1 /* 2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited. 3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. 4 * 5 * This file is released under the GPL. 6 */ 7 8 #include "dm.h" 9 #include "dm-uevent.h" 10 11 #include <linux/init.h> 12 #include <linux/module.h> 13 #include <linux/mutex.h> 14 #include <linux/moduleparam.h> 15 #include <linux/blkpg.h> 16 #include <linux/bio.h> 17 #include <linux/mempool.h> 18 #include <linux/slab.h> 19 #include <linux/idr.h> 20 #include <linux/hdreg.h> 21 #include <linux/delay.h> 22 #include <linux/wait.h> 23 #include <linux/kthread.h> 24 25 #include <trace/events/block.h> 26 27 #define DM_MSG_PREFIX "core" 28 29 #ifdef CONFIG_PRINTK 30 /* 31 * ratelimit state to be used in DMXXX_LIMIT(). 32 */ 33 DEFINE_RATELIMIT_STATE(dm_ratelimit_state, 34 DEFAULT_RATELIMIT_INTERVAL, 35 DEFAULT_RATELIMIT_BURST); 36 EXPORT_SYMBOL(dm_ratelimit_state); 37 #endif 38 39 /* 40 * Cookies are numeric values sent with CHANGE and REMOVE 41 * uevents while resuming, removing or renaming the device. 42 */ 43 #define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE" 44 #define DM_COOKIE_LENGTH 24 45 46 static const char *_name = DM_NAME; 47 48 static unsigned int major = 0; 49 static unsigned int _major = 0; 50 51 static DEFINE_IDR(_minor_idr); 52 53 static DEFINE_SPINLOCK(_minor_lock); 54 55 static void do_deferred_remove(struct work_struct *w); 56 57 static DECLARE_WORK(deferred_remove_work, do_deferred_remove); 58 59 static struct workqueue_struct *deferred_remove_workqueue; 60 61 /* 62 * For bio-based dm. 63 * One of these is allocated per bio. 64 */ 65 struct dm_io { 66 struct mapped_device *md; 67 int error; 68 atomic_t io_count; 69 struct bio *bio; 70 unsigned long start_time; 71 spinlock_t endio_lock; 72 struct dm_stats_aux stats_aux; 73 }; 74 75 /* 76 * For request-based dm. 77 * One of these is allocated per request. 78 */ 79 struct dm_rq_target_io { 80 struct mapped_device *md; 81 struct dm_target *ti; 82 struct request *orig, *clone; 83 struct kthread_work work; 84 int error; 85 union map_info info; 86 }; 87 88 /* 89 * For request-based dm - the bio clones we allocate are embedded in these 90 * structs. 91 * 92 * We allocate these with bio_alloc_bioset, using the front_pad parameter when 93 * the bioset is created - this means the bio has to come at the end of the 94 * struct. 95 */ 96 struct dm_rq_clone_bio_info { 97 struct bio *orig; 98 struct dm_rq_target_io *tio; 99 struct bio clone; 100 }; 101 102 union map_info *dm_get_rq_mapinfo(struct request *rq) 103 { 104 if (rq && rq->end_io_data) 105 return &((struct dm_rq_target_io *)rq->end_io_data)->info; 106 return NULL; 107 } 108 EXPORT_SYMBOL_GPL(dm_get_rq_mapinfo); 109 110 #define MINOR_ALLOCED ((void *)-1) 111 112 /* 113 * Bits for the md->flags field. 114 */ 115 #define DMF_BLOCK_IO_FOR_SUSPEND 0 116 #define DMF_SUSPENDED 1 117 #define DMF_FROZEN 2 118 #define DMF_FREEING 3 119 #define DMF_DELETING 4 120 #define DMF_NOFLUSH_SUSPENDING 5 121 #define DMF_MERGE_IS_OPTIONAL 6 122 #define DMF_DEFERRED_REMOVE 7 123 #define DMF_SUSPENDED_INTERNALLY 8 124 125 /* 126 * A dummy definition to make RCU happy. 127 * struct dm_table should never be dereferenced in this file. 128 */ 129 struct dm_table { 130 int undefined__; 131 }; 132 133 /* 134 * Work processed by per-device workqueue. 135 */ 136 struct mapped_device { 137 struct srcu_struct io_barrier; 138 struct mutex suspend_lock; 139 atomic_t holders; 140 atomic_t open_count; 141 142 /* 143 * The current mapping. 144 * Use dm_get_live_table{_fast} or take suspend_lock for 145 * dereference. 146 */ 147 struct dm_table __rcu *map; 148 149 struct list_head table_devices; 150 struct mutex table_devices_lock; 151 152 unsigned long flags; 153 154 struct request_queue *queue; 155 unsigned type; 156 /* Protect queue and type against concurrent access. */ 157 struct mutex type_lock; 158 159 struct target_type *immutable_target_type; 160 161 struct gendisk *disk; 162 char name[16]; 163 164 void *interface_ptr; 165 166 /* 167 * A list of ios that arrived while we were suspended. 168 */ 169 atomic_t pending[2]; 170 wait_queue_head_t wait; 171 struct work_struct work; 172 struct bio_list deferred; 173 spinlock_t deferred_lock; 174 175 /* 176 * Processing queue (flush) 177 */ 178 struct workqueue_struct *wq; 179 180 /* 181 * io objects are allocated from here. 182 */ 183 mempool_t *io_pool; 184 mempool_t *rq_pool; 185 186 struct bio_set *bs; 187 188 /* 189 * Event handling. 190 */ 191 atomic_t event_nr; 192 wait_queue_head_t eventq; 193 atomic_t uevent_seq; 194 struct list_head uevent_list; 195 spinlock_t uevent_lock; /* Protect access to uevent_list */ 196 197 /* 198 * freeze/thaw support require holding onto a super block 199 */ 200 struct super_block *frozen_sb; 201 struct block_device *bdev; 202 203 /* forced geometry settings */ 204 struct hd_geometry geometry; 205 206 /* kobject and completion */ 207 struct dm_kobject_holder kobj_holder; 208 209 /* zero-length flush that will be cloned and submitted to targets */ 210 struct bio flush_bio; 211 212 /* the number of internal suspends */ 213 unsigned internal_suspend_count; 214 215 struct dm_stats stats; 216 217 struct kthread_worker kworker; 218 struct task_struct *kworker_task; 219 }; 220 221 /* 222 * For mempools pre-allocation at the table loading time. 223 */ 224 struct dm_md_mempools { 225 mempool_t *io_pool; 226 mempool_t *rq_pool; 227 struct bio_set *bs; 228 }; 229 230 struct table_device { 231 struct list_head list; 232 atomic_t count; 233 struct dm_dev dm_dev; 234 }; 235 236 #define RESERVED_BIO_BASED_IOS 16 237 #define RESERVED_REQUEST_BASED_IOS 256 238 #define RESERVED_MAX_IOS 1024 239 static struct kmem_cache *_io_cache; 240 static struct kmem_cache *_rq_tio_cache; 241 static struct kmem_cache *_rq_cache; 242 243 /* 244 * Bio-based DM's mempools' reserved IOs set by the user. 245 */ 246 static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS; 247 248 /* 249 * Request-based DM's mempools' reserved IOs set by the user. 250 */ 251 static unsigned reserved_rq_based_ios = RESERVED_REQUEST_BASED_IOS; 252 253 static unsigned __dm_get_module_param(unsigned *module_param, 254 unsigned def, unsigned max) 255 { 256 unsigned param = ACCESS_ONCE(*module_param); 257 unsigned modified_param = 0; 258 259 if (!param) 260 modified_param = def; 261 else if (param > max) 262 modified_param = max; 263 264 if (modified_param) { 265 (void)cmpxchg(module_param, param, modified_param); 266 param = modified_param; 267 } 268 269 return param; 270 } 271 272 unsigned dm_get_reserved_bio_based_ios(void) 273 { 274 return __dm_get_module_param(&reserved_bio_based_ios, 275 RESERVED_BIO_BASED_IOS, RESERVED_MAX_IOS); 276 } 277 EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios); 278 279 unsigned dm_get_reserved_rq_based_ios(void) 280 { 281 return __dm_get_module_param(&reserved_rq_based_ios, 282 RESERVED_REQUEST_BASED_IOS, RESERVED_MAX_IOS); 283 } 284 EXPORT_SYMBOL_GPL(dm_get_reserved_rq_based_ios); 285 286 static int __init local_init(void) 287 { 288 int r = -ENOMEM; 289 290 /* allocate a slab for the dm_ios */ 291 _io_cache = KMEM_CACHE(dm_io, 0); 292 if (!_io_cache) 293 return r; 294 295 _rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0); 296 if (!_rq_tio_cache) 297 goto out_free_io_cache; 298 299 _rq_cache = kmem_cache_create("dm_clone_request", sizeof(struct request), 300 __alignof__(struct request), 0, NULL); 301 if (!_rq_cache) 302 goto out_free_rq_tio_cache; 303 304 r = dm_uevent_init(); 305 if (r) 306 goto out_free_rq_cache; 307 308 deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1); 309 if (!deferred_remove_workqueue) { 310 r = -ENOMEM; 311 goto out_uevent_exit; 312 } 313 314 _major = major; 315 r = register_blkdev(_major, _name); 316 if (r < 0) 317 goto out_free_workqueue; 318 319 if (!_major) 320 _major = r; 321 322 return 0; 323 324 out_free_workqueue: 325 destroy_workqueue(deferred_remove_workqueue); 326 out_uevent_exit: 327 dm_uevent_exit(); 328 out_free_rq_cache: 329 kmem_cache_destroy(_rq_cache); 330 out_free_rq_tio_cache: 331 kmem_cache_destroy(_rq_tio_cache); 332 out_free_io_cache: 333 kmem_cache_destroy(_io_cache); 334 335 return r; 336 } 337 338 static void local_exit(void) 339 { 340 flush_scheduled_work(); 341 destroy_workqueue(deferred_remove_workqueue); 342 343 kmem_cache_destroy(_rq_cache); 344 kmem_cache_destroy(_rq_tio_cache); 345 kmem_cache_destroy(_io_cache); 346 unregister_blkdev(_major, _name); 347 dm_uevent_exit(); 348 349 _major = 0; 350 351 DMINFO("cleaned up"); 352 } 353 354 static int (*_inits[])(void) __initdata = { 355 local_init, 356 dm_target_init, 357 dm_linear_init, 358 dm_stripe_init, 359 dm_io_init, 360 dm_kcopyd_init, 361 dm_interface_init, 362 dm_statistics_init, 363 }; 364 365 static void (*_exits[])(void) = { 366 local_exit, 367 dm_target_exit, 368 dm_linear_exit, 369 dm_stripe_exit, 370 dm_io_exit, 371 dm_kcopyd_exit, 372 dm_interface_exit, 373 dm_statistics_exit, 374 }; 375 376 static int __init dm_init(void) 377 { 378 const int count = ARRAY_SIZE(_inits); 379 380 int r, i; 381 382 for (i = 0; i < count; i++) { 383 r = _inits[i](); 384 if (r) 385 goto bad; 386 } 387 388 return 0; 389 390 bad: 391 while (i--) 392 _exits[i](); 393 394 return r; 395 } 396 397 static void __exit dm_exit(void) 398 { 399 int i = ARRAY_SIZE(_exits); 400 401 while (i--) 402 _exits[i](); 403 404 /* 405 * Should be empty by this point. 406 */ 407 idr_destroy(&_minor_idr); 408 } 409 410 /* 411 * Block device functions 412 */ 413 int dm_deleting_md(struct mapped_device *md) 414 { 415 return test_bit(DMF_DELETING, &md->flags); 416 } 417 418 static int dm_blk_open(struct block_device *bdev, fmode_t mode) 419 { 420 struct mapped_device *md; 421 422 spin_lock(&_minor_lock); 423 424 md = bdev->bd_disk->private_data; 425 if (!md) 426 goto out; 427 428 if (test_bit(DMF_FREEING, &md->flags) || 429 dm_deleting_md(md)) { 430 md = NULL; 431 goto out; 432 } 433 434 dm_get(md); 435 atomic_inc(&md->open_count); 436 out: 437 spin_unlock(&_minor_lock); 438 439 return md ? 0 : -ENXIO; 440 } 441 442 static void dm_blk_close(struct gendisk *disk, fmode_t mode) 443 { 444 struct mapped_device *md; 445 446 spin_lock(&_minor_lock); 447 448 md = disk->private_data; 449 if (WARN_ON(!md)) 450 goto out; 451 452 if (atomic_dec_and_test(&md->open_count) && 453 (test_bit(DMF_DEFERRED_REMOVE, &md->flags))) 454 queue_work(deferred_remove_workqueue, &deferred_remove_work); 455 456 dm_put(md); 457 out: 458 spin_unlock(&_minor_lock); 459 } 460 461 int dm_open_count(struct mapped_device *md) 462 { 463 return atomic_read(&md->open_count); 464 } 465 466 /* 467 * Guarantees nothing is using the device before it's deleted. 468 */ 469 int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred) 470 { 471 int r = 0; 472 473 spin_lock(&_minor_lock); 474 475 if (dm_open_count(md)) { 476 r = -EBUSY; 477 if (mark_deferred) 478 set_bit(DMF_DEFERRED_REMOVE, &md->flags); 479 } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags)) 480 r = -EEXIST; 481 else 482 set_bit(DMF_DELETING, &md->flags); 483 484 spin_unlock(&_minor_lock); 485 486 return r; 487 } 488 489 int dm_cancel_deferred_remove(struct mapped_device *md) 490 { 491 int r = 0; 492 493 spin_lock(&_minor_lock); 494 495 if (test_bit(DMF_DELETING, &md->flags)) 496 r = -EBUSY; 497 else 498 clear_bit(DMF_DEFERRED_REMOVE, &md->flags); 499 500 spin_unlock(&_minor_lock); 501 502 return r; 503 } 504 505 static void do_deferred_remove(struct work_struct *w) 506 { 507 dm_deferred_remove(); 508 } 509 510 sector_t dm_get_size(struct mapped_device *md) 511 { 512 return get_capacity(md->disk); 513 } 514 515 struct request_queue *dm_get_md_queue(struct mapped_device *md) 516 { 517 return md->queue; 518 } 519 520 struct dm_stats *dm_get_stats(struct mapped_device *md) 521 { 522 return &md->stats; 523 } 524 525 static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo) 526 { 527 struct mapped_device *md = bdev->bd_disk->private_data; 528 529 return dm_get_geometry(md, geo); 530 } 531 532 static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode, 533 unsigned int cmd, unsigned long arg) 534 { 535 struct mapped_device *md = bdev->bd_disk->private_data; 536 int srcu_idx; 537 struct dm_table *map; 538 struct dm_target *tgt; 539 int r = -ENOTTY; 540 541 retry: 542 map = dm_get_live_table(md, &srcu_idx); 543 544 if (!map || !dm_table_get_size(map)) 545 goto out; 546 547 /* We only support devices that have a single target */ 548 if (dm_table_get_num_targets(map) != 1) 549 goto out; 550 551 tgt = dm_table_get_target(map, 0); 552 if (!tgt->type->ioctl) 553 goto out; 554 555 if (dm_suspended_md(md)) { 556 r = -EAGAIN; 557 goto out; 558 } 559 560 r = tgt->type->ioctl(tgt, cmd, arg); 561 562 out: 563 dm_put_live_table(md, srcu_idx); 564 565 if (r == -ENOTCONN) { 566 msleep(10); 567 goto retry; 568 } 569 570 return r; 571 } 572 573 static struct dm_io *alloc_io(struct mapped_device *md) 574 { 575 return mempool_alloc(md->io_pool, GFP_NOIO); 576 } 577 578 static void free_io(struct mapped_device *md, struct dm_io *io) 579 { 580 mempool_free(io, md->io_pool); 581 } 582 583 static void free_tio(struct mapped_device *md, struct dm_target_io *tio) 584 { 585 bio_put(&tio->clone); 586 } 587 588 static struct dm_rq_target_io *alloc_rq_tio(struct mapped_device *md, 589 gfp_t gfp_mask) 590 { 591 return mempool_alloc(md->io_pool, gfp_mask); 592 } 593 594 static void free_rq_tio(struct dm_rq_target_io *tio) 595 { 596 mempool_free(tio, tio->md->io_pool); 597 } 598 599 static struct request *alloc_clone_request(struct mapped_device *md, 600 gfp_t gfp_mask) 601 { 602 return mempool_alloc(md->rq_pool, gfp_mask); 603 } 604 605 static void free_clone_request(struct mapped_device *md, struct request *rq) 606 { 607 mempool_free(rq, md->rq_pool); 608 } 609 610 static int md_in_flight(struct mapped_device *md) 611 { 612 return atomic_read(&md->pending[READ]) + 613 atomic_read(&md->pending[WRITE]); 614 } 615 616 static void start_io_acct(struct dm_io *io) 617 { 618 struct mapped_device *md = io->md; 619 struct bio *bio = io->bio; 620 int cpu; 621 int rw = bio_data_dir(bio); 622 623 io->start_time = jiffies; 624 625 cpu = part_stat_lock(); 626 part_round_stats(cpu, &dm_disk(md)->part0); 627 part_stat_unlock(); 628 atomic_set(&dm_disk(md)->part0.in_flight[rw], 629 atomic_inc_return(&md->pending[rw])); 630 631 if (unlikely(dm_stats_used(&md->stats))) 632 dm_stats_account_io(&md->stats, bio->bi_rw, bio->bi_iter.bi_sector, 633 bio_sectors(bio), false, 0, &io->stats_aux); 634 } 635 636 static void end_io_acct(struct dm_io *io) 637 { 638 struct mapped_device *md = io->md; 639 struct bio *bio = io->bio; 640 unsigned long duration = jiffies - io->start_time; 641 int pending; 642 int rw = bio_data_dir(bio); 643 644 generic_end_io_acct(rw, &dm_disk(md)->part0, io->start_time); 645 646 if (unlikely(dm_stats_used(&md->stats))) 647 dm_stats_account_io(&md->stats, bio->bi_rw, bio->bi_iter.bi_sector, 648 bio_sectors(bio), true, duration, &io->stats_aux); 649 650 /* 651 * After this is decremented the bio must not be touched if it is 652 * a flush. 653 */ 654 pending = atomic_dec_return(&md->pending[rw]); 655 atomic_set(&dm_disk(md)->part0.in_flight[rw], pending); 656 pending += atomic_read(&md->pending[rw^0x1]); 657 658 /* nudge anyone waiting on suspend queue */ 659 if (!pending) 660 wake_up(&md->wait); 661 } 662 663 /* 664 * Add the bio to the list of deferred io. 665 */ 666 static void queue_io(struct mapped_device *md, struct bio *bio) 667 { 668 unsigned long flags; 669 670 spin_lock_irqsave(&md->deferred_lock, flags); 671 bio_list_add(&md->deferred, bio); 672 spin_unlock_irqrestore(&md->deferred_lock, flags); 673 queue_work(md->wq, &md->work); 674 } 675 676 /* 677 * Everyone (including functions in this file), should use this 678 * function to access the md->map field, and make sure they call 679 * dm_put_live_table() when finished. 680 */ 681 struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier) 682 { 683 *srcu_idx = srcu_read_lock(&md->io_barrier); 684 685 return srcu_dereference(md->map, &md->io_barrier); 686 } 687 688 void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier) 689 { 690 srcu_read_unlock(&md->io_barrier, srcu_idx); 691 } 692 693 void dm_sync_table(struct mapped_device *md) 694 { 695 synchronize_srcu(&md->io_barrier); 696 synchronize_rcu_expedited(); 697 } 698 699 /* 700 * A fast alternative to dm_get_live_table/dm_put_live_table. 701 * The caller must not block between these two functions. 702 */ 703 static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU) 704 { 705 rcu_read_lock(); 706 return rcu_dereference(md->map); 707 } 708 709 static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU) 710 { 711 rcu_read_unlock(); 712 } 713 714 /* 715 * Open a table device so we can use it as a map destination. 716 */ 717 static int open_table_device(struct table_device *td, dev_t dev, 718 struct mapped_device *md) 719 { 720 static char *_claim_ptr = "I belong to device-mapper"; 721 struct block_device *bdev; 722 723 int r; 724 725 BUG_ON(td->dm_dev.bdev); 726 727 bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _claim_ptr); 728 if (IS_ERR(bdev)) 729 return PTR_ERR(bdev); 730 731 r = bd_link_disk_holder(bdev, dm_disk(md)); 732 if (r) { 733 blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL); 734 return r; 735 } 736 737 td->dm_dev.bdev = bdev; 738 return 0; 739 } 740 741 /* 742 * Close a table device that we've been using. 743 */ 744 static void close_table_device(struct table_device *td, struct mapped_device *md) 745 { 746 if (!td->dm_dev.bdev) 747 return; 748 749 bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md)); 750 blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL); 751 td->dm_dev.bdev = NULL; 752 } 753 754 static struct table_device *find_table_device(struct list_head *l, dev_t dev, 755 fmode_t mode) { 756 struct table_device *td; 757 758 list_for_each_entry(td, l, list) 759 if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode) 760 return td; 761 762 return NULL; 763 } 764 765 int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode, 766 struct dm_dev **result) { 767 int r; 768 struct table_device *td; 769 770 mutex_lock(&md->table_devices_lock); 771 td = find_table_device(&md->table_devices, dev, mode); 772 if (!td) { 773 td = kmalloc(sizeof(*td), GFP_KERNEL); 774 if (!td) { 775 mutex_unlock(&md->table_devices_lock); 776 return -ENOMEM; 777 } 778 779 td->dm_dev.mode = mode; 780 td->dm_dev.bdev = NULL; 781 782 if ((r = open_table_device(td, dev, md))) { 783 mutex_unlock(&md->table_devices_lock); 784 kfree(td); 785 return r; 786 } 787 788 format_dev_t(td->dm_dev.name, dev); 789 790 atomic_set(&td->count, 0); 791 list_add(&td->list, &md->table_devices); 792 } 793 atomic_inc(&td->count); 794 mutex_unlock(&md->table_devices_lock); 795 796 *result = &td->dm_dev; 797 return 0; 798 } 799 EXPORT_SYMBOL_GPL(dm_get_table_device); 800 801 void dm_put_table_device(struct mapped_device *md, struct dm_dev *d) 802 { 803 struct table_device *td = container_of(d, struct table_device, dm_dev); 804 805 mutex_lock(&md->table_devices_lock); 806 if (atomic_dec_and_test(&td->count)) { 807 close_table_device(td, md); 808 list_del(&td->list); 809 kfree(td); 810 } 811 mutex_unlock(&md->table_devices_lock); 812 } 813 EXPORT_SYMBOL(dm_put_table_device); 814 815 static void free_table_devices(struct list_head *devices) 816 { 817 struct list_head *tmp, *next; 818 819 list_for_each_safe(tmp, next, devices) { 820 struct table_device *td = list_entry(tmp, struct table_device, list); 821 822 DMWARN("dm_destroy: %s still exists with %d references", 823 td->dm_dev.name, atomic_read(&td->count)); 824 kfree(td); 825 } 826 } 827 828 /* 829 * Get the geometry associated with a dm device 830 */ 831 int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo) 832 { 833 *geo = md->geometry; 834 835 return 0; 836 } 837 838 /* 839 * Set the geometry of a device. 840 */ 841 int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo) 842 { 843 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors; 844 845 if (geo->start > sz) { 846 DMWARN("Start sector is beyond the geometry limits."); 847 return -EINVAL; 848 } 849 850 md->geometry = *geo; 851 852 return 0; 853 } 854 855 /*----------------------------------------------------------------- 856 * CRUD START: 857 * A more elegant soln is in the works that uses the queue 858 * merge fn, unfortunately there are a couple of changes to 859 * the block layer that I want to make for this. So in the 860 * interests of getting something for people to use I give 861 * you this clearly demarcated crap. 862 *---------------------------------------------------------------*/ 863 864 static int __noflush_suspending(struct mapped_device *md) 865 { 866 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 867 } 868 869 /* 870 * Decrements the number of outstanding ios that a bio has been 871 * cloned into, completing the original io if necc. 872 */ 873 static void dec_pending(struct dm_io *io, int error) 874 { 875 unsigned long flags; 876 int io_error; 877 struct bio *bio; 878 struct mapped_device *md = io->md; 879 880 /* Push-back supersedes any I/O errors */ 881 if (unlikely(error)) { 882 spin_lock_irqsave(&io->endio_lock, flags); 883 if (!(io->error > 0 && __noflush_suspending(md))) 884 io->error = error; 885 spin_unlock_irqrestore(&io->endio_lock, flags); 886 } 887 888 if (atomic_dec_and_test(&io->io_count)) { 889 if (io->error == DM_ENDIO_REQUEUE) { 890 /* 891 * Target requested pushing back the I/O. 892 */ 893 spin_lock_irqsave(&md->deferred_lock, flags); 894 if (__noflush_suspending(md)) 895 bio_list_add_head(&md->deferred, io->bio); 896 else 897 /* noflush suspend was interrupted. */ 898 io->error = -EIO; 899 spin_unlock_irqrestore(&md->deferred_lock, flags); 900 } 901 902 io_error = io->error; 903 bio = io->bio; 904 end_io_acct(io); 905 free_io(md, io); 906 907 if (io_error == DM_ENDIO_REQUEUE) 908 return; 909 910 if ((bio->bi_rw & REQ_FLUSH) && bio->bi_iter.bi_size) { 911 /* 912 * Preflush done for flush with data, reissue 913 * without REQ_FLUSH. 914 */ 915 bio->bi_rw &= ~REQ_FLUSH; 916 queue_io(md, bio); 917 } else { 918 /* done with normal IO or empty flush */ 919 trace_block_bio_complete(md->queue, bio, io_error); 920 bio_endio(bio, io_error); 921 } 922 } 923 } 924 925 static void disable_write_same(struct mapped_device *md) 926 { 927 struct queue_limits *limits = dm_get_queue_limits(md); 928 929 /* device doesn't really support WRITE SAME, disable it */ 930 limits->max_write_same_sectors = 0; 931 } 932 933 static void clone_endio(struct bio *bio, int error) 934 { 935 int r = error; 936 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone); 937 struct dm_io *io = tio->io; 938 struct mapped_device *md = tio->io->md; 939 dm_endio_fn endio = tio->ti->type->end_io; 940 941 if (!bio_flagged(bio, BIO_UPTODATE) && !error) 942 error = -EIO; 943 944 if (endio) { 945 r = endio(tio->ti, bio, error); 946 if (r < 0 || r == DM_ENDIO_REQUEUE) 947 /* 948 * error and requeue request are handled 949 * in dec_pending(). 950 */ 951 error = r; 952 else if (r == DM_ENDIO_INCOMPLETE) 953 /* The target will handle the io */ 954 return; 955 else if (r) { 956 DMWARN("unimplemented target endio return value: %d", r); 957 BUG(); 958 } 959 } 960 961 if (unlikely(r == -EREMOTEIO && (bio->bi_rw & REQ_WRITE_SAME) && 962 !bdev_get_queue(bio->bi_bdev)->limits.max_write_same_sectors)) 963 disable_write_same(md); 964 965 free_tio(md, tio); 966 dec_pending(io, error); 967 } 968 969 /* 970 * Partial completion handling for request-based dm 971 */ 972 static void end_clone_bio(struct bio *clone, int error) 973 { 974 struct dm_rq_clone_bio_info *info = 975 container_of(clone, struct dm_rq_clone_bio_info, clone); 976 struct dm_rq_target_io *tio = info->tio; 977 struct bio *bio = info->orig; 978 unsigned int nr_bytes = info->orig->bi_iter.bi_size; 979 980 bio_put(clone); 981 982 if (tio->error) 983 /* 984 * An error has already been detected on the request. 985 * Once error occurred, just let clone->end_io() handle 986 * the remainder. 987 */ 988 return; 989 else if (error) { 990 /* 991 * Don't notice the error to the upper layer yet. 992 * The error handling decision is made by the target driver, 993 * when the request is completed. 994 */ 995 tio->error = error; 996 return; 997 } 998 999 /* 1000 * I/O for the bio successfully completed. 1001 * Notice the data completion to the upper layer. 1002 */ 1003 1004 /* 1005 * bios are processed from the head of the list. 1006 * So the completing bio should always be rq->bio. 1007 * If it's not, something wrong is happening. 1008 */ 1009 if (tio->orig->bio != bio) 1010 DMERR("bio completion is going in the middle of the request"); 1011 1012 /* 1013 * Update the original request. 1014 * Do not use blk_end_request() here, because it may complete 1015 * the original request before the clone, and break the ordering. 1016 */ 1017 blk_update_request(tio->orig, 0, nr_bytes); 1018 } 1019 1020 /* 1021 * Don't touch any member of the md after calling this function because 1022 * the md may be freed in dm_put() at the end of this function. 1023 * Or do dm_get() before calling this function and dm_put() later. 1024 */ 1025 static void rq_completed(struct mapped_device *md, int rw, bool run_queue) 1026 { 1027 int nr_requests_pending; 1028 1029 atomic_dec(&md->pending[rw]); 1030 1031 /* nudge anyone waiting on suspend queue */ 1032 nr_requests_pending = md_in_flight(md); 1033 if (!nr_requests_pending) 1034 wake_up(&md->wait); 1035 1036 /* 1037 * Run this off this callpath, as drivers could invoke end_io while 1038 * inside their request_fn (and holding the queue lock). Calling 1039 * back into ->request_fn() could deadlock attempting to grab the 1040 * queue lock again. 1041 */ 1042 if (run_queue) { 1043 if (!nr_requests_pending || 1044 (nr_requests_pending >= md->queue->nr_congestion_on)) 1045 blk_run_queue_async(md->queue); 1046 } 1047 1048 /* 1049 * dm_put() must be at the end of this function. See the comment above 1050 */ 1051 dm_put(md); 1052 } 1053 1054 static void free_rq_clone(struct request *clone) 1055 { 1056 struct dm_rq_target_io *tio = clone->end_io_data; 1057 1058 blk_rq_unprep_clone(clone); 1059 if (clone->q && clone->q->mq_ops) 1060 tio->ti->type->release_clone_rq(clone); 1061 else 1062 free_clone_request(tio->md, clone); 1063 free_rq_tio(tio); 1064 } 1065 1066 /* 1067 * Complete the clone and the original request. 1068 * Must be called without clone's queue lock held, 1069 * see end_clone_request() for more details. 1070 */ 1071 static void dm_end_request(struct request *clone, int error) 1072 { 1073 int rw = rq_data_dir(clone); 1074 struct dm_rq_target_io *tio = clone->end_io_data; 1075 struct mapped_device *md = tio->md; 1076 struct request *rq = tio->orig; 1077 1078 if (rq->cmd_type == REQ_TYPE_BLOCK_PC) { 1079 rq->errors = clone->errors; 1080 rq->resid_len = clone->resid_len; 1081 1082 if (rq->sense) 1083 /* 1084 * We are using the sense buffer of the original 1085 * request. 1086 * So setting the length of the sense data is enough. 1087 */ 1088 rq->sense_len = clone->sense_len; 1089 } 1090 1091 free_rq_clone(clone); 1092 blk_end_request_all(rq, error); 1093 rq_completed(md, rw, true); 1094 } 1095 1096 static void dm_unprep_request(struct request *rq) 1097 { 1098 struct dm_rq_target_io *tio = rq->special; 1099 struct request *clone = tio->clone; 1100 1101 rq->special = NULL; 1102 rq->cmd_flags &= ~REQ_DONTPREP; 1103 1104 if (clone) 1105 free_rq_clone(clone); 1106 } 1107 1108 /* 1109 * Requeue the original request of a clone. 1110 */ 1111 static void dm_requeue_unmapped_original_request(struct mapped_device *md, 1112 struct request *rq) 1113 { 1114 int rw = rq_data_dir(rq); 1115 struct request_queue *q = rq->q; 1116 unsigned long flags; 1117 1118 dm_unprep_request(rq); 1119 1120 spin_lock_irqsave(q->queue_lock, flags); 1121 blk_requeue_request(q, rq); 1122 spin_unlock_irqrestore(q->queue_lock, flags); 1123 1124 rq_completed(md, rw, false); 1125 } 1126 1127 static void dm_requeue_unmapped_request(struct request *clone) 1128 { 1129 struct dm_rq_target_io *tio = clone->end_io_data; 1130 1131 dm_requeue_unmapped_original_request(tio->md, tio->orig); 1132 } 1133 1134 static void __stop_queue(struct request_queue *q) 1135 { 1136 blk_stop_queue(q); 1137 } 1138 1139 static void stop_queue(struct request_queue *q) 1140 { 1141 unsigned long flags; 1142 1143 spin_lock_irqsave(q->queue_lock, flags); 1144 __stop_queue(q); 1145 spin_unlock_irqrestore(q->queue_lock, flags); 1146 } 1147 1148 static void __start_queue(struct request_queue *q) 1149 { 1150 if (blk_queue_stopped(q)) 1151 blk_start_queue(q); 1152 } 1153 1154 static void start_queue(struct request_queue *q) 1155 { 1156 unsigned long flags; 1157 1158 spin_lock_irqsave(q->queue_lock, flags); 1159 __start_queue(q); 1160 spin_unlock_irqrestore(q->queue_lock, flags); 1161 } 1162 1163 static void dm_done(struct request *clone, int error, bool mapped) 1164 { 1165 int r = error; 1166 struct dm_rq_target_io *tio = clone->end_io_data; 1167 dm_request_endio_fn rq_end_io = NULL; 1168 1169 if (tio->ti) { 1170 rq_end_io = tio->ti->type->rq_end_io; 1171 1172 if (mapped && rq_end_io) 1173 r = rq_end_io(tio->ti, clone, error, &tio->info); 1174 } 1175 1176 if (unlikely(r == -EREMOTEIO && (clone->cmd_flags & REQ_WRITE_SAME) && 1177 !clone->q->limits.max_write_same_sectors)) 1178 disable_write_same(tio->md); 1179 1180 if (r <= 0) 1181 /* The target wants to complete the I/O */ 1182 dm_end_request(clone, r); 1183 else if (r == DM_ENDIO_INCOMPLETE) 1184 /* The target will handle the I/O */ 1185 return; 1186 else if (r == DM_ENDIO_REQUEUE) 1187 /* The target wants to requeue the I/O */ 1188 dm_requeue_unmapped_request(clone); 1189 else { 1190 DMWARN("unimplemented target endio return value: %d", r); 1191 BUG(); 1192 } 1193 } 1194 1195 /* 1196 * Request completion handler for request-based dm 1197 */ 1198 static void dm_softirq_done(struct request *rq) 1199 { 1200 bool mapped = true; 1201 struct dm_rq_target_io *tio = rq->special; 1202 struct request *clone = tio->clone; 1203 1204 if (!clone) { 1205 blk_end_request_all(rq, tio->error); 1206 rq_completed(tio->md, rq_data_dir(rq), false); 1207 free_rq_tio(tio); 1208 return; 1209 } 1210 1211 if (rq->cmd_flags & REQ_FAILED) 1212 mapped = false; 1213 1214 dm_done(clone, tio->error, mapped); 1215 } 1216 1217 /* 1218 * Complete the clone and the original request with the error status 1219 * through softirq context. 1220 */ 1221 static void dm_complete_request(struct request *rq, int error) 1222 { 1223 struct dm_rq_target_io *tio = rq->special; 1224 1225 tio->error = error; 1226 blk_complete_request(rq); 1227 } 1228 1229 /* 1230 * Complete the not-mapped clone and the original request with the error status 1231 * through softirq context. 1232 * Target's rq_end_io() function isn't called. 1233 * This may be used when the target's map_rq() or clone_and_map_rq() functions fail. 1234 */ 1235 static void dm_kill_unmapped_request(struct request *rq, int error) 1236 { 1237 rq->cmd_flags |= REQ_FAILED; 1238 dm_complete_request(rq, error); 1239 } 1240 1241 /* 1242 * Called with the clone's queue lock held 1243 */ 1244 static void end_clone_request(struct request *clone, int error) 1245 { 1246 struct dm_rq_target_io *tio = clone->end_io_data; 1247 1248 if (!clone->q->mq_ops) { 1249 /* 1250 * For just cleaning up the information of the queue in which 1251 * the clone was dispatched. 1252 * The clone is *NOT* freed actually here because it is alloced 1253 * from dm own mempool (REQ_ALLOCED isn't set). 1254 */ 1255 __blk_put_request(clone->q, clone); 1256 } 1257 1258 /* 1259 * Actual request completion is done in a softirq context which doesn't 1260 * hold the clone's queue lock. Otherwise, deadlock could occur because: 1261 * - another request may be submitted by the upper level driver 1262 * of the stacking during the completion 1263 * - the submission which requires queue lock may be done 1264 * against this clone's queue 1265 */ 1266 dm_complete_request(tio->orig, error); 1267 } 1268 1269 /* 1270 * Return maximum size of I/O possible at the supplied sector up to the current 1271 * target boundary. 1272 */ 1273 static sector_t max_io_len_target_boundary(sector_t sector, struct dm_target *ti) 1274 { 1275 sector_t target_offset = dm_target_offset(ti, sector); 1276 1277 return ti->len - target_offset; 1278 } 1279 1280 static sector_t max_io_len(sector_t sector, struct dm_target *ti) 1281 { 1282 sector_t len = max_io_len_target_boundary(sector, ti); 1283 sector_t offset, max_len; 1284 1285 /* 1286 * Does the target need to split even further? 1287 */ 1288 if (ti->max_io_len) { 1289 offset = dm_target_offset(ti, sector); 1290 if (unlikely(ti->max_io_len & (ti->max_io_len - 1))) 1291 max_len = sector_div(offset, ti->max_io_len); 1292 else 1293 max_len = offset & (ti->max_io_len - 1); 1294 max_len = ti->max_io_len - max_len; 1295 1296 if (len > max_len) 1297 len = max_len; 1298 } 1299 1300 return len; 1301 } 1302 1303 int dm_set_target_max_io_len(struct dm_target *ti, sector_t len) 1304 { 1305 if (len > UINT_MAX) { 1306 DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)", 1307 (unsigned long long)len, UINT_MAX); 1308 ti->error = "Maximum size of target IO is too large"; 1309 return -EINVAL; 1310 } 1311 1312 ti->max_io_len = (uint32_t) len; 1313 1314 return 0; 1315 } 1316 EXPORT_SYMBOL_GPL(dm_set_target_max_io_len); 1317 1318 /* 1319 * A target may call dm_accept_partial_bio only from the map routine. It is 1320 * allowed for all bio types except REQ_FLUSH. 1321 * 1322 * dm_accept_partial_bio informs the dm that the target only wants to process 1323 * additional n_sectors sectors of the bio and the rest of the data should be 1324 * sent in a next bio. 1325 * 1326 * A diagram that explains the arithmetics: 1327 * +--------------------+---------------+-------+ 1328 * | 1 | 2 | 3 | 1329 * +--------------------+---------------+-------+ 1330 * 1331 * <-------------- *tio->len_ptr ---------------> 1332 * <------- bi_size -------> 1333 * <-- n_sectors --> 1334 * 1335 * Region 1 was already iterated over with bio_advance or similar function. 1336 * (it may be empty if the target doesn't use bio_advance) 1337 * Region 2 is the remaining bio size that the target wants to process. 1338 * (it may be empty if region 1 is non-empty, although there is no reason 1339 * to make it empty) 1340 * The target requires that region 3 is to be sent in the next bio. 1341 * 1342 * If the target wants to receive multiple copies of the bio (via num_*bios, etc), 1343 * the partially processed part (the sum of regions 1+2) must be the same for all 1344 * copies of the bio. 1345 */ 1346 void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors) 1347 { 1348 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone); 1349 unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT; 1350 BUG_ON(bio->bi_rw & REQ_FLUSH); 1351 BUG_ON(bi_size > *tio->len_ptr); 1352 BUG_ON(n_sectors > bi_size); 1353 *tio->len_ptr -= bi_size - n_sectors; 1354 bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT; 1355 } 1356 EXPORT_SYMBOL_GPL(dm_accept_partial_bio); 1357 1358 static void __map_bio(struct dm_target_io *tio) 1359 { 1360 int r; 1361 sector_t sector; 1362 struct mapped_device *md; 1363 struct bio *clone = &tio->clone; 1364 struct dm_target *ti = tio->ti; 1365 1366 clone->bi_end_io = clone_endio; 1367 1368 /* 1369 * Map the clone. If r == 0 we don't need to do 1370 * anything, the target has assumed ownership of 1371 * this io. 1372 */ 1373 atomic_inc(&tio->io->io_count); 1374 sector = clone->bi_iter.bi_sector; 1375 r = ti->type->map(ti, clone); 1376 if (r == DM_MAPIO_REMAPPED) { 1377 /* the bio has been remapped so dispatch it */ 1378 1379 trace_block_bio_remap(bdev_get_queue(clone->bi_bdev), clone, 1380 tio->io->bio->bi_bdev->bd_dev, sector); 1381 1382 generic_make_request(clone); 1383 } else if (r < 0 || r == DM_MAPIO_REQUEUE) { 1384 /* error the io and bail out, or requeue it if needed */ 1385 md = tio->io->md; 1386 dec_pending(tio->io, r); 1387 free_tio(md, tio); 1388 } else if (r) { 1389 DMWARN("unimplemented target map return value: %d", r); 1390 BUG(); 1391 } 1392 } 1393 1394 struct clone_info { 1395 struct mapped_device *md; 1396 struct dm_table *map; 1397 struct bio *bio; 1398 struct dm_io *io; 1399 sector_t sector; 1400 unsigned sector_count; 1401 }; 1402 1403 static void bio_setup_sector(struct bio *bio, sector_t sector, unsigned len) 1404 { 1405 bio->bi_iter.bi_sector = sector; 1406 bio->bi_iter.bi_size = to_bytes(len); 1407 } 1408 1409 /* 1410 * Creates a bio that consists of range of complete bvecs. 1411 */ 1412 static void clone_bio(struct dm_target_io *tio, struct bio *bio, 1413 sector_t sector, unsigned len) 1414 { 1415 struct bio *clone = &tio->clone; 1416 1417 __bio_clone_fast(clone, bio); 1418 1419 if (bio_integrity(bio)) 1420 bio_integrity_clone(clone, bio, GFP_NOIO); 1421 1422 bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector)); 1423 clone->bi_iter.bi_size = to_bytes(len); 1424 1425 if (bio_integrity(bio)) 1426 bio_integrity_trim(clone, 0, len); 1427 } 1428 1429 static struct dm_target_io *alloc_tio(struct clone_info *ci, 1430 struct dm_target *ti, 1431 unsigned target_bio_nr) 1432 { 1433 struct dm_target_io *tio; 1434 struct bio *clone; 1435 1436 clone = bio_alloc_bioset(GFP_NOIO, 0, ci->md->bs); 1437 tio = container_of(clone, struct dm_target_io, clone); 1438 1439 tio->io = ci->io; 1440 tio->ti = ti; 1441 tio->target_bio_nr = target_bio_nr; 1442 1443 return tio; 1444 } 1445 1446 static void __clone_and_map_simple_bio(struct clone_info *ci, 1447 struct dm_target *ti, 1448 unsigned target_bio_nr, unsigned *len) 1449 { 1450 struct dm_target_io *tio = alloc_tio(ci, ti, target_bio_nr); 1451 struct bio *clone = &tio->clone; 1452 1453 tio->len_ptr = len; 1454 1455 __bio_clone_fast(clone, ci->bio); 1456 if (len) 1457 bio_setup_sector(clone, ci->sector, *len); 1458 1459 __map_bio(tio); 1460 } 1461 1462 static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti, 1463 unsigned num_bios, unsigned *len) 1464 { 1465 unsigned target_bio_nr; 1466 1467 for (target_bio_nr = 0; target_bio_nr < num_bios; target_bio_nr++) 1468 __clone_and_map_simple_bio(ci, ti, target_bio_nr, len); 1469 } 1470 1471 static int __send_empty_flush(struct clone_info *ci) 1472 { 1473 unsigned target_nr = 0; 1474 struct dm_target *ti; 1475 1476 BUG_ON(bio_has_data(ci->bio)); 1477 while ((ti = dm_table_get_target(ci->map, target_nr++))) 1478 __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL); 1479 1480 return 0; 1481 } 1482 1483 static void __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti, 1484 sector_t sector, unsigned *len) 1485 { 1486 struct bio *bio = ci->bio; 1487 struct dm_target_io *tio; 1488 unsigned target_bio_nr; 1489 unsigned num_target_bios = 1; 1490 1491 /* 1492 * Does the target want to receive duplicate copies of the bio? 1493 */ 1494 if (bio_data_dir(bio) == WRITE && ti->num_write_bios) 1495 num_target_bios = ti->num_write_bios(ti, bio); 1496 1497 for (target_bio_nr = 0; target_bio_nr < num_target_bios; target_bio_nr++) { 1498 tio = alloc_tio(ci, ti, target_bio_nr); 1499 tio->len_ptr = len; 1500 clone_bio(tio, bio, sector, *len); 1501 __map_bio(tio); 1502 } 1503 } 1504 1505 typedef unsigned (*get_num_bios_fn)(struct dm_target *ti); 1506 1507 static unsigned get_num_discard_bios(struct dm_target *ti) 1508 { 1509 return ti->num_discard_bios; 1510 } 1511 1512 static unsigned get_num_write_same_bios(struct dm_target *ti) 1513 { 1514 return ti->num_write_same_bios; 1515 } 1516 1517 typedef bool (*is_split_required_fn)(struct dm_target *ti); 1518 1519 static bool is_split_required_for_discard(struct dm_target *ti) 1520 { 1521 return ti->split_discard_bios; 1522 } 1523 1524 static int __send_changing_extent_only(struct clone_info *ci, 1525 get_num_bios_fn get_num_bios, 1526 is_split_required_fn is_split_required) 1527 { 1528 struct dm_target *ti; 1529 unsigned len; 1530 unsigned num_bios; 1531 1532 do { 1533 ti = dm_table_find_target(ci->map, ci->sector); 1534 if (!dm_target_is_valid(ti)) 1535 return -EIO; 1536 1537 /* 1538 * Even though the device advertised support for this type of 1539 * request, that does not mean every target supports it, and 1540 * reconfiguration might also have changed that since the 1541 * check was performed. 1542 */ 1543 num_bios = get_num_bios ? get_num_bios(ti) : 0; 1544 if (!num_bios) 1545 return -EOPNOTSUPP; 1546 1547 if (is_split_required && !is_split_required(ti)) 1548 len = min((sector_t)ci->sector_count, max_io_len_target_boundary(ci->sector, ti)); 1549 else 1550 len = min((sector_t)ci->sector_count, max_io_len(ci->sector, ti)); 1551 1552 __send_duplicate_bios(ci, ti, num_bios, &len); 1553 1554 ci->sector += len; 1555 } while (ci->sector_count -= len); 1556 1557 return 0; 1558 } 1559 1560 static int __send_discard(struct clone_info *ci) 1561 { 1562 return __send_changing_extent_only(ci, get_num_discard_bios, 1563 is_split_required_for_discard); 1564 } 1565 1566 static int __send_write_same(struct clone_info *ci) 1567 { 1568 return __send_changing_extent_only(ci, get_num_write_same_bios, NULL); 1569 } 1570 1571 /* 1572 * Select the correct strategy for processing a non-flush bio. 1573 */ 1574 static int __split_and_process_non_flush(struct clone_info *ci) 1575 { 1576 struct bio *bio = ci->bio; 1577 struct dm_target *ti; 1578 unsigned len; 1579 1580 if (unlikely(bio->bi_rw & REQ_DISCARD)) 1581 return __send_discard(ci); 1582 else if (unlikely(bio->bi_rw & REQ_WRITE_SAME)) 1583 return __send_write_same(ci); 1584 1585 ti = dm_table_find_target(ci->map, ci->sector); 1586 if (!dm_target_is_valid(ti)) 1587 return -EIO; 1588 1589 len = min_t(sector_t, max_io_len(ci->sector, ti), ci->sector_count); 1590 1591 __clone_and_map_data_bio(ci, ti, ci->sector, &len); 1592 1593 ci->sector += len; 1594 ci->sector_count -= len; 1595 1596 return 0; 1597 } 1598 1599 /* 1600 * Entry point to split a bio into clones and submit them to the targets. 1601 */ 1602 static void __split_and_process_bio(struct mapped_device *md, 1603 struct dm_table *map, struct bio *bio) 1604 { 1605 struct clone_info ci; 1606 int error = 0; 1607 1608 if (unlikely(!map)) { 1609 bio_io_error(bio); 1610 return; 1611 } 1612 1613 ci.map = map; 1614 ci.md = md; 1615 ci.io = alloc_io(md); 1616 ci.io->error = 0; 1617 atomic_set(&ci.io->io_count, 1); 1618 ci.io->bio = bio; 1619 ci.io->md = md; 1620 spin_lock_init(&ci.io->endio_lock); 1621 ci.sector = bio->bi_iter.bi_sector; 1622 1623 start_io_acct(ci.io); 1624 1625 if (bio->bi_rw & REQ_FLUSH) { 1626 ci.bio = &ci.md->flush_bio; 1627 ci.sector_count = 0; 1628 error = __send_empty_flush(&ci); 1629 /* dec_pending submits any data associated with flush */ 1630 } else { 1631 ci.bio = bio; 1632 ci.sector_count = bio_sectors(bio); 1633 while (ci.sector_count && !error) 1634 error = __split_and_process_non_flush(&ci); 1635 } 1636 1637 /* drop the extra reference count */ 1638 dec_pending(ci.io, error); 1639 } 1640 /*----------------------------------------------------------------- 1641 * CRUD END 1642 *---------------------------------------------------------------*/ 1643 1644 static int dm_merge_bvec(struct request_queue *q, 1645 struct bvec_merge_data *bvm, 1646 struct bio_vec *biovec) 1647 { 1648 struct mapped_device *md = q->queuedata; 1649 struct dm_table *map = dm_get_live_table_fast(md); 1650 struct dm_target *ti; 1651 sector_t max_sectors; 1652 int max_size = 0; 1653 1654 if (unlikely(!map)) 1655 goto out; 1656 1657 ti = dm_table_find_target(map, bvm->bi_sector); 1658 if (!dm_target_is_valid(ti)) 1659 goto out; 1660 1661 /* 1662 * Find maximum amount of I/O that won't need splitting 1663 */ 1664 max_sectors = min(max_io_len(bvm->bi_sector, ti), 1665 (sector_t) queue_max_sectors(q)); 1666 max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size; 1667 if (unlikely(max_size < 0)) /* this shouldn't _ever_ happen */ 1668 max_size = 0; 1669 1670 /* 1671 * merge_bvec_fn() returns number of bytes 1672 * it can accept at this offset 1673 * max is precomputed maximal io size 1674 */ 1675 if (max_size && ti->type->merge) 1676 max_size = ti->type->merge(ti, bvm, biovec, max_size); 1677 /* 1678 * If the target doesn't support merge method and some of the devices 1679 * provided their merge_bvec method (we know this by looking for the 1680 * max_hw_sectors that dm_set_device_limits may set), then we can't 1681 * allow bios with multiple vector entries. So always set max_size 1682 * to 0, and the code below allows just one page. 1683 */ 1684 else if (queue_max_hw_sectors(q) <= PAGE_SIZE >> 9) 1685 max_size = 0; 1686 1687 out: 1688 dm_put_live_table_fast(md); 1689 /* 1690 * Always allow an entire first page 1691 */ 1692 if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT)) 1693 max_size = biovec->bv_len; 1694 1695 return max_size; 1696 } 1697 1698 /* 1699 * The request function that just remaps the bio built up by 1700 * dm_merge_bvec. 1701 */ 1702 static void dm_make_request(struct request_queue *q, struct bio *bio) 1703 { 1704 int rw = bio_data_dir(bio); 1705 struct mapped_device *md = q->queuedata; 1706 int srcu_idx; 1707 struct dm_table *map; 1708 1709 map = dm_get_live_table(md, &srcu_idx); 1710 1711 generic_start_io_acct(rw, bio_sectors(bio), &dm_disk(md)->part0); 1712 1713 /* if we're suspended, we have to queue this io for later */ 1714 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) { 1715 dm_put_live_table(md, srcu_idx); 1716 1717 if (bio_rw(bio) != READA) 1718 queue_io(md, bio); 1719 else 1720 bio_io_error(bio); 1721 return; 1722 } 1723 1724 __split_and_process_bio(md, map, bio); 1725 dm_put_live_table(md, srcu_idx); 1726 return; 1727 } 1728 1729 int dm_request_based(struct mapped_device *md) 1730 { 1731 return blk_queue_stackable(md->queue); 1732 } 1733 1734 static void dm_dispatch_clone_request(struct request *clone, struct request *rq) 1735 { 1736 int r; 1737 1738 if (blk_queue_io_stat(clone->q)) 1739 clone->cmd_flags |= REQ_IO_STAT; 1740 1741 clone->start_time = jiffies; 1742 r = blk_insert_cloned_request(clone->q, clone); 1743 if (r) 1744 /* must complete clone in terms of original request */ 1745 dm_complete_request(rq, r); 1746 } 1747 1748 static int dm_rq_bio_constructor(struct bio *bio, struct bio *bio_orig, 1749 void *data) 1750 { 1751 struct dm_rq_target_io *tio = data; 1752 struct dm_rq_clone_bio_info *info = 1753 container_of(bio, struct dm_rq_clone_bio_info, clone); 1754 1755 info->orig = bio_orig; 1756 info->tio = tio; 1757 bio->bi_end_io = end_clone_bio; 1758 1759 return 0; 1760 } 1761 1762 static int setup_clone(struct request *clone, struct request *rq, 1763 struct dm_rq_target_io *tio, gfp_t gfp_mask) 1764 { 1765 int r; 1766 1767 r = blk_rq_prep_clone(clone, rq, tio->md->bs, gfp_mask, 1768 dm_rq_bio_constructor, tio); 1769 if (r) 1770 return r; 1771 1772 clone->cmd = rq->cmd; 1773 clone->cmd_len = rq->cmd_len; 1774 clone->sense = rq->sense; 1775 clone->end_io = end_clone_request; 1776 clone->end_io_data = tio; 1777 1778 tio->clone = clone; 1779 1780 return 0; 1781 } 1782 1783 static struct request *clone_rq(struct request *rq, struct mapped_device *md, 1784 struct dm_rq_target_io *tio, gfp_t gfp_mask) 1785 { 1786 struct request *clone = alloc_clone_request(md, gfp_mask); 1787 1788 if (!clone) 1789 return NULL; 1790 1791 blk_rq_init(NULL, clone); 1792 if (setup_clone(clone, rq, tio, gfp_mask)) { 1793 /* -ENOMEM */ 1794 free_clone_request(md, clone); 1795 return NULL; 1796 } 1797 1798 return clone; 1799 } 1800 1801 static void map_tio_request(struct kthread_work *work); 1802 1803 static struct dm_rq_target_io *prep_tio(struct request *rq, 1804 struct mapped_device *md, gfp_t gfp_mask) 1805 { 1806 struct dm_rq_target_io *tio; 1807 int srcu_idx; 1808 struct dm_table *table; 1809 1810 tio = alloc_rq_tio(md, gfp_mask); 1811 if (!tio) 1812 return NULL; 1813 1814 tio->md = md; 1815 tio->ti = NULL; 1816 tio->clone = NULL; 1817 tio->orig = rq; 1818 tio->error = 0; 1819 memset(&tio->info, 0, sizeof(tio->info)); 1820 init_kthread_work(&tio->work, map_tio_request); 1821 1822 table = dm_get_live_table(md, &srcu_idx); 1823 if (!dm_table_mq_request_based(table)) { 1824 if (!clone_rq(rq, md, tio, gfp_mask)) { 1825 dm_put_live_table(md, srcu_idx); 1826 free_rq_tio(tio); 1827 return NULL; 1828 } 1829 } 1830 dm_put_live_table(md, srcu_idx); 1831 1832 return tio; 1833 } 1834 1835 /* 1836 * Called with the queue lock held. 1837 */ 1838 static int dm_prep_fn(struct request_queue *q, struct request *rq) 1839 { 1840 struct mapped_device *md = q->queuedata; 1841 struct dm_rq_target_io *tio; 1842 1843 if (unlikely(rq->special)) { 1844 DMWARN("Already has something in rq->special."); 1845 return BLKPREP_KILL; 1846 } 1847 1848 tio = prep_tio(rq, md, GFP_ATOMIC); 1849 if (!tio) 1850 return BLKPREP_DEFER; 1851 1852 rq->special = tio; 1853 rq->cmd_flags |= REQ_DONTPREP; 1854 1855 return BLKPREP_OK; 1856 } 1857 1858 /* 1859 * Returns: 1860 * 0 : the request has been processed 1861 * DM_MAPIO_REQUEUE : the original request needs to be requeued 1862 * < 0 : the request was completed due to failure 1863 */ 1864 static int map_request(struct dm_target *ti, struct request *rq, 1865 struct mapped_device *md) 1866 { 1867 int r; 1868 struct dm_rq_target_io *tio = rq->special; 1869 struct request *clone = NULL; 1870 1871 if (tio->clone) { 1872 clone = tio->clone; 1873 r = ti->type->map_rq(ti, clone, &tio->info); 1874 } else { 1875 r = ti->type->clone_and_map_rq(ti, rq, &tio->info, &clone); 1876 if (r < 0) { 1877 /* The target wants to complete the I/O */ 1878 dm_kill_unmapped_request(rq, r); 1879 return r; 1880 } 1881 if (IS_ERR(clone)) 1882 return DM_MAPIO_REQUEUE; 1883 if (setup_clone(clone, rq, tio, GFP_KERNEL)) { 1884 /* -ENOMEM */ 1885 ti->type->release_clone_rq(clone); 1886 return DM_MAPIO_REQUEUE; 1887 } 1888 } 1889 1890 switch (r) { 1891 case DM_MAPIO_SUBMITTED: 1892 /* The target has taken the I/O to submit by itself later */ 1893 break; 1894 case DM_MAPIO_REMAPPED: 1895 /* The target has remapped the I/O so dispatch it */ 1896 trace_block_rq_remap(clone->q, clone, disk_devt(dm_disk(md)), 1897 blk_rq_pos(rq)); 1898 dm_dispatch_clone_request(clone, rq); 1899 break; 1900 case DM_MAPIO_REQUEUE: 1901 /* The target wants to requeue the I/O */ 1902 dm_requeue_unmapped_request(clone); 1903 break; 1904 default: 1905 if (r > 0) { 1906 DMWARN("unimplemented target map return value: %d", r); 1907 BUG(); 1908 } 1909 1910 /* The target wants to complete the I/O */ 1911 dm_kill_unmapped_request(rq, r); 1912 return r; 1913 } 1914 1915 return 0; 1916 } 1917 1918 static void map_tio_request(struct kthread_work *work) 1919 { 1920 struct dm_rq_target_io *tio = container_of(work, struct dm_rq_target_io, work); 1921 struct request *rq = tio->orig; 1922 struct mapped_device *md = tio->md; 1923 1924 if (map_request(tio->ti, rq, md) == DM_MAPIO_REQUEUE) 1925 dm_requeue_unmapped_original_request(md, rq); 1926 } 1927 1928 static void dm_start_request(struct mapped_device *md, struct request *orig) 1929 { 1930 blk_start_request(orig); 1931 atomic_inc(&md->pending[rq_data_dir(orig)]); 1932 1933 /* 1934 * Hold the md reference here for the in-flight I/O. 1935 * We can't rely on the reference count by device opener, 1936 * because the device may be closed during the request completion 1937 * when all bios are completed. 1938 * See the comment in rq_completed() too. 1939 */ 1940 dm_get(md); 1941 } 1942 1943 /* 1944 * q->request_fn for request-based dm. 1945 * Called with the queue lock held. 1946 */ 1947 static void dm_request_fn(struct request_queue *q) 1948 { 1949 struct mapped_device *md = q->queuedata; 1950 int srcu_idx; 1951 struct dm_table *map = dm_get_live_table(md, &srcu_idx); 1952 struct dm_target *ti; 1953 struct request *rq; 1954 struct dm_rq_target_io *tio; 1955 sector_t pos; 1956 1957 /* 1958 * For suspend, check blk_queue_stopped() and increment 1959 * ->pending within a single queue_lock not to increment the 1960 * number of in-flight I/Os after the queue is stopped in 1961 * dm_suspend(). 1962 */ 1963 while (!blk_queue_stopped(q)) { 1964 rq = blk_peek_request(q); 1965 if (!rq) 1966 goto out; 1967 1968 /* always use block 0 to find the target for flushes for now */ 1969 pos = 0; 1970 if (!(rq->cmd_flags & REQ_FLUSH)) 1971 pos = blk_rq_pos(rq); 1972 1973 ti = dm_table_find_target(map, pos); 1974 if (!dm_target_is_valid(ti)) { 1975 /* 1976 * Must perform setup, that rq_completed() requires, 1977 * before calling dm_kill_unmapped_request 1978 */ 1979 DMERR_LIMIT("request attempted access beyond the end of device"); 1980 dm_start_request(md, rq); 1981 dm_kill_unmapped_request(rq, -EIO); 1982 continue; 1983 } 1984 1985 if (ti->type->busy && ti->type->busy(ti)) 1986 goto delay_and_out; 1987 1988 dm_start_request(md, rq); 1989 1990 tio = rq->special; 1991 /* Establish tio->ti before queuing work (map_tio_request) */ 1992 tio->ti = ti; 1993 queue_kthread_work(&md->kworker, &tio->work); 1994 BUG_ON(!irqs_disabled()); 1995 } 1996 1997 goto out; 1998 1999 delay_and_out: 2000 blk_delay_queue(q, HZ / 100); 2001 out: 2002 dm_put_live_table(md, srcu_idx); 2003 } 2004 2005 static int dm_any_congested(void *congested_data, int bdi_bits) 2006 { 2007 int r = bdi_bits; 2008 struct mapped_device *md = congested_data; 2009 struct dm_table *map; 2010 2011 if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) { 2012 map = dm_get_live_table_fast(md); 2013 if (map) { 2014 /* 2015 * Request-based dm cares about only own queue for 2016 * the query about congestion status of request_queue 2017 */ 2018 if (dm_request_based(md)) 2019 r = md->queue->backing_dev_info.state & 2020 bdi_bits; 2021 else 2022 r = dm_table_any_congested(map, bdi_bits); 2023 } 2024 dm_put_live_table_fast(md); 2025 } 2026 2027 return r; 2028 } 2029 2030 /*----------------------------------------------------------------- 2031 * An IDR is used to keep track of allocated minor numbers. 2032 *---------------------------------------------------------------*/ 2033 static void free_minor(int minor) 2034 { 2035 spin_lock(&_minor_lock); 2036 idr_remove(&_minor_idr, minor); 2037 spin_unlock(&_minor_lock); 2038 } 2039 2040 /* 2041 * See if the device with a specific minor # is free. 2042 */ 2043 static int specific_minor(int minor) 2044 { 2045 int r; 2046 2047 if (minor >= (1 << MINORBITS)) 2048 return -EINVAL; 2049 2050 idr_preload(GFP_KERNEL); 2051 spin_lock(&_minor_lock); 2052 2053 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT); 2054 2055 spin_unlock(&_minor_lock); 2056 idr_preload_end(); 2057 if (r < 0) 2058 return r == -ENOSPC ? -EBUSY : r; 2059 return 0; 2060 } 2061 2062 static int next_free_minor(int *minor) 2063 { 2064 int r; 2065 2066 idr_preload(GFP_KERNEL); 2067 spin_lock(&_minor_lock); 2068 2069 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT); 2070 2071 spin_unlock(&_minor_lock); 2072 idr_preload_end(); 2073 if (r < 0) 2074 return r; 2075 *minor = r; 2076 return 0; 2077 } 2078 2079 static const struct block_device_operations dm_blk_dops; 2080 2081 static void dm_wq_work(struct work_struct *work); 2082 2083 static void dm_init_md_queue(struct mapped_device *md) 2084 { 2085 /* 2086 * Request-based dm devices cannot be stacked on top of bio-based dm 2087 * devices. The type of this dm device has not been decided yet. 2088 * The type is decided at the first table loading time. 2089 * To prevent problematic device stacking, clear the queue flag 2090 * for request stacking support until then. 2091 * 2092 * This queue is new, so no concurrency on the queue_flags. 2093 */ 2094 queue_flag_clear_unlocked(QUEUE_FLAG_STACKABLE, md->queue); 2095 2096 md->queue->queuedata = md; 2097 md->queue->backing_dev_info.congested_fn = dm_any_congested; 2098 md->queue->backing_dev_info.congested_data = md; 2099 2100 blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY); 2101 } 2102 2103 /* 2104 * Allocate and initialise a blank device with a given minor. 2105 */ 2106 static struct mapped_device *alloc_dev(int minor) 2107 { 2108 int r; 2109 struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL); 2110 void *old_md; 2111 2112 if (!md) { 2113 DMWARN("unable to allocate device, out of memory."); 2114 return NULL; 2115 } 2116 2117 if (!try_module_get(THIS_MODULE)) 2118 goto bad_module_get; 2119 2120 /* get a minor number for the dev */ 2121 if (minor == DM_ANY_MINOR) 2122 r = next_free_minor(&minor); 2123 else 2124 r = specific_minor(minor); 2125 if (r < 0) 2126 goto bad_minor; 2127 2128 r = init_srcu_struct(&md->io_barrier); 2129 if (r < 0) 2130 goto bad_io_barrier; 2131 2132 md->type = DM_TYPE_NONE; 2133 mutex_init(&md->suspend_lock); 2134 mutex_init(&md->type_lock); 2135 mutex_init(&md->table_devices_lock); 2136 spin_lock_init(&md->deferred_lock); 2137 atomic_set(&md->holders, 1); 2138 atomic_set(&md->open_count, 0); 2139 atomic_set(&md->event_nr, 0); 2140 atomic_set(&md->uevent_seq, 0); 2141 INIT_LIST_HEAD(&md->uevent_list); 2142 INIT_LIST_HEAD(&md->table_devices); 2143 spin_lock_init(&md->uevent_lock); 2144 2145 md->queue = blk_alloc_queue(GFP_KERNEL); 2146 if (!md->queue) 2147 goto bad_queue; 2148 2149 dm_init_md_queue(md); 2150 2151 md->disk = alloc_disk(1); 2152 if (!md->disk) 2153 goto bad_disk; 2154 2155 atomic_set(&md->pending[0], 0); 2156 atomic_set(&md->pending[1], 0); 2157 init_waitqueue_head(&md->wait); 2158 INIT_WORK(&md->work, dm_wq_work); 2159 init_waitqueue_head(&md->eventq); 2160 init_completion(&md->kobj_holder.completion); 2161 md->kworker_task = NULL; 2162 2163 md->disk->major = _major; 2164 md->disk->first_minor = minor; 2165 md->disk->fops = &dm_blk_dops; 2166 md->disk->queue = md->queue; 2167 md->disk->private_data = md; 2168 sprintf(md->disk->disk_name, "dm-%d", minor); 2169 add_disk(md->disk); 2170 format_dev_t(md->name, MKDEV(_major, minor)); 2171 2172 md->wq = alloc_workqueue("kdmflush", WQ_MEM_RECLAIM, 0); 2173 if (!md->wq) 2174 goto bad_thread; 2175 2176 md->bdev = bdget_disk(md->disk, 0); 2177 if (!md->bdev) 2178 goto bad_bdev; 2179 2180 bio_init(&md->flush_bio); 2181 md->flush_bio.bi_bdev = md->bdev; 2182 md->flush_bio.bi_rw = WRITE_FLUSH; 2183 2184 dm_stats_init(&md->stats); 2185 2186 /* Populate the mapping, nobody knows we exist yet */ 2187 spin_lock(&_minor_lock); 2188 old_md = idr_replace(&_minor_idr, md, minor); 2189 spin_unlock(&_minor_lock); 2190 2191 BUG_ON(old_md != MINOR_ALLOCED); 2192 2193 return md; 2194 2195 bad_bdev: 2196 destroy_workqueue(md->wq); 2197 bad_thread: 2198 del_gendisk(md->disk); 2199 put_disk(md->disk); 2200 bad_disk: 2201 blk_cleanup_queue(md->queue); 2202 bad_queue: 2203 cleanup_srcu_struct(&md->io_barrier); 2204 bad_io_barrier: 2205 free_minor(minor); 2206 bad_minor: 2207 module_put(THIS_MODULE); 2208 bad_module_get: 2209 kfree(md); 2210 return NULL; 2211 } 2212 2213 static void unlock_fs(struct mapped_device *md); 2214 2215 static void free_dev(struct mapped_device *md) 2216 { 2217 int minor = MINOR(disk_devt(md->disk)); 2218 2219 unlock_fs(md); 2220 destroy_workqueue(md->wq); 2221 2222 if (md->kworker_task) 2223 kthread_stop(md->kworker_task); 2224 if (md->io_pool) 2225 mempool_destroy(md->io_pool); 2226 if (md->rq_pool) 2227 mempool_destroy(md->rq_pool); 2228 if (md->bs) 2229 bioset_free(md->bs); 2230 2231 cleanup_srcu_struct(&md->io_barrier); 2232 free_table_devices(&md->table_devices); 2233 dm_stats_cleanup(&md->stats); 2234 2235 spin_lock(&_minor_lock); 2236 md->disk->private_data = NULL; 2237 spin_unlock(&_minor_lock); 2238 if (blk_get_integrity(md->disk)) 2239 blk_integrity_unregister(md->disk); 2240 del_gendisk(md->disk); 2241 put_disk(md->disk); 2242 blk_cleanup_queue(md->queue); 2243 bdput(md->bdev); 2244 free_minor(minor); 2245 2246 module_put(THIS_MODULE); 2247 kfree(md); 2248 } 2249 2250 static void __bind_mempools(struct mapped_device *md, struct dm_table *t) 2251 { 2252 struct dm_md_mempools *p = dm_table_get_md_mempools(t); 2253 2254 if (md->io_pool && md->bs) { 2255 /* The md already has necessary mempools. */ 2256 if (dm_table_get_type(t) == DM_TYPE_BIO_BASED) { 2257 /* 2258 * Reload bioset because front_pad may have changed 2259 * because a different table was loaded. 2260 */ 2261 bioset_free(md->bs); 2262 md->bs = p->bs; 2263 p->bs = NULL; 2264 } 2265 /* 2266 * There's no need to reload with request-based dm 2267 * because the size of front_pad doesn't change. 2268 * Note for future: If you are to reload bioset, 2269 * prep-ed requests in the queue may refer 2270 * to bio from the old bioset, so you must walk 2271 * through the queue to unprep. 2272 */ 2273 goto out; 2274 } 2275 2276 BUG_ON(!p || md->io_pool || md->rq_pool || md->bs); 2277 2278 md->io_pool = p->io_pool; 2279 p->io_pool = NULL; 2280 md->rq_pool = p->rq_pool; 2281 p->rq_pool = NULL; 2282 md->bs = p->bs; 2283 p->bs = NULL; 2284 2285 out: 2286 /* mempool bind completed, now no need any mempools in the table */ 2287 dm_table_free_md_mempools(t); 2288 } 2289 2290 /* 2291 * Bind a table to the device. 2292 */ 2293 static void event_callback(void *context) 2294 { 2295 unsigned long flags; 2296 LIST_HEAD(uevents); 2297 struct mapped_device *md = (struct mapped_device *) context; 2298 2299 spin_lock_irqsave(&md->uevent_lock, flags); 2300 list_splice_init(&md->uevent_list, &uevents); 2301 spin_unlock_irqrestore(&md->uevent_lock, flags); 2302 2303 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj); 2304 2305 atomic_inc(&md->event_nr); 2306 wake_up(&md->eventq); 2307 } 2308 2309 /* 2310 * Protected by md->suspend_lock obtained by dm_swap_table(). 2311 */ 2312 static void __set_size(struct mapped_device *md, sector_t size) 2313 { 2314 set_capacity(md->disk, size); 2315 2316 i_size_write(md->bdev->bd_inode, (loff_t)size << SECTOR_SHIFT); 2317 } 2318 2319 /* 2320 * Return 1 if the queue has a compulsory merge_bvec_fn function. 2321 * 2322 * If this function returns 0, then the device is either a non-dm 2323 * device without a merge_bvec_fn, or it is a dm device that is 2324 * able to split any bios it receives that are too big. 2325 */ 2326 int dm_queue_merge_is_compulsory(struct request_queue *q) 2327 { 2328 struct mapped_device *dev_md; 2329 2330 if (!q->merge_bvec_fn) 2331 return 0; 2332 2333 if (q->make_request_fn == dm_make_request) { 2334 dev_md = q->queuedata; 2335 if (test_bit(DMF_MERGE_IS_OPTIONAL, &dev_md->flags)) 2336 return 0; 2337 } 2338 2339 return 1; 2340 } 2341 2342 static int dm_device_merge_is_compulsory(struct dm_target *ti, 2343 struct dm_dev *dev, sector_t start, 2344 sector_t len, void *data) 2345 { 2346 struct block_device *bdev = dev->bdev; 2347 struct request_queue *q = bdev_get_queue(bdev); 2348 2349 return dm_queue_merge_is_compulsory(q); 2350 } 2351 2352 /* 2353 * Return 1 if it is acceptable to ignore merge_bvec_fn based 2354 * on the properties of the underlying devices. 2355 */ 2356 static int dm_table_merge_is_optional(struct dm_table *table) 2357 { 2358 unsigned i = 0; 2359 struct dm_target *ti; 2360 2361 while (i < dm_table_get_num_targets(table)) { 2362 ti = dm_table_get_target(table, i++); 2363 2364 if (ti->type->iterate_devices && 2365 ti->type->iterate_devices(ti, dm_device_merge_is_compulsory, NULL)) 2366 return 0; 2367 } 2368 2369 return 1; 2370 } 2371 2372 /* 2373 * Returns old map, which caller must destroy. 2374 */ 2375 static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t, 2376 struct queue_limits *limits) 2377 { 2378 struct dm_table *old_map; 2379 struct request_queue *q = md->queue; 2380 sector_t size; 2381 int merge_is_optional; 2382 2383 size = dm_table_get_size(t); 2384 2385 /* 2386 * Wipe any geometry if the size of the table changed. 2387 */ 2388 if (size != dm_get_size(md)) 2389 memset(&md->geometry, 0, sizeof(md->geometry)); 2390 2391 __set_size(md, size); 2392 2393 dm_table_event_callback(t, event_callback, md); 2394 2395 /* 2396 * The queue hasn't been stopped yet, if the old table type wasn't 2397 * for request-based during suspension. So stop it to prevent 2398 * I/O mapping before resume. 2399 * This must be done before setting the queue restrictions, 2400 * because request-based dm may be run just after the setting. 2401 */ 2402 if (dm_table_request_based(t) && !blk_queue_stopped(q)) 2403 stop_queue(q); 2404 2405 __bind_mempools(md, t); 2406 2407 merge_is_optional = dm_table_merge_is_optional(t); 2408 2409 old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2410 rcu_assign_pointer(md->map, t); 2411 md->immutable_target_type = dm_table_get_immutable_target_type(t); 2412 2413 dm_table_set_restrictions(t, q, limits); 2414 if (merge_is_optional) 2415 set_bit(DMF_MERGE_IS_OPTIONAL, &md->flags); 2416 else 2417 clear_bit(DMF_MERGE_IS_OPTIONAL, &md->flags); 2418 if (old_map) 2419 dm_sync_table(md); 2420 2421 return old_map; 2422 } 2423 2424 /* 2425 * Returns unbound table for the caller to free. 2426 */ 2427 static struct dm_table *__unbind(struct mapped_device *md) 2428 { 2429 struct dm_table *map = rcu_dereference_protected(md->map, 1); 2430 2431 if (!map) 2432 return NULL; 2433 2434 dm_table_event_callback(map, NULL, NULL); 2435 RCU_INIT_POINTER(md->map, NULL); 2436 dm_sync_table(md); 2437 2438 return map; 2439 } 2440 2441 /* 2442 * Constructor for a new device. 2443 */ 2444 int dm_create(int minor, struct mapped_device **result) 2445 { 2446 struct mapped_device *md; 2447 2448 md = alloc_dev(minor); 2449 if (!md) 2450 return -ENXIO; 2451 2452 dm_sysfs_init(md); 2453 2454 *result = md; 2455 return 0; 2456 } 2457 2458 /* 2459 * Functions to manage md->type. 2460 * All are required to hold md->type_lock. 2461 */ 2462 void dm_lock_md_type(struct mapped_device *md) 2463 { 2464 mutex_lock(&md->type_lock); 2465 } 2466 2467 void dm_unlock_md_type(struct mapped_device *md) 2468 { 2469 mutex_unlock(&md->type_lock); 2470 } 2471 2472 void dm_set_md_type(struct mapped_device *md, unsigned type) 2473 { 2474 BUG_ON(!mutex_is_locked(&md->type_lock)); 2475 md->type = type; 2476 } 2477 2478 unsigned dm_get_md_type(struct mapped_device *md) 2479 { 2480 BUG_ON(!mutex_is_locked(&md->type_lock)); 2481 return md->type; 2482 } 2483 2484 static bool dm_md_type_request_based(struct mapped_device *md) 2485 { 2486 unsigned table_type = dm_get_md_type(md); 2487 2488 return (table_type == DM_TYPE_REQUEST_BASED || 2489 table_type == DM_TYPE_MQ_REQUEST_BASED); 2490 } 2491 2492 struct target_type *dm_get_immutable_target_type(struct mapped_device *md) 2493 { 2494 return md->immutable_target_type; 2495 } 2496 2497 /* 2498 * The queue_limits are only valid as long as you have a reference 2499 * count on 'md'. 2500 */ 2501 struct queue_limits *dm_get_queue_limits(struct mapped_device *md) 2502 { 2503 BUG_ON(!atomic_read(&md->holders)); 2504 return &md->queue->limits; 2505 } 2506 EXPORT_SYMBOL_GPL(dm_get_queue_limits); 2507 2508 /* 2509 * Fully initialize a request-based queue (->elevator, ->request_fn, etc). 2510 */ 2511 static int dm_init_request_based_queue(struct mapped_device *md) 2512 { 2513 struct request_queue *q = NULL; 2514 2515 if (md->queue->elevator) 2516 return 1; 2517 2518 /* Fully initialize the queue */ 2519 q = blk_init_allocated_queue(md->queue, dm_request_fn, NULL); 2520 if (!q) 2521 return 0; 2522 2523 md->queue = q; 2524 dm_init_md_queue(md); 2525 blk_queue_softirq_done(md->queue, dm_softirq_done); 2526 blk_queue_prep_rq(md->queue, dm_prep_fn); 2527 2528 /* Also initialize the request-based DM worker thread */ 2529 init_kthread_worker(&md->kworker); 2530 md->kworker_task = kthread_run(kthread_worker_fn, &md->kworker, 2531 "kdmwork-%s", dm_device_name(md)); 2532 2533 elv_register_queue(md->queue); 2534 2535 return 1; 2536 } 2537 2538 /* 2539 * Setup the DM device's queue based on md's type 2540 */ 2541 int dm_setup_md_queue(struct mapped_device *md) 2542 { 2543 if (dm_md_type_request_based(md)) { 2544 if (!dm_init_request_based_queue(md)) { 2545 DMWARN("Cannot initialize queue for request-based mapped device"); 2546 return -EINVAL; 2547 } 2548 } else { 2549 /* bio-based specific initialization */ 2550 blk_queue_make_request(md->queue, dm_make_request); 2551 blk_queue_merge_bvec(md->queue, dm_merge_bvec); 2552 } 2553 2554 return 0; 2555 } 2556 2557 struct mapped_device *dm_get_md(dev_t dev) 2558 { 2559 struct mapped_device *md; 2560 unsigned minor = MINOR(dev); 2561 2562 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS)) 2563 return NULL; 2564 2565 spin_lock(&_minor_lock); 2566 2567 md = idr_find(&_minor_idr, minor); 2568 if (md) { 2569 if ((md == MINOR_ALLOCED || 2570 (MINOR(disk_devt(dm_disk(md))) != minor) || 2571 dm_deleting_md(md) || 2572 test_bit(DMF_FREEING, &md->flags))) { 2573 md = NULL; 2574 goto out; 2575 } 2576 dm_get(md); 2577 } 2578 2579 out: 2580 spin_unlock(&_minor_lock); 2581 2582 return md; 2583 } 2584 EXPORT_SYMBOL_GPL(dm_get_md); 2585 2586 void *dm_get_mdptr(struct mapped_device *md) 2587 { 2588 return md->interface_ptr; 2589 } 2590 2591 void dm_set_mdptr(struct mapped_device *md, void *ptr) 2592 { 2593 md->interface_ptr = ptr; 2594 } 2595 2596 void dm_get(struct mapped_device *md) 2597 { 2598 atomic_inc(&md->holders); 2599 BUG_ON(test_bit(DMF_FREEING, &md->flags)); 2600 } 2601 2602 int dm_hold(struct mapped_device *md) 2603 { 2604 spin_lock(&_minor_lock); 2605 if (test_bit(DMF_FREEING, &md->flags)) { 2606 spin_unlock(&_minor_lock); 2607 return -EBUSY; 2608 } 2609 dm_get(md); 2610 spin_unlock(&_minor_lock); 2611 return 0; 2612 } 2613 EXPORT_SYMBOL_GPL(dm_hold); 2614 2615 const char *dm_device_name(struct mapped_device *md) 2616 { 2617 return md->name; 2618 } 2619 EXPORT_SYMBOL_GPL(dm_device_name); 2620 2621 static void __dm_destroy(struct mapped_device *md, bool wait) 2622 { 2623 struct dm_table *map; 2624 int srcu_idx; 2625 2626 might_sleep(); 2627 2628 map = dm_get_live_table(md, &srcu_idx); 2629 2630 spin_lock(&_minor_lock); 2631 idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md)))); 2632 set_bit(DMF_FREEING, &md->flags); 2633 spin_unlock(&_minor_lock); 2634 2635 if (dm_request_based(md)) 2636 flush_kthread_worker(&md->kworker); 2637 2638 /* 2639 * Take suspend_lock so that presuspend and postsuspend methods 2640 * do not race with internal suspend. 2641 */ 2642 mutex_lock(&md->suspend_lock); 2643 if (!dm_suspended_md(md)) { 2644 dm_table_presuspend_targets(map); 2645 dm_table_postsuspend_targets(map); 2646 } 2647 mutex_unlock(&md->suspend_lock); 2648 2649 /* dm_put_live_table must be before msleep, otherwise deadlock is possible */ 2650 dm_put_live_table(md, srcu_idx); 2651 2652 /* 2653 * Rare, but there may be I/O requests still going to complete, 2654 * for example. Wait for all references to disappear. 2655 * No one should increment the reference count of the mapped_device, 2656 * after the mapped_device state becomes DMF_FREEING. 2657 */ 2658 if (wait) 2659 while (atomic_read(&md->holders)) 2660 msleep(1); 2661 else if (atomic_read(&md->holders)) 2662 DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)", 2663 dm_device_name(md), atomic_read(&md->holders)); 2664 2665 dm_sysfs_exit(md); 2666 dm_table_destroy(__unbind(md)); 2667 free_dev(md); 2668 } 2669 2670 void dm_destroy(struct mapped_device *md) 2671 { 2672 __dm_destroy(md, true); 2673 } 2674 2675 void dm_destroy_immediate(struct mapped_device *md) 2676 { 2677 __dm_destroy(md, false); 2678 } 2679 2680 void dm_put(struct mapped_device *md) 2681 { 2682 atomic_dec(&md->holders); 2683 } 2684 EXPORT_SYMBOL_GPL(dm_put); 2685 2686 static int dm_wait_for_completion(struct mapped_device *md, int interruptible) 2687 { 2688 int r = 0; 2689 DECLARE_WAITQUEUE(wait, current); 2690 2691 add_wait_queue(&md->wait, &wait); 2692 2693 while (1) { 2694 set_current_state(interruptible); 2695 2696 if (!md_in_flight(md)) 2697 break; 2698 2699 if (interruptible == TASK_INTERRUPTIBLE && 2700 signal_pending(current)) { 2701 r = -EINTR; 2702 break; 2703 } 2704 2705 io_schedule(); 2706 } 2707 set_current_state(TASK_RUNNING); 2708 2709 remove_wait_queue(&md->wait, &wait); 2710 2711 return r; 2712 } 2713 2714 /* 2715 * Process the deferred bios 2716 */ 2717 static void dm_wq_work(struct work_struct *work) 2718 { 2719 struct mapped_device *md = container_of(work, struct mapped_device, 2720 work); 2721 struct bio *c; 2722 int srcu_idx; 2723 struct dm_table *map; 2724 2725 map = dm_get_live_table(md, &srcu_idx); 2726 2727 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) { 2728 spin_lock_irq(&md->deferred_lock); 2729 c = bio_list_pop(&md->deferred); 2730 spin_unlock_irq(&md->deferred_lock); 2731 2732 if (!c) 2733 break; 2734 2735 if (dm_request_based(md)) 2736 generic_make_request(c); 2737 else 2738 __split_and_process_bio(md, map, c); 2739 } 2740 2741 dm_put_live_table(md, srcu_idx); 2742 } 2743 2744 static void dm_queue_flush(struct mapped_device *md) 2745 { 2746 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 2747 smp_mb__after_atomic(); 2748 queue_work(md->wq, &md->work); 2749 } 2750 2751 /* 2752 * Swap in a new table, returning the old one for the caller to destroy. 2753 */ 2754 struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table) 2755 { 2756 struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL); 2757 struct queue_limits limits; 2758 int r; 2759 2760 mutex_lock(&md->suspend_lock); 2761 2762 /* device must be suspended */ 2763 if (!dm_suspended_md(md)) 2764 goto out; 2765 2766 /* 2767 * If the new table has no data devices, retain the existing limits. 2768 * This helps multipath with queue_if_no_path if all paths disappear, 2769 * then new I/O is queued based on these limits, and then some paths 2770 * reappear. 2771 */ 2772 if (dm_table_has_no_data_devices(table)) { 2773 live_map = dm_get_live_table_fast(md); 2774 if (live_map) 2775 limits = md->queue->limits; 2776 dm_put_live_table_fast(md); 2777 } 2778 2779 if (!live_map) { 2780 r = dm_calculate_queue_limits(table, &limits); 2781 if (r) { 2782 map = ERR_PTR(r); 2783 goto out; 2784 } 2785 } 2786 2787 map = __bind(md, table, &limits); 2788 2789 out: 2790 mutex_unlock(&md->suspend_lock); 2791 return map; 2792 } 2793 2794 /* 2795 * Functions to lock and unlock any filesystem running on the 2796 * device. 2797 */ 2798 static int lock_fs(struct mapped_device *md) 2799 { 2800 int r; 2801 2802 WARN_ON(md->frozen_sb); 2803 2804 md->frozen_sb = freeze_bdev(md->bdev); 2805 if (IS_ERR(md->frozen_sb)) { 2806 r = PTR_ERR(md->frozen_sb); 2807 md->frozen_sb = NULL; 2808 return r; 2809 } 2810 2811 set_bit(DMF_FROZEN, &md->flags); 2812 2813 return 0; 2814 } 2815 2816 static void unlock_fs(struct mapped_device *md) 2817 { 2818 if (!test_bit(DMF_FROZEN, &md->flags)) 2819 return; 2820 2821 thaw_bdev(md->bdev, md->frozen_sb); 2822 md->frozen_sb = NULL; 2823 clear_bit(DMF_FROZEN, &md->flags); 2824 } 2825 2826 /* 2827 * If __dm_suspend returns 0, the device is completely quiescent 2828 * now. There is no request-processing activity. All new requests 2829 * are being added to md->deferred list. 2830 * 2831 * Caller must hold md->suspend_lock 2832 */ 2833 static int __dm_suspend(struct mapped_device *md, struct dm_table *map, 2834 unsigned suspend_flags, int interruptible) 2835 { 2836 bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG; 2837 bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG; 2838 int r; 2839 2840 /* 2841 * DMF_NOFLUSH_SUSPENDING must be set before presuspend. 2842 * This flag is cleared before dm_suspend returns. 2843 */ 2844 if (noflush) 2845 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 2846 2847 /* 2848 * This gets reverted if there's an error later and the targets 2849 * provide the .presuspend_undo hook. 2850 */ 2851 dm_table_presuspend_targets(map); 2852 2853 /* 2854 * Flush I/O to the device. 2855 * Any I/O submitted after lock_fs() may not be flushed. 2856 * noflush takes precedence over do_lockfs. 2857 * (lock_fs() flushes I/Os and waits for them to complete.) 2858 */ 2859 if (!noflush && do_lockfs) { 2860 r = lock_fs(md); 2861 if (r) { 2862 dm_table_presuspend_undo_targets(map); 2863 return r; 2864 } 2865 } 2866 2867 /* 2868 * Here we must make sure that no processes are submitting requests 2869 * to target drivers i.e. no one may be executing 2870 * __split_and_process_bio. This is called from dm_request and 2871 * dm_wq_work. 2872 * 2873 * To get all processes out of __split_and_process_bio in dm_request, 2874 * we take the write lock. To prevent any process from reentering 2875 * __split_and_process_bio from dm_request and quiesce the thread 2876 * (dm_wq_work), we set BMF_BLOCK_IO_FOR_SUSPEND and call 2877 * flush_workqueue(md->wq). 2878 */ 2879 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 2880 if (map) 2881 synchronize_srcu(&md->io_barrier); 2882 2883 /* 2884 * Stop md->queue before flushing md->wq in case request-based 2885 * dm defers requests to md->wq from md->queue. 2886 */ 2887 if (dm_request_based(md)) { 2888 stop_queue(md->queue); 2889 flush_kthread_worker(&md->kworker); 2890 } 2891 2892 flush_workqueue(md->wq); 2893 2894 /* 2895 * At this point no more requests are entering target request routines. 2896 * We call dm_wait_for_completion to wait for all existing requests 2897 * to finish. 2898 */ 2899 r = dm_wait_for_completion(md, interruptible); 2900 2901 if (noflush) 2902 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 2903 if (map) 2904 synchronize_srcu(&md->io_barrier); 2905 2906 /* were we interrupted ? */ 2907 if (r < 0) { 2908 dm_queue_flush(md); 2909 2910 if (dm_request_based(md)) 2911 start_queue(md->queue); 2912 2913 unlock_fs(md); 2914 dm_table_presuspend_undo_targets(map); 2915 /* pushback list is already flushed, so skip flush */ 2916 } 2917 2918 return r; 2919 } 2920 2921 /* 2922 * We need to be able to change a mapping table under a mounted 2923 * filesystem. For example we might want to move some data in 2924 * the background. Before the table can be swapped with 2925 * dm_bind_table, dm_suspend must be called to flush any in 2926 * flight bios and ensure that any further io gets deferred. 2927 */ 2928 /* 2929 * Suspend mechanism in request-based dm. 2930 * 2931 * 1. Flush all I/Os by lock_fs() if needed. 2932 * 2. Stop dispatching any I/O by stopping the request_queue. 2933 * 3. Wait for all in-flight I/Os to be completed or requeued. 2934 * 2935 * To abort suspend, start the request_queue. 2936 */ 2937 int dm_suspend(struct mapped_device *md, unsigned suspend_flags) 2938 { 2939 struct dm_table *map = NULL; 2940 int r = 0; 2941 2942 retry: 2943 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); 2944 2945 if (dm_suspended_md(md)) { 2946 r = -EINVAL; 2947 goto out_unlock; 2948 } 2949 2950 if (dm_suspended_internally_md(md)) { 2951 /* already internally suspended, wait for internal resume */ 2952 mutex_unlock(&md->suspend_lock); 2953 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); 2954 if (r) 2955 return r; 2956 goto retry; 2957 } 2958 2959 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2960 2961 r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE); 2962 if (r) 2963 goto out_unlock; 2964 2965 set_bit(DMF_SUSPENDED, &md->flags); 2966 2967 dm_table_postsuspend_targets(map); 2968 2969 out_unlock: 2970 mutex_unlock(&md->suspend_lock); 2971 return r; 2972 } 2973 2974 static int __dm_resume(struct mapped_device *md, struct dm_table *map) 2975 { 2976 if (map) { 2977 int r = dm_table_resume_targets(map); 2978 if (r) 2979 return r; 2980 } 2981 2982 dm_queue_flush(md); 2983 2984 /* 2985 * Flushing deferred I/Os must be done after targets are resumed 2986 * so that mapping of targets can work correctly. 2987 * Request-based dm is queueing the deferred I/Os in its request_queue. 2988 */ 2989 if (dm_request_based(md)) 2990 start_queue(md->queue); 2991 2992 unlock_fs(md); 2993 2994 return 0; 2995 } 2996 2997 int dm_resume(struct mapped_device *md) 2998 { 2999 int r = -EINVAL; 3000 struct dm_table *map = NULL; 3001 3002 retry: 3003 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); 3004 3005 if (!dm_suspended_md(md)) 3006 goto out; 3007 3008 if (dm_suspended_internally_md(md)) { 3009 /* already internally suspended, wait for internal resume */ 3010 mutex_unlock(&md->suspend_lock); 3011 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); 3012 if (r) 3013 return r; 3014 goto retry; 3015 } 3016 3017 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 3018 if (!map || !dm_table_get_size(map)) 3019 goto out; 3020 3021 r = __dm_resume(md, map); 3022 if (r) 3023 goto out; 3024 3025 clear_bit(DMF_SUSPENDED, &md->flags); 3026 3027 r = 0; 3028 out: 3029 mutex_unlock(&md->suspend_lock); 3030 3031 return r; 3032 } 3033 3034 /* 3035 * Internal suspend/resume works like userspace-driven suspend. It waits 3036 * until all bios finish and prevents issuing new bios to the target drivers. 3037 * It may be used only from the kernel. 3038 */ 3039 3040 static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags) 3041 { 3042 struct dm_table *map = NULL; 3043 3044 if (md->internal_suspend_count++) 3045 return; /* nested internal suspend */ 3046 3047 if (dm_suspended_md(md)) { 3048 set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 3049 return; /* nest suspend */ 3050 } 3051 3052 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 3053 3054 /* 3055 * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is 3056 * supported. Properly supporting a TASK_INTERRUPTIBLE internal suspend 3057 * would require changing .presuspend to return an error -- avoid this 3058 * until there is a need for more elaborate variants of internal suspend. 3059 */ 3060 (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE); 3061 3062 set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 3063 3064 dm_table_postsuspend_targets(map); 3065 } 3066 3067 static void __dm_internal_resume(struct mapped_device *md) 3068 { 3069 BUG_ON(!md->internal_suspend_count); 3070 3071 if (--md->internal_suspend_count) 3072 return; /* resume from nested internal suspend */ 3073 3074 if (dm_suspended_md(md)) 3075 goto done; /* resume from nested suspend */ 3076 3077 /* 3078 * NOTE: existing callers don't need to call dm_table_resume_targets 3079 * (which may fail -- so best to avoid it for now by passing NULL map) 3080 */ 3081 (void) __dm_resume(md, NULL); 3082 3083 done: 3084 clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 3085 smp_mb__after_atomic(); 3086 wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY); 3087 } 3088 3089 void dm_internal_suspend_noflush(struct mapped_device *md) 3090 { 3091 mutex_lock(&md->suspend_lock); 3092 __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG); 3093 mutex_unlock(&md->suspend_lock); 3094 } 3095 EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush); 3096 3097 void dm_internal_resume(struct mapped_device *md) 3098 { 3099 mutex_lock(&md->suspend_lock); 3100 __dm_internal_resume(md); 3101 mutex_unlock(&md->suspend_lock); 3102 } 3103 EXPORT_SYMBOL_GPL(dm_internal_resume); 3104 3105 /* 3106 * Fast variants of internal suspend/resume hold md->suspend_lock, 3107 * which prevents interaction with userspace-driven suspend. 3108 */ 3109 3110 void dm_internal_suspend_fast(struct mapped_device *md) 3111 { 3112 mutex_lock(&md->suspend_lock); 3113 if (dm_suspended_md(md) || dm_suspended_internally_md(md)) 3114 return; 3115 3116 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 3117 synchronize_srcu(&md->io_barrier); 3118 flush_workqueue(md->wq); 3119 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE); 3120 } 3121 EXPORT_SYMBOL_GPL(dm_internal_suspend_fast); 3122 3123 void dm_internal_resume_fast(struct mapped_device *md) 3124 { 3125 if (dm_suspended_md(md) || dm_suspended_internally_md(md)) 3126 goto done; 3127 3128 dm_queue_flush(md); 3129 3130 done: 3131 mutex_unlock(&md->suspend_lock); 3132 } 3133 EXPORT_SYMBOL_GPL(dm_internal_resume_fast); 3134 3135 /*----------------------------------------------------------------- 3136 * Event notification. 3137 *---------------------------------------------------------------*/ 3138 int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action, 3139 unsigned cookie) 3140 { 3141 char udev_cookie[DM_COOKIE_LENGTH]; 3142 char *envp[] = { udev_cookie, NULL }; 3143 3144 if (!cookie) 3145 return kobject_uevent(&disk_to_dev(md->disk)->kobj, action); 3146 else { 3147 snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u", 3148 DM_COOKIE_ENV_VAR_NAME, cookie); 3149 return kobject_uevent_env(&disk_to_dev(md->disk)->kobj, 3150 action, envp); 3151 } 3152 } 3153 3154 uint32_t dm_next_uevent_seq(struct mapped_device *md) 3155 { 3156 return atomic_add_return(1, &md->uevent_seq); 3157 } 3158 3159 uint32_t dm_get_event_nr(struct mapped_device *md) 3160 { 3161 return atomic_read(&md->event_nr); 3162 } 3163 3164 int dm_wait_event(struct mapped_device *md, int event_nr) 3165 { 3166 return wait_event_interruptible(md->eventq, 3167 (event_nr != atomic_read(&md->event_nr))); 3168 } 3169 3170 void dm_uevent_add(struct mapped_device *md, struct list_head *elist) 3171 { 3172 unsigned long flags; 3173 3174 spin_lock_irqsave(&md->uevent_lock, flags); 3175 list_add(elist, &md->uevent_list); 3176 spin_unlock_irqrestore(&md->uevent_lock, flags); 3177 } 3178 3179 /* 3180 * The gendisk is only valid as long as you have a reference 3181 * count on 'md'. 3182 */ 3183 struct gendisk *dm_disk(struct mapped_device *md) 3184 { 3185 return md->disk; 3186 } 3187 3188 struct kobject *dm_kobject(struct mapped_device *md) 3189 { 3190 return &md->kobj_holder.kobj; 3191 } 3192 3193 struct mapped_device *dm_get_from_kobject(struct kobject *kobj) 3194 { 3195 struct mapped_device *md; 3196 3197 md = container_of(kobj, struct mapped_device, kobj_holder.kobj); 3198 3199 if (test_bit(DMF_FREEING, &md->flags) || 3200 dm_deleting_md(md)) 3201 return NULL; 3202 3203 dm_get(md); 3204 return md; 3205 } 3206 3207 int dm_suspended_md(struct mapped_device *md) 3208 { 3209 return test_bit(DMF_SUSPENDED, &md->flags); 3210 } 3211 3212 int dm_suspended_internally_md(struct mapped_device *md) 3213 { 3214 return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 3215 } 3216 3217 int dm_test_deferred_remove_flag(struct mapped_device *md) 3218 { 3219 return test_bit(DMF_DEFERRED_REMOVE, &md->flags); 3220 } 3221 3222 int dm_suspended(struct dm_target *ti) 3223 { 3224 return dm_suspended_md(dm_table_get_md(ti->table)); 3225 } 3226 EXPORT_SYMBOL_GPL(dm_suspended); 3227 3228 int dm_noflush_suspending(struct dm_target *ti) 3229 { 3230 return __noflush_suspending(dm_table_get_md(ti->table)); 3231 } 3232 EXPORT_SYMBOL_GPL(dm_noflush_suspending); 3233 3234 struct dm_md_mempools *dm_alloc_md_mempools(unsigned type, unsigned integrity, unsigned per_bio_data_size) 3235 { 3236 struct dm_md_mempools *pools = kzalloc(sizeof(*pools), GFP_KERNEL); 3237 struct kmem_cache *cachep; 3238 unsigned int pool_size = 0; 3239 unsigned int front_pad; 3240 3241 if (!pools) 3242 return NULL; 3243 3244 switch (type) { 3245 case DM_TYPE_BIO_BASED: 3246 cachep = _io_cache; 3247 pool_size = dm_get_reserved_bio_based_ios(); 3248 front_pad = roundup(per_bio_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone); 3249 break; 3250 case DM_TYPE_REQUEST_BASED: 3251 pool_size = dm_get_reserved_rq_based_ios(); 3252 pools->rq_pool = mempool_create_slab_pool(pool_size, _rq_cache); 3253 if (!pools->rq_pool) 3254 goto out; 3255 /* fall through to setup remaining rq-based pools */ 3256 case DM_TYPE_MQ_REQUEST_BASED: 3257 cachep = _rq_tio_cache; 3258 if (!pool_size) 3259 pool_size = dm_get_reserved_rq_based_ios(); 3260 front_pad = offsetof(struct dm_rq_clone_bio_info, clone); 3261 /* per_bio_data_size is not used. See __bind_mempools(). */ 3262 WARN_ON(per_bio_data_size != 0); 3263 break; 3264 default: 3265 goto out; 3266 } 3267 3268 pools->io_pool = mempool_create_slab_pool(pool_size, cachep); 3269 if (!pools->io_pool) 3270 goto out; 3271 3272 pools->bs = bioset_create_nobvec(pool_size, front_pad); 3273 if (!pools->bs) 3274 goto out; 3275 3276 if (integrity && bioset_integrity_create(pools->bs, pool_size)) 3277 goto out; 3278 3279 return pools; 3280 3281 out: 3282 dm_free_md_mempools(pools); 3283 3284 return NULL; 3285 } 3286 3287 void dm_free_md_mempools(struct dm_md_mempools *pools) 3288 { 3289 if (!pools) 3290 return; 3291 3292 if (pools->io_pool) 3293 mempool_destroy(pools->io_pool); 3294 3295 if (pools->rq_pool) 3296 mempool_destroy(pools->rq_pool); 3297 3298 if (pools->bs) 3299 bioset_free(pools->bs); 3300 3301 kfree(pools); 3302 } 3303 3304 static const struct block_device_operations dm_blk_dops = { 3305 .open = dm_blk_open, 3306 .release = dm_blk_close, 3307 .ioctl = dm_blk_ioctl, 3308 .getgeo = dm_blk_getgeo, 3309 .owner = THIS_MODULE 3310 }; 3311 3312 /* 3313 * module hooks 3314 */ 3315 module_init(dm_init); 3316 module_exit(dm_exit); 3317 3318 module_param(major, uint, 0); 3319 MODULE_PARM_DESC(major, "The major number of the device mapper"); 3320 3321 module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR); 3322 MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools"); 3323 3324 module_param(reserved_rq_based_ios, uint, S_IRUGO | S_IWUSR); 3325 MODULE_PARM_DESC(reserved_rq_based_ios, "Reserved IOs in request-based mempools"); 3326 3327 MODULE_DESCRIPTION(DM_NAME " driver"); 3328 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>"); 3329 MODULE_LICENSE("GPL"); 3330