1 /* 2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited. 3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. 4 * 5 * This file is released under the GPL. 6 */ 7 8 #include "dm.h" 9 #include "dm-uevent.h" 10 11 #include <linux/init.h> 12 #include <linux/module.h> 13 #include <linux/mutex.h> 14 #include <linux/moduleparam.h> 15 #include <linux/blkpg.h> 16 #include <linux/bio.h> 17 #include <linux/mempool.h> 18 #include <linux/slab.h> 19 #include <linux/idr.h> 20 #include <linux/hdreg.h> 21 #include <linux/delay.h> 22 #include <linux/wait.h> 23 #include <linux/kthread.h> 24 #include <linux/ktime.h> 25 #include <linux/elevator.h> /* for rq_end_sector() */ 26 #include <linux/blk-mq.h> 27 28 #include <trace/events/block.h> 29 30 #define DM_MSG_PREFIX "core" 31 32 #ifdef CONFIG_PRINTK 33 /* 34 * ratelimit state to be used in DMXXX_LIMIT(). 35 */ 36 DEFINE_RATELIMIT_STATE(dm_ratelimit_state, 37 DEFAULT_RATELIMIT_INTERVAL, 38 DEFAULT_RATELIMIT_BURST); 39 EXPORT_SYMBOL(dm_ratelimit_state); 40 #endif 41 42 /* 43 * Cookies are numeric values sent with CHANGE and REMOVE 44 * uevents while resuming, removing or renaming the device. 45 */ 46 #define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE" 47 #define DM_COOKIE_LENGTH 24 48 49 static const char *_name = DM_NAME; 50 51 static unsigned int major = 0; 52 static unsigned int _major = 0; 53 54 static DEFINE_IDR(_minor_idr); 55 56 static DEFINE_SPINLOCK(_minor_lock); 57 58 static void do_deferred_remove(struct work_struct *w); 59 60 static DECLARE_WORK(deferred_remove_work, do_deferred_remove); 61 62 static struct workqueue_struct *deferred_remove_workqueue; 63 64 /* 65 * For bio-based dm. 66 * One of these is allocated per bio. 67 */ 68 struct dm_io { 69 struct mapped_device *md; 70 int error; 71 atomic_t io_count; 72 struct bio *bio; 73 unsigned long start_time; 74 spinlock_t endio_lock; 75 struct dm_stats_aux stats_aux; 76 }; 77 78 /* 79 * For request-based dm. 80 * One of these is allocated per request. 81 */ 82 struct dm_rq_target_io { 83 struct mapped_device *md; 84 struct dm_target *ti; 85 struct request *orig, *clone; 86 struct kthread_work work; 87 int error; 88 union map_info info; 89 struct dm_stats_aux stats_aux; 90 unsigned long duration_jiffies; 91 unsigned n_sectors; 92 }; 93 94 /* 95 * For request-based dm - the bio clones we allocate are embedded in these 96 * structs. 97 * 98 * We allocate these with bio_alloc_bioset, using the front_pad parameter when 99 * the bioset is created - this means the bio has to come at the end of the 100 * struct. 101 */ 102 struct dm_rq_clone_bio_info { 103 struct bio *orig; 104 struct dm_rq_target_io *tio; 105 struct bio clone; 106 }; 107 108 union map_info *dm_get_rq_mapinfo(struct request *rq) 109 { 110 if (rq && rq->end_io_data) 111 return &((struct dm_rq_target_io *)rq->end_io_data)->info; 112 return NULL; 113 } 114 EXPORT_SYMBOL_GPL(dm_get_rq_mapinfo); 115 116 #define MINOR_ALLOCED ((void *)-1) 117 118 /* 119 * Bits for the md->flags field. 120 */ 121 #define DMF_BLOCK_IO_FOR_SUSPEND 0 122 #define DMF_SUSPENDED 1 123 #define DMF_FROZEN 2 124 #define DMF_FREEING 3 125 #define DMF_DELETING 4 126 #define DMF_NOFLUSH_SUSPENDING 5 127 #define DMF_MERGE_IS_OPTIONAL 6 128 #define DMF_DEFERRED_REMOVE 7 129 #define DMF_SUSPENDED_INTERNALLY 8 130 131 /* 132 * A dummy definition to make RCU happy. 133 * struct dm_table should never be dereferenced in this file. 134 */ 135 struct dm_table { 136 int undefined__; 137 }; 138 139 /* 140 * Work processed by per-device workqueue. 141 */ 142 struct mapped_device { 143 struct srcu_struct io_barrier; 144 struct mutex suspend_lock; 145 atomic_t holders; 146 atomic_t open_count; 147 148 /* 149 * The current mapping. 150 * Use dm_get_live_table{_fast} or take suspend_lock for 151 * dereference. 152 */ 153 struct dm_table __rcu *map; 154 155 struct list_head table_devices; 156 struct mutex table_devices_lock; 157 158 unsigned long flags; 159 160 struct request_queue *queue; 161 unsigned type; 162 /* Protect queue and type against concurrent access. */ 163 struct mutex type_lock; 164 165 struct target_type *immutable_target_type; 166 167 struct gendisk *disk; 168 char name[16]; 169 170 void *interface_ptr; 171 172 /* 173 * A list of ios that arrived while we were suspended. 174 */ 175 atomic_t pending[2]; 176 wait_queue_head_t wait; 177 struct work_struct work; 178 struct bio_list deferred; 179 spinlock_t deferred_lock; 180 181 /* 182 * Processing queue (flush) 183 */ 184 struct workqueue_struct *wq; 185 186 /* 187 * io objects are allocated from here. 188 */ 189 mempool_t *io_pool; 190 mempool_t *rq_pool; 191 192 struct bio_set *bs; 193 194 /* 195 * Event handling. 196 */ 197 atomic_t event_nr; 198 wait_queue_head_t eventq; 199 atomic_t uevent_seq; 200 struct list_head uevent_list; 201 spinlock_t uevent_lock; /* Protect access to uevent_list */ 202 203 /* 204 * freeze/thaw support require holding onto a super block 205 */ 206 struct super_block *frozen_sb; 207 struct block_device *bdev; 208 209 /* forced geometry settings */ 210 struct hd_geometry geometry; 211 212 /* kobject and completion */ 213 struct dm_kobject_holder kobj_holder; 214 215 /* zero-length flush that will be cloned and submitted to targets */ 216 struct bio flush_bio; 217 218 /* the number of internal suspends */ 219 unsigned internal_suspend_count; 220 221 struct dm_stats stats; 222 223 struct kthread_worker kworker; 224 struct task_struct *kworker_task; 225 226 /* for request-based merge heuristic in dm_request_fn() */ 227 unsigned seq_rq_merge_deadline_usecs; 228 int last_rq_rw; 229 sector_t last_rq_pos; 230 ktime_t last_rq_start_time; 231 232 /* for blk-mq request-based DM support */ 233 struct blk_mq_tag_set tag_set; 234 bool use_blk_mq; 235 }; 236 237 #ifdef CONFIG_DM_MQ_DEFAULT 238 static bool use_blk_mq = true; 239 #else 240 static bool use_blk_mq = false; 241 #endif 242 243 bool dm_use_blk_mq(struct mapped_device *md) 244 { 245 return md->use_blk_mq; 246 } 247 248 /* 249 * For mempools pre-allocation at the table loading time. 250 */ 251 struct dm_md_mempools { 252 mempool_t *io_pool; 253 mempool_t *rq_pool; 254 struct bio_set *bs; 255 }; 256 257 struct table_device { 258 struct list_head list; 259 atomic_t count; 260 struct dm_dev dm_dev; 261 }; 262 263 #define RESERVED_BIO_BASED_IOS 16 264 #define RESERVED_REQUEST_BASED_IOS 256 265 #define RESERVED_MAX_IOS 1024 266 static struct kmem_cache *_io_cache; 267 static struct kmem_cache *_rq_tio_cache; 268 static struct kmem_cache *_rq_cache; 269 270 /* 271 * Bio-based DM's mempools' reserved IOs set by the user. 272 */ 273 static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS; 274 275 /* 276 * Request-based DM's mempools' reserved IOs set by the user. 277 */ 278 static unsigned reserved_rq_based_ios = RESERVED_REQUEST_BASED_IOS; 279 280 static unsigned __dm_get_module_param(unsigned *module_param, 281 unsigned def, unsigned max) 282 { 283 unsigned param = ACCESS_ONCE(*module_param); 284 unsigned modified_param = 0; 285 286 if (!param) 287 modified_param = def; 288 else if (param > max) 289 modified_param = max; 290 291 if (modified_param) { 292 (void)cmpxchg(module_param, param, modified_param); 293 param = modified_param; 294 } 295 296 return param; 297 } 298 299 unsigned dm_get_reserved_bio_based_ios(void) 300 { 301 return __dm_get_module_param(&reserved_bio_based_ios, 302 RESERVED_BIO_BASED_IOS, RESERVED_MAX_IOS); 303 } 304 EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios); 305 306 unsigned dm_get_reserved_rq_based_ios(void) 307 { 308 return __dm_get_module_param(&reserved_rq_based_ios, 309 RESERVED_REQUEST_BASED_IOS, RESERVED_MAX_IOS); 310 } 311 EXPORT_SYMBOL_GPL(dm_get_reserved_rq_based_ios); 312 313 static int __init local_init(void) 314 { 315 int r = -ENOMEM; 316 317 /* allocate a slab for the dm_ios */ 318 _io_cache = KMEM_CACHE(dm_io, 0); 319 if (!_io_cache) 320 return r; 321 322 _rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0); 323 if (!_rq_tio_cache) 324 goto out_free_io_cache; 325 326 _rq_cache = kmem_cache_create("dm_clone_request", sizeof(struct request), 327 __alignof__(struct request), 0, NULL); 328 if (!_rq_cache) 329 goto out_free_rq_tio_cache; 330 331 r = dm_uevent_init(); 332 if (r) 333 goto out_free_rq_cache; 334 335 deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1); 336 if (!deferred_remove_workqueue) { 337 r = -ENOMEM; 338 goto out_uevent_exit; 339 } 340 341 _major = major; 342 r = register_blkdev(_major, _name); 343 if (r < 0) 344 goto out_free_workqueue; 345 346 if (!_major) 347 _major = r; 348 349 return 0; 350 351 out_free_workqueue: 352 destroy_workqueue(deferred_remove_workqueue); 353 out_uevent_exit: 354 dm_uevent_exit(); 355 out_free_rq_cache: 356 kmem_cache_destroy(_rq_cache); 357 out_free_rq_tio_cache: 358 kmem_cache_destroy(_rq_tio_cache); 359 out_free_io_cache: 360 kmem_cache_destroy(_io_cache); 361 362 return r; 363 } 364 365 static void local_exit(void) 366 { 367 flush_scheduled_work(); 368 destroy_workqueue(deferred_remove_workqueue); 369 370 kmem_cache_destroy(_rq_cache); 371 kmem_cache_destroy(_rq_tio_cache); 372 kmem_cache_destroy(_io_cache); 373 unregister_blkdev(_major, _name); 374 dm_uevent_exit(); 375 376 _major = 0; 377 378 DMINFO("cleaned up"); 379 } 380 381 static int (*_inits[])(void) __initdata = { 382 local_init, 383 dm_target_init, 384 dm_linear_init, 385 dm_stripe_init, 386 dm_io_init, 387 dm_kcopyd_init, 388 dm_interface_init, 389 dm_statistics_init, 390 }; 391 392 static void (*_exits[])(void) = { 393 local_exit, 394 dm_target_exit, 395 dm_linear_exit, 396 dm_stripe_exit, 397 dm_io_exit, 398 dm_kcopyd_exit, 399 dm_interface_exit, 400 dm_statistics_exit, 401 }; 402 403 static int __init dm_init(void) 404 { 405 const int count = ARRAY_SIZE(_inits); 406 407 int r, i; 408 409 for (i = 0; i < count; i++) { 410 r = _inits[i](); 411 if (r) 412 goto bad; 413 } 414 415 return 0; 416 417 bad: 418 while (i--) 419 _exits[i](); 420 421 return r; 422 } 423 424 static void __exit dm_exit(void) 425 { 426 int i = ARRAY_SIZE(_exits); 427 428 while (i--) 429 _exits[i](); 430 431 /* 432 * Should be empty by this point. 433 */ 434 idr_destroy(&_minor_idr); 435 } 436 437 /* 438 * Block device functions 439 */ 440 int dm_deleting_md(struct mapped_device *md) 441 { 442 return test_bit(DMF_DELETING, &md->flags); 443 } 444 445 static int dm_blk_open(struct block_device *bdev, fmode_t mode) 446 { 447 struct mapped_device *md; 448 449 spin_lock(&_minor_lock); 450 451 md = bdev->bd_disk->private_data; 452 if (!md) 453 goto out; 454 455 if (test_bit(DMF_FREEING, &md->flags) || 456 dm_deleting_md(md)) { 457 md = NULL; 458 goto out; 459 } 460 461 dm_get(md); 462 atomic_inc(&md->open_count); 463 out: 464 spin_unlock(&_minor_lock); 465 466 return md ? 0 : -ENXIO; 467 } 468 469 static void dm_blk_close(struct gendisk *disk, fmode_t mode) 470 { 471 struct mapped_device *md; 472 473 spin_lock(&_minor_lock); 474 475 md = disk->private_data; 476 if (WARN_ON(!md)) 477 goto out; 478 479 if (atomic_dec_and_test(&md->open_count) && 480 (test_bit(DMF_DEFERRED_REMOVE, &md->flags))) 481 queue_work(deferred_remove_workqueue, &deferred_remove_work); 482 483 dm_put(md); 484 out: 485 spin_unlock(&_minor_lock); 486 } 487 488 int dm_open_count(struct mapped_device *md) 489 { 490 return atomic_read(&md->open_count); 491 } 492 493 /* 494 * Guarantees nothing is using the device before it's deleted. 495 */ 496 int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred) 497 { 498 int r = 0; 499 500 spin_lock(&_minor_lock); 501 502 if (dm_open_count(md)) { 503 r = -EBUSY; 504 if (mark_deferred) 505 set_bit(DMF_DEFERRED_REMOVE, &md->flags); 506 } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags)) 507 r = -EEXIST; 508 else 509 set_bit(DMF_DELETING, &md->flags); 510 511 spin_unlock(&_minor_lock); 512 513 return r; 514 } 515 516 int dm_cancel_deferred_remove(struct mapped_device *md) 517 { 518 int r = 0; 519 520 spin_lock(&_minor_lock); 521 522 if (test_bit(DMF_DELETING, &md->flags)) 523 r = -EBUSY; 524 else 525 clear_bit(DMF_DEFERRED_REMOVE, &md->flags); 526 527 spin_unlock(&_minor_lock); 528 529 return r; 530 } 531 532 static void do_deferred_remove(struct work_struct *w) 533 { 534 dm_deferred_remove(); 535 } 536 537 sector_t dm_get_size(struct mapped_device *md) 538 { 539 return get_capacity(md->disk); 540 } 541 542 struct request_queue *dm_get_md_queue(struct mapped_device *md) 543 { 544 return md->queue; 545 } 546 547 struct dm_stats *dm_get_stats(struct mapped_device *md) 548 { 549 return &md->stats; 550 } 551 552 static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo) 553 { 554 struct mapped_device *md = bdev->bd_disk->private_data; 555 556 return dm_get_geometry(md, geo); 557 } 558 559 static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode, 560 unsigned int cmd, unsigned long arg) 561 { 562 struct mapped_device *md = bdev->bd_disk->private_data; 563 int srcu_idx; 564 struct dm_table *map; 565 struct dm_target *tgt; 566 int r = -ENOTTY; 567 568 retry: 569 map = dm_get_live_table(md, &srcu_idx); 570 571 if (!map || !dm_table_get_size(map)) 572 goto out; 573 574 /* We only support devices that have a single target */ 575 if (dm_table_get_num_targets(map) != 1) 576 goto out; 577 578 tgt = dm_table_get_target(map, 0); 579 if (!tgt->type->ioctl) 580 goto out; 581 582 if (dm_suspended_md(md)) { 583 r = -EAGAIN; 584 goto out; 585 } 586 587 r = tgt->type->ioctl(tgt, cmd, arg); 588 589 out: 590 dm_put_live_table(md, srcu_idx); 591 592 if (r == -ENOTCONN) { 593 msleep(10); 594 goto retry; 595 } 596 597 return r; 598 } 599 600 static struct dm_io *alloc_io(struct mapped_device *md) 601 { 602 return mempool_alloc(md->io_pool, GFP_NOIO); 603 } 604 605 static void free_io(struct mapped_device *md, struct dm_io *io) 606 { 607 mempool_free(io, md->io_pool); 608 } 609 610 static void free_tio(struct mapped_device *md, struct dm_target_io *tio) 611 { 612 bio_put(&tio->clone); 613 } 614 615 static struct dm_rq_target_io *alloc_rq_tio(struct mapped_device *md, 616 gfp_t gfp_mask) 617 { 618 return mempool_alloc(md->io_pool, gfp_mask); 619 } 620 621 static void free_rq_tio(struct dm_rq_target_io *tio) 622 { 623 mempool_free(tio, tio->md->io_pool); 624 } 625 626 static struct request *alloc_clone_request(struct mapped_device *md, 627 gfp_t gfp_mask) 628 { 629 return mempool_alloc(md->rq_pool, gfp_mask); 630 } 631 632 static void free_clone_request(struct mapped_device *md, struct request *rq) 633 { 634 mempool_free(rq, md->rq_pool); 635 } 636 637 static int md_in_flight(struct mapped_device *md) 638 { 639 return atomic_read(&md->pending[READ]) + 640 atomic_read(&md->pending[WRITE]); 641 } 642 643 static void start_io_acct(struct dm_io *io) 644 { 645 struct mapped_device *md = io->md; 646 struct bio *bio = io->bio; 647 int cpu; 648 int rw = bio_data_dir(bio); 649 650 io->start_time = jiffies; 651 652 cpu = part_stat_lock(); 653 part_round_stats(cpu, &dm_disk(md)->part0); 654 part_stat_unlock(); 655 atomic_set(&dm_disk(md)->part0.in_flight[rw], 656 atomic_inc_return(&md->pending[rw])); 657 658 if (unlikely(dm_stats_used(&md->stats))) 659 dm_stats_account_io(&md->stats, bio->bi_rw, bio->bi_iter.bi_sector, 660 bio_sectors(bio), false, 0, &io->stats_aux); 661 } 662 663 static void end_io_acct(struct dm_io *io) 664 { 665 struct mapped_device *md = io->md; 666 struct bio *bio = io->bio; 667 unsigned long duration = jiffies - io->start_time; 668 int pending; 669 int rw = bio_data_dir(bio); 670 671 generic_end_io_acct(rw, &dm_disk(md)->part0, io->start_time); 672 673 if (unlikely(dm_stats_used(&md->stats))) 674 dm_stats_account_io(&md->stats, bio->bi_rw, bio->bi_iter.bi_sector, 675 bio_sectors(bio), true, duration, &io->stats_aux); 676 677 /* 678 * After this is decremented the bio must not be touched if it is 679 * a flush. 680 */ 681 pending = atomic_dec_return(&md->pending[rw]); 682 atomic_set(&dm_disk(md)->part0.in_flight[rw], pending); 683 pending += atomic_read(&md->pending[rw^0x1]); 684 685 /* nudge anyone waiting on suspend queue */ 686 if (!pending) 687 wake_up(&md->wait); 688 } 689 690 /* 691 * Add the bio to the list of deferred io. 692 */ 693 static void queue_io(struct mapped_device *md, struct bio *bio) 694 { 695 unsigned long flags; 696 697 spin_lock_irqsave(&md->deferred_lock, flags); 698 bio_list_add(&md->deferred, bio); 699 spin_unlock_irqrestore(&md->deferred_lock, flags); 700 queue_work(md->wq, &md->work); 701 } 702 703 /* 704 * Everyone (including functions in this file), should use this 705 * function to access the md->map field, and make sure they call 706 * dm_put_live_table() when finished. 707 */ 708 struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier) 709 { 710 *srcu_idx = srcu_read_lock(&md->io_barrier); 711 712 return srcu_dereference(md->map, &md->io_barrier); 713 } 714 715 void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier) 716 { 717 srcu_read_unlock(&md->io_barrier, srcu_idx); 718 } 719 720 void dm_sync_table(struct mapped_device *md) 721 { 722 synchronize_srcu(&md->io_barrier); 723 synchronize_rcu_expedited(); 724 } 725 726 /* 727 * A fast alternative to dm_get_live_table/dm_put_live_table. 728 * The caller must not block between these two functions. 729 */ 730 static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU) 731 { 732 rcu_read_lock(); 733 return rcu_dereference(md->map); 734 } 735 736 static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU) 737 { 738 rcu_read_unlock(); 739 } 740 741 /* 742 * Open a table device so we can use it as a map destination. 743 */ 744 static int open_table_device(struct table_device *td, dev_t dev, 745 struct mapped_device *md) 746 { 747 static char *_claim_ptr = "I belong to device-mapper"; 748 struct block_device *bdev; 749 750 int r; 751 752 BUG_ON(td->dm_dev.bdev); 753 754 bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _claim_ptr); 755 if (IS_ERR(bdev)) 756 return PTR_ERR(bdev); 757 758 r = bd_link_disk_holder(bdev, dm_disk(md)); 759 if (r) { 760 blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL); 761 return r; 762 } 763 764 td->dm_dev.bdev = bdev; 765 return 0; 766 } 767 768 /* 769 * Close a table device that we've been using. 770 */ 771 static void close_table_device(struct table_device *td, struct mapped_device *md) 772 { 773 if (!td->dm_dev.bdev) 774 return; 775 776 bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md)); 777 blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL); 778 td->dm_dev.bdev = NULL; 779 } 780 781 static struct table_device *find_table_device(struct list_head *l, dev_t dev, 782 fmode_t mode) { 783 struct table_device *td; 784 785 list_for_each_entry(td, l, list) 786 if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode) 787 return td; 788 789 return NULL; 790 } 791 792 int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode, 793 struct dm_dev **result) { 794 int r; 795 struct table_device *td; 796 797 mutex_lock(&md->table_devices_lock); 798 td = find_table_device(&md->table_devices, dev, mode); 799 if (!td) { 800 td = kmalloc(sizeof(*td), GFP_KERNEL); 801 if (!td) { 802 mutex_unlock(&md->table_devices_lock); 803 return -ENOMEM; 804 } 805 806 td->dm_dev.mode = mode; 807 td->dm_dev.bdev = NULL; 808 809 if ((r = open_table_device(td, dev, md))) { 810 mutex_unlock(&md->table_devices_lock); 811 kfree(td); 812 return r; 813 } 814 815 format_dev_t(td->dm_dev.name, dev); 816 817 atomic_set(&td->count, 0); 818 list_add(&td->list, &md->table_devices); 819 } 820 atomic_inc(&td->count); 821 mutex_unlock(&md->table_devices_lock); 822 823 *result = &td->dm_dev; 824 return 0; 825 } 826 EXPORT_SYMBOL_GPL(dm_get_table_device); 827 828 void dm_put_table_device(struct mapped_device *md, struct dm_dev *d) 829 { 830 struct table_device *td = container_of(d, struct table_device, dm_dev); 831 832 mutex_lock(&md->table_devices_lock); 833 if (atomic_dec_and_test(&td->count)) { 834 close_table_device(td, md); 835 list_del(&td->list); 836 kfree(td); 837 } 838 mutex_unlock(&md->table_devices_lock); 839 } 840 EXPORT_SYMBOL(dm_put_table_device); 841 842 static void free_table_devices(struct list_head *devices) 843 { 844 struct list_head *tmp, *next; 845 846 list_for_each_safe(tmp, next, devices) { 847 struct table_device *td = list_entry(tmp, struct table_device, list); 848 849 DMWARN("dm_destroy: %s still exists with %d references", 850 td->dm_dev.name, atomic_read(&td->count)); 851 kfree(td); 852 } 853 } 854 855 /* 856 * Get the geometry associated with a dm device 857 */ 858 int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo) 859 { 860 *geo = md->geometry; 861 862 return 0; 863 } 864 865 /* 866 * Set the geometry of a device. 867 */ 868 int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo) 869 { 870 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors; 871 872 if (geo->start > sz) { 873 DMWARN("Start sector is beyond the geometry limits."); 874 return -EINVAL; 875 } 876 877 md->geometry = *geo; 878 879 return 0; 880 } 881 882 /*----------------------------------------------------------------- 883 * CRUD START: 884 * A more elegant soln is in the works that uses the queue 885 * merge fn, unfortunately there are a couple of changes to 886 * the block layer that I want to make for this. So in the 887 * interests of getting something for people to use I give 888 * you this clearly demarcated crap. 889 *---------------------------------------------------------------*/ 890 891 static int __noflush_suspending(struct mapped_device *md) 892 { 893 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 894 } 895 896 /* 897 * Decrements the number of outstanding ios that a bio has been 898 * cloned into, completing the original io if necc. 899 */ 900 static void dec_pending(struct dm_io *io, int error) 901 { 902 unsigned long flags; 903 int io_error; 904 struct bio *bio; 905 struct mapped_device *md = io->md; 906 907 /* Push-back supersedes any I/O errors */ 908 if (unlikely(error)) { 909 spin_lock_irqsave(&io->endio_lock, flags); 910 if (!(io->error > 0 && __noflush_suspending(md))) 911 io->error = error; 912 spin_unlock_irqrestore(&io->endio_lock, flags); 913 } 914 915 if (atomic_dec_and_test(&io->io_count)) { 916 if (io->error == DM_ENDIO_REQUEUE) { 917 /* 918 * Target requested pushing back the I/O. 919 */ 920 spin_lock_irqsave(&md->deferred_lock, flags); 921 if (__noflush_suspending(md)) 922 bio_list_add_head(&md->deferred, io->bio); 923 else 924 /* noflush suspend was interrupted. */ 925 io->error = -EIO; 926 spin_unlock_irqrestore(&md->deferred_lock, flags); 927 } 928 929 io_error = io->error; 930 bio = io->bio; 931 end_io_acct(io); 932 free_io(md, io); 933 934 if (io_error == DM_ENDIO_REQUEUE) 935 return; 936 937 if ((bio->bi_rw & REQ_FLUSH) && bio->bi_iter.bi_size) { 938 /* 939 * Preflush done for flush with data, reissue 940 * without REQ_FLUSH. 941 */ 942 bio->bi_rw &= ~REQ_FLUSH; 943 queue_io(md, bio); 944 } else { 945 /* done with normal IO or empty flush */ 946 trace_block_bio_complete(md->queue, bio, io_error); 947 bio_endio(bio, io_error); 948 } 949 } 950 } 951 952 static void disable_write_same(struct mapped_device *md) 953 { 954 struct queue_limits *limits = dm_get_queue_limits(md); 955 956 /* device doesn't really support WRITE SAME, disable it */ 957 limits->max_write_same_sectors = 0; 958 } 959 960 static void clone_endio(struct bio *bio, int error) 961 { 962 int r = error; 963 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone); 964 struct dm_io *io = tio->io; 965 struct mapped_device *md = tio->io->md; 966 dm_endio_fn endio = tio->ti->type->end_io; 967 968 if (!bio_flagged(bio, BIO_UPTODATE) && !error) 969 error = -EIO; 970 971 if (endio) { 972 r = endio(tio->ti, bio, error); 973 if (r < 0 || r == DM_ENDIO_REQUEUE) 974 /* 975 * error and requeue request are handled 976 * in dec_pending(). 977 */ 978 error = r; 979 else if (r == DM_ENDIO_INCOMPLETE) 980 /* The target will handle the io */ 981 return; 982 else if (r) { 983 DMWARN("unimplemented target endio return value: %d", r); 984 BUG(); 985 } 986 } 987 988 if (unlikely(r == -EREMOTEIO && (bio->bi_rw & REQ_WRITE_SAME) && 989 !bdev_get_queue(bio->bi_bdev)->limits.max_write_same_sectors)) 990 disable_write_same(md); 991 992 free_tio(md, tio); 993 dec_pending(io, error); 994 } 995 996 /* 997 * Partial completion handling for request-based dm 998 */ 999 static void end_clone_bio(struct bio *clone, int error) 1000 { 1001 struct dm_rq_clone_bio_info *info = 1002 container_of(clone, struct dm_rq_clone_bio_info, clone); 1003 struct dm_rq_target_io *tio = info->tio; 1004 struct bio *bio = info->orig; 1005 unsigned int nr_bytes = info->orig->bi_iter.bi_size; 1006 1007 bio_put(clone); 1008 1009 if (tio->error) 1010 /* 1011 * An error has already been detected on the request. 1012 * Once error occurred, just let clone->end_io() handle 1013 * the remainder. 1014 */ 1015 return; 1016 else if (error) { 1017 /* 1018 * Don't notice the error to the upper layer yet. 1019 * The error handling decision is made by the target driver, 1020 * when the request is completed. 1021 */ 1022 tio->error = error; 1023 return; 1024 } 1025 1026 /* 1027 * I/O for the bio successfully completed. 1028 * Notice the data completion to the upper layer. 1029 */ 1030 1031 /* 1032 * bios are processed from the head of the list. 1033 * So the completing bio should always be rq->bio. 1034 * If it's not, something wrong is happening. 1035 */ 1036 if (tio->orig->bio != bio) 1037 DMERR("bio completion is going in the middle of the request"); 1038 1039 /* 1040 * Update the original request. 1041 * Do not use blk_end_request() here, because it may complete 1042 * the original request before the clone, and break the ordering. 1043 */ 1044 blk_update_request(tio->orig, 0, nr_bytes); 1045 } 1046 1047 static struct dm_rq_target_io *tio_from_request(struct request *rq) 1048 { 1049 return (rq->q->mq_ops ? blk_mq_rq_to_pdu(rq) : rq->special); 1050 } 1051 1052 static void rq_end_stats(struct mapped_device *md, struct request *orig) 1053 { 1054 if (unlikely(dm_stats_used(&md->stats))) { 1055 struct dm_rq_target_io *tio = tio_from_request(orig); 1056 tio->duration_jiffies = jiffies - tio->duration_jiffies; 1057 dm_stats_account_io(&md->stats, orig->cmd_flags, blk_rq_pos(orig), 1058 tio->n_sectors, true, tio->duration_jiffies, 1059 &tio->stats_aux); 1060 } 1061 } 1062 1063 /* 1064 * Don't touch any member of the md after calling this function because 1065 * the md may be freed in dm_put() at the end of this function. 1066 * Or do dm_get() before calling this function and dm_put() later. 1067 */ 1068 static void rq_completed(struct mapped_device *md, int rw, bool run_queue) 1069 { 1070 int nr_requests_pending; 1071 1072 atomic_dec(&md->pending[rw]); 1073 1074 /* nudge anyone waiting on suspend queue */ 1075 nr_requests_pending = md_in_flight(md); 1076 if (!nr_requests_pending) 1077 wake_up(&md->wait); 1078 1079 /* 1080 * Run this off this callpath, as drivers could invoke end_io while 1081 * inside their request_fn (and holding the queue lock). Calling 1082 * back into ->request_fn() could deadlock attempting to grab the 1083 * queue lock again. 1084 */ 1085 if (run_queue) { 1086 if (md->queue->mq_ops) 1087 blk_mq_run_hw_queues(md->queue, true); 1088 else if (!nr_requests_pending || 1089 (nr_requests_pending >= md->queue->nr_congestion_on)) 1090 blk_run_queue_async(md->queue); 1091 } 1092 1093 /* 1094 * dm_put() must be at the end of this function. See the comment above 1095 */ 1096 dm_put(md); 1097 } 1098 1099 static void free_rq_clone(struct request *clone) 1100 { 1101 struct dm_rq_target_io *tio = clone->end_io_data; 1102 struct mapped_device *md = tio->md; 1103 1104 blk_rq_unprep_clone(clone); 1105 1106 if (md->type == DM_TYPE_MQ_REQUEST_BASED) 1107 /* stacked on blk-mq queue(s) */ 1108 tio->ti->type->release_clone_rq(clone); 1109 else if (!md->queue->mq_ops) 1110 /* request_fn queue stacked on request_fn queue(s) */ 1111 free_clone_request(md, clone); 1112 /* 1113 * NOTE: for the blk-mq queue stacked on request_fn queue(s) case: 1114 * no need to call free_clone_request() because we leverage blk-mq by 1115 * allocating the clone at the end of the blk-mq pdu (see: clone_rq) 1116 */ 1117 1118 if (!md->queue->mq_ops) 1119 free_rq_tio(tio); 1120 } 1121 1122 /* 1123 * Complete the clone and the original request. 1124 * Must be called without clone's queue lock held, 1125 * see end_clone_request() for more details. 1126 */ 1127 static void dm_end_request(struct request *clone, int error) 1128 { 1129 int rw = rq_data_dir(clone); 1130 struct dm_rq_target_io *tio = clone->end_io_data; 1131 struct mapped_device *md = tio->md; 1132 struct request *rq = tio->orig; 1133 1134 if (rq->cmd_type == REQ_TYPE_BLOCK_PC) { 1135 rq->errors = clone->errors; 1136 rq->resid_len = clone->resid_len; 1137 1138 if (rq->sense) 1139 /* 1140 * We are using the sense buffer of the original 1141 * request. 1142 * So setting the length of the sense data is enough. 1143 */ 1144 rq->sense_len = clone->sense_len; 1145 } 1146 1147 free_rq_clone(clone); 1148 rq_end_stats(md, rq); 1149 if (!rq->q->mq_ops) 1150 blk_end_request_all(rq, error); 1151 else 1152 blk_mq_end_request(rq, error); 1153 rq_completed(md, rw, true); 1154 } 1155 1156 static void dm_unprep_request(struct request *rq) 1157 { 1158 struct dm_rq_target_io *tio = tio_from_request(rq); 1159 struct request *clone = tio->clone; 1160 1161 if (!rq->q->mq_ops) { 1162 rq->special = NULL; 1163 rq->cmd_flags &= ~REQ_DONTPREP; 1164 } 1165 1166 if (clone) 1167 free_rq_clone(clone); 1168 } 1169 1170 /* 1171 * Requeue the original request of a clone. 1172 */ 1173 static void old_requeue_request(struct request *rq) 1174 { 1175 struct request_queue *q = rq->q; 1176 unsigned long flags; 1177 1178 spin_lock_irqsave(q->queue_lock, flags); 1179 blk_requeue_request(q, rq); 1180 blk_run_queue_async(q); 1181 spin_unlock_irqrestore(q->queue_lock, flags); 1182 } 1183 1184 static void dm_requeue_original_request(struct mapped_device *md, 1185 struct request *rq) 1186 { 1187 int rw = rq_data_dir(rq); 1188 1189 dm_unprep_request(rq); 1190 1191 rq_end_stats(md, rq); 1192 if (!rq->q->mq_ops) 1193 old_requeue_request(rq); 1194 else { 1195 blk_mq_requeue_request(rq); 1196 blk_mq_kick_requeue_list(rq->q); 1197 } 1198 1199 rq_completed(md, rw, false); 1200 } 1201 1202 static void old_stop_queue(struct request_queue *q) 1203 { 1204 unsigned long flags; 1205 1206 if (blk_queue_stopped(q)) 1207 return; 1208 1209 spin_lock_irqsave(q->queue_lock, flags); 1210 blk_stop_queue(q); 1211 spin_unlock_irqrestore(q->queue_lock, flags); 1212 } 1213 1214 static void stop_queue(struct request_queue *q) 1215 { 1216 if (!q->mq_ops) 1217 old_stop_queue(q); 1218 else 1219 blk_mq_stop_hw_queues(q); 1220 } 1221 1222 static void old_start_queue(struct request_queue *q) 1223 { 1224 unsigned long flags; 1225 1226 spin_lock_irqsave(q->queue_lock, flags); 1227 if (blk_queue_stopped(q)) 1228 blk_start_queue(q); 1229 spin_unlock_irqrestore(q->queue_lock, flags); 1230 } 1231 1232 static void start_queue(struct request_queue *q) 1233 { 1234 if (!q->mq_ops) 1235 old_start_queue(q); 1236 else 1237 blk_mq_start_stopped_hw_queues(q, true); 1238 } 1239 1240 static void dm_done(struct request *clone, int error, bool mapped) 1241 { 1242 int r = error; 1243 struct dm_rq_target_io *tio = clone->end_io_data; 1244 dm_request_endio_fn rq_end_io = NULL; 1245 1246 if (tio->ti) { 1247 rq_end_io = tio->ti->type->rq_end_io; 1248 1249 if (mapped && rq_end_io) 1250 r = rq_end_io(tio->ti, clone, error, &tio->info); 1251 } 1252 1253 if (unlikely(r == -EREMOTEIO && (clone->cmd_flags & REQ_WRITE_SAME) && 1254 !clone->q->limits.max_write_same_sectors)) 1255 disable_write_same(tio->md); 1256 1257 if (r <= 0) 1258 /* The target wants to complete the I/O */ 1259 dm_end_request(clone, r); 1260 else if (r == DM_ENDIO_INCOMPLETE) 1261 /* The target will handle the I/O */ 1262 return; 1263 else if (r == DM_ENDIO_REQUEUE) 1264 /* The target wants to requeue the I/O */ 1265 dm_requeue_original_request(tio->md, tio->orig); 1266 else { 1267 DMWARN("unimplemented target endio return value: %d", r); 1268 BUG(); 1269 } 1270 } 1271 1272 /* 1273 * Request completion handler for request-based dm 1274 */ 1275 static void dm_softirq_done(struct request *rq) 1276 { 1277 bool mapped = true; 1278 struct dm_rq_target_io *tio = tio_from_request(rq); 1279 struct request *clone = tio->clone; 1280 int rw; 1281 1282 if (!clone) { 1283 rq_end_stats(tio->md, rq); 1284 rw = rq_data_dir(rq); 1285 if (!rq->q->mq_ops) { 1286 blk_end_request_all(rq, tio->error); 1287 rq_completed(tio->md, rw, false); 1288 free_rq_tio(tio); 1289 } else { 1290 blk_mq_end_request(rq, tio->error); 1291 rq_completed(tio->md, rw, false); 1292 } 1293 return; 1294 } 1295 1296 if (rq->cmd_flags & REQ_FAILED) 1297 mapped = false; 1298 1299 dm_done(clone, tio->error, mapped); 1300 } 1301 1302 /* 1303 * Complete the clone and the original request with the error status 1304 * through softirq context. 1305 */ 1306 static void dm_complete_request(struct request *rq, int error) 1307 { 1308 struct dm_rq_target_io *tio = tio_from_request(rq); 1309 1310 tio->error = error; 1311 blk_complete_request(rq); 1312 } 1313 1314 /* 1315 * Complete the not-mapped clone and the original request with the error status 1316 * through softirq context. 1317 * Target's rq_end_io() function isn't called. 1318 * This may be used when the target's map_rq() or clone_and_map_rq() functions fail. 1319 */ 1320 static void dm_kill_unmapped_request(struct request *rq, int error) 1321 { 1322 rq->cmd_flags |= REQ_FAILED; 1323 dm_complete_request(rq, error); 1324 } 1325 1326 /* 1327 * Called with the clone's queue lock held (for non-blk-mq) 1328 */ 1329 static void end_clone_request(struct request *clone, int error) 1330 { 1331 struct dm_rq_target_io *tio = clone->end_io_data; 1332 1333 if (!clone->q->mq_ops) { 1334 /* 1335 * For just cleaning up the information of the queue in which 1336 * the clone was dispatched. 1337 * The clone is *NOT* freed actually here because it is alloced 1338 * from dm own mempool (REQ_ALLOCED isn't set). 1339 */ 1340 __blk_put_request(clone->q, clone); 1341 } 1342 1343 /* 1344 * Actual request completion is done in a softirq context which doesn't 1345 * hold the clone's queue lock. Otherwise, deadlock could occur because: 1346 * - another request may be submitted by the upper level driver 1347 * of the stacking during the completion 1348 * - the submission which requires queue lock may be done 1349 * against this clone's queue 1350 */ 1351 dm_complete_request(tio->orig, error); 1352 } 1353 1354 /* 1355 * Return maximum size of I/O possible at the supplied sector up to the current 1356 * target boundary. 1357 */ 1358 static sector_t max_io_len_target_boundary(sector_t sector, struct dm_target *ti) 1359 { 1360 sector_t target_offset = dm_target_offset(ti, sector); 1361 1362 return ti->len - target_offset; 1363 } 1364 1365 static sector_t max_io_len(sector_t sector, struct dm_target *ti) 1366 { 1367 sector_t len = max_io_len_target_boundary(sector, ti); 1368 sector_t offset, max_len; 1369 1370 /* 1371 * Does the target need to split even further? 1372 */ 1373 if (ti->max_io_len) { 1374 offset = dm_target_offset(ti, sector); 1375 if (unlikely(ti->max_io_len & (ti->max_io_len - 1))) 1376 max_len = sector_div(offset, ti->max_io_len); 1377 else 1378 max_len = offset & (ti->max_io_len - 1); 1379 max_len = ti->max_io_len - max_len; 1380 1381 if (len > max_len) 1382 len = max_len; 1383 } 1384 1385 return len; 1386 } 1387 1388 int dm_set_target_max_io_len(struct dm_target *ti, sector_t len) 1389 { 1390 if (len > UINT_MAX) { 1391 DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)", 1392 (unsigned long long)len, UINT_MAX); 1393 ti->error = "Maximum size of target IO is too large"; 1394 return -EINVAL; 1395 } 1396 1397 ti->max_io_len = (uint32_t) len; 1398 1399 return 0; 1400 } 1401 EXPORT_SYMBOL_GPL(dm_set_target_max_io_len); 1402 1403 /* 1404 * A target may call dm_accept_partial_bio only from the map routine. It is 1405 * allowed for all bio types except REQ_FLUSH. 1406 * 1407 * dm_accept_partial_bio informs the dm that the target only wants to process 1408 * additional n_sectors sectors of the bio and the rest of the data should be 1409 * sent in a next bio. 1410 * 1411 * A diagram that explains the arithmetics: 1412 * +--------------------+---------------+-------+ 1413 * | 1 | 2 | 3 | 1414 * +--------------------+---------------+-------+ 1415 * 1416 * <-------------- *tio->len_ptr ---------------> 1417 * <------- bi_size -------> 1418 * <-- n_sectors --> 1419 * 1420 * Region 1 was already iterated over with bio_advance or similar function. 1421 * (it may be empty if the target doesn't use bio_advance) 1422 * Region 2 is the remaining bio size that the target wants to process. 1423 * (it may be empty if region 1 is non-empty, although there is no reason 1424 * to make it empty) 1425 * The target requires that region 3 is to be sent in the next bio. 1426 * 1427 * If the target wants to receive multiple copies of the bio (via num_*bios, etc), 1428 * the partially processed part (the sum of regions 1+2) must be the same for all 1429 * copies of the bio. 1430 */ 1431 void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors) 1432 { 1433 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone); 1434 unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT; 1435 BUG_ON(bio->bi_rw & REQ_FLUSH); 1436 BUG_ON(bi_size > *tio->len_ptr); 1437 BUG_ON(n_sectors > bi_size); 1438 *tio->len_ptr -= bi_size - n_sectors; 1439 bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT; 1440 } 1441 EXPORT_SYMBOL_GPL(dm_accept_partial_bio); 1442 1443 static void __map_bio(struct dm_target_io *tio) 1444 { 1445 int r; 1446 sector_t sector; 1447 struct mapped_device *md; 1448 struct bio *clone = &tio->clone; 1449 struct dm_target *ti = tio->ti; 1450 1451 clone->bi_end_io = clone_endio; 1452 1453 /* 1454 * Map the clone. If r == 0 we don't need to do 1455 * anything, the target has assumed ownership of 1456 * this io. 1457 */ 1458 atomic_inc(&tio->io->io_count); 1459 sector = clone->bi_iter.bi_sector; 1460 r = ti->type->map(ti, clone); 1461 if (r == DM_MAPIO_REMAPPED) { 1462 /* the bio has been remapped so dispatch it */ 1463 1464 trace_block_bio_remap(bdev_get_queue(clone->bi_bdev), clone, 1465 tio->io->bio->bi_bdev->bd_dev, sector); 1466 1467 generic_make_request(clone); 1468 } else if (r < 0 || r == DM_MAPIO_REQUEUE) { 1469 /* error the io and bail out, or requeue it if needed */ 1470 md = tio->io->md; 1471 dec_pending(tio->io, r); 1472 free_tio(md, tio); 1473 } else if (r) { 1474 DMWARN("unimplemented target map return value: %d", r); 1475 BUG(); 1476 } 1477 } 1478 1479 struct clone_info { 1480 struct mapped_device *md; 1481 struct dm_table *map; 1482 struct bio *bio; 1483 struct dm_io *io; 1484 sector_t sector; 1485 unsigned sector_count; 1486 }; 1487 1488 static void bio_setup_sector(struct bio *bio, sector_t sector, unsigned len) 1489 { 1490 bio->bi_iter.bi_sector = sector; 1491 bio->bi_iter.bi_size = to_bytes(len); 1492 } 1493 1494 /* 1495 * Creates a bio that consists of range of complete bvecs. 1496 */ 1497 static void clone_bio(struct dm_target_io *tio, struct bio *bio, 1498 sector_t sector, unsigned len) 1499 { 1500 struct bio *clone = &tio->clone; 1501 1502 __bio_clone_fast(clone, bio); 1503 1504 if (bio_integrity(bio)) 1505 bio_integrity_clone(clone, bio, GFP_NOIO); 1506 1507 bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector)); 1508 clone->bi_iter.bi_size = to_bytes(len); 1509 1510 if (bio_integrity(bio)) 1511 bio_integrity_trim(clone, 0, len); 1512 } 1513 1514 static struct dm_target_io *alloc_tio(struct clone_info *ci, 1515 struct dm_target *ti, 1516 unsigned target_bio_nr) 1517 { 1518 struct dm_target_io *tio; 1519 struct bio *clone; 1520 1521 clone = bio_alloc_bioset(GFP_NOIO, 0, ci->md->bs); 1522 tio = container_of(clone, struct dm_target_io, clone); 1523 1524 tio->io = ci->io; 1525 tio->ti = ti; 1526 tio->target_bio_nr = target_bio_nr; 1527 1528 return tio; 1529 } 1530 1531 static void __clone_and_map_simple_bio(struct clone_info *ci, 1532 struct dm_target *ti, 1533 unsigned target_bio_nr, unsigned *len) 1534 { 1535 struct dm_target_io *tio = alloc_tio(ci, ti, target_bio_nr); 1536 struct bio *clone = &tio->clone; 1537 1538 tio->len_ptr = len; 1539 1540 __bio_clone_fast(clone, ci->bio); 1541 if (len) 1542 bio_setup_sector(clone, ci->sector, *len); 1543 1544 __map_bio(tio); 1545 } 1546 1547 static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti, 1548 unsigned num_bios, unsigned *len) 1549 { 1550 unsigned target_bio_nr; 1551 1552 for (target_bio_nr = 0; target_bio_nr < num_bios; target_bio_nr++) 1553 __clone_and_map_simple_bio(ci, ti, target_bio_nr, len); 1554 } 1555 1556 static int __send_empty_flush(struct clone_info *ci) 1557 { 1558 unsigned target_nr = 0; 1559 struct dm_target *ti; 1560 1561 BUG_ON(bio_has_data(ci->bio)); 1562 while ((ti = dm_table_get_target(ci->map, target_nr++))) 1563 __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL); 1564 1565 return 0; 1566 } 1567 1568 static void __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti, 1569 sector_t sector, unsigned *len) 1570 { 1571 struct bio *bio = ci->bio; 1572 struct dm_target_io *tio; 1573 unsigned target_bio_nr; 1574 unsigned num_target_bios = 1; 1575 1576 /* 1577 * Does the target want to receive duplicate copies of the bio? 1578 */ 1579 if (bio_data_dir(bio) == WRITE && ti->num_write_bios) 1580 num_target_bios = ti->num_write_bios(ti, bio); 1581 1582 for (target_bio_nr = 0; target_bio_nr < num_target_bios; target_bio_nr++) { 1583 tio = alloc_tio(ci, ti, target_bio_nr); 1584 tio->len_ptr = len; 1585 clone_bio(tio, bio, sector, *len); 1586 __map_bio(tio); 1587 } 1588 } 1589 1590 typedef unsigned (*get_num_bios_fn)(struct dm_target *ti); 1591 1592 static unsigned get_num_discard_bios(struct dm_target *ti) 1593 { 1594 return ti->num_discard_bios; 1595 } 1596 1597 static unsigned get_num_write_same_bios(struct dm_target *ti) 1598 { 1599 return ti->num_write_same_bios; 1600 } 1601 1602 typedef bool (*is_split_required_fn)(struct dm_target *ti); 1603 1604 static bool is_split_required_for_discard(struct dm_target *ti) 1605 { 1606 return ti->split_discard_bios; 1607 } 1608 1609 static int __send_changing_extent_only(struct clone_info *ci, 1610 get_num_bios_fn get_num_bios, 1611 is_split_required_fn is_split_required) 1612 { 1613 struct dm_target *ti; 1614 unsigned len; 1615 unsigned num_bios; 1616 1617 do { 1618 ti = dm_table_find_target(ci->map, ci->sector); 1619 if (!dm_target_is_valid(ti)) 1620 return -EIO; 1621 1622 /* 1623 * Even though the device advertised support for this type of 1624 * request, that does not mean every target supports it, and 1625 * reconfiguration might also have changed that since the 1626 * check was performed. 1627 */ 1628 num_bios = get_num_bios ? get_num_bios(ti) : 0; 1629 if (!num_bios) 1630 return -EOPNOTSUPP; 1631 1632 if (is_split_required && !is_split_required(ti)) 1633 len = min((sector_t)ci->sector_count, max_io_len_target_boundary(ci->sector, ti)); 1634 else 1635 len = min((sector_t)ci->sector_count, max_io_len(ci->sector, ti)); 1636 1637 __send_duplicate_bios(ci, ti, num_bios, &len); 1638 1639 ci->sector += len; 1640 } while (ci->sector_count -= len); 1641 1642 return 0; 1643 } 1644 1645 static int __send_discard(struct clone_info *ci) 1646 { 1647 return __send_changing_extent_only(ci, get_num_discard_bios, 1648 is_split_required_for_discard); 1649 } 1650 1651 static int __send_write_same(struct clone_info *ci) 1652 { 1653 return __send_changing_extent_only(ci, get_num_write_same_bios, NULL); 1654 } 1655 1656 /* 1657 * Select the correct strategy for processing a non-flush bio. 1658 */ 1659 static int __split_and_process_non_flush(struct clone_info *ci) 1660 { 1661 struct bio *bio = ci->bio; 1662 struct dm_target *ti; 1663 unsigned len; 1664 1665 if (unlikely(bio->bi_rw & REQ_DISCARD)) 1666 return __send_discard(ci); 1667 else if (unlikely(bio->bi_rw & REQ_WRITE_SAME)) 1668 return __send_write_same(ci); 1669 1670 ti = dm_table_find_target(ci->map, ci->sector); 1671 if (!dm_target_is_valid(ti)) 1672 return -EIO; 1673 1674 len = min_t(sector_t, max_io_len(ci->sector, ti), ci->sector_count); 1675 1676 __clone_and_map_data_bio(ci, ti, ci->sector, &len); 1677 1678 ci->sector += len; 1679 ci->sector_count -= len; 1680 1681 return 0; 1682 } 1683 1684 /* 1685 * Entry point to split a bio into clones and submit them to the targets. 1686 */ 1687 static void __split_and_process_bio(struct mapped_device *md, 1688 struct dm_table *map, struct bio *bio) 1689 { 1690 struct clone_info ci; 1691 int error = 0; 1692 1693 if (unlikely(!map)) { 1694 bio_io_error(bio); 1695 return; 1696 } 1697 1698 ci.map = map; 1699 ci.md = md; 1700 ci.io = alloc_io(md); 1701 ci.io->error = 0; 1702 atomic_set(&ci.io->io_count, 1); 1703 ci.io->bio = bio; 1704 ci.io->md = md; 1705 spin_lock_init(&ci.io->endio_lock); 1706 ci.sector = bio->bi_iter.bi_sector; 1707 1708 start_io_acct(ci.io); 1709 1710 if (bio->bi_rw & REQ_FLUSH) { 1711 ci.bio = &ci.md->flush_bio; 1712 ci.sector_count = 0; 1713 error = __send_empty_flush(&ci); 1714 /* dec_pending submits any data associated with flush */ 1715 } else { 1716 ci.bio = bio; 1717 ci.sector_count = bio_sectors(bio); 1718 while (ci.sector_count && !error) 1719 error = __split_and_process_non_flush(&ci); 1720 } 1721 1722 /* drop the extra reference count */ 1723 dec_pending(ci.io, error); 1724 } 1725 /*----------------------------------------------------------------- 1726 * CRUD END 1727 *---------------------------------------------------------------*/ 1728 1729 static int dm_merge_bvec(struct request_queue *q, 1730 struct bvec_merge_data *bvm, 1731 struct bio_vec *biovec) 1732 { 1733 struct mapped_device *md = q->queuedata; 1734 struct dm_table *map = dm_get_live_table_fast(md); 1735 struct dm_target *ti; 1736 sector_t max_sectors, max_size = 0; 1737 1738 if (unlikely(!map)) 1739 goto out; 1740 1741 ti = dm_table_find_target(map, bvm->bi_sector); 1742 if (!dm_target_is_valid(ti)) 1743 goto out; 1744 1745 /* 1746 * Find maximum amount of I/O that won't need splitting 1747 */ 1748 max_sectors = min(max_io_len(bvm->bi_sector, ti), 1749 (sector_t) queue_max_sectors(q)); 1750 max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size; 1751 1752 /* 1753 * FIXME: this stop-gap fix _must_ be cleaned up (by passing a sector_t 1754 * to the targets' merge function since it holds sectors not bytes). 1755 * Just doing this as an interim fix for stable@ because the more 1756 * comprehensive cleanup of switching to sector_t will impact every 1757 * DM target that implements a ->merge hook. 1758 */ 1759 if (max_size > INT_MAX) 1760 max_size = INT_MAX; 1761 1762 /* 1763 * merge_bvec_fn() returns number of bytes 1764 * it can accept at this offset 1765 * max is precomputed maximal io size 1766 */ 1767 if (max_size && ti->type->merge) 1768 max_size = ti->type->merge(ti, bvm, biovec, (int) max_size); 1769 /* 1770 * If the target doesn't support merge method and some of the devices 1771 * provided their merge_bvec method (we know this by looking for the 1772 * max_hw_sectors that dm_set_device_limits may set), then we can't 1773 * allow bios with multiple vector entries. So always set max_size 1774 * to 0, and the code below allows just one page. 1775 */ 1776 else if (queue_max_hw_sectors(q) <= PAGE_SIZE >> 9) 1777 max_size = 0; 1778 1779 out: 1780 dm_put_live_table_fast(md); 1781 /* 1782 * Always allow an entire first page 1783 */ 1784 if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT)) 1785 max_size = biovec->bv_len; 1786 1787 return max_size; 1788 } 1789 1790 /* 1791 * The request function that just remaps the bio built up by 1792 * dm_merge_bvec. 1793 */ 1794 static void dm_make_request(struct request_queue *q, struct bio *bio) 1795 { 1796 int rw = bio_data_dir(bio); 1797 struct mapped_device *md = q->queuedata; 1798 int srcu_idx; 1799 struct dm_table *map; 1800 1801 map = dm_get_live_table(md, &srcu_idx); 1802 1803 generic_start_io_acct(rw, bio_sectors(bio), &dm_disk(md)->part0); 1804 1805 /* if we're suspended, we have to queue this io for later */ 1806 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) { 1807 dm_put_live_table(md, srcu_idx); 1808 1809 if (bio_rw(bio) != READA) 1810 queue_io(md, bio); 1811 else 1812 bio_io_error(bio); 1813 return; 1814 } 1815 1816 __split_and_process_bio(md, map, bio); 1817 dm_put_live_table(md, srcu_idx); 1818 return; 1819 } 1820 1821 int dm_request_based(struct mapped_device *md) 1822 { 1823 return blk_queue_stackable(md->queue); 1824 } 1825 1826 static void dm_dispatch_clone_request(struct request *clone, struct request *rq) 1827 { 1828 int r; 1829 1830 if (blk_queue_io_stat(clone->q)) 1831 clone->cmd_flags |= REQ_IO_STAT; 1832 1833 clone->start_time = jiffies; 1834 r = blk_insert_cloned_request(clone->q, clone); 1835 if (r) 1836 /* must complete clone in terms of original request */ 1837 dm_complete_request(rq, r); 1838 } 1839 1840 static int dm_rq_bio_constructor(struct bio *bio, struct bio *bio_orig, 1841 void *data) 1842 { 1843 struct dm_rq_target_io *tio = data; 1844 struct dm_rq_clone_bio_info *info = 1845 container_of(bio, struct dm_rq_clone_bio_info, clone); 1846 1847 info->orig = bio_orig; 1848 info->tio = tio; 1849 bio->bi_end_io = end_clone_bio; 1850 1851 return 0; 1852 } 1853 1854 static int setup_clone(struct request *clone, struct request *rq, 1855 struct dm_rq_target_io *tio, gfp_t gfp_mask) 1856 { 1857 int r; 1858 1859 r = blk_rq_prep_clone(clone, rq, tio->md->bs, gfp_mask, 1860 dm_rq_bio_constructor, tio); 1861 if (r) 1862 return r; 1863 1864 clone->cmd = rq->cmd; 1865 clone->cmd_len = rq->cmd_len; 1866 clone->sense = rq->sense; 1867 clone->end_io = end_clone_request; 1868 clone->end_io_data = tio; 1869 1870 tio->clone = clone; 1871 1872 return 0; 1873 } 1874 1875 static struct request *clone_rq(struct request *rq, struct mapped_device *md, 1876 struct dm_rq_target_io *tio, gfp_t gfp_mask) 1877 { 1878 /* 1879 * Do not allocate a clone if tio->clone was already set 1880 * (see: dm_mq_queue_rq). 1881 */ 1882 bool alloc_clone = !tio->clone; 1883 struct request *clone; 1884 1885 if (alloc_clone) { 1886 clone = alloc_clone_request(md, gfp_mask); 1887 if (!clone) 1888 return NULL; 1889 } else 1890 clone = tio->clone; 1891 1892 blk_rq_init(NULL, clone); 1893 if (setup_clone(clone, rq, tio, gfp_mask)) { 1894 /* -ENOMEM */ 1895 if (alloc_clone) 1896 free_clone_request(md, clone); 1897 return NULL; 1898 } 1899 1900 return clone; 1901 } 1902 1903 static void map_tio_request(struct kthread_work *work); 1904 1905 static void init_tio(struct dm_rq_target_io *tio, struct request *rq, 1906 struct mapped_device *md) 1907 { 1908 tio->md = md; 1909 tio->ti = NULL; 1910 tio->clone = NULL; 1911 tio->orig = rq; 1912 tio->error = 0; 1913 memset(&tio->info, 0, sizeof(tio->info)); 1914 if (md->kworker_task) 1915 init_kthread_work(&tio->work, map_tio_request); 1916 } 1917 1918 static struct dm_rq_target_io *prep_tio(struct request *rq, 1919 struct mapped_device *md, gfp_t gfp_mask) 1920 { 1921 struct dm_rq_target_io *tio; 1922 int srcu_idx; 1923 struct dm_table *table; 1924 1925 tio = alloc_rq_tio(md, gfp_mask); 1926 if (!tio) 1927 return NULL; 1928 1929 init_tio(tio, rq, md); 1930 1931 table = dm_get_live_table(md, &srcu_idx); 1932 if (!dm_table_mq_request_based(table)) { 1933 if (!clone_rq(rq, md, tio, gfp_mask)) { 1934 dm_put_live_table(md, srcu_idx); 1935 free_rq_tio(tio); 1936 return NULL; 1937 } 1938 } 1939 dm_put_live_table(md, srcu_idx); 1940 1941 return tio; 1942 } 1943 1944 /* 1945 * Called with the queue lock held. 1946 */ 1947 static int dm_prep_fn(struct request_queue *q, struct request *rq) 1948 { 1949 struct mapped_device *md = q->queuedata; 1950 struct dm_rq_target_io *tio; 1951 1952 if (unlikely(rq->special)) { 1953 DMWARN("Already has something in rq->special."); 1954 return BLKPREP_KILL; 1955 } 1956 1957 tio = prep_tio(rq, md, GFP_ATOMIC); 1958 if (!tio) 1959 return BLKPREP_DEFER; 1960 1961 rq->special = tio; 1962 rq->cmd_flags |= REQ_DONTPREP; 1963 1964 return BLKPREP_OK; 1965 } 1966 1967 /* 1968 * Returns: 1969 * 0 : the request has been processed 1970 * DM_MAPIO_REQUEUE : the original request needs to be requeued 1971 * < 0 : the request was completed due to failure 1972 */ 1973 static int map_request(struct dm_rq_target_io *tio, struct request *rq, 1974 struct mapped_device *md) 1975 { 1976 int r; 1977 struct dm_target *ti = tio->ti; 1978 struct request *clone = NULL; 1979 1980 if (tio->clone) { 1981 clone = tio->clone; 1982 r = ti->type->map_rq(ti, clone, &tio->info); 1983 } else { 1984 r = ti->type->clone_and_map_rq(ti, rq, &tio->info, &clone); 1985 if (r < 0) { 1986 /* The target wants to complete the I/O */ 1987 dm_kill_unmapped_request(rq, r); 1988 return r; 1989 } 1990 if (r != DM_MAPIO_REMAPPED) 1991 return r; 1992 if (setup_clone(clone, rq, tio, GFP_ATOMIC)) { 1993 /* -ENOMEM */ 1994 ti->type->release_clone_rq(clone); 1995 return DM_MAPIO_REQUEUE; 1996 } 1997 } 1998 1999 switch (r) { 2000 case DM_MAPIO_SUBMITTED: 2001 /* The target has taken the I/O to submit by itself later */ 2002 break; 2003 case DM_MAPIO_REMAPPED: 2004 /* The target has remapped the I/O so dispatch it */ 2005 trace_block_rq_remap(clone->q, clone, disk_devt(dm_disk(md)), 2006 blk_rq_pos(rq)); 2007 dm_dispatch_clone_request(clone, rq); 2008 break; 2009 case DM_MAPIO_REQUEUE: 2010 /* The target wants to requeue the I/O */ 2011 dm_requeue_original_request(md, tio->orig); 2012 break; 2013 default: 2014 if (r > 0) { 2015 DMWARN("unimplemented target map return value: %d", r); 2016 BUG(); 2017 } 2018 2019 /* The target wants to complete the I/O */ 2020 dm_kill_unmapped_request(rq, r); 2021 return r; 2022 } 2023 2024 return 0; 2025 } 2026 2027 static void map_tio_request(struct kthread_work *work) 2028 { 2029 struct dm_rq_target_io *tio = container_of(work, struct dm_rq_target_io, work); 2030 struct request *rq = tio->orig; 2031 struct mapped_device *md = tio->md; 2032 2033 if (map_request(tio, rq, md) == DM_MAPIO_REQUEUE) 2034 dm_requeue_original_request(md, rq); 2035 } 2036 2037 static void dm_start_request(struct mapped_device *md, struct request *orig) 2038 { 2039 if (!orig->q->mq_ops) 2040 blk_start_request(orig); 2041 else 2042 blk_mq_start_request(orig); 2043 atomic_inc(&md->pending[rq_data_dir(orig)]); 2044 2045 if (md->seq_rq_merge_deadline_usecs) { 2046 md->last_rq_pos = rq_end_sector(orig); 2047 md->last_rq_rw = rq_data_dir(orig); 2048 md->last_rq_start_time = ktime_get(); 2049 } 2050 2051 if (unlikely(dm_stats_used(&md->stats))) { 2052 struct dm_rq_target_io *tio = tio_from_request(orig); 2053 tio->duration_jiffies = jiffies; 2054 tio->n_sectors = blk_rq_sectors(orig); 2055 dm_stats_account_io(&md->stats, orig->cmd_flags, blk_rq_pos(orig), 2056 tio->n_sectors, false, 0, &tio->stats_aux); 2057 } 2058 2059 /* 2060 * Hold the md reference here for the in-flight I/O. 2061 * We can't rely on the reference count by device opener, 2062 * because the device may be closed during the request completion 2063 * when all bios are completed. 2064 * See the comment in rq_completed() too. 2065 */ 2066 dm_get(md); 2067 } 2068 2069 #define MAX_SEQ_RQ_MERGE_DEADLINE_USECS 100000 2070 2071 ssize_t dm_attr_rq_based_seq_io_merge_deadline_show(struct mapped_device *md, char *buf) 2072 { 2073 return sprintf(buf, "%u\n", md->seq_rq_merge_deadline_usecs); 2074 } 2075 2076 ssize_t dm_attr_rq_based_seq_io_merge_deadline_store(struct mapped_device *md, 2077 const char *buf, size_t count) 2078 { 2079 unsigned deadline; 2080 2081 if (!dm_request_based(md) || md->use_blk_mq) 2082 return count; 2083 2084 if (kstrtouint(buf, 10, &deadline)) 2085 return -EINVAL; 2086 2087 if (deadline > MAX_SEQ_RQ_MERGE_DEADLINE_USECS) 2088 deadline = MAX_SEQ_RQ_MERGE_DEADLINE_USECS; 2089 2090 md->seq_rq_merge_deadline_usecs = deadline; 2091 2092 return count; 2093 } 2094 2095 static bool dm_request_peeked_before_merge_deadline(struct mapped_device *md) 2096 { 2097 ktime_t kt_deadline; 2098 2099 if (!md->seq_rq_merge_deadline_usecs) 2100 return false; 2101 2102 kt_deadline = ns_to_ktime((u64)md->seq_rq_merge_deadline_usecs * NSEC_PER_USEC); 2103 kt_deadline = ktime_add_safe(md->last_rq_start_time, kt_deadline); 2104 2105 return !ktime_after(ktime_get(), kt_deadline); 2106 } 2107 2108 /* 2109 * q->request_fn for request-based dm. 2110 * Called with the queue lock held. 2111 */ 2112 static void dm_request_fn(struct request_queue *q) 2113 { 2114 struct mapped_device *md = q->queuedata; 2115 int srcu_idx; 2116 struct dm_table *map = dm_get_live_table(md, &srcu_idx); 2117 struct dm_target *ti; 2118 struct request *rq; 2119 struct dm_rq_target_io *tio; 2120 sector_t pos; 2121 2122 /* 2123 * For suspend, check blk_queue_stopped() and increment 2124 * ->pending within a single queue_lock not to increment the 2125 * number of in-flight I/Os after the queue is stopped in 2126 * dm_suspend(). 2127 */ 2128 while (!blk_queue_stopped(q)) { 2129 rq = blk_peek_request(q); 2130 if (!rq) 2131 goto out; 2132 2133 /* always use block 0 to find the target for flushes for now */ 2134 pos = 0; 2135 if (!(rq->cmd_flags & REQ_FLUSH)) 2136 pos = blk_rq_pos(rq); 2137 2138 ti = dm_table_find_target(map, pos); 2139 if (!dm_target_is_valid(ti)) { 2140 /* 2141 * Must perform setup, that rq_completed() requires, 2142 * before calling dm_kill_unmapped_request 2143 */ 2144 DMERR_LIMIT("request attempted access beyond the end of device"); 2145 dm_start_request(md, rq); 2146 dm_kill_unmapped_request(rq, -EIO); 2147 continue; 2148 } 2149 2150 if (dm_request_peeked_before_merge_deadline(md) && 2151 md_in_flight(md) && rq->bio && rq->bio->bi_vcnt == 1 && 2152 md->last_rq_pos == pos && md->last_rq_rw == rq_data_dir(rq)) 2153 goto delay_and_out; 2154 2155 if (ti->type->busy && ti->type->busy(ti)) 2156 goto delay_and_out; 2157 2158 dm_start_request(md, rq); 2159 2160 tio = tio_from_request(rq); 2161 /* Establish tio->ti before queuing work (map_tio_request) */ 2162 tio->ti = ti; 2163 queue_kthread_work(&md->kworker, &tio->work); 2164 BUG_ON(!irqs_disabled()); 2165 } 2166 2167 goto out; 2168 2169 delay_and_out: 2170 blk_delay_queue(q, HZ / 100); 2171 out: 2172 dm_put_live_table(md, srcu_idx); 2173 } 2174 2175 static int dm_any_congested(void *congested_data, int bdi_bits) 2176 { 2177 int r = bdi_bits; 2178 struct mapped_device *md = congested_data; 2179 struct dm_table *map; 2180 2181 if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) { 2182 map = dm_get_live_table_fast(md); 2183 if (map) { 2184 /* 2185 * Request-based dm cares about only own queue for 2186 * the query about congestion status of request_queue 2187 */ 2188 if (dm_request_based(md)) 2189 r = md->queue->backing_dev_info.wb.state & 2190 bdi_bits; 2191 else 2192 r = dm_table_any_congested(map, bdi_bits); 2193 } 2194 dm_put_live_table_fast(md); 2195 } 2196 2197 return r; 2198 } 2199 2200 /*----------------------------------------------------------------- 2201 * An IDR is used to keep track of allocated minor numbers. 2202 *---------------------------------------------------------------*/ 2203 static void free_minor(int minor) 2204 { 2205 spin_lock(&_minor_lock); 2206 idr_remove(&_minor_idr, minor); 2207 spin_unlock(&_minor_lock); 2208 } 2209 2210 /* 2211 * See if the device with a specific minor # is free. 2212 */ 2213 static int specific_minor(int minor) 2214 { 2215 int r; 2216 2217 if (minor >= (1 << MINORBITS)) 2218 return -EINVAL; 2219 2220 idr_preload(GFP_KERNEL); 2221 spin_lock(&_minor_lock); 2222 2223 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT); 2224 2225 spin_unlock(&_minor_lock); 2226 idr_preload_end(); 2227 if (r < 0) 2228 return r == -ENOSPC ? -EBUSY : r; 2229 return 0; 2230 } 2231 2232 static int next_free_minor(int *minor) 2233 { 2234 int r; 2235 2236 idr_preload(GFP_KERNEL); 2237 spin_lock(&_minor_lock); 2238 2239 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT); 2240 2241 spin_unlock(&_minor_lock); 2242 idr_preload_end(); 2243 if (r < 0) 2244 return r; 2245 *minor = r; 2246 return 0; 2247 } 2248 2249 static const struct block_device_operations dm_blk_dops; 2250 2251 static void dm_wq_work(struct work_struct *work); 2252 2253 static void dm_init_md_queue(struct mapped_device *md) 2254 { 2255 /* 2256 * Request-based dm devices cannot be stacked on top of bio-based dm 2257 * devices. The type of this dm device may not have been decided yet. 2258 * The type is decided at the first table loading time. 2259 * To prevent problematic device stacking, clear the queue flag 2260 * for request stacking support until then. 2261 * 2262 * This queue is new, so no concurrency on the queue_flags. 2263 */ 2264 queue_flag_clear_unlocked(QUEUE_FLAG_STACKABLE, md->queue); 2265 } 2266 2267 static void dm_init_old_md_queue(struct mapped_device *md) 2268 { 2269 md->use_blk_mq = false; 2270 dm_init_md_queue(md); 2271 2272 /* 2273 * Initialize aspects of queue that aren't relevant for blk-mq 2274 */ 2275 md->queue->queuedata = md; 2276 md->queue->backing_dev_info.congested_fn = dm_any_congested; 2277 md->queue->backing_dev_info.congested_data = md; 2278 2279 blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY); 2280 } 2281 2282 static void cleanup_mapped_device(struct mapped_device *md) 2283 { 2284 cleanup_srcu_struct(&md->io_barrier); 2285 2286 if (md->wq) 2287 destroy_workqueue(md->wq); 2288 if (md->kworker_task) 2289 kthread_stop(md->kworker_task); 2290 if (md->io_pool) 2291 mempool_destroy(md->io_pool); 2292 if (md->rq_pool) 2293 mempool_destroy(md->rq_pool); 2294 if (md->bs) 2295 bioset_free(md->bs); 2296 2297 if (md->disk) { 2298 spin_lock(&_minor_lock); 2299 md->disk->private_data = NULL; 2300 spin_unlock(&_minor_lock); 2301 if (blk_get_integrity(md->disk)) 2302 blk_integrity_unregister(md->disk); 2303 del_gendisk(md->disk); 2304 put_disk(md->disk); 2305 } 2306 2307 if (md->queue) 2308 blk_cleanup_queue(md->queue); 2309 2310 if (md->bdev) { 2311 bdput(md->bdev); 2312 md->bdev = NULL; 2313 } 2314 } 2315 2316 /* 2317 * Allocate and initialise a blank device with a given minor. 2318 */ 2319 static struct mapped_device *alloc_dev(int minor) 2320 { 2321 int r; 2322 struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL); 2323 void *old_md; 2324 2325 if (!md) { 2326 DMWARN("unable to allocate device, out of memory."); 2327 return NULL; 2328 } 2329 2330 if (!try_module_get(THIS_MODULE)) 2331 goto bad_module_get; 2332 2333 /* get a minor number for the dev */ 2334 if (minor == DM_ANY_MINOR) 2335 r = next_free_minor(&minor); 2336 else 2337 r = specific_minor(minor); 2338 if (r < 0) 2339 goto bad_minor; 2340 2341 r = init_srcu_struct(&md->io_barrier); 2342 if (r < 0) 2343 goto bad_io_barrier; 2344 2345 md->use_blk_mq = use_blk_mq; 2346 md->type = DM_TYPE_NONE; 2347 mutex_init(&md->suspend_lock); 2348 mutex_init(&md->type_lock); 2349 mutex_init(&md->table_devices_lock); 2350 spin_lock_init(&md->deferred_lock); 2351 atomic_set(&md->holders, 1); 2352 atomic_set(&md->open_count, 0); 2353 atomic_set(&md->event_nr, 0); 2354 atomic_set(&md->uevent_seq, 0); 2355 INIT_LIST_HEAD(&md->uevent_list); 2356 INIT_LIST_HEAD(&md->table_devices); 2357 spin_lock_init(&md->uevent_lock); 2358 2359 md->queue = blk_alloc_queue(GFP_KERNEL); 2360 if (!md->queue) 2361 goto bad; 2362 2363 dm_init_md_queue(md); 2364 2365 md->disk = alloc_disk(1); 2366 if (!md->disk) 2367 goto bad; 2368 2369 atomic_set(&md->pending[0], 0); 2370 atomic_set(&md->pending[1], 0); 2371 init_waitqueue_head(&md->wait); 2372 INIT_WORK(&md->work, dm_wq_work); 2373 init_waitqueue_head(&md->eventq); 2374 init_completion(&md->kobj_holder.completion); 2375 md->kworker_task = NULL; 2376 2377 md->disk->major = _major; 2378 md->disk->first_minor = minor; 2379 md->disk->fops = &dm_blk_dops; 2380 md->disk->queue = md->queue; 2381 md->disk->private_data = md; 2382 sprintf(md->disk->disk_name, "dm-%d", minor); 2383 add_disk(md->disk); 2384 format_dev_t(md->name, MKDEV(_major, minor)); 2385 2386 md->wq = alloc_workqueue("kdmflush", WQ_MEM_RECLAIM, 0); 2387 if (!md->wq) 2388 goto bad; 2389 2390 md->bdev = bdget_disk(md->disk, 0); 2391 if (!md->bdev) 2392 goto bad; 2393 2394 bio_init(&md->flush_bio); 2395 md->flush_bio.bi_bdev = md->bdev; 2396 md->flush_bio.bi_rw = WRITE_FLUSH; 2397 2398 dm_stats_init(&md->stats); 2399 2400 /* Populate the mapping, nobody knows we exist yet */ 2401 spin_lock(&_minor_lock); 2402 old_md = idr_replace(&_minor_idr, md, minor); 2403 spin_unlock(&_minor_lock); 2404 2405 BUG_ON(old_md != MINOR_ALLOCED); 2406 2407 return md; 2408 2409 bad: 2410 cleanup_mapped_device(md); 2411 bad_io_barrier: 2412 free_minor(minor); 2413 bad_minor: 2414 module_put(THIS_MODULE); 2415 bad_module_get: 2416 kfree(md); 2417 return NULL; 2418 } 2419 2420 static void unlock_fs(struct mapped_device *md); 2421 2422 static void free_dev(struct mapped_device *md) 2423 { 2424 int minor = MINOR(disk_devt(md->disk)); 2425 2426 unlock_fs(md); 2427 2428 cleanup_mapped_device(md); 2429 if (md->use_blk_mq) 2430 blk_mq_free_tag_set(&md->tag_set); 2431 2432 free_table_devices(&md->table_devices); 2433 dm_stats_cleanup(&md->stats); 2434 free_minor(minor); 2435 2436 module_put(THIS_MODULE); 2437 kfree(md); 2438 } 2439 2440 static void __bind_mempools(struct mapped_device *md, struct dm_table *t) 2441 { 2442 struct dm_md_mempools *p = dm_table_get_md_mempools(t); 2443 2444 if (md->bs) { 2445 /* The md already has necessary mempools. */ 2446 if (dm_table_get_type(t) == DM_TYPE_BIO_BASED) { 2447 /* 2448 * Reload bioset because front_pad may have changed 2449 * because a different table was loaded. 2450 */ 2451 bioset_free(md->bs); 2452 md->bs = p->bs; 2453 p->bs = NULL; 2454 } 2455 /* 2456 * There's no need to reload with request-based dm 2457 * because the size of front_pad doesn't change. 2458 * Note for future: If you are to reload bioset, 2459 * prep-ed requests in the queue may refer 2460 * to bio from the old bioset, so you must walk 2461 * through the queue to unprep. 2462 */ 2463 goto out; 2464 } 2465 2466 BUG_ON(!p || md->io_pool || md->rq_pool || md->bs); 2467 2468 md->io_pool = p->io_pool; 2469 p->io_pool = NULL; 2470 md->rq_pool = p->rq_pool; 2471 p->rq_pool = NULL; 2472 md->bs = p->bs; 2473 p->bs = NULL; 2474 2475 out: 2476 /* mempool bind completed, no longer need any mempools in the table */ 2477 dm_table_free_md_mempools(t); 2478 } 2479 2480 /* 2481 * Bind a table to the device. 2482 */ 2483 static void event_callback(void *context) 2484 { 2485 unsigned long flags; 2486 LIST_HEAD(uevents); 2487 struct mapped_device *md = (struct mapped_device *) context; 2488 2489 spin_lock_irqsave(&md->uevent_lock, flags); 2490 list_splice_init(&md->uevent_list, &uevents); 2491 spin_unlock_irqrestore(&md->uevent_lock, flags); 2492 2493 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj); 2494 2495 atomic_inc(&md->event_nr); 2496 wake_up(&md->eventq); 2497 } 2498 2499 /* 2500 * Protected by md->suspend_lock obtained by dm_swap_table(). 2501 */ 2502 static void __set_size(struct mapped_device *md, sector_t size) 2503 { 2504 set_capacity(md->disk, size); 2505 2506 i_size_write(md->bdev->bd_inode, (loff_t)size << SECTOR_SHIFT); 2507 } 2508 2509 /* 2510 * Return 1 if the queue has a compulsory merge_bvec_fn function. 2511 * 2512 * If this function returns 0, then the device is either a non-dm 2513 * device without a merge_bvec_fn, or it is a dm device that is 2514 * able to split any bios it receives that are too big. 2515 */ 2516 int dm_queue_merge_is_compulsory(struct request_queue *q) 2517 { 2518 struct mapped_device *dev_md; 2519 2520 if (!q->merge_bvec_fn) 2521 return 0; 2522 2523 if (q->make_request_fn == dm_make_request) { 2524 dev_md = q->queuedata; 2525 if (test_bit(DMF_MERGE_IS_OPTIONAL, &dev_md->flags)) 2526 return 0; 2527 } 2528 2529 return 1; 2530 } 2531 2532 static int dm_device_merge_is_compulsory(struct dm_target *ti, 2533 struct dm_dev *dev, sector_t start, 2534 sector_t len, void *data) 2535 { 2536 struct block_device *bdev = dev->bdev; 2537 struct request_queue *q = bdev_get_queue(bdev); 2538 2539 return dm_queue_merge_is_compulsory(q); 2540 } 2541 2542 /* 2543 * Return 1 if it is acceptable to ignore merge_bvec_fn based 2544 * on the properties of the underlying devices. 2545 */ 2546 static int dm_table_merge_is_optional(struct dm_table *table) 2547 { 2548 unsigned i = 0; 2549 struct dm_target *ti; 2550 2551 while (i < dm_table_get_num_targets(table)) { 2552 ti = dm_table_get_target(table, i++); 2553 2554 if (ti->type->iterate_devices && 2555 ti->type->iterate_devices(ti, dm_device_merge_is_compulsory, NULL)) 2556 return 0; 2557 } 2558 2559 return 1; 2560 } 2561 2562 /* 2563 * Returns old map, which caller must destroy. 2564 */ 2565 static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t, 2566 struct queue_limits *limits) 2567 { 2568 struct dm_table *old_map; 2569 struct request_queue *q = md->queue; 2570 sector_t size; 2571 int merge_is_optional; 2572 2573 size = dm_table_get_size(t); 2574 2575 /* 2576 * Wipe any geometry if the size of the table changed. 2577 */ 2578 if (size != dm_get_size(md)) 2579 memset(&md->geometry, 0, sizeof(md->geometry)); 2580 2581 __set_size(md, size); 2582 2583 dm_table_event_callback(t, event_callback, md); 2584 2585 /* 2586 * The queue hasn't been stopped yet, if the old table type wasn't 2587 * for request-based during suspension. So stop it to prevent 2588 * I/O mapping before resume. 2589 * This must be done before setting the queue restrictions, 2590 * because request-based dm may be run just after the setting. 2591 */ 2592 if (dm_table_request_based(t)) 2593 stop_queue(q); 2594 2595 __bind_mempools(md, t); 2596 2597 merge_is_optional = dm_table_merge_is_optional(t); 2598 2599 old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2600 rcu_assign_pointer(md->map, t); 2601 md->immutable_target_type = dm_table_get_immutable_target_type(t); 2602 2603 dm_table_set_restrictions(t, q, limits); 2604 if (merge_is_optional) 2605 set_bit(DMF_MERGE_IS_OPTIONAL, &md->flags); 2606 else 2607 clear_bit(DMF_MERGE_IS_OPTIONAL, &md->flags); 2608 if (old_map) 2609 dm_sync_table(md); 2610 2611 return old_map; 2612 } 2613 2614 /* 2615 * Returns unbound table for the caller to free. 2616 */ 2617 static struct dm_table *__unbind(struct mapped_device *md) 2618 { 2619 struct dm_table *map = rcu_dereference_protected(md->map, 1); 2620 2621 if (!map) 2622 return NULL; 2623 2624 dm_table_event_callback(map, NULL, NULL); 2625 RCU_INIT_POINTER(md->map, NULL); 2626 dm_sync_table(md); 2627 2628 return map; 2629 } 2630 2631 /* 2632 * Constructor for a new device. 2633 */ 2634 int dm_create(int minor, struct mapped_device **result) 2635 { 2636 struct mapped_device *md; 2637 2638 md = alloc_dev(minor); 2639 if (!md) 2640 return -ENXIO; 2641 2642 dm_sysfs_init(md); 2643 2644 *result = md; 2645 return 0; 2646 } 2647 2648 /* 2649 * Functions to manage md->type. 2650 * All are required to hold md->type_lock. 2651 */ 2652 void dm_lock_md_type(struct mapped_device *md) 2653 { 2654 mutex_lock(&md->type_lock); 2655 } 2656 2657 void dm_unlock_md_type(struct mapped_device *md) 2658 { 2659 mutex_unlock(&md->type_lock); 2660 } 2661 2662 void dm_set_md_type(struct mapped_device *md, unsigned type) 2663 { 2664 BUG_ON(!mutex_is_locked(&md->type_lock)); 2665 md->type = type; 2666 } 2667 2668 unsigned dm_get_md_type(struct mapped_device *md) 2669 { 2670 BUG_ON(!mutex_is_locked(&md->type_lock)); 2671 return md->type; 2672 } 2673 2674 struct target_type *dm_get_immutable_target_type(struct mapped_device *md) 2675 { 2676 return md->immutable_target_type; 2677 } 2678 2679 /* 2680 * The queue_limits are only valid as long as you have a reference 2681 * count on 'md'. 2682 */ 2683 struct queue_limits *dm_get_queue_limits(struct mapped_device *md) 2684 { 2685 BUG_ON(!atomic_read(&md->holders)); 2686 return &md->queue->limits; 2687 } 2688 EXPORT_SYMBOL_GPL(dm_get_queue_limits); 2689 2690 static void init_rq_based_worker_thread(struct mapped_device *md) 2691 { 2692 /* Initialize the request-based DM worker thread */ 2693 init_kthread_worker(&md->kworker); 2694 md->kworker_task = kthread_run(kthread_worker_fn, &md->kworker, 2695 "kdmwork-%s", dm_device_name(md)); 2696 } 2697 2698 /* 2699 * Fully initialize a request-based queue (->elevator, ->request_fn, etc). 2700 */ 2701 static int dm_init_request_based_queue(struct mapped_device *md) 2702 { 2703 struct request_queue *q = NULL; 2704 2705 /* Fully initialize the queue */ 2706 q = blk_init_allocated_queue(md->queue, dm_request_fn, NULL); 2707 if (!q) 2708 return -EINVAL; 2709 2710 /* disable dm_request_fn's merge heuristic by default */ 2711 md->seq_rq_merge_deadline_usecs = 0; 2712 2713 md->queue = q; 2714 dm_init_old_md_queue(md); 2715 blk_queue_softirq_done(md->queue, dm_softirq_done); 2716 blk_queue_prep_rq(md->queue, dm_prep_fn); 2717 2718 init_rq_based_worker_thread(md); 2719 2720 elv_register_queue(md->queue); 2721 2722 return 0; 2723 } 2724 2725 static int dm_mq_init_request(void *data, struct request *rq, 2726 unsigned int hctx_idx, unsigned int request_idx, 2727 unsigned int numa_node) 2728 { 2729 struct mapped_device *md = data; 2730 struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq); 2731 2732 /* 2733 * Must initialize md member of tio, otherwise it won't 2734 * be available in dm_mq_queue_rq. 2735 */ 2736 tio->md = md; 2737 2738 return 0; 2739 } 2740 2741 static int dm_mq_queue_rq(struct blk_mq_hw_ctx *hctx, 2742 const struct blk_mq_queue_data *bd) 2743 { 2744 struct request *rq = bd->rq; 2745 struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq); 2746 struct mapped_device *md = tio->md; 2747 int srcu_idx; 2748 struct dm_table *map = dm_get_live_table(md, &srcu_idx); 2749 struct dm_target *ti; 2750 sector_t pos; 2751 2752 /* always use block 0 to find the target for flushes for now */ 2753 pos = 0; 2754 if (!(rq->cmd_flags & REQ_FLUSH)) 2755 pos = blk_rq_pos(rq); 2756 2757 ti = dm_table_find_target(map, pos); 2758 if (!dm_target_is_valid(ti)) { 2759 dm_put_live_table(md, srcu_idx); 2760 DMERR_LIMIT("request attempted access beyond the end of device"); 2761 /* 2762 * Must perform setup, that rq_completed() requires, 2763 * before returning BLK_MQ_RQ_QUEUE_ERROR 2764 */ 2765 dm_start_request(md, rq); 2766 return BLK_MQ_RQ_QUEUE_ERROR; 2767 } 2768 dm_put_live_table(md, srcu_idx); 2769 2770 if (ti->type->busy && ti->type->busy(ti)) 2771 return BLK_MQ_RQ_QUEUE_BUSY; 2772 2773 dm_start_request(md, rq); 2774 2775 /* Init tio using md established in .init_request */ 2776 init_tio(tio, rq, md); 2777 2778 /* 2779 * Establish tio->ti before queuing work (map_tio_request) 2780 * or making direct call to map_request(). 2781 */ 2782 tio->ti = ti; 2783 2784 /* Clone the request if underlying devices aren't blk-mq */ 2785 if (dm_table_get_type(map) == DM_TYPE_REQUEST_BASED) { 2786 /* clone request is allocated at the end of the pdu */ 2787 tio->clone = (void *)blk_mq_rq_to_pdu(rq) + sizeof(struct dm_rq_target_io); 2788 (void) clone_rq(rq, md, tio, GFP_ATOMIC); 2789 queue_kthread_work(&md->kworker, &tio->work); 2790 } else { 2791 /* Direct call is fine since .queue_rq allows allocations */ 2792 if (map_request(tio, rq, md) == DM_MAPIO_REQUEUE) { 2793 /* Undo dm_start_request() before requeuing */ 2794 rq_end_stats(md, rq); 2795 rq_completed(md, rq_data_dir(rq), false); 2796 return BLK_MQ_RQ_QUEUE_BUSY; 2797 } 2798 } 2799 2800 return BLK_MQ_RQ_QUEUE_OK; 2801 } 2802 2803 static struct blk_mq_ops dm_mq_ops = { 2804 .queue_rq = dm_mq_queue_rq, 2805 .map_queue = blk_mq_map_queue, 2806 .complete = dm_softirq_done, 2807 .init_request = dm_mq_init_request, 2808 }; 2809 2810 static int dm_init_request_based_blk_mq_queue(struct mapped_device *md) 2811 { 2812 unsigned md_type = dm_get_md_type(md); 2813 struct request_queue *q; 2814 int err; 2815 2816 memset(&md->tag_set, 0, sizeof(md->tag_set)); 2817 md->tag_set.ops = &dm_mq_ops; 2818 md->tag_set.queue_depth = BLKDEV_MAX_RQ; 2819 md->tag_set.numa_node = NUMA_NO_NODE; 2820 md->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE; 2821 md->tag_set.nr_hw_queues = 1; 2822 if (md_type == DM_TYPE_REQUEST_BASED) { 2823 /* make the memory for non-blk-mq clone part of the pdu */ 2824 md->tag_set.cmd_size = sizeof(struct dm_rq_target_io) + sizeof(struct request); 2825 } else 2826 md->tag_set.cmd_size = sizeof(struct dm_rq_target_io); 2827 md->tag_set.driver_data = md; 2828 2829 err = blk_mq_alloc_tag_set(&md->tag_set); 2830 if (err) 2831 return err; 2832 2833 q = blk_mq_init_allocated_queue(&md->tag_set, md->queue); 2834 if (IS_ERR(q)) { 2835 err = PTR_ERR(q); 2836 goto out_tag_set; 2837 } 2838 md->queue = q; 2839 dm_init_md_queue(md); 2840 2841 /* backfill 'mq' sysfs registration normally done in blk_register_queue */ 2842 blk_mq_register_disk(md->disk); 2843 2844 if (md_type == DM_TYPE_REQUEST_BASED) 2845 init_rq_based_worker_thread(md); 2846 2847 return 0; 2848 2849 out_tag_set: 2850 blk_mq_free_tag_set(&md->tag_set); 2851 return err; 2852 } 2853 2854 static unsigned filter_md_type(unsigned type, struct mapped_device *md) 2855 { 2856 if (type == DM_TYPE_BIO_BASED) 2857 return type; 2858 2859 return !md->use_blk_mq ? DM_TYPE_REQUEST_BASED : DM_TYPE_MQ_REQUEST_BASED; 2860 } 2861 2862 /* 2863 * Setup the DM device's queue based on md's type 2864 */ 2865 int dm_setup_md_queue(struct mapped_device *md) 2866 { 2867 int r; 2868 unsigned md_type = filter_md_type(dm_get_md_type(md), md); 2869 2870 switch (md_type) { 2871 case DM_TYPE_REQUEST_BASED: 2872 r = dm_init_request_based_queue(md); 2873 if (r) { 2874 DMWARN("Cannot initialize queue for request-based mapped device"); 2875 return r; 2876 } 2877 break; 2878 case DM_TYPE_MQ_REQUEST_BASED: 2879 r = dm_init_request_based_blk_mq_queue(md); 2880 if (r) { 2881 DMWARN("Cannot initialize queue for request-based blk-mq mapped device"); 2882 return r; 2883 } 2884 break; 2885 case DM_TYPE_BIO_BASED: 2886 dm_init_old_md_queue(md); 2887 blk_queue_make_request(md->queue, dm_make_request); 2888 blk_queue_merge_bvec(md->queue, dm_merge_bvec); 2889 break; 2890 } 2891 2892 return 0; 2893 } 2894 2895 struct mapped_device *dm_get_md(dev_t dev) 2896 { 2897 struct mapped_device *md; 2898 unsigned minor = MINOR(dev); 2899 2900 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS)) 2901 return NULL; 2902 2903 spin_lock(&_minor_lock); 2904 2905 md = idr_find(&_minor_idr, minor); 2906 if (md) { 2907 if ((md == MINOR_ALLOCED || 2908 (MINOR(disk_devt(dm_disk(md))) != minor) || 2909 dm_deleting_md(md) || 2910 test_bit(DMF_FREEING, &md->flags))) { 2911 md = NULL; 2912 goto out; 2913 } 2914 dm_get(md); 2915 } 2916 2917 out: 2918 spin_unlock(&_minor_lock); 2919 2920 return md; 2921 } 2922 EXPORT_SYMBOL_GPL(dm_get_md); 2923 2924 void *dm_get_mdptr(struct mapped_device *md) 2925 { 2926 return md->interface_ptr; 2927 } 2928 2929 void dm_set_mdptr(struct mapped_device *md, void *ptr) 2930 { 2931 md->interface_ptr = ptr; 2932 } 2933 2934 void dm_get(struct mapped_device *md) 2935 { 2936 atomic_inc(&md->holders); 2937 BUG_ON(test_bit(DMF_FREEING, &md->flags)); 2938 } 2939 2940 int dm_hold(struct mapped_device *md) 2941 { 2942 spin_lock(&_minor_lock); 2943 if (test_bit(DMF_FREEING, &md->flags)) { 2944 spin_unlock(&_minor_lock); 2945 return -EBUSY; 2946 } 2947 dm_get(md); 2948 spin_unlock(&_minor_lock); 2949 return 0; 2950 } 2951 EXPORT_SYMBOL_GPL(dm_hold); 2952 2953 const char *dm_device_name(struct mapped_device *md) 2954 { 2955 return md->name; 2956 } 2957 EXPORT_SYMBOL_GPL(dm_device_name); 2958 2959 static void __dm_destroy(struct mapped_device *md, bool wait) 2960 { 2961 struct dm_table *map; 2962 int srcu_idx; 2963 2964 might_sleep(); 2965 2966 map = dm_get_live_table(md, &srcu_idx); 2967 2968 spin_lock(&_minor_lock); 2969 idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md)))); 2970 set_bit(DMF_FREEING, &md->flags); 2971 spin_unlock(&_minor_lock); 2972 2973 if (dm_request_based(md) && md->kworker_task) 2974 flush_kthread_worker(&md->kworker); 2975 2976 /* 2977 * Take suspend_lock so that presuspend and postsuspend methods 2978 * do not race with internal suspend. 2979 */ 2980 mutex_lock(&md->suspend_lock); 2981 if (!dm_suspended_md(md)) { 2982 dm_table_presuspend_targets(map); 2983 dm_table_postsuspend_targets(map); 2984 } 2985 mutex_unlock(&md->suspend_lock); 2986 2987 /* dm_put_live_table must be before msleep, otherwise deadlock is possible */ 2988 dm_put_live_table(md, srcu_idx); 2989 2990 /* 2991 * Rare, but there may be I/O requests still going to complete, 2992 * for example. Wait for all references to disappear. 2993 * No one should increment the reference count of the mapped_device, 2994 * after the mapped_device state becomes DMF_FREEING. 2995 */ 2996 if (wait) 2997 while (atomic_read(&md->holders)) 2998 msleep(1); 2999 else if (atomic_read(&md->holders)) 3000 DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)", 3001 dm_device_name(md), atomic_read(&md->holders)); 3002 3003 dm_sysfs_exit(md); 3004 dm_table_destroy(__unbind(md)); 3005 free_dev(md); 3006 } 3007 3008 void dm_destroy(struct mapped_device *md) 3009 { 3010 __dm_destroy(md, true); 3011 } 3012 3013 void dm_destroy_immediate(struct mapped_device *md) 3014 { 3015 __dm_destroy(md, false); 3016 } 3017 3018 void dm_put(struct mapped_device *md) 3019 { 3020 atomic_dec(&md->holders); 3021 } 3022 EXPORT_SYMBOL_GPL(dm_put); 3023 3024 static int dm_wait_for_completion(struct mapped_device *md, int interruptible) 3025 { 3026 int r = 0; 3027 DECLARE_WAITQUEUE(wait, current); 3028 3029 add_wait_queue(&md->wait, &wait); 3030 3031 while (1) { 3032 set_current_state(interruptible); 3033 3034 if (!md_in_flight(md)) 3035 break; 3036 3037 if (interruptible == TASK_INTERRUPTIBLE && 3038 signal_pending(current)) { 3039 r = -EINTR; 3040 break; 3041 } 3042 3043 io_schedule(); 3044 } 3045 set_current_state(TASK_RUNNING); 3046 3047 remove_wait_queue(&md->wait, &wait); 3048 3049 return r; 3050 } 3051 3052 /* 3053 * Process the deferred bios 3054 */ 3055 static void dm_wq_work(struct work_struct *work) 3056 { 3057 struct mapped_device *md = container_of(work, struct mapped_device, 3058 work); 3059 struct bio *c; 3060 int srcu_idx; 3061 struct dm_table *map; 3062 3063 map = dm_get_live_table(md, &srcu_idx); 3064 3065 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) { 3066 spin_lock_irq(&md->deferred_lock); 3067 c = bio_list_pop(&md->deferred); 3068 spin_unlock_irq(&md->deferred_lock); 3069 3070 if (!c) 3071 break; 3072 3073 if (dm_request_based(md)) 3074 generic_make_request(c); 3075 else 3076 __split_and_process_bio(md, map, c); 3077 } 3078 3079 dm_put_live_table(md, srcu_idx); 3080 } 3081 3082 static void dm_queue_flush(struct mapped_device *md) 3083 { 3084 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 3085 smp_mb__after_atomic(); 3086 queue_work(md->wq, &md->work); 3087 } 3088 3089 /* 3090 * Swap in a new table, returning the old one for the caller to destroy. 3091 */ 3092 struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table) 3093 { 3094 struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL); 3095 struct queue_limits limits; 3096 int r; 3097 3098 mutex_lock(&md->suspend_lock); 3099 3100 /* device must be suspended */ 3101 if (!dm_suspended_md(md)) 3102 goto out; 3103 3104 /* 3105 * If the new table has no data devices, retain the existing limits. 3106 * This helps multipath with queue_if_no_path if all paths disappear, 3107 * then new I/O is queued based on these limits, and then some paths 3108 * reappear. 3109 */ 3110 if (dm_table_has_no_data_devices(table)) { 3111 live_map = dm_get_live_table_fast(md); 3112 if (live_map) 3113 limits = md->queue->limits; 3114 dm_put_live_table_fast(md); 3115 } 3116 3117 if (!live_map) { 3118 r = dm_calculate_queue_limits(table, &limits); 3119 if (r) { 3120 map = ERR_PTR(r); 3121 goto out; 3122 } 3123 } 3124 3125 map = __bind(md, table, &limits); 3126 3127 out: 3128 mutex_unlock(&md->suspend_lock); 3129 return map; 3130 } 3131 3132 /* 3133 * Functions to lock and unlock any filesystem running on the 3134 * device. 3135 */ 3136 static int lock_fs(struct mapped_device *md) 3137 { 3138 int r; 3139 3140 WARN_ON(md->frozen_sb); 3141 3142 md->frozen_sb = freeze_bdev(md->bdev); 3143 if (IS_ERR(md->frozen_sb)) { 3144 r = PTR_ERR(md->frozen_sb); 3145 md->frozen_sb = NULL; 3146 return r; 3147 } 3148 3149 set_bit(DMF_FROZEN, &md->flags); 3150 3151 return 0; 3152 } 3153 3154 static void unlock_fs(struct mapped_device *md) 3155 { 3156 if (!test_bit(DMF_FROZEN, &md->flags)) 3157 return; 3158 3159 thaw_bdev(md->bdev, md->frozen_sb); 3160 md->frozen_sb = NULL; 3161 clear_bit(DMF_FROZEN, &md->flags); 3162 } 3163 3164 /* 3165 * If __dm_suspend returns 0, the device is completely quiescent 3166 * now. There is no request-processing activity. All new requests 3167 * are being added to md->deferred list. 3168 * 3169 * Caller must hold md->suspend_lock 3170 */ 3171 static int __dm_suspend(struct mapped_device *md, struct dm_table *map, 3172 unsigned suspend_flags, int interruptible) 3173 { 3174 bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG; 3175 bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG; 3176 int r; 3177 3178 /* 3179 * DMF_NOFLUSH_SUSPENDING must be set before presuspend. 3180 * This flag is cleared before dm_suspend returns. 3181 */ 3182 if (noflush) 3183 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 3184 3185 /* 3186 * This gets reverted if there's an error later and the targets 3187 * provide the .presuspend_undo hook. 3188 */ 3189 dm_table_presuspend_targets(map); 3190 3191 /* 3192 * Flush I/O to the device. 3193 * Any I/O submitted after lock_fs() may not be flushed. 3194 * noflush takes precedence over do_lockfs. 3195 * (lock_fs() flushes I/Os and waits for them to complete.) 3196 */ 3197 if (!noflush && do_lockfs) { 3198 r = lock_fs(md); 3199 if (r) { 3200 dm_table_presuspend_undo_targets(map); 3201 return r; 3202 } 3203 } 3204 3205 /* 3206 * Here we must make sure that no processes are submitting requests 3207 * to target drivers i.e. no one may be executing 3208 * __split_and_process_bio. This is called from dm_request and 3209 * dm_wq_work. 3210 * 3211 * To get all processes out of __split_and_process_bio in dm_request, 3212 * we take the write lock. To prevent any process from reentering 3213 * __split_and_process_bio from dm_request and quiesce the thread 3214 * (dm_wq_work), we set BMF_BLOCK_IO_FOR_SUSPEND and call 3215 * flush_workqueue(md->wq). 3216 */ 3217 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 3218 if (map) 3219 synchronize_srcu(&md->io_barrier); 3220 3221 /* 3222 * Stop md->queue before flushing md->wq in case request-based 3223 * dm defers requests to md->wq from md->queue. 3224 */ 3225 if (dm_request_based(md)) { 3226 stop_queue(md->queue); 3227 if (md->kworker_task) 3228 flush_kthread_worker(&md->kworker); 3229 } 3230 3231 flush_workqueue(md->wq); 3232 3233 /* 3234 * At this point no more requests are entering target request routines. 3235 * We call dm_wait_for_completion to wait for all existing requests 3236 * to finish. 3237 */ 3238 r = dm_wait_for_completion(md, interruptible); 3239 3240 if (noflush) 3241 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 3242 if (map) 3243 synchronize_srcu(&md->io_barrier); 3244 3245 /* were we interrupted ? */ 3246 if (r < 0) { 3247 dm_queue_flush(md); 3248 3249 if (dm_request_based(md)) 3250 start_queue(md->queue); 3251 3252 unlock_fs(md); 3253 dm_table_presuspend_undo_targets(map); 3254 /* pushback list is already flushed, so skip flush */ 3255 } 3256 3257 return r; 3258 } 3259 3260 /* 3261 * We need to be able to change a mapping table under a mounted 3262 * filesystem. For example we might want to move some data in 3263 * the background. Before the table can be swapped with 3264 * dm_bind_table, dm_suspend must be called to flush any in 3265 * flight bios and ensure that any further io gets deferred. 3266 */ 3267 /* 3268 * Suspend mechanism in request-based dm. 3269 * 3270 * 1. Flush all I/Os by lock_fs() if needed. 3271 * 2. Stop dispatching any I/O by stopping the request_queue. 3272 * 3. Wait for all in-flight I/Os to be completed or requeued. 3273 * 3274 * To abort suspend, start the request_queue. 3275 */ 3276 int dm_suspend(struct mapped_device *md, unsigned suspend_flags) 3277 { 3278 struct dm_table *map = NULL; 3279 int r = 0; 3280 3281 retry: 3282 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); 3283 3284 if (dm_suspended_md(md)) { 3285 r = -EINVAL; 3286 goto out_unlock; 3287 } 3288 3289 if (dm_suspended_internally_md(md)) { 3290 /* already internally suspended, wait for internal resume */ 3291 mutex_unlock(&md->suspend_lock); 3292 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); 3293 if (r) 3294 return r; 3295 goto retry; 3296 } 3297 3298 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 3299 3300 r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE); 3301 if (r) 3302 goto out_unlock; 3303 3304 set_bit(DMF_SUSPENDED, &md->flags); 3305 3306 dm_table_postsuspend_targets(map); 3307 3308 out_unlock: 3309 mutex_unlock(&md->suspend_lock); 3310 return r; 3311 } 3312 3313 static int __dm_resume(struct mapped_device *md, struct dm_table *map) 3314 { 3315 if (map) { 3316 int r = dm_table_resume_targets(map); 3317 if (r) 3318 return r; 3319 } 3320 3321 dm_queue_flush(md); 3322 3323 /* 3324 * Flushing deferred I/Os must be done after targets are resumed 3325 * so that mapping of targets can work correctly. 3326 * Request-based dm is queueing the deferred I/Os in its request_queue. 3327 */ 3328 if (dm_request_based(md)) 3329 start_queue(md->queue); 3330 3331 unlock_fs(md); 3332 3333 return 0; 3334 } 3335 3336 int dm_resume(struct mapped_device *md) 3337 { 3338 int r = -EINVAL; 3339 struct dm_table *map = NULL; 3340 3341 retry: 3342 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); 3343 3344 if (!dm_suspended_md(md)) 3345 goto out; 3346 3347 if (dm_suspended_internally_md(md)) { 3348 /* already internally suspended, wait for internal resume */ 3349 mutex_unlock(&md->suspend_lock); 3350 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); 3351 if (r) 3352 return r; 3353 goto retry; 3354 } 3355 3356 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 3357 if (!map || !dm_table_get_size(map)) 3358 goto out; 3359 3360 r = __dm_resume(md, map); 3361 if (r) 3362 goto out; 3363 3364 clear_bit(DMF_SUSPENDED, &md->flags); 3365 3366 r = 0; 3367 out: 3368 mutex_unlock(&md->suspend_lock); 3369 3370 return r; 3371 } 3372 3373 /* 3374 * Internal suspend/resume works like userspace-driven suspend. It waits 3375 * until all bios finish and prevents issuing new bios to the target drivers. 3376 * It may be used only from the kernel. 3377 */ 3378 3379 static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags) 3380 { 3381 struct dm_table *map = NULL; 3382 3383 if (md->internal_suspend_count++) 3384 return; /* nested internal suspend */ 3385 3386 if (dm_suspended_md(md)) { 3387 set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 3388 return; /* nest suspend */ 3389 } 3390 3391 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 3392 3393 /* 3394 * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is 3395 * supported. Properly supporting a TASK_INTERRUPTIBLE internal suspend 3396 * would require changing .presuspend to return an error -- avoid this 3397 * until there is a need for more elaborate variants of internal suspend. 3398 */ 3399 (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE); 3400 3401 set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 3402 3403 dm_table_postsuspend_targets(map); 3404 } 3405 3406 static void __dm_internal_resume(struct mapped_device *md) 3407 { 3408 BUG_ON(!md->internal_suspend_count); 3409 3410 if (--md->internal_suspend_count) 3411 return; /* resume from nested internal suspend */ 3412 3413 if (dm_suspended_md(md)) 3414 goto done; /* resume from nested suspend */ 3415 3416 /* 3417 * NOTE: existing callers don't need to call dm_table_resume_targets 3418 * (which may fail -- so best to avoid it for now by passing NULL map) 3419 */ 3420 (void) __dm_resume(md, NULL); 3421 3422 done: 3423 clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 3424 smp_mb__after_atomic(); 3425 wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY); 3426 } 3427 3428 void dm_internal_suspend_noflush(struct mapped_device *md) 3429 { 3430 mutex_lock(&md->suspend_lock); 3431 __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG); 3432 mutex_unlock(&md->suspend_lock); 3433 } 3434 EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush); 3435 3436 void dm_internal_resume(struct mapped_device *md) 3437 { 3438 mutex_lock(&md->suspend_lock); 3439 __dm_internal_resume(md); 3440 mutex_unlock(&md->suspend_lock); 3441 } 3442 EXPORT_SYMBOL_GPL(dm_internal_resume); 3443 3444 /* 3445 * Fast variants of internal suspend/resume hold md->suspend_lock, 3446 * which prevents interaction with userspace-driven suspend. 3447 */ 3448 3449 void dm_internal_suspend_fast(struct mapped_device *md) 3450 { 3451 mutex_lock(&md->suspend_lock); 3452 if (dm_suspended_md(md) || dm_suspended_internally_md(md)) 3453 return; 3454 3455 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 3456 synchronize_srcu(&md->io_barrier); 3457 flush_workqueue(md->wq); 3458 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE); 3459 } 3460 EXPORT_SYMBOL_GPL(dm_internal_suspend_fast); 3461 3462 void dm_internal_resume_fast(struct mapped_device *md) 3463 { 3464 if (dm_suspended_md(md) || dm_suspended_internally_md(md)) 3465 goto done; 3466 3467 dm_queue_flush(md); 3468 3469 done: 3470 mutex_unlock(&md->suspend_lock); 3471 } 3472 EXPORT_SYMBOL_GPL(dm_internal_resume_fast); 3473 3474 /*----------------------------------------------------------------- 3475 * Event notification. 3476 *---------------------------------------------------------------*/ 3477 int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action, 3478 unsigned cookie) 3479 { 3480 char udev_cookie[DM_COOKIE_LENGTH]; 3481 char *envp[] = { udev_cookie, NULL }; 3482 3483 if (!cookie) 3484 return kobject_uevent(&disk_to_dev(md->disk)->kobj, action); 3485 else { 3486 snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u", 3487 DM_COOKIE_ENV_VAR_NAME, cookie); 3488 return kobject_uevent_env(&disk_to_dev(md->disk)->kobj, 3489 action, envp); 3490 } 3491 } 3492 3493 uint32_t dm_next_uevent_seq(struct mapped_device *md) 3494 { 3495 return atomic_add_return(1, &md->uevent_seq); 3496 } 3497 3498 uint32_t dm_get_event_nr(struct mapped_device *md) 3499 { 3500 return atomic_read(&md->event_nr); 3501 } 3502 3503 int dm_wait_event(struct mapped_device *md, int event_nr) 3504 { 3505 return wait_event_interruptible(md->eventq, 3506 (event_nr != atomic_read(&md->event_nr))); 3507 } 3508 3509 void dm_uevent_add(struct mapped_device *md, struct list_head *elist) 3510 { 3511 unsigned long flags; 3512 3513 spin_lock_irqsave(&md->uevent_lock, flags); 3514 list_add(elist, &md->uevent_list); 3515 spin_unlock_irqrestore(&md->uevent_lock, flags); 3516 } 3517 3518 /* 3519 * The gendisk is only valid as long as you have a reference 3520 * count on 'md'. 3521 */ 3522 struct gendisk *dm_disk(struct mapped_device *md) 3523 { 3524 return md->disk; 3525 } 3526 EXPORT_SYMBOL_GPL(dm_disk); 3527 3528 struct kobject *dm_kobject(struct mapped_device *md) 3529 { 3530 return &md->kobj_holder.kobj; 3531 } 3532 3533 struct mapped_device *dm_get_from_kobject(struct kobject *kobj) 3534 { 3535 struct mapped_device *md; 3536 3537 md = container_of(kobj, struct mapped_device, kobj_holder.kobj); 3538 3539 if (test_bit(DMF_FREEING, &md->flags) || 3540 dm_deleting_md(md)) 3541 return NULL; 3542 3543 dm_get(md); 3544 return md; 3545 } 3546 3547 int dm_suspended_md(struct mapped_device *md) 3548 { 3549 return test_bit(DMF_SUSPENDED, &md->flags); 3550 } 3551 3552 int dm_suspended_internally_md(struct mapped_device *md) 3553 { 3554 return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 3555 } 3556 3557 int dm_test_deferred_remove_flag(struct mapped_device *md) 3558 { 3559 return test_bit(DMF_DEFERRED_REMOVE, &md->flags); 3560 } 3561 3562 int dm_suspended(struct dm_target *ti) 3563 { 3564 return dm_suspended_md(dm_table_get_md(ti->table)); 3565 } 3566 EXPORT_SYMBOL_GPL(dm_suspended); 3567 3568 int dm_noflush_suspending(struct dm_target *ti) 3569 { 3570 return __noflush_suspending(dm_table_get_md(ti->table)); 3571 } 3572 EXPORT_SYMBOL_GPL(dm_noflush_suspending); 3573 3574 struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, unsigned type, 3575 unsigned integrity, unsigned per_bio_data_size) 3576 { 3577 struct dm_md_mempools *pools = kzalloc(sizeof(*pools), GFP_KERNEL); 3578 struct kmem_cache *cachep = NULL; 3579 unsigned int pool_size = 0; 3580 unsigned int front_pad; 3581 3582 if (!pools) 3583 return NULL; 3584 3585 type = filter_md_type(type, md); 3586 3587 switch (type) { 3588 case DM_TYPE_BIO_BASED: 3589 cachep = _io_cache; 3590 pool_size = dm_get_reserved_bio_based_ios(); 3591 front_pad = roundup(per_bio_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone); 3592 break; 3593 case DM_TYPE_REQUEST_BASED: 3594 cachep = _rq_tio_cache; 3595 pool_size = dm_get_reserved_rq_based_ios(); 3596 pools->rq_pool = mempool_create_slab_pool(pool_size, _rq_cache); 3597 if (!pools->rq_pool) 3598 goto out; 3599 /* fall through to setup remaining rq-based pools */ 3600 case DM_TYPE_MQ_REQUEST_BASED: 3601 if (!pool_size) 3602 pool_size = dm_get_reserved_rq_based_ios(); 3603 front_pad = offsetof(struct dm_rq_clone_bio_info, clone); 3604 /* per_bio_data_size is not used. See __bind_mempools(). */ 3605 WARN_ON(per_bio_data_size != 0); 3606 break; 3607 default: 3608 BUG(); 3609 } 3610 3611 if (cachep) { 3612 pools->io_pool = mempool_create_slab_pool(pool_size, cachep); 3613 if (!pools->io_pool) 3614 goto out; 3615 } 3616 3617 pools->bs = bioset_create_nobvec(pool_size, front_pad); 3618 if (!pools->bs) 3619 goto out; 3620 3621 if (integrity && bioset_integrity_create(pools->bs, pool_size)) 3622 goto out; 3623 3624 return pools; 3625 3626 out: 3627 dm_free_md_mempools(pools); 3628 3629 return NULL; 3630 } 3631 3632 void dm_free_md_mempools(struct dm_md_mempools *pools) 3633 { 3634 if (!pools) 3635 return; 3636 3637 if (pools->io_pool) 3638 mempool_destroy(pools->io_pool); 3639 3640 if (pools->rq_pool) 3641 mempool_destroy(pools->rq_pool); 3642 3643 if (pools->bs) 3644 bioset_free(pools->bs); 3645 3646 kfree(pools); 3647 } 3648 3649 static const struct block_device_operations dm_blk_dops = { 3650 .open = dm_blk_open, 3651 .release = dm_blk_close, 3652 .ioctl = dm_blk_ioctl, 3653 .getgeo = dm_blk_getgeo, 3654 .owner = THIS_MODULE 3655 }; 3656 3657 /* 3658 * module hooks 3659 */ 3660 module_init(dm_init); 3661 module_exit(dm_exit); 3662 3663 module_param(major, uint, 0); 3664 MODULE_PARM_DESC(major, "The major number of the device mapper"); 3665 3666 module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR); 3667 MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools"); 3668 3669 module_param(reserved_rq_based_ios, uint, S_IRUGO | S_IWUSR); 3670 MODULE_PARM_DESC(reserved_rq_based_ios, "Reserved IOs in request-based mempools"); 3671 3672 module_param(use_blk_mq, bool, S_IRUGO | S_IWUSR); 3673 MODULE_PARM_DESC(use_blk_mq, "Use block multiqueue for request-based DM devices"); 3674 3675 MODULE_DESCRIPTION(DM_NAME " driver"); 3676 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>"); 3677 MODULE_LICENSE("GPL"); 3678