1 /* 2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited. 3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. 4 * 5 * This file is released under the GPL. 6 */ 7 8 #include "dm-core.h" 9 #include "dm-rq.h" 10 #include "dm-uevent.h" 11 12 #include <linux/init.h> 13 #include <linux/module.h> 14 #include <linux/mutex.h> 15 #include <linux/sched/signal.h> 16 #include <linux/blkpg.h> 17 #include <linux/bio.h> 18 #include <linux/mempool.h> 19 #include <linux/dax.h> 20 #include <linux/slab.h> 21 #include <linux/idr.h> 22 #include <linux/uio.h> 23 #include <linux/hdreg.h> 24 #include <linux/delay.h> 25 #include <linux/wait.h> 26 #include <linux/pr.h> 27 #include <linux/refcount.h> 28 29 #define DM_MSG_PREFIX "core" 30 31 /* 32 * Cookies are numeric values sent with CHANGE and REMOVE 33 * uevents while resuming, removing or renaming the device. 34 */ 35 #define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE" 36 #define DM_COOKIE_LENGTH 24 37 38 static const char *_name = DM_NAME; 39 40 static unsigned int major = 0; 41 static unsigned int _major = 0; 42 43 static DEFINE_IDR(_minor_idr); 44 45 static DEFINE_SPINLOCK(_minor_lock); 46 47 static void do_deferred_remove(struct work_struct *w); 48 49 static DECLARE_WORK(deferred_remove_work, do_deferred_remove); 50 51 static struct workqueue_struct *deferred_remove_workqueue; 52 53 atomic_t dm_global_event_nr = ATOMIC_INIT(0); 54 DECLARE_WAIT_QUEUE_HEAD(dm_global_eventq); 55 56 void dm_issue_global_event(void) 57 { 58 atomic_inc(&dm_global_event_nr); 59 wake_up(&dm_global_eventq); 60 } 61 62 /* 63 * One of these is allocated (on-stack) per original bio. 64 */ 65 struct clone_info { 66 struct dm_table *map; 67 struct bio *bio; 68 struct dm_io *io; 69 sector_t sector; 70 unsigned sector_count; 71 }; 72 73 /* 74 * One of these is allocated per clone bio. 75 */ 76 #define DM_TIO_MAGIC 7282014 77 struct dm_target_io { 78 unsigned magic; 79 struct dm_io *io; 80 struct dm_target *ti; 81 unsigned target_bio_nr; 82 unsigned *len_ptr; 83 bool inside_dm_io; 84 struct bio clone; 85 }; 86 87 /* 88 * One of these is allocated per original bio. 89 * It contains the first clone used for that original. 90 */ 91 #define DM_IO_MAGIC 5191977 92 struct dm_io { 93 unsigned magic; 94 struct mapped_device *md; 95 blk_status_t status; 96 atomic_t io_count; 97 struct bio *orig_bio; 98 unsigned long start_time; 99 spinlock_t endio_lock; 100 struct dm_stats_aux stats_aux; 101 /* last member of dm_target_io is 'struct bio' */ 102 struct dm_target_io tio; 103 }; 104 105 void *dm_per_bio_data(struct bio *bio, size_t data_size) 106 { 107 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone); 108 if (!tio->inside_dm_io) 109 return (char *)bio - offsetof(struct dm_target_io, clone) - data_size; 110 return (char *)bio - offsetof(struct dm_target_io, clone) - offsetof(struct dm_io, tio) - data_size; 111 } 112 EXPORT_SYMBOL_GPL(dm_per_bio_data); 113 114 struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size) 115 { 116 struct dm_io *io = (struct dm_io *)((char *)data + data_size); 117 if (io->magic == DM_IO_MAGIC) 118 return (struct bio *)((char *)io + offsetof(struct dm_io, tio) + offsetof(struct dm_target_io, clone)); 119 BUG_ON(io->magic != DM_TIO_MAGIC); 120 return (struct bio *)((char *)io + offsetof(struct dm_target_io, clone)); 121 } 122 EXPORT_SYMBOL_GPL(dm_bio_from_per_bio_data); 123 124 unsigned dm_bio_get_target_bio_nr(const struct bio *bio) 125 { 126 return container_of(bio, struct dm_target_io, clone)->target_bio_nr; 127 } 128 EXPORT_SYMBOL_GPL(dm_bio_get_target_bio_nr); 129 130 #define MINOR_ALLOCED ((void *)-1) 131 132 /* 133 * Bits for the md->flags field. 134 */ 135 #define DMF_BLOCK_IO_FOR_SUSPEND 0 136 #define DMF_SUSPENDED 1 137 #define DMF_FROZEN 2 138 #define DMF_FREEING 3 139 #define DMF_DELETING 4 140 #define DMF_NOFLUSH_SUSPENDING 5 141 #define DMF_DEFERRED_REMOVE 6 142 #define DMF_SUSPENDED_INTERNALLY 7 143 144 #define DM_NUMA_NODE NUMA_NO_NODE 145 static int dm_numa_node = DM_NUMA_NODE; 146 147 /* 148 * For mempools pre-allocation at the table loading time. 149 */ 150 struct dm_md_mempools { 151 struct bio_set bs; 152 struct bio_set io_bs; 153 }; 154 155 struct table_device { 156 struct list_head list; 157 refcount_t count; 158 struct dm_dev dm_dev; 159 }; 160 161 static struct kmem_cache *_rq_tio_cache; 162 static struct kmem_cache *_rq_cache; 163 164 /* 165 * Bio-based DM's mempools' reserved IOs set by the user. 166 */ 167 #define RESERVED_BIO_BASED_IOS 16 168 static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS; 169 170 static int __dm_get_module_param_int(int *module_param, int min, int max) 171 { 172 int param = READ_ONCE(*module_param); 173 int modified_param = 0; 174 bool modified = true; 175 176 if (param < min) 177 modified_param = min; 178 else if (param > max) 179 modified_param = max; 180 else 181 modified = false; 182 183 if (modified) { 184 (void)cmpxchg(module_param, param, modified_param); 185 param = modified_param; 186 } 187 188 return param; 189 } 190 191 unsigned __dm_get_module_param(unsigned *module_param, 192 unsigned def, unsigned max) 193 { 194 unsigned param = READ_ONCE(*module_param); 195 unsigned modified_param = 0; 196 197 if (!param) 198 modified_param = def; 199 else if (param > max) 200 modified_param = max; 201 202 if (modified_param) { 203 (void)cmpxchg(module_param, param, modified_param); 204 param = modified_param; 205 } 206 207 return param; 208 } 209 210 unsigned dm_get_reserved_bio_based_ios(void) 211 { 212 return __dm_get_module_param(&reserved_bio_based_ios, 213 RESERVED_BIO_BASED_IOS, DM_RESERVED_MAX_IOS); 214 } 215 EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios); 216 217 static unsigned dm_get_numa_node(void) 218 { 219 return __dm_get_module_param_int(&dm_numa_node, 220 DM_NUMA_NODE, num_online_nodes() - 1); 221 } 222 223 static int __init local_init(void) 224 { 225 int r = -ENOMEM; 226 227 _rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0); 228 if (!_rq_tio_cache) 229 return r; 230 231 _rq_cache = kmem_cache_create("dm_old_clone_request", sizeof(struct request), 232 __alignof__(struct request), 0, NULL); 233 if (!_rq_cache) 234 goto out_free_rq_tio_cache; 235 236 r = dm_uevent_init(); 237 if (r) 238 goto out_free_rq_cache; 239 240 deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1); 241 if (!deferred_remove_workqueue) { 242 r = -ENOMEM; 243 goto out_uevent_exit; 244 } 245 246 _major = major; 247 r = register_blkdev(_major, _name); 248 if (r < 0) 249 goto out_free_workqueue; 250 251 if (!_major) 252 _major = r; 253 254 return 0; 255 256 out_free_workqueue: 257 destroy_workqueue(deferred_remove_workqueue); 258 out_uevent_exit: 259 dm_uevent_exit(); 260 out_free_rq_cache: 261 kmem_cache_destroy(_rq_cache); 262 out_free_rq_tio_cache: 263 kmem_cache_destroy(_rq_tio_cache); 264 265 return r; 266 } 267 268 static void local_exit(void) 269 { 270 flush_scheduled_work(); 271 destroy_workqueue(deferred_remove_workqueue); 272 273 kmem_cache_destroy(_rq_cache); 274 kmem_cache_destroy(_rq_tio_cache); 275 unregister_blkdev(_major, _name); 276 dm_uevent_exit(); 277 278 _major = 0; 279 280 DMINFO("cleaned up"); 281 } 282 283 static int (*_inits[])(void) __initdata = { 284 local_init, 285 dm_target_init, 286 dm_linear_init, 287 dm_stripe_init, 288 dm_io_init, 289 dm_kcopyd_init, 290 dm_interface_init, 291 dm_statistics_init, 292 }; 293 294 static void (*_exits[])(void) = { 295 local_exit, 296 dm_target_exit, 297 dm_linear_exit, 298 dm_stripe_exit, 299 dm_io_exit, 300 dm_kcopyd_exit, 301 dm_interface_exit, 302 dm_statistics_exit, 303 }; 304 305 static int __init dm_init(void) 306 { 307 const int count = ARRAY_SIZE(_inits); 308 309 int r, i; 310 311 for (i = 0; i < count; i++) { 312 r = _inits[i](); 313 if (r) 314 goto bad; 315 } 316 317 return 0; 318 319 bad: 320 while (i--) 321 _exits[i](); 322 323 return r; 324 } 325 326 static void __exit dm_exit(void) 327 { 328 int i = ARRAY_SIZE(_exits); 329 330 while (i--) 331 _exits[i](); 332 333 /* 334 * Should be empty by this point. 335 */ 336 idr_destroy(&_minor_idr); 337 } 338 339 /* 340 * Block device functions 341 */ 342 int dm_deleting_md(struct mapped_device *md) 343 { 344 return test_bit(DMF_DELETING, &md->flags); 345 } 346 347 static int dm_blk_open(struct block_device *bdev, fmode_t mode) 348 { 349 struct mapped_device *md; 350 351 spin_lock(&_minor_lock); 352 353 md = bdev->bd_disk->private_data; 354 if (!md) 355 goto out; 356 357 if (test_bit(DMF_FREEING, &md->flags) || 358 dm_deleting_md(md)) { 359 md = NULL; 360 goto out; 361 } 362 363 dm_get(md); 364 atomic_inc(&md->open_count); 365 out: 366 spin_unlock(&_minor_lock); 367 368 return md ? 0 : -ENXIO; 369 } 370 371 static void dm_blk_close(struct gendisk *disk, fmode_t mode) 372 { 373 struct mapped_device *md; 374 375 spin_lock(&_minor_lock); 376 377 md = disk->private_data; 378 if (WARN_ON(!md)) 379 goto out; 380 381 if (atomic_dec_and_test(&md->open_count) && 382 (test_bit(DMF_DEFERRED_REMOVE, &md->flags))) 383 queue_work(deferred_remove_workqueue, &deferred_remove_work); 384 385 dm_put(md); 386 out: 387 spin_unlock(&_minor_lock); 388 } 389 390 int dm_open_count(struct mapped_device *md) 391 { 392 return atomic_read(&md->open_count); 393 } 394 395 /* 396 * Guarantees nothing is using the device before it's deleted. 397 */ 398 int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred) 399 { 400 int r = 0; 401 402 spin_lock(&_minor_lock); 403 404 if (dm_open_count(md)) { 405 r = -EBUSY; 406 if (mark_deferred) 407 set_bit(DMF_DEFERRED_REMOVE, &md->flags); 408 } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags)) 409 r = -EEXIST; 410 else 411 set_bit(DMF_DELETING, &md->flags); 412 413 spin_unlock(&_minor_lock); 414 415 return r; 416 } 417 418 int dm_cancel_deferred_remove(struct mapped_device *md) 419 { 420 int r = 0; 421 422 spin_lock(&_minor_lock); 423 424 if (test_bit(DMF_DELETING, &md->flags)) 425 r = -EBUSY; 426 else 427 clear_bit(DMF_DEFERRED_REMOVE, &md->flags); 428 429 spin_unlock(&_minor_lock); 430 431 return r; 432 } 433 434 static void do_deferred_remove(struct work_struct *w) 435 { 436 dm_deferred_remove(); 437 } 438 439 sector_t dm_get_size(struct mapped_device *md) 440 { 441 return get_capacity(md->disk); 442 } 443 444 struct request_queue *dm_get_md_queue(struct mapped_device *md) 445 { 446 return md->queue; 447 } 448 449 struct dm_stats *dm_get_stats(struct mapped_device *md) 450 { 451 return &md->stats; 452 } 453 454 static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo) 455 { 456 struct mapped_device *md = bdev->bd_disk->private_data; 457 458 return dm_get_geometry(md, geo); 459 } 460 461 static int dm_blk_report_zones(struct gendisk *disk, sector_t sector, 462 struct blk_zone *zones, unsigned int *nr_zones, 463 gfp_t gfp_mask) 464 { 465 #ifdef CONFIG_BLK_DEV_ZONED 466 struct mapped_device *md = disk->private_data; 467 struct dm_target *tgt; 468 struct dm_table *map; 469 int srcu_idx, ret; 470 471 if (dm_suspended_md(md)) 472 return -EAGAIN; 473 474 map = dm_get_live_table(md, &srcu_idx); 475 if (!map) 476 return -EIO; 477 478 tgt = dm_table_find_target(map, sector); 479 if (!dm_target_is_valid(tgt)) { 480 ret = -EIO; 481 goto out; 482 } 483 484 /* 485 * If we are executing this, we already know that the block device 486 * is a zoned device and so each target should have support for that 487 * type of drive. A missing report_zones method means that the target 488 * driver has a problem. 489 */ 490 if (WARN_ON(!tgt->type->report_zones)) { 491 ret = -EIO; 492 goto out; 493 } 494 495 /* 496 * blkdev_report_zones() will loop and call this again to cover all the 497 * zones of the target, eventually moving on to the next target. 498 * So there is no need to loop here trying to fill the entire array 499 * of zones. 500 */ 501 ret = tgt->type->report_zones(tgt, sector, zones, 502 nr_zones, gfp_mask); 503 504 out: 505 dm_put_live_table(md, srcu_idx); 506 return ret; 507 #else 508 return -ENOTSUPP; 509 #endif 510 } 511 512 static int dm_prepare_ioctl(struct mapped_device *md, int *srcu_idx, 513 struct block_device **bdev) 514 __acquires(md->io_barrier) 515 { 516 struct dm_target *tgt; 517 struct dm_table *map; 518 int r; 519 520 retry: 521 r = -ENOTTY; 522 map = dm_get_live_table(md, srcu_idx); 523 if (!map || !dm_table_get_size(map)) 524 return r; 525 526 /* We only support devices that have a single target */ 527 if (dm_table_get_num_targets(map) != 1) 528 return r; 529 530 tgt = dm_table_get_target(map, 0); 531 if (!tgt->type->prepare_ioctl) 532 return r; 533 534 if (dm_suspended_md(md)) 535 return -EAGAIN; 536 537 r = tgt->type->prepare_ioctl(tgt, bdev); 538 if (r == -ENOTCONN && !fatal_signal_pending(current)) { 539 dm_put_live_table(md, *srcu_idx); 540 msleep(10); 541 goto retry; 542 } 543 544 return r; 545 } 546 547 static void dm_unprepare_ioctl(struct mapped_device *md, int srcu_idx) 548 __releases(md->io_barrier) 549 { 550 dm_put_live_table(md, srcu_idx); 551 } 552 553 static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode, 554 unsigned int cmd, unsigned long arg) 555 { 556 struct mapped_device *md = bdev->bd_disk->private_data; 557 int r, srcu_idx; 558 559 r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 560 if (r < 0) 561 goto out; 562 563 if (r > 0) { 564 /* 565 * Target determined this ioctl is being issued against a 566 * subset of the parent bdev; require extra privileges. 567 */ 568 if (!capable(CAP_SYS_RAWIO)) { 569 DMWARN_LIMIT( 570 "%s: sending ioctl %x to DM device without required privilege.", 571 current->comm, cmd); 572 r = -ENOIOCTLCMD; 573 goto out; 574 } 575 } 576 577 r = __blkdev_driver_ioctl(bdev, mode, cmd, arg); 578 out: 579 dm_unprepare_ioctl(md, srcu_idx); 580 return r; 581 } 582 583 static void start_io_acct(struct dm_io *io); 584 585 static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio) 586 { 587 struct dm_io *io; 588 struct dm_target_io *tio; 589 struct bio *clone; 590 591 clone = bio_alloc_bioset(GFP_NOIO, 0, &md->io_bs); 592 if (!clone) 593 return NULL; 594 595 tio = container_of(clone, struct dm_target_io, clone); 596 tio->inside_dm_io = true; 597 tio->io = NULL; 598 599 io = container_of(tio, struct dm_io, tio); 600 io->magic = DM_IO_MAGIC; 601 io->status = 0; 602 atomic_set(&io->io_count, 1); 603 io->orig_bio = bio; 604 io->md = md; 605 spin_lock_init(&io->endio_lock); 606 607 start_io_acct(io); 608 609 return io; 610 } 611 612 static void free_io(struct mapped_device *md, struct dm_io *io) 613 { 614 bio_put(&io->tio.clone); 615 } 616 617 static struct dm_target_io *alloc_tio(struct clone_info *ci, struct dm_target *ti, 618 unsigned target_bio_nr, gfp_t gfp_mask) 619 { 620 struct dm_target_io *tio; 621 622 if (!ci->io->tio.io) { 623 /* the dm_target_io embedded in ci->io is available */ 624 tio = &ci->io->tio; 625 } else { 626 struct bio *clone = bio_alloc_bioset(gfp_mask, 0, &ci->io->md->bs); 627 if (!clone) 628 return NULL; 629 630 tio = container_of(clone, struct dm_target_io, clone); 631 tio->inside_dm_io = false; 632 } 633 634 tio->magic = DM_TIO_MAGIC; 635 tio->io = ci->io; 636 tio->ti = ti; 637 tio->target_bio_nr = target_bio_nr; 638 639 return tio; 640 } 641 642 static void free_tio(struct dm_target_io *tio) 643 { 644 if (tio->inside_dm_io) 645 return; 646 bio_put(&tio->clone); 647 } 648 649 int md_in_flight(struct mapped_device *md) 650 { 651 return atomic_read(&md->pending[READ]) + 652 atomic_read(&md->pending[WRITE]); 653 } 654 655 static void start_io_acct(struct dm_io *io) 656 { 657 struct mapped_device *md = io->md; 658 struct bio *bio = io->orig_bio; 659 int rw = bio_data_dir(bio); 660 661 io->start_time = jiffies; 662 663 generic_start_io_acct(md->queue, bio_op(bio), bio_sectors(bio), 664 &dm_disk(md)->part0); 665 666 atomic_set(&dm_disk(md)->part0.in_flight[rw], 667 atomic_inc_return(&md->pending[rw])); 668 669 if (unlikely(dm_stats_used(&md->stats))) 670 dm_stats_account_io(&md->stats, bio_data_dir(bio), 671 bio->bi_iter.bi_sector, bio_sectors(bio), 672 false, 0, &io->stats_aux); 673 } 674 675 static void end_io_acct(struct dm_io *io) 676 { 677 struct mapped_device *md = io->md; 678 struct bio *bio = io->orig_bio; 679 unsigned long duration = jiffies - io->start_time; 680 int pending; 681 int rw = bio_data_dir(bio); 682 683 generic_end_io_acct(md->queue, bio_op(bio), &dm_disk(md)->part0, 684 io->start_time); 685 686 if (unlikely(dm_stats_used(&md->stats))) 687 dm_stats_account_io(&md->stats, bio_data_dir(bio), 688 bio->bi_iter.bi_sector, bio_sectors(bio), 689 true, duration, &io->stats_aux); 690 691 /* 692 * After this is decremented the bio must not be touched if it is 693 * a flush. 694 */ 695 pending = atomic_dec_return(&md->pending[rw]); 696 atomic_set(&dm_disk(md)->part0.in_flight[rw], pending); 697 pending += atomic_read(&md->pending[rw^0x1]); 698 699 /* nudge anyone waiting on suspend queue */ 700 if (!pending) 701 wake_up(&md->wait); 702 } 703 704 /* 705 * Add the bio to the list of deferred io. 706 */ 707 static void queue_io(struct mapped_device *md, struct bio *bio) 708 { 709 unsigned long flags; 710 711 spin_lock_irqsave(&md->deferred_lock, flags); 712 bio_list_add(&md->deferred, bio); 713 spin_unlock_irqrestore(&md->deferred_lock, flags); 714 queue_work(md->wq, &md->work); 715 } 716 717 /* 718 * Everyone (including functions in this file), should use this 719 * function to access the md->map field, and make sure they call 720 * dm_put_live_table() when finished. 721 */ 722 struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier) 723 { 724 *srcu_idx = srcu_read_lock(&md->io_barrier); 725 726 return srcu_dereference(md->map, &md->io_barrier); 727 } 728 729 void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier) 730 { 731 srcu_read_unlock(&md->io_barrier, srcu_idx); 732 } 733 734 void dm_sync_table(struct mapped_device *md) 735 { 736 synchronize_srcu(&md->io_barrier); 737 synchronize_rcu_expedited(); 738 } 739 740 /* 741 * A fast alternative to dm_get_live_table/dm_put_live_table. 742 * The caller must not block between these two functions. 743 */ 744 static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU) 745 { 746 rcu_read_lock(); 747 return rcu_dereference(md->map); 748 } 749 750 static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU) 751 { 752 rcu_read_unlock(); 753 } 754 755 static char *_dm_claim_ptr = "I belong to device-mapper"; 756 757 /* 758 * Open a table device so we can use it as a map destination. 759 */ 760 static int open_table_device(struct table_device *td, dev_t dev, 761 struct mapped_device *md) 762 { 763 struct block_device *bdev; 764 765 int r; 766 767 BUG_ON(td->dm_dev.bdev); 768 769 bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _dm_claim_ptr); 770 if (IS_ERR(bdev)) 771 return PTR_ERR(bdev); 772 773 r = bd_link_disk_holder(bdev, dm_disk(md)); 774 if (r) { 775 blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL); 776 return r; 777 } 778 779 td->dm_dev.bdev = bdev; 780 td->dm_dev.dax_dev = dax_get_by_host(bdev->bd_disk->disk_name); 781 return 0; 782 } 783 784 /* 785 * Close a table device that we've been using. 786 */ 787 static void close_table_device(struct table_device *td, struct mapped_device *md) 788 { 789 if (!td->dm_dev.bdev) 790 return; 791 792 bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md)); 793 blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL); 794 put_dax(td->dm_dev.dax_dev); 795 td->dm_dev.bdev = NULL; 796 td->dm_dev.dax_dev = NULL; 797 } 798 799 static struct table_device *find_table_device(struct list_head *l, dev_t dev, 800 fmode_t mode) { 801 struct table_device *td; 802 803 list_for_each_entry(td, l, list) 804 if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode) 805 return td; 806 807 return NULL; 808 } 809 810 int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode, 811 struct dm_dev **result) { 812 int r; 813 struct table_device *td; 814 815 mutex_lock(&md->table_devices_lock); 816 td = find_table_device(&md->table_devices, dev, mode); 817 if (!td) { 818 td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id); 819 if (!td) { 820 mutex_unlock(&md->table_devices_lock); 821 return -ENOMEM; 822 } 823 824 td->dm_dev.mode = mode; 825 td->dm_dev.bdev = NULL; 826 827 if ((r = open_table_device(td, dev, md))) { 828 mutex_unlock(&md->table_devices_lock); 829 kfree(td); 830 return r; 831 } 832 833 format_dev_t(td->dm_dev.name, dev); 834 835 refcount_set(&td->count, 1); 836 list_add(&td->list, &md->table_devices); 837 } else { 838 refcount_inc(&td->count); 839 } 840 mutex_unlock(&md->table_devices_lock); 841 842 *result = &td->dm_dev; 843 return 0; 844 } 845 EXPORT_SYMBOL_GPL(dm_get_table_device); 846 847 void dm_put_table_device(struct mapped_device *md, struct dm_dev *d) 848 { 849 struct table_device *td = container_of(d, struct table_device, dm_dev); 850 851 mutex_lock(&md->table_devices_lock); 852 if (refcount_dec_and_test(&td->count)) { 853 close_table_device(td, md); 854 list_del(&td->list); 855 kfree(td); 856 } 857 mutex_unlock(&md->table_devices_lock); 858 } 859 EXPORT_SYMBOL(dm_put_table_device); 860 861 static void free_table_devices(struct list_head *devices) 862 { 863 struct list_head *tmp, *next; 864 865 list_for_each_safe(tmp, next, devices) { 866 struct table_device *td = list_entry(tmp, struct table_device, list); 867 868 DMWARN("dm_destroy: %s still exists with %d references", 869 td->dm_dev.name, refcount_read(&td->count)); 870 kfree(td); 871 } 872 } 873 874 /* 875 * Get the geometry associated with a dm device 876 */ 877 int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo) 878 { 879 *geo = md->geometry; 880 881 return 0; 882 } 883 884 /* 885 * Set the geometry of a device. 886 */ 887 int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo) 888 { 889 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors; 890 891 if (geo->start > sz) { 892 DMWARN("Start sector is beyond the geometry limits."); 893 return -EINVAL; 894 } 895 896 md->geometry = *geo; 897 898 return 0; 899 } 900 901 static int __noflush_suspending(struct mapped_device *md) 902 { 903 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 904 } 905 906 /* 907 * Decrements the number of outstanding ios that a bio has been 908 * cloned into, completing the original io if necc. 909 */ 910 static void dec_pending(struct dm_io *io, blk_status_t error) 911 { 912 unsigned long flags; 913 blk_status_t io_error; 914 struct bio *bio; 915 struct mapped_device *md = io->md; 916 917 /* Push-back supersedes any I/O errors */ 918 if (unlikely(error)) { 919 spin_lock_irqsave(&io->endio_lock, flags); 920 if (!(io->status == BLK_STS_DM_REQUEUE && __noflush_suspending(md))) 921 io->status = error; 922 spin_unlock_irqrestore(&io->endio_lock, flags); 923 } 924 925 if (atomic_dec_and_test(&io->io_count)) { 926 if (io->status == BLK_STS_DM_REQUEUE) { 927 /* 928 * Target requested pushing back the I/O. 929 */ 930 spin_lock_irqsave(&md->deferred_lock, flags); 931 if (__noflush_suspending(md)) 932 /* NOTE early return due to BLK_STS_DM_REQUEUE below */ 933 bio_list_add_head(&md->deferred, io->orig_bio); 934 else 935 /* noflush suspend was interrupted. */ 936 io->status = BLK_STS_IOERR; 937 spin_unlock_irqrestore(&md->deferred_lock, flags); 938 } 939 940 io_error = io->status; 941 bio = io->orig_bio; 942 end_io_acct(io); 943 free_io(md, io); 944 945 if (io_error == BLK_STS_DM_REQUEUE) 946 return; 947 948 if ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size) { 949 /* 950 * Preflush done for flush with data, reissue 951 * without REQ_PREFLUSH. 952 */ 953 bio->bi_opf &= ~REQ_PREFLUSH; 954 queue_io(md, bio); 955 } else { 956 /* done with normal IO or empty flush */ 957 if (io_error) 958 bio->bi_status = io_error; 959 bio_endio(bio); 960 } 961 } 962 } 963 964 void disable_write_same(struct mapped_device *md) 965 { 966 struct queue_limits *limits = dm_get_queue_limits(md); 967 968 /* device doesn't really support WRITE SAME, disable it */ 969 limits->max_write_same_sectors = 0; 970 } 971 972 void disable_write_zeroes(struct mapped_device *md) 973 { 974 struct queue_limits *limits = dm_get_queue_limits(md); 975 976 /* device doesn't really support WRITE ZEROES, disable it */ 977 limits->max_write_zeroes_sectors = 0; 978 } 979 980 static void clone_endio(struct bio *bio) 981 { 982 blk_status_t error = bio->bi_status; 983 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone); 984 struct dm_io *io = tio->io; 985 struct mapped_device *md = tio->io->md; 986 dm_endio_fn endio = tio->ti->type->end_io; 987 988 if (unlikely(error == BLK_STS_TARGET) && md->type != DM_TYPE_NVME_BIO_BASED) { 989 if (bio_op(bio) == REQ_OP_WRITE_SAME && 990 !bio->bi_disk->queue->limits.max_write_same_sectors) 991 disable_write_same(md); 992 if (bio_op(bio) == REQ_OP_WRITE_ZEROES && 993 !bio->bi_disk->queue->limits.max_write_zeroes_sectors) 994 disable_write_zeroes(md); 995 } 996 997 if (endio) { 998 int r = endio(tio->ti, bio, &error); 999 switch (r) { 1000 case DM_ENDIO_REQUEUE: 1001 error = BLK_STS_DM_REQUEUE; 1002 /*FALLTHRU*/ 1003 case DM_ENDIO_DONE: 1004 break; 1005 case DM_ENDIO_INCOMPLETE: 1006 /* The target will handle the io */ 1007 return; 1008 default: 1009 DMWARN("unimplemented target endio return value: %d", r); 1010 BUG(); 1011 } 1012 } 1013 1014 free_tio(tio); 1015 dec_pending(io, error); 1016 } 1017 1018 /* 1019 * Return maximum size of I/O possible at the supplied sector up to the current 1020 * target boundary. 1021 */ 1022 static sector_t max_io_len_target_boundary(sector_t sector, struct dm_target *ti) 1023 { 1024 sector_t target_offset = dm_target_offset(ti, sector); 1025 1026 return ti->len - target_offset; 1027 } 1028 1029 static sector_t max_io_len(sector_t sector, struct dm_target *ti) 1030 { 1031 sector_t len = max_io_len_target_boundary(sector, ti); 1032 sector_t offset, max_len; 1033 1034 /* 1035 * Does the target need to split even further? 1036 */ 1037 if (ti->max_io_len) { 1038 offset = dm_target_offset(ti, sector); 1039 if (unlikely(ti->max_io_len & (ti->max_io_len - 1))) 1040 max_len = sector_div(offset, ti->max_io_len); 1041 else 1042 max_len = offset & (ti->max_io_len - 1); 1043 max_len = ti->max_io_len - max_len; 1044 1045 if (len > max_len) 1046 len = max_len; 1047 } 1048 1049 return len; 1050 } 1051 1052 int dm_set_target_max_io_len(struct dm_target *ti, sector_t len) 1053 { 1054 if (len > UINT_MAX) { 1055 DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)", 1056 (unsigned long long)len, UINT_MAX); 1057 ti->error = "Maximum size of target IO is too large"; 1058 return -EINVAL; 1059 } 1060 1061 /* 1062 * BIO based queue uses its own splitting. When multipage bvecs 1063 * is switched on, size of the incoming bio may be too big to 1064 * be handled in some targets, such as crypt. 1065 * 1066 * When these targets are ready for the big bio, we can remove 1067 * the limit. 1068 */ 1069 ti->max_io_len = min_t(uint32_t, len, BIO_MAX_PAGES * PAGE_SIZE); 1070 1071 return 0; 1072 } 1073 EXPORT_SYMBOL_GPL(dm_set_target_max_io_len); 1074 1075 static struct dm_target *dm_dax_get_live_target(struct mapped_device *md, 1076 sector_t sector, int *srcu_idx) 1077 __acquires(md->io_barrier) 1078 { 1079 struct dm_table *map; 1080 struct dm_target *ti; 1081 1082 map = dm_get_live_table(md, srcu_idx); 1083 if (!map) 1084 return NULL; 1085 1086 ti = dm_table_find_target(map, sector); 1087 if (!dm_target_is_valid(ti)) 1088 return NULL; 1089 1090 return ti; 1091 } 1092 1093 static long dm_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, 1094 long nr_pages, void **kaddr, pfn_t *pfn) 1095 { 1096 struct mapped_device *md = dax_get_private(dax_dev); 1097 sector_t sector = pgoff * PAGE_SECTORS; 1098 struct dm_target *ti; 1099 long len, ret = -EIO; 1100 int srcu_idx; 1101 1102 ti = dm_dax_get_live_target(md, sector, &srcu_idx); 1103 1104 if (!ti) 1105 goto out; 1106 if (!ti->type->direct_access) 1107 goto out; 1108 len = max_io_len(sector, ti) / PAGE_SECTORS; 1109 if (len < 1) 1110 goto out; 1111 nr_pages = min(len, nr_pages); 1112 ret = ti->type->direct_access(ti, pgoff, nr_pages, kaddr, pfn); 1113 1114 out: 1115 dm_put_live_table(md, srcu_idx); 1116 1117 return ret; 1118 } 1119 1120 static size_t dm_dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, 1121 void *addr, size_t bytes, struct iov_iter *i) 1122 { 1123 struct mapped_device *md = dax_get_private(dax_dev); 1124 sector_t sector = pgoff * PAGE_SECTORS; 1125 struct dm_target *ti; 1126 long ret = 0; 1127 int srcu_idx; 1128 1129 ti = dm_dax_get_live_target(md, sector, &srcu_idx); 1130 1131 if (!ti) 1132 goto out; 1133 if (!ti->type->dax_copy_from_iter) { 1134 ret = copy_from_iter(addr, bytes, i); 1135 goto out; 1136 } 1137 ret = ti->type->dax_copy_from_iter(ti, pgoff, addr, bytes, i); 1138 out: 1139 dm_put_live_table(md, srcu_idx); 1140 1141 return ret; 1142 } 1143 1144 static size_t dm_dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, 1145 void *addr, size_t bytes, struct iov_iter *i) 1146 { 1147 struct mapped_device *md = dax_get_private(dax_dev); 1148 sector_t sector = pgoff * PAGE_SECTORS; 1149 struct dm_target *ti; 1150 long ret = 0; 1151 int srcu_idx; 1152 1153 ti = dm_dax_get_live_target(md, sector, &srcu_idx); 1154 1155 if (!ti) 1156 goto out; 1157 if (!ti->type->dax_copy_to_iter) { 1158 ret = copy_to_iter(addr, bytes, i); 1159 goto out; 1160 } 1161 ret = ti->type->dax_copy_to_iter(ti, pgoff, addr, bytes, i); 1162 out: 1163 dm_put_live_table(md, srcu_idx); 1164 1165 return ret; 1166 } 1167 1168 /* 1169 * A target may call dm_accept_partial_bio only from the map routine. It is 1170 * allowed for all bio types except REQ_PREFLUSH and REQ_OP_ZONE_RESET. 1171 * 1172 * dm_accept_partial_bio informs the dm that the target only wants to process 1173 * additional n_sectors sectors of the bio and the rest of the data should be 1174 * sent in a next bio. 1175 * 1176 * A diagram that explains the arithmetics: 1177 * +--------------------+---------------+-------+ 1178 * | 1 | 2 | 3 | 1179 * +--------------------+---------------+-------+ 1180 * 1181 * <-------------- *tio->len_ptr ---------------> 1182 * <------- bi_size -------> 1183 * <-- n_sectors --> 1184 * 1185 * Region 1 was already iterated over with bio_advance or similar function. 1186 * (it may be empty if the target doesn't use bio_advance) 1187 * Region 2 is the remaining bio size that the target wants to process. 1188 * (it may be empty if region 1 is non-empty, although there is no reason 1189 * to make it empty) 1190 * The target requires that region 3 is to be sent in the next bio. 1191 * 1192 * If the target wants to receive multiple copies of the bio (via num_*bios, etc), 1193 * the partially processed part (the sum of regions 1+2) must be the same for all 1194 * copies of the bio. 1195 */ 1196 void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors) 1197 { 1198 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone); 1199 unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT; 1200 BUG_ON(bio->bi_opf & REQ_PREFLUSH); 1201 BUG_ON(bi_size > *tio->len_ptr); 1202 BUG_ON(n_sectors > bi_size); 1203 *tio->len_ptr -= bi_size - n_sectors; 1204 bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT; 1205 } 1206 EXPORT_SYMBOL_GPL(dm_accept_partial_bio); 1207 1208 /* 1209 * The zone descriptors obtained with a zone report indicate 1210 * zone positions within the underlying device of the target. The zone 1211 * descriptors must be remapped to match their position within the dm device. 1212 * The caller target should obtain the zones information using 1213 * blkdev_report_zones() to ensure that remapping for partition offset is 1214 * already handled. 1215 */ 1216 void dm_remap_zone_report(struct dm_target *ti, sector_t start, 1217 struct blk_zone *zones, unsigned int *nr_zones) 1218 { 1219 #ifdef CONFIG_BLK_DEV_ZONED 1220 struct blk_zone *zone; 1221 unsigned int nrz = *nr_zones; 1222 int i; 1223 1224 /* 1225 * Remap the start sector and write pointer position of the zones in 1226 * the array. Since we may have obtained from the target underlying 1227 * device more zones that the target size, also adjust the number 1228 * of zones. 1229 */ 1230 for (i = 0; i < nrz; i++) { 1231 zone = zones + i; 1232 if (zone->start >= start + ti->len) { 1233 memset(zone, 0, sizeof(struct blk_zone) * (nrz - i)); 1234 break; 1235 } 1236 1237 zone->start = zone->start + ti->begin - start; 1238 if (zone->type == BLK_ZONE_TYPE_CONVENTIONAL) 1239 continue; 1240 1241 if (zone->cond == BLK_ZONE_COND_FULL) 1242 zone->wp = zone->start + zone->len; 1243 else if (zone->cond == BLK_ZONE_COND_EMPTY) 1244 zone->wp = zone->start; 1245 else 1246 zone->wp = zone->wp + ti->begin - start; 1247 } 1248 1249 *nr_zones = i; 1250 #else /* !CONFIG_BLK_DEV_ZONED */ 1251 *nr_zones = 0; 1252 #endif 1253 } 1254 EXPORT_SYMBOL_GPL(dm_remap_zone_report); 1255 1256 static blk_qc_t __map_bio(struct dm_target_io *tio) 1257 { 1258 int r; 1259 sector_t sector; 1260 struct bio *clone = &tio->clone; 1261 struct dm_io *io = tio->io; 1262 struct mapped_device *md = io->md; 1263 struct dm_target *ti = tio->ti; 1264 blk_qc_t ret = BLK_QC_T_NONE; 1265 1266 clone->bi_end_io = clone_endio; 1267 1268 /* 1269 * Map the clone. If r == 0 we don't need to do 1270 * anything, the target has assumed ownership of 1271 * this io. 1272 */ 1273 atomic_inc(&io->io_count); 1274 sector = clone->bi_iter.bi_sector; 1275 1276 r = ti->type->map(ti, clone); 1277 switch (r) { 1278 case DM_MAPIO_SUBMITTED: 1279 break; 1280 case DM_MAPIO_REMAPPED: 1281 /* the bio has been remapped so dispatch it */ 1282 trace_block_bio_remap(clone->bi_disk->queue, clone, 1283 bio_dev(io->orig_bio), sector); 1284 if (md->type == DM_TYPE_NVME_BIO_BASED) 1285 ret = direct_make_request(clone); 1286 else 1287 ret = generic_make_request(clone); 1288 break; 1289 case DM_MAPIO_KILL: 1290 free_tio(tio); 1291 dec_pending(io, BLK_STS_IOERR); 1292 break; 1293 case DM_MAPIO_REQUEUE: 1294 free_tio(tio); 1295 dec_pending(io, BLK_STS_DM_REQUEUE); 1296 break; 1297 default: 1298 DMWARN("unimplemented target map return value: %d", r); 1299 BUG(); 1300 } 1301 1302 return ret; 1303 } 1304 1305 static void bio_setup_sector(struct bio *bio, sector_t sector, unsigned len) 1306 { 1307 bio->bi_iter.bi_sector = sector; 1308 bio->bi_iter.bi_size = to_bytes(len); 1309 } 1310 1311 /* 1312 * Creates a bio that consists of range of complete bvecs. 1313 */ 1314 static int clone_bio(struct dm_target_io *tio, struct bio *bio, 1315 sector_t sector, unsigned len) 1316 { 1317 struct bio *clone = &tio->clone; 1318 1319 __bio_clone_fast(clone, bio); 1320 1321 if (unlikely(bio_integrity(bio) != NULL)) { 1322 int r; 1323 1324 if (unlikely(!dm_target_has_integrity(tio->ti->type) && 1325 !dm_target_passes_integrity(tio->ti->type))) { 1326 DMWARN("%s: the target %s doesn't support integrity data.", 1327 dm_device_name(tio->io->md), 1328 tio->ti->type->name); 1329 return -EIO; 1330 } 1331 1332 r = bio_integrity_clone(clone, bio, GFP_NOIO); 1333 if (r < 0) 1334 return r; 1335 } 1336 1337 bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector)); 1338 clone->bi_iter.bi_size = to_bytes(len); 1339 1340 if (unlikely(bio_integrity(bio) != NULL)) 1341 bio_integrity_trim(clone); 1342 1343 return 0; 1344 } 1345 1346 static void alloc_multiple_bios(struct bio_list *blist, struct clone_info *ci, 1347 struct dm_target *ti, unsigned num_bios) 1348 { 1349 struct dm_target_io *tio; 1350 int try; 1351 1352 if (!num_bios) 1353 return; 1354 1355 if (num_bios == 1) { 1356 tio = alloc_tio(ci, ti, 0, GFP_NOIO); 1357 bio_list_add(blist, &tio->clone); 1358 return; 1359 } 1360 1361 for (try = 0; try < 2; try++) { 1362 int bio_nr; 1363 struct bio *bio; 1364 1365 if (try) 1366 mutex_lock(&ci->io->md->table_devices_lock); 1367 for (bio_nr = 0; bio_nr < num_bios; bio_nr++) { 1368 tio = alloc_tio(ci, ti, bio_nr, try ? GFP_NOIO : GFP_NOWAIT); 1369 if (!tio) 1370 break; 1371 1372 bio_list_add(blist, &tio->clone); 1373 } 1374 if (try) 1375 mutex_unlock(&ci->io->md->table_devices_lock); 1376 if (bio_nr == num_bios) 1377 return; 1378 1379 while ((bio = bio_list_pop(blist))) { 1380 tio = container_of(bio, struct dm_target_io, clone); 1381 free_tio(tio); 1382 } 1383 } 1384 } 1385 1386 static blk_qc_t __clone_and_map_simple_bio(struct clone_info *ci, 1387 struct dm_target_io *tio, unsigned *len) 1388 { 1389 struct bio *clone = &tio->clone; 1390 1391 tio->len_ptr = len; 1392 1393 __bio_clone_fast(clone, ci->bio); 1394 if (len) 1395 bio_setup_sector(clone, ci->sector, *len); 1396 1397 return __map_bio(tio); 1398 } 1399 1400 static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti, 1401 unsigned num_bios, unsigned *len) 1402 { 1403 struct bio_list blist = BIO_EMPTY_LIST; 1404 struct bio *bio; 1405 struct dm_target_io *tio; 1406 1407 alloc_multiple_bios(&blist, ci, ti, num_bios); 1408 1409 while ((bio = bio_list_pop(&blist))) { 1410 tio = container_of(bio, struct dm_target_io, clone); 1411 (void) __clone_and_map_simple_bio(ci, tio, len); 1412 } 1413 } 1414 1415 static int __send_empty_flush(struct clone_info *ci) 1416 { 1417 unsigned target_nr = 0; 1418 struct dm_target *ti; 1419 1420 BUG_ON(bio_has_data(ci->bio)); 1421 while ((ti = dm_table_get_target(ci->map, target_nr++))) 1422 __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL); 1423 1424 return 0; 1425 } 1426 1427 static int __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti, 1428 sector_t sector, unsigned *len) 1429 { 1430 struct bio *bio = ci->bio; 1431 struct dm_target_io *tio; 1432 int r; 1433 1434 tio = alloc_tio(ci, ti, 0, GFP_NOIO); 1435 tio->len_ptr = len; 1436 r = clone_bio(tio, bio, sector, *len); 1437 if (r < 0) { 1438 free_tio(tio); 1439 return r; 1440 } 1441 (void) __map_bio(tio); 1442 1443 return 0; 1444 } 1445 1446 typedef unsigned (*get_num_bios_fn)(struct dm_target *ti); 1447 1448 static unsigned get_num_discard_bios(struct dm_target *ti) 1449 { 1450 return ti->num_discard_bios; 1451 } 1452 1453 static unsigned get_num_secure_erase_bios(struct dm_target *ti) 1454 { 1455 return ti->num_secure_erase_bios; 1456 } 1457 1458 static unsigned get_num_write_same_bios(struct dm_target *ti) 1459 { 1460 return ti->num_write_same_bios; 1461 } 1462 1463 static unsigned get_num_write_zeroes_bios(struct dm_target *ti) 1464 { 1465 return ti->num_write_zeroes_bios; 1466 } 1467 1468 typedef bool (*is_split_required_fn)(struct dm_target *ti); 1469 1470 static bool is_split_required_for_discard(struct dm_target *ti) 1471 { 1472 return ti->split_discard_bios; 1473 } 1474 1475 static int __send_changing_extent_only(struct clone_info *ci, struct dm_target *ti, 1476 get_num_bios_fn get_num_bios, 1477 is_split_required_fn is_split_required) 1478 { 1479 unsigned len; 1480 unsigned num_bios; 1481 1482 /* 1483 * Even though the device advertised support for this type of 1484 * request, that does not mean every target supports it, and 1485 * reconfiguration might also have changed that since the 1486 * check was performed. 1487 */ 1488 num_bios = get_num_bios ? get_num_bios(ti) : 0; 1489 if (!num_bios) 1490 return -EOPNOTSUPP; 1491 1492 if (is_split_required && !is_split_required(ti)) 1493 len = min((sector_t)ci->sector_count, max_io_len_target_boundary(ci->sector, ti)); 1494 else 1495 len = min((sector_t)ci->sector_count, max_io_len(ci->sector, ti)); 1496 1497 __send_duplicate_bios(ci, ti, num_bios, &len); 1498 1499 ci->sector += len; 1500 ci->sector_count -= len; 1501 1502 return 0; 1503 } 1504 1505 static int __send_discard(struct clone_info *ci, struct dm_target *ti) 1506 { 1507 return __send_changing_extent_only(ci, ti, get_num_discard_bios, 1508 is_split_required_for_discard); 1509 } 1510 1511 static int __send_secure_erase(struct clone_info *ci, struct dm_target *ti) 1512 { 1513 return __send_changing_extent_only(ci, ti, get_num_secure_erase_bios, NULL); 1514 } 1515 1516 static int __send_write_same(struct clone_info *ci, struct dm_target *ti) 1517 { 1518 return __send_changing_extent_only(ci, ti, get_num_write_same_bios, NULL); 1519 } 1520 1521 static int __send_write_zeroes(struct clone_info *ci, struct dm_target *ti) 1522 { 1523 return __send_changing_extent_only(ci, ti, get_num_write_zeroes_bios, NULL); 1524 } 1525 1526 static bool __process_abnormal_io(struct clone_info *ci, struct dm_target *ti, 1527 int *result) 1528 { 1529 struct bio *bio = ci->bio; 1530 1531 if (bio_op(bio) == REQ_OP_DISCARD) 1532 *result = __send_discard(ci, ti); 1533 else if (bio_op(bio) == REQ_OP_SECURE_ERASE) 1534 *result = __send_secure_erase(ci, ti); 1535 else if (bio_op(bio) == REQ_OP_WRITE_SAME) 1536 *result = __send_write_same(ci, ti); 1537 else if (bio_op(bio) == REQ_OP_WRITE_ZEROES) 1538 *result = __send_write_zeroes(ci, ti); 1539 else 1540 return false; 1541 1542 return true; 1543 } 1544 1545 /* 1546 * Select the correct strategy for processing a non-flush bio. 1547 */ 1548 static int __split_and_process_non_flush(struct clone_info *ci) 1549 { 1550 struct dm_target *ti; 1551 unsigned len; 1552 int r; 1553 1554 ti = dm_table_find_target(ci->map, ci->sector); 1555 if (!dm_target_is_valid(ti)) 1556 return -EIO; 1557 1558 if (unlikely(__process_abnormal_io(ci, ti, &r))) 1559 return r; 1560 1561 len = min_t(sector_t, max_io_len(ci->sector, ti), ci->sector_count); 1562 1563 r = __clone_and_map_data_bio(ci, ti, ci->sector, &len); 1564 if (r < 0) 1565 return r; 1566 1567 ci->sector += len; 1568 ci->sector_count -= len; 1569 1570 return 0; 1571 } 1572 1573 static void init_clone_info(struct clone_info *ci, struct mapped_device *md, 1574 struct dm_table *map, struct bio *bio) 1575 { 1576 ci->map = map; 1577 ci->io = alloc_io(md, bio); 1578 ci->sector = bio->bi_iter.bi_sector; 1579 } 1580 1581 /* 1582 * Entry point to split a bio into clones and submit them to the targets. 1583 */ 1584 static blk_qc_t __split_and_process_bio(struct mapped_device *md, 1585 struct dm_table *map, struct bio *bio) 1586 { 1587 struct clone_info ci; 1588 blk_qc_t ret = BLK_QC_T_NONE; 1589 int error = 0; 1590 1591 if (unlikely(!map)) { 1592 bio_io_error(bio); 1593 return ret; 1594 } 1595 1596 blk_queue_split(md->queue, &bio); 1597 1598 init_clone_info(&ci, md, map, bio); 1599 1600 if (bio->bi_opf & REQ_PREFLUSH) { 1601 ci.bio = &ci.io->md->flush_bio; 1602 ci.sector_count = 0; 1603 error = __send_empty_flush(&ci); 1604 /* dec_pending submits any data associated with flush */ 1605 } else if (bio_op(bio) == REQ_OP_ZONE_RESET) { 1606 ci.bio = bio; 1607 ci.sector_count = 0; 1608 error = __split_and_process_non_flush(&ci); 1609 } else { 1610 ci.bio = bio; 1611 ci.sector_count = bio_sectors(bio); 1612 while (ci.sector_count && !error) { 1613 error = __split_and_process_non_flush(&ci); 1614 if (current->bio_list && ci.sector_count && !error) { 1615 /* 1616 * Remainder must be passed to generic_make_request() 1617 * so that it gets handled *after* bios already submitted 1618 * have been completely processed. 1619 * We take a clone of the original to store in 1620 * ci.io->orig_bio to be used by end_io_acct() and 1621 * for dec_pending to use for completion handling. 1622 */ 1623 struct bio *b = bio_split(bio, bio_sectors(bio) - ci.sector_count, 1624 GFP_NOIO, &md->queue->bio_split); 1625 ci.io->orig_bio = b; 1626 bio_chain(b, bio); 1627 ret = generic_make_request(bio); 1628 break; 1629 } 1630 } 1631 } 1632 1633 /* drop the extra reference count */ 1634 dec_pending(ci.io, errno_to_blk_status(error)); 1635 return ret; 1636 } 1637 1638 /* 1639 * Optimized variant of __split_and_process_bio that leverages the 1640 * fact that targets that use it do _not_ have a need to split bios. 1641 */ 1642 static blk_qc_t __process_bio(struct mapped_device *md, 1643 struct dm_table *map, struct bio *bio) 1644 { 1645 struct clone_info ci; 1646 blk_qc_t ret = BLK_QC_T_NONE; 1647 int error = 0; 1648 1649 if (unlikely(!map)) { 1650 bio_io_error(bio); 1651 return ret; 1652 } 1653 1654 init_clone_info(&ci, md, map, bio); 1655 1656 if (bio->bi_opf & REQ_PREFLUSH) { 1657 ci.bio = &ci.io->md->flush_bio; 1658 ci.sector_count = 0; 1659 error = __send_empty_flush(&ci); 1660 /* dec_pending submits any data associated with flush */ 1661 } else { 1662 struct dm_target *ti = md->immutable_target; 1663 struct dm_target_io *tio; 1664 1665 /* 1666 * Defend against IO still getting in during teardown 1667 * - as was seen for a time with nvme-fcloop 1668 */ 1669 if (WARN_ON_ONCE(!ti || !dm_target_is_valid(ti))) { 1670 error = -EIO; 1671 goto out; 1672 } 1673 1674 ci.bio = bio; 1675 ci.sector_count = bio_sectors(bio); 1676 if (unlikely(__process_abnormal_io(&ci, ti, &error))) 1677 goto out; 1678 1679 tio = alloc_tio(&ci, ti, 0, GFP_NOIO); 1680 ret = __clone_and_map_simple_bio(&ci, tio, NULL); 1681 } 1682 out: 1683 /* drop the extra reference count */ 1684 dec_pending(ci.io, errno_to_blk_status(error)); 1685 return ret; 1686 } 1687 1688 typedef blk_qc_t (process_bio_fn)(struct mapped_device *, struct dm_table *, struct bio *); 1689 1690 static blk_qc_t __dm_make_request(struct request_queue *q, struct bio *bio, 1691 process_bio_fn process_bio) 1692 { 1693 struct mapped_device *md = q->queuedata; 1694 blk_qc_t ret = BLK_QC_T_NONE; 1695 int srcu_idx; 1696 struct dm_table *map; 1697 1698 map = dm_get_live_table(md, &srcu_idx); 1699 1700 /* if we're suspended, we have to queue this io for later */ 1701 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) { 1702 dm_put_live_table(md, srcu_idx); 1703 1704 if (!(bio->bi_opf & REQ_RAHEAD)) 1705 queue_io(md, bio); 1706 else 1707 bio_io_error(bio); 1708 return ret; 1709 } 1710 1711 ret = process_bio(md, map, bio); 1712 1713 dm_put_live_table(md, srcu_idx); 1714 return ret; 1715 } 1716 1717 /* 1718 * The request function that remaps the bio to one target and 1719 * splits off any remainder. 1720 */ 1721 static blk_qc_t dm_make_request(struct request_queue *q, struct bio *bio) 1722 { 1723 return __dm_make_request(q, bio, __split_and_process_bio); 1724 } 1725 1726 static blk_qc_t dm_make_request_nvme(struct request_queue *q, struct bio *bio) 1727 { 1728 return __dm_make_request(q, bio, __process_bio); 1729 } 1730 1731 static int dm_any_congested(void *congested_data, int bdi_bits) 1732 { 1733 int r = bdi_bits; 1734 struct mapped_device *md = congested_data; 1735 struct dm_table *map; 1736 1737 if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) { 1738 if (dm_request_based(md)) { 1739 /* 1740 * With request-based DM we only need to check the 1741 * top-level queue for congestion. 1742 */ 1743 r = md->queue->backing_dev_info->wb.state & bdi_bits; 1744 } else { 1745 map = dm_get_live_table_fast(md); 1746 if (map) 1747 r = dm_table_any_congested(map, bdi_bits); 1748 dm_put_live_table_fast(md); 1749 } 1750 } 1751 1752 return r; 1753 } 1754 1755 /*----------------------------------------------------------------- 1756 * An IDR is used to keep track of allocated minor numbers. 1757 *---------------------------------------------------------------*/ 1758 static void free_minor(int minor) 1759 { 1760 spin_lock(&_minor_lock); 1761 idr_remove(&_minor_idr, minor); 1762 spin_unlock(&_minor_lock); 1763 } 1764 1765 /* 1766 * See if the device with a specific minor # is free. 1767 */ 1768 static int specific_minor(int minor) 1769 { 1770 int r; 1771 1772 if (minor >= (1 << MINORBITS)) 1773 return -EINVAL; 1774 1775 idr_preload(GFP_KERNEL); 1776 spin_lock(&_minor_lock); 1777 1778 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT); 1779 1780 spin_unlock(&_minor_lock); 1781 idr_preload_end(); 1782 if (r < 0) 1783 return r == -ENOSPC ? -EBUSY : r; 1784 return 0; 1785 } 1786 1787 static int next_free_minor(int *minor) 1788 { 1789 int r; 1790 1791 idr_preload(GFP_KERNEL); 1792 spin_lock(&_minor_lock); 1793 1794 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT); 1795 1796 spin_unlock(&_minor_lock); 1797 idr_preload_end(); 1798 if (r < 0) 1799 return r; 1800 *minor = r; 1801 return 0; 1802 } 1803 1804 static const struct block_device_operations dm_blk_dops; 1805 static const struct dax_operations dm_dax_ops; 1806 1807 static void dm_wq_work(struct work_struct *work); 1808 1809 static void dm_init_normal_md_queue(struct mapped_device *md) 1810 { 1811 /* 1812 * Initialize aspects of queue that aren't relevant for blk-mq 1813 */ 1814 md->queue->backing_dev_info->congested_fn = dm_any_congested; 1815 } 1816 1817 static void cleanup_mapped_device(struct mapped_device *md) 1818 { 1819 if (md->wq) 1820 destroy_workqueue(md->wq); 1821 bioset_exit(&md->bs); 1822 bioset_exit(&md->io_bs); 1823 1824 if (md->dax_dev) { 1825 kill_dax(md->dax_dev); 1826 put_dax(md->dax_dev); 1827 md->dax_dev = NULL; 1828 } 1829 1830 if (md->disk) { 1831 spin_lock(&_minor_lock); 1832 md->disk->private_data = NULL; 1833 spin_unlock(&_minor_lock); 1834 del_gendisk(md->disk); 1835 put_disk(md->disk); 1836 } 1837 1838 if (md->queue) 1839 blk_cleanup_queue(md->queue); 1840 1841 cleanup_srcu_struct(&md->io_barrier); 1842 1843 if (md->bdev) { 1844 bdput(md->bdev); 1845 md->bdev = NULL; 1846 } 1847 1848 mutex_destroy(&md->suspend_lock); 1849 mutex_destroy(&md->type_lock); 1850 mutex_destroy(&md->table_devices_lock); 1851 1852 dm_mq_cleanup_mapped_device(md); 1853 } 1854 1855 /* 1856 * Allocate and initialise a blank device with a given minor. 1857 */ 1858 static struct mapped_device *alloc_dev(int minor) 1859 { 1860 int r, numa_node_id = dm_get_numa_node(); 1861 struct dax_device *dax_dev = NULL; 1862 struct mapped_device *md; 1863 void *old_md; 1864 1865 md = kvzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id); 1866 if (!md) { 1867 DMWARN("unable to allocate device, out of memory."); 1868 return NULL; 1869 } 1870 1871 if (!try_module_get(THIS_MODULE)) 1872 goto bad_module_get; 1873 1874 /* get a minor number for the dev */ 1875 if (minor == DM_ANY_MINOR) 1876 r = next_free_minor(&minor); 1877 else 1878 r = specific_minor(minor); 1879 if (r < 0) 1880 goto bad_minor; 1881 1882 r = init_srcu_struct(&md->io_barrier); 1883 if (r < 0) 1884 goto bad_io_barrier; 1885 1886 md->numa_node_id = numa_node_id; 1887 md->init_tio_pdu = false; 1888 md->type = DM_TYPE_NONE; 1889 mutex_init(&md->suspend_lock); 1890 mutex_init(&md->type_lock); 1891 mutex_init(&md->table_devices_lock); 1892 spin_lock_init(&md->deferred_lock); 1893 atomic_set(&md->holders, 1); 1894 atomic_set(&md->open_count, 0); 1895 atomic_set(&md->event_nr, 0); 1896 atomic_set(&md->uevent_seq, 0); 1897 INIT_LIST_HEAD(&md->uevent_list); 1898 INIT_LIST_HEAD(&md->table_devices); 1899 spin_lock_init(&md->uevent_lock); 1900 1901 md->queue = blk_alloc_queue_node(GFP_KERNEL, numa_node_id, NULL); 1902 if (!md->queue) 1903 goto bad; 1904 md->queue->queuedata = md; 1905 md->queue->backing_dev_info->congested_data = md; 1906 1907 md->disk = alloc_disk_node(1, md->numa_node_id); 1908 if (!md->disk) 1909 goto bad; 1910 1911 atomic_set(&md->pending[0], 0); 1912 atomic_set(&md->pending[1], 0); 1913 init_waitqueue_head(&md->wait); 1914 INIT_WORK(&md->work, dm_wq_work); 1915 init_waitqueue_head(&md->eventq); 1916 init_completion(&md->kobj_holder.completion); 1917 1918 md->disk->major = _major; 1919 md->disk->first_minor = minor; 1920 md->disk->fops = &dm_blk_dops; 1921 md->disk->queue = md->queue; 1922 md->disk->private_data = md; 1923 sprintf(md->disk->disk_name, "dm-%d", minor); 1924 1925 if (IS_ENABLED(CONFIG_DAX_DRIVER)) { 1926 dax_dev = alloc_dax(md, md->disk->disk_name, &dm_dax_ops); 1927 if (!dax_dev) 1928 goto bad; 1929 } 1930 md->dax_dev = dax_dev; 1931 1932 add_disk_no_queue_reg(md->disk); 1933 format_dev_t(md->name, MKDEV(_major, minor)); 1934 1935 md->wq = alloc_workqueue("kdmflush", WQ_MEM_RECLAIM, 0); 1936 if (!md->wq) 1937 goto bad; 1938 1939 md->bdev = bdget_disk(md->disk, 0); 1940 if (!md->bdev) 1941 goto bad; 1942 1943 bio_init(&md->flush_bio, NULL, 0); 1944 bio_set_dev(&md->flush_bio, md->bdev); 1945 md->flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC; 1946 1947 dm_stats_init(&md->stats); 1948 1949 /* Populate the mapping, nobody knows we exist yet */ 1950 spin_lock(&_minor_lock); 1951 old_md = idr_replace(&_minor_idr, md, minor); 1952 spin_unlock(&_minor_lock); 1953 1954 BUG_ON(old_md != MINOR_ALLOCED); 1955 1956 return md; 1957 1958 bad: 1959 cleanup_mapped_device(md); 1960 bad_io_barrier: 1961 free_minor(minor); 1962 bad_minor: 1963 module_put(THIS_MODULE); 1964 bad_module_get: 1965 kvfree(md); 1966 return NULL; 1967 } 1968 1969 static void unlock_fs(struct mapped_device *md); 1970 1971 static void free_dev(struct mapped_device *md) 1972 { 1973 int minor = MINOR(disk_devt(md->disk)); 1974 1975 unlock_fs(md); 1976 1977 cleanup_mapped_device(md); 1978 1979 free_table_devices(&md->table_devices); 1980 dm_stats_cleanup(&md->stats); 1981 free_minor(minor); 1982 1983 module_put(THIS_MODULE); 1984 kvfree(md); 1985 } 1986 1987 static int __bind_mempools(struct mapped_device *md, struct dm_table *t) 1988 { 1989 struct dm_md_mempools *p = dm_table_get_md_mempools(t); 1990 int ret = 0; 1991 1992 if (dm_table_bio_based(t)) { 1993 /* 1994 * The md may already have mempools that need changing. 1995 * If so, reload bioset because front_pad may have changed 1996 * because a different table was loaded. 1997 */ 1998 bioset_exit(&md->bs); 1999 bioset_exit(&md->io_bs); 2000 2001 } else if (bioset_initialized(&md->bs)) { 2002 /* 2003 * There's no need to reload with request-based dm 2004 * because the size of front_pad doesn't change. 2005 * Note for future: If you are to reload bioset, 2006 * prep-ed requests in the queue may refer 2007 * to bio from the old bioset, so you must walk 2008 * through the queue to unprep. 2009 */ 2010 goto out; 2011 } 2012 2013 BUG_ON(!p || 2014 bioset_initialized(&md->bs) || 2015 bioset_initialized(&md->io_bs)); 2016 2017 ret = bioset_init_from_src(&md->bs, &p->bs); 2018 if (ret) 2019 goto out; 2020 ret = bioset_init_from_src(&md->io_bs, &p->io_bs); 2021 if (ret) 2022 bioset_exit(&md->bs); 2023 out: 2024 /* mempool bind completed, no longer need any mempools in the table */ 2025 dm_table_free_md_mempools(t); 2026 return ret; 2027 } 2028 2029 /* 2030 * Bind a table to the device. 2031 */ 2032 static void event_callback(void *context) 2033 { 2034 unsigned long flags; 2035 LIST_HEAD(uevents); 2036 struct mapped_device *md = (struct mapped_device *) context; 2037 2038 spin_lock_irqsave(&md->uevent_lock, flags); 2039 list_splice_init(&md->uevent_list, &uevents); 2040 spin_unlock_irqrestore(&md->uevent_lock, flags); 2041 2042 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj); 2043 2044 atomic_inc(&md->event_nr); 2045 wake_up(&md->eventq); 2046 dm_issue_global_event(); 2047 } 2048 2049 /* 2050 * Protected by md->suspend_lock obtained by dm_swap_table(). 2051 */ 2052 static void __set_size(struct mapped_device *md, sector_t size) 2053 { 2054 lockdep_assert_held(&md->suspend_lock); 2055 2056 set_capacity(md->disk, size); 2057 2058 i_size_write(md->bdev->bd_inode, (loff_t)size << SECTOR_SHIFT); 2059 } 2060 2061 /* 2062 * Returns old map, which caller must destroy. 2063 */ 2064 static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t, 2065 struct queue_limits *limits) 2066 { 2067 struct dm_table *old_map; 2068 struct request_queue *q = md->queue; 2069 bool request_based = dm_table_request_based(t); 2070 sector_t size; 2071 int ret; 2072 2073 lockdep_assert_held(&md->suspend_lock); 2074 2075 size = dm_table_get_size(t); 2076 2077 /* 2078 * Wipe any geometry if the size of the table changed. 2079 */ 2080 if (size != dm_get_size(md)) 2081 memset(&md->geometry, 0, sizeof(md->geometry)); 2082 2083 __set_size(md, size); 2084 2085 dm_table_event_callback(t, event_callback, md); 2086 2087 /* 2088 * The queue hasn't been stopped yet, if the old table type wasn't 2089 * for request-based during suspension. So stop it to prevent 2090 * I/O mapping before resume. 2091 * This must be done before setting the queue restrictions, 2092 * because request-based dm may be run just after the setting. 2093 */ 2094 if (request_based) 2095 dm_stop_queue(q); 2096 2097 if (request_based || md->type == DM_TYPE_NVME_BIO_BASED) { 2098 /* 2099 * Leverage the fact that request-based DM targets and 2100 * NVMe bio based targets are immutable singletons 2101 * - used to optimize both dm_request_fn and dm_mq_queue_rq; 2102 * and __process_bio. 2103 */ 2104 md->immutable_target = dm_table_get_immutable_target(t); 2105 } 2106 2107 ret = __bind_mempools(md, t); 2108 if (ret) { 2109 old_map = ERR_PTR(ret); 2110 goto out; 2111 } 2112 2113 old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2114 rcu_assign_pointer(md->map, (void *)t); 2115 md->immutable_target_type = dm_table_get_immutable_target_type(t); 2116 2117 dm_table_set_restrictions(t, q, limits); 2118 if (old_map) 2119 dm_sync_table(md); 2120 2121 out: 2122 return old_map; 2123 } 2124 2125 /* 2126 * Returns unbound table for the caller to free. 2127 */ 2128 static struct dm_table *__unbind(struct mapped_device *md) 2129 { 2130 struct dm_table *map = rcu_dereference_protected(md->map, 1); 2131 2132 if (!map) 2133 return NULL; 2134 2135 dm_table_event_callback(map, NULL, NULL); 2136 RCU_INIT_POINTER(md->map, NULL); 2137 dm_sync_table(md); 2138 2139 return map; 2140 } 2141 2142 /* 2143 * Constructor for a new device. 2144 */ 2145 int dm_create(int minor, struct mapped_device **result) 2146 { 2147 int r; 2148 struct mapped_device *md; 2149 2150 md = alloc_dev(minor); 2151 if (!md) 2152 return -ENXIO; 2153 2154 r = dm_sysfs_init(md); 2155 if (r) { 2156 free_dev(md); 2157 return r; 2158 } 2159 2160 *result = md; 2161 return 0; 2162 } 2163 2164 /* 2165 * Functions to manage md->type. 2166 * All are required to hold md->type_lock. 2167 */ 2168 void dm_lock_md_type(struct mapped_device *md) 2169 { 2170 mutex_lock(&md->type_lock); 2171 } 2172 2173 void dm_unlock_md_type(struct mapped_device *md) 2174 { 2175 mutex_unlock(&md->type_lock); 2176 } 2177 2178 void dm_set_md_type(struct mapped_device *md, enum dm_queue_mode type) 2179 { 2180 BUG_ON(!mutex_is_locked(&md->type_lock)); 2181 md->type = type; 2182 } 2183 2184 enum dm_queue_mode dm_get_md_type(struct mapped_device *md) 2185 { 2186 return md->type; 2187 } 2188 2189 struct target_type *dm_get_immutable_target_type(struct mapped_device *md) 2190 { 2191 return md->immutable_target_type; 2192 } 2193 2194 /* 2195 * The queue_limits are only valid as long as you have a reference 2196 * count on 'md'. 2197 */ 2198 struct queue_limits *dm_get_queue_limits(struct mapped_device *md) 2199 { 2200 BUG_ON(!atomic_read(&md->holders)); 2201 return &md->queue->limits; 2202 } 2203 EXPORT_SYMBOL_GPL(dm_get_queue_limits); 2204 2205 /* 2206 * Setup the DM device's queue based on md's type 2207 */ 2208 int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t) 2209 { 2210 int r; 2211 struct queue_limits limits; 2212 enum dm_queue_mode type = dm_get_md_type(md); 2213 2214 switch (type) { 2215 case DM_TYPE_REQUEST_BASED: 2216 r = dm_mq_init_request_queue(md, t); 2217 if (r) { 2218 DMERR("Cannot initialize queue for request-based dm-mq mapped device"); 2219 return r; 2220 } 2221 break; 2222 case DM_TYPE_BIO_BASED: 2223 case DM_TYPE_DAX_BIO_BASED: 2224 dm_init_normal_md_queue(md); 2225 blk_queue_make_request(md->queue, dm_make_request); 2226 break; 2227 case DM_TYPE_NVME_BIO_BASED: 2228 dm_init_normal_md_queue(md); 2229 blk_queue_make_request(md->queue, dm_make_request_nvme); 2230 break; 2231 case DM_TYPE_NONE: 2232 WARN_ON_ONCE(true); 2233 break; 2234 } 2235 2236 r = dm_calculate_queue_limits(t, &limits); 2237 if (r) { 2238 DMERR("Cannot calculate initial queue limits"); 2239 return r; 2240 } 2241 dm_table_set_restrictions(t, md->queue, &limits); 2242 blk_register_queue(md->disk); 2243 2244 return 0; 2245 } 2246 2247 struct mapped_device *dm_get_md(dev_t dev) 2248 { 2249 struct mapped_device *md; 2250 unsigned minor = MINOR(dev); 2251 2252 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS)) 2253 return NULL; 2254 2255 spin_lock(&_minor_lock); 2256 2257 md = idr_find(&_minor_idr, minor); 2258 if (!md || md == MINOR_ALLOCED || (MINOR(disk_devt(dm_disk(md))) != minor) || 2259 test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) { 2260 md = NULL; 2261 goto out; 2262 } 2263 dm_get(md); 2264 out: 2265 spin_unlock(&_minor_lock); 2266 2267 return md; 2268 } 2269 EXPORT_SYMBOL_GPL(dm_get_md); 2270 2271 void *dm_get_mdptr(struct mapped_device *md) 2272 { 2273 return md->interface_ptr; 2274 } 2275 2276 void dm_set_mdptr(struct mapped_device *md, void *ptr) 2277 { 2278 md->interface_ptr = ptr; 2279 } 2280 2281 void dm_get(struct mapped_device *md) 2282 { 2283 atomic_inc(&md->holders); 2284 BUG_ON(test_bit(DMF_FREEING, &md->flags)); 2285 } 2286 2287 int dm_hold(struct mapped_device *md) 2288 { 2289 spin_lock(&_minor_lock); 2290 if (test_bit(DMF_FREEING, &md->flags)) { 2291 spin_unlock(&_minor_lock); 2292 return -EBUSY; 2293 } 2294 dm_get(md); 2295 spin_unlock(&_minor_lock); 2296 return 0; 2297 } 2298 EXPORT_SYMBOL_GPL(dm_hold); 2299 2300 const char *dm_device_name(struct mapped_device *md) 2301 { 2302 return md->name; 2303 } 2304 EXPORT_SYMBOL_GPL(dm_device_name); 2305 2306 static void __dm_destroy(struct mapped_device *md, bool wait) 2307 { 2308 struct dm_table *map; 2309 int srcu_idx; 2310 2311 might_sleep(); 2312 2313 spin_lock(&_minor_lock); 2314 idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md)))); 2315 set_bit(DMF_FREEING, &md->flags); 2316 spin_unlock(&_minor_lock); 2317 2318 blk_set_queue_dying(md->queue); 2319 2320 /* 2321 * Take suspend_lock so that presuspend and postsuspend methods 2322 * do not race with internal suspend. 2323 */ 2324 mutex_lock(&md->suspend_lock); 2325 map = dm_get_live_table(md, &srcu_idx); 2326 if (!dm_suspended_md(md)) { 2327 dm_table_presuspend_targets(map); 2328 dm_table_postsuspend_targets(map); 2329 } 2330 /* dm_put_live_table must be before msleep, otherwise deadlock is possible */ 2331 dm_put_live_table(md, srcu_idx); 2332 mutex_unlock(&md->suspend_lock); 2333 2334 /* 2335 * Rare, but there may be I/O requests still going to complete, 2336 * for example. Wait for all references to disappear. 2337 * No one should increment the reference count of the mapped_device, 2338 * after the mapped_device state becomes DMF_FREEING. 2339 */ 2340 if (wait) 2341 while (atomic_read(&md->holders)) 2342 msleep(1); 2343 else if (atomic_read(&md->holders)) 2344 DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)", 2345 dm_device_name(md), atomic_read(&md->holders)); 2346 2347 dm_sysfs_exit(md); 2348 dm_table_destroy(__unbind(md)); 2349 free_dev(md); 2350 } 2351 2352 void dm_destroy(struct mapped_device *md) 2353 { 2354 __dm_destroy(md, true); 2355 } 2356 2357 void dm_destroy_immediate(struct mapped_device *md) 2358 { 2359 __dm_destroy(md, false); 2360 } 2361 2362 void dm_put(struct mapped_device *md) 2363 { 2364 atomic_dec(&md->holders); 2365 } 2366 EXPORT_SYMBOL_GPL(dm_put); 2367 2368 static int dm_wait_for_completion(struct mapped_device *md, long task_state) 2369 { 2370 int r = 0; 2371 DEFINE_WAIT(wait); 2372 2373 while (1) { 2374 prepare_to_wait(&md->wait, &wait, task_state); 2375 2376 if (!md_in_flight(md)) 2377 break; 2378 2379 if (signal_pending_state(task_state, current)) { 2380 r = -EINTR; 2381 break; 2382 } 2383 2384 io_schedule(); 2385 } 2386 finish_wait(&md->wait, &wait); 2387 2388 return r; 2389 } 2390 2391 /* 2392 * Process the deferred bios 2393 */ 2394 static void dm_wq_work(struct work_struct *work) 2395 { 2396 struct mapped_device *md = container_of(work, struct mapped_device, 2397 work); 2398 struct bio *c; 2399 int srcu_idx; 2400 struct dm_table *map; 2401 2402 map = dm_get_live_table(md, &srcu_idx); 2403 2404 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) { 2405 spin_lock_irq(&md->deferred_lock); 2406 c = bio_list_pop(&md->deferred); 2407 spin_unlock_irq(&md->deferred_lock); 2408 2409 if (!c) 2410 break; 2411 2412 if (dm_request_based(md)) 2413 generic_make_request(c); 2414 else 2415 __split_and_process_bio(md, map, c); 2416 } 2417 2418 dm_put_live_table(md, srcu_idx); 2419 } 2420 2421 static void dm_queue_flush(struct mapped_device *md) 2422 { 2423 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 2424 smp_mb__after_atomic(); 2425 queue_work(md->wq, &md->work); 2426 } 2427 2428 /* 2429 * Swap in a new table, returning the old one for the caller to destroy. 2430 */ 2431 struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table) 2432 { 2433 struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL); 2434 struct queue_limits limits; 2435 int r; 2436 2437 mutex_lock(&md->suspend_lock); 2438 2439 /* device must be suspended */ 2440 if (!dm_suspended_md(md)) 2441 goto out; 2442 2443 /* 2444 * If the new table has no data devices, retain the existing limits. 2445 * This helps multipath with queue_if_no_path if all paths disappear, 2446 * then new I/O is queued based on these limits, and then some paths 2447 * reappear. 2448 */ 2449 if (dm_table_has_no_data_devices(table)) { 2450 live_map = dm_get_live_table_fast(md); 2451 if (live_map) 2452 limits = md->queue->limits; 2453 dm_put_live_table_fast(md); 2454 } 2455 2456 if (!live_map) { 2457 r = dm_calculate_queue_limits(table, &limits); 2458 if (r) { 2459 map = ERR_PTR(r); 2460 goto out; 2461 } 2462 } 2463 2464 map = __bind(md, table, &limits); 2465 dm_issue_global_event(); 2466 2467 out: 2468 mutex_unlock(&md->suspend_lock); 2469 return map; 2470 } 2471 2472 /* 2473 * Functions to lock and unlock any filesystem running on the 2474 * device. 2475 */ 2476 static int lock_fs(struct mapped_device *md) 2477 { 2478 int r; 2479 2480 WARN_ON(md->frozen_sb); 2481 2482 md->frozen_sb = freeze_bdev(md->bdev); 2483 if (IS_ERR(md->frozen_sb)) { 2484 r = PTR_ERR(md->frozen_sb); 2485 md->frozen_sb = NULL; 2486 return r; 2487 } 2488 2489 set_bit(DMF_FROZEN, &md->flags); 2490 2491 return 0; 2492 } 2493 2494 static void unlock_fs(struct mapped_device *md) 2495 { 2496 if (!test_bit(DMF_FROZEN, &md->flags)) 2497 return; 2498 2499 thaw_bdev(md->bdev, md->frozen_sb); 2500 md->frozen_sb = NULL; 2501 clear_bit(DMF_FROZEN, &md->flags); 2502 } 2503 2504 /* 2505 * @suspend_flags: DM_SUSPEND_LOCKFS_FLAG and/or DM_SUSPEND_NOFLUSH_FLAG 2506 * @task_state: e.g. TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE 2507 * @dmf_suspended_flag: DMF_SUSPENDED or DMF_SUSPENDED_INTERNALLY 2508 * 2509 * If __dm_suspend returns 0, the device is completely quiescent 2510 * now. There is no request-processing activity. All new requests 2511 * are being added to md->deferred list. 2512 */ 2513 static int __dm_suspend(struct mapped_device *md, struct dm_table *map, 2514 unsigned suspend_flags, long task_state, 2515 int dmf_suspended_flag) 2516 { 2517 bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG; 2518 bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG; 2519 int r; 2520 2521 lockdep_assert_held(&md->suspend_lock); 2522 2523 /* 2524 * DMF_NOFLUSH_SUSPENDING must be set before presuspend. 2525 * This flag is cleared before dm_suspend returns. 2526 */ 2527 if (noflush) 2528 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 2529 else 2530 pr_debug("%s: suspending with flush\n", dm_device_name(md)); 2531 2532 /* 2533 * This gets reverted if there's an error later and the targets 2534 * provide the .presuspend_undo hook. 2535 */ 2536 dm_table_presuspend_targets(map); 2537 2538 /* 2539 * Flush I/O to the device. 2540 * Any I/O submitted after lock_fs() may not be flushed. 2541 * noflush takes precedence over do_lockfs. 2542 * (lock_fs() flushes I/Os and waits for them to complete.) 2543 */ 2544 if (!noflush && do_lockfs) { 2545 r = lock_fs(md); 2546 if (r) { 2547 dm_table_presuspend_undo_targets(map); 2548 return r; 2549 } 2550 } 2551 2552 /* 2553 * Here we must make sure that no processes are submitting requests 2554 * to target drivers i.e. no one may be executing 2555 * __split_and_process_bio. This is called from dm_request and 2556 * dm_wq_work. 2557 * 2558 * To get all processes out of __split_and_process_bio in dm_request, 2559 * we take the write lock. To prevent any process from reentering 2560 * __split_and_process_bio from dm_request and quiesce the thread 2561 * (dm_wq_work), we set BMF_BLOCK_IO_FOR_SUSPEND and call 2562 * flush_workqueue(md->wq). 2563 */ 2564 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 2565 if (map) 2566 synchronize_srcu(&md->io_barrier); 2567 2568 /* 2569 * Stop md->queue before flushing md->wq in case request-based 2570 * dm defers requests to md->wq from md->queue. 2571 */ 2572 if (dm_request_based(md)) 2573 dm_stop_queue(md->queue); 2574 2575 flush_workqueue(md->wq); 2576 2577 /* 2578 * At this point no more requests are entering target request routines. 2579 * We call dm_wait_for_completion to wait for all existing requests 2580 * to finish. 2581 */ 2582 r = dm_wait_for_completion(md, task_state); 2583 if (!r) 2584 set_bit(dmf_suspended_flag, &md->flags); 2585 2586 if (noflush) 2587 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 2588 if (map) 2589 synchronize_srcu(&md->io_barrier); 2590 2591 /* were we interrupted ? */ 2592 if (r < 0) { 2593 dm_queue_flush(md); 2594 2595 if (dm_request_based(md)) 2596 dm_start_queue(md->queue); 2597 2598 unlock_fs(md); 2599 dm_table_presuspend_undo_targets(map); 2600 /* pushback list is already flushed, so skip flush */ 2601 } 2602 2603 return r; 2604 } 2605 2606 /* 2607 * We need to be able to change a mapping table under a mounted 2608 * filesystem. For example we might want to move some data in 2609 * the background. Before the table can be swapped with 2610 * dm_bind_table, dm_suspend must be called to flush any in 2611 * flight bios and ensure that any further io gets deferred. 2612 */ 2613 /* 2614 * Suspend mechanism in request-based dm. 2615 * 2616 * 1. Flush all I/Os by lock_fs() if needed. 2617 * 2. Stop dispatching any I/O by stopping the request_queue. 2618 * 3. Wait for all in-flight I/Os to be completed or requeued. 2619 * 2620 * To abort suspend, start the request_queue. 2621 */ 2622 int dm_suspend(struct mapped_device *md, unsigned suspend_flags) 2623 { 2624 struct dm_table *map = NULL; 2625 int r = 0; 2626 2627 retry: 2628 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); 2629 2630 if (dm_suspended_md(md)) { 2631 r = -EINVAL; 2632 goto out_unlock; 2633 } 2634 2635 if (dm_suspended_internally_md(md)) { 2636 /* already internally suspended, wait for internal resume */ 2637 mutex_unlock(&md->suspend_lock); 2638 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); 2639 if (r) 2640 return r; 2641 goto retry; 2642 } 2643 2644 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2645 2646 r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED); 2647 if (r) 2648 goto out_unlock; 2649 2650 dm_table_postsuspend_targets(map); 2651 2652 out_unlock: 2653 mutex_unlock(&md->suspend_lock); 2654 return r; 2655 } 2656 2657 static int __dm_resume(struct mapped_device *md, struct dm_table *map) 2658 { 2659 if (map) { 2660 int r = dm_table_resume_targets(map); 2661 if (r) 2662 return r; 2663 } 2664 2665 dm_queue_flush(md); 2666 2667 /* 2668 * Flushing deferred I/Os must be done after targets are resumed 2669 * so that mapping of targets can work correctly. 2670 * Request-based dm is queueing the deferred I/Os in its request_queue. 2671 */ 2672 if (dm_request_based(md)) 2673 dm_start_queue(md->queue); 2674 2675 unlock_fs(md); 2676 2677 return 0; 2678 } 2679 2680 int dm_resume(struct mapped_device *md) 2681 { 2682 int r; 2683 struct dm_table *map = NULL; 2684 2685 retry: 2686 r = -EINVAL; 2687 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); 2688 2689 if (!dm_suspended_md(md)) 2690 goto out; 2691 2692 if (dm_suspended_internally_md(md)) { 2693 /* already internally suspended, wait for internal resume */ 2694 mutex_unlock(&md->suspend_lock); 2695 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); 2696 if (r) 2697 return r; 2698 goto retry; 2699 } 2700 2701 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2702 if (!map || !dm_table_get_size(map)) 2703 goto out; 2704 2705 r = __dm_resume(md, map); 2706 if (r) 2707 goto out; 2708 2709 clear_bit(DMF_SUSPENDED, &md->flags); 2710 out: 2711 mutex_unlock(&md->suspend_lock); 2712 2713 return r; 2714 } 2715 2716 /* 2717 * Internal suspend/resume works like userspace-driven suspend. It waits 2718 * until all bios finish and prevents issuing new bios to the target drivers. 2719 * It may be used only from the kernel. 2720 */ 2721 2722 static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags) 2723 { 2724 struct dm_table *map = NULL; 2725 2726 lockdep_assert_held(&md->suspend_lock); 2727 2728 if (md->internal_suspend_count++) 2729 return; /* nested internal suspend */ 2730 2731 if (dm_suspended_md(md)) { 2732 set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 2733 return; /* nest suspend */ 2734 } 2735 2736 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2737 2738 /* 2739 * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is 2740 * supported. Properly supporting a TASK_INTERRUPTIBLE internal suspend 2741 * would require changing .presuspend to return an error -- avoid this 2742 * until there is a need for more elaborate variants of internal suspend. 2743 */ 2744 (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE, 2745 DMF_SUSPENDED_INTERNALLY); 2746 2747 dm_table_postsuspend_targets(map); 2748 } 2749 2750 static void __dm_internal_resume(struct mapped_device *md) 2751 { 2752 BUG_ON(!md->internal_suspend_count); 2753 2754 if (--md->internal_suspend_count) 2755 return; /* resume from nested internal suspend */ 2756 2757 if (dm_suspended_md(md)) 2758 goto done; /* resume from nested suspend */ 2759 2760 /* 2761 * NOTE: existing callers don't need to call dm_table_resume_targets 2762 * (which may fail -- so best to avoid it for now by passing NULL map) 2763 */ 2764 (void) __dm_resume(md, NULL); 2765 2766 done: 2767 clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 2768 smp_mb__after_atomic(); 2769 wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY); 2770 } 2771 2772 void dm_internal_suspend_noflush(struct mapped_device *md) 2773 { 2774 mutex_lock(&md->suspend_lock); 2775 __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG); 2776 mutex_unlock(&md->suspend_lock); 2777 } 2778 EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush); 2779 2780 void dm_internal_resume(struct mapped_device *md) 2781 { 2782 mutex_lock(&md->suspend_lock); 2783 __dm_internal_resume(md); 2784 mutex_unlock(&md->suspend_lock); 2785 } 2786 EXPORT_SYMBOL_GPL(dm_internal_resume); 2787 2788 /* 2789 * Fast variants of internal suspend/resume hold md->suspend_lock, 2790 * which prevents interaction with userspace-driven suspend. 2791 */ 2792 2793 void dm_internal_suspend_fast(struct mapped_device *md) 2794 { 2795 mutex_lock(&md->suspend_lock); 2796 if (dm_suspended_md(md) || dm_suspended_internally_md(md)) 2797 return; 2798 2799 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 2800 synchronize_srcu(&md->io_barrier); 2801 flush_workqueue(md->wq); 2802 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE); 2803 } 2804 EXPORT_SYMBOL_GPL(dm_internal_suspend_fast); 2805 2806 void dm_internal_resume_fast(struct mapped_device *md) 2807 { 2808 if (dm_suspended_md(md) || dm_suspended_internally_md(md)) 2809 goto done; 2810 2811 dm_queue_flush(md); 2812 2813 done: 2814 mutex_unlock(&md->suspend_lock); 2815 } 2816 EXPORT_SYMBOL_GPL(dm_internal_resume_fast); 2817 2818 /*----------------------------------------------------------------- 2819 * Event notification. 2820 *---------------------------------------------------------------*/ 2821 int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action, 2822 unsigned cookie) 2823 { 2824 char udev_cookie[DM_COOKIE_LENGTH]; 2825 char *envp[] = { udev_cookie, NULL }; 2826 2827 if (!cookie) 2828 return kobject_uevent(&disk_to_dev(md->disk)->kobj, action); 2829 else { 2830 snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u", 2831 DM_COOKIE_ENV_VAR_NAME, cookie); 2832 return kobject_uevent_env(&disk_to_dev(md->disk)->kobj, 2833 action, envp); 2834 } 2835 } 2836 2837 uint32_t dm_next_uevent_seq(struct mapped_device *md) 2838 { 2839 return atomic_add_return(1, &md->uevent_seq); 2840 } 2841 2842 uint32_t dm_get_event_nr(struct mapped_device *md) 2843 { 2844 return atomic_read(&md->event_nr); 2845 } 2846 2847 int dm_wait_event(struct mapped_device *md, int event_nr) 2848 { 2849 return wait_event_interruptible(md->eventq, 2850 (event_nr != atomic_read(&md->event_nr))); 2851 } 2852 2853 void dm_uevent_add(struct mapped_device *md, struct list_head *elist) 2854 { 2855 unsigned long flags; 2856 2857 spin_lock_irqsave(&md->uevent_lock, flags); 2858 list_add(elist, &md->uevent_list); 2859 spin_unlock_irqrestore(&md->uevent_lock, flags); 2860 } 2861 2862 /* 2863 * The gendisk is only valid as long as you have a reference 2864 * count on 'md'. 2865 */ 2866 struct gendisk *dm_disk(struct mapped_device *md) 2867 { 2868 return md->disk; 2869 } 2870 EXPORT_SYMBOL_GPL(dm_disk); 2871 2872 struct kobject *dm_kobject(struct mapped_device *md) 2873 { 2874 return &md->kobj_holder.kobj; 2875 } 2876 2877 struct mapped_device *dm_get_from_kobject(struct kobject *kobj) 2878 { 2879 struct mapped_device *md; 2880 2881 md = container_of(kobj, struct mapped_device, kobj_holder.kobj); 2882 2883 spin_lock(&_minor_lock); 2884 if (test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) { 2885 md = NULL; 2886 goto out; 2887 } 2888 dm_get(md); 2889 out: 2890 spin_unlock(&_minor_lock); 2891 2892 return md; 2893 } 2894 2895 int dm_suspended_md(struct mapped_device *md) 2896 { 2897 return test_bit(DMF_SUSPENDED, &md->flags); 2898 } 2899 2900 int dm_suspended_internally_md(struct mapped_device *md) 2901 { 2902 return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 2903 } 2904 2905 int dm_test_deferred_remove_flag(struct mapped_device *md) 2906 { 2907 return test_bit(DMF_DEFERRED_REMOVE, &md->flags); 2908 } 2909 2910 int dm_suspended(struct dm_target *ti) 2911 { 2912 return dm_suspended_md(dm_table_get_md(ti->table)); 2913 } 2914 EXPORT_SYMBOL_GPL(dm_suspended); 2915 2916 int dm_noflush_suspending(struct dm_target *ti) 2917 { 2918 return __noflush_suspending(dm_table_get_md(ti->table)); 2919 } 2920 EXPORT_SYMBOL_GPL(dm_noflush_suspending); 2921 2922 struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_queue_mode type, 2923 unsigned integrity, unsigned per_io_data_size, 2924 unsigned min_pool_size) 2925 { 2926 struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id); 2927 unsigned int pool_size = 0; 2928 unsigned int front_pad, io_front_pad; 2929 int ret; 2930 2931 if (!pools) 2932 return NULL; 2933 2934 switch (type) { 2935 case DM_TYPE_BIO_BASED: 2936 case DM_TYPE_DAX_BIO_BASED: 2937 case DM_TYPE_NVME_BIO_BASED: 2938 pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size); 2939 front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone); 2940 io_front_pad = roundup(front_pad, __alignof__(struct dm_io)) + offsetof(struct dm_io, tio); 2941 ret = bioset_init(&pools->io_bs, pool_size, io_front_pad, 0); 2942 if (ret) 2943 goto out; 2944 if (integrity && bioset_integrity_create(&pools->io_bs, pool_size)) 2945 goto out; 2946 break; 2947 case DM_TYPE_REQUEST_BASED: 2948 pool_size = max(dm_get_reserved_rq_based_ios(), min_pool_size); 2949 front_pad = offsetof(struct dm_rq_clone_bio_info, clone); 2950 /* per_io_data_size is used for blk-mq pdu at queue allocation */ 2951 break; 2952 default: 2953 BUG(); 2954 } 2955 2956 ret = bioset_init(&pools->bs, pool_size, front_pad, 0); 2957 if (ret) 2958 goto out; 2959 2960 if (integrity && bioset_integrity_create(&pools->bs, pool_size)) 2961 goto out; 2962 2963 return pools; 2964 2965 out: 2966 dm_free_md_mempools(pools); 2967 2968 return NULL; 2969 } 2970 2971 void dm_free_md_mempools(struct dm_md_mempools *pools) 2972 { 2973 if (!pools) 2974 return; 2975 2976 bioset_exit(&pools->bs); 2977 bioset_exit(&pools->io_bs); 2978 2979 kfree(pools); 2980 } 2981 2982 struct dm_pr { 2983 u64 old_key; 2984 u64 new_key; 2985 u32 flags; 2986 bool fail_early; 2987 }; 2988 2989 static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn, 2990 void *data) 2991 { 2992 struct mapped_device *md = bdev->bd_disk->private_data; 2993 struct dm_table *table; 2994 struct dm_target *ti; 2995 int ret = -ENOTTY, srcu_idx; 2996 2997 table = dm_get_live_table(md, &srcu_idx); 2998 if (!table || !dm_table_get_size(table)) 2999 goto out; 3000 3001 /* We only support devices that have a single target */ 3002 if (dm_table_get_num_targets(table) != 1) 3003 goto out; 3004 ti = dm_table_get_target(table, 0); 3005 3006 ret = -EINVAL; 3007 if (!ti->type->iterate_devices) 3008 goto out; 3009 3010 ret = ti->type->iterate_devices(ti, fn, data); 3011 out: 3012 dm_put_live_table(md, srcu_idx); 3013 return ret; 3014 } 3015 3016 /* 3017 * For register / unregister we need to manually call out to every path. 3018 */ 3019 static int __dm_pr_register(struct dm_target *ti, struct dm_dev *dev, 3020 sector_t start, sector_t len, void *data) 3021 { 3022 struct dm_pr *pr = data; 3023 const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops; 3024 3025 if (!ops || !ops->pr_register) 3026 return -EOPNOTSUPP; 3027 return ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags); 3028 } 3029 3030 static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key, 3031 u32 flags) 3032 { 3033 struct dm_pr pr = { 3034 .old_key = old_key, 3035 .new_key = new_key, 3036 .flags = flags, 3037 .fail_early = true, 3038 }; 3039 int ret; 3040 3041 ret = dm_call_pr(bdev, __dm_pr_register, &pr); 3042 if (ret && new_key) { 3043 /* unregister all paths if we failed to register any path */ 3044 pr.old_key = new_key; 3045 pr.new_key = 0; 3046 pr.flags = 0; 3047 pr.fail_early = false; 3048 dm_call_pr(bdev, __dm_pr_register, &pr); 3049 } 3050 3051 return ret; 3052 } 3053 3054 static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type, 3055 u32 flags) 3056 { 3057 struct mapped_device *md = bdev->bd_disk->private_data; 3058 const struct pr_ops *ops; 3059 int r, srcu_idx; 3060 3061 r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 3062 if (r < 0) 3063 goto out; 3064 3065 ops = bdev->bd_disk->fops->pr_ops; 3066 if (ops && ops->pr_reserve) 3067 r = ops->pr_reserve(bdev, key, type, flags); 3068 else 3069 r = -EOPNOTSUPP; 3070 out: 3071 dm_unprepare_ioctl(md, srcu_idx); 3072 return r; 3073 } 3074 3075 static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type) 3076 { 3077 struct mapped_device *md = bdev->bd_disk->private_data; 3078 const struct pr_ops *ops; 3079 int r, srcu_idx; 3080 3081 r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 3082 if (r < 0) 3083 goto out; 3084 3085 ops = bdev->bd_disk->fops->pr_ops; 3086 if (ops && ops->pr_release) 3087 r = ops->pr_release(bdev, key, type); 3088 else 3089 r = -EOPNOTSUPP; 3090 out: 3091 dm_unprepare_ioctl(md, srcu_idx); 3092 return r; 3093 } 3094 3095 static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key, 3096 enum pr_type type, bool abort) 3097 { 3098 struct mapped_device *md = bdev->bd_disk->private_data; 3099 const struct pr_ops *ops; 3100 int r, srcu_idx; 3101 3102 r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 3103 if (r < 0) 3104 goto out; 3105 3106 ops = bdev->bd_disk->fops->pr_ops; 3107 if (ops && ops->pr_preempt) 3108 r = ops->pr_preempt(bdev, old_key, new_key, type, abort); 3109 else 3110 r = -EOPNOTSUPP; 3111 out: 3112 dm_unprepare_ioctl(md, srcu_idx); 3113 return r; 3114 } 3115 3116 static int dm_pr_clear(struct block_device *bdev, u64 key) 3117 { 3118 struct mapped_device *md = bdev->bd_disk->private_data; 3119 const struct pr_ops *ops; 3120 int r, srcu_idx; 3121 3122 r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 3123 if (r < 0) 3124 goto out; 3125 3126 ops = bdev->bd_disk->fops->pr_ops; 3127 if (ops && ops->pr_clear) 3128 r = ops->pr_clear(bdev, key); 3129 else 3130 r = -EOPNOTSUPP; 3131 out: 3132 dm_unprepare_ioctl(md, srcu_idx); 3133 return r; 3134 } 3135 3136 static const struct pr_ops dm_pr_ops = { 3137 .pr_register = dm_pr_register, 3138 .pr_reserve = dm_pr_reserve, 3139 .pr_release = dm_pr_release, 3140 .pr_preempt = dm_pr_preempt, 3141 .pr_clear = dm_pr_clear, 3142 }; 3143 3144 static const struct block_device_operations dm_blk_dops = { 3145 .open = dm_blk_open, 3146 .release = dm_blk_close, 3147 .ioctl = dm_blk_ioctl, 3148 .getgeo = dm_blk_getgeo, 3149 .report_zones = dm_blk_report_zones, 3150 .pr_ops = &dm_pr_ops, 3151 .owner = THIS_MODULE 3152 }; 3153 3154 static const struct dax_operations dm_dax_ops = { 3155 .direct_access = dm_dax_direct_access, 3156 .copy_from_iter = dm_dax_copy_from_iter, 3157 .copy_to_iter = dm_dax_copy_to_iter, 3158 }; 3159 3160 /* 3161 * module hooks 3162 */ 3163 module_init(dm_init); 3164 module_exit(dm_exit); 3165 3166 module_param(major, uint, 0); 3167 MODULE_PARM_DESC(major, "The major number of the device mapper"); 3168 3169 module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR); 3170 MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools"); 3171 3172 module_param(dm_numa_node, int, S_IRUGO | S_IWUSR); 3173 MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations"); 3174 3175 MODULE_DESCRIPTION(DM_NAME " driver"); 3176 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>"); 3177 MODULE_LICENSE("GPL"); 3178