1 /* 2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited. 3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. 4 * 5 * This file is released under the GPL. 6 */ 7 8 #include "dm-core.h" 9 #include "dm-rq.h" 10 #include "dm-uevent.h" 11 #include "dm-ima.h" 12 13 #include <linux/init.h> 14 #include <linux/module.h> 15 #include <linux/mutex.h> 16 #include <linux/sched/mm.h> 17 #include <linux/sched/signal.h> 18 #include <linux/blkpg.h> 19 #include <linux/bio.h> 20 #include <linux/mempool.h> 21 #include <linux/dax.h> 22 #include <linux/slab.h> 23 #include <linux/idr.h> 24 #include <linux/uio.h> 25 #include <linux/hdreg.h> 26 #include <linux/delay.h> 27 #include <linux/wait.h> 28 #include <linux/pr.h> 29 #include <linux/refcount.h> 30 #include <linux/part_stat.h> 31 #include <linux/blk-crypto.h> 32 #include <linux/blk-crypto-profile.h> 33 34 #define DM_MSG_PREFIX "core" 35 36 /* 37 * Cookies are numeric values sent with CHANGE and REMOVE 38 * uevents while resuming, removing or renaming the device. 39 */ 40 #define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE" 41 #define DM_COOKIE_LENGTH 24 42 43 static const char *_name = DM_NAME; 44 45 static unsigned int major = 0; 46 static unsigned int _major = 0; 47 48 static DEFINE_IDR(_minor_idr); 49 50 static DEFINE_SPINLOCK(_minor_lock); 51 52 static void do_deferred_remove(struct work_struct *w); 53 54 static DECLARE_WORK(deferred_remove_work, do_deferred_remove); 55 56 static struct workqueue_struct *deferred_remove_workqueue; 57 58 atomic_t dm_global_event_nr = ATOMIC_INIT(0); 59 DECLARE_WAIT_QUEUE_HEAD(dm_global_eventq); 60 61 void dm_issue_global_event(void) 62 { 63 atomic_inc(&dm_global_event_nr); 64 wake_up(&dm_global_eventq); 65 } 66 67 /* 68 * One of these is allocated (on-stack) per original bio. 69 */ 70 struct clone_info { 71 struct dm_table *map; 72 struct bio *bio; 73 struct dm_io *io; 74 sector_t sector; 75 unsigned sector_count; 76 }; 77 78 #define DM_TARGET_IO_BIO_OFFSET (offsetof(struct dm_target_io, clone)) 79 #define DM_IO_BIO_OFFSET \ 80 (offsetof(struct dm_target_io, clone) + offsetof(struct dm_io, tio)) 81 82 static inline struct dm_target_io *clone_to_tio(struct bio *clone) 83 { 84 return container_of(clone, struct dm_target_io, clone); 85 } 86 87 void *dm_per_bio_data(struct bio *bio, size_t data_size) 88 { 89 if (!clone_to_tio(bio)->inside_dm_io) 90 return (char *)bio - DM_TARGET_IO_BIO_OFFSET - data_size; 91 return (char *)bio - DM_IO_BIO_OFFSET - data_size; 92 } 93 EXPORT_SYMBOL_GPL(dm_per_bio_data); 94 95 struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size) 96 { 97 struct dm_io *io = (struct dm_io *)((char *)data + data_size); 98 if (io->magic == DM_IO_MAGIC) 99 return (struct bio *)((char *)io + DM_IO_BIO_OFFSET); 100 BUG_ON(io->magic != DM_TIO_MAGIC); 101 return (struct bio *)((char *)io + DM_TARGET_IO_BIO_OFFSET); 102 } 103 EXPORT_SYMBOL_GPL(dm_bio_from_per_bio_data); 104 105 unsigned dm_bio_get_target_bio_nr(const struct bio *bio) 106 { 107 return container_of(bio, struct dm_target_io, clone)->target_bio_nr; 108 } 109 EXPORT_SYMBOL_GPL(dm_bio_get_target_bio_nr); 110 111 #define MINOR_ALLOCED ((void *)-1) 112 113 #define DM_NUMA_NODE NUMA_NO_NODE 114 static int dm_numa_node = DM_NUMA_NODE; 115 116 #define DEFAULT_SWAP_BIOS (8 * 1048576 / PAGE_SIZE) 117 static int swap_bios = DEFAULT_SWAP_BIOS; 118 static int get_swap_bios(void) 119 { 120 int latch = READ_ONCE(swap_bios); 121 if (unlikely(latch <= 0)) 122 latch = DEFAULT_SWAP_BIOS; 123 return latch; 124 } 125 126 /* 127 * For mempools pre-allocation at the table loading time. 128 */ 129 struct dm_md_mempools { 130 struct bio_set bs; 131 struct bio_set io_bs; 132 }; 133 134 struct table_device { 135 struct list_head list; 136 refcount_t count; 137 struct dm_dev dm_dev; 138 }; 139 140 /* 141 * Bio-based DM's mempools' reserved IOs set by the user. 142 */ 143 #define RESERVED_BIO_BASED_IOS 16 144 static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS; 145 146 static int __dm_get_module_param_int(int *module_param, int min, int max) 147 { 148 int param = READ_ONCE(*module_param); 149 int modified_param = 0; 150 bool modified = true; 151 152 if (param < min) 153 modified_param = min; 154 else if (param > max) 155 modified_param = max; 156 else 157 modified = false; 158 159 if (modified) { 160 (void)cmpxchg(module_param, param, modified_param); 161 param = modified_param; 162 } 163 164 return param; 165 } 166 167 unsigned __dm_get_module_param(unsigned *module_param, 168 unsigned def, unsigned max) 169 { 170 unsigned param = READ_ONCE(*module_param); 171 unsigned modified_param = 0; 172 173 if (!param) 174 modified_param = def; 175 else if (param > max) 176 modified_param = max; 177 178 if (modified_param) { 179 (void)cmpxchg(module_param, param, modified_param); 180 param = modified_param; 181 } 182 183 return param; 184 } 185 186 unsigned dm_get_reserved_bio_based_ios(void) 187 { 188 return __dm_get_module_param(&reserved_bio_based_ios, 189 RESERVED_BIO_BASED_IOS, DM_RESERVED_MAX_IOS); 190 } 191 EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios); 192 193 static unsigned dm_get_numa_node(void) 194 { 195 return __dm_get_module_param_int(&dm_numa_node, 196 DM_NUMA_NODE, num_online_nodes() - 1); 197 } 198 199 static int __init local_init(void) 200 { 201 int r; 202 203 r = dm_uevent_init(); 204 if (r) 205 return r; 206 207 deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1); 208 if (!deferred_remove_workqueue) { 209 r = -ENOMEM; 210 goto out_uevent_exit; 211 } 212 213 _major = major; 214 r = register_blkdev(_major, _name); 215 if (r < 0) 216 goto out_free_workqueue; 217 218 if (!_major) 219 _major = r; 220 221 return 0; 222 223 out_free_workqueue: 224 destroy_workqueue(deferred_remove_workqueue); 225 out_uevent_exit: 226 dm_uevent_exit(); 227 228 return r; 229 } 230 231 static void local_exit(void) 232 { 233 flush_scheduled_work(); 234 destroy_workqueue(deferred_remove_workqueue); 235 236 unregister_blkdev(_major, _name); 237 dm_uevent_exit(); 238 239 _major = 0; 240 241 DMINFO("cleaned up"); 242 } 243 244 static int (*_inits[])(void) __initdata = { 245 local_init, 246 dm_target_init, 247 dm_linear_init, 248 dm_stripe_init, 249 dm_io_init, 250 dm_kcopyd_init, 251 dm_interface_init, 252 dm_statistics_init, 253 }; 254 255 static void (*_exits[])(void) = { 256 local_exit, 257 dm_target_exit, 258 dm_linear_exit, 259 dm_stripe_exit, 260 dm_io_exit, 261 dm_kcopyd_exit, 262 dm_interface_exit, 263 dm_statistics_exit, 264 }; 265 266 static int __init dm_init(void) 267 { 268 const int count = ARRAY_SIZE(_inits); 269 int r, i; 270 271 #if (IS_ENABLED(CONFIG_IMA) && !IS_ENABLED(CONFIG_IMA_DISABLE_HTABLE)) 272 DMWARN("CONFIG_IMA_DISABLE_HTABLE is disabled." 273 " Duplicate IMA measurements will not be recorded in the IMA log."); 274 #endif 275 276 for (i = 0; i < count; i++) { 277 r = _inits[i](); 278 if (r) 279 goto bad; 280 } 281 282 return 0; 283 bad: 284 while (i--) 285 _exits[i](); 286 287 return r; 288 } 289 290 static void __exit dm_exit(void) 291 { 292 int i = ARRAY_SIZE(_exits); 293 294 while (i--) 295 _exits[i](); 296 297 /* 298 * Should be empty by this point. 299 */ 300 idr_destroy(&_minor_idr); 301 } 302 303 /* 304 * Block device functions 305 */ 306 int dm_deleting_md(struct mapped_device *md) 307 { 308 return test_bit(DMF_DELETING, &md->flags); 309 } 310 311 static int dm_blk_open(struct block_device *bdev, fmode_t mode) 312 { 313 struct mapped_device *md; 314 315 spin_lock(&_minor_lock); 316 317 md = bdev->bd_disk->private_data; 318 if (!md) 319 goto out; 320 321 if (test_bit(DMF_FREEING, &md->flags) || 322 dm_deleting_md(md)) { 323 md = NULL; 324 goto out; 325 } 326 327 dm_get(md); 328 atomic_inc(&md->open_count); 329 out: 330 spin_unlock(&_minor_lock); 331 332 return md ? 0 : -ENXIO; 333 } 334 335 static void dm_blk_close(struct gendisk *disk, fmode_t mode) 336 { 337 struct mapped_device *md; 338 339 spin_lock(&_minor_lock); 340 341 md = disk->private_data; 342 if (WARN_ON(!md)) 343 goto out; 344 345 if (atomic_dec_and_test(&md->open_count) && 346 (test_bit(DMF_DEFERRED_REMOVE, &md->flags))) 347 queue_work(deferred_remove_workqueue, &deferred_remove_work); 348 349 dm_put(md); 350 out: 351 spin_unlock(&_minor_lock); 352 } 353 354 int dm_open_count(struct mapped_device *md) 355 { 356 return atomic_read(&md->open_count); 357 } 358 359 /* 360 * Guarantees nothing is using the device before it's deleted. 361 */ 362 int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred) 363 { 364 int r = 0; 365 366 spin_lock(&_minor_lock); 367 368 if (dm_open_count(md)) { 369 r = -EBUSY; 370 if (mark_deferred) 371 set_bit(DMF_DEFERRED_REMOVE, &md->flags); 372 } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags)) 373 r = -EEXIST; 374 else 375 set_bit(DMF_DELETING, &md->flags); 376 377 spin_unlock(&_minor_lock); 378 379 return r; 380 } 381 382 int dm_cancel_deferred_remove(struct mapped_device *md) 383 { 384 int r = 0; 385 386 spin_lock(&_minor_lock); 387 388 if (test_bit(DMF_DELETING, &md->flags)) 389 r = -EBUSY; 390 else 391 clear_bit(DMF_DEFERRED_REMOVE, &md->flags); 392 393 spin_unlock(&_minor_lock); 394 395 return r; 396 } 397 398 static void do_deferred_remove(struct work_struct *w) 399 { 400 dm_deferred_remove(); 401 } 402 403 static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo) 404 { 405 struct mapped_device *md = bdev->bd_disk->private_data; 406 407 return dm_get_geometry(md, geo); 408 } 409 410 static int dm_prepare_ioctl(struct mapped_device *md, int *srcu_idx, 411 struct block_device **bdev) 412 { 413 struct dm_target *tgt; 414 struct dm_table *map; 415 int r; 416 417 retry: 418 r = -ENOTTY; 419 map = dm_get_live_table(md, srcu_idx); 420 if (!map || !dm_table_get_size(map)) 421 return r; 422 423 /* We only support devices that have a single target */ 424 if (dm_table_get_num_targets(map) != 1) 425 return r; 426 427 tgt = dm_table_get_target(map, 0); 428 if (!tgt->type->prepare_ioctl) 429 return r; 430 431 if (dm_suspended_md(md)) 432 return -EAGAIN; 433 434 r = tgt->type->prepare_ioctl(tgt, bdev); 435 if (r == -ENOTCONN && !fatal_signal_pending(current)) { 436 dm_put_live_table(md, *srcu_idx); 437 msleep(10); 438 goto retry; 439 } 440 441 return r; 442 } 443 444 static void dm_unprepare_ioctl(struct mapped_device *md, int srcu_idx) 445 { 446 dm_put_live_table(md, srcu_idx); 447 } 448 449 static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode, 450 unsigned int cmd, unsigned long arg) 451 { 452 struct mapped_device *md = bdev->bd_disk->private_data; 453 int r, srcu_idx; 454 455 r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 456 if (r < 0) 457 goto out; 458 459 if (r > 0) { 460 /* 461 * Target determined this ioctl is being issued against a 462 * subset of the parent bdev; require extra privileges. 463 */ 464 if (!capable(CAP_SYS_RAWIO)) { 465 DMDEBUG_LIMIT( 466 "%s: sending ioctl %x to DM device without required privilege.", 467 current->comm, cmd); 468 r = -ENOIOCTLCMD; 469 goto out; 470 } 471 } 472 473 if (!bdev->bd_disk->fops->ioctl) 474 r = -ENOTTY; 475 else 476 r = bdev->bd_disk->fops->ioctl(bdev, mode, cmd, arg); 477 out: 478 dm_unprepare_ioctl(md, srcu_idx); 479 return r; 480 } 481 482 u64 dm_start_time_ns_from_clone(struct bio *bio) 483 { 484 return jiffies_to_nsecs(clone_to_tio(bio)->io->start_time); 485 } 486 EXPORT_SYMBOL_GPL(dm_start_time_ns_from_clone); 487 488 static bool bio_is_flush_with_data(struct bio *bio) 489 { 490 return ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size); 491 } 492 493 static void dm_io_acct(bool end, struct mapped_device *md, struct bio *bio, 494 unsigned long start_time, struct dm_stats_aux *stats_aux) 495 { 496 bool is_flush_with_data; 497 unsigned int bi_size; 498 499 /* If REQ_PREFLUSH set save any payload but do not account it */ 500 is_flush_with_data = bio_is_flush_with_data(bio); 501 if (is_flush_with_data) { 502 bi_size = bio->bi_iter.bi_size; 503 bio->bi_iter.bi_size = 0; 504 } 505 506 if (!end) 507 bio_start_io_acct_time(bio, start_time); 508 else 509 bio_end_io_acct(bio, start_time); 510 511 if (unlikely(dm_stats_used(&md->stats))) 512 dm_stats_account_io(&md->stats, bio_data_dir(bio), 513 bio->bi_iter.bi_sector, bio_sectors(bio), 514 end, start_time, stats_aux); 515 516 /* Restore bio's payload so it does get accounted upon requeue */ 517 if (is_flush_with_data) 518 bio->bi_iter.bi_size = bi_size; 519 } 520 521 static void dm_start_io_acct(struct dm_io *io) 522 { 523 dm_io_acct(false, io->md, io->orig_bio, io->start_time, &io->stats_aux); 524 } 525 526 static void dm_end_io_acct(struct dm_io *io) 527 { 528 dm_io_acct(true, io->md, io->orig_bio, io->start_time, &io->stats_aux); 529 } 530 531 static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio) 532 { 533 struct dm_io *io; 534 struct dm_target_io *tio; 535 struct bio *clone; 536 537 clone = bio_alloc_clone(bio->bi_bdev, bio, GFP_NOIO, &md->io_bs); 538 539 tio = clone_to_tio(clone); 540 tio->inside_dm_io = true; 541 tio->io = NULL; 542 543 io = container_of(tio, struct dm_io, tio); 544 io->magic = DM_IO_MAGIC; 545 io->status = 0; 546 atomic_set(&io->io_count, 1); 547 this_cpu_inc(*md->pending_io); 548 io->orig_bio = bio; 549 io->md = md; 550 spin_lock_init(&io->endio_lock); 551 552 io->start_time = jiffies; 553 554 dm_stats_record_start(&md->stats, &io->stats_aux); 555 556 return io; 557 } 558 559 static void free_io(struct dm_io *io) 560 { 561 bio_put(&io->tio.clone); 562 } 563 564 static struct bio *alloc_tio(struct clone_info *ci, struct dm_target *ti, 565 unsigned target_bio_nr, unsigned *len, gfp_t gfp_mask) 566 { 567 struct dm_target_io *tio; 568 569 if (!ci->io->tio.io) { 570 /* the dm_target_io embedded in ci->io is available */ 571 tio = &ci->io->tio; 572 } else { 573 struct bio *clone = bio_alloc_clone(ci->bio->bi_bdev, ci->bio, 574 gfp_mask, &ci->io->md->bs); 575 if (!clone) 576 return NULL; 577 578 tio = clone_to_tio(clone); 579 tio->inside_dm_io = false; 580 } 581 582 tio->magic = DM_TIO_MAGIC; 583 tio->io = ci->io; 584 tio->ti = ti; 585 tio->target_bio_nr = target_bio_nr; 586 tio->len_ptr = len; 587 tio->old_sector = 0; 588 589 return &tio->clone; 590 } 591 592 static void free_tio(struct bio *clone) 593 { 594 if (clone_to_tio(clone)->inside_dm_io) 595 return; 596 bio_put(clone); 597 } 598 599 /* 600 * Add the bio to the list of deferred io. 601 */ 602 static void queue_io(struct mapped_device *md, struct bio *bio) 603 { 604 unsigned long flags; 605 606 spin_lock_irqsave(&md->deferred_lock, flags); 607 bio_list_add(&md->deferred, bio); 608 spin_unlock_irqrestore(&md->deferred_lock, flags); 609 queue_work(md->wq, &md->work); 610 } 611 612 /* 613 * Everyone (including functions in this file), should use this 614 * function to access the md->map field, and make sure they call 615 * dm_put_live_table() when finished. 616 */ 617 struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier) 618 { 619 *srcu_idx = srcu_read_lock(&md->io_barrier); 620 621 return srcu_dereference(md->map, &md->io_barrier); 622 } 623 624 void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier) 625 { 626 srcu_read_unlock(&md->io_barrier, srcu_idx); 627 } 628 629 void dm_sync_table(struct mapped_device *md) 630 { 631 synchronize_srcu(&md->io_barrier); 632 synchronize_rcu_expedited(); 633 } 634 635 /* 636 * A fast alternative to dm_get_live_table/dm_put_live_table. 637 * The caller must not block between these two functions. 638 */ 639 static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU) 640 { 641 rcu_read_lock(); 642 return rcu_dereference(md->map); 643 } 644 645 static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU) 646 { 647 rcu_read_unlock(); 648 } 649 650 static char *_dm_claim_ptr = "I belong to device-mapper"; 651 652 /* 653 * Open a table device so we can use it as a map destination. 654 */ 655 static int open_table_device(struct table_device *td, dev_t dev, 656 struct mapped_device *md) 657 { 658 struct block_device *bdev; 659 u64 part_off; 660 int r; 661 662 BUG_ON(td->dm_dev.bdev); 663 664 bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _dm_claim_ptr); 665 if (IS_ERR(bdev)) 666 return PTR_ERR(bdev); 667 668 r = bd_link_disk_holder(bdev, dm_disk(md)); 669 if (r) { 670 blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL); 671 return r; 672 } 673 674 td->dm_dev.bdev = bdev; 675 td->dm_dev.dax_dev = fs_dax_get_by_bdev(bdev, &part_off); 676 return 0; 677 } 678 679 /* 680 * Close a table device that we've been using. 681 */ 682 static void close_table_device(struct table_device *td, struct mapped_device *md) 683 { 684 if (!td->dm_dev.bdev) 685 return; 686 687 bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md)); 688 blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL); 689 put_dax(td->dm_dev.dax_dev); 690 td->dm_dev.bdev = NULL; 691 td->dm_dev.dax_dev = NULL; 692 } 693 694 static struct table_device *find_table_device(struct list_head *l, dev_t dev, 695 fmode_t mode) 696 { 697 struct table_device *td; 698 699 list_for_each_entry(td, l, list) 700 if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode) 701 return td; 702 703 return NULL; 704 } 705 706 int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode, 707 struct dm_dev **result) 708 { 709 int r; 710 struct table_device *td; 711 712 mutex_lock(&md->table_devices_lock); 713 td = find_table_device(&md->table_devices, dev, mode); 714 if (!td) { 715 td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id); 716 if (!td) { 717 mutex_unlock(&md->table_devices_lock); 718 return -ENOMEM; 719 } 720 721 td->dm_dev.mode = mode; 722 td->dm_dev.bdev = NULL; 723 724 if ((r = open_table_device(td, dev, md))) { 725 mutex_unlock(&md->table_devices_lock); 726 kfree(td); 727 return r; 728 } 729 730 format_dev_t(td->dm_dev.name, dev); 731 732 refcount_set(&td->count, 1); 733 list_add(&td->list, &md->table_devices); 734 } else { 735 refcount_inc(&td->count); 736 } 737 mutex_unlock(&md->table_devices_lock); 738 739 *result = &td->dm_dev; 740 return 0; 741 } 742 743 void dm_put_table_device(struct mapped_device *md, struct dm_dev *d) 744 { 745 struct table_device *td = container_of(d, struct table_device, dm_dev); 746 747 mutex_lock(&md->table_devices_lock); 748 if (refcount_dec_and_test(&td->count)) { 749 close_table_device(td, md); 750 list_del(&td->list); 751 kfree(td); 752 } 753 mutex_unlock(&md->table_devices_lock); 754 } 755 756 static void free_table_devices(struct list_head *devices) 757 { 758 struct list_head *tmp, *next; 759 760 list_for_each_safe(tmp, next, devices) { 761 struct table_device *td = list_entry(tmp, struct table_device, list); 762 763 DMWARN("dm_destroy: %s still exists with %d references", 764 td->dm_dev.name, refcount_read(&td->count)); 765 kfree(td); 766 } 767 } 768 769 /* 770 * Get the geometry associated with a dm device 771 */ 772 int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo) 773 { 774 *geo = md->geometry; 775 776 return 0; 777 } 778 779 /* 780 * Set the geometry of a device. 781 */ 782 int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo) 783 { 784 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors; 785 786 if (geo->start > sz) { 787 DMWARN("Start sector is beyond the geometry limits."); 788 return -EINVAL; 789 } 790 791 md->geometry = *geo; 792 793 return 0; 794 } 795 796 static int __noflush_suspending(struct mapped_device *md) 797 { 798 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 799 } 800 801 /* 802 * Decrements the number of outstanding ios that a bio has been 803 * cloned into, completing the original io if necc. 804 */ 805 void dm_io_dec_pending(struct dm_io *io, blk_status_t error) 806 { 807 unsigned long flags; 808 blk_status_t io_error; 809 struct bio *bio; 810 struct mapped_device *md = io->md; 811 812 /* Push-back supersedes any I/O errors */ 813 if (unlikely(error)) { 814 spin_lock_irqsave(&io->endio_lock, flags); 815 if (!(io->status == BLK_STS_DM_REQUEUE && __noflush_suspending(md))) 816 io->status = error; 817 spin_unlock_irqrestore(&io->endio_lock, flags); 818 } 819 820 if (atomic_dec_and_test(&io->io_count)) { 821 bio = io->orig_bio; 822 if (io->status == BLK_STS_DM_REQUEUE) { 823 /* 824 * Target requested pushing back the I/O. 825 */ 826 spin_lock_irqsave(&md->deferred_lock, flags); 827 if (__noflush_suspending(md) && 828 !WARN_ON_ONCE(dm_is_zone_write(md, bio))) { 829 /* NOTE early return due to BLK_STS_DM_REQUEUE below */ 830 bio_list_add_head(&md->deferred, bio); 831 } else { 832 /* 833 * noflush suspend was interrupted or this is 834 * a write to a zoned target. 835 */ 836 io->status = BLK_STS_IOERR; 837 } 838 spin_unlock_irqrestore(&md->deferred_lock, flags); 839 } 840 841 io_error = io->status; 842 dm_end_io_acct(io); 843 free_io(io); 844 smp_wmb(); 845 this_cpu_dec(*md->pending_io); 846 847 /* nudge anyone waiting on suspend queue */ 848 if (unlikely(wq_has_sleeper(&md->wait))) 849 wake_up(&md->wait); 850 851 if (io_error == BLK_STS_DM_REQUEUE) 852 return; 853 854 if (bio_is_flush_with_data(bio)) { 855 /* 856 * Preflush done for flush with data, reissue 857 * without REQ_PREFLUSH. 858 */ 859 bio->bi_opf &= ~REQ_PREFLUSH; 860 queue_io(md, bio); 861 } else { 862 /* done with normal IO or empty flush */ 863 if (io_error) 864 bio->bi_status = io_error; 865 bio_endio(bio); 866 } 867 } 868 } 869 870 void disable_discard(struct mapped_device *md) 871 { 872 struct queue_limits *limits = dm_get_queue_limits(md); 873 874 /* device doesn't really support DISCARD, disable it */ 875 limits->max_discard_sectors = 0; 876 blk_queue_flag_clear(QUEUE_FLAG_DISCARD, md->queue); 877 } 878 879 void disable_write_same(struct mapped_device *md) 880 { 881 struct queue_limits *limits = dm_get_queue_limits(md); 882 883 /* device doesn't really support WRITE SAME, disable it */ 884 limits->max_write_same_sectors = 0; 885 } 886 887 void disable_write_zeroes(struct mapped_device *md) 888 { 889 struct queue_limits *limits = dm_get_queue_limits(md); 890 891 /* device doesn't really support WRITE ZEROES, disable it */ 892 limits->max_write_zeroes_sectors = 0; 893 } 894 895 static bool swap_bios_limit(struct dm_target *ti, struct bio *bio) 896 { 897 return unlikely((bio->bi_opf & REQ_SWAP) != 0) && unlikely(ti->limit_swap_bios); 898 } 899 900 static void clone_endio(struct bio *bio) 901 { 902 blk_status_t error = bio->bi_status; 903 struct dm_target_io *tio = clone_to_tio(bio); 904 struct dm_io *io = tio->io; 905 struct mapped_device *md = tio->io->md; 906 dm_endio_fn endio = tio->ti->type->end_io; 907 struct request_queue *q = bio->bi_bdev->bd_disk->queue; 908 909 if (unlikely(error == BLK_STS_TARGET)) { 910 if (bio_op(bio) == REQ_OP_DISCARD && 911 !q->limits.max_discard_sectors) 912 disable_discard(md); 913 else if (bio_op(bio) == REQ_OP_WRITE_SAME && 914 !q->limits.max_write_same_sectors) 915 disable_write_same(md); 916 else if (bio_op(bio) == REQ_OP_WRITE_ZEROES && 917 !q->limits.max_write_zeroes_sectors) 918 disable_write_zeroes(md); 919 } 920 921 if (blk_queue_is_zoned(q)) 922 dm_zone_endio(io, bio); 923 924 if (endio) { 925 int r = endio(tio->ti, bio, &error); 926 switch (r) { 927 case DM_ENDIO_REQUEUE: 928 /* 929 * Requeuing writes to a sequential zone of a zoned 930 * target will break the sequential write pattern: 931 * fail such IO. 932 */ 933 if (WARN_ON_ONCE(dm_is_zone_write(md, bio))) 934 error = BLK_STS_IOERR; 935 else 936 error = BLK_STS_DM_REQUEUE; 937 fallthrough; 938 case DM_ENDIO_DONE: 939 break; 940 case DM_ENDIO_INCOMPLETE: 941 /* The target will handle the io */ 942 return; 943 default: 944 DMWARN("unimplemented target endio return value: %d", r); 945 BUG(); 946 } 947 } 948 949 if (unlikely(swap_bios_limit(tio->ti, bio))) { 950 struct mapped_device *md = io->md; 951 up(&md->swap_bios_semaphore); 952 } 953 954 free_tio(bio); 955 dm_io_dec_pending(io, error); 956 } 957 958 /* 959 * Return maximum size of I/O possible at the supplied sector up to the current 960 * target boundary. 961 */ 962 static inline sector_t max_io_len_target_boundary(struct dm_target *ti, 963 sector_t target_offset) 964 { 965 return ti->len - target_offset; 966 } 967 968 static sector_t max_io_len(struct dm_target *ti, sector_t sector) 969 { 970 sector_t target_offset = dm_target_offset(ti, sector); 971 sector_t len = max_io_len_target_boundary(ti, target_offset); 972 sector_t max_len; 973 974 /* 975 * Does the target need to split IO even further? 976 * - varied (per target) IO splitting is a tenet of DM; this 977 * explains why stacked chunk_sectors based splitting via 978 * blk_max_size_offset() isn't possible here. So pass in 979 * ti->max_io_len to override stacked chunk_sectors. 980 */ 981 if (ti->max_io_len) { 982 max_len = blk_max_size_offset(ti->table->md->queue, 983 target_offset, ti->max_io_len); 984 if (len > max_len) 985 len = max_len; 986 } 987 988 return len; 989 } 990 991 int dm_set_target_max_io_len(struct dm_target *ti, sector_t len) 992 { 993 if (len > UINT_MAX) { 994 DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)", 995 (unsigned long long)len, UINT_MAX); 996 ti->error = "Maximum size of target IO is too large"; 997 return -EINVAL; 998 } 999 1000 ti->max_io_len = (uint32_t) len; 1001 1002 return 0; 1003 } 1004 EXPORT_SYMBOL_GPL(dm_set_target_max_io_len); 1005 1006 static struct dm_target *dm_dax_get_live_target(struct mapped_device *md, 1007 sector_t sector, int *srcu_idx) 1008 __acquires(md->io_barrier) 1009 { 1010 struct dm_table *map; 1011 struct dm_target *ti; 1012 1013 map = dm_get_live_table(md, srcu_idx); 1014 if (!map) 1015 return NULL; 1016 1017 ti = dm_table_find_target(map, sector); 1018 if (!ti) 1019 return NULL; 1020 1021 return ti; 1022 } 1023 1024 static long dm_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, 1025 long nr_pages, void **kaddr, pfn_t *pfn) 1026 { 1027 struct mapped_device *md = dax_get_private(dax_dev); 1028 sector_t sector = pgoff * PAGE_SECTORS; 1029 struct dm_target *ti; 1030 long len, ret = -EIO; 1031 int srcu_idx; 1032 1033 ti = dm_dax_get_live_target(md, sector, &srcu_idx); 1034 1035 if (!ti) 1036 goto out; 1037 if (!ti->type->direct_access) 1038 goto out; 1039 len = max_io_len(ti, sector) / PAGE_SECTORS; 1040 if (len < 1) 1041 goto out; 1042 nr_pages = min(len, nr_pages); 1043 ret = ti->type->direct_access(ti, pgoff, nr_pages, kaddr, pfn); 1044 1045 out: 1046 dm_put_live_table(md, srcu_idx); 1047 1048 return ret; 1049 } 1050 1051 static int dm_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, 1052 size_t nr_pages) 1053 { 1054 struct mapped_device *md = dax_get_private(dax_dev); 1055 sector_t sector = pgoff * PAGE_SECTORS; 1056 struct dm_target *ti; 1057 int ret = -EIO; 1058 int srcu_idx; 1059 1060 ti = dm_dax_get_live_target(md, sector, &srcu_idx); 1061 1062 if (!ti) 1063 goto out; 1064 if (WARN_ON(!ti->type->dax_zero_page_range)) { 1065 /* 1066 * ->zero_page_range() is mandatory dax operation. If we are 1067 * here, something is wrong. 1068 */ 1069 goto out; 1070 } 1071 ret = ti->type->dax_zero_page_range(ti, pgoff, nr_pages); 1072 out: 1073 dm_put_live_table(md, srcu_idx); 1074 1075 return ret; 1076 } 1077 1078 /* 1079 * A target may call dm_accept_partial_bio only from the map routine. It is 1080 * allowed for all bio types except REQ_PREFLUSH, REQ_OP_ZONE_* zone management 1081 * operations and REQ_OP_ZONE_APPEND (zone append writes). 1082 * 1083 * dm_accept_partial_bio informs the dm that the target only wants to process 1084 * additional n_sectors sectors of the bio and the rest of the data should be 1085 * sent in a next bio. 1086 * 1087 * A diagram that explains the arithmetics: 1088 * +--------------------+---------------+-------+ 1089 * | 1 | 2 | 3 | 1090 * +--------------------+---------------+-------+ 1091 * 1092 * <-------------- *tio->len_ptr ---------------> 1093 * <------- bi_size -------> 1094 * <-- n_sectors --> 1095 * 1096 * Region 1 was already iterated over with bio_advance or similar function. 1097 * (it may be empty if the target doesn't use bio_advance) 1098 * Region 2 is the remaining bio size that the target wants to process. 1099 * (it may be empty if region 1 is non-empty, although there is no reason 1100 * to make it empty) 1101 * The target requires that region 3 is to be sent in the next bio. 1102 * 1103 * If the target wants to receive multiple copies of the bio (via num_*bios, etc), 1104 * the partially processed part (the sum of regions 1+2) must be the same for all 1105 * copies of the bio. 1106 */ 1107 void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors) 1108 { 1109 struct dm_target_io *tio = clone_to_tio(bio); 1110 unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT; 1111 1112 BUG_ON(bio->bi_opf & REQ_PREFLUSH); 1113 BUG_ON(op_is_zone_mgmt(bio_op(bio))); 1114 BUG_ON(bio_op(bio) == REQ_OP_ZONE_APPEND); 1115 BUG_ON(bi_size > *tio->len_ptr); 1116 BUG_ON(n_sectors > bi_size); 1117 1118 *tio->len_ptr -= bi_size - n_sectors; 1119 bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT; 1120 } 1121 EXPORT_SYMBOL_GPL(dm_accept_partial_bio); 1122 1123 static noinline void __set_swap_bios_limit(struct mapped_device *md, int latch) 1124 { 1125 mutex_lock(&md->swap_bios_lock); 1126 while (latch < md->swap_bios) { 1127 cond_resched(); 1128 down(&md->swap_bios_semaphore); 1129 md->swap_bios--; 1130 } 1131 while (latch > md->swap_bios) { 1132 cond_resched(); 1133 up(&md->swap_bios_semaphore); 1134 md->swap_bios++; 1135 } 1136 mutex_unlock(&md->swap_bios_lock); 1137 } 1138 1139 static void __map_bio(struct bio *clone) 1140 { 1141 struct dm_target_io *tio = clone_to_tio(clone); 1142 int r; 1143 struct dm_io *io = tio->io; 1144 struct dm_target *ti = tio->ti; 1145 1146 clone->bi_end_io = clone_endio; 1147 1148 /* 1149 * Map the clone. If r == 0 we don't need to do 1150 * anything, the target has assumed ownership of 1151 * this io. 1152 */ 1153 dm_io_inc_pending(io); 1154 tio->old_sector = clone->bi_iter.bi_sector; 1155 1156 if (unlikely(swap_bios_limit(ti, clone))) { 1157 struct mapped_device *md = io->md; 1158 int latch = get_swap_bios(); 1159 if (unlikely(latch != md->swap_bios)) 1160 __set_swap_bios_limit(md, latch); 1161 down(&md->swap_bios_semaphore); 1162 } 1163 1164 /* 1165 * Check if the IO needs a special mapping due to zone append emulation 1166 * on zoned target. In this case, dm_zone_map_bio() calls the target 1167 * map operation. 1168 */ 1169 if (dm_emulate_zone_append(io->md)) 1170 r = dm_zone_map_bio(tio); 1171 else 1172 r = ti->type->map(ti, clone); 1173 1174 switch (r) { 1175 case DM_MAPIO_SUBMITTED: 1176 break; 1177 case DM_MAPIO_REMAPPED: 1178 /* the bio has been remapped so dispatch it */ 1179 trace_block_bio_remap(clone, bio_dev(io->orig_bio), 1180 tio->old_sector); 1181 submit_bio_noacct(clone); 1182 break; 1183 case DM_MAPIO_KILL: 1184 case DM_MAPIO_REQUEUE: 1185 if (unlikely(swap_bios_limit(ti, clone))) 1186 up(&io->md->swap_bios_semaphore); 1187 free_tio(clone); 1188 if (r == DM_MAPIO_KILL) 1189 dm_io_dec_pending(io, BLK_STS_IOERR); 1190 else 1191 dm_io_dec_pending(io, BLK_STS_DM_REQUEUE); 1192 break; 1193 default: 1194 DMWARN("unimplemented target map return value: %d", r); 1195 BUG(); 1196 } 1197 } 1198 1199 static void alloc_multiple_bios(struct bio_list *blist, struct clone_info *ci, 1200 struct dm_target *ti, unsigned num_bios, 1201 unsigned *len) 1202 { 1203 struct bio *bio; 1204 int try; 1205 1206 for (try = 0; try < 2; try++) { 1207 int bio_nr; 1208 1209 if (try) 1210 mutex_lock(&ci->io->md->table_devices_lock); 1211 for (bio_nr = 0; bio_nr < num_bios; bio_nr++) { 1212 bio = alloc_tio(ci, ti, bio_nr, len, 1213 try ? GFP_NOIO : GFP_NOWAIT); 1214 if (!bio) 1215 break; 1216 1217 bio_list_add(blist, bio); 1218 } 1219 if (try) 1220 mutex_unlock(&ci->io->md->table_devices_lock); 1221 if (bio_nr == num_bios) 1222 return; 1223 1224 while ((bio = bio_list_pop(blist))) 1225 free_tio(bio); 1226 } 1227 } 1228 1229 static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti, 1230 unsigned num_bios, unsigned *len) 1231 { 1232 struct bio_list blist = BIO_EMPTY_LIST; 1233 struct bio *clone; 1234 1235 switch (num_bios) { 1236 case 0: 1237 break; 1238 case 1: 1239 clone = alloc_tio(ci, ti, 0, len, GFP_NOIO); 1240 if (len) 1241 clone->bi_iter.bi_size = to_bytes(*len); 1242 __map_bio(clone); 1243 break; 1244 default: 1245 alloc_multiple_bios(&blist, ci, ti, num_bios, len); 1246 while ((clone = bio_list_pop(&blist))) { 1247 if (len) 1248 clone->bi_iter.bi_size = to_bytes(*len); 1249 __map_bio(clone); 1250 } 1251 break; 1252 } 1253 } 1254 1255 static int __send_empty_flush(struct clone_info *ci) 1256 { 1257 unsigned target_nr = 0; 1258 struct dm_target *ti; 1259 struct bio flush_bio; 1260 1261 /* 1262 * Use an on-stack bio for this, it's safe since we don't 1263 * need to reference it after submit. It's just used as 1264 * the basis for the clone(s). 1265 */ 1266 bio_init(&flush_bio, ci->io->md->disk->part0, NULL, 0, 1267 REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC); 1268 1269 ci->bio = &flush_bio; 1270 ci->sector_count = 0; 1271 1272 while ((ti = dm_table_get_target(ci->map, target_nr++))) 1273 __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL); 1274 1275 bio_uninit(ci->bio); 1276 return 0; 1277 } 1278 1279 static int __send_changing_extent_only(struct clone_info *ci, struct dm_target *ti, 1280 unsigned num_bios) 1281 { 1282 unsigned len; 1283 1284 /* 1285 * Even though the device advertised support for this type of 1286 * request, that does not mean every target supports it, and 1287 * reconfiguration might also have changed that since the 1288 * check was performed. 1289 */ 1290 if (!num_bios) 1291 return -EOPNOTSUPP; 1292 1293 len = min_t(sector_t, ci->sector_count, 1294 max_io_len_target_boundary(ti, dm_target_offset(ti, ci->sector))); 1295 1296 __send_duplicate_bios(ci, ti, num_bios, &len); 1297 1298 ci->sector += len; 1299 ci->sector_count -= len; 1300 1301 return 0; 1302 } 1303 1304 static bool is_abnormal_io(struct bio *bio) 1305 { 1306 bool r = false; 1307 1308 switch (bio_op(bio)) { 1309 case REQ_OP_DISCARD: 1310 case REQ_OP_SECURE_ERASE: 1311 case REQ_OP_WRITE_SAME: 1312 case REQ_OP_WRITE_ZEROES: 1313 r = true; 1314 break; 1315 } 1316 1317 return r; 1318 } 1319 1320 static bool __process_abnormal_io(struct clone_info *ci, struct dm_target *ti, 1321 int *result) 1322 { 1323 struct bio *bio = ci->bio; 1324 unsigned num_bios = 0; 1325 1326 switch (bio_op(bio)) { 1327 case REQ_OP_DISCARD: 1328 num_bios = ti->num_discard_bios; 1329 break; 1330 case REQ_OP_SECURE_ERASE: 1331 num_bios = ti->num_secure_erase_bios; 1332 break; 1333 case REQ_OP_WRITE_SAME: 1334 num_bios = ti->num_write_same_bios; 1335 break; 1336 case REQ_OP_WRITE_ZEROES: 1337 num_bios = ti->num_write_zeroes_bios; 1338 break; 1339 default: 1340 return false; 1341 } 1342 1343 *result = __send_changing_extent_only(ci, ti, num_bios); 1344 return true; 1345 } 1346 1347 /* 1348 * Select the correct strategy for processing a non-flush bio. 1349 */ 1350 static int __split_and_process_bio(struct clone_info *ci) 1351 { 1352 struct bio *clone; 1353 struct dm_target *ti; 1354 unsigned len; 1355 int r; 1356 1357 ti = dm_table_find_target(ci->map, ci->sector); 1358 if (!ti) 1359 return -EIO; 1360 1361 if (__process_abnormal_io(ci, ti, &r)) 1362 return r; 1363 1364 len = min_t(sector_t, max_io_len(ti, ci->sector), ci->sector_count); 1365 1366 clone = alloc_tio(ci, ti, 0, &len, GFP_NOIO); 1367 clone->bi_iter.bi_size = to_bytes(len); 1368 if (bio_integrity(clone)) 1369 bio_integrity_trim(clone); 1370 1371 __map_bio(clone); 1372 1373 ci->sector += len; 1374 ci->sector_count -= len; 1375 1376 return 0; 1377 } 1378 1379 static void init_clone_info(struct clone_info *ci, struct mapped_device *md, 1380 struct dm_table *map, struct bio *bio) 1381 { 1382 ci->map = map; 1383 ci->io = alloc_io(md, bio); 1384 ci->bio = bio; 1385 ci->sector = bio->bi_iter.bi_sector; 1386 ci->sector_count = bio_sectors(bio); 1387 1388 /* Shouldn't happen but sector_count was being set to 0 so... */ 1389 if (WARN_ON_ONCE(op_is_zone_mgmt(bio_op(bio)) && ci->sector_count)) 1390 ci->sector_count = 0; 1391 } 1392 1393 /* 1394 * Entry point to split a bio into clones and submit them to the targets. 1395 */ 1396 static void dm_split_and_process_bio(struct mapped_device *md, 1397 struct dm_table *map, struct bio *bio) 1398 { 1399 struct clone_info ci; 1400 struct bio *b; 1401 int error = 0; 1402 1403 init_clone_info(&ci, md, map, bio); 1404 1405 if (bio->bi_opf & REQ_PREFLUSH) { 1406 error = __send_empty_flush(&ci); 1407 /* dm_io_dec_pending submits any data associated with flush */ 1408 goto out; 1409 } 1410 1411 error = __split_and_process_bio(&ci); 1412 if (error || !ci.sector_count) 1413 goto out; 1414 1415 /* 1416 * Remainder must be passed to submit_bio_noacct() so it gets handled 1417 * *after* bios already submitted have been completely processed. 1418 * We take a clone of the original to store in ci.io->orig_bio to be 1419 * used by dm_end_io_acct() and for dm_io_dec_pending() to use for 1420 * completion handling. 1421 */ 1422 b = bio_split(bio, bio_sectors(bio) - ci.sector_count, 1423 GFP_NOIO, &md->queue->bio_split); 1424 ci.io->orig_bio = b; 1425 1426 bio_chain(b, bio); 1427 trace_block_split(b, bio->bi_iter.bi_sector); 1428 submit_bio_noacct(bio); 1429 out: 1430 dm_start_io_acct(ci.io); 1431 /* drop the extra reference count */ 1432 dm_io_dec_pending(ci.io, errno_to_blk_status(error)); 1433 } 1434 1435 static void dm_submit_bio(struct bio *bio) 1436 { 1437 struct mapped_device *md = bio->bi_bdev->bd_disk->private_data; 1438 int srcu_idx; 1439 struct dm_table *map; 1440 1441 map = dm_get_live_table(md, &srcu_idx); 1442 if (unlikely(!map)) { 1443 DMERR_LIMIT("%s: mapping table unavailable, erroring io", 1444 dm_device_name(md)); 1445 bio_io_error(bio); 1446 goto out; 1447 } 1448 1449 /* If suspended, queue this IO for later */ 1450 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) { 1451 if (bio->bi_opf & REQ_NOWAIT) 1452 bio_wouldblock_error(bio); 1453 else if (bio->bi_opf & REQ_RAHEAD) 1454 bio_io_error(bio); 1455 else 1456 queue_io(md, bio); 1457 goto out; 1458 } 1459 1460 /* 1461 * Use blk_queue_split() for abnormal IO (e.g. discard, writesame, etc) 1462 * otherwise associated queue_limits won't be imposed. 1463 */ 1464 if (is_abnormal_io(bio)) 1465 blk_queue_split(&bio); 1466 1467 dm_split_and_process_bio(md, map, bio); 1468 out: 1469 dm_put_live_table(md, srcu_idx); 1470 } 1471 1472 /*----------------------------------------------------------------- 1473 * An IDR is used to keep track of allocated minor numbers. 1474 *---------------------------------------------------------------*/ 1475 static void free_minor(int minor) 1476 { 1477 spin_lock(&_minor_lock); 1478 idr_remove(&_minor_idr, minor); 1479 spin_unlock(&_minor_lock); 1480 } 1481 1482 /* 1483 * See if the device with a specific minor # is free. 1484 */ 1485 static int specific_minor(int minor) 1486 { 1487 int r; 1488 1489 if (minor >= (1 << MINORBITS)) 1490 return -EINVAL; 1491 1492 idr_preload(GFP_KERNEL); 1493 spin_lock(&_minor_lock); 1494 1495 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT); 1496 1497 spin_unlock(&_minor_lock); 1498 idr_preload_end(); 1499 if (r < 0) 1500 return r == -ENOSPC ? -EBUSY : r; 1501 return 0; 1502 } 1503 1504 static int next_free_minor(int *minor) 1505 { 1506 int r; 1507 1508 idr_preload(GFP_KERNEL); 1509 spin_lock(&_minor_lock); 1510 1511 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT); 1512 1513 spin_unlock(&_minor_lock); 1514 idr_preload_end(); 1515 if (r < 0) 1516 return r; 1517 *minor = r; 1518 return 0; 1519 } 1520 1521 static const struct block_device_operations dm_blk_dops; 1522 static const struct block_device_operations dm_rq_blk_dops; 1523 static const struct dax_operations dm_dax_ops; 1524 1525 static void dm_wq_work(struct work_struct *work); 1526 1527 #ifdef CONFIG_BLK_INLINE_ENCRYPTION 1528 static void dm_queue_destroy_crypto_profile(struct request_queue *q) 1529 { 1530 dm_destroy_crypto_profile(q->crypto_profile); 1531 } 1532 1533 #else /* CONFIG_BLK_INLINE_ENCRYPTION */ 1534 1535 static inline void dm_queue_destroy_crypto_profile(struct request_queue *q) 1536 { 1537 } 1538 #endif /* !CONFIG_BLK_INLINE_ENCRYPTION */ 1539 1540 static void cleanup_mapped_device(struct mapped_device *md) 1541 { 1542 if (md->wq) 1543 destroy_workqueue(md->wq); 1544 bioset_exit(&md->bs); 1545 bioset_exit(&md->io_bs); 1546 1547 if (md->dax_dev) { 1548 dax_remove_host(md->disk); 1549 kill_dax(md->dax_dev); 1550 put_dax(md->dax_dev); 1551 md->dax_dev = NULL; 1552 } 1553 1554 if (md->disk) { 1555 spin_lock(&_minor_lock); 1556 md->disk->private_data = NULL; 1557 spin_unlock(&_minor_lock); 1558 if (dm_get_md_type(md) != DM_TYPE_NONE) { 1559 dm_sysfs_exit(md); 1560 del_gendisk(md->disk); 1561 } 1562 dm_queue_destroy_crypto_profile(md->queue); 1563 blk_cleanup_disk(md->disk); 1564 } 1565 1566 if (md->pending_io) { 1567 free_percpu(md->pending_io); 1568 md->pending_io = NULL; 1569 } 1570 1571 cleanup_srcu_struct(&md->io_barrier); 1572 1573 mutex_destroy(&md->suspend_lock); 1574 mutex_destroy(&md->type_lock); 1575 mutex_destroy(&md->table_devices_lock); 1576 mutex_destroy(&md->swap_bios_lock); 1577 1578 dm_mq_cleanup_mapped_device(md); 1579 dm_cleanup_zoned_dev(md); 1580 } 1581 1582 /* 1583 * Allocate and initialise a blank device with a given minor. 1584 */ 1585 static struct mapped_device *alloc_dev(int minor) 1586 { 1587 int r, numa_node_id = dm_get_numa_node(); 1588 struct mapped_device *md; 1589 void *old_md; 1590 1591 md = kvzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id); 1592 if (!md) { 1593 DMWARN("unable to allocate device, out of memory."); 1594 return NULL; 1595 } 1596 1597 if (!try_module_get(THIS_MODULE)) 1598 goto bad_module_get; 1599 1600 /* get a minor number for the dev */ 1601 if (minor == DM_ANY_MINOR) 1602 r = next_free_minor(&minor); 1603 else 1604 r = specific_minor(minor); 1605 if (r < 0) 1606 goto bad_minor; 1607 1608 r = init_srcu_struct(&md->io_barrier); 1609 if (r < 0) 1610 goto bad_io_barrier; 1611 1612 md->numa_node_id = numa_node_id; 1613 md->init_tio_pdu = false; 1614 md->type = DM_TYPE_NONE; 1615 mutex_init(&md->suspend_lock); 1616 mutex_init(&md->type_lock); 1617 mutex_init(&md->table_devices_lock); 1618 spin_lock_init(&md->deferred_lock); 1619 atomic_set(&md->holders, 1); 1620 atomic_set(&md->open_count, 0); 1621 atomic_set(&md->event_nr, 0); 1622 atomic_set(&md->uevent_seq, 0); 1623 INIT_LIST_HEAD(&md->uevent_list); 1624 INIT_LIST_HEAD(&md->table_devices); 1625 spin_lock_init(&md->uevent_lock); 1626 1627 /* 1628 * default to bio-based until DM table is loaded and md->type 1629 * established. If request-based table is loaded: blk-mq will 1630 * override accordingly. 1631 */ 1632 md->disk = blk_alloc_disk(md->numa_node_id); 1633 if (!md->disk) 1634 goto bad; 1635 md->queue = md->disk->queue; 1636 1637 init_waitqueue_head(&md->wait); 1638 INIT_WORK(&md->work, dm_wq_work); 1639 init_waitqueue_head(&md->eventq); 1640 init_completion(&md->kobj_holder.completion); 1641 1642 md->swap_bios = get_swap_bios(); 1643 sema_init(&md->swap_bios_semaphore, md->swap_bios); 1644 mutex_init(&md->swap_bios_lock); 1645 1646 md->disk->major = _major; 1647 md->disk->first_minor = minor; 1648 md->disk->minors = 1; 1649 md->disk->flags |= GENHD_FL_NO_PART; 1650 md->disk->fops = &dm_blk_dops; 1651 md->disk->queue = md->queue; 1652 md->disk->private_data = md; 1653 sprintf(md->disk->disk_name, "dm-%d", minor); 1654 1655 if (IS_ENABLED(CONFIG_FS_DAX)) { 1656 md->dax_dev = alloc_dax(md, &dm_dax_ops); 1657 if (IS_ERR(md->dax_dev)) { 1658 md->dax_dev = NULL; 1659 goto bad; 1660 } 1661 set_dax_nocache(md->dax_dev); 1662 set_dax_nomc(md->dax_dev); 1663 if (dax_add_host(md->dax_dev, md->disk)) 1664 goto bad; 1665 } 1666 1667 format_dev_t(md->name, MKDEV(_major, minor)); 1668 1669 md->wq = alloc_workqueue("kdmflush/%s", WQ_MEM_RECLAIM, 0, md->name); 1670 if (!md->wq) 1671 goto bad; 1672 1673 md->pending_io = alloc_percpu(unsigned long); 1674 if (!md->pending_io) 1675 goto bad; 1676 1677 dm_stats_init(&md->stats); 1678 1679 /* Populate the mapping, nobody knows we exist yet */ 1680 spin_lock(&_minor_lock); 1681 old_md = idr_replace(&_minor_idr, md, minor); 1682 spin_unlock(&_minor_lock); 1683 1684 BUG_ON(old_md != MINOR_ALLOCED); 1685 1686 return md; 1687 1688 bad: 1689 cleanup_mapped_device(md); 1690 bad_io_barrier: 1691 free_minor(minor); 1692 bad_minor: 1693 module_put(THIS_MODULE); 1694 bad_module_get: 1695 kvfree(md); 1696 return NULL; 1697 } 1698 1699 static void unlock_fs(struct mapped_device *md); 1700 1701 static void free_dev(struct mapped_device *md) 1702 { 1703 int minor = MINOR(disk_devt(md->disk)); 1704 1705 unlock_fs(md); 1706 1707 cleanup_mapped_device(md); 1708 1709 free_table_devices(&md->table_devices); 1710 dm_stats_cleanup(&md->stats); 1711 free_minor(minor); 1712 1713 module_put(THIS_MODULE); 1714 kvfree(md); 1715 } 1716 1717 static int __bind_mempools(struct mapped_device *md, struct dm_table *t) 1718 { 1719 struct dm_md_mempools *p = dm_table_get_md_mempools(t); 1720 int ret = 0; 1721 1722 if (dm_table_bio_based(t)) { 1723 /* 1724 * The md may already have mempools that need changing. 1725 * If so, reload bioset because front_pad may have changed 1726 * because a different table was loaded. 1727 */ 1728 bioset_exit(&md->bs); 1729 bioset_exit(&md->io_bs); 1730 1731 } else if (bioset_initialized(&md->bs)) { 1732 /* 1733 * There's no need to reload with request-based dm 1734 * because the size of front_pad doesn't change. 1735 * Note for future: If you are to reload bioset, 1736 * prep-ed requests in the queue may refer 1737 * to bio from the old bioset, so you must walk 1738 * through the queue to unprep. 1739 */ 1740 goto out; 1741 } 1742 1743 BUG_ON(!p || 1744 bioset_initialized(&md->bs) || 1745 bioset_initialized(&md->io_bs)); 1746 1747 ret = bioset_init_from_src(&md->bs, &p->bs); 1748 if (ret) 1749 goto out; 1750 ret = bioset_init_from_src(&md->io_bs, &p->io_bs); 1751 if (ret) 1752 bioset_exit(&md->bs); 1753 out: 1754 /* mempool bind completed, no longer need any mempools in the table */ 1755 dm_table_free_md_mempools(t); 1756 return ret; 1757 } 1758 1759 /* 1760 * Bind a table to the device. 1761 */ 1762 static void event_callback(void *context) 1763 { 1764 unsigned long flags; 1765 LIST_HEAD(uevents); 1766 struct mapped_device *md = (struct mapped_device *) context; 1767 1768 spin_lock_irqsave(&md->uevent_lock, flags); 1769 list_splice_init(&md->uevent_list, &uevents); 1770 spin_unlock_irqrestore(&md->uevent_lock, flags); 1771 1772 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj); 1773 1774 atomic_inc(&md->event_nr); 1775 wake_up(&md->eventq); 1776 dm_issue_global_event(); 1777 } 1778 1779 /* 1780 * Returns old map, which caller must destroy. 1781 */ 1782 static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t, 1783 struct queue_limits *limits) 1784 { 1785 struct dm_table *old_map; 1786 struct request_queue *q = md->queue; 1787 bool request_based = dm_table_request_based(t); 1788 sector_t size; 1789 int ret; 1790 1791 lockdep_assert_held(&md->suspend_lock); 1792 1793 size = dm_table_get_size(t); 1794 1795 /* 1796 * Wipe any geometry if the size of the table changed. 1797 */ 1798 if (size != dm_get_size(md)) 1799 memset(&md->geometry, 0, sizeof(md->geometry)); 1800 1801 if (!get_capacity(md->disk)) 1802 set_capacity(md->disk, size); 1803 else 1804 set_capacity_and_notify(md->disk, size); 1805 1806 dm_table_event_callback(t, event_callback, md); 1807 1808 if (request_based) { 1809 /* 1810 * Leverage the fact that request-based DM targets are 1811 * immutable singletons - used to optimize dm_mq_queue_rq. 1812 */ 1813 md->immutable_target = dm_table_get_immutable_target(t); 1814 } 1815 1816 ret = __bind_mempools(md, t); 1817 if (ret) { 1818 old_map = ERR_PTR(ret); 1819 goto out; 1820 } 1821 1822 ret = dm_table_set_restrictions(t, q, limits); 1823 if (ret) { 1824 old_map = ERR_PTR(ret); 1825 goto out; 1826 } 1827 1828 old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 1829 rcu_assign_pointer(md->map, (void *)t); 1830 md->immutable_target_type = dm_table_get_immutable_target_type(t); 1831 1832 if (old_map) 1833 dm_sync_table(md); 1834 1835 out: 1836 return old_map; 1837 } 1838 1839 /* 1840 * Returns unbound table for the caller to free. 1841 */ 1842 static struct dm_table *__unbind(struct mapped_device *md) 1843 { 1844 struct dm_table *map = rcu_dereference_protected(md->map, 1); 1845 1846 if (!map) 1847 return NULL; 1848 1849 dm_table_event_callback(map, NULL, NULL); 1850 RCU_INIT_POINTER(md->map, NULL); 1851 dm_sync_table(md); 1852 1853 return map; 1854 } 1855 1856 /* 1857 * Constructor for a new device. 1858 */ 1859 int dm_create(int minor, struct mapped_device **result) 1860 { 1861 struct mapped_device *md; 1862 1863 md = alloc_dev(minor); 1864 if (!md) 1865 return -ENXIO; 1866 1867 dm_ima_reset_data(md); 1868 1869 *result = md; 1870 return 0; 1871 } 1872 1873 /* 1874 * Functions to manage md->type. 1875 * All are required to hold md->type_lock. 1876 */ 1877 void dm_lock_md_type(struct mapped_device *md) 1878 { 1879 mutex_lock(&md->type_lock); 1880 } 1881 1882 void dm_unlock_md_type(struct mapped_device *md) 1883 { 1884 mutex_unlock(&md->type_lock); 1885 } 1886 1887 void dm_set_md_type(struct mapped_device *md, enum dm_queue_mode type) 1888 { 1889 BUG_ON(!mutex_is_locked(&md->type_lock)); 1890 md->type = type; 1891 } 1892 1893 enum dm_queue_mode dm_get_md_type(struct mapped_device *md) 1894 { 1895 return md->type; 1896 } 1897 1898 struct target_type *dm_get_immutable_target_type(struct mapped_device *md) 1899 { 1900 return md->immutable_target_type; 1901 } 1902 1903 /* 1904 * The queue_limits are only valid as long as you have a reference 1905 * count on 'md'. 1906 */ 1907 struct queue_limits *dm_get_queue_limits(struct mapped_device *md) 1908 { 1909 BUG_ON(!atomic_read(&md->holders)); 1910 return &md->queue->limits; 1911 } 1912 EXPORT_SYMBOL_GPL(dm_get_queue_limits); 1913 1914 /* 1915 * Setup the DM device's queue based on md's type 1916 */ 1917 int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t) 1918 { 1919 enum dm_queue_mode type = dm_table_get_type(t); 1920 struct queue_limits limits; 1921 int r; 1922 1923 switch (type) { 1924 case DM_TYPE_REQUEST_BASED: 1925 md->disk->fops = &dm_rq_blk_dops; 1926 r = dm_mq_init_request_queue(md, t); 1927 if (r) { 1928 DMERR("Cannot initialize queue for request-based dm mapped device"); 1929 return r; 1930 } 1931 break; 1932 case DM_TYPE_BIO_BASED: 1933 case DM_TYPE_DAX_BIO_BASED: 1934 break; 1935 case DM_TYPE_NONE: 1936 WARN_ON_ONCE(true); 1937 break; 1938 } 1939 1940 r = dm_calculate_queue_limits(t, &limits); 1941 if (r) { 1942 DMERR("Cannot calculate initial queue limits"); 1943 return r; 1944 } 1945 r = dm_table_set_restrictions(t, md->queue, &limits); 1946 if (r) 1947 return r; 1948 1949 r = add_disk(md->disk); 1950 if (r) 1951 return r; 1952 1953 r = dm_sysfs_init(md); 1954 if (r) { 1955 del_gendisk(md->disk); 1956 return r; 1957 } 1958 md->type = type; 1959 return 0; 1960 } 1961 1962 struct mapped_device *dm_get_md(dev_t dev) 1963 { 1964 struct mapped_device *md; 1965 unsigned minor = MINOR(dev); 1966 1967 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS)) 1968 return NULL; 1969 1970 spin_lock(&_minor_lock); 1971 1972 md = idr_find(&_minor_idr, minor); 1973 if (!md || md == MINOR_ALLOCED || (MINOR(disk_devt(dm_disk(md))) != minor) || 1974 test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) { 1975 md = NULL; 1976 goto out; 1977 } 1978 dm_get(md); 1979 out: 1980 spin_unlock(&_minor_lock); 1981 1982 return md; 1983 } 1984 EXPORT_SYMBOL_GPL(dm_get_md); 1985 1986 void *dm_get_mdptr(struct mapped_device *md) 1987 { 1988 return md->interface_ptr; 1989 } 1990 1991 void dm_set_mdptr(struct mapped_device *md, void *ptr) 1992 { 1993 md->interface_ptr = ptr; 1994 } 1995 1996 void dm_get(struct mapped_device *md) 1997 { 1998 atomic_inc(&md->holders); 1999 BUG_ON(test_bit(DMF_FREEING, &md->flags)); 2000 } 2001 2002 int dm_hold(struct mapped_device *md) 2003 { 2004 spin_lock(&_minor_lock); 2005 if (test_bit(DMF_FREEING, &md->flags)) { 2006 spin_unlock(&_minor_lock); 2007 return -EBUSY; 2008 } 2009 dm_get(md); 2010 spin_unlock(&_minor_lock); 2011 return 0; 2012 } 2013 EXPORT_SYMBOL_GPL(dm_hold); 2014 2015 const char *dm_device_name(struct mapped_device *md) 2016 { 2017 return md->name; 2018 } 2019 EXPORT_SYMBOL_GPL(dm_device_name); 2020 2021 static void __dm_destroy(struct mapped_device *md, bool wait) 2022 { 2023 struct dm_table *map; 2024 int srcu_idx; 2025 2026 might_sleep(); 2027 2028 spin_lock(&_minor_lock); 2029 idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md)))); 2030 set_bit(DMF_FREEING, &md->flags); 2031 spin_unlock(&_minor_lock); 2032 2033 blk_set_queue_dying(md->queue); 2034 2035 /* 2036 * Take suspend_lock so that presuspend and postsuspend methods 2037 * do not race with internal suspend. 2038 */ 2039 mutex_lock(&md->suspend_lock); 2040 map = dm_get_live_table(md, &srcu_idx); 2041 if (!dm_suspended_md(md)) { 2042 dm_table_presuspend_targets(map); 2043 set_bit(DMF_SUSPENDED, &md->flags); 2044 set_bit(DMF_POST_SUSPENDING, &md->flags); 2045 dm_table_postsuspend_targets(map); 2046 } 2047 /* dm_put_live_table must be before msleep, otherwise deadlock is possible */ 2048 dm_put_live_table(md, srcu_idx); 2049 mutex_unlock(&md->suspend_lock); 2050 2051 /* 2052 * Rare, but there may be I/O requests still going to complete, 2053 * for example. Wait for all references to disappear. 2054 * No one should increment the reference count of the mapped_device, 2055 * after the mapped_device state becomes DMF_FREEING. 2056 */ 2057 if (wait) 2058 while (atomic_read(&md->holders)) 2059 msleep(1); 2060 else if (atomic_read(&md->holders)) 2061 DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)", 2062 dm_device_name(md), atomic_read(&md->holders)); 2063 2064 dm_table_destroy(__unbind(md)); 2065 free_dev(md); 2066 } 2067 2068 void dm_destroy(struct mapped_device *md) 2069 { 2070 __dm_destroy(md, true); 2071 } 2072 2073 void dm_destroy_immediate(struct mapped_device *md) 2074 { 2075 __dm_destroy(md, false); 2076 } 2077 2078 void dm_put(struct mapped_device *md) 2079 { 2080 atomic_dec(&md->holders); 2081 } 2082 EXPORT_SYMBOL_GPL(dm_put); 2083 2084 static bool dm_in_flight_bios(struct mapped_device *md) 2085 { 2086 int cpu; 2087 unsigned long sum = 0; 2088 2089 for_each_possible_cpu(cpu) 2090 sum += *per_cpu_ptr(md->pending_io, cpu); 2091 2092 return sum != 0; 2093 } 2094 2095 static int dm_wait_for_bios_completion(struct mapped_device *md, unsigned int task_state) 2096 { 2097 int r = 0; 2098 DEFINE_WAIT(wait); 2099 2100 while (true) { 2101 prepare_to_wait(&md->wait, &wait, task_state); 2102 2103 if (!dm_in_flight_bios(md)) 2104 break; 2105 2106 if (signal_pending_state(task_state, current)) { 2107 r = -EINTR; 2108 break; 2109 } 2110 2111 io_schedule(); 2112 } 2113 finish_wait(&md->wait, &wait); 2114 2115 smp_rmb(); 2116 2117 return r; 2118 } 2119 2120 static int dm_wait_for_completion(struct mapped_device *md, unsigned int task_state) 2121 { 2122 int r = 0; 2123 2124 if (!queue_is_mq(md->queue)) 2125 return dm_wait_for_bios_completion(md, task_state); 2126 2127 while (true) { 2128 if (!blk_mq_queue_inflight(md->queue)) 2129 break; 2130 2131 if (signal_pending_state(task_state, current)) { 2132 r = -EINTR; 2133 break; 2134 } 2135 2136 msleep(5); 2137 } 2138 2139 return r; 2140 } 2141 2142 /* 2143 * Process the deferred bios 2144 */ 2145 static void dm_wq_work(struct work_struct *work) 2146 { 2147 struct mapped_device *md = container_of(work, struct mapped_device, work); 2148 struct bio *bio; 2149 2150 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) { 2151 spin_lock_irq(&md->deferred_lock); 2152 bio = bio_list_pop(&md->deferred); 2153 spin_unlock_irq(&md->deferred_lock); 2154 2155 if (!bio) 2156 break; 2157 2158 submit_bio_noacct(bio); 2159 } 2160 } 2161 2162 static void dm_queue_flush(struct mapped_device *md) 2163 { 2164 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 2165 smp_mb__after_atomic(); 2166 queue_work(md->wq, &md->work); 2167 } 2168 2169 /* 2170 * Swap in a new table, returning the old one for the caller to destroy. 2171 */ 2172 struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table) 2173 { 2174 struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL); 2175 struct queue_limits limits; 2176 int r; 2177 2178 mutex_lock(&md->suspend_lock); 2179 2180 /* device must be suspended */ 2181 if (!dm_suspended_md(md)) 2182 goto out; 2183 2184 /* 2185 * If the new table has no data devices, retain the existing limits. 2186 * This helps multipath with queue_if_no_path if all paths disappear, 2187 * then new I/O is queued based on these limits, and then some paths 2188 * reappear. 2189 */ 2190 if (dm_table_has_no_data_devices(table)) { 2191 live_map = dm_get_live_table_fast(md); 2192 if (live_map) 2193 limits = md->queue->limits; 2194 dm_put_live_table_fast(md); 2195 } 2196 2197 if (!live_map) { 2198 r = dm_calculate_queue_limits(table, &limits); 2199 if (r) { 2200 map = ERR_PTR(r); 2201 goto out; 2202 } 2203 } 2204 2205 map = __bind(md, table, &limits); 2206 dm_issue_global_event(); 2207 2208 out: 2209 mutex_unlock(&md->suspend_lock); 2210 return map; 2211 } 2212 2213 /* 2214 * Functions to lock and unlock any filesystem running on the 2215 * device. 2216 */ 2217 static int lock_fs(struct mapped_device *md) 2218 { 2219 int r; 2220 2221 WARN_ON(test_bit(DMF_FROZEN, &md->flags)); 2222 2223 r = freeze_bdev(md->disk->part0); 2224 if (!r) 2225 set_bit(DMF_FROZEN, &md->flags); 2226 return r; 2227 } 2228 2229 static void unlock_fs(struct mapped_device *md) 2230 { 2231 if (!test_bit(DMF_FROZEN, &md->flags)) 2232 return; 2233 thaw_bdev(md->disk->part0); 2234 clear_bit(DMF_FROZEN, &md->flags); 2235 } 2236 2237 /* 2238 * @suspend_flags: DM_SUSPEND_LOCKFS_FLAG and/or DM_SUSPEND_NOFLUSH_FLAG 2239 * @task_state: e.g. TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE 2240 * @dmf_suspended_flag: DMF_SUSPENDED or DMF_SUSPENDED_INTERNALLY 2241 * 2242 * If __dm_suspend returns 0, the device is completely quiescent 2243 * now. There is no request-processing activity. All new requests 2244 * are being added to md->deferred list. 2245 */ 2246 static int __dm_suspend(struct mapped_device *md, struct dm_table *map, 2247 unsigned suspend_flags, unsigned int task_state, 2248 int dmf_suspended_flag) 2249 { 2250 bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG; 2251 bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG; 2252 int r; 2253 2254 lockdep_assert_held(&md->suspend_lock); 2255 2256 /* 2257 * DMF_NOFLUSH_SUSPENDING must be set before presuspend. 2258 * This flag is cleared before dm_suspend returns. 2259 */ 2260 if (noflush) 2261 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 2262 else 2263 DMDEBUG("%s: suspending with flush", dm_device_name(md)); 2264 2265 /* 2266 * This gets reverted if there's an error later and the targets 2267 * provide the .presuspend_undo hook. 2268 */ 2269 dm_table_presuspend_targets(map); 2270 2271 /* 2272 * Flush I/O to the device. 2273 * Any I/O submitted after lock_fs() may not be flushed. 2274 * noflush takes precedence over do_lockfs. 2275 * (lock_fs() flushes I/Os and waits for them to complete.) 2276 */ 2277 if (!noflush && do_lockfs) { 2278 r = lock_fs(md); 2279 if (r) { 2280 dm_table_presuspend_undo_targets(map); 2281 return r; 2282 } 2283 } 2284 2285 /* 2286 * Here we must make sure that no processes are submitting requests 2287 * to target drivers i.e. no one may be executing 2288 * dm_split_and_process_bio from dm_submit_bio. 2289 * 2290 * To get all processes out of dm_split_and_process_bio in dm_submit_bio, 2291 * we take the write lock. To prevent any process from reentering 2292 * dm_split_and_process_bio from dm_submit_bio and quiesce the thread 2293 * (dm_wq_work), we set DMF_BLOCK_IO_FOR_SUSPEND and call 2294 * flush_workqueue(md->wq). 2295 */ 2296 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 2297 if (map) 2298 synchronize_srcu(&md->io_barrier); 2299 2300 /* 2301 * Stop md->queue before flushing md->wq in case request-based 2302 * dm defers requests to md->wq from md->queue. 2303 */ 2304 if (dm_request_based(md)) 2305 dm_stop_queue(md->queue); 2306 2307 flush_workqueue(md->wq); 2308 2309 /* 2310 * At this point no more requests are entering target request routines. 2311 * We call dm_wait_for_completion to wait for all existing requests 2312 * to finish. 2313 */ 2314 r = dm_wait_for_completion(md, task_state); 2315 if (!r) 2316 set_bit(dmf_suspended_flag, &md->flags); 2317 2318 if (noflush) 2319 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 2320 if (map) 2321 synchronize_srcu(&md->io_barrier); 2322 2323 /* were we interrupted ? */ 2324 if (r < 0) { 2325 dm_queue_flush(md); 2326 2327 if (dm_request_based(md)) 2328 dm_start_queue(md->queue); 2329 2330 unlock_fs(md); 2331 dm_table_presuspend_undo_targets(map); 2332 /* pushback list is already flushed, so skip flush */ 2333 } 2334 2335 return r; 2336 } 2337 2338 /* 2339 * We need to be able to change a mapping table under a mounted 2340 * filesystem. For example we might want to move some data in 2341 * the background. Before the table can be swapped with 2342 * dm_bind_table, dm_suspend must be called to flush any in 2343 * flight bios and ensure that any further io gets deferred. 2344 */ 2345 /* 2346 * Suspend mechanism in request-based dm. 2347 * 2348 * 1. Flush all I/Os by lock_fs() if needed. 2349 * 2. Stop dispatching any I/O by stopping the request_queue. 2350 * 3. Wait for all in-flight I/Os to be completed or requeued. 2351 * 2352 * To abort suspend, start the request_queue. 2353 */ 2354 int dm_suspend(struct mapped_device *md, unsigned suspend_flags) 2355 { 2356 struct dm_table *map = NULL; 2357 int r = 0; 2358 2359 retry: 2360 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); 2361 2362 if (dm_suspended_md(md)) { 2363 r = -EINVAL; 2364 goto out_unlock; 2365 } 2366 2367 if (dm_suspended_internally_md(md)) { 2368 /* already internally suspended, wait for internal resume */ 2369 mutex_unlock(&md->suspend_lock); 2370 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); 2371 if (r) 2372 return r; 2373 goto retry; 2374 } 2375 2376 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2377 2378 r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED); 2379 if (r) 2380 goto out_unlock; 2381 2382 set_bit(DMF_POST_SUSPENDING, &md->flags); 2383 dm_table_postsuspend_targets(map); 2384 clear_bit(DMF_POST_SUSPENDING, &md->flags); 2385 2386 out_unlock: 2387 mutex_unlock(&md->suspend_lock); 2388 return r; 2389 } 2390 2391 static int __dm_resume(struct mapped_device *md, struct dm_table *map) 2392 { 2393 if (map) { 2394 int r = dm_table_resume_targets(map); 2395 if (r) 2396 return r; 2397 } 2398 2399 dm_queue_flush(md); 2400 2401 /* 2402 * Flushing deferred I/Os must be done after targets are resumed 2403 * so that mapping of targets can work correctly. 2404 * Request-based dm is queueing the deferred I/Os in its request_queue. 2405 */ 2406 if (dm_request_based(md)) 2407 dm_start_queue(md->queue); 2408 2409 unlock_fs(md); 2410 2411 return 0; 2412 } 2413 2414 int dm_resume(struct mapped_device *md) 2415 { 2416 int r; 2417 struct dm_table *map = NULL; 2418 2419 retry: 2420 r = -EINVAL; 2421 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); 2422 2423 if (!dm_suspended_md(md)) 2424 goto out; 2425 2426 if (dm_suspended_internally_md(md)) { 2427 /* already internally suspended, wait for internal resume */ 2428 mutex_unlock(&md->suspend_lock); 2429 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); 2430 if (r) 2431 return r; 2432 goto retry; 2433 } 2434 2435 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2436 if (!map || !dm_table_get_size(map)) 2437 goto out; 2438 2439 r = __dm_resume(md, map); 2440 if (r) 2441 goto out; 2442 2443 clear_bit(DMF_SUSPENDED, &md->flags); 2444 out: 2445 mutex_unlock(&md->suspend_lock); 2446 2447 return r; 2448 } 2449 2450 /* 2451 * Internal suspend/resume works like userspace-driven suspend. It waits 2452 * until all bios finish and prevents issuing new bios to the target drivers. 2453 * It may be used only from the kernel. 2454 */ 2455 2456 static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags) 2457 { 2458 struct dm_table *map = NULL; 2459 2460 lockdep_assert_held(&md->suspend_lock); 2461 2462 if (md->internal_suspend_count++) 2463 return; /* nested internal suspend */ 2464 2465 if (dm_suspended_md(md)) { 2466 set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 2467 return; /* nest suspend */ 2468 } 2469 2470 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2471 2472 /* 2473 * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is 2474 * supported. Properly supporting a TASK_INTERRUPTIBLE internal suspend 2475 * would require changing .presuspend to return an error -- avoid this 2476 * until there is a need for more elaborate variants of internal suspend. 2477 */ 2478 (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE, 2479 DMF_SUSPENDED_INTERNALLY); 2480 2481 set_bit(DMF_POST_SUSPENDING, &md->flags); 2482 dm_table_postsuspend_targets(map); 2483 clear_bit(DMF_POST_SUSPENDING, &md->flags); 2484 } 2485 2486 static void __dm_internal_resume(struct mapped_device *md) 2487 { 2488 BUG_ON(!md->internal_suspend_count); 2489 2490 if (--md->internal_suspend_count) 2491 return; /* resume from nested internal suspend */ 2492 2493 if (dm_suspended_md(md)) 2494 goto done; /* resume from nested suspend */ 2495 2496 /* 2497 * NOTE: existing callers don't need to call dm_table_resume_targets 2498 * (which may fail -- so best to avoid it for now by passing NULL map) 2499 */ 2500 (void) __dm_resume(md, NULL); 2501 2502 done: 2503 clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 2504 smp_mb__after_atomic(); 2505 wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY); 2506 } 2507 2508 void dm_internal_suspend_noflush(struct mapped_device *md) 2509 { 2510 mutex_lock(&md->suspend_lock); 2511 __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG); 2512 mutex_unlock(&md->suspend_lock); 2513 } 2514 EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush); 2515 2516 void dm_internal_resume(struct mapped_device *md) 2517 { 2518 mutex_lock(&md->suspend_lock); 2519 __dm_internal_resume(md); 2520 mutex_unlock(&md->suspend_lock); 2521 } 2522 EXPORT_SYMBOL_GPL(dm_internal_resume); 2523 2524 /* 2525 * Fast variants of internal suspend/resume hold md->suspend_lock, 2526 * which prevents interaction with userspace-driven suspend. 2527 */ 2528 2529 void dm_internal_suspend_fast(struct mapped_device *md) 2530 { 2531 mutex_lock(&md->suspend_lock); 2532 if (dm_suspended_md(md) || dm_suspended_internally_md(md)) 2533 return; 2534 2535 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 2536 synchronize_srcu(&md->io_barrier); 2537 flush_workqueue(md->wq); 2538 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE); 2539 } 2540 EXPORT_SYMBOL_GPL(dm_internal_suspend_fast); 2541 2542 void dm_internal_resume_fast(struct mapped_device *md) 2543 { 2544 if (dm_suspended_md(md) || dm_suspended_internally_md(md)) 2545 goto done; 2546 2547 dm_queue_flush(md); 2548 2549 done: 2550 mutex_unlock(&md->suspend_lock); 2551 } 2552 EXPORT_SYMBOL_GPL(dm_internal_resume_fast); 2553 2554 /*----------------------------------------------------------------- 2555 * Event notification. 2556 *---------------------------------------------------------------*/ 2557 int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action, 2558 unsigned cookie) 2559 { 2560 int r; 2561 unsigned noio_flag; 2562 char udev_cookie[DM_COOKIE_LENGTH]; 2563 char *envp[] = { udev_cookie, NULL }; 2564 2565 noio_flag = memalloc_noio_save(); 2566 2567 if (!cookie) 2568 r = kobject_uevent(&disk_to_dev(md->disk)->kobj, action); 2569 else { 2570 snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u", 2571 DM_COOKIE_ENV_VAR_NAME, cookie); 2572 r = kobject_uevent_env(&disk_to_dev(md->disk)->kobj, 2573 action, envp); 2574 } 2575 2576 memalloc_noio_restore(noio_flag); 2577 2578 return r; 2579 } 2580 2581 uint32_t dm_next_uevent_seq(struct mapped_device *md) 2582 { 2583 return atomic_add_return(1, &md->uevent_seq); 2584 } 2585 2586 uint32_t dm_get_event_nr(struct mapped_device *md) 2587 { 2588 return atomic_read(&md->event_nr); 2589 } 2590 2591 int dm_wait_event(struct mapped_device *md, int event_nr) 2592 { 2593 return wait_event_interruptible(md->eventq, 2594 (event_nr != atomic_read(&md->event_nr))); 2595 } 2596 2597 void dm_uevent_add(struct mapped_device *md, struct list_head *elist) 2598 { 2599 unsigned long flags; 2600 2601 spin_lock_irqsave(&md->uevent_lock, flags); 2602 list_add(elist, &md->uevent_list); 2603 spin_unlock_irqrestore(&md->uevent_lock, flags); 2604 } 2605 2606 /* 2607 * The gendisk is only valid as long as you have a reference 2608 * count on 'md'. 2609 */ 2610 struct gendisk *dm_disk(struct mapped_device *md) 2611 { 2612 return md->disk; 2613 } 2614 EXPORT_SYMBOL_GPL(dm_disk); 2615 2616 struct kobject *dm_kobject(struct mapped_device *md) 2617 { 2618 return &md->kobj_holder.kobj; 2619 } 2620 2621 struct mapped_device *dm_get_from_kobject(struct kobject *kobj) 2622 { 2623 struct mapped_device *md; 2624 2625 md = container_of(kobj, struct mapped_device, kobj_holder.kobj); 2626 2627 spin_lock(&_minor_lock); 2628 if (test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) { 2629 md = NULL; 2630 goto out; 2631 } 2632 dm_get(md); 2633 out: 2634 spin_unlock(&_minor_lock); 2635 2636 return md; 2637 } 2638 2639 int dm_suspended_md(struct mapped_device *md) 2640 { 2641 return test_bit(DMF_SUSPENDED, &md->flags); 2642 } 2643 2644 static int dm_post_suspending_md(struct mapped_device *md) 2645 { 2646 return test_bit(DMF_POST_SUSPENDING, &md->flags); 2647 } 2648 2649 int dm_suspended_internally_md(struct mapped_device *md) 2650 { 2651 return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 2652 } 2653 2654 int dm_test_deferred_remove_flag(struct mapped_device *md) 2655 { 2656 return test_bit(DMF_DEFERRED_REMOVE, &md->flags); 2657 } 2658 2659 int dm_suspended(struct dm_target *ti) 2660 { 2661 return dm_suspended_md(ti->table->md); 2662 } 2663 EXPORT_SYMBOL_GPL(dm_suspended); 2664 2665 int dm_post_suspending(struct dm_target *ti) 2666 { 2667 return dm_post_suspending_md(ti->table->md); 2668 } 2669 EXPORT_SYMBOL_GPL(dm_post_suspending); 2670 2671 int dm_noflush_suspending(struct dm_target *ti) 2672 { 2673 return __noflush_suspending(ti->table->md); 2674 } 2675 EXPORT_SYMBOL_GPL(dm_noflush_suspending); 2676 2677 struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_queue_mode type, 2678 unsigned integrity, unsigned per_io_data_size, 2679 unsigned min_pool_size) 2680 { 2681 struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id); 2682 unsigned int pool_size = 0; 2683 unsigned int front_pad, io_front_pad; 2684 int ret; 2685 2686 if (!pools) 2687 return NULL; 2688 2689 switch (type) { 2690 case DM_TYPE_BIO_BASED: 2691 case DM_TYPE_DAX_BIO_BASED: 2692 pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size); 2693 front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + DM_TARGET_IO_BIO_OFFSET; 2694 io_front_pad = roundup(per_io_data_size, __alignof__(struct dm_io)) + DM_IO_BIO_OFFSET; 2695 ret = bioset_init(&pools->io_bs, pool_size, io_front_pad, 0); 2696 if (ret) 2697 goto out; 2698 if (integrity && bioset_integrity_create(&pools->io_bs, pool_size)) 2699 goto out; 2700 break; 2701 case DM_TYPE_REQUEST_BASED: 2702 pool_size = max(dm_get_reserved_rq_based_ios(), min_pool_size); 2703 front_pad = offsetof(struct dm_rq_clone_bio_info, clone); 2704 /* per_io_data_size is used for blk-mq pdu at queue allocation */ 2705 break; 2706 default: 2707 BUG(); 2708 } 2709 2710 ret = bioset_init(&pools->bs, pool_size, front_pad, 0); 2711 if (ret) 2712 goto out; 2713 2714 if (integrity && bioset_integrity_create(&pools->bs, pool_size)) 2715 goto out; 2716 2717 return pools; 2718 2719 out: 2720 dm_free_md_mempools(pools); 2721 2722 return NULL; 2723 } 2724 2725 void dm_free_md_mempools(struct dm_md_mempools *pools) 2726 { 2727 if (!pools) 2728 return; 2729 2730 bioset_exit(&pools->bs); 2731 bioset_exit(&pools->io_bs); 2732 2733 kfree(pools); 2734 } 2735 2736 struct dm_pr { 2737 u64 old_key; 2738 u64 new_key; 2739 u32 flags; 2740 bool fail_early; 2741 }; 2742 2743 static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn, 2744 void *data) 2745 { 2746 struct mapped_device *md = bdev->bd_disk->private_data; 2747 struct dm_table *table; 2748 struct dm_target *ti; 2749 int ret = -ENOTTY, srcu_idx; 2750 2751 table = dm_get_live_table(md, &srcu_idx); 2752 if (!table || !dm_table_get_size(table)) 2753 goto out; 2754 2755 /* We only support devices that have a single target */ 2756 if (dm_table_get_num_targets(table) != 1) 2757 goto out; 2758 ti = dm_table_get_target(table, 0); 2759 2760 ret = -EINVAL; 2761 if (!ti->type->iterate_devices) 2762 goto out; 2763 2764 ret = ti->type->iterate_devices(ti, fn, data); 2765 out: 2766 dm_put_live_table(md, srcu_idx); 2767 return ret; 2768 } 2769 2770 /* 2771 * For register / unregister we need to manually call out to every path. 2772 */ 2773 static int __dm_pr_register(struct dm_target *ti, struct dm_dev *dev, 2774 sector_t start, sector_t len, void *data) 2775 { 2776 struct dm_pr *pr = data; 2777 const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops; 2778 2779 if (!ops || !ops->pr_register) 2780 return -EOPNOTSUPP; 2781 return ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags); 2782 } 2783 2784 static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key, 2785 u32 flags) 2786 { 2787 struct dm_pr pr = { 2788 .old_key = old_key, 2789 .new_key = new_key, 2790 .flags = flags, 2791 .fail_early = true, 2792 }; 2793 int ret; 2794 2795 ret = dm_call_pr(bdev, __dm_pr_register, &pr); 2796 if (ret && new_key) { 2797 /* unregister all paths if we failed to register any path */ 2798 pr.old_key = new_key; 2799 pr.new_key = 0; 2800 pr.flags = 0; 2801 pr.fail_early = false; 2802 dm_call_pr(bdev, __dm_pr_register, &pr); 2803 } 2804 2805 return ret; 2806 } 2807 2808 static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type, 2809 u32 flags) 2810 { 2811 struct mapped_device *md = bdev->bd_disk->private_data; 2812 const struct pr_ops *ops; 2813 int r, srcu_idx; 2814 2815 r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 2816 if (r < 0) 2817 goto out; 2818 2819 ops = bdev->bd_disk->fops->pr_ops; 2820 if (ops && ops->pr_reserve) 2821 r = ops->pr_reserve(bdev, key, type, flags); 2822 else 2823 r = -EOPNOTSUPP; 2824 out: 2825 dm_unprepare_ioctl(md, srcu_idx); 2826 return r; 2827 } 2828 2829 static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type) 2830 { 2831 struct mapped_device *md = bdev->bd_disk->private_data; 2832 const struct pr_ops *ops; 2833 int r, srcu_idx; 2834 2835 r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 2836 if (r < 0) 2837 goto out; 2838 2839 ops = bdev->bd_disk->fops->pr_ops; 2840 if (ops && ops->pr_release) 2841 r = ops->pr_release(bdev, key, type); 2842 else 2843 r = -EOPNOTSUPP; 2844 out: 2845 dm_unprepare_ioctl(md, srcu_idx); 2846 return r; 2847 } 2848 2849 static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key, 2850 enum pr_type type, bool abort) 2851 { 2852 struct mapped_device *md = bdev->bd_disk->private_data; 2853 const struct pr_ops *ops; 2854 int r, srcu_idx; 2855 2856 r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 2857 if (r < 0) 2858 goto out; 2859 2860 ops = bdev->bd_disk->fops->pr_ops; 2861 if (ops && ops->pr_preempt) 2862 r = ops->pr_preempt(bdev, old_key, new_key, type, abort); 2863 else 2864 r = -EOPNOTSUPP; 2865 out: 2866 dm_unprepare_ioctl(md, srcu_idx); 2867 return r; 2868 } 2869 2870 static int dm_pr_clear(struct block_device *bdev, u64 key) 2871 { 2872 struct mapped_device *md = bdev->bd_disk->private_data; 2873 const struct pr_ops *ops; 2874 int r, srcu_idx; 2875 2876 r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 2877 if (r < 0) 2878 goto out; 2879 2880 ops = bdev->bd_disk->fops->pr_ops; 2881 if (ops && ops->pr_clear) 2882 r = ops->pr_clear(bdev, key); 2883 else 2884 r = -EOPNOTSUPP; 2885 out: 2886 dm_unprepare_ioctl(md, srcu_idx); 2887 return r; 2888 } 2889 2890 static const struct pr_ops dm_pr_ops = { 2891 .pr_register = dm_pr_register, 2892 .pr_reserve = dm_pr_reserve, 2893 .pr_release = dm_pr_release, 2894 .pr_preempt = dm_pr_preempt, 2895 .pr_clear = dm_pr_clear, 2896 }; 2897 2898 static const struct block_device_operations dm_blk_dops = { 2899 .submit_bio = dm_submit_bio, 2900 .open = dm_blk_open, 2901 .release = dm_blk_close, 2902 .ioctl = dm_blk_ioctl, 2903 .getgeo = dm_blk_getgeo, 2904 .report_zones = dm_blk_report_zones, 2905 .pr_ops = &dm_pr_ops, 2906 .owner = THIS_MODULE 2907 }; 2908 2909 static const struct block_device_operations dm_rq_blk_dops = { 2910 .open = dm_blk_open, 2911 .release = dm_blk_close, 2912 .ioctl = dm_blk_ioctl, 2913 .getgeo = dm_blk_getgeo, 2914 .pr_ops = &dm_pr_ops, 2915 .owner = THIS_MODULE 2916 }; 2917 2918 static const struct dax_operations dm_dax_ops = { 2919 .direct_access = dm_dax_direct_access, 2920 .zero_page_range = dm_dax_zero_page_range, 2921 }; 2922 2923 /* 2924 * module hooks 2925 */ 2926 module_init(dm_init); 2927 module_exit(dm_exit); 2928 2929 module_param(major, uint, 0); 2930 MODULE_PARM_DESC(major, "The major number of the device mapper"); 2931 2932 module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR); 2933 MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools"); 2934 2935 module_param(dm_numa_node, int, S_IRUGO | S_IWUSR); 2936 MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations"); 2937 2938 module_param(swap_bios, int, S_IRUGO | S_IWUSR); 2939 MODULE_PARM_DESC(swap_bios, "Maximum allowed inflight swap IOs"); 2940 2941 MODULE_DESCRIPTION(DM_NAME " driver"); 2942 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>"); 2943 MODULE_LICENSE("GPL"); 2944