1 /* 2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited. 3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. 4 * 5 * This file is released under the GPL. 6 */ 7 8 #include "dm-core.h" 9 #include "dm-rq.h" 10 #include "dm-uevent.h" 11 #include "dm-ima.h" 12 13 #include <linux/init.h> 14 #include <linux/module.h> 15 #include <linux/mutex.h> 16 #include <linux/sched/mm.h> 17 #include <linux/sched/signal.h> 18 #include <linux/blkpg.h> 19 #include <linux/bio.h> 20 #include <linux/mempool.h> 21 #include <linux/dax.h> 22 #include <linux/slab.h> 23 #include <linux/idr.h> 24 #include <linux/uio.h> 25 #include <linux/hdreg.h> 26 #include <linux/delay.h> 27 #include <linux/wait.h> 28 #include <linux/pr.h> 29 #include <linux/refcount.h> 30 #include <linux/part_stat.h> 31 #include <linux/blk-crypto.h> 32 #include <linux/blk-crypto-profile.h> 33 34 #define DM_MSG_PREFIX "core" 35 36 /* 37 * Cookies are numeric values sent with CHANGE and REMOVE 38 * uevents while resuming, removing or renaming the device. 39 */ 40 #define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE" 41 #define DM_COOKIE_LENGTH 24 42 43 /* 44 * For REQ_POLLED fs bio, this flag is set if we link mapped underlying 45 * dm_io into one list, and reuse bio->bi_private as the list head. Before 46 * ending this fs bio, we will recover its ->bi_private. 47 */ 48 #define REQ_DM_POLL_LIST REQ_DRV 49 50 static const char *_name = DM_NAME; 51 52 static unsigned int major = 0; 53 static unsigned int _major = 0; 54 55 static DEFINE_IDR(_minor_idr); 56 57 static DEFINE_SPINLOCK(_minor_lock); 58 59 static void do_deferred_remove(struct work_struct *w); 60 61 static DECLARE_WORK(deferred_remove_work, do_deferred_remove); 62 63 static struct workqueue_struct *deferred_remove_workqueue; 64 65 atomic_t dm_global_event_nr = ATOMIC_INIT(0); 66 DECLARE_WAIT_QUEUE_HEAD(dm_global_eventq); 67 68 void dm_issue_global_event(void) 69 { 70 atomic_inc(&dm_global_event_nr); 71 wake_up(&dm_global_eventq); 72 } 73 74 DEFINE_STATIC_KEY_FALSE(stats_enabled); 75 DEFINE_STATIC_KEY_FALSE(swap_bios_enabled); 76 DEFINE_STATIC_KEY_FALSE(zoned_enabled); 77 78 /* 79 * One of these is allocated (on-stack) per original bio. 80 */ 81 struct clone_info { 82 struct dm_table *map; 83 struct bio *bio; 84 struct dm_io *io; 85 sector_t sector; 86 unsigned sector_count; 87 bool is_abnormal_io:1; 88 bool submit_as_polled:1; 89 }; 90 91 #define DM_TARGET_IO_BIO_OFFSET (offsetof(struct dm_target_io, clone)) 92 #define DM_IO_BIO_OFFSET \ 93 (offsetof(struct dm_target_io, clone) + offsetof(struct dm_io, tio)) 94 95 static inline struct dm_target_io *clone_to_tio(struct bio *clone) 96 { 97 return container_of(clone, struct dm_target_io, clone); 98 } 99 100 void *dm_per_bio_data(struct bio *bio, size_t data_size) 101 { 102 if (!dm_tio_flagged(clone_to_tio(bio), DM_TIO_INSIDE_DM_IO)) 103 return (char *)bio - DM_TARGET_IO_BIO_OFFSET - data_size; 104 return (char *)bio - DM_IO_BIO_OFFSET - data_size; 105 } 106 EXPORT_SYMBOL_GPL(dm_per_bio_data); 107 108 struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size) 109 { 110 struct dm_io *io = (struct dm_io *)((char *)data + data_size); 111 if (io->magic == DM_IO_MAGIC) 112 return (struct bio *)((char *)io + DM_IO_BIO_OFFSET); 113 BUG_ON(io->magic != DM_TIO_MAGIC); 114 return (struct bio *)((char *)io + DM_TARGET_IO_BIO_OFFSET); 115 } 116 EXPORT_SYMBOL_GPL(dm_bio_from_per_bio_data); 117 118 unsigned dm_bio_get_target_bio_nr(const struct bio *bio) 119 { 120 return container_of(bio, struct dm_target_io, clone)->target_bio_nr; 121 } 122 EXPORT_SYMBOL_GPL(dm_bio_get_target_bio_nr); 123 124 #define MINOR_ALLOCED ((void *)-1) 125 126 #define DM_NUMA_NODE NUMA_NO_NODE 127 static int dm_numa_node = DM_NUMA_NODE; 128 129 #define DEFAULT_SWAP_BIOS (8 * 1048576 / PAGE_SIZE) 130 static int swap_bios = DEFAULT_SWAP_BIOS; 131 static int get_swap_bios(void) 132 { 133 int latch = READ_ONCE(swap_bios); 134 if (unlikely(latch <= 0)) 135 latch = DEFAULT_SWAP_BIOS; 136 return latch; 137 } 138 139 /* 140 * For mempools pre-allocation at the table loading time. 141 */ 142 struct dm_md_mempools { 143 struct bio_set bs; 144 struct bio_set io_bs; 145 }; 146 147 struct table_device { 148 struct list_head list; 149 refcount_t count; 150 struct dm_dev dm_dev; 151 }; 152 153 /* 154 * Bio-based DM's mempools' reserved IOs set by the user. 155 */ 156 #define RESERVED_BIO_BASED_IOS 16 157 static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS; 158 159 static int __dm_get_module_param_int(int *module_param, int min, int max) 160 { 161 int param = READ_ONCE(*module_param); 162 int modified_param = 0; 163 bool modified = true; 164 165 if (param < min) 166 modified_param = min; 167 else if (param > max) 168 modified_param = max; 169 else 170 modified = false; 171 172 if (modified) { 173 (void)cmpxchg(module_param, param, modified_param); 174 param = modified_param; 175 } 176 177 return param; 178 } 179 180 unsigned __dm_get_module_param(unsigned *module_param, 181 unsigned def, unsigned max) 182 { 183 unsigned param = READ_ONCE(*module_param); 184 unsigned modified_param = 0; 185 186 if (!param) 187 modified_param = def; 188 else if (param > max) 189 modified_param = max; 190 191 if (modified_param) { 192 (void)cmpxchg(module_param, param, modified_param); 193 param = modified_param; 194 } 195 196 return param; 197 } 198 199 unsigned dm_get_reserved_bio_based_ios(void) 200 { 201 return __dm_get_module_param(&reserved_bio_based_ios, 202 RESERVED_BIO_BASED_IOS, DM_RESERVED_MAX_IOS); 203 } 204 EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios); 205 206 static unsigned dm_get_numa_node(void) 207 { 208 return __dm_get_module_param_int(&dm_numa_node, 209 DM_NUMA_NODE, num_online_nodes() - 1); 210 } 211 212 static int __init local_init(void) 213 { 214 int r; 215 216 r = dm_uevent_init(); 217 if (r) 218 return r; 219 220 deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1); 221 if (!deferred_remove_workqueue) { 222 r = -ENOMEM; 223 goto out_uevent_exit; 224 } 225 226 _major = major; 227 r = register_blkdev(_major, _name); 228 if (r < 0) 229 goto out_free_workqueue; 230 231 if (!_major) 232 _major = r; 233 234 return 0; 235 236 out_free_workqueue: 237 destroy_workqueue(deferred_remove_workqueue); 238 out_uevent_exit: 239 dm_uevent_exit(); 240 241 return r; 242 } 243 244 static void local_exit(void) 245 { 246 flush_scheduled_work(); 247 destroy_workqueue(deferred_remove_workqueue); 248 249 unregister_blkdev(_major, _name); 250 dm_uevent_exit(); 251 252 _major = 0; 253 254 DMINFO("cleaned up"); 255 } 256 257 static int (*_inits[])(void) __initdata = { 258 local_init, 259 dm_target_init, 260 dm_linear_init, 261 dm_stripe_init, 262 dm_io_init, 263 dm_kcopyd_init, 264 dm_interface_init, 265 dm_statistics_init, 266 }; 267 268 static void (*_exits[])(void) = { 269 local_exit, 270 dm_target_exit, 271 dm_linear_exit, 272 dm_stripe_exit, 273 dm_io_exit, 274 dm_kcopyd_exit, 275 dm_interface_exit, 276 dm_statistics_exit, 277 }; 278 279 static int __init dm_init(void) 280 { 281 const int count = ARRAY_SIZE(_inits); 282 int r, i; 283 284 #if (IS_ENABLED(CONFIG_IMA) && !IS_ENABLED(CONFIG_IMA_DISABLE_HTABLE)) 285 DMWARN("CONFIG_IMA_DISABLE_HTABLE is disabled." 286 " Duplicate IMA measurements will not be recorded in the IMA log."); 287 #endif 288 289 for (i = 0; i < count; i++) { 290 r = _inits[i](); 291 if (r) 292 goto bad; 293 } 294 295 return 0; 296 bad: 297 while (i--) 298 _exits[i](); 299 300 return r; 301 } 302 303 static void __exit dm_exit(void) 304 { 305 int i = ARRAY_SIZE(_exits); 306 307 while (i--) 308 _exits[i](); 309 310 /* 311 * Should be empty by this point. 312 */ 313 idr_destroy(&_minor_idr); 314 } 315 316 /* 317 * Block device functions 318 */ 319 int dm_deleting_md(struct mapped_device *md) 320 { 321 return test_bit(DMF_DELETING, &md->flags); 322 } 323 324 static int dm_blk_open(struct block_device *bdev, fmode_t mode) 325 { 326 struct mapped_device *md; 327 328 spin_lock(&_minor_lock); 329 330 md = bdev->bd_disk->private_data; 331 if (!md) 332 goto out; 333 334 if (test_bit(DMF_FREEING, &md->flags) || 335 dm_deleting_md(md)) { 336 md = NULL; 337 goto out; 338 } 339 340 dm_get(md); 341 atomic_inc(&md->open_count); 342 out: 343 spin_unlock(&_minor_lock); 344 345 return md ? 0 : -ENXIO; 346 } 347 348 static void dm_blk_close(struct gendisk *disk, fmode_t mode) 349 { 350 struct mapped_device *md; 351 352 spin_lock(&_minor_lock); 353 354 md = disk->private_data; 355 if (WARN_ON(!md)) 356 goto out; 357 358 if (atomic_dec_and_test(&md->open_count) && 359 (test_bit(DMF_DEFERRED_REMOVE, &md->flags))) 360 queue_work(deferred_remove_workqueue, &deferred_remove_work); 361 362 dm_put(md); 363 out: 364 spin_unlock(&_minor_lock); 365 } 366 367 int dm_open_count(struct mapped_device *md) 368 { 369 return atomic_read(&md->open_count); 370 } 371 372 /* 373 * Guarantees nothing is using the device before it's deleted. 374 */ 375 int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred) 376 { 377 int r = 0; 378 379 spin_lock(&_minor_lock); 380 381 if (dm_open_count(md)) { 382 r = -EBUSY; 383 if (mark_deferred) 384 set_bit(DMF_DEFERRED_REMOVE, &md->flags); 385 } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags)) 386 r = -EEXIST; 387 else 388 set_bit(DMF_DELETING, &md->flags); 389 390 spin_unlock(&_minor_lock); 391 392 return r; 393 } 394 395 int dm_cancel_deferred_remove(struct mapped_device *md) 396 { 397 int r = 0; 398 399 spin_lock(&_minor_lock); 400 401 if (test_bit(DMF_DELETING, &md->flags)) 402 r = -EBUSY; 403 else 404 clear_bit(DMF_DEFERRED_REMOVE, &md->flags); 405 406 spin_unlock(&_minor_lock); 407 408 return r; 409 } 410 411 static void do_deferred_remove(struct work_struct *w) 412 { 413 dm_deferred_remove(); 414 } 415 416 static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo) 417 { 418 struct mapped_device *md = bdev->bd_disk->private_data; 419 420 return dm_get_geometry(md, geo); 421 } 422 423 static int dm_prepare_ioctl(struct mapped_device *md, int *srcu_idx, 424 struct block_device **bdev) 425 { 426 struct dm_target *tgt; 427 struct dm_table *map; 428 int r; 429 430 retry: 431 r = -ENOTTY; 432 map = dm_get_live_table(md, srcu_idx); 433 if (!map || !dm_table_get_size(map)) 434 return r; 435 436 /* We only support devices that have a single target */ 437 if (dm_table_get_num_targets(map) != 1) 438 return r; 439 440 tgt = dm_table_get_target(map, 0); 441 if (!tgt->type->prepare_ioctl) 442 return r; 443 444 if (dm_suspended_md(md)) 445 return -EAGAIN; 446 447 r = tgt->type->prepare_ioctl(tgt, bdev); 448 if (r == -ENOTCONN && !fatal_signal_pending(current)) { 449 dm_put_live_table(md, *srcu_idx); 450 msleep(10); 451 goto retry; 452 } 453 454 return r; 455 } 456 457 static void dm_unprepare_ioctl(struct mapped_device *md, int srcu_idx) 458 { 459 dm_put_live_table(md, srcu_idx); 460 } 461 462 static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode, 463 unsigned int cmd, unsigned long arg) 464 { 465 struct mapped_device *md = bdev->bd_disk->private_data; 466 int r, srcu_idx; 467 468 r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 469 if (r < 0) 470 goto out; 471 472 if (r > 0) { 473 /* 474 * Target determined this ioctl is being issued against a 475 * subset of the parent bdev; require extra privileges. 476 */ 477 if (!capable(CAP_SYS_RAWIO)) { 478 DMDEBUG_LIMIT( 479 "%s: sending ioctl %x to DM device without required privilege.", 480 current->comm, cmd); 481 r = -ENOIOCTLCMD; 482 goto out; 483 } 484 } 485 486 if (!bdev->bd_disk->fops->ioctl) 487 r = -ENOTTY; 488 else 489 r = bdev->bd_disk->fops->ioctl(bdev, mode, cmd, arg); 490 out: 491 dm_unprepare_ioctl(md, srcu_idx); 492 return r; 493 } 494 495 u64 dm_start_time_ns_from_clone(struct bio *bio) 496 { 497 return jiffies_to_nsecs(clone_to_tio(bio)->io->start_time); 498 } 499 EXPORT_SYMBOL_GPL(dm_start_time_ns_from_clone); 500 501 static bool bio_is_flush_with_data(struct bio *bio) 502 { 503 return ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size); 504 } 505 506 static void dm_io_acct(struct dm_io *io, bool end) 507 { 508 struct dm_stats_aux *stats_aux = &io->stats_aux; 509 unsigned long start_time = io->start_time; 510 struct mapped_device *md = io->md; 511 struct bio *bio = io->orig_bio; 512 unsigned int sectors; 513 514 /* 515 * If REQ_PREFLUSH set, don't account payload, it will be 516 * submitted (and accounted) after this flush completes. 517 */ 518 if (bio_is_flush_with_data(bio)) 519 sectors = 0; 520 else if (likely(!(dm_io_flagged(io, DM_IO_WAS_SPLIT)))) 521 sectors = bio_sectors(bio); 522 else 523 sectors = io->sectors; 524 525 if (!end) 526 bdev_start_io_acct(bio->bi_bdev, sectors, bio_op(bio), 527 start_time); 528 else 529 bdev_end_io_acct(bio->bi_bdev, bio_op(bio), start_time); 530 531 if (static_branch_unlikely(&stats_enabled) && 532 unlikely(dm_stats_used(&md->stats))) { 533 sector_t sector; 534 535 if (likely(!dm_io_flagged(io, DM_IO_WAS_SPLIT))) 536 sector = bio->bi_iter.bi_sector; 537 else 538 sector = bio_end_sector(bio) - io->sector_offset; 539 540 dm_stats_account_io(&md->stats, bio_data_dir(bio), 541 sector, sectors, 542 end, start_time, stats_aux); 543 } 544 } 545 546 static void __dm_start_io_acct(struct dm_io *io) 547 { 548 dm_io_acct(io, false); 549 } 550 551 static void dm_start_io_acct(struct dm_io *io, struct bio *clone) 552 { 553 /* 554 * Ensure IO accounting is only ever started once. 555 */ 556 if (dm_io_flagged(io, DM_IO_ACCOUNTED)) 557 return; 558 559 /* Expect no possibility for race unless DM_TIO_IS_DUPLICATE_BIO. */ 560 if (!clone || likely(dm_tio_is_normal(clone_to_tio(clone)))) { 561 dm_io_set_flag(io, DM_IO_ACCOUNTED); 562 } else { 563 unsigned long flags; 564 /* Can afford locking given DM_TIO_IS_DUPLICATE_BIO */ 565 spin_lock_irqsave(&io->lock, flags); 566 dm_io_set_flag(io, DM_IO_ACCOUNTED); 567 spin_unlock_irqrestore(&io->lock, flags); 568 } 569 570 __dm_start_io_acct(io); 571 } 572 573 static void dm_end_io_acct(struct dm_io *io) 574 { 575 dm_io_acct(io, true); 576 } 577 578 static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio) 579 { 580 struct dm_io *io; 581 struct dm_target_io *tio; 582 struct bio *clone; 583 584 clone = bio_alloc_clone(NULL, bio, GFP_NOIO, &md->io_bs); 585 /* Set default bdev, but target must bio_set_dev() before issuing IO */ 586 clone->bi_bdev = md->disk->part0; 587 588 tio = clone_to_tio(clone); 589 tio->flags = 0; 590 dm_tio_set_flag(tio, DM_TIO_INSIDE_DM_IO); 591 tio->io = NULL; 592 593 io = container_of(tio, struct dm_io, tio); 594 io->magic = DM_IO_MAGIC; 595 io->status = BLK_STS_OK; 596 597 /* one ref is for submission, the other is for completion */ 598 atomic_set(&io->io_count, 2); 599 this_cpu_inc(*md->pending_io); 600 io->orig_bio = bio; 601 io->md = md; 602 spin_lock_init(&io->lock); 603 io->start_time = jiffies; 604 io->flags = 0; 605 606 if (static_branch_unlikely(&stats_enabled)) 607 dm_stats_record_start(&md->stats, &io->stats_aux); 608 609 return io; 610 } 611 612 static void free_io(struct dm_io *io) 613 { 614 bio_put(&io->tio.clone); 615 } 616 617 static struct bio *alloc_tio(struct clone_info *ci, struct dm_target *ti, 618 unsigned target_bio_nr, unsigned *len, gfp_t gfp_mask) 619 { 620 struct dm_target_io *tio; 621 struct bio *clone; 622 623 if (!ci->io->tio.io) { 624 /* the dm_target_io embedded in ci->io is available */ 625 tio = &ci->io->tio; 626 /* alloc_io() already initialized embedded clone */ 627 clone = &tio->clone; 628 } else { 629 struct mapped_device *md = ci->io->md; 630 631 clone = bio_alloc_clone(NULL, ci->bio, gfp_mask, &md->bs); 632 if (!clone) 633 return NULL; 634 /* Set default bdev, but target must bio_set_dev() before issuing IO */ 635 clone->bi_bdev = md->disk->part0; 636 637 /* REQ_DM_POLL_LIST shouldn't be inherited */ 638 clone->bi_opf &= ~REQ_DM_POLL_LIST; 639 640 tio = clone_to_tio(clone); 641 tio->flags = 0; /* also clears DM_TIO_INSIDE_DM_IO */ 642 } 643 644 tio->magic = DM_TIO_MAGIC; 645 tio->io = ci->io; 646 tio->ti = ti; 647 tio->target_bio_nr = target_bio_nr; 648 tio->len_ptr = len; 649 tio->old_sector = 0; 650 651 if (len) { 652 clone->bi_iter.bi_size = to_bytes(*len); 653 if (bio_integrity(clone)) 654 bio_integrity_trim(clone); 655 } 656 657 return clone; 658 } 659 660 static void free_tio(struct bio *clone) 661 { 662 if (dm_tio_flagged(clone_to_tio(clone), DM_TIO_INSIDE_DM_IO)) 663 return; 664 bio_put(clone); 665 } 666 667 /* 668 * Add the bio to the list of deferred io. 669 */ 670 static void queue_io(struct mapped_device *md, struct bio *bio) 671 { 672 unsigned long flags; 673 674 spin_lock_irqsave(&md->deferred_lock, flags); 675 bio_list_add(&md->deferred, bio); 676 spin_unlock_irqrestore(&md->deferred_lock, flags); 677 queue_work(md->wq, &md->work); 678 } 679 680 /* 681 * Everyone (including functions in this file), should use this 682 * function to access the md->map field, and make sure they call 683 * dm_put_live_table() when finished. 684 */ 685 struct dm_table *dm_get_live_table(struct mapped_device *md, 686 int *srcu_idx) __acquires(md->io_barrier) 687 { 688 *srcu_idx = srcu_read_lock(&md->io_barrier); 689 690 return srcu_dereference(md->map, &md->io_barrier); 691 } 692 693 void dm_put_live_table(struct mapped_device *md, 694 int srcu_idx) __releases(md->io_barrier) 695 { 696 srcu_read_unlock(&md->io_barrier, srcu_idx); 697 } 698 699 void dm_sync_table(struct mapped_device *md) 700 { 701 synchronize_srcu(&md->io_barrier); 702 synchronize_rcu_expedited(); 703 } 704 705 /* 706 * A fast alternative to dm_get_live_table/dm_put_live_table. 707 * The caller must not block between these two functions. 708 */ 709 static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU) 710 { 711 rcu_read_lock(); 712 return rcu_dereference(md->map); 713 } 714 715 static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU) 716 { 717 rcu_read_unlock(); 718 } 719 720 static inline struct dm_table *dm_get_live_table_bio(struct mapped_device *md, 721 int *srcu_idx, struct bio *bio) 722 { 723 if (bio->bi_opf & REQ_NOWAIT) 724 return dm_get_live_table_fast(md); 725 else 726 return dm_get_live_table(md, srcu_idx); 727 } 728 729 static inline void dm_put_live_table_bio(struct mapped_device *md, int srcu_idx, 730 struct bio *bio) 731 { 732 if (bio->bi_opf & REQ_NOWAIT) 733 dm_put_live_table_fast(md); 734 else 735 dm_put_live_table(md, srcu_idx); 736 } 737 738 static char *_dm_claim_ptr = "I belong to device-mapper"; 739 740 /* 741 * Open a table device so we can use it as a map destination. 742 */ 743 static int open_table_device(struct table_device *td, dev_t dev, 744 struct mapped_device *md) 745 { 746 struct block_device *bdev; 747 u64 part_off; 748 int r; 749 750 BUG_ON(td->dm_dev.bdev); 751 752 bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _dm_claim_ptr); 753 if (IS_ERR(bdev)) 754 return PTR_ERR(bdev); 755 756 r = bd_link_disk_holder(bdev, dm_disk(md)); 757 if (r) { 758 blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL); 759 return r; 760 } 761 762 td->dm_dev.bdev = bdev; 763 td->dm_dev.dax_dev = fs_dax_get_by_bdev(bdev, &part_off); 764 return 0; 765 } 766 767 /* 768 * Close a table device that we've been using. 769 */ 770 static void close_table_device(struct table_device *td, struct mapped_device *md) 771 { 772 if (!td->dm_dev.bdev) 773 return; 774 775 bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md)); 776 blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL); 777 put_dax(td->dm_dev.dax_dev); 778 td->dm_dev.bdev = NULL; 779 td->dm_dev.dax_dev = NULL; 780 } 781 782 static struct table_device *find_table_device(struct list_head *l, dev_t dev, 783 fmode_t mode) 784 { 785 struct table_device *td; 786 787 list_for_each_entry(td, l, list) 788 if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode) 789 return td; 790 791 return NULL; 792 } 793 794 int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode, 795 struct dm_dev **result) 796 { 797 int r; 798 struct table_device *td; 799 800 mutex_lock(&md->table_devices_lock); 801 td = find_table_device(&md->table_devices, dev, mode); 802 if (!td) { 803 td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id); 804 if (!td) { 805 mutex_unlock(&md->table_devices_lock); 806 return -ENOMEM; 807 } 808 809 td->dm_dev.mode = mode; 810 td->dm_dev.bdev = NULL; 811 812 if ((r = open_table_device(td, dev, md))) { 813 mutex_unlock(&md->table_devices_lock); 814 kfree(td); 815 return r; 816 } 817 818 format_dev_t(td->dm_dev.name, dev); 819 820 refcount_set(&td->count, 1); 821 list_add(&td->list, &md->table_devices); 822 } else { 823 refcount_inc(&td->count); 824 } 825 mutex_unlock(&md->table_devices_lock); 826 827 *result = &td->dm_dev; 828 return 0; 829 } 830 831 void dm_put_table_device(struct mapped_device *md, struct dm_dev *d) 832 { 833 struct table_device *td = container_of(d, struct table_device, dm_dev); 834 835 mutex_lock(&md->table_devices_lock); 836 if (refcount_dec_and_test(&td->count)) { 837 close_table_device(td, md); 838 list_del(&td->list); 839 kfree(td); 840 } 841 mutex_unlock(&md->table_devices_lock); 842 } 843 844 static void free_table_devices(struct list_head *devices) 845 { 846 struct list_head *tmp, *next; 847 848 list_for_each_safe(tmp, next, devices) { 849 struct table_device *td = list_entry(tmp, struct table_device, list); 850 851 DMWARN("dm_destroy: %s still exists with %d references", 852 td->dm_dev.name, refcount_read(&td->count)); 853 kfree(td); 854 } 855 } 856 857 /* 858 * Get the geometry associated with a dm device 859 */ 860 int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo) 861 { 862 *geo = md->geometry; 863 864 return 0; 865 } 866 867 /* 868 * Set the geometry of a device. 869 */ 870 int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo) 871 { 872 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors; 873 874 if (geo->start > sz) { 875 DMWARN("Start sector is beyond the geometry limits."); 876 return -EINVAL; 877 } 878 879 md->geometry = *geo; 880 881 return 0; 882 } 883 884 static int __noflush_suspending(struct mapped_device *md) 885 { 886 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 887 } 888 889 static void dm_io_complete(struct dm_io *io) 890 { 891 blk_status_t io_error; 892 struct mapped_device *md = io->md; 893 struct bio *bio = io->orig_bio; 894 895 if (io->status == BLK_STS_DM_REQUEUE) { 896 unsigned long flags; 897 /* 898 * Target requested pushing back the I/O. 899 */ 900 spin_lock_irqsave(&md->deferred_lock, flags); 901 if (__noflush_suspending(md) && 902 !WARN_ON_ONCE(dm_is_zone_write(md, bio))) { 903 /* NOTE early return due to BLK_STS_DM_REQUEUE below */ 904 bio_list_add_head(&md->deferred, bio); 905 } else { 906 /* 907 * noflush suspend was interrupted or this is 908 * a write to a zoned target. 909 */ 910 io->status = BLK_STS_IOERR; 911 } 912 spin_unlock_irqrestore(&md->deferred_lock, flags); 913 } 914 915 io_error = io->status; 916 if (dm_io_flagged(io, DM_IO_ACCOUNTED)) 917 dm_end_io_acct(io); 918 else if (!io_error) { 919 /* 920 * Must handle target that DM_MAPIO_SUBMITTED only to 921 * then bio_endio() rather than dm_submit_bio_remap() 922 */ 923 __dm_start_io_acct(io); 924 dm_end_io_acct(io); 925 } 926 free_io(io); 927 smp_wmb(); 928 this_cpu_dec(*md->pending_io); 929 930 /* nudge anyone waiting on suspend queue */ 931 if (unlikely(wq_has_sleeper(&md->wait))) 932 wake_up(&md->wait); 933 934 if (io_error == BLK_STS_DM_REQUEUE || io_error == BLK_STS_AGAIN) { 935 if (bio->bi_opf & REQ_POLLED) { 936 /* 937 * Upper layer won't help us poll split bio (io->orig_bio 938 * may only reflect a subset of the pre-split original) 939 * so clear REQ_POLLED in case of requeue. 940 */ 941 bio_clear_polled(bio); 942 if (io_error == BLK_STS_AGAIN) { 943 /* io_uring doesn't handle BLK_STS_AGAIN (yet) */ 944 queue_io(md, bio); 945 } 946 } 947 return; 948 } 949 950 if (bio_is_flush_with_data(bio)) { 951 /* 952 * Preflush done for flush with data, reissue 953 * without REQ_PREFLUSH. 954 */ 955 bio->bi_opf &= ~REQ_PREFLUSH; 956 queue_io(md, bio); 957 } else { 958 /* done with normal IO or empty flush */ 959 if (io_error) 960 bio->bi_status = io_error; 961 bio_endio(bio); 962 } 963 } 964 965 /* 966 * Decrements the number of outstanding ios that a bio has been 967 * cloned into, completing the original io if necc. 968 */ 969 static inline void __dm_io_dec_pending(struct dm_io *io) 970 { 971 if (atomic_dec_and_test(&io->io_count)) 972 dm_io_complete(io); 973 } 974 975 static void dm_io_set_error(struct dm_io *io, blk_status_t error) 976 { 977 unsigned long flags; 978 979 /* Push-back supersedes any I/O errors */ 980 spin_lock_irqsave(&io->lock, flags); 981 if (!(io->status == BLK_STS_DM_REQUEUE && 982 __noflush_suspending(io->md))) { 983 io->status = error; 984 } 985 spin_unlock_irqrestore(&io->lock, flags); 986 } 987 988 static void dm_io_dec_pending(struct dm_io *io, blk_status_t error) 989 { 990 if (unlikely(error)) 991 dm_io_set_error(io, error); 992 993 __dm_io_dec_pending(io); 994 } 995 996 void disable_discard(struct mapped_device *md) 997 { 998 struct queue_limits *limits = dm_get_queue_limits(md); 999 1000 /* device doesn't really support DISCARD, disable it */ 1001 limits->max_discard_sectors = 0; 1002 } 1003 1004 void disable_write_zeroes(struct mapped_device *md) 1005 { 1006 struct queue_limits *limits = dm_get_queue_limits(md); 1007 1008 /* device doesn't really support WRITE ZEROES, disable it */ 1009 limits->max_write_zeroes_sectors = 0; 1010 } 1011 1012 static bool swap_bios_limit(struct dm_target *ti, struct bio *bio) 1013 { 1014 return unlikely((bio->bi_opf & REQ_SWAP) != 0) && unlikely(ti->limit_swap_bios); 1015 } 1016 1017 static void clone_endio(struct bio *bio) 1018 { 1019 blk_status_t error = bio->bi_status; 1020 struct dm_target_io *tio = clone_to_tio(bio); 1021 struct dm_target *ti = tio->ti; 1022 dm_endio_fn endio = ti->type->end_io; 1023 struct dm_io *io = tio->io; 1024 struct mapped_device *md = io->md; 1025 1026 if (likely(bio->bi_bdev != md->disk->part0)) { 1027 struct request_queue *q = bdev_get_queue(bio->bi_bdev); 1028 1029 if (unlikely(error == BLK_STS_TARGET)) { 1030 if (bio_op(bio) == REQ_OP_DISCARD && 1031 !bdev_max_discard_sectors(bio->bi_bdev)) 1032 disable_discard(md); 1033 else if (bio_op(bio) == REQ_OP_WRITE_ZEROES && 1034 !q->limits.max_write_zeroes_sectors) 1035 disable_write_zeroes(md); 1036 } 1037 1038 if (static_branch_unlikely(&zoned_enabled) && 1039 unlikely(blk_queue_is_zoned(q))) 1040 dm_zone_endio(io, bio); 1041 } 1042 1043 if (endio) { 1044 int r = endio(ti, bio, &error); 1045 switch (r) { 1046 case DM_ENDIO_REQUEUE: 1047 if (static_branch_unlikely(&zoned_enabled)) { 1048 /* 1049 * Requeuing writes to a sequential zone of a zoned 1050 * target will break the sequential write pattern: 1051 * fail such IO. 1052 */ 1053 if (WARN_ON_ONCE(dm_is_zone_write(md, bio))) 1054 error = BLK_STS_IOERR; 1055 else 1056 error = BLK_STS_DM_REQUEUE; 1057 } else 1058 error = BLK_STS_DM_REQUEUE; 1059 fallthrough; 1060 case DM_ENDIO_DONE: 1061 break; 1062 case DM_ENDIO_INCOMPLETE: 1063 /* The target will handle the io */ 1064 return; 1065 default: 1066 DMWARN("unimplemented target endio return value: %d", r); 1067 BUG(); 1068 } 1069 } 1070 1071 if (static_branch_unlikely(&swap_bios_enabled) && 1072 unlikely(swap_bios_limit(ti, bio))) 1073 up(&md->swap_bios_semaphore); 1074 1075 free_tio(bio); 1076 dm_io_dec_pending(io, error); 1077 } 1078 1079 /* 1080 * Return maximum size of I/O possible at the supplied sector up to the current 1081 * target boundary. 1082 */ 1083 static inline sector_t max_io_len_target_boundary(struct dm_target *ti, 1084 sector_t target_offset) 1085 { 1086 return ti->len - target_offset; 1087 } 1088 1089 static sector_t max_io_len(struct dm_target *ti, sector_t sector) 1090 { 1091 sector_t target_offset = dm_target_offset(ti, sector); 1092 sector_t len = max_io_len_target_boundary(ti, target_offset); 1093 sector_t max_len; 1094 1095 /* 1096 * Does the target need to split IO even further? 1097 * - varied (per target) IO splitting is a tenet of DM; this 1098 * explains why stacked chunk_sectors based splitting via 1099 * blk_max_size_offset() isn't possible here. So pass in 1100 * ti->max_io_len to override stacked chunk_sectors. 1101 */ 1102 if (ti->max_io_len) { 1103 max_len = blk_max_size_offset(ti->table->md->queue, 1104 target_offset, ti->max_io_len); 1105 if (len > max_len) 1106 len = max_len; 1107 } 1108 1109 return len; 1110 } 1111 1112 int dm_set_target_max_io_len(struct dm_target *ti, sector_t len) 1113 { 1114 if (len > UINT_MAX) { 1115 DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)", 1116 (unsigned long long)len, UINT_MAX); 1117 ti->error = "Maximum size of target IO is too large"; 1118 return -EINVAL; 1119 } 1120 1121 ti->max_io_len = (uint32_t) len; 1122 1123 return 0; 1124 } 1125 EXPORT_SYMBOL_GPL(dm_set_target_max_io_len); 1126 1127 static struct dm_target *dm_dax_get_live_target(struct mapped_device *md, 1128 sector_t sector, int *srcu_idx) 1129 __acquires(md->io_barrier) 1130 { 1131 struct dm_table *map; 1132 struct dm_target *ti; 1133 1134 map = dm_get_live_table(md, srcu_idx); 1135 if (!map) 1136 return NULL; 1137 1138 ti = dm_table_find_target(map, sector); 1139 if (!ti) 1140 return NULL; 1141 1142 return ti; 1143 } 1144 1145 static long dm_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, 1146 long nr_pages, void **kaddr, pfn_t *pfn) 1147 { 1148 struct mapped_device *md = dax_get_private(dax_dev); 1149 sector_t sector = pgoff * PAGE_SECTORS; 1150 struct dm_target *ti; 1151 long len, ret = -EIO; 1152 int srcu_idx; 1153 1154 ti = dm_dax_get_live_target(md, sector, &srcu_idx); 1155 1156 if (!ti) 1157 goto out; 1158 if (!ti->type->direct_access) 1159 goto out; 1160 len = max_io_len(ti, sector) / PAGE_SECTORS; 1161 if (len < 1) 1162 goto out; 1163 nr_pages = min(len, nr_pages); 1164 ret = ti->type->direct_access(ti, pgoff, nr_pages, kaddr, pfn); 1165 1166 out: 1167 dm_put_live_table(md, srcu_idx); 1168 1169 return ret; 1170 } 1171 1172 static int dm_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, 1173 size_t nr_pages) 1174 { 1175 struct mapped_device *md = dax_get_private(dax_dev); 1176 sector_t sector = pgoff * PAGE_SECTORS; 1177 struct dm_target *ti; 1178 int ret = -EIO; 1179 int srcu_idx; 1180 1181 ti = dm_dax_get_live_target(md, sector, &srcu_idx); 1182 1183 if (!ti) 1184 goto out; 1185 if (WARN_ON(!ti->type->dax_zero_page_range)) { 1186 /* 1187 * ->zero_page_range() is mandatory dax operation. If we are 1188 * here, something is wrong. 1189 */ 1190 goto out; 1191 } 1192 ret = ti->type->dax_zero_page_range(ti, pgoff, nr_pages); 1193 out: 1194 dm_put_live_table(md, srcu_idx); 1195 1196 return ret; 1197 } 1198 1199 /* 1200 * A target may call dm_accept_partial_bio only from the map routine. It is 1201 * allowed for all bio types except REQ_PREFLUSH, REQ_OP_ZONE_* zone management 1202 * operations, REQ_OP_ZONE_APPEND (zone append writes) and any bio serviced by 1203 * __send_duplicate_bios(). 1204 * 1205 * dm_accept_partial_bio informs the dm that the target only wants to process 1206 * additional n_sectors sectors of the bio and the rest of the data should be 1207 * sent in a next bio. 1208 * 1209 * A diagram that explains the arithmetics: 1210 * +--------------------+---------------+-------+ 1211 * | 1 | 2 | 3 | 1212 * +--------------------+---------------+-------+ 1213 * 1214 * <-------------- *tio->len_ptr ---------------> 1215 * <----- bio_sectors -----> 1216 * <-- n_sectors --> 1217 * 1218 * Region 1 was already iterated over with bio_advance or similar function. 1219 * (it may be empty if the target doesn't use bio_advance) 1220 * Region 2 is the remaining bio size that the target wants to process. 1221 * (it may be empty if region 1 is non-empty, although there is no reason 1222 * to make it empty) 1223 * The target requires that region 3 is to be sent in the next bio. 1224 * 1225 * If the target wants to receive multiple copies of the bio (via num_*bios, etc), 1226 * the partially processed part (the sum of regions 1+2) must be the same for all 1227 * copies of the bio. 1228 */ 1229 void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors) 1230 { 1231 struct dm_target_io *tio = clone_to_tio(bio); 1232 unsigned bio_sectors = bio_sectors(bio); 1233 1234 BUG_ON(dm_tio_flagged(tio, DM_TIO_IS_DUPLICATE_BIO)); 1235 BUG_ON(op_is_zone_mgmt(bio_op(bio))); 1236 BUG_ON(bio_op(bio) == REQ_OP_ZONE_APPEND); 1237 BUG_ON(bio_sectors > *tio->len_ptr); 1238 BUG_ON(n_sectors > bio_sectors); 1239 1240 *tio->len_ptr -= bio_sectors - n_sectors; 1241 bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT; 1242 1243 /* 1244 * __split_and_process_bio() may have already saved mapped part 1245 * for accounting but it is being reduced so update accordingly. 1246 */ 1247 dm_io_set_flag(tio->io, DM_IO_WAS_SPLIT); 1248 tio->io->sectors = n_sectors; 1249 } 1250 EXPORT_SYMBOL_GPL(dm_accept_partial_bio); 1251 1252 /* 1253 * @clone: clone bio that DM core passed to target's .map function 1254 * @tgt_clone: clone of @clone bio that target needs submitted 1255 * 1256 * Targets should use this interface to submit bios they take 1257 * ownership of when returning DM_MAPIO_SUBMITTED. 1258 * 1259 * Target should also enable ti->accounts_remapped_io 1260 */ 1261 void dm_submit_bio_remap(struct bio *clone, struct bio *tgt_clone) 1262 { 1263 struct dm_target_io *tio = clone_to_tio(clone); 1264 struct dm_io *io = tio->io; 1265 1266 /* establish bio that will get submitted */ 1267 if (!tgt_clone) 1268 tgt_clone = clone; 1269 1270 /* 1271 * Account io->origin_bio to DM dev on behalf of target 1272 * that took ownership of IO with DM_MAPIO_SUBMITTED. 1273 */ 1274 dm_start_io_acct(io, clone); 1275 1276 trace_block_bio_remap(tgt_clone, disk_devt(io->md->disk), 1277 tio->old_sector); 1278 submit_bio_noacct(tgt_clone); 1279 } 1280 EXPORT_SYMBOL_GPL(dm_submit_bio_remap); 1281 1282 static noinline void __set_swap_bios_limit(struct mapped_device *md, int latch) 1283 { 1284 mutex_lock(&md->swap_bios_lock); 1285 while (latch < md->swap_bios) { 1286 cond_resched(); 1287 down(&md->swap_bios_semaphore); 1288 md->swap_bios--; 1289 } 1290 while (latch > md->swap_bios) { 1291 cond_resched(); 1292 up(&md->swap_bios_semaphore); 1293 md->swap_bios++; 1294 } 1295 mutex_unlock(&md->swap_bios_lock); 1296 } 1297 1298 static void __map_bio(struct bio *clone) 1299 { 1300 struct dm_target_io *tio = clone_to_tio(clone); 1301 struct dm_target *ti = tio->ti; 1302 struct dm_io *io = tio->io; 1303 struct mapped_device *md = io->md; 1304 int r; 1305 1306 clone->bi_end_io = clone_endio; 1307 1308 /* 1309 * Map the clone. 1310 */ 1311 tio->old_sector = clone->bi_iter.bi_sector; 1312 1313 if (static_branch_unlikely(&swap_bios_enabled) && 1314 unlikely(swap_bios_limit(ti, clone))) { 1315 int latch = get_swap_bios(); 1316 if (unlikely(latch != md->swap_bios)) 1317 __set_swap_bios_limit(md, latch); 1318 down(&md->swap_bios_semaphore); 1319 } 1320 1321 if (static_branch_unlikely(&zoned_enabled)) { 1322 /* 1323 * Check if the IO needs a special mapping due to zone append 1324 * emulation on zoned target. In this case, dm_zone_map_bio() 1325 * calls the target map operation. 1326 */ 1327 if (unlikely(dm_emulate_zone_append(md))) 1328 r = dm_zone_map_bio(tio); 1329 else 1330 r = ti->type->map(ti, clone); 1331 } else 1332 r = ti->type->map(ti, clone); 1333 1334 switch (r) { 1335 case DM_MAPIO_SUBMITTED: 1336 /* target has assumed ownership of this io */ 1337 if (!ti->accounts_remapped_io) 1338 dm_start_io_acct(io, clone); 1339 break; 1340 case DM_MAPIO_REMAPPED: 1341 dm_submit_bio_remap(clone, NULL); 1342 break; 1343 case DM_MAPIO_KILL: 1344 case DM_MAPIO_REQUEUE: 1345 if (static_branch_unlikely(&swap_bios_enabled) && 1346 unlikely(swap_bios_limit(ti, clone))) 1347 up(&md->swap_bios_semaphore); 1348 free_tio(clone); 1349 if (r == DM_MAPIO_KILL) 1350 dm_io_dec_pending(io, BLK_STS_IOERR); 1351 else 1352 dm_io_dec_pending(io, BLK_STS_DM_REQUEUE); 1353 break; 1354 default: 1355 DMWARN("unimplemented target map return value: %d", r); 1356 BUG(); 1357 } 1358 } 1359 1360 static void setup_split_accounting(struct clone_info *ci, unsigned len) 1361 { 1362 struct dm_io *io = ci->io; 1363 1364 if (ci->sector_count > len) { 1365 /* 1366 * Split needed, save the mapped part for accounting. 1367 * NOTE: dm_accept_partial_bio() will update accordingly. 1368 */ 1369 dm_io_set_flag(io, DM_IO_WAS_SPLIT); 1370 io->sectors = len; 1371 } 1372 1373 if (static_branch_unlikely(&stats_enabled) && 1374 unlikely(dm_stats_used(&io->md->stats))) { 1375 /* 1376 * Save bi_sector in terms of its offset from end of 1377 * original bio, only needed for DM-stats' benefit. 1378 * - saved regardless of whether split needed so that 1379 * dm_accept_partial_bio() doesn't need to. 1380 */ 1381 io->sector_offset = bio_end_sector(ci->bio) - ci->sector; 1382 } 1383 } 1384 1385 static void alloc_multiple_bios(struct bio_list *blist, struct clone_info *ci, 1386 struct dm_target *ti, unsigned num_bios) 1387 { 1388 struct bio *bio; 1389 int try; 1390 1391 for (try = 0; try < 2; try++) { 1392 int bio_nr; 1393 1394 if (try) 1395 mutex_lock(&ci->io->md->table_devices_lock); 1396 for (bio_nr = 0; bio_nr < num_bios; bio_nr++) { 1397 bio = alloc_tio(ci, ti, bio_nr, NULL, 1398 try ? GFP_NOIO : GFP_NOWAIT); 1399 if (!bio) 1400 break; 1401 1402 bio_list_add(blist, bio); 1403 } 1404 if (try) 1405 mutex_unlock(&ci->io->md->table_devices_lock); 1406 if (bio_nr == num_bios) 1407 return; 1408 1409 while ((bio = bio_list_pop(blist))) 1410 free_tio(bio); 1411 } 1412 } 1413 1414 static int __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti, 1415 unsigned num_bios, unsigned *len) 1416 { 1417 struct bio_list blist = BIO_EMPTY_LIST; 1418 struct bio *clone; 1419 int ret = 0; 1420 1421 switch (num_bios) { 1422 case 0: 1423 break; 1424 case 1: 1425 if (len) 1426 setup_split_accounting(ci, *len); 1427 clone = alloc_tio(ci, ti, 0, len, GFP_NOIO); 1428 __map_bio(clone); 1429 ret = 1; 1430 break; 1431 default: 1432 /* dm_accept_partial_bio() is not supported with shared tio->len_ptr */ 1433 alloc_multiple_bios(&blist, ci, ti, num_bios); 1434 while ((clone = bio_list_pop(&blist))) { 1435 dm_tio_set_flag(clone_to_tio(clone), DM_TIO_IS_DUPLICATE_BIO); 1436 __map_bio(clone); 1437 ret += 1; 1438 } 1439 break; 1440 } 1441 1442 return ret; 1443 } 1444 1445 static void __send_empty_flush(struct clone_info *ci) 1446 { 1447 unsigned target_nr = 0; 1448 struct dm_target *ti; 1449 struct bio flush_bio; 1450 1451 /* 1452 * Use an on-stack bio for this, it's safe since we don't 1453 * need to reference it after submit. It's just used as 1454 * the basis for the clone(s). 1455 */ 1456 bio_init(&flush_bio, ci->io->md->disk->part0, NULL, 0, 1457 REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC); 1458 1459 ci->bio = &flush_bio; 1460 ci->sector_count = 0; 1461 ci->io->tio.clone.bi_iter.bi_size = 0; 1462 1463 while ((ti = dm_table_get_target(ci->map, target_nr++))) { 1464 int bios; 1465 1466 atomic_add(ti->num_flush_bios, &ci->io->io_count); 1467 bios = __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL); 1468 atomic_sub(ti->num_flush_bios - bios, &ci->io->io_count); 1469 } 1470 1471 /* 1472 * alloc_io() takes one extra reference for submission, so the 1473 * reference won't reach 0 without the following subtraction 1474 */ 1475 atomic_sub(1, &ci->io->io_count); 1476 1477 bio_uninit(ci->bio); 1478 } 1479 1480 static void __send_changing_extent_only(struct clone_info *ci, struct dm_target *ti, 1481 unsigned num_bios) 1482 { 1483 unsigned len; 1484 int bios; 1485 1486 len = min_t(sector_t, ci->sector_count, 1487 max_io_len_target_boundary(ti, dm_target_offset(ti, ci->sector))); 1488 1489 atomic_add(num_bios, &ci->io->io_count); 1490 bios = __send_duplicate_bios(ci, ti, num_bios, &len); 1491 /* 1492 * alloc_io() takes one extra reference for submission, so the 1493 * reference won't reach 0 without the following (+1) subtraction 1494 */ 1495 atomic_sub(num_bios - bios + 1, &ci->io->io_count); 1496 1497 ci->sector += len; 1498 ci->sector_count -= len; 1499 } 1500 1501 static bool is_abnormal_io(struct bio *bio) 1502 { 1503 unsigned int op = bio_op(bio); 1504 1505 if (op != REQ_OP_READ && op != REQ_OP_WRITE && op != REQ_OP_FLUSH) { 1506 switch (op) { 1507 case REQ_OP_DISCARD: 1508 case REQ_OP_SECURE_ERASE: 1509 case REQ_OP_WRITE_ZEROES: 1510 return true; 1511 default: 1512 break; 1513 } 1514 } 1515 1516 return false; 1517 } 1518 1519 static blk_status_t __process_abnormal_io(struct clone_info *ci, 1520 struct dm_target *ti) 1521 { 1522 unsigned num_bios = 0; 1523 1524 switch (bio_op(ci->bio)) { 1525 case REQ_OP_DISCARD: 1526 num_bios = ti->num_discard_bios; 1527 break; 1528 case REQ_OP_SECURE_ERASE: 1529 num_bios = ti->num_secure_erase_bios; 1530 break; 1531 case REQ_OP_WRITE_ZEROES: 1532 num_bios = ti->num_write_zeroes_bios; 1533 break; 1534 } 1535 1536 /* 1537 * Even though the device advertised support for this type of 1538 * request, that does not mean every target supports it, and 1539 * reconfiguration might also have changed that since the 1540 * check was performed. 1541 */ 1542 if (unlikely(!num_bios)) 1543 return BLK_STS_NOTSUPP; 1544 1545 __send_changing_extent_only(ci, ti, num_bios); 1546 return BLK_STS_OK; 1547 } 1548 1549 /* 1550 * Reuse ->bi_private as dm_io list head for storing all dm_io instances 1551 * associated with this bio, and this bio's bi_private needs to be 1552 * stored in dm_io->data before the reuse. 1553 * 1554 * bio->bi_private is owned by fs or upper layer, so block layer won't 1555 * touch it after splitting. Meantime it won't be changed by anyone after 1556 * bio is submitted. So this reuse is safe. 1557 */ 1558 static inline struct dm_io **dm_poll_list_head(struct bio *bio) 1559 { 1560 return (struct dm_io **)&bio->bi_private; 1561 } 1562 1563 static void dm_queue_poll_io(struct bio *bio, struct dm_io *io) 1564 { 1565 struct dm_io **head = dm_poll_list_head(bio); 1566 1567 if (!(bio->bi_opf & REQ_DM_POLL_LIST)) { 1568 bio->bi_opf |= REQ_DM_POLL_LIST; 1569 /* 1570 * Save .bi_private into dm_io, so that we can reuse 1571 * .bi_private as dm_io list head for storing dm_io list 1572 */ 1573 io->data = bio->bi_private; 1574 1575 /* tell block layer to poll for completion */ 1576 bio->bi_cookie = ~BLK_QC_T_NONE; 1577 1578 io->next = NULL; 1579 } else { 1580 /* 1581 * bio recursed due to split, reuse original poll list, 1582 * and save bio->bi_private too. 1583 */ 1584 io->data = (*head)->data; 1585 io->next = *head; 1586 } 1587 1588 *head = io; 1589 } 1590 1591 /* 1592 * Select the correct strategy for processing a non-flush bio. 1593 */ 1594 static blk_status_t __split_and_process_bio(struct clone_info *ci) 1595 { 1596 struct bio *clone; 1597 struct dm_target *ti; 1598 unsigned len; 1599 1600 ti = dm_table_find_target(ci->map, ci->sector); 1601 if (unlikely(!ti)) 1602 return BLK_STS_IOERR; 1603 else if (unlikely(ci->is_abnormal_io)) 1604 return __process_abnormal_io(ci, ti); 1605 1606 /* 1607 * Only support bio polling for normal IO, and the target io is 1608 * exactly inside the dm_io instance (verified in dm_poll_dm_io) 1609 */ 1610 ci->submit_as_polled = ci->bio->bi_opf & REQ_POLLED; 1611 1612 len = min_t(sector_t, max_io_len(ti, ci->sector), ci->sector_count); 1613 setup_split_accounting(ci, len); 1614 clone = alloc_tio(ci, ti, 0, &len, GFP_NOIO); 1615 __map_bio(clone); 1616 1617 ci->sector += len; 1618 ci->sector_count -= len; 1619 1620 return BLK_STS_OK; 1621 } 1622 1623 static void init_clone_info(struct clone_info *ci, struct mapped_device *md, 1624 struct dm_table *map, struct bio *bio, bool is_abnormal) 1625 { 1626 ci->map = map; 1627 ci->io = alloc_io(md, bio); 1628 ci->bio = bio; 1629 ci->is_abnormal_io = is_abnormal; 1630 ci->submit_as_polled = false; 1631 ci->sector = bio->bi_iter.bi_sector; 1632 ci->sector_count = bio_sectors(bio); 1633 1634 /* Shouldn't happen but sector_count was being set to 0 so... */ 1635 if (static_branch_unlikely(&zoned_enabled) && 1636 WARN_ON_ONCE(op_is_zone_mgmt(bio_op(bio)) && ci->sector_count)) 1637 ci->sector_count = 0; 1638 } 1639 1640 /* 1641 * Entry point to split a bio into clones and submit them to the targets. 1642 */ 1643 static void dm_split_and_process_bio(struct mapped_device *md, 1644 struct dm_table *map, struct bio *bio) 1645 { 1646 struct clone_info ci; 1647 struct dm_io *io; 1648 blk_status_t error = BLK_STS_OK; 1649 bool is_abnormal; 1650 1651 is_abnormal = is_abnormal_io(bio); 1652 if (unlikely(is_abnormal)) { 1653 /* 1654 * Use blk_queue_split() for abnormal IO (e.g. discard, etc) 1655 * otherwise associated queue_limits won't be imposed. 1656 */ 1657 blk_queue_split(&bio); 1658 } 1659 1660 init_clone_info(&ci, md, map, bio, is_abnormal); 1661 io = ci.io; 1662 1663 if (bio->bi_opf & REQ_PREFLUSH) { 1664 __send_empty_flush(&ci); 1665 /* dm_io_complete submits any data associated with flush */ 1666 goto out; 1667 } 1668 1669 error = __split_and_process_bio(&ci); 1670 if (error || !ci.sector_count) 1671 goto out; 1672 /* 1673 * Remainder must be passed to submit_bio_noacct() so it gets handled 1674 * *after* bios already submitted have been completely processed. 1675 */ 1676 bio_trim(bio, io->sectors, ci.sector_count); 1677 trace_block_split(bio, bio->bi_iter.bi_sector); 1678 bio_inc_remaining(bio); 1679 submit_bio_noacct(bio); 1680 out: 1681 /* 1682 * Drop the extra reference count for non-POLLED bio, and hold one 1683 * reference for POLLED bio, which will be released in dm_poll_bio 1684 * 1685 * Add every dm_io instance into the dm_io list head which is stored 1686 * in bio->bi_private, so that dm_poll_bio can poll them all. 1687 */ 1688 if (error || !ci.submit_as_polled) { 1689 /* 1690 * In case of submission failure, the extra reference for 1691 * submitting io isn't consumed yet 1692 */ 1693 if (error) 1694 atomic_dec(&io->io_count); 1695 dm_io_dec_pending(io, error); 1696 } else 1697 dm_queue_poll_io(bio, io); 1698 } 1699 1700 static void dm_submit_bio(struct bio *bio) 1701 { 1702 struct mapped_device *md = bio->bi_bdev->bd_disk->private_data; 1703 int srcu_idx; 1704 struct dm_table *map; 1705 1706 map = dm_get_live_table_bio(md, &srcu_idx, bio); 1707 1708 /* If suspended, or map not yet available, queue this IO for later */ 1709 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) || 1710 unlikely(!map)) { 1711 if (bio->bi_opf & REQ_NOWAIT) 1712 bio_wouldblock_error(bio); 1713 else if (bio->bi_opf & REQ_RAHEAD) 1714 bio_io_error(bio); 1715 else 1716 queue_io(md, bio); 1717 goto out; 1718 } 1719 1720 dm_split_and_process_bio(md, map, bio); 1721 out: 1722 dm_put_live_table_bio(md, srcu_idx, bio); 1723 } 1724 1725 static bool dm_poll_dm_io(struct dm_io *io, struct io_comp_batch *iob, 1726 unsigned int flags) 1727 { 1728 WARN_ON_ONCE(!dm_tio_is_normal(&io->tio)); 1729 1730 /* don't poll if the mapped io is done */ 1731 if (atomic_read(&io->io_count) > 1) 1732 bio_poll(&io->tio.clone, iob, flags); 1733 1734 /* bio_poll holds the last reference */ 1735 return atomic_read(&io->io_count) == 1; 1736 } 1737 1738 static int dm_poll_bio(struct bio *bio, struct io_comp_batch *iob, 1739 unsigned int flags) 1740 { 1741 struct dm_io **head = dm_poll_list_head(bio); 1742 struct dm_io *list = *head; 1743 struct dm_io *tmp = NULL; 1744 struct dm_io *curr, *next; 1745 1746 /* Only poll normal bio which was marked as REQ_DM_POLL_LIST */ 1747 if (!(bio->bi_opf & REQ_DM_POLL_LIST)) 1748 return 0; 1749 1750 WARN_ON_ONCE(!list); 1751 1752 /* 1753 * Restore .bi_private before possibly completing dm_io. 1754 * 1755 * bio_poll() is only possible once @bio has been completely 1756 * submitted via submit_bio_noacct()'s depth-first submission. 1757 * So there is no dm_queue_poll_io() race associated with 1758 * clearing REQ_DM_POLL_LIST here. 1759 */ 1760 bio->bi_opf &= ~REQ_DM_POLL_LIST; 1761 bio->bi_private = list->data; 1762 1763 for (curr = list, next = curr->next; curr; curr = next, next = 1764 curr ? curr->next : NULL) { 1765 if (dm_poll_dm_io(curr, iob, flags)) { 1766 /* 1767 * clone_endio() has already occurred, so no 1768 * error handling is needed here. 1769 */ 1770 __dm_io_dec_pending(curr); 1771 } else { 1772 curr->next = tmp; 1773 tmp = curr; 1774 } 1775 } 1776 1777 /* Not done? */ 1778 if (tmp) { 1779 bio->bi_opf |= REQ_DM_POLL_LIST; 1780 /* Reset bio->bi_private to dm_io list head */ 1781 *head = tmp; 1782 return 0; 1783 } 1784 return 1; 1785 } 1786 1787 /*----------------------------------------------------------------- 1788 * An IDR is used to keep track of allocated minor numbers. 1789 *---------------------------------------------------------------*/ 1790 static void free_minor(int minor) 1791 { 1792 spin_lock(&_minor_lock); 1793 idr_remove(&_minor_idr, minor); 1794 spin_unlock(&_minor_lock); 1795 } 1796 1797 /* 1798 * See if the device with a specific minor # is free. 1799 */ 1800 static int specific_minor(int minor) 1801 { 1802 int r; 1803 1804 if (minor >= (1 << MINORBITS)) 1805 return -EINVAL; 1806 1807 idr_preload(GFP_KERNEL); 1808 spin_lock(&_minor_lock); 1809 1810 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT); 1811 1812 spin_unlock(&_minor_lock); 1813 idr_preload_end(); 1814 if (r < 0) 1815 return r == -ENOSPC ? -EBUSY : r; 1816 return 0; 1817 } 1818 1819 static int next_free_minor(int *minor) 1820 { 1821 int r; 1822 1823 idr_preload(GFP_KERNEL); 1824 spin_lock(&_minor_lock); 1825 1826 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT); 1827 1828 spin_unlock(&_minor_lock); 1829 idr_preload_end(); 1830 if (r < 0) 1831 return r; 1832 *minor = r; 1833 return 0; 1834 } 1835 1836 static const struct block_device_operations dm_blk_dops; 1837 static const struct block_device_operations dm_rq_blk_dops; 1838 static const struct dax_operations dm_dax_ops; 1839 1840 static void dm_wq_work(struct work_struct *work); 1841 1842 #ifdef CONFIG_BLK_INLINE_ENCRYPTION 1843 static void dm_queue_destroy_crypto_profile(struct request_queue *q) 1844 { 1845 dm_destroy_crypto_profile(q->crypto_profile); 1846 } 1847 1848 #else /* CONFIG_BLK_INLINE_ENCRYPTION */ 1849 1850 static inline void dm_queue_destroy_crypto_profile(struct request_queue *q) 1851 { 1852 } 1853 #endif /* !CONFIG_BLK_INLINE_ENCRYPTION */ 1854 1855 static void cleanup_mapped_device(struct mapped_device *md) 1856 { 1857 if (md->wq) 1858 destroy_workqueue(md->wq); 1859 bioset_exit(&md->bs); 1860 bioset_exit(&md->io_bs); 1861 1862 if (md->dax_dev) { 1863 dax_remove_host(md->disk); 1864 kill_dax(md->dax_dev); 1865 put_dax(md->dax_dev); 1866 md->dax_dev = NULL; 1867 } 1868 1869 dm_cleanup_zoned_dev(md); 1870 if (md->disk) { 1871 spin_lock(&_minor_lock); 1872 md->disk->private_data = NULL; 1873 spin_unlock(&_minor_lock); 1874 if (dm_get_md_type(md) != DM_TYPE_NONE) { 1875 dm_sysfs_exit(md); 1876 del_gendisk(md->disk); 1877 } 1878 dm_queue_destroy_crypto_profile(md->queue); 1879 blk_cleanup_disk(md->disk); 1880 } 1881 1882 if (md->pending_io) { 1883 free_percpu(md->pending_io); 1884 md->pending_io = NULL; 1885 } 1886 1887 cleanup_srcu_struct(&md->io_barrier); 1888 1889 mutex_destroy(&md->suspend_lock); 1890 mutex_destroy(&md->type_lock); 1891 mutex_destroy(&md->table_devices_lock); 1892 mutex_destroy(&md->swap_bios_lock); 1893 1894 dm_mq_cleanup_mapped_device(md); 1895 } 1896 1897 /* 1898 * Allocate and initialise a blank device with a given minor. 1899 */ 1900 static struct mapped_device *alloc_dev(int minor) 1901 { 1902 int r, numa_node_id = dm_get_numa_node(); 1903 struct mapped_device *md; 1904 void *old_md; 1905 1906 md = kvzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id); 1907 if (!md) { 1908 DMWARN("unable to allocate device, out of memory."); 1909 return NULL; 1910 } 1911 1912 if (!try_module_get(THIS_MODULE)) 1913 goto bad_module_get; 1914 1915 /* get a minor number for the dev */ 1916 if (minor == DM_ANY_MINOR) 1917 r = next_free_minor(&minor); 1918 else 1919 r = specific_minor(minor); 1920 if (r < 0) 1921 goto bad_minor; 1922 1923 r = init_srcu_struct(&md->io_barrier); 1924 if (r < 0) 1925 goto bad_io_barrier; 1926 1927 md->numa_node_id = numa_node_id; 1928 md->init_tio_pdu = false; 1929 md->type = DM_TYPE_NONE; 1930 mutex_init(&md->suspend_lock); 1931 mutex_init(&md->type_lock); 1932 mutex_init(&md->table_devices_lock); 1933 spin_lock_init(&md->deferred_lock); 1934 atomic_set(&md->holders, 1); 1935 atomic_set(&md->open_count, 0); 1936 atomic_set(&md->event_nr, 0); 1937 atomic_set(&md->uevent_seq, 0); 1938 INIT_LIST_HEAD(&md->uevent_list); 1939 INIT_LIST_HEAD(&md->table_devices); 1940 spin_lock_init(&md->uevent_lock); 1941 1942 /* 1943 * default to bio-based until DM table is loaded and md->type 1944 * established. If request-based table is loaded: blk-mq will 1945 * override accordingly. 1946 */ 1947 md->disk = blk_alloc_disk(md->numa_node_id); 1948 if (!md->disk) 1949 goto bad; 1950 md->queue = md->disk->queue; 1951 1952 init_waitqueue_head(&md->wait); 1953 INIT_WORK(&md->work, dm_wq_work); 1954 init_waitqueue_head(&md->eventq); 1955 init_completion(&md->kobj_holder.completion); 1956 1957 md->swap_bios = get_swap_bios(); 1958 sema_init(&md->swap_bios_semaphore, md->swap_bios); 1959 mutex_init(&md->swap_bios_lock); 1960 1961 md->disk->major = _major; 1962 md->disk->first_minor = minor; 1963 md->disk->minors = 1; 1964 md->disk->flags |= GENHD_FL_NO_PART; 1965 md->disk->fops = &dm_blk_dops; 1966 md->disk->queue = md->queue; 1967 md->disk->private_data = md; 1968 sprintf(md->disk->disk_name, "dm-%d", minor); 1969 1970 if (IS_ENABLED(CONFIG_FS_DAX)) { 1971 md->dax_dev = alloc_dax(md, &dm_dax_ops); 1972 if (IS_ERR(md->dax_dev)) { 1973 md->dax_dev = NULL; 1974 goto bad; 1975 } 1976 set_dax_nocache(md->dax_dev); 1977 set_dax_nomc(md->dax_dev); 1978 if (dax_add_host(md->dax_dev, md->disk)) 1979 goto bad; 1980 } 1981 1982 format_dev_t(md->name, MKDEV(_major, minor)); 1983 1984 md->wq = alloc_workqueue("kdmflush/%s", WQ_MEM_RECLAIM, 0, md->name); 1985 if (!md->wq) 1986 goto bad; 1987 1988 md->pending_io = alloc_percpu(unsigned long); 1989 if (!md->pending_io) 1990 goto bad; 1991 1992 dm_stats_init(&md->stats); 1993 1994 /* Populate the mapping, nobody knows we exist yet */ 1995 spin_lock(&_minor_lock); 1996 old_md = idr_replace(&_minor_idr, md, minor); 1997 spin_unlock(&_minor_lock); 1998 1999 BUG_ON(old_md != MINOR_ALLOCED); 2000 2001 return md; 2002 2003 bad: 2004 cleanup_mapped_device(md); 2005 bad_io_barrier: 2006 free_minor(minor); 2007 bad_minor: 2008 module_put(THIS_MODULE); 2009 bad_module_get: 2010 kvfree(md); 2011 return NULL; 2012 } 2013 2014 static void unlock_fs(struct mapped_device *md); 2015 2016 static void free_dev(struct mapped_device *md) 2017 { 2018 int minor = MINOR(disk_devt(md->disk)); 2019 2020 unlock_fs(md); 2021 2022 cleanup_mapped_device(md); 2023 2024 free_table_devices(&md->table_devices); 2025 dm_stats_cleanup(&md->stats); 2026 free_minor(minor); 2027 2028 module_put(THIS_MODULE); 2029 kvfree(md); 2030 } 2031 2032 static int __bind_mempools(struct mapped_device *md, struct dm_table *t) 2033 { 2034 struct dm_md_mempools *p = dm_table_get_md_mempools(t); 2035 int ret = 0; 2036 2037 if (dm_table_bio_based(t)) { 2038 /* 2039 * The md may already have mempools that need changing. 2040 * If so, reload bioset because front_pad may have changed 2041 * because a different table was loaded. 2042 */ 2043 bioset_exit(&md->bs); 2044 bioset_exit(&md->io_bs); 2045 2046 } else if (bioset_initialized(&md->bs)) { 2047 /* 2048 * There's no need to reload with request-based dm 2049 * because the size of front_pad doesn't change. 2050 * Note for future: If you are to reload bioset, 2051 * prep-ed requests in the queue may refer 2052 * to bio from the old bioset, so you must walk 2053 * through the queue to unprep. 2054 */ 2055 goto out; 2056 } 2057 2058 BUG_ON(!p || 2059 bioset_initialized(&md->bs) || 2060 bioset_initialized(&md->io_bs)); 2061 2062 ret = bioset_init_from_src(&md->bs, &p->bs); 2063 if (ret) 2064 goto out; 2065 ret = bioset_init_from_src(&md->io_bs, &p->io_bs); 2066 if (ret) 2067 bioset_exit(&md->bs); 2068 out: 2069 /* mempool bind completed, no longer need any mempools in the table */ 2070 dm_table_free_md_mempools(t); 2071 return ret; 2072 } 2073 2074 /* 2075 * Bind a table to the device. 2076 */ 2077 static void event_callback(void *context) 2078 { 2079 unsigned long flags; 2080 LIST_HEAD(uevents); 2081 struct mapped_device *md = (struct mapped_device *) context; 2082 2083 spin_lock_irqsave(&md->uevent_lock, flags); 2084 list_splice_init(&md->uevent_list, &uevents); 2085 spin_unlock_irqrestore(&md->uevent_lock, flags); 2086 2087 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj); 2088 2089 atomic_inc(&md->event_nr); 2090 wake_up(&md->eventq); 2091 dm_issue_global_event(); 2092 } 2093 2094 /* 2095 * Returns old map, which caller must destroy. 2096 */ 2097 static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t, 2098 struct queue_limits *limits) 2099 { 2100 struct dm_table *old_map; 2101 sector_t size; 2102 int ret; 2103 2104 lockdep_assert_held(&md->suspend_lock); 2105 2106 size = dm_table_get_size(t); 2107 2108 /* 2109 * Wipe any geometry if the size of the table changed. 2110 */ 2111 if (size != dm_get_size(md)) 2112 memset(&md->geometry, 0, sizeof(md->geometry)); 2113 2114 if (!get_capacity(md->disk)) 2115 set_capacity(md->disk, size); 2116 else 2117 set_capacity_and_notify(md->disk, size); 2118 2119 dm_table_event_callback(t, event_callback, md); 2120 2121 if (dm_table_request_based(t)) { 2122 /* 2123 * Leverage the fact that request-based DM targets are 2124 * immutable singletons - used to optimize dm_mq_queue_rq. 2125 */ 2126 md->immutable_target = dm_table_get_immutable_target(t); 2127 } 2128 2129 ret = __bind_mempools(md, t); 2130 if (ret) { 2131 old_map = ERR_PTR(ret); 2132 goto out; 2133 } 2134 2135 ret = dm_table_set_restrictions(t, md->queue, limits); 2136 if (ret) { 2137 old_map = ERR_PTR(ret); 2138 goto out; 2139 } 2140 2141 old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2142 rcu_assign_pointer(md->map, (void *)t); 2143 md->immutable_target_type = dm_table_get_immutable_target_type(t); 2144 2145 if (old_map) 2146 dm_sync_table(md); 2147 out: 2148 return old_map; 2149 } 2150 2151 /* 2152 * Returns unbound table for the caller to free. 2153 */ 2154 static struct dm_table *__unbind(struct mapped_device *md) 2155 { 2156 struct dm_table *map = rcu_dereference_protected(md->map, 1); 2157 2158 if (!map) 2159 return NULL; 2160 2161 dm_table_event_callback(map, NULL, NULL); 2162 RCU_INIT_POINTER(md->map, NULL); 2163 dm_sync_table(md); 2164 2165 return map; 2166 } 2167 2168 /* 2169 * Constructor for a new device. 2170 */ 2171 int dm_create(int minor, struct mapped_device **result) 2172 { 2173 struct mapped_device *md; 2174 2175 md = alloc_dev(minor); 2176 if (!md) 2177 return -ENXIO; 2178 2179 dm_ima_reset_data(md); 2180 2181 *result = md; 2182 return 0; 2183 } 2184 2185 /* 2186 * Functions to manage md->type. 2187 * All are required to hold md->type_lock. 2188 */ 2189 void dm_lock_md_type(struct mapped_device *md) 2190 { 2191 mutex_lock(&md->type_lock); 2192 } 2193 2194 void dm_unlock_md_type(struct mapped_device *md) 2195 { 2196 mutex_unlock(&md->type_lock); 2197 } 2198 2199 void dm_set_md_type(struct mapped_device *md, enum dm_queue_mode type) 2200 { 2201 BUG_ON(!mutex_is_locked(&md->type_lock)); 2202 md->type = type; 2203 } 2204 2205 enum dm_queue_mode dm_get_md_type(struct mapped_device *md) 2206 { 2207 return md->type; 2208 } 2209 2210 struct target_type *dm_get_immutable_target_type(struct mapped_device *md) 2211 { 2212 return md->immutable_target_type; 2213 } 2214 2215 /* 2216 * The queue_limits are only valid as long as you have a reference 2217 * count on 'md'. 2218 */ 2219 struct queue_limits *dm_get_queue_limits(struct mapped_device *md) 2220 { 2221 BUG_ON(!atomic_read(&md->holders)); 2222 return &md->queue->limits; 2223 } 2224 EXPORT_SYMBOL_GPL(dm_get_queue_limits); 2225 2226 /* 2227 * Setup the DM device's queue based on md's type 2228 */ 2229 int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t) 2230 { 2231 enum dm_queue_mode type = dm_table_get_type(t); 2232 struct queue_limits limits; 2233 int r; 2234 2235 switch (type) { 2236 case DM_TYPE_REQUEST_BASED: 2237 md->disk->fops = &dm_rq_blk_dops; 2238 r = dm_mq_init_request_queue(md, t); 2239 if (r) { 2240 DMERR("Cannot initialize queue for request-based dm mapped device"); 2241 return r; 2242 } 2243 break; 2244 case DM_TYPE_BIO_BASED: 2245 case DM_TYPE_DAX_BIO_BASED: 2246 break; 2247 case DM_TYPE_NONE: 2248 WARN_ON_ONCE(true); 2249 break; 2250 } 2251 2252 r = dm_calculate_queue_limits(t, &limits); 2253 if (r) { 2254 DMERR("Cannot calculate initial queue limits"); 2255 return r; 2256 } 2257 r = dm_table_set_restrictions(t, md->queue, &limits); 2258 if (r) 2259 return r; 2260 2261 r = add_disk(md->disk); 2262 if (r) 2263 return r; 2264 2265 r = dm_sysfs_init(md); 2266 if (r) { 2267 del_gendisk(md->disk); 2268 return r; 2269 } 2270 md->type = type; 2271 return 0; 2272 } 2273 2274 struct mapped_device *dm_get_md(dev_t dev) 2275 { 2276 struct mapped_device *md; 2277 unsigned minor = MINOR(dev); 2278 2279 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS)) 2280 return NULL; 2281 2282 spin_lock(&_minor_lock); 2283 2284 md = idr_find(&_minor_idr, minor); 2285 if (!md || md == MINOR_ALLOCED || (MINOR(disk_devt(dm_disk(md))) != minor) || 2286 test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) { 2287 md = NULL; 2288 goto out; 2289 } 2290 dm_get(md); 2291 out: 2292 spin_unlock(&_minor_lock); 2293 2294 return md; 2295 } 2296 EXPORT_SYMBOL_GPL(dm_get_md); 2297 2298 void *dm_get_mdptr(struct mapped_device *md) 2299 { 2300 return md->interface_ptr; 2301 } 2302 2303 void dm_set_mdptr(struct mapped_device *md, void *ptr) 2304 { 2305 md->interface_ptr = ptr; 2306 } 2307 2308 void dm_get(struct mapped_device *md) 2309 { 2310 atomic_inc(&md->holders); 2311 BUG_ON(test_bit(DMF_FREEING, &md->flags)); 2312 } 2313 2314 int dm_hold(struct mapped_device *md) 2315 { 2316 spin_lock(&_minor_lock); 2317 if (test_bit(DMF_FREEING, &md->flags)) { 2318 spin_unlock(&_minor_lock); 2319 return -EBUSY; 2320 } 2321 dm_get(md); 2322 spin_unlock(&_minor_lock); 2323 return 0; 2324 } 2325 EXPORT_SYMBOL_GPL(dm_hold); 2326 2327 const char *dm_device_name(struct mapped_device *md) 2328 { 2329 return md->name; 2330 } 2331 EXPORT_SYMBOL_GPL(dm_device_name); 2332 2333 static void __dm_destroy(struct mapped_device *md, bool wait) 2334 { 2335 struct dm_table *map; 2336 int srcu_idx; 2337 2338 might_sleep(); 2339 2340 spin_lock(&_minor_lock); 2341 idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md)))); 2342 set_bit(DMF_FREEING, &md->flags); 2343 spin_unlock(&_minor_lock); 2344 2345 blk_mark_disk_dead(md->disk); 2346 2347 /* 2348 * Take suspend_lock so that presuspend and postsuspend methods 2349 * do not race with internal suspend. 2350 */ 2351 mutex_lock(&md->suspend_lock); 2352 map = dm_get_live_table(md, &srcu_idx); 2353 if (!dm_suspended_md(md)) { 2354 dm_table_presuspend_targets(map); 2355 set_bit(DMF_SUSPENDED, &md->flags); 2356 set_bit(DMF_POST_SUSPENDING, &md->flags); 2357 dm_table_postsuspend_targets(map); 2358 } 2359 /* dm_put_live_table must be before msleep, otherwise deadlock is possible */ 2360 dm_put_live_table(md, srcu_idx); 2361 mutex_unlock(&md->suspend_lock); 2362 2363 /* 2364 * Rare, but there may be I/O requests still going to complete, 2365 * for example. Wait for all references to disappear. 2366 * No one should increment the reference count of the mapped_device, 2367 * after the mapped_device state becomes DMF_FREEING. 2368 */ 2369 if (wait) 2370 while (atomic_read(&md->holders)) 2371 msleep(1); 2372 else if (atomic_read(&md->holders)) 2373 DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)", 2374 dm_device_name(md), atomic_read(&md->holders)); 2375 2376 dm_table_destroy(__unbind(md)); 2377 free_dev(md); 2378 } 2379 2380 void dm_destroy(struct mapped_device *md) 2381 { 2382 __dm_destroy(md, true); 2383 } 2384 2385 void dm_destroy_immediate(struct mapped_device *md) 2386 { 2387 __dm_destroy(md, false); 2388 } 2389 2390 void dm_put(struct mapped_device *md) 2391 { 2392 atomic_dec(&md->holders); 2393 } 2394 EXPORT_SYMBOL_GPL(dm_put); 2395 2396 static bool dm_in_flight_bios(struct mapped_device *md) 2397 { 2398 int cpu; 2399 unsigned long sum = 0; 2400 2401 for_each_possible_cpu(cpu) 2402 sum += *per_cpu_ptr(md->pending_io, cpu); 2403 2404 return sum != 0; 2405 } 2406 2407 static int dm_wait_for_bios_completion(struct mapped_device *md, unsigned int task_state) 2408 { 2409 int r = 0; 2410 DEFINE_WAIT(wait); 2411 2412 while (true) { 2413 prepare_to_wait(&md->wait, &wait, task_state); 2414 2415 if (!dm_in_flight_bios(md)) 2416 break; 2417 2418 if (signal_pending_state(task_state, current)) { 2419 r = -EINTR; 2420 break; 2421 } 2422 2423 io_schedule(); 2424 } 2425 finish_wait(&md->wait, &wait); 2426 2427 smp_rmb(); 2428 2429 return r; 2430 } 2431 2432 static int dm_wait_for_completion(struct mapped_device *md, unsigned int task_state) 2433 { 2434 int r = 0; 2435 2436 if (!queue_is_mq(md->queue)) 2437 return dm_wait_for_bios_completion(md, task_state); 2438 2439 while (true) { 2440 if (!blk_mq_queue_inflight(md->queue)) 2441 break; 2442 2443 if (signal_pending_state(task_state, current)) { 2444 r = -EINTR; 2445 break; 2446 } 2447 2448 msleep(5); 2449 } 2450 2451 return r; 2452 } 2453 2454 /* 2455 * Process the deferred bios 2456 */ 2457 static void dm_wq_work(struct work_struct *work) 2458 { 2459 struct mapped_device *md = container_of(work, struct mapped_device, work); 2460 struct bio *bio; 2461 2462 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) { 2463 spin_lock_irq(&md->deferred_lock); 2464 bio = bio_list_pop(&md->deferred); 2465 spin_unlock_irq(&md->deferred_lock); 2466 2467 if (!bio) 2468 break; 2469 2470 submit_bio_noacct(bio); 2471 } 2472 } 2473 2474 static void dm_queue_flush(struct mapped_device *md) 2475 { 2476 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 2477 smp_mb__after_atomic(); 2478 queue_work(md->wq, &md->work); 2479 } 2480 2481 /* 2482 * Swap in a new table, returning the old one for the caller to destroy. 2483 */ 2484 struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table) 2485 { 2486 struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL); 2487 struct queue_limits limits; 2488 int r; 2489 2490 mutex_lock(&md->suspend_lock); 2491 2492 /* device must be suspended */ 2493 if (!dm_suspended_md(md)) 2494 goto out; 2495 2496 /* 2497 * If the new table has no data devices, retain the existing limits. 2498 * This helps multipath with queue_if_no_path if all paths disappear, 2499 * then new I/O is queued based on these limits, and then some paths 2500 * reappear. 2501 */ 2502 if (dm_table_has_no_data_devices(table)) { 2503 live_map = dm_get_live_table_fast(md); 2504 if (live_map) 2505 limits = md->queue->limits; 2506 dm_put_live_table_fast(md); 2507 } 2508 2509 if (!live_map) { 2510 r = dm_calculate_queue_limits(table, &limits); 2511 if (r) { 2512 map = ERR_PTR(r); 2513 goto out; 2514 } 2515 } 2516 2517 map = __bind(md, table, &limits); 2518 dm_issue_global_event(); 2519 2520 out: 2521 mutex_unlock(&md->suspend_lock); 2522 return map; 2523 } 2524 2525 /* 2526 * Functions to lock and unlock any filesystem running on the 2527 * device. 2528 */ 2529 static int lock_fs(struct mapped_device *md) 2530 { 2531 int r; 2532 2533 WARN_ON(test_bit(DMF_FROZEN, &md->flags)); 2534 2535 r = freeze_bdev(md->disk->part0); 2536 if (!r) 2537 set_bit(DMF_FROZEN, &md->flags); 2538 return r; 2539 } 2540 2541 static void unlock_fs(struct mapped_device *md) 2542 { 2543 if (!test_bit(DMF_FROZEN, &md->flags)) 2544 return; 2545 thaw_bdev(md->disk->part0); 2546 clear_bit(DMF_FROZEN, &md->flags); 2547 } 2548 2549 /* 2550 * @suspend_flags: DM_SUSPEND_LOCKFS_FLAG and/or DM_SUSPEND_NOFLUSH_FLAG 2551 * @task_state: e.g. TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE 2552 * @dmf_suspended_flag: DMF_SUSPENDED or DMF_SUSPENDED_INTERNALLY 2553 * 2554 * If __dm_suspend returns 0, the device is completely quiescent 2555 * now. There is no request-processing activity. All new requests 2556 * are being added to md->deferred list. 2557 */ 2558 static int __dm_suspend(struct mapped_device *md, struct dm_table *map, 2559 unsigned suspend_flags, unsigned int task_state, 2560 int dmf_suspended_flag) 2561 { 2562 bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG; 2563 bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG; 2564 int r; 2565 2566 lockdep_assert_held(&md->suspend_lock); 2567 2568 /* 2569 * DMF_NOFLUSH_SUSPENDING must be set before presuspend. 2570 * This flag is cleared before dm_suspend returns. 2571 */ 2572 if (noflush) 2573 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 2574 else 2575 DMDEBUG("%s: suspending with flush", dm_device_name(md)); 2576 2577 /* 2578 * This gets reverted if there's an error later and the targets 2579 * provide the .presuspend_undo hook. 2580 */ 2581 dm_table_presuspend_targets(map); 2582 2583 /* 2584 * Flush I/O to the device. 2585 * Any I/O submitted after lock_fs() may not be flushed. 2586 * noflush takes precedence over do_lockfs. 2587 * (lock_fs() flushes I/Os and waits for them to complete.) 2588 */ 2589 if (!noflush && do_lockfs) { 2590 r = lock_fs(md); 2591 if (r) { 2592 dm_table_presuspend_undo_targets(map); 2593 return r; 2594 } 2595 } 2596 2597 /* 2598 * Here we must make sure that no processes are submitting requests 2599 * to target drivers i.e. no one may be executing 2600 * dm_split_and_process_bio from dm_submit_bio. 2601 * 2602 * To get all processes out of dm_split_and_process_bio in dm_submit_bio, 2603 * we take the write lock. To prevent any process from reentering 2604 * dm_split_and_process_bio from dm_submit_bio and quiesce the thread 2605 * (dm_wq_work), we set DMF_BLOCK_IO_FOR_SUSPEND and call 2606 * flush_workqueue(md->wq). 2607 */ 2608 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 2609 if (map) 2610 synchronize_srcu(&md->io_barrier); 2611 2612 /* 2613 * Stop md->queue before flushing md->wq in case request-based 2614 * dm defers requests to md->wq from md->queue. 2615 */ 2616 if (dm_request_based(md)) 2617 dm_stop_queue(md->queue); 2618 2619 flush_workqueue(md->wq); 2620 2621 /* 2622 * At this point no more requests are entering target request routines. 2623 * We call dm_wait_for_completion to wait for all existing requests 2624 * to finish. 2625 */ 2626 r = dm_wait_for_completion(md, task_state); 2627 if (!r) 2628 set_bit(dmf_suspended_flag, &md->flags); 2629 2630 if (noflush) 2631 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 2632 if (map) 2633 synchronize_srcu(&md->io_barrier); 2634 2635 /* were we interrupted ? */ 2636 if (r < 0) { 2637 dm_queue_flush(md); 2638 2639 if (dm_request_based(md)) 2640 dm_start_queue(md->queue); 2641 2642 unlock_fs(md); 2643 dm_table_presuspend_undo_targets(map); 2644 /* pushback list is already flushed, so skip flush */ 2645 } 2646 2647 return r; 2648 } 2649 2650 /* 2651 * We need to be able to change a mapping table under a mounted 2652 * filesystem. For example we might want to move some data in 2653 * the background. Before the table can be swapped with 2654 * dm_bind_table, dm_suspend must be called to flush any in 2655 * flight bios and ensure that any further io gets deferred. 2656 */ 2657 /* 2658 * Suspend mechanism in request-based dm. 2659 * 2660 * 1. Flush all I/Os by lock_fs() if needed. 2661 * 2. Stop dispatching any I/O by stopping the request_queue. 2662 * 3. Wait for all in-flight I/Os to be completed or requeued. 2663 * 2664 * To abort suspend, start the request_queue. 2665 */ 2666 int dm_suspend(struct mapped_device *md, unsigned suspend_flags) 2667 { 2668 struct dm_table *map = NULL; 2669 int r = 0; 2670 2671 retry: 2672 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); 2673 2674 if (dm_suspended_md(md)) { 2675 r = -EINVAL; 2676 goto out_unlock; 2677 } 2678 2679 if (dm_suspended_internally_md(md)) { 2680 /* already internally suspended, wait for internal resume */ 2681 mutex_unlock(&md->suspend_lock); 2682 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); 2683 if (r) 2684 return r; 2685 goto retry; 2686 } 2687 2688 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2689 2690 r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED); 2691 if (r) 2692 goto out_unlock; 2693 2694 set_bit(DMF_POST_SUSPENDING, &md->flags); 2695 dm_table_postsuspend_targets(map); 2696 clear_bit(DMF_POST_SUSPENDING, &md->flags); 2697 2698 out_unlock: 2699 mutex_unlock(&md->suspend_lock); 2700 return r; 2701 } 2702 2703 static int __dm_resume(struct mapped_device *md, struct dm_table *map) 2704 { 2705 if (map) { 2706 int r = dm_table_resume_targets(map); 2707 if (r) 2708 return r; 2709 } 2710 2711 dm_queue_flush(md); 2712 2713 /* 2714 * Flushing deferred I/Os must be done after targets are resumed 2715 * so that mapping of targets can work correctly. 2716 * Request-based dm is queueing the deferred I/Os in its request_queue. 2717 */ 2718 if (dm_request_based(md)) 2719 dm_start_queue(md->queue); 2720 2721 unlock_fs(md); 2722 2723 return 0; 2724 } 2725 2726 int dm_resume(struct mapped_device *md) 2727 { 2728 int r; 2729 struct dm_table *map = NULL; 2730 2731 retry: 2732 r = -EINVAL; 2733 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); 2734 2735 if (!dm_suspended_md(md)) 2736 goto out; 2737 2738 if (dm_suspended_internally_md(md)) { 2739 /* already internally suspended, wait for internal resume */ 2740 mutex_unlock(&md->suspend_lock); 2741 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); 2742 if (r) 2743 return r; 2744 goto retry; 2745 } 2746 2747 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2748 if (!map || !dm_table_get_size(map)) 2749 goto out; 2750 2751 r = __dm_resume(md, map); 2752 if (r) 2753 goto out; 2754 2755 clear_bit(DMF_SUSPENDED, &md->flags); 2756 out: 2757 mutex_unlock(&md->suspend_lock); 2758 2759 return r; 2760 } 2761 2762 /* 2763 * Internal suspend/resume works like userspace-driven suspend. It waits 2764 * until all bios finish and prevents issuing new bios to the target drivers. 2765 * It may be used only from the kernel. 2766 */ 2767 2768 static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags) 2769 { 2770 struct dm_table *map = NULL; 2771 2772 lockdep_assert_held(&md->suspend_lock); 2773 2774 if (md->internal_suspend_count++) 2775 return; /* nested internal suspend */ 2776 2777 if (dm_suspended_md(md)) { 2778 set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 2779 return; /* nest suspend */ 2780 } 2781 2782 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2783 2784 /* 2785 * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is 2786 * supported. Properly supporting a TASK_INTERRUPTIBLE internal suspend 2787 * would require changing .presuspend to return an error -- avoid this 2788 * until there is a need for more elaborate variants of internal suspend. 2789 */ 2790 (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE, 2791 DMF_SUSPENDED_INTERNALLY); 2792 2793 set_bit(DMF_POST_SUSPENDING, &md->flags); 2794 dm_table_postsuspend_targets(map); 2795 clear_bit(DMF_POST_SUSPENDING, &md->flags); 2796 } 2797 2798 static void __dm_internal_resume(struct mapped_device *md) 2799 { 2800 BUG_ON(!md->internal_suspend_count); 2801 2802 if (--md->internal_suspend_count) 2803 return; /* resume from nested internal suspend */ 2804 2805 if (dm_suspended_md(md)) 2806 goto done; /* resume from nested suspend */ 2807 2808 /* 2809 * NOTE: existing callers don't need to call dm_table_resume_targets 2810 * (which may fail -- so best to avoid it for now by passing NULL map) 2811 */ 2812 (void) __dm_resume(md, NULL); 2813 2814 done: 2815 clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 2816 smp_mb__after_atomic(); 2817 wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY); 2818 } 2819 2820 void dm_internal_suspend_noflush(struct mapped_device *md) 2821 { 2822 mutex_lock(&md->suspend_lock); 2823 __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG); 2824 mutex_unlock(&md->suspend_lock); 2825 } 2826 EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush); 2827 2828 void dm_internal_resume(struct mapped_device *md) 2829 { 2830 mutex_lock(&md->suspend_lock); 2831 __dm_internal_resume(md); 2832 mutex_unlock(&md->suspend_lock); 2833 } 2834 EXPORT_SYMBOL_GPL(dm_internal_resume); 2835 2836 /* 2837 * Fast variants of internal suspend/resume hold md->suspend_lock, 2838 * which prevents interaction with userspace-driven suspend. 2839 */ 2840 2841 void dm_internal_suspend_fast(struct mapped_device *md) 2842 { 2843 mutex_lock(&md->suspend_lock); 2844 if (dm_suspended_md(md) || dm_suspended_internally_md(md)) 2845 return; 2846 2847 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 2848 synchronize_srcu(&md->io_barrier); 2849 flush_workqueue(md->wq); 2850 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE); 2851 } 2852 EXPORT_SYMBOL_GPL(dm_internal_suspend_fast); 2853 2854 void dm_internal_resume_fast(struct mapped_device *md) 2855 { 2856 if (dm_suspended_md(md) || dm_suspended_internally_md(md)) 2857 goto done; 2858 2859 dm_queue_flush(md); 2860 2861 done: 2862 mutex_unlock(&md->suspend_lock); 2863 } 2864 EXPORT_SYMBOL_GPL(dm_internal_resume_fast); 2865 2866 /*----------------------------------------------------------------- 2867 * Event notification. 2868 *---------------------------------------------------------------*/ 2869 int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action, 2870 unsigned cookie) 2871 { 2872 int r; 2873 unsigned noio_flag; 2874 char udev_cookie[DM_COOKIE_LENGTH]; 2875 char *envp[] = { udev_cookie, NULL }; 2876 2877 noio_flag = memalloc_noio_save(); 2878 2879 if (!cookie) 2880 r = kobject_uevent(&disk_to_dev(md->disk)->kobj, action); 2881 else { 2882 snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u", 2883 DM_COOKIE_ENV_VAR_NAME, cookie); 2884 r = kobject_uevent_env(&disk_to_dev(md->disk)->kobj, 2885 action, envp); 2886 } 2887 2888 memalloc_noio_restore(noio_flag); 2889 2890 return r; 2891 } 2892 2893 uint32_t dm_next_uevent_seq(struct mapped_device *md) 2894 { 2895 return atomic_add_return(1, &md->uevent_seq); 2896 } 2897 2898 uint32_t dm_get_event_nr(struct mapped_device *md) 2899 { 2900 return atomic_read(&md->event_nr); 2901 } 2902 2903 int dm_wait_event(struct mapped_device *md, int event_nr) 2904 { 2905 return wait_event_interruptible(md->eventq, 2906 (event_nr != atomic_read(&md->event_nr))); 2907 } 2908 2909 void dm_uevent_add(struct mapped_device *md, struct list_head *elist) 2910 { 2911 unsigned long flags; 2912 2913 spin_lock_irqsave(&md->uevent_lock, flags); 2914 list_add(elist, &md->uevent_list); 2915 spin_unlock_irqrestore(&md->uevent_lock, flags); 2916 } 2917 2918 /* 2919 * The gendisk is only valid as long as you have a reference 2920 * count on 'md'. 2921 */ 2922 struct gendisk *dm_disk(struct mapped_device *md) 2923 { 2924 return md->disk; 2925 } 2926 EXPORT_SYMBOL_GPL(dm_disk); 2927 2928 struct kobject *dm_kobject(struct mapped_device *md) 2929 { 2930 return &md->kobj_holder.kobj; 2931 } 2932 2933 struct mapped_device *dm_get_from_kobject(struct kobject *kobj) 2934 { 2935 struct mapped_device *md; 2936 2937 md = container_of(kobj, struct mapped_device, kobj_holder.kobj); 2938 2939 spin_lock(&_minor_lock); 2940 if (test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) { 2941 md = NULL; 2942 goto out; 2943 } 2944 dm_get(md); 2945 out: 2946 spin_unlock(&_minor_lock); 2947 2948 return md; 2949 } 2950 2951 int dm_suspended_md(struct mapped_device *md) 2952 { 2953 return test_bit(DMF_SUSPENDED, &md->flags); 2954 } 2955 2956 static int dm_post_suspending_md(struct mapped_device *md) 2957 { 2958 return test_bit(DMF_POST_SUSPENDING, &md->flags); 2959 } 2960 2961 int dm_suspended_internally_md(struct mapped_device *md) 2962 { 2963 return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 2964 } 2965 2966 int dm_test_deferred_remove_flag(struct mapped_device *md) 2967 { 2968 return test_bit(DMF_DEFERRED_REMOVE, &md->flags); 2969 } 2970 2971 int dm_suspended(struct dm_target *ti) 2972 { 2973 return dm_suspended_md(ti->table->md); 2974 } 2975 EXPORT_SYMBOL_GPL(dm_suspended); 2976 2977 int dm_post_suspending(struct dm_target *ti) 2978 { 2979 return dm_post_suspending_md(ti->table->md); 2980 } 2981 EXPORT_SYMBOL_GPL(dm_post_suspending); 2982 2983 int dm_noflush_suspending(struct dm_target *ti) 2984 { 2985 return __noflush_suspending(ti->table->md); 2986 } 2987 EXPORT_SYMBOL_GPL(dm_noflush_suspending); 2988 2989 struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_queue_mode type, 2990 unsigned per_io_data_size, unsigned min_pool_size, 2991 bool integrity, bool poll) 2992 { 2993 struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id); 2994 unsigned int pool_size = 0; 2995 unsigned int front_pad, io_front_pad; 2996 int ret; 2997 2998 if (!pools) 2999 return NULL; 3000 3001 switch (type) { 3002 case DM_TYPE_BIO_BASED: 3003 case DM_TYPE_DAX_BIO_BASED: 3004 pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size); 3005 front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + DM_TARGET_IO_BIO_OFFSET; 3006 io_front_pad = roundup(per_io_data_size, __alignof__(struct dm_io)) + DM_IO_BIO_OFFSET; 3007 ret = bioset_init(&pools->io_bs, pool_size, io_front_pad, poll ? BIOSET_PERCPU_CACHE : 0); 3008 if (ret) 3009 goto out; 3010 if (integrity && bioset_integrity_create(&pools->io_bs, pool_size)) 3011 goto out; 3012 break; 3013 case DM_TYPE_REQUEST_BASED: 3014 pool_size = max(dm_get_reserved_rq_based_ios(), min_pool_size); 3015 front_pad = offsetof(struct dm_rq_clone_bio_info, clone); 3016 /* per_io_data_size is used for blk-mq pdu at queue allocation */ 3017 break; 3018 default: 3019 BUG(); 3020 } 3021 3022 ret = bioset_init(&pools->bs, pool_size, front_pad, 0); 3023 if (ret) 3024 goto out; 3025 3026 if (integrity && bioset_integrity_create(&pools->bs, pool_size)) 3027 goto out; 3028 3029 return pools; 3030 3031 out: 3032 dm_free_md_mempools(pools); 3033 3034 return NULL; 3035 } 3036 3037 void dm_free_md_mempools(struct dm_md_mempools *pools) 3038 { 3039 if (!pools) 3040 return; 3041 3042 bioset_exit(&pools->bs); 3043 bioset_exit(&pools->io_bs); 3044 3045 kfree(pools); 3046 } 3047 3048 struct dm_pr { 3049 u64 old_key; 3050 u64 new_key; 3051 u32 flags; 3052 bool fail_early; 3053 }; 3054 3055 static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn, 3056 void *data) 3057 { 3058 struct mapped_device *md = bdev->bd_disk->private_data; 3059 struct dm_table *table; 3060 struct dm_target *ti; 3061 int ret = -ENOTTY, srcu_idx; 3062 3063 table = dm_get_live_table(md, &srcu_idx); 3064 if (!table || !dm_table_get_size(table)) 3065 goto out; 3066 3067 /* We only support devices that have a single target */ 3068 if (dm_table_get_num_targets(table) != 1) 3069 goto out; 3070 ti = dm_table_get_target(table, 0); 3071 3072 ret = -EINVAL; 3073 if (!ti->type->iterate_devices) 3074 goto out; 3075 3076 ret = ti->type->iterate_devices(ti, fn, data); 3077 out: 3078 dm_put_live_table(md, srcu_idx); 3079 return ret; 3080 } 3081 3082 /* 3083 * For register / unregister we need to manually call out to every path. 3084 */ 3085 static int __dm_pr_register(struct dm_target *ti, struct dm_dev *dev, 3086 sector_t start, sector_t len, void *data) 3087 { 3088 struct dm_pr *pr = data; 3089 const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops; 3090 3091 if (!ops || !ops->pr_register) 3092 return -EOPNOTSUPP; 3093 return ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags); 3094 } 3095 3096 static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key, 3097 u32 flags) 3098 { 3099 struct dm_pr pr = { 3100 .old_key = old_key, 3101 .new_key = new_key, 3102 .flags = flags, 3103 .fail_early = true, 3104 }; 3105 int ret; 3106 3107 ret = dm_call_pr(bdev, __dm_pr_register, &pr); 3108 if (ret && new_key) { 3109 /* unregister all paths if we failed to register any path */ 3110 pr.old_key = new_key; 3111 pr.new_key = 0; 3112 pr.flags = 0; 3113 pr.fail_early = false; 3114 dm_call_pr(bdev, __dm_pr_register, &pr); 3115 } 3116 3117 return ret; 3118 } 3119 3120 static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type, 3121 u32 flags) 3122 { 3123 struct mapped_device *md = bdev->bd_disk->private_data; 3124 const struct pr_ops *ops; 3125 int r, srcu_idx; 3126 3127 r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 3128 if (r < 0) 3129 goto out; 3130 3131 ops = bdev->bd_disk->fops->pr_ops; 3132 if (ops && ops->pr_reserve) 3133 r = ops->pr_reserve(bdev, key, type, flags); 3134 else 3135 r = -EOPNOTSUPP; 3136 out: 3137 dm_unprepare_ioctl(md, srcu_idx); 3138 return r; 3139 } 3140 3141 static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type) 3142 { 3143 struct mapped_device *md = bdev->bd_disk->private_data; 3144 const struct pr_ops *ops; 3145 int r, srcu_idx; 3146 3147 r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 3148 if (r < 0) 3149 goto out; 3150 3151 ops = bdev->bd_disk->fops->pr_ops; 3152 if (ops && ops->pr_release) 3153 r = ops->pr_release(bdev, key, type); 3154 else 3155 r = -EOPNOTSUPP; 3156 out: 3157 dm_unprepare_ioctl(md, srcu_idx); 3158 return r; 3159 } 3160 3161 static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key, 3162 enum pr_type type, bool abort) 3163 { 3164 struct mapped_device *md = bdev->bd_disk->private_data; 3165 const struct pr_ops *ops; 3166 int r, srcu_idx; 3167 3168 r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 3169 if (r < 0) 3170 goto out; 3171 3172 ops = bdev->bd_disk->fops->pr_ops; 3173 if (ops && ops->pr_preempt) 3174 r = ops->pr_preempt(bdev, old_key, new_key, type, abort); 3175 else 3176 r = -EOPNOTSUPP; 3177 out: 3178 dm_unprepare_ioctl(md, srcu_idx); 3179 return r; 3180 } 3181 3182 static int dm_pr_clear(struct block_device *bdev, u64 key) 3183 { 3184 struct mapped_device *md = bdev->bd_disk->private_data; 3185 const struct pr_ops *ops; 3186 int r, srcu_idx; 3187 3188 r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 3189 if (r < 0) 3190 goto out; 3191 3192 ops = bdev->bd_disk->fops->pr_ops; 3193 if (ops && ops->pr_clear) 3194 r = ops->pr_clear(bdev, key); 3195 else 3196 r = -EOPNOTSUPP; 3197 out: 3198 dm_unprepare_ioctl(md, srcu_idx); 3199 return r; 3200 } 3201 3202 static const struct pr_ops dm_pr_ops = { 3203 .pr_register = dm_pr_register, 3204 .pr_reserve = dm_pr_reserve, 3205 .pr_release = dm_pr_release, 3206 .pr_preempt = dm_pr_preempt, 3207 .pr_clear = dm_pr_clear, 3208 }; 3209 3210 static const struct block_device_operations dm_blk_dops = { 3211 .submit_bio = dm_submit_bio, 3212 .poll_bio = dm_poll_bio, 3213 .open = dm_blk_open, 3214 .release = dm_blk_close, 3215 .ioctl = dm_blk_ioctl, 3216 .getgeo = dm_blk_getgeo, 3217 .report_zones = dm_blk_report_zones, 3218 .pr_ops = &dm_pr_ops, 3219 .owner = THIS_MODULE 3220 }; 3221 3222 static const struct block_device_operations dm_rq_blk_dops = { 3223 .open = dm_blk_open, 3224 .release = dm_blk_close, 3225 .ioctl = dm_blk_ioctl, 3226 .getgeo = dm_blk_getgeo, 3227 .pr_ops = &dm_pr_ops, 3228 .owner = THIS_MODULE 3229 }; 3230 3231 static const struct dax_operations dm_dax_ops = { 3232 .direct_access = dm_dax_direct_access, 3233 .zero_page_range = dm_dax_zero_page_range, 3234 }; 3235 3236 /* 3237 * module hooks 3238 */ 3239 module_init(dm_init); 3240 module_exit(dm_exit); 3241 3242 module_param(major, uint, 0); 3243 MODULE_PARM_DESC(major, "The major number of the device mapper"); 3244 3245 module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR); 3246 MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools"); 3247 3248 module_param(dm_numa_node, int, S_IRUGO | S_IWUSR); 3249 MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations"); 3250 3251 module_param(swap_bios, int, S_IRUGO | S_IWUSR); 3252 MODULE_PARM_DESC(swap_bios, "Maximum allowed inflight swap IOs"); 3253 3254 MODULE_DESCRIPTION(DM_NAME " driver"); 3255 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>"); 3256 MODULE_LICENSE("GPL"); 3257