1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited. 4 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. 5 * 6 * This file is released under the GPL. 7 */ 8 9 #include "dm-core.h" 10 #include "dm-rq.h" 11 #include "dm-uevent.h" 12 #include "dm-ima.h" 13 14 #include <linux/init.h> 15 #include <linux/module.h> 16 #include <linux/mutex.h> 17 #include <linux/sched/mm.h> 18 #include <linux/sched/signal.h> 19 #include <linux/blkpg.h> 20 #include <linux/bio.h> 21 #include <linux/mempool.h> 22 #include <linux/dax.h> 23 #include <linux/slab.h> 24 #include <linux/idr.h> 25 #include <linux/uio.h> 26 #include <linux/hdreg.h> 27 #include <linux/delay.h> 28 #include <linux/wait.h> 29 #include <linux/pr.h> 30 #include <linux/refcount.h> 31 #include <linux/part_stat.h> 32 #include <linux/blk-crypto.h> 33 #include <linux/blk-crypto-profile.h> 34 35 #define DM_MSG_PREFIX "core" 36 37 /* 38 * Cookies are numeric values sent with CHANGE and REMOVE 39 * uevents while resuming, removing or renaming the device. 40 */ 41 #define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE" 42 #define DM_COOKIE_LENGTH 24 43 44 /* 45 * For REQ_POLLED fs bio, this flag is set if we link mapped underlying 46 * dm_io into one list, and reuse bio->bi_private as the list head. Before 47 * ending this fs bio, we will recover its ->bi_private. 48 */ 49 #define REQ_DM_POLL_LIST REQ_DRV 50 51 static const char *_name = DM_NAME; 52 53 static unsigned int major; 54 static unsigned int _major; 55 56 static DEFINE_IDR(_minor_idr); 57 58 static DEFINE_SPINLOCK(_minor_lock); 59 60 static void do_deferred_remove(struct work_struct *w); 61 62 static DECLARE_WORK(deferred_remove_work, do_deferred_remove); 63 64 static struct workqueue_struct *deferred_remove_workqueue; 65 66 atomic_t dm_global_event_nr = ATOMIC_INIT(0); 67 DECLARE_WAIT_QUEUE_HEAD(dm_global_eventq); 68 69 void dm_issue_global_event(void) 70 { 71 atomic_inc(&dm_global_event_nr); 72 wake_up(&dm_global_eventq); 73 } 74 75 DEFINE_STATIC_KEY_FALSE(stats_enabled); 76 DEFINE_STATIC_KEY_FALSE(swap_bios_enabled); 77 DEFINE_STATIC_KEY_FALSE(zoned_enabled); 78 79 /* 80 * One of these is allocated (on-stack) per original bio. 81 */ 82 struct clone_info { 83 struct dm_table *map; 84 struct bio *bio; 85 struct dm_io *io; 86 sector_t sector; 87 unsigned int sector_count; 88 bool is_abnormal_io:1; 89 bool submit_as_polled:1; 90 }; 91 92 static inline struct dm_target_io *clone_to_tio(struct bio *clone) 93 { 94 return container_of(clone, struct dm_target_io, clone); 95 } 96 97 void *dm_per_bio_data(struct bio *bio, size_t data_size) 98 { 99 if (!dm_tio_flagged(clone_to_tio(bio), DM_TIO_INSIDE_DM_IO)) 100 return (char *)bio - DM_TARGET_IO_BIO_OFFSET - data_size; 101 return (char *)bio - DM_IO_BIO_OFFSET - data_size; 102 } 103 EXPORT_SYMBOL_GPL(dm_per_bio_data); 104 105 struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size) 106 { 107 struct dm_io *io = (struct dm_io *)((char *)data + data_size); 108 109 if (io->magic == DM_IO_MAGIC) 110 return (struct bio *)((char *)io + DM_IO_BIO_OFFSET); 111 BUG_ON(io->magic != DM_TIO_MAGIC); 112 return (struct bio *)((char *)io + DM_TARGET_IO_BIO_OFFSET); 113 } 114 EXPORT_SYMBOL_GPL(dm_bio_from_per_bio_data); 115 116 unsigned int dm_bio_get_target_bio_nr(const struct bio *bio) 117 { 118 return container_of(bio, struct dm_target_io, clone)->target_bio_nr; 119 } 120 EXPORT_SYMBOL_GPL(dm_bio_get_target_bio_nr); 121 122 #define MINOR_ALLOCED ((void *)-1) 123 124 #define DM_NUMA_NODE NUMA_NO_NODE 125 static int dm_numa_node = DM_NUMA_NODE; 126 127 #define DEFAULT_SWAP_BIOS (8 * 1048576 / PAGE_SIZE) 128 static int swap_bios = DEFAULT_SWAP_BIOS; 129 static int get_swap_bios(void) 130 { 131 int latch = READ_ONCE(swap_bios); 132 133 if (unlikely(latch <= 0)) 134 latch = DEFAULT_SWAP_BIOS; 135 return latch; 136 } 137 138 struct table_device { 139 struct list_head list; 140 refcount_t count; 141 struct dm_dev dm_dev; 142 }; 143 144 /* 145 * Bio-based DM's mempools' reserved IOs set by the user. 146 */ 147 #define RESERVED_BIO_BASED_IOS 16 148 static unsigned int reserved_bio_based_ios = RESERVED_BIO_BASED_IOS; 149 150 static int __dm_get_module_param_int(int *module_param, int min, int max) 151 { 152 int param = READ_ONCE(*module_param); 153 int modified_param = 0; 154 bool modified = true; 155 156 if (param < min) 157 modified_param = min; 158 else if (param > max) 159 modified_param = max; 160 else 161 modified = false; 162 163 if (modified) { 164 (void)cmpxchg(module_param, param, modified_param); 165 param = modified_param; 166 } 167 168 return param; 169 } 170 171 unsigned int __dm_get_module_param(unsigned int *module_param, unsigned int def, unsigned int max) 172 { 173 unsigned int param = READ_ONCE(*module_param); 174 unsigned int modified_param = 0; 175 176 if (!param) 177 modified_param = def; 178 else if (param > max) 179 modified_param = max; 180 181 if (modified_param) { 182 (void)cmpxchg(module_param, param, modified_param); 183 param = modified_param; 184 } 185 186 return param; 187 } 188 189 unsigned int dm_get_reserved_bio_based_ios(void) 190 { 191 return __dm_get_module_param(&reserved_bio_based_ios, 192 RESERVED_BIO_BASED_IOS, DM_RESERVED_MAX_IOS); 193 } 194 EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios); 195 196 static unsigned int dm_get_numa_node(void) 197 { 198 return __dm_get_module_param_int(&dm_numa_node, 199 DM_NUMA_NODE, num_online_nodes() - 1); 200 } 201 202 static int __init local_init(void) 203 { 204 int r; 205 206 r = dm_uevent_init(); 207 if (r) 208 return r; 209 210 deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1); 211 if (!deferred_remove_workqueue) { 212 r = -ENOMEM; 213 goto out_uevent_exit; 214 } 215 216 _major = major; 217 r = register_blkdev(_major, _name); 218 if (r < 0) 219 goto out_free_workqueue; 220 221 if (!_major) 222 _major = r; 223 224 return 0; 225 226 out_free_workqueue: 227 destroy_workqueue(deferred_remove_workqueue); 228 out_uevent_exit: 229 dm_uevent_exit(); 230 231 return r; 232 } 233 234 static void local_exit(void) 235 { 236 destroy_workqueue(deferred_remove_workqueue); 237 238 unregister_blkdev(_major, _name); 239 dm_uevent_exit(); 240 241 _major = 0; 242 243 DMINFO("cleaned up"); 244 } 245 246 static int (*_inits[])(void) __initdata = { 247 local_init, 248 dm_target_init, 249 dm_linear_init, 250 dm_stripe_init, 251 dm_io_init, 252 dm_kcopyd_init, 253 dm_interface_init, 254 dm_statistics_init, 255 }; 256 257 static void (*_exits[])(void) = { 258 local_exit, 259 dm_target_exit, 260 dm_linear_exit, 261 dm_stripe_exit, 262 dm_io_exit, 263 dm_kcopyd_exit, 264 dm_interface_exit, 265 dm_statistics_exit, 266 }; 267 268 static int __init dm_init(void) 269 { 270 const int count = ARRAY_SIZE(_inits); 271 int r, i; 272 273 #if (IS_ENABLED(CONFIG_IMA) && !IS_ENABLED(CONFIG_IMA_DISABLE_HTABLE)) 274 DMWARN("CONFIG_IMA_DISABLE_HTABLE is disabled." 275 " Duplicate IMA measurements will not be recorded in the IMA log."); 276 #endif 277 278 for (i = 0; i < count; i++) { 279 r = _inits[i](); 280 if (r) 281 goto bad; 282 } 283 284 return 0; 285 bad: 286 while (i--) 287 _exits[i](); 288 289 return r; 290 } 291 292 static void __exit dm_exit(void) 293 { 294 int i = ARRAY_SIZE(_exits); 295 296 while (i--) 297 _exits[i](); 298 299 /* 300 * Should be empty by this point. 301 */ 302 idr_destroy(&_minor_idr); 303 } 304 305 /* 306 * Block device functions 307 */ 308 int dm_deleting_md(struct mapped_device *md) 309 { 310 return test_bit(DMF_DELETING, &md->flags); 311 } 312 313 static int dm_blk_open(struct block_device *bdev, fmode_t mode) 314 { 315 struct mapped_device *md; 316 317 spin_lock(&_minor_lock); 318 319 md = bdev->bd_disk->private_data; 320 if (!md) 321 goto out; 322 323 if (test_bit(DMF_FREEING, &md->flags) || 324 dm_deleting_md(md)) { 325 md = NULL; 326 goto out; 327 } 328 329 dm_get(md); 330 atomic_inc(&md->open_count); 331 out: 332 spin_unlock(&_minor_lock); 333 334 return md ? 0 : -ENXIO; 335 } 336 337 static void dm_blk_close(struct gendisk *disk, fmode_t mode) 338 { 339 struct mapped_device *md; 340 341 spin_lock(&_minor_lock); 342 343 md = disk->private_data; 344 if (WARN_ON(!md)) 345 goto out; 346 347 if (atomic_dec_and_test(&md->open_count) && 348 (test_bit(DMF_DEFERRED_REMOVE, &md->flags))) 349 queue_work(deferred_remove_workqueue, &deferred_remove_work); 350 351 dm_put(md); 352 out: 353 spin_unlock(&_minor_lock); 354 } 355 356 int dm_open_count(struct mapped_device *md) 357 { 358 return atomic_read(&md->open_count); 359 } 360 361 /* 362 * Guarantees nothing is using the device before it's deleted. 363 */ 364 int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred) 365 { 366 int r = 0; 367 368 spin_lock(&_minor_lock); 369 370 if (dm_open_count(md)) { 371 r = -EBUSY; 372 if (mark_deferred) 373 set_bit(DMF_DEFERRED_REMOVE, &md->flags); 374 } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags)) 375 r = -EEXIST; 376 else 377 set_bit(DMF_DELETING, &md->flags); 378 379 spin_unlock(&_minor_lock); 380 381 return r; 382 } 383 384 int dm_cancel_deferred_remove(struct mapped_device *md) 385 { 386 int r = 0; 387 388 spin_lock(&_minor_lock); 389 390 if (test_bit(DMF_DELETING, &md->flags)) 391 r = -EBUSY; 392 else 393 clear_bit(DMF_DEFERRED_REMOVE, &md->flags); 394 395 spin_unlock(&_minor_lock); 396 397 return r; 398 } 399 400 static void do_deferred_remove(struct work_struct *w) 401 { 402 dm_deferred_remove(); 403 } 404 405 static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo) 406 { 407 struct mapped_device *md = bdev->bd_disk->private_data; 408 409 return dm_get_geometry(md, geo); 410 } 411 412 static int dm_prepare_ioctl(struct mapped_device *md, int *srcu_idx, 413 struct block_device **bdev) 414 { 415 struct dm_target *ti; 416 struct dm_table *map; 417 int r; 418 419 retry: 420 r = -ENOTTY; 421 map = dm_get_live_table(md, srcu_idx); 422 if (!map || !dm_table_get_size(map)) 423 return r; 424 425 /* We only support devices that have a single target */ 426 if (map->num_targets != 1) 427 return r; 428 429 ti = dm_table_get_target(map, 0); 430 if (!ti->type->prepare_ioctl) 431 return r; 432 433 if (dm_suspended_md(md)) 434 return -EAGAIN; 435 436 r = ti->type->prepare_ioctl(ti, bdev); 437 if (r == -ENOTCONN && !fatal_signal_pending(current)) { 438 dm_put_live_table(md, *srcu_idx); 439 fsleep(10000); 440 goto retry; 441 } 442 443 return r; 444 } 445 446 static void dm_unprepare_ioctl(struct mapped_device *md, int srcu_idx) 447 { 448 dm_put_live_table(md, srcu_idx); 449 } 450 451 static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode, 452 unsigned int cmd, unsigned long arg) 453 { 454 struct mapped_device *md = bdev->bd_disk->private_data; 455 int r, srcu_idx; 456 457 r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 458 if (r < 0) 459 goto out; 460 461 if (r > 0) { 462 /* 463 * Target determined this ioctl is being issued against a 464 * subset of the parent bdev; require extra privileges. 465 */ 466 if (!capable(CAP_SYS_RAWIO)) { 467 DMDEBUG_LIMIT( 468 "%s: sending ioctl %x to DM device without required privilege.", 469 current->comm, cmd); 470 r = -ENOIOCTLCMD; 471 goto out; 472 } 473 } 474 475 if (!bdev->bd_disk->fops->ioctl) 476 r = -ENOTTY; 477 else 478 r = bdev->bd_disk->fops->ioctl(bdev, mode, cmd, arg); 479 out: 480 dm_unprepare_ioctl(md, srcu_idx); 481 return r; 482 } 483 484 u64 dm_start_time_ns_from_clone(struct bio *bio) 485 { 486 return jiffies_to_nsecs(clone_to_tio(bio)->io->start_time); 487 } 488 EXPORT_SYMBOL_GPL(dm_start_time_ns_from_clone); 489 490 static bool bio_is_flush_with_data(struct bio *bio) 491 { 492 return ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size); 493 } 494 495 static void dm_io_acct(struct dm_io *io, bool end) 496 { 497 struct dm_stats_aux *stats_aux = &io->stats_aux; 498 unsigned long start_time = io->start_time; 499 struct mapped_device *md = io->md; 500 struct bio *bio = io->orig_bio; 501 unsigned int sectors; 502 503 /* 504 * If REQ_PREFLUSH set, don't account payload, it will be 505 * submitted (and accounted) after this flush completes. 506 */ 507 if (bio_is_flush_with_data(bio)) 508 sectors = 0; 509 else if (likely(!(dm_io_flagged(io, DM_IO_WAS_SPLIT)))) 510 sectors = bio_sectors(bio); 511 else 512 sectors = io->sectors; 513 514 if (!end) 515 bdev_start_io_acct(bio->bi_bdev, bio_op(bio), start_time); 516 else 517 bdev_end_io_acct(bio->bi_bdev, bio_op(bio), sectors, 518 start_time); 519 520 if (static_branch_unlikely(&stats_enabled) && 521 unlikely(dm_stats_used(&md->stats))) { 522 sector_t sector; 523 524 if (likely(!dm_io_flagged(io, DM_IO_WAS_SPLIT))) 525 sector = bio->bi_iter.bi_sector; 526 else 527 sector = bio_end_sector(bio) - io->sector_offset; 528 529 dm_stats_account_io(&md->stats, bio_data_dir(bio), 530 sector, sectors, 531 end, start_time, stats_aux); 532 } 533 } 534 535 static void __dm_start_io_acct(struct dm_io *io) 536 { 537 dm_io_acct(io, false); 538 } 539 540 static void dm_start_io_acct(struct dm_io *io, struct bio *clone) 541 { 542 /* 543 * Ensure IO accounting is only ever started once. 544 */ 545 if (dm_io_flagged(io, DM_IO_ACCOUNTED)) 546 return; 547 548 /* Expect no possibility for race unless DM_TIO_IS_DUPLICATE_BIO. */ 549 if (!clone || likely(dm_tio_is_normal(clone_to_tio(clone)))) { 550 dm_io_set_flag(io, DM_IO_ACCOUNTED); 551 } else { 552 unsigned long flags; 553 /* Can afford locking given DM_TIO_IS_DUPLICATE_BIO */ 554 spin_lock_irqsave(&io->lock, flags); 555 if (dm_io_flagged(io, DM_IO_ACCOUNTED)) { 556 spin_unlock_irqrestore(&io->lock, flags); 557 return; 558 } 559 dm_io_set_flag(io, DM_IO_ACCOUNTED); 560 spin_unlock_irqrestore(&io->lock, flags); 561 } 562 563 __dm_start_io_acct(io); 564 } 565 566 static void dm_end_io_acct(struct dm_io *io) 567 { 568 dm_io_acct(io, true); 569 } 570 571 static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio) 572 { 573 struct dm_io *io; 574 struct dm_target_io *tio; 575 struct bio *clone; 576 577 clone = bio_alloc_clone(NULL, bio, GFP_NOIO, &md->mempools->io_bs); 578 tio = clone_to_tio(clone); 579 tio->flags = 0; 580 dm_tio_set_flag(tio, DM_TIO_INSIDE_DM_IO); 581 tio->io = NULL; 582 583 io = container_of(tio, struct dm_io, tio); 584 io->magic = DM_IO_MAGIC; 585 io->status = BLK_STS_OK; 586 587 /* one ref is for submission, the other is for completion */ 588 atomic_set(&io->io_count, 2); 589 this_cpu_inc(*md->pending_io); 590 io->orig_bio = bio; 591 io->md = md; 592 spin_lock_init(&io->lock); 593 io->start_time = jiffies; 594 io->flags = 0; 595 596 if (static_branch_unlikely(&stats_enabled)) 597 dm_stats_record_start(&md->stats, &io->stats_aux); 598 599 return io; 600 } 601 602 static void free_io(struct dm_io *io) 603 { 604 bio_put(&io->tio.clone); 605 } 606 607 static struct bio *alloc_tio(struct clone_info *ci, struct dm_target *ti, 608 unsigned int target_bio_nr, unsigned int *len, gfp_t gfp_mask) 609 { 610 struct mapped_device *md = ci->io->md; 611 struct dm_target_io *tio; 612 struct bio *clone; 613 614 if (!ci->io->tio.io) { 615 /* the dm_target_io embedded in ci->io is available */ 616 tio = &ci->io->tio; 617 /* alloc_io() already initialized embedded clone */ 618 clone = &tio->clone; 619 } else { 620 clone = bio_alloc_clone(NULL, ci->bio, gfp_mask, 621 &md->mempools->bs); 622 if (!clone) 623 return NULL; 624 625 /* REQ_DM_POLL_LIST shouldn't be inherited */ 626 clone->bi_opf &= ~REQ_DM_POLL_LIST; 627 628 tio = clone_to_tio(clone); 629 tio->flags = 0; /* also clears DM_TIO_INSIDE_DM_IO */ 630 } 631 632 tio->magic = DM_TIO_MAGIC; 633 tio->io = ci->io; 634 tio->ti = ti; 635 tio->target_bio_nr = target_bio_nr; 636 tio->len_ptr = len; 637 tio->old_sector = 0; 638 639 /* Set default bdev, but target must bio_set_dev() before issuing IO */ 640 clone->bi_bdev = md->disk->part0; 641 if (unlikely(ti->needs_bio_set_dev)) 642 bio_set_dev(clone, md->disk->part0); 643 644 if (len) { 645 clone->bi_iter.bi_size = to_bytes(*len); 646 if (bio_integrity(clone)) 647 bio_integrity_trim(clone); 648 } 649 650 return clone; 651 } 652 653 static void free_tio(struct bio *clone) 654 { 655 if (dm_tio_flagged(clone_to_tio(clone), DM_TIO_INSIDE_DM_IO)) 656 return; 657 bio_put(clone); 658 } 659 660 /* 661 * Add the bio to the list of deferred io. 662 */ 663 static void queue_io(struct mapped_device *md, struct bio *bio) 664 { 665 unsigned long flags; 666 667 spin_lock_irqsave(&md->deferred_lock, flags); 668 bio_list_add(&md->deferred, bio); 669 spin_unlock_irqrestore(&md->deferred_lock, flags); 670 queue_work(md->wq, &md->work); 671 } 672 673 /* 674 * Everyone (including functions in this file), should use this 675 * function to access the md->map field, and make sure they call 676 * dm_put_live_table() when finished. 677 */ 678 struct dm_table *dm_get_live_table(struct mapped_device *md, 679 int *srcu_idx) __acquires(md->io_barrier) 680 { 681 *srcu_idx = srcu_read_lock(&md->io_barrier); 682 683 return srcu_dereference(md->map, &md->io_barrier); 684 } 685 686 void dm_put_live_table(struct mapped_device *md, 687 int srcu_idx) __releases(md->io_barrier) 688 { 689 srcu_read_unlock(&md->io_barrier, srcu_idx); 690 } 691 692 void dm_sync_table(struct mapped_device *md) 693 { 694 synchronize_srcu(&md->io_barrier); 695 synchronize_rcu_expedited(); 696 } 697 698 /* 699 * A fast alternative to dm_get_live_table/dm_put_live_table. 700 * The caller must not block between these two functions. 701 */ 702 static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU) 703 { 704 rcu_read_lock(); 705 return rcu_dereference(md->map); 706 } 707 708 static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU) 709 { 710 rcu_read_unlock(); 711 } 712 713 static inline struct dm_table *dm_get_live_table_bio(struct mapped_device *md, 714 int *srcu_idx, blk_opf_t bio_opf) 715 { 716 if (bio_opf & REQ_NOWAIT) 717 return dm_get_live_table_fast(md); 718 else 719 return dm_get_live_table(md, srcu_idx); 720 } 721 722 static inline void dm_put_live_table_bio(struct mapped_device *md, int srcu_idx, 723 blk_opf_t bio_opf) 724 { 725 if (bio_opf & REQ_NOWAIT) 726 dm_put_live_table_fast(md); 727 else 728 dm_put_live_table(md, srcu_idx); 729 } 730 731 static char *_dm_claim_ptr = "I belong to device-mapper"; 732 733 /* 734 * Open a table device so we can use it as a map destination. 735 */ 736 static struct table_device *open_table_device(struct mapped_device *md, 737 dev_t dev, fmode_t mode) 738 { 739 struct table_device *td; 740 struct block_device *bdev; 741 u64 part_off; 742 int r; 743 744 td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id); 745 if (!td) 746 return ERR_PTR(-ENOMEM); 747 refcount_set(&td->count, 1); 748 749 bdev = blkdev_get_by_dev(dev, mode | FMODE_EXCL, _dm_claim_ptr); 750 if (IS_ERR(bdev)) { 751 r = PTR_ERR(bdev); 752 goto out_free_td; 753 } 754 755 /* 756 * We can be called before the dm disk is added. In that case we can't 757 * register the holder relation here. It will be done once add_disk was 758 * called. 759 */ 760 if (md->disk->slave_dir) { 761 r = bd_link_disk_holder(bdev, md->disk); 762 if (r) 763 goto out_blkdev_put; 764 } 765 766 td->dm_dev.mode = mode; 767 td->dm_dev.bdev = bdev; 768 td->dm_dev.dax_dev = fs_dax_get_by_bdev(bdev, &part_off, NULL, NULL); 769 format_dev_t(td->dm_dev.name, dev); 770 list_add(&td->list, &md->table_devices); 771 return td; 772 773 out_blkdev_put: 774 blkdev_put(bdev, mode | FMODE_EXCL); 775 out_free_td: 776 kfree(td); 777 return ERR_PTR(r); 778 } 779 780 /* 781 * Close a table device that we've been using. 782 */ 783 static void close_table_device(struct table_device *td, struct mapped_device *md) 784 { 785 if (md->disk->slave_dir) 786 bd_unlink_disk_holder(td->dm_dev.bdev, md->disk); 787 blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL); 788 put_dax(td->dm_dev.dax_dev); 789 list_del(&td->list); 790 kfree(td); 791 } 792 793 static struct table_device *find_table_device(struct list_head *l, dev_t dev, 794 fmode_t mode) 795 { 796 struct table_device *td; 797 798 list_for_each_entry(td, l, list) 799 if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode) 800 return td; 801 802 return NULL; 803 } 804 805 int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode, 806 struct dm_dev **result) 807 { 808 struct table_device *td; 809 810 mutex_lock(&md->table_devices_lock); 811 td = find_table_device(&md->table_devices, dev, mode); 812 if (!td) { 813 td = open_table_device(md, dev, mode); 814 if (IS_ERR(td)) { 815 mutex_unlock(&md->table_devices_lock); 816 return PTR_ERR(td); 817 } 818 } else { 819 refcount_inc(&td->count); 820 } 821 mutex_unlock(&md->table_devices_lock); 822 823 *result = &td->dm_dev; 824 return 0; 825 } 826 827 void dm_put_table_device(struct mapped_device *md, struct dm_dev *d) 828 { 829 struct table_device *td = container_of(d, struct table_device, dm_dev); 830 831 mutex_lock(&md->table_devices_lock); 832 if (refcount_dec_and_test(&td->count)) 833 close_table_device(td, md); 834 mutex_unlock(&md->table_devices_lock); 835 } 836 837 /* 838 * Get the geometry associated with a dm device 839 */ 840 int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo) 841 { 842 *geo = md->geometry; 843 844 return 0; 845 } 846 847 /* 848 * Set the geometry of a device. 849 */ 850 int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo) 851 { 852 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors; 853 854 if (geo->start > sz) { 855 DMERR("Start sector is beyond the geometry limits."); 856 return -EINVAL; 857 } 858 859 md->geometry = *geo; 860 861 return 0; 862 } 863 864 static int __noflush_suspending(struct mapped_device *md) 865 { 866 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 867 } 868 869 static void dm_requeue_add_io(struct dm_io *io, bool first_stage) 870 { 871 struct mapped_device *md = io->md; 872 873 if (first_stage) { 874 struct dm_io *next = md->requeue_list; 875 876 md->requeue_list = io; 877 io->next = next; 878 } else { 879 bio_list_add_head(&md->deferred, io->orig_bio); 880 } 881 } 882 883 static void dm_kick_requeue(struct mapped_device *md, bool first_stage) 884 { 885 if (first_stage) 886 queue_work(md->wq, &md->requeue_work); 887 else 888 queue_work(md->wq, &md->work); 889 } 890 891 /* 892 * Return true if the dm_io's original bio is requeued. 893 * io->status is updated with error if requeue disallowed. 894 */ 895 static bool dm_handle_requeue(struct dm_io *io, bool first_stage) 896 { 897 struct bio *bio = io->orig_bio; 898 bool handle_requeue = (io->status == BLK_STS_DM_REQUEUE); 899 bool handle_polled_eagain = ((io->status == BLK_STS_AGAIN) && 900 (bio->bi_opf & REQ_POLLED)); 901 struct mapped_device *md = io->md; 902 bool requeued = false; 903 904 if (handle_requeue || handle_polled_eagain) { 905 unsigned long flags; 906 907 if (bio->bi_opf & REQ_POLLED) { 908 /* 909 * Upper layer won't help us poll split bio 910 * (io->orig_bio may only reflect a subset of the 911 * pre-split original) so clear REQ_POLLED. 912 */ 913 bio_clear_polled(bio); 914 } 915 916 /* 917 * Target requested pushing back the I/O or 918 * polled IO hit BLK_STS_AGAIN. 919 */ 920 spin_lock_irqsave(&md->deferred_lock, flags); 921 if ((__noflush_suspending(md) && 922 !WARN_ON_ONCE(dm_is_zone_write(md, bio))) || 923 handle_polled_eagain || first_stage) { 924 dm_requeue_add_io(io, first_stage); 925 requeued = true; 926 } else { 927 /* 928 * noflush suspend was interrupted or this is 929 * a write to a zoned target. 930 */ 931 io->status = BLK_STS_IOERR; 932 } 933 spin_unlock_irqrestore(&md->deferred_lock, flags); 934 } 935 936 if (requeued) 937 dm_kick_requeue(md, first_stage); 938 939 return requeued; 940 } 941 942 static void __dm_io_complete(struct dm_io *io, bool first_stage) 943 { 944 struct bio *bio = io->orig_bio; 945 struct mapped_device *md = io->md; 946 blk_status_t io_error; 947 bool requeued; 948 949 requeued = dm_handle_requeue(io, first_stage); 950 if (requeued && first_stage) 951 return; 952 953 io_error = io->status; 954 if (dm_io_flagged(io, DM_IO_ACCOUNTED)) 955 dm_end_io_acct(io); 956 else if (!io_error) { 957 /* 958 * Must handle target that DM_MAPIO_SUBMITTED only to 959 * then bio_endio() rather than dm_submit_bio_remap() 960 */ 961 __dm_start_io_acct(io); 962 dm_end_io_acct(io); 963 } 964 free_io(io); 965 smp_wmb(); 966 this_cpu_dec(*md->pending_io); 967 968 /* nudge anyone waiting on suspend queue */ 969 if (unlikely(wq_has_sleeper(&md->wait))) 970 wake_up(&md->wait); 971 972 /* Return early if the original bio was requeued */ 973 if (requeued) 974 return; 975 976 if (bio_is_flush_with_data(bio)) { 977 /* 978 * Preflush done for flush with data, reissue 979 * without REQ_PREFLUSH. 980 */ 981 bio->bi_opf &= ~REQ_PREFLUSH; 982 queue_io(md, bio); 983 } else { 984 /* done with normal IO or empty flush */ 985 if (io_error) 986 bio->bi_status = io_error; 987 bio_endio(bio); 988 } 989 } 990 991 static void dm_wq_requeue_work(struct work_struct *work) 992 { 993 struct mapped_device *md = container_of(work, struct mapped_device, 994 requeue_work); 995 unsigned long flags; 996 struct dm_io *io; 997 998 /* reuse deferred lock to simplify dm_handle_requeue */ 999 spin_lock_irqsave(&md->deferred_lock, flags); 1000 io = md->requeue_list; 1001 md->requeue_list = NULL; 1002 spin_unlock_irqrestore(&md->deferred_lock, flags); 1003 1004 while (io) { 1005 struct dm_io *next = io->next; 1006 1007 dm_io_rewind(io, &md->disk->bio_split); 1008 1009 io->next = NULL; 1010 __dm_io_complete(io, false); 1011 io = next; 1012 cond_resched(); 1013 } 1014 } 1015 1016 /* 1017 * Two staged requeue: 1018 * 1019 * 1) io->orig_bio points to the real original bio, and the part mapped to 1020 * this io must be requeued, instead of other parts of the original bio. 1021 * 1022 * 2) io->orig_bio points to new cloned bio which matches the requeued dm_io. 1023 */ 1024 static void dm_io_complete(struct dm_io *io) 1025 { 1026 bool first_requeue; 1027 1028 /* 1029 * Only dm_io that has been split needs two stage requeue, otherwise 1030 * we may run into long bio clone chain during suspend and OOM could 1031 * be triggered. 1032 * 1033 * Also flush data dm_io won't be marked as DM_IO_WAS_SPLIT, so they 1034 * also aren't handled via the first stage requeue. 1035 */ 1036 if (dm_io_flagged(io, DM_IO_WAS_SPLIT)) 1037 first_requeue = true; 1038 else 1039 first_requeue = false; 1040 1041 __dm_io_complete(io, first_requeue); 1042 } 1043 1044 /* 1045 * Decrements the number of outstanding ios that a bio has been 1046 * cloned into, completing the original io if necc. 1047 */ 1048 static inline void __dm_io_dec_pending(struct dm_io *io) 1049 { 1050 if (atomic_dec_and_test(&io->io_count)) 1051 dm_io_complete(io); 1052 } 1053 1054 static void dm_io_set_error(struct dm_io *io, blk_status_t error) 1055 { 1056 unsigned long flags; 1057 1058 /* Push-back supersedes any I/O errors */ 1059 spin_lock_irqsave(&io->lock, flags); 1060 if (!(io->status == BLK_STS_DM_REQUEUE && 1061 __noflush_suspending(io->md))) { 1062 io->status = error; 1063 } 1064 spin_unlock_irqrestore(&io->lock, flags); 1065 } 1066 1067 static void dm_io_dec_pending(struct dm_io *io, blk_status_t error) 1068 { 1069 if (unlikely(error)) 1070 dm_io_set_error(io, error); 1071 1072 __dm_io_dec_pending(io); 1073 } 1074 1075 void disable_discard(struct mapped_device *md) 1076 { 1077 struct queue_limits *limits = dm_get_queue_limits(md); 1078 1079 /* device doesn't really support DISCARD, disable it */ 1080 limits->max_discard_sectors = 0; 1081 } 1082 1083 void disable_write_zeroes(struct mapped_device *md) 1084 { 1085 struct queue_limits *limits = dm_get_queue_limits(md); 1086 1087 /* device doesn't really support WRITE ZEROES, disable it */ 1088 limits->max_write_zeroes_sectors = 0; 1089 } 1090 1091 static bool swap_bios_limit(struct dm_target *ti, struct bio *bio) 1092 { 1093 return unlikely((bio->bi_opf & REQ_SWAP) != 0) && unlikely(ti->limit_swap_bios); 1094 } 1095 1096 static void clone_endio(struct bio *bio) 1097 { 1098 blk_status_t error = bio->bi_status; 1099 struct dm_target_io *tio = clone_to_tio(bio); 1100 struct dm_target *ti = tio->ti; 1101 dm_endio_fn endio = ti->type->end_io; 1102 struct dm_io *io = tio->io; 1103 struct mapped_device *md = io->md; 1104 1105 if (unlikely(error == BLK_STS_TARGET)) { 1106 if (bio_op(bio) == REQ_OP_DISCARD && 1107 !bdev_max_discard_sectors(bio->bi_bdev)) 1108 disable_discard(md); 1109 else if (bio_op(bio) == REQ_OP_WRITE_ZEROES && 1110 !bdev_write_zeroes_sectors(bio->bi_bdev)) 1111 disable_write_zeroes(md); 1112 } 1113 1114 if (static_branch_unlikely(&zoned_enabled) && 1115 unlikely(bdev_is_zoned(bio->bi_bdev))) 1116 dm_zone_endio(io, bio); 1117 1118 if (endio) { 1119 int r = endio(ti, bio, &error); 1120 1121 switch (r) { 1122 case DM_ENDIO_REQUEUE: 1123 if (static_branch_unlikely(&zoned_enabled)) { 1124 /* 1125 * Requeuing writes to a sequential zone of a zoned 1126 * target will break the sequential write pattern: 1127 * fail such IO. 1128 */ 1129 if (WARN_ON_ONCE(dm_is_zone_write(md, bio))) 1130 error = BLK_STS_IOERR; 1131 else 1132 error = BLK_STS_DM_REQUEUE; 1133 } else 1134 error = BLK_STS_DM_REQUEUE; 1135 fallthrough; 1136 case DM_ENDIO_DONE: 1137 break; 1138 case DM_ENDIO_INCOMPLETE: 1139 /* The target will handle the io */ 1140 return; 1141 default: 1142 DMCRIT("unimplemented target endio return value: %d", r); 1143 BUG(); 1144 } 1145 } 1146 1147 if (static_branch_unlikely(&swap_bios_enabled) && 1148 unlikely(swap_bios_limit(ti, bio))) 1149 up(&md->swap_bios_semaphore); 1150 1151 free_tio(bio); 1152 dm_io_dec_pending(io, error); 1153 } 1154 1155 /* 1156 * Return maximum size of I/O possible at the supplied sector up to the current 1157 * target boundary. 1158 */ 1159 static inline sector_t max_io_len_target_boundary(struct dm_target *ti, 1160 sector_t target_offset) 1161 { 1162 return ti->len - target_offset; 1163 } 1164 1165 static sector_t max_io_len(struct dm_target *ti, sector_t sector) 1166 { 1167 sector_t target_offset = dm_target_offset(ti, sector); 1168 sector_t len = max_io_len_target_boundary(ti, target_offset); 1169 1170 /* 1171 * Does the target need to split IO even further? 1172 * - varied (per target) IO splitting is a tenet of DM; this 1173 * explains why stacked chunk_sectors based splitting via 1174 * bio_split_to_limits() isn't possible here. 1175 */ 1176 if (!ti->max_io_len) 1177 return len; 1178 return min_t(sector_t, len, 1179 min(queue_max_sectors(ti->table->md->queue), 1180 blk_chunk_sectors_left(target_offset, ti->max_io_len))); 1181 } 1182 1183 int dm_set_target_max_io_len(struct dm_target *ti, sector_t len) 1184 { 1185 if (len > UINT_MAX) { 1186 DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)", 1187 (unsigned long long)len, UINT_MAX); 1188 ti->error = "Maximum size of target IO is too large"; 1189 return -EINVAL; 1190 } 1191 1192 ti->max_io_len = (uint32_t) len; 1193 1194 return 0; 1195 } 1196 EXPORT_SYMBOL_GPL(dm_set_target_max_io_len); 1197 1198 static struct dm_target *dm_dax_get_live_target(struct mapped_device *md, 1199 sector_t sector, int *srcu_idx) 1200 __acquires(md->io_barrier) 1201 { 1202 struct dm_table *map; 1203 struct dm_target *ti; 1204 1205 map = dm_get_live_table(md, srcu_idx); 1206 if (!map) 1207 return NULL; 1208 1209 ti = dm_table_find_target(map, sector); 1210 if (!ti) 1211 return NULL; 1212 1213 return ti; 1214 } 1215 1216 static long dm_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, 1217 long nr_pages, enum dax_access_mode mode, void **kaddr, 1218 pfn_t *pfn) 1219 { 1220 struct mapped_device *md = dax_get_private(dax_dev); 1221 sector_t sector = pgoff * PAGE_SECTORS; 1222 struct dm_target *ti; 1223 long len, ret = -EIO; 1224 int srcu_idx; 1225 1226 ti = dm_dax_get_live_target(md, sector, &srcu_idx); 1227 1228 if (!ti) 1229 goto out; 1230 if (!ti->type->direct_access) 1231 goto out; 1232 len = max_io_len(ti, sector) / PAGE_SECTORS; 1233 if (len < 1) 1234 goto out; 1235 nr_pages = min(len, nr_pages); 1236 ret = ti->type->direct_access(ti, pgoff, nr_pages, mode, kaddr, pfn); 1237 1238 out: 1239 dm_put_live_table(md, srcu_idx); 1240 1241 return ret; 1242 } 1243 1244 static int dm_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, 1245 size_t nr_pages) 1246 { 1247 struct mapped_device *md = dax_get_private(dax_dev); 1248 sector_t sector = pgoff * PAGE_SECTORS; 1249 struct dm_target *ti; 1250 int ret = -EIO; 1251 int srcu_idx; 1252 1253 ti = dm_dax_get_live_target(md, sector, &srcu_idx); 1254 1255 if (!ti) 1256 goto out; 1257 if (WARN_ON(!ti->type->dax_zero_page_range)) { 1258 /* 1259 * ->zero_page_range() is mandatory dax operation. If we are 1260 * here, something is wrong. 1261 */ 1262 goto out; 1263 } 1264 ret = ti->type->dax_zero_page_range(ti, pgoff, nr_pages); 1265 out: 1266 dm_put_live_table(md, srcu_idx); 1267 1268 return ret; 1269 } 1270 1271 static size_t dm_dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff, 1272 void *addr, size_t bytes, struct iov_iter *i) 1273 { 1274 struct mapped_device *md = dax_get_private(dax_dev); 1275 sector_t sector = pgoff * PAGE_SECTORS; 1276 struct dm_target *ti; 1277 int srcu_idx; 1278 long ret = 0; 1279 1280 ti = dm_dax_get_live_target(md, sector, &srcu_idx); 1281 if (!ti || !ti->type->dax_recovery_write) 1282 goto out; 1283 1284 ret = ti->type->dax_recovery_write(ti, pgoff, addr, bytes, i); 1285 out: 1286 dm_put_live_table(md, srcu_idx); 1287 return ret; 1288 } 1289 1290 /* 1291 * A target may call dm_accept_partial_bio only from the map routine. It is 1292 * allowed for all bio types except REQ_PREFLUSH, REQ_OP_ZONE_* zone management 1293 * operations, REQ_OP_ZONE_APPEND (zone append writes) and any bio serviced by 1294 * __send_duplicate_bios(). 1295 * 1296 * dm_accept_partial_bio informs the dm that the target only wants to process 1297 * additional n_sectors sectors of the bio and the rest of the data should be 1298 * sent in a next bio. 1299 * 1300 * A diagram that explains the arithmetics: 1301 * +--------------------+---------------+-------+ 1302 * | 1 | 2 | 3 | 1303 * +--------------------+---------------+-------+ 1304 * 1305 * <-------------- *tio->len_ptr ---------------> 1306 * <----- bio_sectors -----> 1307 * <-- n_sectors --> 1308 * 1309 * Region 1 was already iterated over with bio_advance or similar function. 1310 * (it may be empty if the target doesn't use bio_advance) 1311 * Region 2 is the remaining bio size that the target wants to process. 1312 * (it may be empty if region 1 is non-empty, although there is no reason 1313 * to make it empty) 1314 * The target requires that region 3 is to be sent in the next bio. 1315 * 1316 * If the target wants to receive multiple copies of the bio (via num_*bios, etc), 1317 * the partially processed part (the sum of regions 1+2) must be the same for all 1318 * copies of the bio. 1319 */ 1320 void dm_accept_partial_bio(struct bio *bio, unsigned int n_sectors) 1321 { 1322 struct dm_target_io *tio = clone_to_tio(bio); 1323 struct dm_io *io = tio->io; 1324 unsigned int bio_sectors = bio_sectors(bio); 1325 1326 BUG_ON(dm_tio_flagged(tio, DM_TIO_IS_DUPLICATE_BIO)); 1327 BUG_ON(op_is_zone_mgmt(bio_op(bio))); 1328 BUG_ON(bio_op(bio) == REQ_OP_ZONE_APPEND); 1329 BUG_ON(bio_sectors > *tio->len_ptr); 1330 BUG_ON(n_sectors > bio_sectors); 1331 1332 *tio->len_ptr -= bio_sectors - n_sectors; 1333 bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT; 1334 1335 /* 1336 * __split_and_process_bio() may have already saved mapped part 1337 * for accounting but it is being reduced so update accordingly. 1338 */ 1339 dm_io_set_flag(io, DM_IO_WAS_SPLIT); 1340 io->sectors = n_sectors; 1341 io->sector_offset = bio_sectors(io->orig_bio); 1342 } 1343 EXPORT_SYMBOL_GPL(dm_accept_partial_bio); 1344 1345 /* 1346 * @clone: clone bio that DM core passed to target's .map function 1347 * @tgt_clone: clone of @clone bio that target needs submitted 1348 * 1349 * Targets should use this interface to submit bios they take 1350 * ownership of when returning DM_MAPIO_SUBMITTED. 1351 * 1352 * Target should also enable ti->accounts_remapped_io 1353 */ 1354 void dm_submit_bio_remap(struct bio *clone, struct bio *tgt_clone) 1355 { 1356 struct dm_target_io *tio = clone_to_tio(clone); 1357 struct dm_io *io = tio->io; 1358 1359 /* establish bio that will get submitted */ 1360 if (!tgt_clone) 1361 tgt_clone = clone; 1362 1363 /* 1364 * Account io->origin_bio to DM dev on behalf of target 1365 * that took ownership of IO with DM_MAPIO_SUBMITTED. 1366 */ 1367 dm_start_io_acct(io, clone); 1368 1369 trace_block_bio_remap(tgt_clone, disk_devt(io->md->disk), 1370 tio->old_sector); 1371 submit_bio_noacct(tgt_clone); 1372 } 1373 EXPORT_SYMBOL_GPL(dm_submit_bio_remap); 1374 1375 static noinline void __set_swap_bios_limit(struct mapped_device *md, int latch) 1376 { 1377 mutex_lock(&md->swap_bios_lock); 1378 while (latch < md->swap_bios) { 1379 cond_resched(); 1380 down(&md->swap_bios_semaphore); 1381 md->swap_bios--; 1382 } 1383 while (latch > md->swap_bios) { 1384 cond_resched(); 1385 up(&md->swap_bios_semaphore); 1386 md->swap_bios++; 1387 } 1388 mutex_unlock(&md->swap_bios_lock); 1389 } 1390 1391 static void __map_bio(struct bio *clone) 1392 { 1393 struct dm_target_io *tio = clone_to_tio(clone); 1394 struct dm_target *ti = tio->ti; 1395 struct dm_io *io = tio->io; 1396 struct mapped_device *md = io->md; 1397 int r; 1398 1399 clone->bi_end_io = clone_endio; 1400 1401 /* 1402 * Map the clone. 1403 */ 1404 tio->old_sector = clone->bi_iter.bi_sector; 1405 1406 if (static_branch_unlikely(&swap_bios_enabled) && 1407 unlikely(swap_bios_limit(ti, clone))) { 1408 int latch = get_swap_bios(); 1409 1410 if (unlikely(latch != md->swap_bios)) 1411 __set_swap_bios_limit(md, latch); 1412 down(&md->swap_bios_semaphore); 1413 } 1414 1415 if (static_branch_unlikely(&zoned_enabled)) { 1416 /* 1417 * Check if the IO needs a special mapping due to zone append 1418 * emulation on zoned target. In this case, dm_zone_map_bio() 1419 * calls the target map operation. 1420 */ 1421 if (unlikely(dm_emulate_zone_append(md))) 1422 r = dm_zone_map_bio(tio); 1423 else 1424 r = ti->type->map(ti, clone); 1425 } else 1426 r = ti->type->map(ti, clone); 1427 1428 switch (r) { 1429 case DM_MAPIO_SUBMITTED: 1430 /* target has assumed ownership of this io */ 1431 if (!ti->accounts_remapped_io) 1432 dm_start_io_acct(io, clone); 1433 break; 1434 case DM_MAPIO_REMAPPED: 1435 dm_submit_bio_remap(clone, NULL); 1436 break; 1437 case DM_MAPIO_KILL: 1438 case DM_MAPIO_REQUEUE: 1439 if (static_branch_unlikely(&swap_bios_enabled) && 1440 unlikely(swap_bios_limit(ti, clone))) 1441 up(&md->swap_bios_semaphore); 1442 free_tio(clone); 1443 if (r == DM_MAPIO_KILL) 1444 dm_io_dec_pending(io, BLK_STS_IOERR); 1445 else 1446 dm_io_dec_pending(io, BLK_STS_DM_REQUEUE); 1447 break; 1448 default: 1449 DMCRIT("unimplemented target map return value: %d", r); 1450 BUG(); 1451 } 1452 } 1453 1454 static void setup_split_accounting(struct clone_info *ci, unsigned int len) 1455 { 1456 struct dm_io *io = ci->io; 1457 1458 if (ci->sector_count > len) { 1459 /* 1460 * Split needed, save the mapped part for accounting. 1461 * NOTE: dm_accept_partial_bio() will update accordingly. 1462 */ 1463 dm_io_set_flag(io, DM_IO_WAS_SPLIT); 1464 io->sectors = len; 1465 io->sector_offset = bio_sectors(ci->bio); 1466 } 1467 } 1468 1469 static void alloc_multiple_bios(struct bio_list *blist, struct clone_info *ci, 1470 struct dm_target *ti, unsigned int num_bios, 1471 unsigned *len) 1472 { 1473 struct bio *bio; 1474 int try; 1475 1476 for (try = 0; try < 2; try++) { 1477 int bio_nr; 1478 1479 if (try) 1480 mutex_lock(&ci->io->md->table_devices_lock); 1481 for (bio_nr = 0; bio_nr < num_bios; bio_nr++) { 1482 bio = alloc_tio(ci, ti, bio_nr, len, 1483 try ? GFP_NOIO : GFP_NOWAIT); 1484 if (!bio) 1485 break; 1486 1487 bio_list_add(blist, bio); 1488 } 1489 if (try) 1490 mutex_unlock(&ci->io->md->table_devices_lock); 1491 if (bio_nr == num_bios) 1492 return; 1493 1494 while ((bio = bio_list_pop(blist))) 1495 free_tio(bio); 1496 } 1497 } 1498 1499 static int __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti, 1500 unsigned int num_bios, unsigned int *len) 1501 { 1502 struct bio_list blist = BIO_EMPTY_LIST; 1503 struct bio *clone; 1504 unsigned int ret = 0; 1505 1506 switch (num_bios) { 1507 case 0: 1508 break; 1509 case 1: 1510 if (len) 1511 setup_split_accounting(ci, *len); 1512 clone = alloc_tio(ci, ti, 0, len, GFP_NOIO); 1513 __map_bio(clone); 1514 ret = 1; 1515 break; 1516 default: 1517 if (len) 1518 setup_split_accounting(ci, *len); 1519 /* dm_accept_partial_bio() is not supported with shared tio->len_ptr */ 1520 alloc_multiple_bios(&blist, ci, ti, num_bios, len); 1521 while ((clone = bio_list_pop(&blist))) { 1522 dm_tio_set_flag(clone_to_tio(clone), DM_TIO_IS_DUPLICATE_BIO); 1523 __map_bio(clone); 1524 ret += 1; 1525 } 1526 break; 1527 } 1528 1529 return ret; 1530 } 1531 1532 static void __send_empty_flush(struct clone_info *ci) 1533 { 1534 struct dm_table *t = ci->map; 1535 struct bio flush_bio; 1536 1537 /* 1538 * Use an on-stack bio for this, it's safe since we don't 1539 * need to reference it after submit. It's just used as 1540 * the basis for the clone(s). 1541 */ 1542 bio_init(&flush_bio, ci->io->md->disk->part0, NULL, 0, 1543 REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC); 1544 1545 ci->bio = &flush_bio; 1546 ci->sector_count = 0; 1547 ci->io->tio.clone.bi_iter.bi_size = 0; 1548 1549 for (unsigned int i = 0; i < t->num_targets; i++) { 1550 unsigned int bios; 1551 struct dm_target *ti = dm_table_get_target(t, i); 1552 1553 atomic_add(ti->num_flush_bios, &ci->io->io_count); 1554 bios = __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL); 1555 atomic_sub(ti->num_flush_bios - bios, &ci->io->io_count); 1556 } 1557 1558 /* 1559 * alloc_io() takes one extra reference for submission, so the 1560 * reference won't reach 0 without the following subtraction 1561 */ 1562 atomic_sub(1, &ci->io->io_count); 1563 1564 bio_uninit(ci->bio); 1565 } 1566 1567 static void __send_changing_extent_only(struct clone_info *ci, struct dm_target *ti, 1568 unsigned int num_bios) 1569 { 1570 unsigned int len, bios; 1571 1572 len = min_t(sector_t, ci->sector_count, 1573 max_io_len_target_boundary(ti, dm_target_offset(ti, ci->sector))); 1574 1575 atomic_add(num_bios, &ci->io->io_count); 1576 bios = __send_duplicate_bios(ci, ti, num_bios, &len); 1577 /* 1578 * alloc_io() takes one extra reference for submission, so the 1579 * reference won't reach 0 without the following (+1) subtraction 1580 */ 1581 atomic_sub(num_bios - bios + 1, &ci->io->io_count); 1582 1583 ci->sector += len; 1584 ci->sector_count -= len; 1585 } 1586 1587 static bool is_abnormal_io(struct bio *bio) 1588 { 1589 enum req_op op = bio_op(bio); 1590 1591 if (op != REQ_OP_READ && op != REQ_OP_WRITE && op != REQ_OP_FLUSH) { 1592 switch (op) { 1593 case REQ_OP_DISCARD: 1594 case REQ_OP_SECURE_ERASE: 1595 case REQ_OP_WRITE_ZEROES: 1596 return true; 1597 default: 1598 break; 1599 } 1600 } 1601 1602 return false; 1603 } 1604 1605 static blk_status_t __process_abnormal_io(struct clone_info *ci, 1606 struct dm_target *ti) 1607 { 1608 unsigned int num_bios = 0; 1609 1610 switch (bio_op(ci->bio)) { 1611 case REQ_OP_DISCARD: 1612 num_bios = ti->num_discard_bios; 1613 break; 1614 case REQ_OP_SECURE_ERASE: 1615 num_bios = ti->num_secure_erase_bios; 1616 break; 1617 case REQ_OP_WRITE_ZEROES: 1618 num_bios = ti->num_write_zeroes_bios; 1619 break; 1620 default: 1621 break; 1622 } 1623 1624 /* 1625 * Even though the device advertised support for this type of 1626 * request, that does not mean every target supports it, and 1627 * reconfiguration might also have changed that since the 1628 * check was performed. 1629 */ 1630 if (unlikely(!num_bios)) 1631 return BLK_STS_NOTSUPP; 1632 1633 __send_changing_extent_only(ci, ti, num_bios); 1634 return BLK_STS_OK; 1635 } 1636 1637 /* 1638 * Reuse ->bi_private as dm_io list head for storing all dm_io instances 1639 * associated with this bio, and this bio's bi_private needs to be 1640 * stored in dm_io->data before the reuse. 1641 * 1642 * bio->bi_private is owned by fs or upper layer, so block layer won't 1643 * touch it after splitting. Meantime it won't be changed by anyone after 1644 * bio is submitted. So this reuse is safe. 1645 */ 1646 static inline struct dm_io **dm_poll_list_head(struct bio *bio) 1647 { 1648 return (struct dm_io **)&bio->bi_private; 1649 } 1650 1651 static void dm_queue_poll_io(struct bio *bio, struct dm_io *io) 1652 { 1653 struct dm_io **head = dm_poll_list_head(bio); 1654 1655 if (!(bio->bi_opf & REQ_DM_POLL_LIST)) { 1656 bio->bi_opf |= REQ_DM_POLL_LIST; 1657 /* 1658 * Save .bi_private into dm_io, so that we can reuse 1659 * .bi_private as dm_io list head for storing dm_io list 1660 */ 1661 io->data = bio->bi_private; 1662 1663 /* tell block layer to poll for completion */ 1664 bio->bi_cookie = ~BLK_QC_T_NONE; 1665 1666 io->next = NULL; 1667 } else { 1668 /* 1669 * bio recursed due to split, reuse original poll list, 1670 * and save bio->bi_private too. 1671 */ 1672 io->data = (*head)->data; 1673 io->next = *head; 1674 } 1675 1676 *head = io; 1677 } 1678 1679 /* 1680 * Select the correct strategy for processing a non-flush bio. 1681 */ 1682 static blk_status_t __split_and_process_bio(struct clone_info *ci) 1683 { 1684 struct bio *clone; 1685 struct dm_target *ti; 1686 unsigned int len; 1687 1688 ti = dm_table_find_target(ci->map, ci->sector); 1689 if (unlikely(!ti)) 1690 return BLK_STS_IOERR; 1691 1692 if (unlikely((ci->bio->bi_opf & REQ_NOWAIT) != 0) && 1693 unlikely(!dm_target_supports_nowait(ti->type))) 1694 return BLK_STS_NOTSUPP; 1695 1696 if (unlikely(ci->is_abnormal_io)) 1697 return __process_abnormal_io(ci, ti); 1698 1699 /* 1700 * Only support bio polling for normal IO, and the target io is 1701 * exactly inside the dm_io instance (verified in dm_poll_dm_io) 1702 */ 1703 ci->submit_as_polled = !!(ci->bio->bi_opf & REQ_POLLED); 1704 1705 len = min_t(sector_t, max_io_len(ti, ci->sector), ci->sector_count); 1706 setup_split_accounting(ci, len); 1707 clone = alloc_tio(ci, ti, 0, &len, GFP_NOIO); 1708 __map_bio(clone); 1709 1710 ci->sector += len; 1711 ci->sector_count -= len; 1712 1713 return BLK_STS_OK; 1714 } 1715 1716 static void init_clone_info(struct clone_info *ci, struct mapped_device *md, 1717 struct dm_table *map, struct bio *bio, bool is_abnormal) 1718 { 1719 ci->map = map; 1720 ci->io = alloc_io(md, bio); 1721 ci->bio = bio; 1722 ci->is_abnormal_io = is_abnormal; 1723 ci->submit_as_polled = false; 1724 ci->sector = bio->bi_iter.bi_sector; 1725 ci->sector_count = bio_sectors(bio); 1726 1727 /* Shouldn't happen but sector_count was being set to 0 so... */ 1728 if (static_branch_unlikely(&zoned_enabled) && 1729 WARN_ON_ONCE(op_is_zone_mgmt(bio_op(bio)) && ci->sector_count)) 1730 ci->sector_count = 0; 1731 } 1732 1733 /* 1734 * Entry point to split a bio into clones and submit them to the targets. 1735 */ 1736 static void dm_split_and_process_bio(struct mapped_device *md, 1737 struct dm_table *map, struct bio *bio) 1738 { 1739 struct clone_info ci; 1740 struct dm_io *io; 1741 blk_status_t error = BLK_STS_OK; 1742 bool is_abnormal; 1743 1744 is_abnormal = is_abnormal_io(bio); 1745 if (unlikely(is_abnormal)) { 1746 /* 1747 * Use bio_split_to_limits() for abnormal IO (e.g. discard, etc) 1748 * otherwise associated queue_limits won't be imposed. 1749 */ 1750 bio = bio_split_to_limits(bio); 1751 if (!bio) 1752 return; 1753 } 1754 1755 init_clone_info(&ci, md, map, bio, is_abnormal); 1756 io = ci.io; 1757 1758 if (bio->bi_opf & REQ_PREFLUSH) { 1759 __send_empty_flush(&ci); 1760 /* dm_io_complete submits any data associated with flush */ 1761 goto out; 1762 } 1763 1764 error = __split_and_process_bio(&ci); 1765 if (error || !ci.sector_count) 1766 goto out; 1767 /* 1768 * Remainder must be passed to submit_bio_noacct() so it gets handled 1769 * *after* bios already submitted have been completely processed. 1770 */ 1771 bio_trim(bio, io->sectors, ci.sector_count); 1772 trace_block_split(bio, bio->bi_iter.bi_sector); 1773 bio_inc_remaining(bio); 1774 submit_bio_noacct(bio); 1775 out: 1776 /* 1777 * Drop the extra reference count for non-POLLED bio, and hold one 1778 * reference for POLLED bio, which will be released in dm_poll_bio 1779 * 1780 * Add every dm_io instance into the dm_io list head which is stored 1781 * in bio->bi_private, so that dm_poll_bio can poll them all. 1782 */ 1783 if (error || !ci.submit_as_polled) { 1784 /* 1785 * In case of submission failure, the extra reference for 1786 * submitting io isn't consumed yet 1787 */ 1788 if (error) 1789 atomic_dec(&io->io_count); 1790 dm_io_dec_pending(io, error); 1791 } else 1792 dm_queue_poll_io(bio, io); 1793 } 1794 1795 static void dm_submit_bio(struct bio *bio) 1796 { 1797 struct mapped_device *md = bio->bi_bdev->bd_disk->private_data; 1798 int srcu_idx; 1799 struct dm_table *map; 1800 blk_opf_t bio_opf = bio->bi_opf; 1801 1802 map = dm_get_live_table_bio(md, &srcu_idx, bio_opf); 1803 1804 /* If suspended, or map not yet available, queue this IO for later */ 1805 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) || 1806 unlikely(!map)) { 1807 if (bio->bi_opf & REQ_NOWAIT) 1808 bio_wouldblock_error(bio); 1809 else if (bio->bi_opf & REQ_RAHEAD) 1810 bio_io_error(bio); 1811 else 1812 queue_io(md, bio); 1813 goto out; 1814 } 1815 1816 dm_split_and_process_bio(md, map, bio); 1817 out: 1818 dm_put_live_table_bio(md, srcu_idx, bio_opf); 1819 } 1820 1821 static bool dm_poll_dm_io(struct dm_io *io, struct io_comp_batch *iob, 1822 unsigned int flags) 1823 { 1824 WARN_ON_ONCE(!dm_tio_is_normal(&io->tio)); 1825 1826 /* don't poll if the mapped io is done */ 1827 if (atomic_read(&io->io_count) > 1) 1828 bio_poll(&io->tio.clone, iob, flags); 1829 1830 /* bio_poll holds the last reference */ 1831 return atomic_read(&io->io_count) == 1; 1832 } 1833 1834 static int dm_poll_bio(struct bio *bio, struct io_comp_batch *iob, 1835 unsigned int flags) 1836 { 1837 struct dm_io **head = dm_poll_list_head(bio); 1838 struct dm_io *list = *head; 1839 struct dm_io *tmp = NULL; 1840 struct dm_io *curr, *next; 1841 1842 /* Only poll normal bio which was marked as REQ_DM_POLL_LIST */ 1843 if (!(bio->bi_opf & REQ_DM_POLL_LIST)) 1844 return 0; 1845 1846 WARN_ON_ONCE(!list); 1847 1848 /* 1849 * Restore .bi_private before possibly completing dm_io. 1850 * 1851 * bio_poll() is only possible once @bio has been completely 1852 * submitted via submit_bio_noacct()'s depth-first submission. 1853 * So there is no dm_queue_poll_io() race associated with 1854 * clearing REQ_DM_POLL_LIST here. 1855 */ 1856 bio->bi_opf &= ~REQ_DM_POLL_LIST; 1857 bio->bi_private = list->data; 1858 1859 for (curr = list, next = curr->next; curr; curr = next, next = 1860 curr ? curr->next : NULL) { 1861 if (dm_poll_dm_io(curr, iob, flags)) { 1862 /* 1863 * clone_endio() has already occurred, so no 1864 * error handling is needed here. 1865 */ 1866 __dm_io_dec_pending(curr); 1867 } else { 1868 curr->next = tmp; 1869 tmp = curr; 1870 } 1871 } 1872 1873 /* Not done? */ 1874 if (tmp) { 1875 bio->bi_opf |= REQ_DM_POLL_LIST; 1876 /* Reset bio->bi_private to dm_io list head */ 1877 *head = tmp; 1878 return 0; 1879 } 1880 return 1; 1881 } 1882 1883 /* 1884 *--------------------------------------------------------------- 1885 * An IDR is used to keep track of allocated minor numbers. 1886 *--------------------------------------------------------------- 1887 */ 1888 static void free_minor(int minor) 1889 { 1890 spin_lock(&_minor_lock); 1891 idr_remove(&_minor_idr, minor); 1892 spin_unlock(&_minor_lock); 1893 } 1894 1895 /* 1896 * See if the device with a specific minor # is free. 1897 */ 1898 static int specific_minor(int minor) 1899 { 1900 int r; 1901 1902 if (minor >= (1 << MINORBITS)) 1903 return -EINVAL; 1904 1905 idr_preload(GFP_KERNEL); 1906 spin_lock(&_minor_lock); 1907 1908 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT); 1909 1910 spin_unlock(&_minor_lock); 1911 idr_preload_end(); 1912 if (r < 0) 1913 return r == -ENOSPC ? -EBUSY : r; 1914 return 0; 1915 } 1916 1917 static int next_free_minor(int *minor) 1918 { 1919 int r; 1920 1921 idr_preload(GFP_KERNEL); 1922 spin_lock(&_minor_lock); 1923 1924 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT); 1925 1926 spin_unlock(&_minor_lock); 1927 idr_preload_end(); 1928 if (r < 0) 1929 return r; 1930 *minor = r; 1931 return 0; 1932 } 1933 1934 static const struct block_device_operations dm_blk_dops; 1935 static const struct block_device_operations dm_rq_blk_dops; 1936 static const struct dax_operations dm_dax_ops; 1937 1938 static void dm_wq_work(struct work_struct *work); 1939 1940 #ifdef CONFIG_BLK_INLINE_ENCRYPTION 1941 static void dm_queue_destroy_crypto_profile(struct request_queue *q) 1942 { 1943 dm_destroy_crypto_profile(q->crypto_profile); 1944 } 1945 1946 #else /* CONFIG_BLK_INLINE_ENCRYPTION */ 1947 1948 static inline void dm_queue_destroy_crypto_profile(struct request_queue *q) 1949 { 1950 } 1951 #endif /* !CONFIG_BLK_INLINE_ENCRYPTION */ 1952 1953 static void cleanup_mapped_device(struct mapped_device *md) 1954 { 1955 if (md->wq) 1956 destroy_workqueue(md->wq); 1957 dm_free_md_mempools(md->mempools); 1958 1959 if (md->dax_dev) { 1960 dax_remove_host(md->disk); 1961 kill_dax(md->dax_dev); 1962 put_dax(md->dax_dev); 1963 md->dax_dev = NULL; 1964 } 1965 1966 dm_cleanup_zoned_dev(md); 1967 if (md->disk) { 1968 spin_lock(&_minor_lock); 1969 md->disk->private_data = NULL; 1970 spin_unlock(&_minor_lock); 1971 if (dm_get_md_type(md) != DM_TYPE_NONE) { 1972 struct table_device *td; 1973 1974 dm_sysfs_exit(md); 1975 list_for_each_entry(td, &md->table_devices, list) { 1976 bd_unlink_disk_holder(td->dm_dev.bdev, 1977 md->disk); 1978 } 1979 1980 /* 1981 * Hold lock to make sure del_gendisk() won't concurrent 1982 * with open/close_table_device(). 1983 */ 1984 mutex_lock(&md->table_devices_lock); 1985 del_gendisk(md->disk); 1986 mutex_unlock(&md->table_devices_lock); 1987 } 1988 dm_queue_destroy_crypto_profile(md->queue); 1989 put_disk(md->disk); 1990 } 1991 1992 if (md->pending_io) { 1993 free_percpu(md->pending_io); 1994 md->pending_io = NULL; 1995 } 1996 1997 cleanup_srcu_struct(&md->io_barrier); 1998 1999 mutex_destroy(&md->suspend_lock); 2000 mutex_destroy(&md->type_lock); 2001 mutex_destroy(&md->table_devices_lock); 2002 mutex_destroy(&md->swap_bios_lock); 2003 2004 dm_mq_cleanup_mapped_device(md); 2005 } 2006 2007 /* 2008 * Allocate and initialise a blank device with a given minor. 2009 */ 2010 static struct mapped_device *alloc_dev(int minor) 2011 { 2012 int r, numa_node_id = dm_get_numa_node(); 2013 struct mapped_device *md; 2014 void *old_md; 2015 2016 md = kvzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id); 2017 if (!md) { 2018 DMERR("unable to allocate device, out of memory."); 2019 return NULL; 2020 } 2021 2022 if (!try_module_get(THIS_MODULE)) 2023 goto bad_module_get; 2024 2025 /* get a minor number for the dev */ 2026 if (minor == DM_ANY_MINOR) 2027 r = next_free_minor(&minor); 2028 else 2029 r = specific_minor(minor); 2030 if (r < 0) 2031 goto bad_minor; 2032 2033 r = init_srcu_struct(&md->io_barrier); 2034 if (r < 0) 2035 goto bad_io_barrier; 2036 2037 md->numa_node_id = numa_node_id; 2038 md->init_tio_pdu = false; 2039 md->type = DM_TYPE_NONE; 2040 mutex_init(&md->suspend_lock); 2041 mutex_init(&md->type_lock); 2042 mutex_init(&md->table_devices_lock); 2043 spin_lock_init(&md->deferred_lock); 2044 atomic_set(&md->holders, 1); 2045 atomic_set(&md->open_count, 0); 2046 atomic_set(&md->event_nr, 0); 2047 atomic_set(&md->uevent_seq, 0); 2048 INIT_LIST_HEAD(&md->uevent_list); 2049 INIT_LIST_HEAD(&md->table_devices); 2050 spin_lock_init(&md->uevent_lock); 2051 2052 /* 2053 * default to bio-based until DM table is loaded and md->type 2054 * established. If request-based table is loaded: blk-mq will 2055 * override accordingly. 2056 */ 2057 md->disk = blk_alloc_disk(md->numa_node_id); 2058 if (!md->disk) 2059 goto bad; 2060 md->queue = md->disk->queue; 2061 2062 init_waitqueue_head(&md->wait); 2063 INIT_WORK(&md->work, dm_wq_work); 2064 INIT_WORK(&md->requeue_work, dm_wq_requeue_work); 2065 init_waitqueue_head(&md->eventq); 2066 init_completion(&md->kobj_holder.completion); 2067 2068 md->requeue_list = NULL; 2069 md->swap_bios = get_swap_bios(); 2070 sema_init(&md->swap_bios_semaphore, md->swap_bios); 2071 mutex_init(&md->swap_bios_lock); 2072 2073 md->disk->major = _major; 2074 md->disk->first_minor = minor; 2075 md->disk->minors = 1; 2076 md->disk->flags |= GENHD_FL_NO_PART; 2077 md->disk->fops = &dm_blk_dops; 2078 md->disk->private_data = md; 2079 sprintf(md->disk->disk_name, "dm-%d", minor); 2080 2081 if (IS_ENABLED(CONFIG_FS_DAX)) { 2082 md->dax_dev = alloc_dax(md, &dm_dax_ops); 2083 if (IS_ERR(md->dax_dev)) { 2084 md->dax_dev = NULL; 2085 goto bad; 2086 } 2087 set_dax_nocache(md->dax_dev); 2088 set_dax_nomc(md->dax_dev); 2089 if (dax_add_host(md->dax_dev, md->disk)) 2090 goto bad; 2091 } 2092 2093 format_dev_t(md->name, MKDEV(_major, minor)); 2094 2095 md->wq = alloc_workqueue("kdmflush/%s", WQ_MEM_RECLAIM, 0, md->name); 2096 if (!md->wq) 2097 goto bad; 2098 2099 md->pending_io = alloc_percpu(unsigned long); 2100 if (!md->pending_io) 2101 goto bad; 2102 2103 r = dm_stats_init(&md->stats); 2104 if (r < 0) 2105 goto bad; 2106 2107 /* Populate the mapping, nobody knows we exist yet */ 2108 spin_lock(&_minor_lock); 2109 old_md = idr_replace(&_minor_idr, md, minor); 2110 spin_unlock(&_minor_lock); 2111 2112 BUG_ON(old_md != MINOR_ALLOCED); 2113 2114 return md; 2115 2116 bad: 2117 cleanup_mapped_device(md); 2118 bad_io_barrier: 2119 free_minor(minor); 2120 bad_minor: 2121 module_put(THIS_MODULE); 2122 bad_module_get: 2123 kvfree(md); 2124 return NULL; 2125 } 2126 2127 static void unlock_fs(struct mapped_device *md); 2128 2129 static void free_dev(struct mapped_device *md) 2130 { 2131 int minor = MINOR(disk_devt(md->disk)); 2132 2133 unlock_fs(md); 2134 2135 cleanup_mapped_device(md); 2136 2137 WARN_ON_ONCE(!list_empty(&md->table_devices)); 2138 dm_stats_cleanup(&md->stats); 2139 free_minor(minor); 2140 2141 module_put(THIS_MODULE); 2142 kvfree(md); 2143 } 2144 2145 /* 2146 * Bind a table to the device. 2147 */ 2148 static void event_callback(void *context) 2149 { 2150 unsigned long flags; 2151 LIST_HEAD(uevents); 2152 struct mapped_device *md = context; 2153 2154 spin_lock_irqsave(&md->uevent_lock, flags); 2155 list_splice_init(&md->uevent_list, &uevents); 2156 spin_unlock_irqrestore(&md->uevent_lock, flags); 2157 2158 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj); 2159 2160 atomic_inc(&md->event_nr); 2161 wake_up(&md->eventq); 2162 dm_issue_global_event(); 2163 } 2164 2165 /* 2166 * Returns old map, which caller must destroy. 2167 */ 2168 static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t, 2169 struct queue_limits *limits) 2170 { 2171 struct dm_table *old_map; 2172 sector_t size; 2173 int ret; 2174 2175 lockdep_assert_held(&md->suspend_lock); 2176 2177 size = dm_table_get_size(t); 2178 2179 /* 2180 * Wipe any geometry if the size of the table changed. 2181 */ 2182 if (size != dm_get_size(md)) 2183 memset(&md->geometry, 0, sizeof(md->geometry)); 2184 2185 set_capacity(md->disk, size); 2186 2187 dm_table_event_callback(t, event_callback, md); 2188 2189 if (dm_table_request_based(t)) { 2190 /* 2191 * Leverage the fact that request-based DM targets are 2192 * immutable singletons - used to optimize dm_mq_queue_rq. 2193 */ 2194 md->immutable_target = dm_table_get_immutable_target(t); 2195 2196 /* 2197 * There is no need to reload with request-based dm because the 2198 * size of front_pad doesn't change. 2199 * 2200 * Note for future: If you are to reload bioset, prep-ed 2201 * requests in the queue may refer to bio from the old bioset, 2202 * so you must walk through the queue to unprep. 2203 */ 2204 if (!md->mempools) { 2205 md->mempools = t->mempools; 2206 t->mempools = NULL; 2207 } 2208 } else { 2209 /* 2210 * The md may already have mempools that need changing. 2211 * If so, reload bioset because front_pad may have changed 2212 * because a different table was loaded. 2213 */ 2214 dm_free_md_mempools(md->mempools); 2215 md->mempools = t->mempools; 2216 t->mempools = NULL; 2217 } 2218 2219 ret = dm_table_set_restrictions(t, md->queue, limits); 2220 if (ret) { 2221 old_map = ERR_PTR(ret); 2222 goto out; 2223 } 2224 2225 old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2226 rcu_assign_pointer(md->map, (void *)t); 2227 md->immutable_target_type = dm_table_get_immutable_target_type(t); 2228 2229 if (old_map) 2230 dm_sync_table(md); 2231 out: 2232 return old_map; 2233 } 2234 2235 /* 2236 * Returns unbound table for the caller to free. 2237 */ 2238 static struct dm_table *__unbind(struct mapped_device *md) 2239 { 2240 struct dm_table *map = rcu_dereference_protected(md->map, 1); 2241 2242 if (!map) 2243 return NULL; 2244 2245 dm_table_event_callback(map, NULL, NULL); 2246 RCU_INIT_POINTER(md->map, NULL); 2247 dm_sync_table(md); 2248 2249 return map; 2250 } 2251 2252 /* 2253 * Constructor for a new device. 2254 */ 2255 int dm_create(int minor, struct mapped_device **result) 2256 { 2257 struct mapped_device *md; 2258 2259 md = alloc_dev(minor); 2260 if (!md) 2261 return -ENXIO; 2262 2263 dm_ima_reset_data(md); 2264 2265 *result = md; 2266 return 0; 2267 } 2268 2269 /* 2270 * Functions to manage md->type. 2271 * All are required to hold md->type_lock. 2272 */ 2273 void dm_lock_md_type(struct mapped_device *md) 2274 { 2275 mutex_lock(&md->type_lock); 2276 } 2277 2278 void dm_unlock_md_type(struct mapped_device *md) 2279 { 2280 mutex_unlock(&md->type_lock); 2281 } 2282 2283 void dm_set_md_type(struct mapped_device *md, enum dm_queue_mode type) 2284 { 2285 BUG_ON(!mutex_is_locked(&md->type_lock)); 2286 md->type = type; 2287 } 2288 2289 enum dm_queue_mode dm_get_md_type(struct mapped_device *md) 2290 { 2291 return md->type; 2292 } 2293 2294 struct target_type *dm_get_immutable_target_type(struct mapped_device *md) 2295 { 2296 return md->immutable_target_type; 2297 } 2298 2299 /* 2300 * The queue_limits are only valid as long as you have a reference 2301 * count on 'md'. 2302 */ 2303 struct queue_limits *dm_get_queue_limits(struct mapped_device *md) 2304 { 2305 BUG_ON(!atomic_read(&md->holders)); 2306 return &md->queue->limits; 2307 } 2308 EXPORT_SYMBOL_GPL(dm_get_queue_limits); 2309 2310 /* 2311 * Setup the DM device's queue based on md's type 2312 */ 2313 int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t) 2314 { 2315 enum dm_queue_mode type = dm_table_get_type(t); 2316 struct queue_limits limits; 2317 struct table_device *td; 2318 int r; 2319 2320 switch (type) { 2321 case DM_TYPE_REQUEST_BASED: 2322 md->disk->fops = &dm_rq_blk_dops; 2323 r = dm_mq_init_request_queue(md, t); 2324 if (r) { 2325 DMERR("Cannot initialize queue for request-based dm mapped device"); 2326 return r; 2327 } 2328 break; 2329 case DM_TYPE_BIO_BASED: 2330 case DM_TYPE_DAX_BIO_BASED: 2331 break; 2332 case DM_TYPE_NONE: 2333 WARN_ON_ONCE(true); 2334 break; 2335 } 2336 2337 r = dm_calculate_queue_limits(t, &limits); 2338 if (r) { 2339 DMERR("Cannot calculate initial queue limits"); 2340 return r; 2341 } 2342 r = dm_table_set_restrictions(t, md->queue, &limits); 2343 if (r) 2344 return r; 2345 2346 /* 2347 * Hold lock to make sure add_disk() and del_gendisk() won't concurrent 2348 * with open_table_device() and close_table_device(). 2349 */ 2350 mutex_lock(&md->table_devices_lock); 2351 r = add_disk(md->disk); 2352 mutex_unlock(&md->table_devices_lock); 2353 if (r) 2354 return r; 2355 2356 /* 2357 * Register the holder relationship for devices added before the disk 2358 * was live. 2359 */ 2360 list_for_each_entry(td, &md->table_devices, list) { 2361 r = bd_link_disk_holder(td->dm_dev.bdev, md->disk); 2362 if (r) 2363 goto out_undo_holders; 2364 } 2365 2366 r = dm_sysfs_init(md); 2367 if (r) 2368 goto out_undo_holders; 2369 2370 md->type = type; 2371 return 0; 2372 2373 out_undo_holders: 2374 list_for_each_entry_continue_reverse(td, &md->table_devices, list) 2375 bd_unlink_disk_holder(td->dm_dev.bdev, md->disk); 2376 mutex_lock(&md->table_devices_lock); 2377 del_gendisk(md->disk); 2378 mutex_unlock(&md->table_devices_lock); 2379 return r; 2380 } 2381 2382 struct mapped_device *dm_get_md(dev_t dev) 2383 { 2384 struct mapped_device *md; 2385 unsigned int minor = MINOR(dev); 2386 2387 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS)) 2388 return NULL; 2389 2390 spin_lock(&_minor_lock); 2391 2392 md = idr_find(&_minor_idr, minor); 2393 if (!md || md == MINOR_ALLOCED || (MINOR(disk_devt(dm_disk(md))) != minor) || 2394 test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) { 2395 md = NULL; 2396 goto out; 2397 } 2398 dm_get(md); 2399 out: 2400 spin_unlock(&_minor_lock); 2401 2402 return md; 2403 } 2404 EXPORT_SYMBOL_GPL(dm_get_md); 2405 2406 void *dm_get_mdptr(struct mapped_device *md) 2407 { 2408 return md->interface_ptr; 2409 } 2410 2411 void dm_set_mdptr(struct mapped_device *md, void *ptr) 2412 { 2413 md->interface_ptr = ptr; 2414 } 2415 2416 void dm_get(struct mapped_device *md) 2417 { 2418 atomic_inc(&md->holders); 2419 BUG_ON(test_bit(DMF_FREEING, &md->flags)); 2420 } 2421 2422 int dm_hold(struct mapped_device *md) 2423 { 2424 spin_lock(&_minor_lock); 2425 if (test_bit(DMF_FREEING, &md->flags)) { 2426 spin_unlock(&_minor_lock); 2427 return -EBUSY; 2428 } 2429 dm_get(md); 2430 spin_unlock(&_minor_lock); 2431 return 0; 2432 } 2433 EXPORT_SYMBOL_GPL(dm_hold); 2434 2435 const char *dm_device_name(struct mapped_device *md) 2436 { 2437 return md->name; 2438 } 2439 EXPORT_SYMBOL_GPL(dm_device_name); 2440 2441 static void __dm_destroy(struct mapped_device *md, bool wait) 2442 { 2443 struct dm_table *map; 2444 int srcu_idx; 2445 2446 might_sleep(); 2447 2448 spin_lock(&_minor_lock); 2449 idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md)))); 2450 set_bit(DMF_FREEING, &md->flags); 2451 spin_unlock(&_minor_lock); 2452 2453 blk_mark_disk_dead(md->disk); 2454 2455 /* 2456 * Take suspend_lock so that presuspend and postsuspend methods 2457 * do not race with internal suspend. 2458 */ 2459 mutex_lock(&md->suspend_lock); 2460 map = dm_get_live_table(md, &srcu_idx); 2461 if (!dm_suspended_md(md)) { 2462 dm_table_presuspend_targets(map); 2463 set_bit(DMF_SUSPENDED, &md->flags); 2464 set_bit(DMF_POST_SUSPENDING, &md->flags); 2465 dm_table_postsuspend_targets(map); 2466 } 2467 /* dm_put_live_table must be before fsleep, otherwise deadlock is possible */ 2468 dm_put_live_table(md, srcu_idx); 2469 mutex_unlock(&md->suspend_lock); 2470 2471 /* 2472 * Rare, but there may be I/O requests still going to complete, 2473 * for example. Wait for all references to disappear. 2474 * No one should increment the reference count of the mapped_device, 2475 * after the mapped_device state becomes DMF_FREEING. 2476 */ 2477 if (wait) 2478 while (atomic_read(&md->holders)) 2479 fsleep(1000); 2480 else if (atomic_read(&md->holders)) 2481 DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)", 2482 dm_device_name(md), atomic_read(&md->holders)); 2483 2484 dm_table_destroy(__unbind(md)); 2485 free_dev(md); 2486 } 2487 2488 void dm_destroy(struct mapped_device *md) 2489 { 2490 __dm_destroy(md, true); 2491 } 2492 2493 void dm_destroy_immediate(struct mapped_device *md) 2494 { 2495 __dm_destroy(md, false); 2496 } 2497 2498 void dm_put(struct mapped_device *md) 2499 { 2500 atomic_dec(&md->holders); 2501 } 2502 EXPORT_SYMBOL_GPL(dm_put); 2503 2504 static bool dm_in_flight_bios(struct mapped_device *md) 2505 { 2506 int cpu; 2507 unsigned long sum = 0; 2508 2509 for_each_possible_cpu(cpu) 2510 sum += *per_cpu_ptr(md->pending_io, cpu); 2511 2512 return sum != 0; 2513 } 2514 2515 static int dm_wait_for_bios_completion(struct mapped_device *md, unsigned int task_state) 2516 { 2517 int r = 0; 2518 DEFINE_WAIT(wait); 2519 2520 while (true) { 2521 prepare_to_wait(&md->wait, &wait, task_state); 2522 2523 if (!dm_in_flight_bios(md)) 2524 break; 2525 2526 if (signal_pending_state(task_state, current)) { 2527 r = -EINTR; 2528 break; 2529 } 2530 2531 io_schedule(); 2532 } 2533 finish_wait(&md->wait, &wait); 2534 2535 smp_rmb(); 2536 2537 return r; 2538 } 2539 2540 static int dm_wait_for_completion(struct mapped_device *md, unsigned int task_state) 2541 { 2542 int r = 0; 2543 2544 if (!queue_is_mq(md->queue)) 2545 return dm_wait_for_bios_completion(md, task_state); 2546 2547 while (true) { 2548 if (!blk_mq_queue_inflight(md->queue)) 2549 break; 2550 2551 if (signal_pending_state(task_state, current)) { 2552 r = -EINTR; 2553 break; 2554 } 2555 2556 fsleep(5000); 2557 } 2558 2559 return r; 2560 } 2561 2562 /* 2563 * Process the deferred bios 2564 */ 2565 static void dm_wq_work(struct work_struct *work) 2566 { 2567 struct mapped_device *md = container_of(work, struct mapped_device, work); 2568 struct bio *bio; 2569 2570 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) { 2571 spin_lock_irq(&md->deferred_lock); 2572 bio = bio_list_pop(&md->deferred); 2573 spin_unlock_irq(&md->deferred_lock); 2574 2575 if (!bio) 2576 break; 2577 2578 submit_bio_noacct(bio); 2579 cond_resched(); 2580 } 2581 } 2582 2583 static void dm_queue_flush(struct mapped_device *md) 2584 { 2585 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 2586 smp_mb__after_atomic(); 2587 queue_work(md->wq, &md->work); 2588 } 2589 2590 /* 2591 * Swap in a new table, returning the old one for the caller to destroy. 2592 */ 2593 struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table) 2594 { 2595 struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL); 2596 struct queue_limits limits; 2597 int r; 2598 2599 mutex_lock(&md->suspend_lock); 2600 2601 /* device must be suspended */ 2602 if (!dm_suspended_md(md)) 2603 goto out; 2604 2605 /* 2606 * If the new table has no data devices, retain the existing limits. 2607 * This helps multipath with queue_if_no_path if all paths disappear, 2608 * then new I/O is queued based on these limits, and then some paths 2609 * reappear. 2610 */ 2611 if (dm_table_has_no_data_devices(table)) { 2612 live_map = dm_get_live_table_fast(md); 2613 if (live_map) 2614 limits = md->queue->limits; 2615 dm_put_live_table_fast(md); 2616 } 2617 2618 if (!live_map) { 2619 r = dm_calculate_queue_limits(table, &limits); 2620 if (r) { 2621 map = ERR_PTR(r); 2622 goto out; 2623 } 2624 } 2625 2626 map = __bind(md, table, &limits); 2627 dm_issue_global_event(); 2628 2629 out: 2630 mutex_unlock(&md->suspend_lock); 2631 return map; 2632 } 2633 2634 /* 2635 * Functions to lock and unlock any filesystem running on the 2636 * device. 2637 */ 2638 static int lock_fs(struct mapped_device *md) 2639 { 2640 int r; 2641 2642 WARN_ON(test_bit(DMF_FROZEN, &md->flags)); 2643 2644 r = freeze_bdev(md->disk->part0); 2645 if (!r) 2646 set_bit(DMF_FROZEN, &md->flags); 2647 return r; 2648 } 2649 2650 static void unlock_fs(struct mapped_device *md) 2651 { 2652 if (!test_bit(DMF_FROZEN, &md->flags)) 2653 return; 2654 thaw_bdev(md->disk->part0); 2655 clear_bit(DMF_FROZEN, &md->flags); 2656 } 2657 2658 /* 2659 * @suspend_flags: DM_SUSPEND_LOCKFS_FLAG and/or DM_SUSPEND_NOFLUSH_FLAG 2660 * @task_state: e.g. TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE 2661 * @dmf_suspended_flag: DMF_SUSPENDED or DMF_SUSPENDED_INTERNALLY 2662 * 2663 * If __dm_suspend returns 0, the device is completely quiescent 2664 * now. There is no request-processing activity. All new requests 2665 * are being added to md->deferred list. 2666 */ 2667 static int __dm_suspend(struct mapped_device *md, struct dm_table *map, 2668 unsigned int suspend_flags, unsigned int task_state, 2669 int dmf_suspended_flag) 2670 { 2671 bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG; 2672 bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG; 2673 int r; 2674 2675 lockdep_assert_held(&md->suspend_lock); 2676 2677 /* 2678 * DMF_NOFLUSH_SUSPENDING must be set before presuspend. 2679 * This flag is cleared before dm_suspend returns. 2680 */ 2681 if (noflush) 2682 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 2683 else 2684 DMDEBUG("%s: suspending with flush", dm_device_name(md)); 2685 2686 /* 2687 * This gets reverted if there's an error later and the targets 2688 * provide the .presuspend_undo hook. 2689 */ 2690 dm_table_presuspend_targets(map); 2691 2692 /* 2693 * Flush I/O to the device. 2694 * Any I/O submitted after lock_fs() may not be flushed. 2695 * noflush takes precedence over do_lockfs. 2696 * (lock_fs() flushes I/Os and waits for them to complete.) 2697 */ 2698 if (!noflush && do_lockfs) { 2699 r = lock_fs(md); 2700 if (r) { 2701 dm_table_presuspend_undo_targets(map); 2702 return r; 2703 } 2704 } 2705 2706 /* 2707 * Here we must make sure that no processes are submitting requests 2708 * to target drivers i.e. no one may be executing 2709 * dm_split_and_process_bio from dm_submit_bio. 2710 * 2711 * To get all processes out of dm_split_and_process_bio in dm_submit_bio, 2712 * we take the write lock. To prevent any process from reentering 2713 * dm_split_and_process_bio from dm_submit_bio and quiesce the thread 2714 * (dm_wq_work), we set DMF_BLOCK_IO_FOR_SUSPEND and call 2715 * flush_workqueue(md->wq). 2716 */ 2717 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 2718 if (map) 2719 synchronize_srcu(&md->io_barrier); 2720 2721 /* 2722 * Stop md->queue before flushing md->wq in case request-based 2723 * dm defers requests to md->wq from md->queue. 2724 */ 2725 if (dm_request_based(md)) 2726 dm_stop_queue(md->queue); 2727 2728 flush_workqueue(md->wq); 2729 2730 /* 2731 * At this point no more requests are entering target request routines. 2732 * We call dm_wait_for_completion to wait for all existing requests 2733 * to finish. 2734 */ 2735 r = dm_wait_for_completion(md, task_state); 2736 if (!r) 2737 set_bit(dmf_suspended_flag, &md->flags); 2738 2739 if (noflush) 2740 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 2741 if (map) 2742 synchronize_srcu(&md->io_barrier); 2743 2744 /* were we interrupted ? */ 2745 if (r < 0) { 2746 dm_queue_flush(md); 2747 2748 if (dm_request_based(md)) 2749 dm_start_queue(md->queue); 2750 2751 unlock_fs(md); 2752 dm_table_presuspend_undo_targets(map); 2753 /* pushback list is already flushed, so skip flush */ 2754 } 2755 2756 return r; 2757 } 2758 2759 /* 2760 * We need to be able to change a mapping table under a mounted 2761 * filesystem. For example we might want to move some data in 2762 * the background. Before the table can be swapped with 2763 * dm_bind_table, dm_suspend must be called to flush any in 2764 * flight bios and ensure that any further io gets deferred. 2765 */ 2766 /* 2767 * Suspend mechanism in request-based dm. 2768 * 2769 * 1. Flush all I/Os by lock_fs() if needed. 2770 * 2. Stop dispatching any I/O by stopping the request_queue. 2771 * 3. Wait for all in-flight I/Os to be completed or requeued. 2772 * 2773 * To abort suspend, start the request_queue. 2774 */ 2775 int dm_suspend(struct mapped_device *md, unsigned int suspend_flags) 2776 { 2777 struct dm_table *map = NULL; 2778 int r = 0; 2779 2780 retry: 2781 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); 2782 2783 if (dm_suspended_md(md)) { 2784 r = -EINVAL; 2785 goto out_unlock; 2786 } 2787 2788 if (dm_suspended_internally_md(md)) { 2789 /* already internally suspended, wait for internal resume */ 2790 mutex_unlock(&md->suspend_lock); 2791 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); 2792 if (r) 2793 return r; 2794 goto retry; 2795 } 2796 2797 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2798 2799 r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED); 2800 if (r) 2801 goto out_unlock; 2802 2803 set_bit(DMF_POST_SUSPENDING, &md->flags); 2804 dm_table_postsuspend_targets(map); 2805 clear_bit(DMF_POST_SUSPENDING, &md->flags); 2806 2807 out_unlock: 2808 mutex_unlock(&md->suspend_lock); 2809 return r; 2810 } 2811 2812 static int __dm_resume(struct mapped_device *md, struct dm_table *map) 2813 { 2814 if (map) { 2815 int r = dm_table_resume_targets(map); 2816 2817 if (r) 2818 return r; 2819 } 2820 2821 dm_queue_flush(md); 2822 2823 /* 2824 * Flushing deferred I/Os must be done after targets are resumed 2825 * so that mapping of targets can work correctly. 2826 * Request-based dm is queueing the deferred I/Os in its request_queue. 2827 */ 2828 if (dm_request_based(md)) 2829 dm_start_queue(md->queue); 2830 2831 unlock_fs(md); 2832 2833 return 0; 2834 } 2835 2836 int dm_resume(struct mapped_device *md) 2837 { 2838 int r; 2839 struct dm_table *map = NULL; 2840 2841 retry: 2842 r = -EINVAL; 2843 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); 2844 2845 if (!dm_suspended_md(md)) 2846 goto out; 2847 2848 if (dm_suspended_internally_md(md)) { 2849 /* already internally suspended, wait for internal resume */ 2850 mutex_unlock(&md->suspend_lock); 2851 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); 2852 if (r) 2853 return r; 2854 goto retry; 2855 } 2856 2857 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2858 if (!map || !dm_table_get_size(map)) 2859 goto out; 2860 2861 r = __dm_resume(md, map); 2862 if (r) 2863 goto out; 2864 2865 clear_bit(DMF_SUSPENDED, &md->flags); 2866 out: 2867 mutex_unlock(&md->suspend_lock); 2868 2869 return r; 2870 } 2871 2872 /* 2873 * Internal suspend/resume works like userspace-driven suspend. It waits 2874 * until all bios finish and prevents issuing new bios to the target drivers. 2875 * It may be used only from the kernel. 2876 */ 2877 2878 static void __dm_internal_suspend(struct mapped_device *md, unsigned int suspend_flags) 2879 { 2880 struct dm_table *map = NULL; 2881 2882 lockdep_assert_held(&md->suspend_lock); 2883 2884 if (md->internal_suspend_count++) 2885 return; /* nested internal suspend */ 2886 2887 if (dm_suspended_md(md)) { 2888 set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 2889 return; /* nest suspend */ 2890 } 2891 2892 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2893 2894 /* 2895 * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is 2896 * supported. Properly supporting a TASK_INTERRUPTIBLE internal suspend 2897 * would require changing .presuspend to return an error -- avoid this 2898 * until there is a need for more elaborate variants of internal suspend. 2899 */ 2900 (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE, 2901 DMF_SUSPENDED_INTERNALLY); 2902 2903 set_bit(DMF_POST_SUSPENDING, &md->flags); 2904 dm_table_postsuspend_targets(map); 2905 clear_bit(DMF_POST_SUSPENDING, &md->flags); 2906 } 2907 2908 static void __dm_internal_resume(struct mapped_device *md) 2909 { 2910 BUG_ON(!md->internal_suspend_count); 2911 2912 if (--md->internal_suspend_count) 2913 return; /* resume from nested internal suspend */ 2914 2915 if (dm_suspended_md(md)) 2916 goto done; /* resume from nested suspend */ 2917 2918 /* 2919 * NOTE: existing callers don't need to call dm_table_resume_targets 2920 * (which may fail -- so best to avoid it for now by passing NULL map) 2921 */ 2922 (void) __dm_resume(md, NULL); 2923 2924 done: 2925 clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 2926 smp_mb__after_atomic(); 2927 wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY); 2928 } 2929 2930 void dm_internal_suspend_noflush(struct mapped_device *md) 2931 { 2932 mutex_lock(&md->suspend_lock); 2933 __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG); 2934 mutex_unlock(&md->suspend_lock); 2935 } 2936 EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush); 2937 2938 void dm_internal_resume(struct mapped_device *md) 2939 { 2940 mutex_lock(&md->suspend_lock); 2941 __dm_internal_resume(md); 2942 mutex_unlock(&md->suspend_lock); 2943 } 2944 EXPORT_SYMBOL_GPL(dm_internal_resume); 2945 2946 /* 2947 * Fast variants of internal suspend/resume hold md->suspend_lock, 2948 * which prevents interaction with userspace-driven suspend. 2949 */ 2950 2951 void dm_internal_suspend_fast(struct mapped_device *md) 2952 { 2953 mutex_lock(&md->suspend_lock); 2954 if (dm_suspended_md(md) || dm_suspended_internally_md(md)) 2955 return; 2956 2957 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 2958 synchronize_srcu(&md->io_barrier); 2959 flush_workqueue(md->wq); 2960 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE); 2961 } 2962 EXPORT_SYMBOL_GPL(dm_internal_suspend_fast); 2963 2964 void dm_internal_resume_fast(struct mapped_device *md) 2965 { 2966 if (dm_suspended_md(md) || dm_suspended_internally_md(md)) 2967 goto done; 2968 2969 dm_queue_flush(md); 2970 2971 done: 2972 mutex_unlock(&md->suspend_lock); 2973 } 2974 EXPORT_SYMBOL_GPL(dm_internal_resume_fast); 2975 2976 /* 2977 *--------------------------------------------------------------- 2978 * Event notification. 2979 *--------------------------------------------------------------- 2980 */ 2981 int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action, 2982 unsigned int cookie, bool need_resize_uevent) 2983 { 2984 int r; 2985 unsigned int noio_flag; 2986 char udev_cookie[DM_COOKIE_LENGTH]; 2987 char *envp[3] = { NULL, NULL, NULL }; 2988 char **envpp = envp; 2989 if (cookie) { 2990 snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u", 2991 DM_COOKIE_ENV_VAR_NAME, cookie); 2992 *envpp++ = udev_cookie; 2993 } 2994 if (need_resize_uevent) { 2995 *envpp++ = "RESIZE=1"; 2996 } 2997 2998 noio_flag = memalloc_noio_save(); 2999 3000 r = kobject_uevent_env(&disk_to_dev(md->disk)->kobj, action, envp); 3001 3002 memalloc_noio_restore(noio_flag); 3003 3004 return r; 3005 } 3006 3007 uint32_t dm_next_uevent_seq(struct mapped_device *md) 3008 { 3009 return atomic_add_return(1, &md->uevent_seq); 3010 } 3011 3012 uint32_t dm_get_event_nr(struct mapped_device *md) 3013 { 3014 return atomic_read(&md->event_nr); 3015 } 3016 3017 int dm_wait_event(struct mapped_device *md, int event_nr) 3018 { 3019 return wait_event_interruptible(md->eventq, 3020 (event_nr != atomic_read(&md->event_nr))); 3021 } 3022 3023 void dm_uevent_add(struct mapped_device *md, struct list_head *elist) 3024 { 3025 unsigned long flags; 3026 3027 spin_lock_irqsave(&md->uevent_lock, flags); 3028 list_add(elist, &md->uevent_list); 3029 spin_unlock_irqrestore(&md->uevent_lock, flags); 3030 } 3031 3032 /* 3033 * The gendisk is only valid as long as you have a reference 3034 * count on 'md'. 3035 */ 3036 struct gendisk *dm_disk(struct mapped_device *md) 3037 { 3038 return md->disk; 3039 } 3040 EXPORT_SYMBOL_GPL(dm_disk); 3041 3042 struct kobject *dm_kobject(struct mapped_device *md) 3043 { 3044 return &md->kobj_holder.kobj; 3045 } 3046 3047 struct mapped_device *dm_get_from_kobject(struct kobject *kobj) 3048 { 3049 struct mapped_device *md; 3050 3051 md = container_of(kobj, struct mapped_device, kobj_holder.kobj); 3052 3053 spin_lock(&_minor_lock); 3054 if (test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) { 3055 md = NULL; 3056 goto out; 3057 } 3058 dm_get(md); 3059 out: 3060 spin_unlock(&_minor_lock); 3061 3062 return md; 3063 } 3064 3065 int dm_suspended_md(struct mapped_device *md) 3066 { 3067 return test_bit(DMF_SUSPENDED, &md->flags); 3068 } 3069 3070 static int dm_post_suspending_md(struct mapped_device *md) 3071 { 3072 return test_bit(DMF_POST_SUSPENDING, &md->flags); 3073 } 3074 3075 int dm_suspended_internally_md(struct mapped_device *md) 3076 { 3077 return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 3078 } 3079 3080 int dm_test_deferred_remove_flag(struct mapped_device *md) 3081 { 3082 return test_bit(DMF_DEFERRED_REMOVE, &md->flags); 3083 } 3084 3085 int dm_suspended(struct dm_target *ti) 3086 { 3087 return dm_suspended_md(ti->table->md); 3088 } 3089 EXPORT_SYMBOL_GPL(dm_suspended); 3090 3091 int dm_post_suspending(struct dm_target *ti) 3092 { 3093 return dm_post_suspending_md(ti->table->md); 3094 } 3095 EXPORT_SYMBOL_GPL(dm_post_suspending); 3096 3097 int dm_noflush_suspending(struct dm_target *ti) 3098 { 3099 return __noflush_suspending(ti->table->md); 3100 } 3101 EXPORT_SYMBOL_GPL(dm_noflush_suspending); 3102 3103 void dm_free_md_mempools(struct dm_md_mempools *pools) 3104 { 3105 if (!pools) 3106 return; 3107 3108 bioset_exit(&pools->bs); 3109 bioset_exit(&pools->io_bs); 3110 3111 kfree(pools); 3112 } 3113 3114 struct dm_pr { 3115 u64 old_key; 3116 u64 new_key; 3117 u32 flags; 3118 bool abort; 3119 bool fail_early; 3120 int ret; 3121 enum pr_type type; 3122 }; 3123 3124 static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn, 3125 struct dm_pr *pr) 3126 { 3127 struct mapped_device *md = bdev->bd_disk->private_data; 3128 struct dm_table *table; 3129 struct dm_target *ti; 3130 int ret = -ENOTTY, srcu_idx; 3131 3132 table = dm_get_live_table(md, &srcu_idx); 3133 if (!table || !dm_table_get_size(table)) 3134 goto out; 3135 3136 /* We only support devices that have a single target */ 3137 if (table->num_targets != 1) 3138 goto out; 3139 ti = dm_table_get_target(table, 0); 3140 3141 if (dm_suspended_md(md)) { 3142 ret = -EAGAIN; 3143 goto out; 3144 } 3145 3146 ret = -EINVAL; 3147 if (!ti->type->iterate_devices) 3148 goto out; 3149 3150 ti->type->iterate_devices(ti, fn, pr); 3151 ret = 0; 3152 out: 3153 dm_put_live_table(md, srcu_idx); 3154 return ret; 3155 } 3156 3157 /* 3158 * For register / unregister we need to manually call out to every path. 3159 */ 3160 static int __dm_pr_register(struct dm_target *ti, struct dm_dev *dev, 3161 sector_t start, sector_t len, void *data) 3162 { 3163 struct dm_pr *pr = data; 3164 const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops; 3165 int ret; 3166 3167 if (!ops || !ops->pr_register) { 3168 pr->ret = -EOPNOTSUPP; 3169 return -1; 3170 } 3171 3172 ret = ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags); 3173 if (!ret) 3174 return 0; 3175 3176 if (!pr->ret) 3177 pr->ret = ret; 3178 3179 if (pr->fail_early) 3180 return -1; 3181 3182 return 0; 3183 } 3184 3185 static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key, 3186 u32 flags) 3187 { 3188 struct dm_pr pr = { 3189 .old_key = old_key, 3190 .new_key = new_key, 3191 .flags = flags, 3192 .fail_early = true, 3193 .ret = 0, 3194 }; 3195 int ret; 3196 3197 ret = dm_call_pr(bdev, __dm_pr_register, &pr); 3198 if (ret) { 3199 /* Didn't even get to register a path */ 3200 return ret; 3201 } 3202 3203 if (!pr.ret) 3204 return 0; 3205 ret = pr.ret; 3206 3207 if (!new_key) 3208 return ret; 3209 3210 /* unregister all paths if we failed to register any path */ 3211 pr.old_key = new_key; 3212 pr.new_key = 0; 3213 pr.flags = 0; 3214 pr.fail_early = false; 3215 (void) dm_call_pr(bdev, __dm_pr_register, &pr); 3216 return ret; 3217 } 3218 3219 3220 static int __dm_pr_reserve(struct dm_target *ti, struct dm_dev *dev, 3221 sector_t start, sector_t len, void *data) 3222 { 3223 struct dm_pr *pr = data; 3224 const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops; 3225 3226 if (!ops || !ops->pr_reserve) { 3227 pr->ret = -EOPNOTSUPP; 3228 return -1; 3229 } 3230 3231 pr->ret = ops->pr_reserve(dev->bdev, pr->old_key, pr->type, pr->flags); 3232 if (!pr->ret) 3233 return -1; 3234 3235 return 0; 3236 } 3237 3238 static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type, 3239 u32 flags) 3240 { 3241 struct dm_pr pr = { 3242 .old_key = key, 3243 .flags = flags, 3244 .type = type, 3245 .fail_early = false, 3246 .ret = 0, 3247 }; 3248 int ret; 3249 3250 ret = dm_call_pr(bdev, __dm_pr_reserve, &pr); 3251 if (ret) 3252 return ret; 3253 3254 return pr.ret; 3255 } 3256 3257 /* 3258 * If there is a non-All Registrants type of reservation, the release must be 3259 * sent down the holding path. For the cases where there is no reservation or 3260 * the path is not the holder the device will also return success, so we must 3261 * try each path to make sure we got the correct path. 3262 */ 3263 static int __dm_pr_release(struct dm_target *ti, struct dm_dev *dev, 3264 sector_t start, sector_t len, void *data) 3265 { 3266 struct dm_pr *pr = data; 3267 const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops; 3268 3269 if (!ops || !ops->pr_release) { 3270 pr->ret = -EOPNOTSUPP; 3271 return -1; 3272 } 3273 3274 pr->ret = ops->pr_release(dev->bdev, pr->old_key, pr->type); 3275 if (pr->ret) 3276 return -1; 3277 3278 return 0; 3279 } 3280 3281 static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type) 3282 { 3283 struct dm_pr pr = { 3284 .old_key = key, 3285 .type = type, 3286 .fail_early = false, 3287 }; 3288 int ret; 3289 3290 ret = dm_call_pr(bdev, __dm_pr_release, &pr); 3291 if (ret) 3292 return ret; 3293 3294 return pr.ret; 3295 } 3296 3297 static int __dm_pr_preempt(struct dm_target *ti, struct dm_dev *dev, 3298 sector_t start, sector_t len, void *data) 3299 { 3300 struct dm_pr *pr = data; 3301 const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops; 3302 3303 if (!ops || !ops->pr_preempt) { 3304 pr->ret = -EOPNOTSUPP; 3305 return -1; 3306 } 3307 3308 pr->ret = ops->pr_preempt(dev->bdev, pr->old_key, pr->new_key, pr->type, 3309 pr->abort); 3310 if (!pr->ret) 3311 return -1; 3312 3313 return 0; 3314 } 3315 3316 static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key, 3317 enum pr_type type, bool abort) 3318 { 3319 struct dm_pr pr = { 3320 .new_key = new_key, 3321 .old_key = old_key, 3322 .type = type, 3323 .fail_early = false, 3324 }; 3325 int ret; 3326 3327 ret = dm_call_pr(bdev, __dm_pr_preempt, &pr); 3328 if (ret) 3329 return ret; 3330 3331 return pr.ret; 3332 } 3333 3334 static int dm_pr_clear(struct block_device *bdev, u64 key) 3335 { 3336 struct mapped_device *md = bdev->bd_disk->private_data; 3337 const struct pr_ops *ops; 3338 int r, srcu_idx; 3339 3340 r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 3341 if (r < 0) 3342 goto out; 3343 3344 ops = bdev->bd_disk->fops->pr_ops; 3345 if (ops && ops->pr_clear) 3346 r = ops->pr_clear(bdev, key); 3347 else 3348 r = -EOPNOTSUPP; 3349 out: 3350 dm_unprepare_ioctl(md, srcu_idx); 3351 return r; 3352 } 3353 3354 static const struct pr_ops dm_pr_ops = { 3355 .pr_register = dm_pr_register, 3356 .pr_reserve = dm_pr_reserve, 3357 .pr_release = dm_pr_release, 3358 .pr_preempt = dm_pr_preempt, 3359 .pr_clear = dm_pr_clear, 3360 }; 3361 3362 static const struct block_device_operations dm_blk_dops = { 3363 .submit_bio = dm_submit_bio, 3364 .poll_bio = dm_poll_bio, 3365 .open = dm_blk_open, 3366 .release = dm_blk_close, 3367 .ioctl = dm_blk_ioctl, 3368 .getgeo = dm_blk_getgeo, 3369 .report_zones = dm_blk_report_zones, 3370 .pr_ops = &dm_pr_ops, 3371 .owner = THIS_MODULE 3372 }; 3373 3374 static const struct block_device_operations dm_rq_blk_dops = { 3375 .open = dm_blk_open, 3376 .release = dm_blk_close, 3377 .ioctl = dm_blk_ioctl, 3378 .getgeo = dm_blk_getgeo, 3379 .pr_ops = &dm_pr_ops, 3380 .owner = THIS_MODULE 3381 }; 3382 3383 static const struct dax_operations dm_dax_ops = { 3384 .direct_access = dm_dax_direct_access, 3385 .zero_page_range = dm_dax_zero_page_range, 3386 .recovery_write = dm_dax_recovery_write, 3387 }; 3388 3389 /* 3390 * module hooks 3391 */ 3392 module_init(dm_init); 3393 module_exit(dm_exit); 3394 3395 module_param(major, uint, 0); 3396 MODULE_PARM_DESC(major, "The major number of the device mapper"); 3397 3398 module_param(reserved_bio_based_ios, uint, 0644); 3399 MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools"); 3400 3401 module_param(dm_numa_node, int, 0644); 3402 MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations"); 3403 3404 module_param(swap_bios, int, 0644); 3405 MODULE_PARM_DESC(swap_bios, "Maximum allowed inflight swap IOs"); 3406 3407 MODULE_DESCRIPTION(DM_NAME " driver"); 3408 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>"); 3409 MODULE_LICENSE("GPL"); 3410