13bd94003SHeinz Mauelshagen // SPDX-License-Identifier: GPL-2.0-only 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * Copyright (C) 2001, 2002 Sistina Software (UK) Limited. 4784aae73SMilan Broz * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * This file is released under the GPL. 71da177e4SLinus Torvalds */ 81da177e4SLinus Torvalds 94cc96131SMike Snitzer #include "dm-core.h" 104cc96131SMike Snitzer #include "dm-rq.h" 1151e5b2bdSMike Anderson #include "dm-uevent.h" 1291ccbbacSTushar Sugandhi #include "dm-ima.h" 131da177e4SLinus Torvalds 141da177e4SLinus Torvalds #include <linux/init.h> 151da177e4SLinus Torvalds #include <linux/module.h> 1648c9c27bSArjan van de Ven #include <linux/mutex.h> 176958c1c6SMikulas Patocka #include <linux/sched/mm.h> 18174cd4b1SIngo Molnar #include <linux/sched/signal.h> 191da177e4SLinus Torvalds #include <linux/blkpg.h> 201da177e4SLinus Torvalds #include <linux/bio.h> 211da177e4SLinus Torvalds #include <linux/mempool.h> 22f26c5719SDan Williams #include <linux/dax.h> 231da177e4SLinus Torvalds #include <linux/slab.h> 241da177e4SLinus Torvalds #include <linux/idr.h> 257e026c8cSDan Williams #include <linux/uio.h> 263ac51e74SDarrick J. Wong #include <linux/hdreg.h> 273f77316dSKiyoshi Ueda #include <linux/delay.h> 28ffcc3936SMike Snitzer #include <linux/wait.h> 2971cdb697SChristoph Hellwig #include <linux/pr.h> 30b0b4d7c6SElena Reshetova #include <linux/refcount.h> 31c6a564ffSChristoph Hellwig #include <linux/part_stat.h> 32a892c8d5SSatya Tangirala #include <linux/blk-crypto.h> 331e8d44bdSEric Biggers #include <linux/blk-crypto-profile.h> 3455782138SLi Zefan 3572d94861SAlasdair G Kergon #define DM_MSG_PREFIX "core" 3672d94861SAlasdair G Kergon 3760935eb2SMilan Broz /* 3860935eb2SMilan Broz * Cookies are numeric values sent with CHANGE and REMOVE 3960935eb2SMilan Broz * uevents while resuming, removing or renaming the device. 4060935eb2SMilan Broz */ 4160935eb2SMilan Broz #define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE" 4260935eb2SMilan Broz #define DM_COOKIE_LENGTH 24 4360935eb2SMilan Broz 44b99fdcdcSMing Lei /* 45b99fdcdcSMing Lei * For REQ_POLLED fs bio, this flag is set if we link mapped underlying 46b99fdcdcSMing Lei * dm_io into one list, and reuse bio->bi_private as the list head. Before 47b99fdcdcSMing Lei * ending this fs bio, we will recover its ->bi_private. 48b99fdcdcSMing Lei */ 49b99fdcdcSMing Lei #define REQ_DM_POLL_LIST REQ_DRV 50b99fdcdcSMing Lei 511da177e4SLinus Torvalds static const char *_name = DM_NAME; 521da177e4SLinus Torvalds 532f06cd12SHeinz Mauelshagen static unsigned int major; 542f06cd12SHeinz Mauelshagen static unsigned int _major; 551da177e4SLinus Torvalds 56d15b774cSAlasdair G Kergon static DEFINE_IDR(_minor_idr); 57d15b774cSAlasdair G Kergon 58f32c10b0SJeff Mahoney static DEFINE_SPINLOCK(_minor_lock); 592c140a24SMikulas Patocka 602c140a24SMikulas Patocka static void do_deferred_remove(struct work_struct *w); 612c140a24SMikulas Patocka 622c140a24SMikulas Patocka static DECLARE_WORK(deferred_remove_work, do_deferred_remove); 632c140a24SMikulas Patocka 64acfe0ad7SMikulas Patocka static struct workqueue_struct *deferred_remove_workqueue; 65acfe0ad7SMikulas Patocka 6693e6442cSMikulas Patocka atomic_t dm_global_event_nr = ATOMIC_INIT(0); 6793e6442cSMikulas Patocka DECLARE_WAIT_QUEUE_HEAD(dm_global_eventq); 6893e6442cSMikulas Patocka 6962e08243SMikulas Patocka void dm_issue_global_event(void) 7062e08243SMikulas Patocka { 7162e08243SMikulas Patocka atomic_inc(&dm_global_event_nr); 7262e08243SMikulas Patocka wake_up(&dm_global_eventq); 7362e08243SMikulas Patocka } 7462e08243SMikulas Patocka 75442761fdSMike Snitzer DEFINE_STATIC_KEY_FALSE(stats_enabled); 76442761fdSMike Snitzer DEFINE_STATIC_KEY_FALSE(swap_bios_enabled); 77442761fdSMike Snitzer DEFINE_STATIC_KEY_FALSE(zoned_enabled); 78442761fdSMike Snitzer 791da177e4SLinus Torvalds /* 8064f52b0eSMike Snitzer * One of these is allocated (on-stack) per original bio. 811da177e4SLinus Torvalds */ 8264f52b0eSMike Snitzer struct clone_info { 8364f52b0eSMike Snitzer struct dm_table *map; 8464f52b0eSMike Snitzer struct bio *bio; 8564f52b0eSMike Snitzer struct dm_io *io; 8664f52b0eSMike Snitzer sector_t sector; 8786a3238cSHeinz Mauelshagen unsigned int sector_count; 884edadf6dSMike Snitzer bool is_abnormal_io:1; 894edadf6dSMike Snitzer bool submit_as_polled:1; 9064f52b0eSMike Snitzer }; 9164f52b0eSMike Snitzer 926c23f0bdSChristoph Hellwig static inline struct dm_target_io *clone_to_tio(struct bio *clone) 936c23f0bdSChristoph Hellwig { 946c23f0bdSChristoph Hellwig return container_of(clone, struct dm_target_io, clone); 956c23f0bdSChristoph Hellwig } 966c23f0bdSChristoph Hellwig 9764f52b0eSMike Snitzer void *dm_per_bio_data(struct bio *bio, size_t data_size) 9864f52b0eSMike Snitzer { 99655f3aadSMike Snitzer if (!dm_tio_flagged(clone_to_tio(bio), DM_TIO_INSIDE_DM_IO)) 10062f26317SJeffle Xu return (char *)bio - DM_TARGET_IO_BIO_OFFSET - data_size; 10162f26317SJeffle Xu return (char *)bio - DM_IO_BIO_OFFSET - data_size; 10264f52b0eSMike Snitzer } 10364f52b0eSMike Snitzer EXPORT_SYMBOL_GPL(dm_per_bio_data); 10464f52b0eSMike Snitzer 10564f52b0eSMike Snitzer struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size) 10664f52b0eSMike Snitzer { 10764f52b0eSMike Snitzer struct dm_io *io = (struct dm_io *)((char *)data + data_size); 108b30f1607SHeinz Mauelshagen 10964f52b0eSMike Snitzer if (io->magic == DM_IO_MAGIC) 11062f26317SJeffle Xu return (struct bio *)((char *)io + DM_IO_BIO_OFFSET); 11164f52b0eSMike Snitzer BUG_ON(io->magic != DM_TIO_MAGIC); 11262f26317SJeffle Xu return (struct bio *)((char *)io + DM_TARGET_IO_BIO_OFFSET); 11364f52b0eSMike Snitzer } 11464f52b0eSMike Snitzer EXPORT_SYMBOL_GPL(dm_bio_from_per_bio_data); 11564f52b0eSMike Snitzer 11686a3238cSHeinz Mauelshagen unsigned int dm_bio_get_target_bio_nr(const struct bio *bio) 11764f52b0eSMike Snitzer { 11864f52b0eSMike Snitzer return container_of(bio, struct dm_target_io, clone)->target_bio_nr; 11964f52b0eSMike Snitzer } 12064f52b0eSMike Snitzer EXPORT_SYMBOL_GPL(dm_bio_get_target_bio_nr); 12164f52b0eSMike Snitzer 122ba61fdd1SJeff Mahoney #define MINOR_ALLOCED ((void *)-1) 123ba61fdd1SJeff Mahoney 124115485e8SMike Snitzer #define DM_NUMA_NODE NUMA_NO_NODE 125115485e8SMike Snitzer static int dm_numa_node = DM_NUMA_NODE; 126faad87dfSMike Snitzer 127a666e5c0SMikulas Patocka #define DEFAULT_SWAP_BIOS (8 * 1048576 / PAGE_SIZE) 128a666e5c0SMikulas Patocka static int swap_bios = DEFAULT_SWAP_BIOS; 129a666e5c0SMikulas Patocka static int get_swap_bios(void) 130a666e5c0SMikulas Patocka { 131a666e5c0SMikulas Patocka int latch = READ_ONCE(swap_bios); 132b30f1607SHeinz Mauelshagen 133a666e5c0SMikulas Patocka if (unlikely(latch <= 0)) 134a666e5c0SMikulas Patocka latch = DEFAULT_SWAP_BIOS; 135a666e5c0SMikulas Patocka return latch; 136a666e5c0SMikulas Patocka } 137a666e5c0SMikulas Patocka 13886f1152bSBenjamin Marzinski struct table_device { 13986f1152bSBenjamin Marzinski struct list_head list; 140b0b4d7c6SElena Reshetova refcount_t count; 14186f1152bSBenjamin Marzinski struct dm_dev dm_dev; 14286f1152bSBenjamin Marzinski }; 14386f1152bSBenjamin Marzinski 144f4790826SMike Snitzer /* 145e8603136SMike Snitzer * Bio-based DM's mempools' reserved IOs set by the user. 146e8603136SMike Snitzer */ 1474cc96131SMike Snitzer #define RESERVED_BIO_BASED_IOS 16 14886a3238cSHeinz Mauelshagen static unsigned int reserved_bio_based_ios = RESERVED_BIO_BASED_IOS; 149e8603136SMike Snitzer 150115485e8SMike Snitzer static int __dm_get_module_param_int(int *module_param, int min, int max) 151115485e8SMike Snitzer { 1526aa7de05SMark Rutland int param = READ_ONCE(*module_param); 153115485e8SMike Snitzer int modified_param = 0; 154115485e8SMike Snitzer bool modified = true; 155115485e8SMike Snitzer 156115485e8SMike Snitzer if (param < min) 157115485e8SMike Snitzer modified_param = min; 158115485e8SMike Snitzer else if (param > max) 159115485e8SMike Snitzer modified_param = max; 160115485e8SMike Snitzer else 161115485e8SMike Snitzer modified = false; 162115485e8SMike Snitzer 163115485e8SMike Snitzer if (modified) { 164115485e8SMike Snitzer (void)cmpxchg(module_param, param, modified_param); 165115485e8SMike Snitzer param = modified_param; 166115485e8SMike Snitzer } 167115485e8SMike Snitzer 168115485e8SMike Snitzer return param; 169115485e8SMike Snitzer } 170115485e8SMike Snitzer 17186a3238cSHeinz Mauelshagen unsigned int __dm_get_module_param(unsigned int *module_param, unsigned int def, unsigned int max) 172f4790826SMike Snitzer { 17386a3238cSHeinz Mauelshagen unsigned int param = READ_ONCE(*module_param); 17486a3238cSHeinz Mauelshagen unsigned int modified_param = 0; 175f4790826SMike Snitzer 17609c2d531SMike Snitzer if (!param) 17709c2d531SMike Snitzer modified_param = def; 17809c2d531SMike Snitzer else if (param > max) 17909c2d531SMike Snitzer modified_param = max; 180f4790826SMike Snitzer 18109c2d531SMike Snitzer if (modified_param) { 18209c2d531SMike Snitzer (void)cmpxchg(module_param, param, modified_param); 18309c2d531SMike Snitzer param = modified_param; 184f4790826SMike Snitzer } 185f4790826SMike Snitzer 18609c2d531SMike Snitzer return param; 187f4790826SMike Snitzer } 188f4790826SMike Snitzer 18986a3238cSHeinz Mauelshagen unsigned int dm_get_reserved_bio_based_ios(void) 190e8603136SMike Snitzer { 19109c2d531SMike Snitzer return __dm_get_module_param(&reserved_bio_based_ios, 1924cc96131SMike Snitzer RESERVED_BIO_BASED_IOS, DM_RESERVED_MAX_IOS); 193e8603136SMike Snitzer } 194e8603136SMike Snitzer EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios); 195e8603136SMike Snitzer 19686a3238cSHeinz Mauelshagen static unsigned int dm_get_numa_node(void) 197115485e8SMike Snitzer { 198115485e8SMike Snitzer return __dm_get_module_param_int(&dm_numa_node, 199115485e8SMike Snitzer DM_NUMA_NODE, num_online_nodes() - 1); 200115485e8SMike Snitzer } 201115485e8SMike Snitzer 2021da177e4SLinus Torvalds static int __init local_init(void) 2031da177e4SLinus Torvalds { 204e689fbabSMike Snitzer int r; 2051ae49ea2SMike Snitzer 20651e5b2bdSMike Anderson r = dm_uevent_init(); 20751157b4aSKiyoshi Ueda if (r) 208e689fbabSMike Snitzer return r; 20951e5b2bdSMike Anderson 21057bbf99cSTejun Heo deferred_remove_workqueue = alloc_ordered_workqueue("kdmremove", 0); 211acfe0ad7SMikulas Patocka if (!deferred_remove_workqueue) { 212acfe0ad7SMikulas Patocka r = -ENOMEM; 213acfe0ad7SMikulas Patocka goto out_uevent_exit; 214acfe0ad7SMikulas Patocka } 215acfe0ad7SMikulas Patocka 2161da177e4SLinus Torvalds _major = major; 2171da177e4SLinus Torvalds r = register_blkdev(_major, _name); 21851157b4aSKiyoshi Ueda if (r < 0) 219acfe0ad7SMikulas Patocka goto out_free_workqueue; 2201da177e4SLinus Torvalds 2211da177e4SLinus Torvalds if (!_major) 2221da177e4SLinus Torvalds _major = r; 2231da177e4SLinus Torvalds 2241da177e4SLinus Torvalds return 0; 22551157b4aSKiyoshi Ueda 226acfe0ad7SMikulas Patocka out_free_workqueue: 227acfe0ad7SMikulas Patocka destroy_workqueue(deferred_remove_workqueue); 22851157b4aSKiyoshi Ueda out_uevent_exit: 22951157b4aSKiyoshi Ueda dm_uevent_exit(); 23051157b4aSKiyoshi Ueda 23151157b4aSKiyoshi Ueda return r; 2321da177e4SLinus Torvalds } 2331da177e4SLinus Torvalds 2341da177e4SLinus Torvalds static void local_exit(void) 2351da177e4SLinus Torvalds { 236acfe0ad7SMikulas Patocka destroy_workqueue(deferred_remove_workqueue); 2372c140a24SMikulas Patocka 23800d59405SAkinobu Mita unregister_blkdev(_major, _name); 23951e5b2bdSMike Anderson dm_uevent_exit(); 2401da177e4SLinus Torvalds 2411da177e4SLinus Torvalds _major = 0; 2421da177e4SLinus Torvalds 2431da177e4SLinus Torvalds DMINFO("cleaned up"); 2441da177e4SLinus Torvalds } 2451da177e4SLinus Torvalds 246b9249e55SAlasdair G Kergon static int (*_inits[])(void) __initdata = { 2471da177e4SLinus Torvalds local_init, 2481da177e4SLinus Torvalds dm_target_init, 2491da177e4SLinus Torvalds dm_linear_init, 2501da177e4SLinus Torvalds dm_stripe_init, 251952b3557SMikulas Patocka dm_io_init, 252945fa4d2SMikulas Patocka dm_kcopyd_init, 2531da177e4SLinus Torvalds dm_interface_init, 254fd2ed4d2SMikulas Patocka dm_statistics_init, 2551da177e4SLinus Torvalds }; 2561da177e4SLinus Torvalds 257b9249e55SAlasdair G Kergon static void (*_exits[])(void) = { 2581da177e4SLinus Torvalds local_exit, 2591da177e4SLinus Torvalds dm_target_exit, 2601da177e4SLinus Torvalds dm_linear_exit, 2611da177e4SLinus Torvalds dm_stripe_exit, 262952b3557SMikulas Patocka dm_io_exit, 263945fa4d2SMikulas Patocka dm_kcopyd_exit, 2641da177e4SLinus Torvalds dm_interface_exit, 265fd2ed4d2SMikulas Patocka dm_statistics_exit, 2661da177e4SLinus Torvalds }; 2671da177e4SLinus Torvalds 2681da177e4SLinus Torvalds static int __init dm_init(void) 2691da177e4SLinus Torvalds { 2701da177e4SLinus Torvalds const int count = ARRAY_SIZE(_inits); 2711da177e4SLinus Torvalds int r, i; 2721da177e4SLinus Torvalds 273f1cd6cb2STushar Sugandhi #if (IS_ENABLED(CONFIG_IMA) && !IS_ENABLED(CONFIG_IMA_DISABLE_HTABLE)) 274f1cd6cb2STushar Sugandhi DMWARN("CONFIG_IMA_DISABLE_HTABLE is disabled." 275f1cd6cb2STushar Sugandhi " Duplicate IMA measurements will not be recorded in the IMA log."); 276f1cd6cb2STushar Sugandhi #endif 277f1cd6cb2STushar Sugandhi 2781da177e4SLinus Torvalds for (i = 0; i < count; i++) { 2791da177e4SLinus Torvalds r = _inits[i](); 2801da177e4SLinus Torvalds if (r) 2811da177e4SLinus Torvalds goto bad; 2821da177e4SLinus Torvalds } 2831da177e4SLinus Torvalds 2841da177e4SLinus Torvalds return 0; 2851da177e4SLinus Torvalds bad: 2861da177e4SLinus Torvalds while (i--) 2871da177e4SLinus Torvalds _exits[i](); 2881da177e4SLinus Torvalds 2891da177e4SLinus Torvalds return r; 2901da177e4SLinus Torvalds } 2911da177e4SLinus Torvalds 2921da177e4SLinus Torvalds static void __exit dm_exit(void) 2931da177e4SLinus Torvalds { 2941da177e4SLinus Torvalds int i = ARRAY_SIZE(_exits); 2951da177e4SLinus Torvalds 2961da177e4SLinus Torvalds while (i--) 2971da177e4SLinus Torvalds _exits[i](); 298d15b774cSAlasdair G Kergon 299d15b774cSAlasdair G Kergon /* 300d15b774cSAlasdair G Kergon * Should be empty by this point. 301d15b774cSAlasdair G Kergon */ 302d15b774cSAlasdair G Kergon idr_destroy(&_minor_idr); 3031da177e4SLinus Torvalds } 3041da177e4SLinus Torvalds 3051da177e4SLinus Torvalds /* 3061da177e4SLinus Torvalds * Block device functions 3071da177e4SLinus Torvalds */ 308432a212cSMike Anderson int dm_deleting_md(struct mapped_device *md) 309432a212cSMike Anderson { 310432a212cSMike Anderson return test_bit(DMF_DELETING, &md->flags); 311432a212cSMike Anderson } 312432a212cSMike Anderson 31305bdb996SChristoph Hellwig static int dm_blk_open(struct gendisk *disk, blk_mode_t mode) 3141da177e4SLinus Torvalds { 3151da177e4SLinus Torvalds struct mapped_device *md; 3161da177e4SLinus Torvalds 317fba9f90eSJeff Mahoney spin_lock(&_minor_lock); 318fba9f90eSJeff Mahoney 319d32e2bf8SChristoph Hellwig md = disk->private_data; 320fba9f90eSJeff Mahoney if (!md) 321fba9f90eSJeff Mahoney goto out; 322fba9f90eSJeff Mahoney 3235c6bd75dSAlasdair G Kergon if (test_bit(DMF_FREEING, &md->flags) || 324432a212cSMike Anderson dm_deleting_md(md)) { 325fba9f90eSJeff Mahoney md = NULL; 326fba9f90eSJeff Mahoney goto out; 327fba9f90eSJeff Mahoney } 328fba9f90eSJeff Mahoney 3291da177e4SLinus Torvalds dm_get(md); 3305c6bd75dSAlasdair G Kergon atomic_inc(&md->open_count); 331fba9f90eSJeff Mahoney out: 332fba9f90eSJeff Mahoney spin_unlock(&_minor_lock); 333fba9f90eSJeff Mahoney 334fba9f90eSJeff Mahoney return md ? 0 : -ENXIO; 3351da177e4SLinus Torvalds } 3361da177e4SLinus Torvalds 337ae220766SChristoph Hellwig static void dm_blk_close(struct gendisk *disk) 3381da177e4SLinus Torvalds { 33963a4f065SMike Snitzer struct mapped_device *md; 3406e9624b8SArnd Bergmann 3414a1aeb98SMilan Broz spin_lock(&_minor_lock); 3424a1aeb98SMilan Broz 34363a4f065SMike Snitzer md = disk->private_data; 34463a4f065SMike Snitzer if (WARN_ON(!md)) 34563a4f065SMike Snitzer goto out; 34663a4f065SMike Snitzer 3472c140a24SMikulas Patocka if (atomic_dec_and_test(&md->open_count) && 3482c140a24SMikulas Patocka (test_bit(DMF_DEFERRED_REMOVE, &md->flags))) 349acfe0ad7SMikulas Patocka queue_work(deferred_remove_workqueue, &deferred_remove_work); 3502c140a24SMikulas Patocka 3511da177e4SLinus Torvalds dm_put(md); 35263a4f065SMike Snitzer out: 3534a1aeb98SMilan Broz spin_unlock(&_minor_lock); 3541da177e4SLinus Torvalds } 3551da177e4SLinus Torvalds 3565c6bd75dSAlasdair G Kergon int dm_open_count(struct mapped_device *md) 3575c6bd75dSAlasdair G Kergon { 3585c6bd75dSAlasdair G Kergon return atomic_read(&md->open_count); 3595c6bd75dSAlasdair G Kergon } 3605c6bd75dSAlasdair G Kergon 3615c6bd75dSAlasdair G Kergon /* 3625c6bd75dSAlasdair G Kergon * Guarantees nothing is using the device before it's deleted. 3635c6bd75dSAlasdair G Kergon */ 3642c140a24SMikulas Patocka int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred) 3655c6bd75dSAlasdair G Kergon { 3665c6bd75dSAlasdair G Kergon int r = 0; 3675c6bd75dSAlasdair G Kergon 3685c6bd75dSAlasdair G Kergon spin_lock(&_minor_lock); 3695c6bd75dSAlasdair G Kergon 3702c140a24SMikulas Patocka if (dm_open_count(md)) { 3715c6bd75dSAlasdair G Kergon r = -EBUSY; 3722c140a24SMikulas Patocka if (mark_deferred) 3732c140a24SMikulas Patocka set_bit(DMF_DEFERRED_REMOVE, &md->flags); 3742c140a24SMikulas Patocka } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags)) 3752c140a24SMikulas Patocka r = -EEXIST; 3765c6bd75dSAlasdair G Kergon else 3775c6bd75dSAlasdair G Kergon set_bit(DMF_DELETING, &md->flags); 3785c6bd75dSAlasdair G Kergon 3795c6bd75dSAlasdair G Kergon spin_unlock(&_minor_lock); 3805c6bd75dSAlasdair G Kergon 3815c6bd75dSAlasdair G Kergon return r; 3825c6bd75dSAlasdair G Kergon } 3835c6bd75dSAlasdair G Kergon 3842c140a24SMikulas Patocka int dm_cancel_deferred_remove(struct mapped_device *md) 3852c140a24SMikulas Patocka { 3862c140a24SMikulas Patocka int r = 0; 3872c140a24SMikulas Patocka 3882c140a24SMikulas Patocka spin_lock(&_minor_lock); 3892c140a24SMikulas Patocka 3902c140a24SMikulas Patocka if (test_bit(DMF_DELETING, &md->flags)) 3912c140a24SMikulas Patocka r = -EBUSY; 3922c140a24SMikulas Patocka else 3932c140a24SMikulas Patocka clear_bit(DMF_DEFERRED_REMOVE, &md->flags); 3942c140a24SMikulas Patocka 3952c140a24SMikulas Patocka spin_unlock(&_minor_lock); 3962c140a24SMikulas Patocka 3972c140a24SMikulas Patocka return r; 3982c140a24SMikulas Patocka } 3992c140a24SMikulas Patocka 4002c140a24SMikulas Patocka static void do_deferred_remove(struct work_struct *w) 4012c140a24SMikulas Patocka { 4022c140a24SMikulas Patocka dm_deferred_remove(); 4032c140a24SMikulas Patocka } 4042c140a24SMikulas Patocka 4053ac51e74SDarrick J. Wong static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo) 4063ac51e74SDarrick J. Wong { 4073ac51e74SDarrick J. Wong struct mapped_device *md = bdev->bd_disk->private_data; 4083ac51e74SDarrick J. Wong 4093ac51e74SDarrick J. Wong return dm_get_geometry(md, geo); 4103ac51e74SDarrick J. Wong } 4113ac51e74SDarrick J. Wong 412971888c4SMike Snitzer static int dm_prepare_ioctl(struct mapped_device *md, int *srcu_idx, 4135bd5e8d8SMike Snitzer struct block_device **bdev) 414aa129a22SMilan Broz { 415564b5c54SMike Snitzer struct dm_target *ti; 4166c182cd8SHannes Reinecke struct dm_table *map; 417971888c4SMike Snitzer int r; 418aa129a22SMilan Broz 4196c182cd8SHannes Reinecke retry: 420e56f81e0SChristoph Hellwig r = -ENOTTY; 421971888c4SMike Snitzer map = dm_get_live_table(md, srcu_idx); 422aa129a22SMilan Broz if (!map || !dm_table_get_size(map)) 423971888c4SMike Snitzer return r; 424aa129a22SMilan Broz 425aa129a22SMilan Broz /* We only support devices that have a single target */ 4262aec377aSMike Snitzer if (map->num_targets != 1) 427971888c4SMike Snitzer return r; 428aa129a22SMilan Broz 429564b5c54SMike Snitzer ti = dm_table_get_target(map, 0); 430564b5c54SMike Snitzer if (!ti->type->prepare_ioctl) 431e56f81e0SChristoph Hellwig return r; 432aa129a22SMilan Broz 433971888c4SMike Snitzer if (dm_suspended_md(md)) 434971888c4SMike Snitzer return -EAGAIN; 435971888c4SMike Snitzer 436564b5c54SMike Snitzer r = ti->type->prepare_ioctl(ti, bdev); 4375bbbfdf6SJunichi Nomura if (r == -ENOTCONN && !fatal_signal_pending(current)) { 438971888c4SMike Snitzer dm_put_live_table(md, *srcu_idx); 439238d991fSHeinz Mauelshagen fsleep(10000); 4406c182cd8SHannes Reinecke goto retry; 4416c182cd8SHannes Reinecke } 442971888c4SMike Snitzer 443e56f81e0SChristoph Hellwig return r; 444e56f81e0SChristoph Hellwig } 4456c182cd8SHannes Reinecke 446971888c4SMike Snitzer static void dm_unprepare_ioctl(struct mapped_device *md, int srcu_idx) 447971888c4SMike Snitzer { 448971888c4SMike Snitzer dm_put_live_table(md, srcu_idx); 449971888c4SMike Snitzer } 450971888c4SMike Snitzer 45105bdb996SChristoph Hellwig static int dm_blk_ioctl(struct block_device *bdev, blk_mode_t mode, 452e56f81e0SChristoph Hellwig unsigned int cmd, unsigned long arg) 453e56f81e0SChristoph Hellwig { 454e56f81e0SChristoph Hellwig struct mapped_device *md = bdev->bd_disk->private_data; 455971888c4SMike Snitzer int r, srcu_idx; 456e56f81e0SChristoph Hellwig 4575bd5e8d8SMike Snitzer r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 458e56f81e0SChristoph Hellwig if (r < 0) 459971888c4SMike Snitzer goto out; 460e56f81e0SChristoph Hellwig 461e56f81e0SChristoph Hellwig if (r > 0) { 462e56f81e0SChristoph Hellwig /* 463e980f623SChristoph Hellwig * Target determined this ioctl is being issued against a 464e980f623SChristoph Hellwig * subset of the parent bdev; require extra privileges. 465e56f81e0SChristoph Hellwig */ 466e980f623SChristoph Hellwig if (!capable(CAP_SYS_RAWIO)) { 4670378c625SMike Snitzer DMDEBUG_LIMIT( 468e980f623SChristoph Hellwig "%s: sending ioctl %x to DM device without required privilege.", 469e980f623SChristoph Hellwig current->comm, cmd); 470e980f623SChristoph Hellwig r = -ENOIOCTLCMD; 471e56f81e0SChristoph Hellwig goto out; 472e56f81e0SChristoph Hellwig } 473e980f623SChristoph Hellwig } 474e56f81e0SChristoph Hellwig 475a7cb3d2fSChristoph Hellwig if (!bdev->bd_disk->fops->ioctl) 476a7cb3d2fSChristoph Hellwig r = -ENOTTY; 477a7cb3d2fSChristoph Hellwig else 478a7cb3d2fSChristoph Hellwig r = bdev->bd_disk->fops->ioctl(bdev, mode, cmd, arg); 479e56f81e0SChristoph Hellwig out: 480971888c4SMike Snitzer dm_unprepare_ioctl(md, srcu_idx); 481aa129a22SMilan Broz return r; 482aa129a22SMilan Broz } 483aa129a22SMilan Broz 4847465d7acSMike Snitzer u64 dm_start_time_ns_from_clone(struct bio *bio) 4857465d7acSMike Snitzer { 4866c23f0bdSChristoph Hellwig return jiffies_to_nsecs(clone_to_tio(bio)->io->start_time); 4877465d7acSMike Snitzer } 4887465d7acSMike Snitzer EXPORT_SYMBOL_GPL(dm_start_time_ns_from_clone); 4897465d7acSMike Snitzer 49006eed768SMike Snitzer static inline bool bio_is_flush_with_data(struct bio *bio) 4917465d7acSMike Snitzer { 4928d394bc4SMike Snitzer return ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size); 4937465d7acSMike Snitzer } 4947465d7acSMike Snitzer 49506eed768SMike Snitzer static inline unsigned int dm_io_sectors(struct dm_io *io, struct bio *bio) 4967465d7acSMike Snitzer { 497d3de6d12SMing Lei /* 498d3de6d12SMing Lei * If REQ_PREFLUSH set, don't account payload, it will be 499d3de6d12SMing Lei * submitted (and accounted) after this flush completes. 500d3de6d12SMing Lei */ 501d3de6d12SMing Lei if (bio_is_flush_with_data(bio)) 50206eed768SMike Snitzer return 0; 50306eed768SMike Snitzer if (unlikely(dm_io_flagged(io, DM_IO_WAS_SPLIT))) 50406eed768SMike Snitzer return io->sectors; 50506eed768SMike Snitzer return bio_sectors(bio); 50606eed768SMike Snitzer } 5078d394bc4SMike Snitzer 50806eed768SMike Snitzer static void dm_io_acct(struct dm_io *io, bool end) 50906eed768SMike Snitzer { 51006eed768SMike Snitzer struct bio *bio = io->orig_bio; 5118d394bc4SMike Snitzer 512526d1006SLi Nan if (dm_io_flagged(io, DM_IO_BLK_STAT)) { 5138d394bc4SMike Snitzer if (!end) 514526d1006SLi Nan bdev_start_io_acct(bio->bi_bdev, bio_op(bio), 51506eed768SMike Snitzer io->start_time); 5168d394bc4SMike Snitzer else 517526d1006SLi Nan bdev_end_io_acct(bio->bi_bdev, bio_op(bio), 51806eed768SMike Snitzer dm_io_sectors(io, bio), 51906eed768SMike Snitzer io->start_time); 520526d1006SLi Nan } 5217465d7acSMike Snitzer 522442761fdSMike Snitzer if (static_branch_unlikely(&stats_enabled) && 52306eed768SMike Snitzer unlikely(dm_stats_used(&io->md->stats))) { 5247dd76d1fSMing Lei sector_t sector; 5257dd76d1fSMing Lei 52606eed768SMike Snitzer if (unlikely(dm_io_flagged(io, DM_IO_WAS_SPLIT))) 5277dd76d1fSMing Lei sector = bio_end_sector(bio) - io->sector_offset; 52806eed768SMike Snitzer else 52906eed768SMike Snitzer sector = bio->bi_iter.bi_sector; 5307dd76d1fSMing Lei 53106eed768SMike Snitzer dm_stats_account_io(&io->md->stats, bio_data_dir(bio), 53206eed768SMike Snitzer sector, dm_io_sectors(io, bio), 53306eed768SMike Snitzer end, io->start_time, &io->stats_aux); 5348d394bc4SMike Snitzer } 5357dd76d1fSMing Lei } 5368d394bc4SMike Snitzer 537b992b40dSMing Lei static void __dm_start_io_acct(struct dm_io *io) 5388d394bc4SMike Snitzer { 539e6926ad0SMing Lei dm_io_acct(io, false); 5408d394bc4SMike Snitzer } 5418d394bc4SMike Snitzer 5420fbb4d93SMike Snitzer static void dm_start_io_acct(struct dm_io *io, struct bio *clone) 5438d394bc4SMike Snitzer { 5440fbb4d93SMike Snitzer /* 5450fbb4d93SMike Snitzer * Ensure IO accounting is only ever started once. 5460fbb4d93SMike Snitzer */ 5473b03f7c1SMike Snitzer if (dm_io_flagged(io, DM_IO_ACCOUNTED)) 5480fbb4d93SMike Snitzer return; 5493b03f7c1SMike Snitzer 5503b03f7c1SMike Snitzer /* Expect no possibility for race unless DM_TIO_IS_DUPLICATE_BIO. */ 5513b03f7c1SMike Snitzer if (!clone || likely(dm_tio_is_normal(clone_to_tio(clone)))) { 55282f6cdccSMike Snitzer dm_io_set_flag(io, DM_IO_ACCOUNTED); 55382f6cdccSMike Snitzer } else { 55482f6cdccSMike Snitzer unsigned long flags; 555655f3aadSMike Snitzer /* Can afford locking given DM_TIO_IS_DUPLICATE_BIO */ 5564d7bca13SMike Snitzer spin_lock_irqsave(&io->lock, flags); 55710eb3a0dSBenjamin Marzinski if (dm_io_flagged(io, DM_IO_ACCOUNTED)) { 55810eb3a0dSBenjamin Marzinski spin_unlock_irqrestore(&io->lock, flags); 55910eb3a0dSBenjamin Marzinski return; 56010eb3a0dSBenjamin Marzinski } 56182f6cdccSMike Snitzer dm_io_set_flag(io, DM_IO_ACCOUNTED); 5624d7bca13SMike Snitzer spin_unlock_irqrestore(&io->lock, flags); 56382f6cdccSMike Snitzer } 5640fbb4d93SMike Snitzer 565b992b40dSMing Lei __dm_start_io_acct(io); 5660fbb4d93SMike Snitzer } 5670fbb4d93SMike Snitzer 568b992b40dSMing Lei static void dm_end_io_acct(struct dm_io *io) 5690fbb4d93SMike Snitzer { 570e6926ad0SMing Lei dm_io_acct(io, true); 5717465d7acSMike Snitzer } 572978e51baSMike Snitzer 573978e51baSMike Snitzer static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio) 5741da177e4SLinus Torvalds { 57564f52b0eSMike Snitzer struct dm_io *io; 57664f52b0eSMike Snitzer struct dm_target_io *tio; 57764f52b0eSMike Snitzer struct bio *clone; 57864f52b0eSMike Snitzer 57929dec90aSChristoph Hellwig clone = bio_alloc_clone(NULL, bio, GFP_NOIO, &md->mempools->io_bs); 5806c23f0bdSChristoph Hellwig tio = clone_to_tio(clone); 581655f3aadSMike Snitzer tio->flags = 0; 582655f3aadSMike Snitzer dm_tio_set_flag(tio, DM_TIO_INSIDE_DM_IO); 58364f52b0eSMike Snitzer tio->io = NULL; 58464f52b0eSMike Snitzer 58564f52b0eSMike Snitzer io = container_of(tio, struct dm_io, tio); 58664f52b0eSMike Snitzer io->magic = DM_IO_MAGIC; 58784b98f4cSMike Snitzer io->status = BLK_STS_OK; 5880f14d60aSMing Lei 5890f14d60aSMing Lei /* one ref is for submission, the other is for completion */ 5900f14d60aSMing Lei atomic_set(&io->io_count, 2); 5919f6dc633SMike Snitzer this_cpu_inc(*md->pending_io); 5927dd76d1fSMing Lei io->orig_bio = bio; 593978e51baSMike Snitzer io->md = md; 5944d7bca13SMike Snitzer spin_lock_init(&io->lock); 595b879f915SMike Snitzer io->start_time = jiffies; 59682f6cdccSMike Snitzer io->flags = 0; 597526d1006SLi Nan if (blk_queue_io_stat(md->queue)) 598526d1006SLi Nan dm_io_set_flag(io, DM_IO_BLK_STAT); 59964f52b0eSMike Snitzer 600c4f512d2SMike Snitzer if (static_branch_unlikely(&stats_enabled) && 601c4f512d2SMike Snitzer unlikely(dm_stats_used(&md->stats))) 6020cdb90f0SMike Snitzer dm_stats_record_start(&md->stats, &io->stats_aux); 60364f52b0eSMike Snitzer 60464f52b0eSMike Snitzer return io; 6051da177e4SLinus Torvalds } 6061da177e4SLinus Torvalds 6070119ab14SMike Snitzer static void free_io(struct dm_io *io) 6081da177e4SLinus Torvalds { 60964f52b0eSMike Snitzer bio_put(&io->tio.clone); 61064f52b0eSMike Snitzer } 61164f52b0eSMike Snitzer 6121d1068ceSChristoph Hellwig static struct bio *alloc_tio(struct clone_info *ci, struct dm_target *ti, 61386a3238cSHeinz Mauelshagen unsigned int target_bio_nr, unsigned int *len, gfp_t gfp_mask) 61464f52b0eSMike Snitzer { 6159dd1cd32SMike Snitzer struct mapped_device *md = ci->io->md; 61664f52b0eSMike Snitzer struct dm_target_io *tio; 617018b05ebSMike Snitzer struct bio *clone; 61864f52b0eSMike Snitzer 61964f52b0eSMike Snitzer if (!ci->io->tio.io) { 62064f52b0eSMike Snitzer /* the dm_target_io embedded in ci->io is available */ 62164f52b0eSMike Snitzer tio = &ci->io->tio; 622018b05ebSMike Snitzer /* alloc_io() already initialized embedded clone */ 623018b05ebSMike Snitzer clone = &tio->clone; 62464f52b0eSMike Snitzer } else { 62529dec90aSChristoph Hellwig clone = bio_alloc_clone(NULL, ci->bio, gfp_mask, 62629dec90aSChristoph Hellwig &md->mempools->bs); 62764f52b0eSMike Snitzer if (!clone) 62864f52b0eSMike Snitzer return NULL; 62964f52b0eSMike Snitzer 630b99fdcdcSMing Lei /* REQ_DM_POLL_LIST shouldn't be inherited */ 631b99fdcdcSMing Lei clone->bi_opf &= ~REQ_DM_POLL_LIST; 632b99fdcdcSMing Lei 6336c23f0bdSChristoph Hellwig tio = clone_to_tio(clone); 634655f3aadSMike Snitzer tio->flags = 0; /* also clears DM_TIO_INSIDE_DM_IO */ 63564f52b0eSMike Snitzer } 63664f52b0eSMike Snitzer 63764f52b0eSMike Snitzer tio->magic = DM_TIO_MAGIC; 63864f52b0eSMike Snitzer tio->io = ci->io; 63964f52b0eSMike Snitzer tio->ti = ti; 64064f52b0eSMike Snitzer tio->target_bio_nr = target_bio_nr; 641dc8e2021SChristoph Hellwig tio->len_ptr = len; 642743598f0SMike Snitzer tio->old_sector = 0; 64364f52b0eSMike Snitzer 6449dd1cd32SMike Snitzer /* Set default bdev, but target must bio_set_dev() before issuing IO */ 6459dd1cd32SMike Snitzer clone->bi_bdev = md->disk->part0; 6469dd1cd32SMike Snitzer if (unlikely(ti->needs_bio_set_dev)) 6479dd1cd32SMike Snitzer bio_set_dev(clone, md->disk->part0); 6489dd1cd32SMike Snitzer 649018b05ebSMike Snitzer if (len) { 650018b05ebSMike Snitzer clone->bi_iter.bi_size = to_bytes(*len); 651018b05ebSMike Snitzer if (bio_integrity(clone)) 652018b05ebSMike Snitzer bio_integrity_trim(clone); 653018b05ebSMike Snitzer } 654018b05ebSMike Snitzer 655018b05ebSMike Snitzer return clone; 6561da177e4SLinus Torvalds } 6571da177e4SLinus Torvalds 6581d1068ceSChristoph Hellwig static void free_tio(struct bio *clone) 6591da177e4SLinus Torvalds { 660655f3aadSMike Snitzer if (dm_tio_flagged(clone_to_tio(clone), DM_TIO_INSIDE_DM_IO)) 66164f52b0eSMike Snitzer return; 6621d1068ceSChristoph Hellwig bio_put(clone); 6631da177e4SLinus Torvalds } 6641da177e4SLinus Torvalds 6651da177e4SLinus Torvalds /* 6661da177e4SLinus Torvalds * Add the bio to the list of deferred io. 6671da177e4SLinus Torvalds */ 66892c63902SMikulas Patocka static void queue_io(struct mapped_device *md, struct bio *bio) 6691da177e4SLinus Torvalds { 67005447420SKiyoshi Ueda unsigned long flags; 6711da177e4SLinus Torvalds 67205447420SKiyoshi Ueda spin_lock_irqsave(&md->deferred_lock, flags); 6731da177e4SLinus Torvalds bio_list_add(&md->deferred, bio); 67405447420SKiyoshi Ueda spin_unlock_irqrestore(&md->deferred_lock, flags); 67592c63902SMikulas Patocka queue_work(md->wq, &md->work); 6761da177e4SLinus Torvalds } 6771da177e4SLinus Torvalds 6781da177e4SLinus Torvalds /* 6791da177e4SLinus Torvalds * Everyone (including functions in this file), should use this 6801da177e4SLinus Torvalds * function to access the md->map field, and make sure they call 68183d5e5b0SMikulas Patocka * dm_put_live_table() when finished. 6821da177e4SLinus Torvalds */ 683563a225cSMike Snitzer struct dm_table *dm_get_live_table(struct mapped_device *md, 684563a225cSMike Snitzer int *srcu_idx) __acquires(md->io_barrier) 6851da177e4SLinus Torvalds { 68683d5e5b0SMikulas Patocka *srcu_idx = srcu_read_lock(&md->io_barrier); 6871da177e4SLinus Torvalds 68883d5e5b0SMikulas Patocka return srcu_dereference(md->map, &md->io_barrier); 68983d5e5b0SMikulas Patocka } 6901da177e4SLinus Torvalds 691563a225cSMike Snitzer void dm_put_live_table(struct mapped_device *md, 692563a225cSMike Snitzer int srcu_idx) __releases(md->io_barrier) 69383d5e5b0SMikulas Patocka { 69483d5e5b0SMikulas Patocka srcu_read_unlock(&md->io_barrier, srcu_idx); 69583d5e5b0SMikulas Patocka } 69683d5e5b0SMikulas Patocka 69783d5e5b0SMikulas Patocka void dm_sync_table(struct mapped_device *md) 69883d5e5b0SMikulas Patocka { 69983d5e5b0SMikulas Patocka synchronize_srcu(&md->io_barrier); 70083d5e5b0SMikulas Patocka synchronize_rcu_expedited(); 70183d5e5b0SMikulas Patocka } 70283d5e5b0SMikulas Patocka 70383d5e5b0SMikulas Patocka /* 70483d5e5b0SMikulas Patocka * A fast alternative to dm_get_live_table/dm_put_live_table. 70583d5e5b0SMikulas Patocka * The caller must not block between these two functions. 70683d5e5b0SMikulas Patocka */ 70783d5e5b0SMikulas Patocka static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU) 70883d5e5b0SMikulas Patocka { 70983d5e5b0SMikulas Patocka rcu_read_lock(); 71083d5e5b0SMikulas Patocka return rcu_dereference(md->map); 71183d5e5b0SMikulas Patocka } 71283d5e5b0SMikulas Patocka 71383d5e5b0SMikulas Patocka static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU) 71483d5e5b0SMikulas Patocka { 71583d5e5b0SMikulas Patocka rcu_read_unlock(); 7161da177e4SLinus Torvalds } 7171da177e4SLinus Torvalds 718971888c4SMike Snitzer static char *_dm_claim_ptr = "I belong to device-mapper"; 719971888c4SMike Snitzer 7203ac51e74SDarrick J. Wong /* 72186f1152bSBenjamin Marzinski * Open a table device so we can use it as a map destination. 72286f1152bSBenjamin Marzinski */ 723b9a785d2SChristoph Hellwig static struct table_device *open_table_device(struct mapped_device *md, 72405bdb996SChristoph Hellwig dev_t dev, blk_mode_t mode) 72586f1152bSBenjamin Marzinski { 726b9a785d2SChristoph Hellwig struct table_device *td; 72786f1152bSBenjamin Marzinski struct block_device *bdev; 728cd913c76SChristoph Hellwig u64 part_off; 72986f1152bSBenjamin Marzinski int r; 73086f1152bSBenjamin Marzinski 731b9a785d2SChristoph Hellwig td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id); 732b9a785d2SChristoph Hellwig if (!td) 733b9a785d2SChristoph Hellwig return ERR_PTR(-ENOMEM); 734b9a785d2SChristoph Hellwig refcount_set(&td->count, 1); 73586f1152bSBenjamin Marzinski 7362736e8eeSChristoph Hellwig bdev = blkdev_get_by_dev(dev, mode, _dm_claim_ptr, NULL); 737b9a785d2SChristoph Hellwig if (IS_ERR(bdev)) { 738b9a785d2SChristoph Hellwig r = PTR_ERR(bdev); 739b9a785d2SChristoph Hellwig goto out_free_td; 74086f1152bSBenjamin Marzinski } 74186f1152bSBenjamin Marzinski 7421a581b72SChristoph Hellwig /* 7431a581b72SChristoph Hellwig * We can be called before the dm disk is added. In that case we can't 7441a581b72SChristoph Hellwig * register the holder relation here. It will be done once add_disk was 7451a581b72SChristoph Hellwig * called. 7461a581b72SChristoph Hellwig */ 7471a581b72SChristoph Hellwig if (md->disk->slave_dir) { 7481a581b72SChristoph Hellwig r = bd_link_disk_holder(bdev, md->disk); 749b9a785d2SChristoph Hellwig if (r) 750b9a785d2SChristoph Hellwig goto out_blkdev_put; 7511a581b72SChristoph Hellwig } 752b9a785d2SChristoph Hellwig 753b9a785d2SChristoph Hellwig td->dm_dev.mode = mode; 75486f1152bSBenjamin Marzinski td->dm_dev.bdev = bdev; 7558012b866SShiyang Ruan td->dm_dev.dax_dev = fs_dax_get_by_bdev(bdev, &part_off, NULL, NULL); 756b9a785d2SChristoph Hellwig format_dev_t(td->dm_dev.name, dev); 757b9a785d2SChristoph Hellwig list_add(&td->list, &md->table_devices); 758b9a785d2SChristoph Hellwig return td; 759b9a785d2SChristoph Hellwig 760b9a785d2SChristoph Hellwig out_blkdev_put: 7612736e8eeSChristoph Hellwig blkdev_put(bdev, _dm_claim_ptr); 762b9a785d2SChristoph Hellwig out_free_td: 763b9a785d2SChristoph Hellwig kfree(td); 764b9a785d2SChristoph Hellwig return ERR_PTR(r); 76586f1152bSBenjamin Marzinski } 76686f1152bSBenjamin Marzinski 76786f1152bSBenjamin Marzinski /* 76886f1152bSBenjamin Marzinski * Close a table device that we've been using. 76986f1152bSBenjamin Marzinski */ 77086f1152bSBenjamin Marzinski static void close_table_device(struct table_device *td, struct mapped_device *md) 77186f1152bSBenjamin Marzinski { 7721a581b72SChristoph Hellwig if (md->disk->slave_dir) 7731a581b72SChristoph Hellwig bd_unlink_disk_holder(td->dm_dev.bdev, md->disk); 7742736e8eeSChristoph Hellwig blkdev_put(td->dm_dev.bdev, _dm_claim_ptr); 775817bf402SDan Williams put_dax(td->dm_dev.dax_dev); 7767b586583SChristoph Hellwig list_del(&td->list); 7777b586583SChristoph Hellwig kfree(td); 77886f1152bSBenjamin Marzinski } 77986f1152bSBenjamin Marzinski 78086f1152bSBenjamin Marzinski static struct table_device *find_table_device(struct list_head *l, dev_t dev, 78105bdb996SChristoph Hellwig blk_mode_t mode) 7828454fca4SSheetal Singala { 78386f1152bSBenjamin Marzinski struct table_device *td; 78486f1152bSBenjamin Marzinski 78586f1152bSBenjamin Marzinski list_for_each_entry(td, l, list) 78686f1152bSBenjamin Marzinski if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode) 78786f1152bSBenjamin Marzinski return td; 78886f1152bSBenjamin Marzinski 78986f1152bSBenjamin Marzinski return NULL; 79086f1152bSBenjamin Marzinski } 79186f1152bSBenjamin Marzinski 79205bdb996SChristoph Hellwig int dm_get_table_device(struct mapped_device *md, dev_t dev, blk_mode_t mode, 7938454fca4SSheetal Singala struct dm_dev **result) 7948454fca4SSheetal Singala { 79586f1152bSBenjamin Marzinski struct table_device *td; 79686f1152bSBenjamin Marzinski 79786f1152bSBenjamin Marzinski mutex_lock(&md->table_devices_lock); 79886f1152bSBenjamin Marzinski td = find_table_device(&md->table_devices, dev, mode); 79986f1152bSBenjamin Marzinski if (!td) { 800b9a785d2SChristoph Hellwig td = open_table_device(md, dev, mode); 801b9a785d2SChristoph Hellwig if (IS_ERR(td)) { 80286f1152bSBenjamin Marzinski mutex_unlock(&md->table_devices_lock); 803b9a785d2SChristoph Hellwig return PTR_ERR(td); 80486f1152bSBenjamin Marzinski } 805b0b4d7c6SElena Reshetova } else { 806b0b4d7c6SElena Reshetova refcount_inc(&td->count); 80786f1152bSBenjamin Marzinski } 80886f1152bSBenjamin Marzinski mutex_unlock(&md->table_devices_lock); 80986f1152bSBenjamin Marzinski 81086f1152bSBenjamin Marzinski *result = &td->dm_dev; 81186f1152bSBenjamin Marzinski return 0; 81286f1152bSBenjamin Marzinski } 81386f1152bSBenjamin Marzinski 81486f1152bSBenjamin Marzinski void dm_put_table_device(struct mapped_device *md, struct dm_dev *d) 81586f1152bSBenjamin Marzinski { 81686f1152bSBenjamin Marzinski struct table_device *td = container_of(d, struct table_device, dm_dev); 81786f1152bSBenjamin Marzinski 81886f1152bSBenjamin Marzinski mutex_lock(&md->table_devices_lock); 8197b586583SChristoph Hellwig if (refcount_dec_and_test(&td->count)) 82086f1152bSBenjamin Marzinski close_table_device(td, md); 82186f1152bSBenjamin Marzinski mutex_unlock(&md->table_devices_lock); 82286f1152bSBenjamin Marzinski } 82386f1152bSBenjamin Marzinski 82486f1152bSBenjamin Marzinski /* 8253ac51e74SDarrick J. Wong * Get the geometry associated with a dm device 8263ac51e74SDarrick J. Wong */ 8273ac51e74SDarrick J. Wong int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo) 8283ac51e74SDarrick J. Wong { 8293ac51e74SDarrick J. Wong *geo = md->geometry; 8303ac51e74SDarrick J. Wong 8313ac51e74SDarrick J. Wong return 0; 8323ac51e74SDarrick J. Wong } 8333ac51e74SDarrick J. Wong 8343ac51e74SDarrick J. Wong /* 8353ac51e74SDarrick J. Wong * Set the geometry of a device. 8363ac51e74SDarrick J. Wong */ 8373ac51e74SDarrick J. Wong int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo) 8383ac51e74SDarrick J. Wong { 8393ac51e74SDarrick J. Wong sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors; 8403ac51e74SDarrick J. Wong 8413ac51e74SDarrick J. Wong if (geo->start > sz) { 84243e6c111SMikulas Patocka DMERR("Start sector is beyond the geometry limits."); 8433ac51e74SDarrick J. Wong return -EINVAL; 8443ac51e74SDarrick J. Wong } 8453ac51e74SDarrick J. Wong 8463ac51e74SDarrick J. Wong md->geometry = *geo; 8473ac51e74SDarrick J. Wong 8483ac51e74SDarrick J. Wong return 0; 8493ac51e74SDarrick J. Wong } 8503ac51e74SDarrick J. Wong 8512e93ccc1SKiyoshi Ueda static int __noflush_suspending(struct mapped_device *md) 8522e93ccc1SKiyoshi Ueda { 8532e93ccc1SKiyoshi Ueda return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 8542e93ccc1SKiyoshi Ueda } 8552e93ccc1SKiyoshi Ueda 8568b211aacSMing Lei static void dm_requeue_add_io(struct dm_io *io, bool first_stage) 8571da177e4SLinus Torvalds { 858b35f8caaSMilan Broz struct mapped_device *md = io->md; 8592e93ccc1SKiyoshi Ueda 8608b211aacSMing Lei if (first_stage) { 8618b211aacSMing Lei struct dm_io *next = md->requeue_list; 8628b211aacSMing Lei 8638b211aacSMing Lei md->requeue_list = io; 8648b211aacSMing Lei io->next = next; 8658b211aacSMing Lei } else { 8668b211aacSMing Lei bio_list_add_head(&md->deferred, io->orig_bio); 8678b211aacSMing Lei } 8688b211aacSMing Lei } 8698b211aacSMing Lei 8708b211aacSMing Lei static void dm_kick_requeue(struct mapped_device *md, bool first_stage) 8718b211aacSMing Lei { 8728b211aacSMing Lei if (first_stage) 8738b211aacSMing Lei queue_work(md->wq, &md->requeue_work); 8748b211aacSMing Lei else 8758b211aacSMing Lei queue_work(md->wq, &md->work); 8768b211aacSMing Lei } 8778b211aacSMing Lei 8782e93ccc1SKiyoshi Ueda /* 879444fe04fSMing Lei * Return true if the dm_io's original bio is requeued. 880444fe04fSMing Lei * io->status is updated with error if requeue disallowed. 881444fe04fSMing Lei */ 8828b211aacSMing Lei static bool dm_handle_requeue(struct dm_io *io, bool first_stage) 883444fe04fSMing Lei { 8848b211aacSMing Lei struct bio *bio = io->orig_bio; 885444fe04fSMing Lei bool handle_requeue = (io->status == BLK_STS_DM_REQUEUE); 886444fe04fSMing Lei bool handle_polled_eagain = ((io->status == BLK_STS_AGAIN) && 887444fe04fSMing Lei (bio->bi_opf & REQ_POLLED)); 888444fe04fSMing Lei struct mapped_device *md = io->md; 889444fe04fSMing Lei bool requeued = false; 890444fe04fSMing Lei 891444fe04fSMing Lei if (handle_requeue || handle_polled_eagain) { 892444fe04fSMing Lei unsigned long flags; 893444fe04fSMing Lei 894444fe04fSMing Lei if (bio->bi_opf & REQ_POLLED) { 895444fe04fSMing Lei /* 896444fe04fSMing Lei * Upper layer won't help us poll split bio 897444fe04fSMing Lei * (io->orig_bio may only reflect a subset of the 898444fe04fSMing Lei * pre-split original) so clear REQ_POLLED. 899444fe04fSMing Lei */ 900444fe04fSMing Lei bio_clear_polled(bio); 901444fe04fSMing Lei } 902444fe04fSMing Lei 903444fe04fSMing Lei /* 904444fe04fSMing Lei * Target requested pushing back the I/O or 905444fe04fSMing Lei * polled IO hit BLK_STS_AGAIN. 9062e93ccc1SKiyoshi Ueda */ 907022c2611SMikulas Patocka spin_lock_irqsave(&md->deferred_lock, flags); 908444fe04fSMing Lei if ((__noflush_suspending(md) && 909444fe04fSMing Lei !WARN_ON_ONCE(dm_is_zone_write(md, bio))) || 9108b211aacSMing Lei handle_polled_eagain || first_stage) { 9118b211aacSMing Lei dm_requeue_add_io(io, first_stage); 912444fe04fSMing Lei requeued = true; 913bf14e2b2SDamien Le Moal } else { 914bf14e2b2SDamien Le Moal /* 915bf14e2b2SDamien Le Moal * noflush suspend was interrupted or this is 916bf14e2b2SDamien Le Moal * a write to a zoned target. 917bf14e2b2SDamien Le Moal */ 9184e4cbee9SChristoph Hellwig io->status = BLK_STS_IOERR; 919bf14e2b2SDamien Le Moal } 920022c2611SMikulas Patocka spin_unlock_irqrestore(&md->deferred_lock, flags); 9212e93ccc1SKiyoshi Ueda } 9222e93ccc1SKiyoshi Ueda 923444fe04fSMing Lei if (requeued) 9248b211aacSMing Lei dm_kick_requeue(md, first_stage); 925444fe04fSMing Lei 926444fe04fSMing Lei return requeued; 927444fe04fSMing Lei } 928444fe04fSMing Lei 9298b211aacSMing Lei static void __dm_io_complete(struct dm_io *io, bool first_stage) 930444fe04fSMing Lei { 9318b211aacSMing Lei struct bio *bio = io->orig_bio; 932444fe04fSMing Lei struct mapped_device *md = io->md; 933444fe04fSMing Lei blk_status_t io_error; 934444fe04fSMing Lei bool requeued; 935444fe04fSMing Lei 9368b211aacSMing Lei requeued = dm_handle_requeue(io, first_stage); 9378b211aacSMing Lei if (requeued && first_stage) 9388b211aacSMing Lei return; 939444fe04fSMing Lei 9404e4cbee9SChristoph Hellwig io_error = io->status; 94182f6cdccSMike Snitzer if (dm_io_flagged(io, DM_IO_ACCOUNTED)) 942b992b40dSMing Lei dm_end_io_acct(io); 9430fbb4d93SMike Snitzer else if (!io_error) { 9440fbb4d93SMike Snitzer /* 9450fbb4d93SMike Snitzer * Must handle target that DM_MAPIO_SUBMITTED only to 9460fbb4d93SMike Snitzer * then bio_endio() rather than dm_submit_bio_remap() 9470fbb4d93SMike Snitzer */ 948b992b40dSMing Lei __dm_start_io_acct(io); 949b992b40dSMing Lei dm_end_io_acct(io); 9500fbb4d93SMike Snitzer } 9510119ab14SMike Snitzer free_io(io); 9529f6dc633SMike Snitzer smp_wmb(); 9539f6dc633SMike Snitzer this_cpu_dec(*md->pending_io); 9542056a782SJens Axboe 9559f6dc633SMike Snitzer /* nudge anyone waiting on suspend queue */ 9569f6dc633SMike Snitzer if (unlikely(wq_has_sleeper(&md->wait))) 9579f6dc633SMike Snitzer wake_up(&md->wait); 9581da177e4SLinus Torvalds 959444fe04fSMing Lei /* Return early if the original bio was requeued */ 960444fe04fSMing Lei if (requeued) 96178ccef91SMike Snitzer return; 9626a8736d1STejun Heo 9638d394bc4SMike Snitzer if (bio_is_flush_with_data(bio)) { 964af7e466aSMikulas Patocka /* 9656a8736d1STejun Heo * Preflush done for flush with data, reissue 96628a8f0d3SMike Christie * without REQ_PREFLUSH. 967af7e466aSMikulas Patocka */ 9681eff9d32SJens Axboe bio->bi_opf &= ~REQ_PREFLUSH; 9696a8736d1STejun Heo queue_io(md, bio); 970af7e466aSMikulas Patocka } else { 971b372d360SMike Snitzer /* done with normal IO or empty flush */ 9728dd601faSNeilBrown if (io_error) 9734e4cbee9SChristoph Hellwig bio->bi_status = io_error; 9744246a0b6SChristoph Hellwig bio_endio(bio); 9752e93ccc1SKiyoshi Ueda } 9761da177e4SLinus Torvalds } 977e2736347SMike Snitzer 9788b211aacSMing Lei static void dm_wq_requeue_work(struct work_struct *work) 9798b211aacSMing Lei { 9808b211aacSMing Lei struct mapped_device *md = container_of(work, struct mapped_device, 9818b211aacSMing Lei requeue_work); 9828b211aacSMing Lei unsigned long flags; 9838b211aacSMing Lei struct dm_io *io; 9848b211aacSMing Lei 9858b211aacSMing Lei /* reuse deferred lock to simplify dm_handle_requeue */ 9868b211aacSMing Lei spin_lock_irqsave(&md->deferred_lock, flags); 9878b211aacSMing Lei io = md->requeue_list; 9888b211aacSMing Lei md->requeue_list = NULL; 9898b211aacSMing Lei spin_unlock_irqrestore(&md->deferred_lock, flags); 9908b211aacSMing Lei 9918b211aacSMing Lei while (io) { 9928b211aacSMing Lei struct dm_io *next = io->next; 9938b211aacSMing Lei 99446754bd0SChristoph Hellwig dm_io_rewind(io, &md->disk->bio_split); 9958b211aacSMing Lei 9968b211aacSMing Lei io->next = NULL; 9978b211aacSMing Lei __dm_io_complete(io, false); 9988b211aacSMing Lei io = next; 999f77692d6SMike Snitzer cond_resched(); 10008b211aacSMing Lei } 10018b211aacSMing Lei } 10028b211aacSMing Lei 10038b211aacSMing Lei /* 10048b211aacSMing Lei * Two staged requeue: 10058b211aacSMing Lei * 10068b211aacSMing Lei * 1) io->orig_bio points to the real original bio, and the part mapped to 10078b211aacSMing Lei * this io must be requeued, instead of other parts of the original bio. 10088b211aacSMing Lei * 10098b211aacSMing Lei * 2) io->orig_bio points to new cloned bio which matches the requeued dm_io. 10108b211aacSMing Lei */ 10118b211aacSMing Lei static void dm_io_complete(struct dm_io *io) 10128b211aacSMing Lei { 10138b211aacSMing Lei bool first_requeue; 10148b211aacSMing Lei 10158b211aacSMing Lei /* 10168b211aacSMing Lei * Only dm_io that has been split needs two stage requeue, otherwise 10178b211aacSMing Lei * we may run into long bio clone chain during suspend and OOM could 10188b211aacSMing Lei * be triggered. 10198b211aacSMing Lei * 10208b211aacSMing Lei * Also flush data dm_io won't be marked as DM_IO_WAS_SPLIT, so they 10218b211aacSMing Lei * also aren't handled via the first stage requeue. 10228b211aacSMing Lei */ 10238b211aacSMing Lei if (dm_io_flagged(io, DM_IO_WAS_SPLIT)) 10248b211aacSMing Lei first_requeue = true; 10258b211aacSMing Lei else 10268b211aacSMing Lei first_requeue = false; 10278b211aacSMing Lei 10288b211aacSMing Lei __dm_io_complete(io, first_requeue); 10298b211aacSMing Lei } 10308b211aacSMing Lei 1031e2736347SMike Snitzer /* 1032e2736347SMike Snitzer * Decrements the number of outstanding ios that a bio has been 1033e2736347SMike Snitzer * cloned into, completing the original io if necc. 1034e2736347SMike Snitzer */ 103584b98f4cSMike Snitzer static inline void __dm_io_dec_pending(struct dm_io *io) 1036e2736347SMike Snitzer { 103784b98f4cSMike Snitzer if (atomic_dec_and_test(&io->io_count)) 103884b98f4cSMike Snitzer dm_io_complete(io); 103984b98f4cSMike Snitzer } 104084b98f4cSMike Snitzer 104184b98f4cSMike Snitzer static void dm_io_set_error(struct dm_io *io, blk_status_t error) 104284b98f4cSMike Snitzer { 1043e2736347SMike Snitzer unsigned long flags; 104484b98f4cSMike Snitzer 104584b98f4cSMike Snitzer /* Push-back supersedes any I/O errors */ 10464d7bca13SMike Snitzer spin_lock_irqsave(&io->lock, flags); 1047e2736347SMike Snitzer if (!(io->status == BLK_STS_DM_REQUEUE && 104884b98f4cSMike Snitzer __noflush_suspending(io->md))) { 1049e2736347SMike Snitzer io->status = error; 105084b98f4cSMike Snitzer } 10514d7bca13SMike Snitzer spin_unlock_irqrestore(&io->lock, flags); 1052e2736347SMike Snitzer } 1053e2736347SMike Snitzer 10542e803cd9SMing Lei static void dm_io_dec_pending(struct dm_io *io, blk_status_t error) 105584b98f4cSMike Snitzer { 105684b98f4cSMike Snitzer if (unlikely(error)) 105784b98f4cSMike Snitzer dm_io_set_error(io, error); 105884b98f4cSMike Snitzer 105984b98f4cSMike Snitzer __dm_io_dec_pending(io); 1060af7e466aSMikulas Patocka } 10611da177e4SLinus Torvalds 1062f7995089SMike Snitzer /* 1063f7995089SMike Snitzer * The queue_limits are only valid as long as you have a reference 1064f7995089SMike Snitzer * count on 'md'. But _not_ imposing verification to avoid atomic_read(), 1065f7995089SMike Snitzer */ 1066f7995089SMike Snitzer static inline struct queue_limits *dm_get_queue_limits(struct mapped_device *md) 1067f7995089SMike Snitzer { 1068f7995089SMike Snitzer return &md->queue->limits; 1069f7995089SMike Snitzer } 1070f7995089SMike Snitzer 1071bcb44433SMike Snitzer void disable_discard(struct mapped_device *md) 1072bcb44433SMike Snitzer { 1073bcb44433SMike Snitzer struct queue_limits *limits = dm_get_queue_limits(md); 1074bcb44433SMike Snitzer 1075bcb44433SMike Snitzer /* device doesn't really support DISCARD, disable it */ 1076bcb44433SMike Snitzer limits->max_discard_sectors = 0; 1077bcb44433SMike Snitzer } 1078bcb44433SMike Snitzer 1079ac62d620SChristoph Hellwig void disable_write_zeroes(struct mapped_device *md) 1080ac62d620SChristoph Hellwig { 1081ac62d620SChristoph Hellwig struct queue_limits *limits = dm_get_queue_limits(md); 1082ac62d620SChristoph Hellwig 1083ac62d620SChristoph Hellwig /* device doesn't really support WRITE ZEROES, disable it */ 1084ac62d620SChristoph Hellwig limits->max_write_zeroes_sectors = 0; 1085ac62d620SChristoph Hellwig } 1086ac62d620SChristoph Hellwig 1087a666e5c0SMikulas Patocka static bool swap_bios_limit(struct dm_target *ti, struct bio *bio) 1088a666e5c0SMikulas Patocka { 1089a666e5c0SMikulas Patocka return unlikely((bio->bi_opf & REQ_SWAP) != 0) && unlikely(ti->limit_swap_bios); 1090a666e5c0SMikulas Patocka } 1091a666e5c0SMikulas Patocka 10924246a0b6SChristoph Hellwig static void clone_endio(struct bio *bio) 10931da177e4SLinus Torvalds { 10944e4cbee9SChristoph Hellwig blk_status_t error = bio->bi_status; 10956cbce280SMike Snitzer struct dm_target_io *tio = clone_to_tio(bio); 10966cbce280SMike Snitzer struct dm_target *ti = tio->ti; 10976cbce280SMike Snitzer dm_endio_fn endio = ti->type->end_io; 10986cbce280SMike Snitzer struct dm_io *io = tio->io; 10996cbce280SMike Snitzer struct mapped_device *md = io->md; 11001da177e4SLinus Torvalds 11019c37de29SMike Snitzer if (unlikely(error == BLK_STS_TARGET)) { 1102bcb44433SMike Snitzer if (bio_op(bio) == REQ_OP_DISCARD && 110370200574SChristoph Hellwig !bdev_max_discard_sectors(bio->bi_bdev)) 1104bcb44433SMike Snitzer disable_discard(md); 1105bcb44433SMike Snitzer else if (bio_op(bio) == REQ_OP_WRITE_ZEROES && 1106dddf3056SMike Snitzer !bdev_write_zeroes_sectors(bio->bi_bdev)) 1107ac62d620SChristoph Hellwig disable_write_zeroes(md); 1108ac62d620SChristoph Hellwig } 11097eee4ae2SMike Snitzer 1110442761fdSMike Snitzer if (static_branch_unlikely(&zoned_enabled) && 1111edd1dbc8SChristoph Hellwig unlikely(bdev_is_zoned(bio->bi_bdev))) 1112bb37d772SDamien Le Moal dm_zone_endio(io, bio); 1113415c79e1SJohannes Thumshirn 11141be56909SChristoph Hellwig if (endio) { 11156cbce280SMike Snitzer int r = endio(ti, bio, &error); 1116b30f1607SHeinz Mauelshagen 11171be56909SChristoph Hellwig switch (r) { 11181be56909SChristoph Hellwig case DM_ENDIO_REQUEUE: 1119442761fdSMike Snitzer if (static_branch_unlikely(&zoned_enabled)) { 1120bf14e2b2SDamien Le Moal /* 1121bf14e2b2SDamien Le Moal * Requeuing writes to a sequential zone of a zoned 1122bf14e2b2SDamien Le Moal * target will break the sequential write pattern: 1123bf14e2b2SDamien Le Moal * fail such IO. 1124bf14e2b2SDamien Le Moal */ 1125bf14e2b2SDamien Le Moal if (WARN_ON_ONCE(dm_is_zone_write(md, bio))) 1126bf14e2b2SDamien Le Moal error = BLK_STS_IOERR; 1127bf14e2b2SDamien Le Moal else 11284e4cbee9SChristoph Hellwig error = BLK_STS_DM_REQUEUE; 1129442761fdSMike Snitzer } else 1130442761fdSMike Snitzer error = BLK_STS_DM_REQUEUE; 1131df561f66SGustavo A. R. Silva fallthrough; 11321be56909SChristoph Hellwig case DM_ENDIO_DONE: 11331be56909SChristoph Hellwig break; 11341be56909SChristoph Hellwig case DM_ENDIO_INCOMPLETE: 11351be56909SChristoph Hellwig /* The target will handle the io */ 11361be56909SChristoph Hellwig return; 11371be56909SChristoph Hellwig default: 113843e6c111SMikulas Patocka DMCRIT("unimplemented target endio return value: %d", r); 11391be56909SChristoph Hellwig BUG(); 11401be56909SChristoph Hellwig } 11411be56909SChristoph Hellwig } 11421be56909SChristoph Hellwig 1143442761fdSMike Snitzer if (static_branch_unlikely(&swap_bios_enabled) && 1144442761fdSMike Snitzer unlikely(swap_bios_limit(ti, bio))) 1145a666e5c0SMikulas Patocka up(&md->swap_bios_semaphore); 1146a666e5c0SMikulas Patocka 11471d1068ceSChristoph Hellwig free_tio(bio); 1148e2118b3cSDamien Le Moal dm_io_dec_pending(io, error); 11491da177e4SLinus Torvalds } 11501da177e4SLinus Torvalds 115178d8e58aSMike Snitzer /* 115256a67df7SMike Snitzer * Return maximum size of I/O possible at the supplied sector up to the current 115356a67df7SMike Snitzer * target boundary. 115456a67df7SMike Snitzer */ 11553720281dSMike Snitzer static inline sector_t max_io_len_target_boundary(struct dm_target *ti, 11563720281dSMike Snitzer sector_t target_offset) 11571da177e4SLinus Torvalds { 115856a67df7SMike Snitzer return ti->len - target_offset; 115956a67df7SMike Snitzer } 116056a67df7SMike Snitzer 116106961c48SMike Snitzer static sector_t __max_io_len(struct dm_target *ti, sector_t sector, 1162be04c14aSMike Snitzer unsigned int max_granularity, 1163be04c14aSMike Snitzer unsigned int max_sectors) 116456a67df7SMike Snitzer { 11653720281dSMike Snitzer sector_t target_offset = dm_target_offset(ti, sector); 11663720281dSMike Snitzer sector_t len = max_io_len_target_boundary(ti, target_offset); 11671da177e4SLinus Torvalds 11681da177e4SLinus Torvalds /* 11693ee16db3SMike Snitzer * Does the target need to split IO even further? 11703ee16db3SMike Snitzer * - varied (per target) IO splitting is a tenet of DM; this 11713ee16db3SMike Snitzer * explains why stacked chunk_sectors based splitting via 11725a97806fSChristoph Hellwig * bio_split_to_limits() isn't possible here. 11731da177e4SLinus Torvalds */ 117406961c48SMike Snitzer if (!max_granularity) 11751da177e4SLinus Torvalds return len; 1176c3949322SChristoph Hellwig return min_t(sector_t, len, 1177be04c14aSMike Snitzer min(max_sectors ? : queue_max_sectors(ti->table->md->queue), 117806961c48SMike Snitzer blk_chunk_sectors_left(target_offset, max_granularity))); 117906961c48SMike Snitzer } 118006961c48SMike Snitzer 118106961c48SMike Snitzer static inline sector_t max_io_len(struct dm_target *ti, sector_t sector) 118206961c48SMike Snitzer { 1183be04c14aSMike Snitzer return __max_io_len(ti, sector, ti->max_io_len, 0); 11841da177e4SLinus Torvalds } 11851da177e4SLinus Torvalds 1186542f9038SMike Snitzer int dm_set_target_max_io_len(struct dm_target *ti, sector_t len) 1187542f9038SMike Snitzer { 1188542f9038SMike Snitzer if (len > UINT_MAX) { 1189542f9038SMike Snitzer DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)", 1190542f9038SMike Snitzer (unsigned long long)len, UINT_MAX); 1191542f9038SMike Snitzer ti->error = "Maximum size of target IO is too large"; 1192542f9038SMike Snitzer return -EINVAL; 1193542f9038SMike Snitzer } 1194542f9038SMike Snitzer 119575ae1936SMikulas Patocka ti->max_io_len = (uint32_t) len; 1196542f9038SMike Snitzer 1197542f9038SMike Snitzer return 0; 1198542f9038SMike Snitzer } 1199542f9038SMike Snitzer EXPORT_SYMBOL_GPL(dm_set_target_max_io_len); 1200542f9038SMike Snitzer 1201f26c5719SDan Williams static struct dm_target *dm_dax_get_live_target(struct mapped_device *md, 1202f26c5719SDan Williams sector_t sector, int *srcu_idx) 12033d97c829SMike Snitzer __acquires(md->io_barrier) 1204545ed20eSToshi Kani { 1205545ed20eSToshi Kani struct dm_table *map; 1206545ed20eSToshi Kani struct dm_target *ti; 1207545ed20eSToshi Kani 1208f26c5719SDan Williams map = dm_get_live_table(md, srcu_idx); 1209545ed20eSToshi Kani if (!map) 1210f26c5719SDan Williams return NULL; 1211545ed20eSToshi Kani 1212545ed20eSToshi Kani ti = dm_table_find_target(map, sector); 1213123d87d5SMikulas Patocka if (!ti) 1214f26c5719SDan Williams return NULL; 1215f26c5719SDan Williams 1216f26c5719SDan Williams return ti; 1217f26c5719SDan Williams } 1218f26c5719SDan Williams 1219f26c5719SDan Williams static long dm_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, 1220e511c4a3SJane Chu long nr_pages, enum dax_access_mode mode, void **kaddr, 1221e511c4a3SJane Chu pfn_t *pfn) 1222f26c5719SDan Williams { 1223f26c5719SDan Williams struct mapped_device *md = dax_get_private(dax_dev); 1224f26c5719SDan Williams sector_t sector = pgoff * PAGE_SECTORS; 1225f26c5719SDan Williams struct dm_target *ti; 1226f26c5719SDan Williams long len, ret = -EIO; 1227f26c5719SDan Williams int srcu_idx; 1228f26c5719SDan Williams 1229f26c5719SDan Williams ti = dm_dax_get_live_target(md, sector, &srcu_idx); 1230f26c5719SDan Williams 1231f26c5719SDan Williams if (!ti) 1232545ed20eSToshi Kani goto out; 1233f26c5719SDan Williams if (!ti->type->direct_access) 1234f26c5719SDan Williams goto out; 12353720281dSMike Snitzer len = max_io_len(ti, sector) / PAGE_SECTORS; 1236f26c5719SDan Williams if (len < 1) 1237f26c5719SDan Williams goto out; 1238f26c5719SDan Williams nr_pages = min(len, nr_pages); 1239e511c4a3SJane Chu ret = ti->type->direct_access(ti, pgoff, nr_pages, mode, kaddr, pfn); 1240817bf402SDan Williams 1241545ed20eSToshi Kani out: 1242545ed20eSToshi Kani dm_put_live_table(md, srcu_idx); 1243f26c5719SDan Williams 1244f26c5719SDan Williams return ret; 1245545ed20eSToshi Kani } 1246545ed20eSToshi Kani 1247cdf6cdcdSVivek Goyal static int dm_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, 1248cdf6cdcdSVivek Goyal size_t nr_pages) 1249cdf6cdcdSVivek Goyal { 1250cdf6cdcdSVivek Goyal struct mapped_device *md = dax_get_private(dax_dev); 1251cdf6cdcdSVivek Goyal sector_t sector = pgoff * PAGE_SECTORS; 1252cdf6cdcdSVivek Goyal struct dm_target *ti; 1253cdf6cdcdSVivek Goyal int ret = -EIO; 1254cdf6cdcdSVivek Goyal int srcu_idx; 1255cdf6cdcdSVivek Goyal 1256cdf6cdcdSVivek Goyal ti = dm_dax_get_live_target(md, sector, &srcu_idx); 1257cdf6cdcdSVivek Goyal 1258cdf6cdcdSVivek Goyal if (!ti) 1259cdf6cdcdSVivek Goyal goto out; 1260cdf6cdcdSVivek Goyal if (WARN_ON(!ti->type->dax_zero_page_range)) { 1261cdf6cdcdSVivek Goyal /* 1262cdf6cdcdSVivek Goyal * ->zero_page_range() is mandatory dax operation. If we are 1263cdf6cdcdSVivek Goyal * here, something is wrong. 1264cdf6cdcdSVivek Goyal */ 1265cdf6cdcdSVivek Goyal goto out; 1266cdf6cdcdSVivek Goyal } 1267cdf6cdcdSVivek Goyal ret = ti->type->dax_zero_page_range(ti, pgoff, nr_pages); 1268cdf6cdcdSVivek Goyal out: 1269cdf6cdcdSVivek Goyal dm_put_live_table(md, srcu_idx); 1270cdf6cdcdSVivek Goyal 1271cdf6cdcdSVivek Goyal return ret; 1272cdf6cdcdSVivek Goyal } 1273cdf6cdcdSVivek Goyal 1274047218ecSJane Chu static size_t dm_dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff, 1275047218ecSJane Chu void *addr, size_t bytes, struct iov_iter *i) 1276047218ecSJane Chu { 1277047218ecSJane Chu struct mapped_device *md = dax_get_private(dax_dev); 1278047218ecSJane Chu sector_t sector = pgoff * PAGE_SECTORS; 1279047218ecSJane Chu struct dm_target *ti; 1280047218ecSJane Chu int srcu_idx; 1281047218ecSJane Chu long ret = 0; 1282047218ecSJane Chu 1283047218ecSJane Chu ti = dm_dax_get_live_target(md, sector, &srcu_idx); 1284047218ecSJane Chu if (!ti || !ti->type->dax_recovery_write) 1285047218ecSJane Chu goto out; 1286047218ecSJane Chu 1287047218ecSJane Chu ret = ti->type->dax_recovery_write(ti, pgoff, addr, bytes, i); 1288047218ecSJane Chu out: 1289047218ecSJane Chu dm_put_live_table(md, srcu_idx); 1290047218ecSJane Chu return ret; 1291047218ecSJane Chu } 1292047218ecSJane Chu 12931dd40c3eSMikulas Patocka /* 12941dd40c3eSMikulas Patocka * A target may call dm_accept_partial_bio only from the map routine. It is 12956842d264SDamien Le Moal * allowed for all bio types except REQ_PREFLUSH, REQ_OP_ZONE_* zone management 1296e6fc9f62SMike Snitzer * operations, REQ_OP_ZONE_APPEND (zone append writes) and any bio serviced by 1297e6fc9f62SMike Snitzer * __send_duplicate_bios(). 12981dd40c3eSMikulas Patocka * 12991dd40c3eSMikulas Patocka * dm_accept_partial_bio informs the dm that the target only wants to process 13001dd40c3eSMikulas Patocka * additional n_sectors sectors of the bio and the rest of the data should be 13011dd40c3eSMikulas Patocka * sent in a next bio. 13021dd40c3eSMikulas Patocka * 13031dd40c3eSMikulas Patocka * A diagram that explains the arithmetics: 13041dd40c3eSMikulas Patocka * +--------------------+---------------+-------+ 13051dd40c3eSMikulas Patocka * | 1 | 2 | 3 | 13061dd40c3eSMikulas Patocka * +--------------------+---------------+-------+ 13071dd40c3eSMikulas Patocka * 13081dd40c3eSMikulas Patocka * <-------------- *tio->len_ptr ---------------> 1309bdb34759SMike Snitzer * <----- bio_sectors -----> 13101dd40c3eSMikulas Patocka * <-- n_sectors --> 13111dd40c3eSMikulas Patocka * 13121dd40c3eSMikulas Patocka * Region 1 was already iterated over with bio_advance or similar function. 13131dd40c3eSMikulas Patocka * (it may be empty if the target doesn't use bio_advance) 13141dd40c3eSMikulas Patocka * Region 2 is the remaining bio size that the target wants to process. 13151dd40c3eSMikulas Patocka * (it may be empty if region 1 is non-empty, although there is no reason 13161dd40c3eSMikulas Patocka * to make it empty) 13171dd40c3eSMikulas Patocka * The target requires that region 3 is to be sent in the next bio. 13181dd40c3eSMikulas Patocka * 13191dd40c3eSMikulas Patocka * If the target wants to receive multiple copies of the bio (via num_*bios, etc), 13201dd40c3eSMikulas Patocka * the partially processed part (the sum of regions 1+2) must be the same for all 13211dd40c3eSMikulas Patocka * copies of the bio. 13221dd40c3eSMikulas Patocka */ 132386a3238cSHeinz Mauelshagen void dm_accept_partial_bio(struct bio *bio, unsigned int n_sectors) 13241dd40c3eSMikulas Patocka { 13256c23f0bdSChristoph Hellwig struct dm_target_io *tio = clone_to_tio(bio); 13268b211aacSMing Lei struct dm_io *io = tio->io; 132786a3238cSHeinz Mauelshagen unsigned int bio_sectors = bio_sectors(bio); 13286842d264SDamien Le Moal 1329655f3aadSMike Snitzer BUG_ON(dm_tio_flagged(tio, DM_TIO_IS_DUPLICATE_BIO)); 13306842d264SDamien Le Moal BUG_ON(op_is_zone_mgmt(bio_op(bio))); 13316842d264SDamien Le Moal BUG_ON(bio_op(bio) == REQ_OP_ZONE_APPEND); 1332bdb34759SMike Snitzer BUG_ON(bio_sectors > *tio->len_ptr); 1333bdb34759SMike Snitzer BUG_ON(n_sectors > bio_sectors); 13346842d264SDamien Le Moal 1335bdb34759SMike Snitzer *tio->len_ptr -= bio_sectors - n_sectors; 13361dd40c3eSMikulas Patocka bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT; 13377dd76d1fSMing Lei 13387dd76d1fSMing Lei /* 13397dd76d1fSMing Lei * __split_and_process_bio() may have already saved mapped part 13407dd76d1fSMing Lei * for accounting but it is being reduced so update accordingly. 13417dd76d1fSMing Lei */ 13428b211aacSMing Lei dm_io_set_flag(io, DM_IO_WAS_SPLIT); 13438b211aacSMing Lei io->sectors = n_sectors; 13448b211aacSMing Lei io->sector_offset = bio_sectors(io->orig_bio); 13451dd40c3eSMikulas Patocka } 13461dd40c3eSMikulas Patocka EXPORT_SYMBOL_GPL(dm_accept_partial_bio); 13471dd40c3eSMikulas Patocka 13480fbb4d93SMike Snitzer /* 13490fbb4d93SMike Snitzer * @clone: clone bio that DM core passed to target's .map function 13500fbb4d93SMike Snitzer * @tgt_clone: clone of @clone bio that target needs submitted 13510fbb4d93SMike Snitzer * 13520fbb4d93SMike Snitzer * Targets should use this interface to submit bios they take 13530fbb4d93SMike Snitzer * ownership of when returning DM_MAPIO_SUBMITTED. 13540fbb4d93SMike Snitzer * 13550fbb4d93SMike Snitzer * Target should also enable ti->accounts_remapped_io 13560fbb4d93SMike Snitzer */ 1357b7f8dff0SMike Snitzer void dm_submit_bio_remap(struct bio *clone, struct bio *tgt_clone) 13580fbb4d93SMike Snitzer { 13590fbb4d93SMike Snitzer struct dm_target_io *tio = clone_to_tio(clone); 13600fbb4d93SMike Snitzer struct dm_io *io = tio->io; 13610fbb4d93SMike Snitzer 13620fbb4d93SMike Snitzer /* establish bio that will get submitted */ 13630fbb4d93SMike Snitzer if (!tgt_clone) 13640fbb4d93SMike Snitzer tgt_clone = clone; 13650fbb4d93SMike Snitzer 13660fbb4d93SMike Snitzer /* 13670fbb4d93SMike Snitzer * Account io->origin_bio to DM dev on behalf of target 13680fbb4d93SMike Snitzer * that took ownership of IO with DM_MAPIO_SUBMITTED. 13690fbb4d93SMike Snitzer */ 13700fbb4d93SMike Snitzer dm_start_io_acct(io, clone); 13710fbb4d93SMike Snitzer 13729d20653fSMike Snitzer trace_block_bio_remap(tgt_clone, disk_devt(io->md->disk), 13730fbb4d93SMike Snitzer tio->old_sector); 13749d20653fSMike Snitzer submit_bio_noacct(tgt_clone); 13750fbb4d93SMike Snitzer } 13760fbb4d93SMike Snitzer EXPORT_SYMBOL_GPL(dm_submit_bio_remap); 13770fbb4d93SMike Snitzer 1378a666e5c0SMikulas Patocka static noinline void __set_swap_bios_limit(struct mapped_device *md, int latch) 1379a666e5c0SMikulas Patocka { 1380a666e5c0SMikulas Patocka mutex_lock(&md->swap_bios_lock); 1381a666e5c0SMikulas Patocka while (latch < md->swap_bios) { 1382a666e5c0SMikulas Patocka cond_resched(); 1383a666e5c0SMikulas Patocka down(&md->swap_bios_semaphore); 1384a666e5c0SMikulas Patocka md->swap_bios--; 1385a666e5c0SMikulas Patocka } 1386a666e5c0SMikulas Patocka while (latch > md->swap_bios) { 1387a666e5c0SMikulas Patocka cond_resched(); 1388a666e5c0SMikulas Patocka up(&md->swap_bios_semaphore); 1389a666e5c0SMikulas Patocka md->swap_bios++; 1390a666e5c0SMikulas Patocka } 1391a666e5c0SMikulas Patocka mutex_unlock(&md->swap_bios_lock); 1392a666e5c0SMikulas Patocka } 1393a666e5c0SMikulas Patocka 13941561b396SChristoph Hellwig static void __map_bio(struct bio *clone) 13951da177e4SLinus Torvalds { 13961561b396SChristoph Hellwig struct dm_target_io *tio = clone_to_tio(clone); 1397bd2a49b8SAlasdair G Kergon struct dm_target *ti = tio->ti; 13986cbce280SMike Snitzer struct dm_io *io = tio->io; 13996cbce280SMike Snitzer struct mapped_device *md = io->md; 14006cbce280SMike Snitzer int r; 14011da177e4SLinus Torvalds 14021da177e4SLinus Torvalds clone->bi_end_io = clone_endio; 14031da177e4SLinus Torvalds 14041da177e4SLinus Torvalds /* 14050fbb4d93SMike Snitzer * Map the clone. 14061da177e4SLinus Torvalds */ 1407743598f0SMike Snitzer tio->old_sector = clone->bi_iter.bi_sector; 1408d67a5f4bSMikulas Patocka 1409442761fdSMike Snitzer if (static_branch_unlikely(&swap_bios_enabled) && 1410442761fdSMike Snitzer unlikely(swap_bios_limit(ti, clone))) { 1411a666e5c0SMikulas Patocka int latch = get_swap_bios(); 1412b30f1607SHeinz Mauelshagen 1413a666e5c0SMikulas Patocka if (unlikely(latch != md->swap_bios)) 1414a666e5c0SMikulas Patocka __set_swap_bios_limit(md, latch); 1415a666e5c0SMikulas Patocka down(&md->swap_bios_semaphore); 1416a666e5c0SMikulas Patocka } 1417a666e5c0SMikulas Patocka 1418442761fdSMike Snitzer if (static_branch_unlikely(&zoned_enabled)) { 1419bb37d772SDamien Le Moal /* 1420442761fdSMike Snitzer * Check if the IO needs a special mapping due to zone append 1421442761fdSMike Snitzer * emulation on zoned target. In this case, dm_zone_map_bio() 1422442761fdSMike Snitzer * calls the target map operation. 1423bb37d772SDamien Le Moal */ 14246cbce280SMike Snitzer if (unlikely(dm_emulate_zone_append(md))) 1425bb37d772SDamien Le Moal r = dm_zone_map_bio(tio); 1426bb37d772SDamien Le Moal else 14277de3ee57SMikulas Patocka r = ti->type->map(ti, clone); 1428442761fdSMike Snitzer } else 1429442761fdSMike Snitzer r = ti->type->map(ti, clone); 1430bb37d772SDamien Le Moal 1431846785e6SChristoph Hellwig switch (r) { 1432846785e6SChristoph Hellwig case DM_MAPIO_SUBMITTED: 14330fbb4d93SMike Snitzer /* target has assumed ownership of this io */ 14340fbb4d93SMike Snitzer if (!ti->accounts_remapped_io) 14359d20653fSMike Snitzer dm_start_io_acct(io, clone); 1436846785e6SChristoph Hellwig break; 1437846785e6SChristoph Hellwig case DM_MAPIO_REMAPPED: 14389d20653fSMike Snitzer dm_submit_bio_remap(clone, NULL); 1439846785e6SChristoph Hellwig break; 1440846785e6SChristoph Hellwig case DM_MAPIO_KILL: 1441846785e6SChristoph Hellwig case DM_MAPIO_REQUEUE: 1442442761fdSMike Snitzer if (static_branch_unlikely(&swap_bios_enabled) && 1443442761fdSMike Snitzer unlikely(swap_bios_limit(ti, clone))) 14446cbce280SMike Snitzer up(&md->swap_bios_semaphore); 14451d1068ceSChristoph Hellwig free_tio(clone); 144690a2326eSMike Snitzer if (r == DM_MAPIO_KILL) 144790a2326eSMike Snitzer dm_io_dec_pending(io, BLK_STS_IOERR); 144890a2326eSMike Snitzer else 1449e2118b3cSDamien Le Moal dm_io_dec_pending(io, BLK_STS_DM_REQUEUE); 1450846785e6SChristoph Hellwig break; 1451846785e6SChristoph Hellwig default: 145243e6c111SMikulas Patocka DMCRIT("unimplemented target map return value: %d", r); 145345cbcd79SKiyoshi Ueda BUG(); 14541da177e4SLinus Torvalds } 14551da177e4SLinus Torvalds } 14561da177e4SLinus Torvalds 145786a3238cSHeinz Mauelshagen static void setup_split_accounting(struct clone_info *ci, unsigned int len) 14587dd76d1fSMing Lei { 14597dd76d1fSMing Lei struct dm_io *io = ci->io; 14607dd76d1fSMing Lei 14617dd76d1fSMing Lei if (ci->sector_count > len) { 14627dd76d1fSMing Lei /* 14637dd76d1fSMing Lei * Split needed, save the mapped part for accounting. 14647dd76d1fSMing Lei * NOTE: dm_accept_partial_bio() will update accordingly. 14657dd76d1fSMing Lei */ 14667dd76d1fSMing Lei dm_io_set_flag(io, DM_IO_WAS_SPLIT); 14677dd76d1fSMing Lei io->sectors = len; 14688b211aacSMing Lei io->sector_offset = bio_sectors(ci->bio); 14697dd76d1fSMing Lei } 14707dd76d1fSMing Lei } 14717dd76d1fSMing Lei 1472318716ddSMike Snitzer static void alloc_multiple_bios(struct bio_list *blist, struct clone_info *ci, 1473f7b58a69SMike Snitzer struct dm_target *ti, unsigned int num_bios, 1474f7b58a69SMike Snitzer unsigned *len) 1475f9ab94ceSMikulas Patocka { 14761d1068ceSChristoph Hellwig struct bio *bio; 1477318716ddSMike Snitzer int try; 1478dba14160SMikulas Patocka 1479318716ddSMike Snitzer for (try = 0; try < 2; try++) { 1480318716ddSMike Snitzer int bio_nr; 1481318716ddSMike Snitzer 1482318716ddSMike Snitzer if (try) 1483bc02cdbeSMike Snitzer mutex_lock(&ci->io->md->table_devices_lock); 1484318716ddSMike Snitzer for (bio_nr = 0; bio_nr < num_bios; bio_nr++) { 1485f7b58a69SMike Snitzer bio = alloc_tio(ci, ti, bio_nr, len, 1486dc8e2021SChristoph Hellwig try ? GFP_NOIO : GFP_NOWAIT); 14871d1068ceSChristoph Hellwig if (!bio) 1488318716ddSMike Snitzer break; 1489318716ddSMike Snitzer 14901d1068ceSChristoph Hellwig bio_list_add(blist, bio); 1491318716ddSMike Snitzer } 1492318716ddSMike Snitzer if (try) 1493bc02cdbeSMike Snitzer mutex_unlock(&ci->io->md->table_devices_lock); 1494318716ddSMike Snitzer if (bio_nr == num_bios) 1495318716ddSMike Snitzer return; 1496318716ddSMike Snitzer 14976c23f0bdSChristoph Hellwig while ((bio = bio_list_pop(blist))) 14981d1068ceSChristoph Hellwig free_tio(bio); 1499318716ddSMike Snitzer } 1500318716ddSMike Snitzer } 1501f9ab94ceSMikulas Patocka 15020f14d60aSMing Lei static int __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti, 150386a3238cSHeinz Mauelshagen unsigned int num_bios, unsigned int *len) 150406a426ceSMike Snitzer { 1505318716ddSMike Snitzer struct bio_list blist = BIO_EMPTY_LIST; 15068eabf5d0SChristoph Hellwig struct bio *clone; 1507564b5c54SMike Snitzer unsigned int ret = 0; 150806a426ceSMike Snitzer 1509891fced6SChristoph Hellwig switch (num_bios) { 1510891fced6SChristoph Hellwig case 0: 1511891fced6SChristoph Hellwig break; 1512891fced6SChristoph Hellwig case 1: 15137dd76d1fSMing Lei if (len) 15147dd76d1fSMing Lei setup_split_accounting(ci, *len); 1515891fced6SChristoph Hellwig clone = alloc_tio(ci, ti, 0, len, GFP_NOIO); 1516891fced6SChristoph Hellwig __map_bio(clone); 15170f14d60aSMing Lei ret = 1; 1518891fced6SChristoph Hellwig break; 1519891fced6SChristoph Hellwig default: 1520666eed46SMike Snitzer if (len) 1521666eed46SMike Snitzer setup_split_accounting(ci, *len); 15227dd06a25SMike Snitzer /* dm_accept_partial_bio() is not supported with shared tio->len_ptr */ 1523f7b58a69SMike Snitzer alloc_multiple_bios(&blist, ci, ti, num_bios, len); 15248eabf5d0SChristoph Hellwig while ((clone = bio_list_pop(&blist))) { 1525655f3aadSMike Snitzer dm_tio_set_flag(clone_to_tio(clone), DM_TIO_IS_DUPLICATE_BIO); 15261561b396SChristoph Hellwig __map_bio(clone); 15270f14d60aSMing Lei ret += 1; 1528f9ab94ceSMikulas Patocka } 1529891fced6SChristoph Hellwig break; 1530318716ddSMike Snitzer } 15310f14d60aSMing Lei 15320f14d60aSMing Lei return ret; 153306a426ceSMike Snitzer } 153406a426ceSMike Snitzer 1535332f2b1eSMike Snitzer static void __send_empty_flush(struct clone_info *ci) 1536f9ab94ceSMikulas Patocka { 1537564b5c54SMike Snitzer struct dm_table *t = ci->map; 1538828678b8SMike Snitzer struct bio flush_bio; 1539828678b8SMike Snitzer 1540828678b8SMike Snitzer /* 1541828678b8SMike Snitzer * Use an on-stack bio for this, it's safe since we don't 1542828678b8SMike Snitzer * need to reference it after submit. It's just used as 1543828678b8SMike Snitzer * the basis for the clone(s). 1544828678b8SMike Snitzer */ 154549add496SChristoph Hellwig bio_init(&flush_bio, ci->io->md->disk->part0, NULL, 0, 154649add496SChristoph Hellwig REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC); 154747d95102SChristoph Hellwig 1548828678b8SMike Snitzer ci->bio = &flush_bio; 1549828678b8SMike Snitzer ci->sector_count = 0; 155092b914e2SShin'ichiro Kawasaki ci->io->tio.clone.bi_iter.bi_size = 0; 1551f9ab94ceSMikulas Patocka 1552564b5c54SMike Snitzer for (unsigned int i = 0; i < t->num_targets; i++) { 1553564b5c54SMike Snitzer unsigned int bios; 1554564b5c54SMike Snitzer struct dm_target *ti = dm_table_get_target(t, i); 15550f14d60aSMing Lei 15560f14d60aSMing Lei atomic_add(ti->num_flush_bios, &ci->io->io_count); 15570f14d60aSMing Lei bios = __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL); 15580f14d60aSMing Lei atomic_sub(ti->num_flush_bios - bios, &ci->io->io_count); 15590f14d60aSMing Lei } 15600f14d60aSMing Lei 15610f14d60aSMing Lei /* 15620f14d60aSMing Lei * alloc_io() takes one extra reference for submission, so the 15630f14d60aSMing Lei * reference won't reach 0 without the following subtraction 15640f14d60aSMing Lei */ 15650f14d60aSMing Lei atomic_sub(1, &ci->io->io_count); 1566828678b8SMike Snitzer 1567828678b8SMike Snitzer bio_uninit(ci->bio); 1568f9ab94ceSMikulas Patocka } 1569f9ab94ceSMikulas Patocka 1570e6fc9f62SMike Snitzer static void __send_changing_extent_only(struct clone_info *ci, struct dm_target *ti, 157106961c48SMike Snitzer unsigned int num_bios, 1572be04c14aSMike Snitzer unsigned int max_granularity, 1573be04c14aSMike Snitzer unsigned int max_sectors) 15745ae89a87SMike Snitzer { 157586a3238cSHeinz Mauelshagen unsigned int len, bios; 15765ae89a87SMike Snitzer 15773720281dSMike Snitzer len = min_t(sector_t, ci->sector_count, 1578be04c14aSMike Snitzer __max_io_len(ti, ci->sector, max_granularity, max_sectors)); 157951b86f9aSMichael Lass 15800f14d60aSMing Lei atomic_add(num_bios, &ci->io->io_count); 15810f14d60aSMing Lei bios = __send_duplicate_bios(ci, ti, num_bios, &len); 15820f14d60aSMing Lei /* 15830f14d60aSMing Lei * alloc_io() takes one extra reference for submission, so the 15840f14d60aSMing Lei * reference won't reach 0 without the following (+1) subtraction 15850f14d60aSMing Lei */ 15860f14d60aSMing Lei atomic_sub(num_bios - bios + 1, &ci->io->io_count); 15877dd06a25SMike Snitzer 1588a79245b3SMike Snitzer ci->sector += len; 15893d7f4562SMike Snitzer ci->sector_count -= len; 15905ae89a87SMike Snitzer } 15915ae89a87SMike Snitzer 1592568c73a3SMike Snitzer static bool is_abnormal_io(struct bio *bio) 1593568c73a3SMike Snitzer { 1594a3282b43SBart Van Assche enum req_op op = bio_op(bio); 1595568c73a3SMike Snitzer 15964edadf6dSMike Snitzer if (op != REQ_OP_READ && op != REQ_OP_WRITE && op != REQ_OP_FLUSH) { 15974edadf6dSMike Snitzer switch (op) { 1598568c73a3SMike Snitzer case REQ_OP_DISCARD: 1599568c73a3SMike Snitzer case REQ_OP_SECURE_ERASE: 1600568c73a3SMike Snitzer case REQ_OP_WRITE_ZEROES: 16014edadf6dSMike Snitzer return true; 16024edadf6dSMike Snitzer default: 1603568c73a3SMike Snitzer break; 1604568c73a3SMike Snitzer } 1605568c73a3SMike Snitzer } 1606568c73a3SMike Snitzer 16074edadf6dSMike Snitzer return false; 16084edadf6dSMike Snitzer } 16094edadf6dSMike Snitzer 16104edadf6dSMike Snitzer static blk_status_t __process_abnormal_io(struct clone_info *ci, 16114edadf6dSMike Snitzer struct dm_target *ti) 16120519c71eSMike Snitzer { 161386a3238cSHeinz Mauelshagen unsigned int num_bios = 0; 161406961c48SMike Snitzer unsigned int max_granularity = 0; 1615be04c14aSMike Snitzer unsigned int max_sectors = 0; 161613f6facfSMike Snitzer struct queue_limits *limits = dm_get_queue_limits(ti->table->md); 16170519c71eSMike Snitzer 1618e6fc9f62SMike Snitzer switch (bio_op(ci->bio)) { 16199679b5a7SMike Snitzer case REQ_OP_DISCARD: 16209679b5a7SMike Snitzer num_bios = ti->num_discard_bios; 1621be04c14aSMike Snitzer max_sectors = limits->max_discard_sectors; 162213f6facfSMike Snitzer if (ti->max_discard_granularity) 1623be04c14aSMike Snitzer max_granularity = max_sectors; 16249679b5a7SMike Snitzer break; 16259679b5a7SMike Snitzer case REQ_OP_SECURE_ERASE: 16269679b5a7SMike Snitzer num_bios = ti->num_secure_erase_bios; 1627be04c14aSMike Snitzer max_sectors = limits->max_secure_erase_sectors; 162813f6facfSMike Snitzer if (ti->max_secure_erase_granularity) 1629be04c14aSMike Snitzer max_granularity = max_sectors; 16309679b5a7SMike Snitzer break; 16319679b5a7SMike Snitzer case REQ_OP_WRITE_ZEROES: 16329679b5a7SMike Snitzer num_bios = ti->num_write_zeroes_bios; 1633be04c14aSMike Snitzer max_sectors = limits->max_write_zeroes_sectors; 163413f6facfSMike Snitzer if (ti->max_write_zeroes_granularity) 1635be04c14aSMike Snitzer max_granularity = max_sectors; 16369679b5a7SMike Snitzer break; 16372d9b02beSBart Van Assche default: 16382d9b02beSBart Van Assche break; 16399679b5a7SMike Snitzer } 16400519c71eSMike Snitzer 1641e6fc9f62SMike Snitzer /* 1642e6fc9f62SMike Snitzer * Even though the device advertised support for this type of 1643e6fc9f62SMike Snitzer * request, that does not mean every target supports it, and 1644e6fc9f62SMike Snitzer * reconfiguration might also have changed that since the 1645e6fc9f62SMike Snitzer * check was performed. 1646e6fc9f62SMike Snitzer */ 164784b98f4cSMike Snitzer if (unlikely(!num_bios)) 16484edadf6dSMike Snitzer return BLK_STS_NOTSUPP; 16494edadf6dSMike Snitzer 1650be04c14aSMike Snitzer __send_changing_extent_only(ci, ti, num_bios, 1651be04c14aSMike Snitzer max_granularity, max_sectors); 16524edadf6dSMike Snitzer return BLK_STS_OK; 16530519c71eSMike Snitzer } 16540519c71eSMike Snitzer 1655e4c93811SAlasdair G Kergon /* 1656ec211631SMing Lei * Reuse ->bi_private as dm_io list head for storing all dm_io instances 1657b99fdcdcSMing Lei * associated with this bio, and this bio's bi_private needs to be 1658b99fdcdcSMing Lei * stored in dm_io->data before the reuse. 1659b99fdcdcSMing Lei * 1660b99fdcdcSMing Lei * bio->bi_private is owned by fs or upper layer, so block layer won't 1661b99fdcdcSMing Lei * touch it after splitting. Meantime it won't be changed by anyone after 1662b99fdcdcSMing Lei * bio is submitted. So this reuse is safe. 1663b99fdcdcSMing Lei */ 1664ec211631SMing Lei static inline struct dm_io **dm_poll_list_head(struct bio *bio) 1665b99fdcdcSMing Lei { 1666ec211631SMing Lei return (struct dm_io **)&bio->bi_private; 1667b99fdcdcSMing Lei } 1668b99fdcdcSMing Lei 1669b99fdcdcSMing Lei static void dm_queue_poll_io(struct bio *bio, struct dm_io *io) 1670b99fdcdcSMing Lei { 1671ec211631SMing Lei struct dm_io **head = dm_poll_list_head(bio); 1672b99fdcdcSMing Lei 1673b99fdcdcSMing Lei if (!(bio->bi_opf & REQ_DM_POLL_LIST)) { 1674b99fdcdcSMing Lei bio->bi_opf |= REQ_DM_POLL_LIST; 1675b99fdcdcSMing Lei /* 1676b99fdcdcSMing Lei * Save .bi_private into dm_io, so that we can reuse 1677ec211631SMing Lei * .bi_private as dm_io list head for storing dm_io list 1678b99fdcdcSMing Lei */ 1679b99fdcdcSMing Lei io->data = bio->bi_private; 1680b99fdcdcSMing Lei 1681b99fdcdcSMing Lei /* tell block layer to poll for completion */ 1682b99fdcdcSMing Lei bio->bi_cookie = ~BLK_QC_T_NONE; 1683ec211631SMing Lei 1684ec211631SMing Lei io->next = NULL; 1685b99fdcdcSMing Lei } else { 1686b99fdcdcSMing Lei /* 1687b99fdcdcSMing Lei * bio recursed due to split, reuse original poll list, 1688b99fdcdcSMing Lei * and save bio->bi_private too. 1689b99fdcdcSMing Lei */ 1690ec211631SMing Lei io->data = (*head)->data; 1691ec211631SMing Lei io->next = *head; 1692b99fdcdcSMing Lei } 1693b99fdcdcSMing Lei 1694ec211631SMing Lei *head = io; 1695b99fdcdcSMing Lei } 1696b99fdcdcSMing Lei 1697b99fdcdcSMing Lei /* 1698e4c93811SAlasdair G Kergon * Select the correct strategy for processing a non-flush bio. 1699e4c93811SAlasdair G Kergon */ 170084b98f4cSMike Snitzer static blk_status_t __split_and_process_bio(struct clone_info *ci) 1701e4c93811SAlasdair G Kergon { 170266bdaa43SMike Snitzer struct bio *clone; 1703e4c93811SAlasdair G Kergon struct dm_target *ti; 170486a3238cSHeinz Mauelshagen unsigned int len; 1705e4c93811SAlasdair G Kergon 1706e4c93811SAlasdair G Kergon ti = dm_table_find_target(ci->map, ci->sector); 17074edadf6dSMike Snitzer if (unlikely(!ti)) 17084edadf6dSMike Snitzer return BLK_STS_IOERR; 17091ee88de3SMikulas Patocka 17101ee88de3SMikulas Patocka if (unlikely((ci->bio->bi_opf & REQ_NOWAIT) != 0) && 17111ee88de3SMikulas Patocka unlikely(!dm_target_supports_nowait(ti->type))) 17121ee88de3SMikulas Patocka return BLK_STS_NOTSUPP; 17131ee88de3SMikulas Patocka 17141ee88de3SMikulas Patocka if (unlikely(ci->is_abnormal_io)) 17154edadf6dSMike Snitzer return __process_abnormal_io(ci, ti); 17163d7f4562SMike Snitzer 1717b99fdcdcSMing Lei /* 1718b99fdcdcSMing Lei * Only support bio polling for normal IO, and the target io is 1719b99fdcdcSMing Lei * exactly inside the dm_io instance (verified in dm_poll_dm_io) 1720b99fdcdcSMing Lei */ 1721a3282b43SBart Van Assche ci->submit_as_polled = !!(ci->bio->bi_opf & REQ_POLLED); 1722e4c93811SAlasdair G Kergon 1723e4c93811SAlasdair G Kergon len = min_t(sector_t, max_io_len(ti, ci->sector), ci->sector_count); 17247dd76d1fSMing Lei setup_split_accounting(ci, len); 172566bdaa43SMike Snitzer clone = alloc_tio(ci, ti, 0, &len, GFP_NOIO); 172666bdaa43SMike Snitzer __map_bio(clone); 1727e4c93811SAlasdair G Kergon 1728e4c93811SAlasdair G Kergon ci->sector += len; 1729e4c93811SAlasdair G Kergon ci->sector_count -= len; 1730e4c93811SAlasdair G Kergon 173184b98f4cSMike Snitzer return BLK_STS_OK; 1732e4c93811SAlasdair G Kergon } 1733e4c93811SAlasdair G Kergon 1734978e51baSMike Snitzer static void init_clone_info(struct clone_info *ci, struct mapped_device *md, 17354edadf6dSMike Snitzer struct dm_table *map, struct bio *bio, bool is_abnormal) 1736978e51baSMike Snitzer { 1737978e51baSMike Snitzer ci->map = map; 1738978e51baSMike Snitzer ci->io = alloc_io(md, bio); 1739d41e077aSMike Snitzer ci->bio = bio; 17404edadf6dSMike Snitzer ci->is_abnormal_io = is_abnormal; 1741b99fdcdcSMing Lei ci->submit_as_polled = false; 1742978e51baSMike Snitzer ci->sector = bio->bi_iter.bi_sector; 1743d41e077aSMike Snitzer ci->sector_count = bio_sectors(bio); 1744d41e077aSMike Snitzer 1745d41e077aSMike Snitzer /* Shouldn't happen but sector_count was being set to 0 so... */ 1746442761fdSMike Snitzer if (static_branch_unlikely(&zoned_enabled) && 1747442761fdSMike Snitzer WARN_ON_ONCE(op_is_zone_mgmt(bio_op(bio)) && ci->sector_count)) 1748d41e077aSMike Snitzer ci->sector_count = 0; 1749978e51baSMike Snitzer } 1750978e51baSMike Snitzer 1751e4c93811SAlasdair G Kergon /* 175214fe594dSAlasdair G Kergon * Entry point to split a bio into clones and submit them to the targets. 17531da177e4SLinus Torvalds */ 175496c9865cSMike Snitzer static void dm_split_and_process_bio(struct mapped_device *md, 175583d5e5b0SMikulas Patocka struct dm_table *map, struct bio *bio) 17561da177e4SLinus Torvalds { 17571da177e4SLinus Torvalds struct clone_info ci; 17584857abf6SMike Snitzer struct dm_io *io; 175984b98f4cSMike Snitzer blk_status_t error = BLK_STS_OK; 17604edadf6dSMike Snitzer bool is_abnormal; 17611da177e4SLinus Torvalds 17624edadf6dSMike Snitzer is_abnormal = is_abnormal_io(bio); 17634edadf6dSMike Snitzer if (unlikely(is_abnormal)) { 17644edadf6dSMike Snitzer /* 17655a97806fSChristoph Hellwig * Use bio_split_to_limits() for abnormal IO (e.g. discard, etc) 17664edadf6dSMike Snitzer * otherwise associated queue_limits won't be imposed. 17674edadf6dSMike Snitzer */ 17685a97806fSChristoph Hellwig bio = bio_split_to_limits(bio); 1769613b1488SJens Axboe if (!bio) 1770613b1488SJens Axboe return; 17714edadf6dSMike Snitzer } 17724edadf6dSMike Snitzer 17734edadf6dSMike Snitzer init_clone_info(&ci, md, map, bio, is_abnormal); 17744857abf6SMike Snitzer io = ci.io; 1775bd2a49b8SAlasdair G Kergon 17761eff9d32SJens Axboe if (bio->bi_opf & REQ_PREFLUSH) { 1777332f2b1eSMike Snitzer __send_empty_flush(&ci); 1778e2736347SMike Snitzer /* dm_io_complete submits any data associated with flush */ 1779d41e077aSMike Snitzer goto out; 1780d41e077aSMike Snitzer } 1781d41e077aSMike Snitzer 178296c9865cSMike Snitzer error = __split_and_process_bio(&ci); 1783d41e077aSMike Snitzer if (error || !ci.sector_count) 1784d41e077aSMike Snitzer goto out; 178518a25da8SNeilBrown /* 1786d41e077aSMike Snitzer * Remainder must be passed to submit_bio_noacct() so it gets handled 1787d41e077aSMike Snitzer * *after* bios already submitted have been completely processed. 178818a25da8SNeilBrown */ 17898b211aacSMing Lei bio_trim(bio, io->sectors, ci.sector_count); 17908b211aacSMing Lei trace_block_split(bio, bio->bi_iter.bi_sector); 17918b211aacSMing Lei bio_inc_remaining(bio); 17923e08773cSChristoph Hellwig submit_bio_noacct(bio); 1793d41e077aSMike Snitzer out: 1794b99fdcdcSMing Lei /* 1795b99fdcdcSMing Lei * Drop the extra reference count for non-POLLED bio, and hold one 1796b99fdcdcSMing Lei * reference for POLLED bio, which will be released in dm_poll_bio 1797b99fdcdcSMing Lei * 1798ec211631SMing Lei * Add every dm_io instance into the dm_io list head which is stored 1799ec211631SMing Lei * in bio->bi_private, so that dm_poll_bio can poll them all. 1800b99fdcdcSMing Lei */ 18010f14d60aSMing Lei if (error || !ci.submit_as_polled) { 18020f14d60aSMing Lei /* 18030f14d60aSMing Lei * In case of submission failure, the extra reference for 18040f14d60aSMing Lei * submitting io isn't consumed yet 18050f14d60aSMing Lei */ 18060f14d60aSMing Lei if (error) 18070f14d60aSMing Lei atomic_dec(&io->io_count); 18080f14d60aSMing Lei dm_io_dec_pending(io, error); 18090f14d60aSMing Lei } else 18104857abf6SMike Snitzer dm_queue_poll_io(bio, io); 18111da177e4SLinus Torvalds } 18121da177e4SLinus Torvalds 18133e08773cSChristoph Hellwig static void dm_submit_bio(struct bio *bio) 18141da177e4SLinus Torvalds { 1815309dca30SChristoph Hellwig struct mapped_device *md = bio->bi_bdev->bd_disk->private_data; 181683d5e5b0SMikulas Patocka int srcu_idx; 181783d5e5b0SMikulas Patocka struct dm_table *map; 18181da177e4SLinus Torvalds 1819a9ce3853SJens Axboe map = dm_get_live_table(md, &srcu_idx); 18208cf7961dSChristoph Hellwig 1821fa247089SMike Snitzer /* If suspended, or map not yet available, queue this IO for later */ 1822fa247089SMike Snitzer if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) || 1823fa247089SMike Snitzer unlikely(!map)) { 18246abc4946SKonstantin Khlebnikov if (bio->bi_opf & REQ_NOWAIT) 18256abc4946SKonstantin Khlebnikov bio_wouldblock_error(bio); 1826b2abdb1bSMike Snitzer else if (bio->bi_opf & REQ_RAHEAD) 18276a8736d1STejun Heo bio_io_error(bio); 1828b2abdb1bSMike Snitzer else 1829b2abdb1bSMike Snitzer queue_io(md, bio); 1830b2abdb1bSMike Snitzer goto out; 18311da177e4SLinus Torvalds } 18321da177e4SLinus Torvalds 183396c9865cSMike Snitzer dm_split_and_process_bio(md, map, bio); 1834b2abdb1bSMike Snitzer out: 1835a9ce3853SJens Axboe dm_put_live_table(md, srcu_idx); 1836978e51baSMike Snitzer } 1837978e51baSMike Snitzer 1838b99fdcdcSMing Lei static bool dm_poll_dm_io(struct dm_io *io, struct io_comp_batch *iob, 1839b99fdcdcSMing Lei unsigned int flags) 1840b99fdcdcSMing Lei { 1841655f3aadSMike Snitzer WARN_ON_ONCE(!dm_tio_is_normal(&io->tio)); 1842b99fdcdcSMing Lei 1843b99fdcdcSMing Lei /* don't poll if the mapped io is done */ 1844b99fdcdcSMing Lei if (atomic_read(&io->io_count) > 1) 1845b99fdcdcSMing Lei bio_poll(&io->tio.clone, iob, flags); 1846b99fdcdcSMing Lei 1847b99fdcdcSMing Lei /* bio_poll holds the last reference */ 1848b99fdcdcSMing Lei return atomic_read(&io->io_count) == 1; 1849b99fdcdcSMing Lei } 1850b99fdcdcSMing Lei 1851b99fdcdcSMing Lei static int dm_poll_bio(struct bio *bio, struct io_comp_batch *iob, 1852b99fdcdcSMing Lei unsigned int flags) 1853b99fdcdcSMing Lei { 1854ec211631SMing Lei struct dm_io **head = dm_poll_list_head(bio); 1855ec211631SMing Lei struct dm_io *list = *head; 1856ec211631SMing Lei struct dm_io *tmp = NULL; 1857ec211631SMing Lei struct dm_io *curr, *next; 1858b99fdcdcSMing Lei 1859b99fdcdcSMing Lei /* Only poll normal bio which was marked as REQ_DM_POLL_LIST */ 1860b99fdcdcSMing Lei if (!(bio->bi_opf & REQ_DM_POLL_LIST)) 1861b99fdcdcSMing Lei return 0; 1862b99fdcdcSMing Lei 1863ec211631SMing Lei WARN_ON_ONCE(!list); 1864b99fdcdcSMing Lei 1865b99fdcdcSMing Lei /* 1866b99fdcdcSMing Lei * Restore .bi_private before possibly completing dm_io. 1867b99fdcdcSMing Lei * 1868b99fdcdcSMing Lei * bio_poll() is only possible once @bio has been completely 1869b99fdcdcSMing Lei * submitted via submit_bio_noacct()'s depth-first submission. 1870b99fdcdcSMing Lei * So there is no dm_queue_poll_io() race associated with 1871b99fdcdcSMing Lei * clearing REQ_DM_POLL_LIST here. 1872b99fdcdcSMing Lei */ 1873b99fdcdcSMing Lei bio->bi_opf &= ~REQ_DM_POLL_LIST; 1874ec211631SMing Lei bio->bi_private = list->data; 1875b99fdcdcSMing Lei 1876ec211631SMing Lei for (curr = list, next = curr->next; curr; curr = next, next = 1877ec211631SMing Lei curr ? curr->next : NULL) { 1878ec211631SMing Lei if (dm_poll_dm_io(curr, iob, flags)) { 1879b99fdcdcSMing Lei /* 188084b98f4cSMike Snitzer * clone_endio() has already occurred, so no 188184b98f4cSMike Snitzer * error handling is needed here. 1882b99fdcdcSMing Lei */ 1883ec211631SMing Lei __dm_io_dec_pending(curr); 1884ec211631SMing Lei } else { 1885ec211631SMing Lei curr->next = tmp; 1886ec211631SMing Lei tmp = curr; 1887b99fdcdcSMing Lei } 1888b99fdcdcSMing Lei } 1889b99fdcdcSMing Lei 1890b99fdcdcSMing Lei /* Not done? */ 1891ec211631SMing Lei if (tmp) { 1892b99fdcdcSMing Lei bio->bi_opf |= REQ_DM_POLL_LIST; 1893b99fdcdcSMing Lei /* Reset bio->bi_private to dm_io list head */ 1894ec211631SMing Lei *head = tmp; 1895b99fdcdcSMing Lei return 0; 1896b99fdcdcSMing Lei } 1897b99fdcdcSMing Lei return 1; 1898b99fdcdcSMing Lei } 1899b99fdcdcSMing Lei 1900a4a82ce3SHeinz Mauelshagen /* 1901a4a82ce3SHeinz Mauelshagen *--------------------------------------------------------------- 19021da177e4SLinus Torvalds * An IDR is used to keep track of allocated minor numbers. 1903a4a82ce3SHeinz Mauelshagen *--------------------------------------------------------------- 1904a4a82ce3SHeinz Mauelshagen */ 19052b06cfffSAlasdair G Kergon static void free_minor(int minor) 19061da177e4SLinus Torvalds { 1907f32c10b0SJeff Mahoney spin_lock(&_minor_lock); 19081da177e4SLinus Torvalds idr_remove(&_minor_idr, minor); 1909f32c10b0SJeff Mahoney spin_unlock(&_minor_lock); 19101da177e4SLinus Torvalds } 19111da177e4SLinus Torvalds 19121da177e4SLinus Torvalds /* 19131da177e4SLinus Torvalds * See if the device with a specific minor # is free. 19141da177e4SLinus Torvalds */ 1915cf13ab8eSFrederik Deweerdt static int specific_minor(int minor) 19161da177e4SLinus Torvalds { 1917c9d76be6STejun Heo int r; 19181da177e4SLinus Torvalds 19191da177e4SLinus Torvalds if (minor >= (1 << MINORBITS)) 19201da177e4SLinus Torvalds return -EINVAL; 19211da177e4SLinus Torvalds 1922c9d76be6STejun Heo idr_preload(GFP_KERNEL); 1923f32c10b0SJeff Mahoney spin_lock(&_minor_lock); 19241da177e4SLinus Torvalds 1925c9d76be6STejun Heo r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT); 19261da177e4SLinus Torvalds 1927f32c10b0SJeff Mahoney spin_unlock(&_minor_lock); 1928c9d76be6STejun Heo idr_preload_end(); 1929c9d76be6STejun Heo if (r < 0) 1930c9d76be6STejun Heo return r == -ENOSPC ? -EBUSY : r; 1931c9d76be6STejun Heo return 0; 19321da177e4SLinus Torvalds } 19331da177e4SLinus Torvalds 1934cf13ab8eSFrederik Deweerdt static int next_free_minor(int *minor) 19351da177e4SLinus Torvalds { 1936c9d76be6STejun Heo int r; 19371da177e4SLinus Torvalds 1938c9d76be6STejun Heo idr_preload(GFP_KERNEL); 1939f32c10b0SJeff Mahoney spin_lock(&_minor_lock); 19401da177e4SLinus Torvalds 1941c9d76be6STejun Heo r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT); 19421da177e4SLinus Torvalds 1943f32c10b0SJeff Mahoney spin_unlock(&_minor_lock); 1944c9d76be6STejun Heo idr_preload_end(); 1945c9d76be6STejun Heo if (r < 0) 19461da177e4SLinus Torvalds return r; 1947c9d76be6STejun Heo *minor = r; 1948c9d76be6STejun Heo return 0; 19491da177e4SLinus Torvalds } 19501da177e4SLinus Torvalds 195183d5cde4SAlexey Dobriyan static const struct block_device_operations dm_blk_dops; 1952681cc5e8SMike Snitzer static const struct block_device_operations dm_rq_blk_dops; 1953f26c5719SDan Williams static const struct dax_operations dm_dax_ops; 19541da177e4SLinus Torvalds 195553d5914fSMikulas Patocka static void dm_wq_work(struct work_struct *work); 195653d5914fSMikulas Patocka 1957aa6ce87aSSatya Tangirala #ifdef CONFIG_BLK_INLINE_ENCRYPTION 1958cb77cb5aSEric Biggers static void dm_queue_destroy_crypto_profile(struct request_queue *q) 1959aa6ce87aSSatya Tangirala { 1960cb77cb5aSEric Biggers dm_destroy_crypto_profile(q->crypto_profile); 1961aa6ce87aSSatya Tangirala } 1962aa6ce87aSSatya Tangirala 1963aa6ce87aSSatya Tangirala #else /* CONFIG_BLK_INLINE_ENCRYPTION */ 1964aa6ce87aSSatya Tangirala 1965cb77cb5aSEric Biggers static inline void dm_queue_destroy_crypto_profile(struct request_queue *q) 1966aa6ce87aSSatya Tangirala { 1967aa6ce87aSSatya Tangirala } 1968aa6ce87aSSatya Tangirala #endif /* !CONFIG_BLK_INLINE_ENCRYPTION */ 1969aa6ce87aSSatya Tangirala 19700f20972fSMike Snitzer static void cleanup_mapped_device(struct mapped_device *md) 19710f20972fSMike Snitzer { 19720f20972fSMike Snitzer if (md->wq) 19730f20972fSMike Snitzer destroy_workqueue(md->wq); 197429dec90aSChristoph Hellwig dm_free_md_mempools(md->mempools); 19750f20972fSMike Snitzer 1976f26c5719SDan Williams if (md->dax_dev) { 1977fb08a190SChristoph Hellwig dax_remove_host(md->disk); 1978f26c5719SDan Williams kill_dax(md->dax_dev); 1979f26c5719SDan Williams put_dax(md->dax_dev); 1980f26c5719SDan Williams md->dax_dev = NULL; 1981f26c5719SDan Williams } 1982f26c5719SDan Williams 1983588b7f5dSKirill Tkhai dm_cleanup_zoned_dev(md); 19840f20972fSMike Snitzer if (md->disk) { 19850f20972fSMike Snitzer spin_lock(&_minor_lock); 19860f20972fSMike Snitzer md->disk->private_data = NULL; 19870f20972fSMike Snitzer spin_unlock(&_minor_lock); 198889f871afSChristoph Hellwig if (dm_get_md_type(md) != DM_TYPE_NONE) { 19891a581b72SChristoph Hellwig struct table_device *td; 19901a581b72SChristoph Hellwig 199189f871afSChristoph Hellwig dm_sysfs_exit(md); 19921a581b72SChristoph Hellwig list_for_each_entry(td, &md->table_devices, list) { 19931a581b72SChristoph Hellwig bd_unlink_disk_holder(td->dm_dev.bdev, 19941a581b72SChristoph Hellwig md->disk); 19951a581b72SChristoph Hellwig } 1996d563792cSYu Kuai 1997d563792cSYu Kuai /* 1998d563792cSYu Kuai * Hold lock to make sure del_gendisk() won't concurrent 1999d563792cSYu Kuai * with open/close_table_device(). 2000d563792cSYu Kuai */ 2001d563792cSYu Kuai mutex_lock(&md->table_devices_lock); 20020f20972fSMike Snitzer del_gendisk(md->disk); 2003d563792cSYu Kuai mutex_unlock(&md->table_devices_lock); 200489f871afSChristoph Hellwig } 2005cb77cb5aSEric Biggers dm_queue_destroy_crypto_profile(md->queue); 20068b9ab626SChristoph Hellwig put_disk(md->disk); 200774a2b6ecSChristoph Hellwig } 20080f20972fSMike Snitzer 20099f6dc633SMike Snitzer if (md->pending_io) { 20109f6dc633SMike Snitzer free_percpu(md->pending_io); 20119f6dc633SMike Snitzer md->pending_io = NULL; 20129f6dc633SMike Snitzer } 20139f6dc633SMike Snitzer 2014d09960b0STahsin Erdogan cleanup_srcu_struct(&md->io_barrier); 2015d09960b0STahsin Erdogan 2016d5ffebddSMike Snitzer mutex_destroy(&md->suspend_lock); 2017d5ffebddSMike Snitzer mutex_destroy(&md->type_lock); 2018d5ffebddSMike Snitzer mutex_destroy(&md->table_devices_lock); 2019a666e5c0SMikulas Patocka mutex_destroy(&md->swap_bios_lock); 2020d5ffebddSMike Snitzer 20214cc96131SMike Snitzer dm_mq_cleanup_mapped_device(md); 20220f20972fSMike Snitzer } 20230f20972fSMike Snitzer 20241da177e4SLinus Torvalds /* 20251da177e4SLinus Torvalds * Allocate and initialise a blank device with a given minor. 20261da177e4SLinus Torvalds */ 20272b06cfffSAlasdair G Kergon static struct mapped_device *alloc_dev(int minor) 20281da177e4SLinus Torvalds { 2029115485e8SMike Snitzer int r, numa_node_id = dm_get_numa_node(); 2030115485e8SMike Snitzer struct mapped_device *md; 2031ba61fdd1SJeff Mahoney void *old_md; 20321da177e4SLinus Torvalds 2033856eb091SMikulas Patocka md = kvzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id); 20341da177e4SLinus Torvalds if (!md) { 203543e6c111SMikulas Patocka DMERR("unable to allocate device, out of memory."); 20361da177e4SLinus Torvalds return NULL; 20371da177e4SLinus Torvalds } 20381da177e4SLinus Torvalds 203910da4f79SJeff Mahoney if (!try_module_get(THIS_MODULE)) 20406ed7ade8SMilan Broz goto bad_module_get; 204110da4f79SJeff Mahoney 20421da177e4SLinus Torvalds /* get a minor number for the dev */ 20432b06cfffSAlasdair G Kergon if (minor == DM_ANY_MINOR) 2044cf13ab8eSFrederik Deweerdt r = next_free_minor(&minor); 20452b06cfffSAlasdair G Kergon else 2046cf13ab8eSFrederik Deweerdt r = specific_minor(minor); 20471da177e4SLinus Torvalds if (r < 0) 20486ed7ade8SMilan Broz goto bad_minor; 20491da177e4SLinus Torvalds 205083d5e5b0SMikulas Patocka r = init_srcu_struct(&md->io_barrier); 205183d5e5b0SMikulas Patocka if (r < 0) 205283d5e5b0SMikulas Patocka goto bad_io_barrier; 205383d5e5b0SMikulas Patocka 2054115485e8SMike Snitzer md->numa_node_id = numa_node_id; 2055591ddcfcSMike Snitzer md->init_tio_pdu = false; 2056a5664dadSMike Snitzer md->type = DM_TYPE_NONE; 2057e61290a4SDaniel Walker mutex_init(&md->suspend_lock); 2058a5664dadSMike Snitzer mutex_init(&md->type_lock); 205986f1152bSBenjamin Marzinski mutex_init(&md->table_devices_lock); 2060022c2611SMikulas Patocka spin_lock_init(&md->deferred_lock); 20611da177e4SLinus Torvalds atomic_set(&md->holders, 1); 20625c6bd75dSAlasdair G Kergon atomic_set(&md->open_count, 0); 20631da177e4SLinus Torvalds atomic_set(&md->event_nr, 0); 20647a8c3d3bSMike Anderson atomic_set(&md->uevent_seq, 0); 20657a8c3d3bSMike Anderson INIT_LIST_HEAD(&md->uevent_list); 206686f1152bSBenjamin Marzinski INIT_LIST_HEAD(&md->table_devices); 20677a8c3d3bSMike Anderson spin_lock_init(&md->uevent_lock); 20681da177e4SLinus Torvalds 206947ace7e0SMike Snitzer /* 2070c62b37d9SChristoph Hellwig * default to bio-based until DM table is loaded and md->type 2071c62b37d9SChristoph Hellwig * established. If request-based table is loaded: blk-mq will 2072c62b37d9SChristoph Hellwig * override accordingly. 207347ace7e0SMike Snitzer */ 207474fe6ba9SChristoph Hellwig md->disk = blk_alloc_disk(md->numa_node_id); 20751da177e4SLinus Torvalds if (!md->disk) 20760f20972fSMike Snitzer goto bad; 207774fe6ba9SChristoph Hellwig md->queue = md->disk->queue; 20781da177e4SLinus Torvalds 2079f0b04115SJeff Mahoney init_waitqueue_head(&md->wait); 208053d5914fSMikulas Patocka INIT_WORK(&md->work, dm_wq_work); 20818b211aacSMing Lei INIT_WORK(&md->requeue_work, dm_wq_requeue_work); 2082f0b04115SJeff Mahoney init_waitqueue_head(&md->eventq); 20832995fa78SMikulas Patocka init_completion(&md->kobj_holder.completion); 2084f0b04115SJeff Mahoney 20858b211aacSMing Lei md->requeue_list = NULL; 2086a666e5c0SMikulas Patocka md->swap_bios = get_swap_bios(); 2087a666e5c0SMikulas Patocka sema_init(&md->swap_bios_semaphore, md->swap_bios); 2088a666e5c0SMikulas Patocka mutex_init(&md->swap_bios_lock); 2089a666e5c0SMikulas Patocka 20901da177e4SLinus Torvalds md->disk->major = _major; 20911da177e4SLinus Torvalds md->disk->first_minor = minor; 209274fe6ba9SChristoph Hellwig md->disk->minors = 1; 20931ebe2e5fSChristoph Hellwig md->disk->flags |= GENHD_FL_NO_PART; 20941da177e4SLinus Torvalds md->disk->fops = &dm_blk_dops; 20951da177e4SLinus Torvalds md->disk->private_data = md; 20961da177e4SLinus Torvalds sprintf(md->disk->disk_name, "dm-%d", minor); 2097f26c5719SDan Williams 20985d2a228bSChristoph Hellwig if (IS_ENABLED(CONFIG_FS_DAX)) { 209930c6828aSChristoph Hellwig md->dax_dev = alloc_dax(md, &dm_dax_ops); 2100d7519392SChristoph Hellwig if (IS_ERR(md->dax_dev)) { 2101d7519392SChristoph Hellwig md->dax_dev = NULL; 2102f26c5719SDan Williams goto bad; 2103976431b0SDan Williams } 21047ac5360cSChristoph Hellwig set_dax_nocache(md->dax_dev); 21057ac5360cSChristoph Hellwig set_dax_nomc(md->dax_dev); 2106fb08a190SChristoph Hellwig if (dax_add_host(md->dax_dev, md->disk)) 2107f26c5719SDan Williams goto bad; 2108f26c5719SDan Williams } 21091da177e4SLinus Torvalds 21107e51f257SMike Anderson format_dev_t(md->name, MKDEV(_major, minor)); 21111da177e4SLinus Torvalds 2112c7c879eeSMichał Mirosław md->wq = alloc_workqueue("kdmflush/%s", WQ_MEM_RECLAIM, 0, md->name); 2113304f3f6aSMilan Broz if (!md->wq) 21140f20972fSMike Snitzer goto bad; 2115304f3f6aSMilan Broz 21169f6dc633SMike Snitzer md->pending_io = alloc_percpu(unsigned long); 21179f6dc633SMike Snitzer if (!md->pending_io) 21189f6dc633SMike Snitzer goto bad; 21199f6dc633SMike Snitzer 2120d3aa3e06SJiasheng Jiang r = dm_stats_init(&md->stats); 2121d3aa3e06SJiasheng Jiang if (r < 0) 2122d3aa3e06SJiasheng Jiang goto bad; 2123fd2ed4d2SMikulas Patocka 2124ba61fdd1SJeff Mahoney /* Populate the mapping, nobody knows we exist yet */ 2125f32c10b0SJeff Mahoney spin_lock(&_minor_lock); 2126ba61fdd1SJeff Mahoney old_md = idr_replace(&_minor_idr, md, minor); 2127f32c10b0SJeff Mahoney spin_unlock(&_minor_lock); 2128ba61fdd1SJeff Mahoney 2129ba61fdd1SJeff Mahoney BUG_ON(old_md != MINOR_ALLOCED); 2130ba61fdd1SJeff Mahoney 21311da177e4SLinus Torvalds return md; 21321da177e4SLinus Torvalds 21330f20972fSMike Snitzer bad: 21340f20972fSMike Snitzer cleanup_mapped_device(md); 213583d5e5b0SMikulas Patocka bad_io_barrier: 21361da177e4SLinus Torvalds free_minor(minor); 21376ed7ade8SMilan Broz bad_minor: 213810da4f79SJeff Mahoney module_put(THIS_MODULE); 21396ed7ade8SMilan Broz bad_module_get: 2140856eb091SMikulas Patocka kvfree(md); 21411da177e4SLinus Torvalds return NULL; 21421da177e4SLinus Torvalds } 21431da177e4SLinus Torvalds 2144ae9da83fSJun'ichi Nomura static void unlock_fs(struct mapped_device *md); 2145ae9da83fSJun'ichi Nomura 21461da177e4SLinus Torvalds static void free_dev(struct mapped_device *md) 21471da177e4SLinus Torvalds { 2148f331c029STejun Heo int minor = MINOR(disk_devt(md->disk)); 214963d94e48SJun'ichi Nomura 2150ae9da83fSJun'ichi Nomura unlock_fs(md); 21512eb6e1e3SKeith Busch 21520f20972fSMike Snitzer cleanup_mapped_device(md); 21530f20972fSMike Snitzer 2154992ec6a9SChristoph Hellwig WARN_ON_ONCE(!list_empty(&md->table_devices)); 21550f20972fSMike Snitzer dm_stats_cleanup(&md->stats); 215663a4f065SMike Snitzer free_minor(minor); 215763a4f065SMike Snitzer 215810da4f79SJeff Mahoney module_put(THIS_MODULE); 2159856eb091SMikulas Patocka kvfree(md); 21601da177e4SLinus Torvalds } 21611da177e4SLinus Torvalds 21621da177e4SLinus Torvalds /* 21631da177e4SLinus Torvalds * Bind a table to the device. 21641da177e4SLinus Torvalds */ 21651da177e4SLinus Torvalds static void event_callback(void *context) 21661da177e4SLinus Torvalds { 21677a8c3d3bSMike Anderson unsigned long flags; 21687a8c3d3bSMike Anderson LIST_HEAD(uevents); 2169d695e441SXU pengfei struct mapped_device *md = context; 21701da177e4SLinus Torvalds 21717a8c3d3bSMike Anderson spin_lock_irqsave(&md->uevent_lock, flags); 21727a8c3d3bSMike Anderson list_splice_init(&md->uevent_list, &uevents); 21737a8c3d3bSMike Anderson spin_unlock_irqrestore(&md->uevent_lock, flags); 21747a8c3d3bSMike Anderson 2175ed9e1982STejun Heo dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj); 21767a8c3d3bSMike Anderson 21771da177e4SLinus Torvalds atomic_inc(&md->event_nr); 21781da177e4SLinus Torvalds wake_up(&md->eventq); 217962e08243SMikulas Patocka dm_issue_global_event(); 21801da177e4SLinus Torvalds } 21811da177e4SLinus Torvalds 2182c217649bSMike Snitzer /* 2183042d2a9bSAlasdair G Kergon * Returns old map, which caller must destroy. 2184042d2a9bSAlasdair G Kergon */ 2185042d2a9bSAlasdair G Kergon static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t, 2186754c5fc7SMike Snitzer struct queue_limits *limits) 21871da177e4SLinus Torvalds { 2188042d2a9bSAlasdair G Kergon struct dm_table *old_map; 21891da177e4SLinus Torvalds sector_t size; 21902a2a4c51SJens Axboe int ret; 21911da177e4SLinus Torvalds 21925a8f1f80SBart Van Assche lockdep_assert_held(&md->suspend_lock); 21935a8f1f80SBart Van Assche 21941da177e4SLinus Torvalds size = dm_table_get_size(t); 21953ac51e74SDarrick J. Wong 21963ac51e74SDarrick J. Wong /* 21973ac51e74SDarrick J. Wong * Wipe any geometry if the size of the table changed. 21983ac51e74SDarrick J. Wong */ 2199fd2ed4d2SMikulas Patocka if (size != dm_get_size(md)) 22003ac51e74SDarrick J. Wong memset(&md->geometry, 0, sizeof(md->geometry)); 22013ac51e74SDarrick J. Wong 22025424a0b8SMikulas Patocka set_capacity(md->disk, size); 22031da177e4SLinus Torvalds 2204cf222b37SAlasdair G Kergon dm_table_event_callback(t, event_callback, md); 22052ca3310eSAlasdair G Kergon 2206f5b4aee1SMike Snitzer if (dm_table_request_based(t)) { 220716f12266SMike Snitzer /* 22089c37de29SMike Snitzer * Leverage the fact that request-based DM targets are 22099c37de29SMike Snitzer * immutable singletons - used to optimize dm_mq_queue_rq. 221016f12266SMike Snitzer */ 221116f12266SMike Snitzer md->immutable_target = dm_table_get_immutable_target(t); 2212e6ee8c0bSKiyoshi Ueda 221329dec90aSChristoph Hellwig /* 221429dec90aSChristoph Hellwig * There is no need to reload with request-based dm because the 221529dec90aSChristoph Hellwig * size of front_pad doesn't change. 221629dec90aSChristoph Hellwig * 221729dec90aSChristoph Hellwig * Note for future: If you are to reload bioset, prep-ed 221829dec90aSChristoph Hellwig * requests in the queue may refer to bio from the old bioset, 221929dec90aSChristoph Hellwig * so you must walk through the queue to unprep. 222029dec90aSChristoph Hellwig */ 222129dec90aSChristoph Hellwig if (!md->mempools) { 222229dec90aSChristoph Hellwig md->mempools = t->mempools; 222329dec90aSChristoph Hellwig t->mempools = NULL; 222429dec90aSChristoph Hellwig } 222529dec90aSChristoph Hellwig } else { 222629dec90aSChristoph Hellwig /* 222729dec90aSChristoph Hellwig * The md may already have mempools that need changing. 222829dec90aSChristoph Hellwig * If so, reload bioset because front_pad may have changed 222929dec90aSChristoph Hellwig * because a different table was loaded. 223029dec90aSChristoph Hellwig */ 223129dec90aSChristoph Hellwig dm_free_md_mempools(md->mempools); 223229dec90aSChristoph Hellwig md->mempools = t->mempools; 223329dec90aSChristoph Hellwig t->mempools = NULL; 22342a2a4c51SJens Axboe } 2235e6ee8c0bSKiyoshi Ueda 2236f5b4aee1SMike Snitzer ret = dm_table_set_restrictions(t, md->queue, limits); 2237bb37d772SDamien Le Moal if (ret) { 2238bb37d772SDamien Le Moal old_map = ERR_PTR(ret); 2239bb37d772SDamien Le Moal goto out; 2240bb37d772SDamien Le Moal } 2241bb37d772SDamien Le Moal 2242a12f5d48SEric Dumazet old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 22431d3aa6f6SMike Snitzer rcu_assign_pointer(md->map, (void *)t); 224436a0456fSAlasdair G Kergon md->immutable_target_type = dm_table_get_immutable_target_type(t); 224536a0456fSAlasdair G Kergon 224641abc4e1SHannes Reinecke if (old_map) 224783d5e5b0SMikulas Patocka dm_sync_table(md); 22482a2a4c51SJens Axboe out: 2249042d2a9bSAlasdair G Kergon return old_map; 22501da177e4SLinus Torvalds } 22511da177e4SLinus Torvalds 2252a7940155SAlasdair G Kergon /* 2253a7940155SAlasdair G Kergon * Returns unbound table for the caller to free. 2254a7940155SAlasdair G Kergon */ 2255a7940155SAlasdair G Kergon static struct dm_table *__unbind(struct mapped_device *md) 22561da177e4SLinus Torvalds { 2257a12f5d48SEric Dumazet struct dm_table *map = rcu_dereference_protected(md->map, 1); 22581da177e4SLinus Torvalds 22591da177e4SLinus Torvalds if (!map) 2260a7940155SAlasdair G Kergon return NULL; 22611da177e4SLinus Torvalds 22621da177e4SLinus Torvalds dm_table_event_callback(map, NULL, NULL); 22639cdb8520SMonam Agarwal RCU_INIT_POINTER(md->map, NULL); 226483d5e5b0SMikulas Patocka dm_sync_table(md); 2265a7940155SAlasdair G Kergon 2266a7940155SAlasdair G Kergon return map; 22671da177e4SLinus Torvalds } 22681da177e4SLinus Torvalds 22691da177e4SLinus Torvalds /* 22701da177e4SLinus Torvalds * Constructor for a new device. 22711da177e4SLinus Torvalds */ 22722b06cfffSAlasdair G Kergon int dm_create(int minor, struct mapped_device **result) 22731da177e4SLinus Torvalds { 22741da177e4SLinus Torvalds struct mapped_device *md; 22751da177e4SLinus Torvalds 22762b06cfffSAlasdair G Kergon md = alloc_dev(minor); 22771da177e4SLinus Torvalds if (!md) 22781da177e4SLinus Torvalds return -ENXIO; 22791da177e4SLinus Torvalds 228091ccbbacSTushar Sugandhi dm_ima_reset_data(md); 228191ccbbacSTushar Sugandhi 22821da177e4SLinus Torvalds *result = md; 22831da177e4SLinus Torvalds return 0; 22841da177e4SLinus Torvalds } 22851da177e4SLinus Torvalds 2286a5664dadSMike Snitzer /* 2287a5664dadSMike Snitzer * Functions to manage md->type. 2288a5664dadSMike Snitzer * All are required to hold md->type_lock. 2289a5664dadSMike Snitzer */ 2290a5664dadSMike Snitzer void dm_lock_md_type(struct mapped_device *md) 2291a5664dadSMike Snitzer { 2292a5664dadSMike Snitzer mutex_lock(&md->type_lock); 2293a5664dadSMike Snitzer } 2294a5664dadSMike Snitzer 2295a5664dadSMike Snitzer void dm_unlock_md_type(struct mapped_device *md) 2296a5664dadSMike Snitzer { 2297a5664dadSMike Snitzer mutex_unlock(&md->type_lock); 2298a5664dadSMike Snitzer } 2299a5664dadSMike Snitzer 23007e0d574fSBart Van Assche void dm_set_md_type(struct mapped_device *md, enum dm_queue_mode type) 2301a5664dadSMike Snitzer { 230200c4fc3bSMike Snitzer BUG_ON(!mutex_is_locked(&md->type_lock)); 2303a5664dadSMike Snitzer md->type = type; 2304a5664dadSMike Snitzer } 2305a5664dadSMike Snitzer 23067e0d574fSBart Van Assche enum dm_queue_mode dm_get_md_type(struct mapped_device *md) 2307a5664dadSMike Snitzer { 2308a5664dadSMike Snitzer return md->type; 2309a5664dadSMike Snitzer } 2310a5664dadSMike Snitzer 231136a0456fSAlasdair G Kergon struct target_type *dm_get_immutable_target_type(struct mapped_device *md) 231236a0456fSAlasdair G Kergon { 231336a0456fSAlasdair G Kergon return md->immutable_target_type; 231436a0456fSAlasdair G Kergon } 231536a0456fSAlasdair G Kergon 23164a0b4ddfSMike Snitzer /* 23174a0b4ddfSMike Snitzer * Setup the DM device's queue based on md's type 23184a0b4ddfSMike Snitzer */ 2319591ddcfcSMike Snitzer int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t) 23204a0b4ddfSMike Snitzer { 2321ba305859SChristoph Hellwig enum dm_queue_mode type = dm_table_get_type(t); 2322c100ec49SMike Snitzer struct queue_limits limits; 23231a581b72SChristoph Hellwig struct table_device *td; 2324ba305859SChristoph Hellwig int r; 2325bfebd1cdSMike Snitzer 2326545ed20eSToshi Kani switch (type) { 2327bfebd1cdSMike Snitzer case DM_TYPE_REQUEST_BASED: 2328681cc5e8SMike Snitzer md->disk->fops = &dm_rq_blk_dops; 2329e83068a5SMike Snitzer r = dm_mq_init_request_queue(md, t); 2330bfebd1cdSMike Snitzer if (r) { 2331681cc5e8SMike Snitzer DMERR("Cannot initialize queue for request-based dm mapped device"); 2332bfebd1cdSMike Snitzer return r; 2333bfebd1cdSMike Snitzer } 2334bfebd1cdSMike Snitzer break; 2335bfebd1cdSMike Snitzer case DM_TYPE_BIO_BASED: 2336545ed20eSToshi Kani case DM_TYPE_DAX_BIO_BASED: 2337526d1006SLi Nan blk_queue_flag_set(QUEUE_FLAG_IO_STAT, md->queue); 2338bfebd1cdSMike Snitzer break; 23397e0d574fSBart Van Assche case DM_TYPE_NONE: 23407e0d574fSBart Van Assche WARN_ON_ONCE(true); 23417e0d574fSBart Van Assche break; 2342ff36ab34SMike Snitzer } 23434a0b4ddfSMike Snitzer 2344c100ec49SMike Snitzer r = dm_calculate_queue_limits(t, &limits); 2345c100ec49SMike Snitzer if (r) { 2346c100ec49SMike Snitzer DMERR("Cannot calculate initial queue limits"); 2347c100ec49SMike Snitzer return r; 2348c100ec49SMike Snitzer } 2349bb37d772SDamien Le Moal r = dm_table_set_restrictions(t, md->queue, &limits); 2350bb37d772SDamien Le Moal if (r) 2351bb37d772SDamien Le Moal return r; 235289f871afSChristoph Hellwig 2353d563792cSYu Kuai /* 2354d563792cSYu Kuai * Hold lock to make sure add_disk() and del_gendisk() won't concurrent 2355d563792cSYu Kuai * with open_table_device() and close_table_device(). 2356d563792cSYu Kuai */ 2357d563792cSYu Kuai mutex_lock(&md->table_devices_lock); 2358e7089f65SLuis Chamberlain r = add_disk(md->disk); 2359d563792cSYu Kuai mutex_unlock(&md->table_devices_lock); 2360e7089f65SLuis Chamberlain if (r) 2361e7089f65SLuis Chamberlain return r; 236289f871afSChristoph Hellwig 23631a581b72SChristoph Hellwig /* 23641a581b72SChristoph Hellwig * Register the holder relationship for devices added before the disk 23651a581b72SChristoph Hellwig * was live. 23661a581b72SChristoph Hellwig */ 23671a581b72SChristoph Hellwig list_for_each_entry(td, &md->table_devices, list) { 23681a581b72SChristoph Hellwig r = bd_link_disk_holder(td->dm_dev.bdev, md->disk); 23691a581b72SChristoph Hellwig if (r) 23701a581b72SChristoph Hellwig goto out_undo_holders; 23711a581b72SChristoph Hellwig } 23721a581b72SChristoph Hellwig 237389f871afSChristoph Hellwig r = dm_sysfs_init(md); 23741a581b72SChristoph Hellwig if (r) 23751a581b72SChristoph Hellwig goto out_undo_holders; 23761a581b72SChristoph Hellwig 23771a581b72SChristoph Hellwig md->type = type; 23781a581b72SChristoph Hellwig return 0; 23791a581b72SChristoph Hellwig 23801a581b72SChristoph Hellwig out_undo_holders: 23811a581b72SChristoph Hellwig list_for_each_entry_continue_reverse(td, &md->table_devices, list) 23821a581b72SChristoph Hellwig bd_unlink_disk_holder(td->dm_dev.bdev, md->disk); 2383d563792cSYu Kuai mutex_lock(&md->table_devices_lock); 238489f871afSChristoph Hellwig del_gendisk(md->disk); 2385d563792cSYu Kuai mutex_unlock(&md->table_devices_lock); 238689f871afSChristoph Hellwig return r; 238789f871afSChristoph Hellwig } 2388d563792cSYu Kuai 23892bec1f4aSMikulas Patocka struct mapped_device *dm_get_md(dev_t dev) 23901da177e4SLinus Torvalds { 23911da177e4SLinus Torvalds struct mapped_device *md; 239286a3238cSHeinz Mauelshagen unsigned int minor = MINOR(dev); 23931da177e4SLinus Torvalds 23941da177e4SLinus Torvalds if (MAJOR(dev) != _major || minor >= (1 << MINORBITS)) 23951da177e4SLinus Torvalds return NULL; 23961da177e4SLinus Torvalds 2397f32c10b0SJeff Mahoney spin_lock(&_minor_lock); 23981da177e4SLinus Torvalds 23991da177e4SLinus Torvalds md = idr_find(&_minor_idr, minor); 240049de5769SMike Snitzer if (!md || md == MINOR_ALLOCED || (MINOR(disk_devt(dm_disk(md))) != minor) || 240149de5769SMike Snitzer test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) { 2402637842cfSDavid Teigland md = NULL; 2403fba9f90eSJeff Mahoney goto out; 2404fba9f90eSJeff Mahoney } 24052bec1f4aSMikulas Patocka dm_get(md); 2406fba9f90eSJeff Mahoney out: 2407f32c10b0SJeff Mahoney spin_unlock(&_minor_lock); 24081da177e4SLinus Torvalds 2409637842cfSDavid Teigland return md; 2410637842cfSDavid Teigland } 24113cf2e4baSAlasdair G Kergon EXPORT_SYMBOL_GPL(dm_get_md); 2412d229a958SDavid Teigland 24139ade92a9SAlasdair G Kergon void *dm_get_mdptr(struct mapped_device *md) 2414637842cfSDavid Teigland { 24159ade92a9SAlasdair G Kergon return md->interface_ptr; 24161da177e4SLinus Torvalds } 24171da177e4SLinus Torvalds 24181da177e4SLinus Torvalds void dm_set_mdptr(struct mapped_device *md, void *ptr) 24191da177e4SLinus Torvalds { 24201da177e4SLinus Torvalds md->interface_ptr = ptr; 24211da177e4SLinus Torvalds } 24221da177e4SLinus Torvalds 24231da177e4SLinus Torvalds void dm_get(struct mapped_device *md) 24241da177e4SLinus Torvalds { 24251da177e4SLinus Torvalds atomic_inc(&md->holders); 24263f77316dSKiyoshi Ueda BUG_ON(test_bit(DMF_FREEING, &md->flags)); 24271da177e4SLinus Torvalds } 24281da177e4SLinus Torvalds 242909ee96b2SMikulas Patocka int dm_hold(struct mapped_device *md) 243009ee96b2SMikulas Patocka { 243109ee96b2SMikulas Patocka spin_lock(&_minor_lock); 243209ee96b2SMikulas Patocka if (test_bit(DMF_FREEING, &md->flags)) { 243309ee96b2SMikulas Patocka spin_unlock(&_minor_lock); 243409ee96b2SMikulas Patocka return -EBUSY; 243509ee96b2SMikulas Patocka } 243609ee96b2SMikulas Patocka dm_get(md); 243709ee96b2SMikulas Patocka spin_unlock(&_minor_lock); 243809ee96b2SMikulas Patocka return 0; 243909ee96b2SMikulas Patocka } 244009ee96b2SMikulas Patocka EXPORT_SYMBOL_GPL(dm_hold); 244109ee96b2SMikulas Patocka 244272d94861SAlasdair G Kergon const char *dm_device_name(struct mapped_device *md) 244372d94861SAlasdair G Kergon { 244472d94861SAlasdair G Kergon return md->name; 244572d94861SAlasdair G Kergon } 244672d94861SAlasdair G Kergon EXPORT_SYMBOL_GPL(dm_device_name); 244772d94861SAlasdair G Kergon 24483f77316dSKiyoshi Ueda static void __dm_destroy(struct mapped_device *md, bool wait) 24491da177e4SLinus Torvalds { 24501134e5aeSMike Anderson struct dm_table *map; 245183d5e5b0SMikulas Patocka int srcu_idx; 24521da177e4SLinus Torvalds 24533f77316dSKiyoshi Ueda might_sleep(); 2454fba9f90eSJeff Mahoney 245563a4f065SMike Snitzer spin_lock(&_minor_lock); 24563f77316dSKiyoshi Ueda idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md)))); 2457fba9f90eSJeff Mahoney set_bit(DMF_FREEING, &md->flags); 2458f32c10b0SJeff Mahoney spin_unlock(&_minor_lock); 24593f77316dSKiyoshi Ueda 24607a5428dcSChristoph Hellwig blk_mark_disk_dead(md->disk); 24613b785fbcSBart Van Assche 2462ab7c7bb6SMikulas Patocka /* 2463ab7c7bb6SMikulas Patocka * Take suspend_lock so that presuspend and postsuspend methods 2464ab7c7bb6SMikulas Patocka * do not race with internal suspend. 2465ab7c7bb6SMikulas Patocka */ 2466ab7c7bb6SMikulas Patocka mutex_lock(&md->suspend_lock); 24672a708cffSJunichi Nomura map = dm_get_live_table(md, &srcu_idx); 24684f186f8bSKiyoshi Ueda if (!dm_suspended_md(md)) { 24691da177e4SLinus Torvalds dm_table_presuspend_targets(map); 2470adc0daadSMikulas Patocka set_bit(DMF_SUSPENDED, &md->flags); 24715df96f2bSMikulas Patocka set_bit(DMF_POST_SUSPENDING, &md->flags); 24721da177e4SLinus Torvalds dm_table_postsuspend_targets(map); 24731da177e4SLinus Torvalds } 2474238d991fSHeinz Mauelshagen /* dm_put_live_table must be before fsleep, otherwise deadlock is possible */ 247583d5e5b0SMikulas Patocka dm_put_live_table(md, srcu_idx); 24762a708cffSJunichi Nomura mutex_unlock(&md->suspend_lock); 247783d5e5b0SMikulas Patocka 24783f77316dSKiyoshi Ueda /* 24793f77316dSKiyoshi Ueda * Rare, but there may be I/O requests still going to complete, 24803f77316dSKiyoshi Ueda * for example. Wait for all references to disappear. 24813f77316dSKiyoshi Ueda * No one should increment the reference count of the mapped_device, 24823f77316dSKiyoshi Ueda * after the mapped_device state becomes DMF_FREEING. 24833f77316dSKiyoshi Ueda */ 24843f77316dSKiyoshi Ueda if (wait) 24853f77316dSKiyoshi Ueda while (atomic_read(&md->holders)) 2486238d991fSHeinz Mauelshagen fsleep(1000); 24873f77316dSKiyoshi Ueda else if (atomic_read(&md->holders)) 24883f77316dSKiyoshi Ueda DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)", 24893f77316dSKiyoshi Ueda dm_device_name(md), atomic_read(&md->holders)); 24903f77316dSKiyoshi Ueda 2491a7940155SAlasdair G Kergon dm_table_destroy(__unbind(md)); 24921da177e4SLinus Torvalds free_dev(md); 24931da177e4SLinus Torvalds } 24943f77316dSKiyoshi Ueda 24953f77316dSKiyoshi Ueda void dm_destroy(struct mapped_device *md) 24963f77316dSKiyoshi Ueda { 24973f77316dSKiyoshi Ueda __dm_destroy(md, true); 24983f77316dSKiyoshi Ueda } 24993f77316dSKiyoshi Ueda 25003f77316dSKiyoshi Ueda void dm_destroy_immediate(struct mapped_device *md) 25013f77316dSKiyoshi Ueda { 25023f77316dSKiyoshi Ueda __dm_destroy(md, false); 25033f77316dSKiyoshi Ueda } 25043f77316dSKiyoshi Ueda 25053f77316dSKiyoshi Ueda void dm_put(struct mapped_device *md) 25063f77316dSKiyoshi Ueda { 25073f77316dSKiyoshi Ueda atomic_dec(&md->holders); 25081da177e4SLinus Torvalds } 250979eb885cSEdward Goggin EXPORT_SYMBOL_GPL(dm_put); 25101da177e4SLinus Torvalds 25119f6dc633SMike Snitzer static bool dm_in_flight_bios(struct mapped_device *md) 251285067747SMing Lei { 251385067747SMing Lei int cpu; 25149f6dc633SMike Snitzer unsigned long sum = 0; 251585067747SMing Lei 25169f6dc633SMike Snitzer for_each_possible_cpu(cpu) 25179f6dc633SMike Snitzer sum += *per_cpu_ptr(md->pending_io, cpu); 251885067747SMing Lei 251985067747SMing Lei return sum != 0; 252085067747SMing Lei } 252185067747SMing Lei 25222f064a59SPeter Zijlstra static int dm_wait_for_bios_completion(struct mapped_device *md, unsigned int task_state) 252346125c1cSMilan Broz { 252446125c1cSMilan Broz int r = 0; 25259f4c3f87SBart Van Assche DEFINE_WAIT(wait); 252646125c1cSMilan Broz 252785067747SMing Lei while (true) { 25289f4c3f87SBart Van Assche prepare_to_wait(&md->wait, &wait, task_state); 252946125c1cSMilan Broz 25309f6dc633SMike Snitzer if (!dm_in_flight_bios(md)) 253146125c1cSMilan Broz break; 253246125c1cSMilan Broz 2533e3fabdfdSBart Van Assche if (signal_pending_state(task_state, current)) { 253446125c1cSMilan Broz r = -EINTR; 253546125c1cSMilan Broz break; 253646125c1cSMilan Broz } 253746125c1cSMilan Broz 253846125c1cSMilan Broz io_schedule(); 253946125c1cSMilan Broz } 25409f4c3f87SBart Van Assche finish_wait(&md->wait, &wait); 2541b44ebeb0SMikulas Patocka 25429f6dc633SMike Snitzer smp_rmb(); 25439f6dc633SMike Snitzer 254446125c1cSMilan Broz return r; 254546125c1cSMilan Broz } 254646125c1cSMilan Broz 25472f064a59SPeter Zijlstra static int dm_wait_for_completion(struct mapped_device *md, unsigned int task_state) 254885067747SMing Lei { 254985067747SMing Lei int r = 0; 255085067747SMing Lei 255185067747SMing Lei if (!queue_is_mq(md->queue)) 255285067747SMing Lei return dm_wait_for_bios_completion(md, task_state); 255385067747SMing Lei 255485067747SMing Lei while (true) { 255585067747SMing Lei if (!blk_mq_queue_inflight(md->queue)) 255685067747SMing Lei break; 255785067747SMing Lei 255885067747SMing Lei if (signal_pending_state(task_state, current)) { 255985067747SMing Lei r = -EINTR; 256085067747SMing Lei break; 256185067747SMing Lei } 256285067747SMing Lei 2563238d991fSHeinz Mauelshagen fsleep(5000); 256485067747SMing Lei } 256585067747SMing Lei 256685067747SMing Lei return r; 256785067747SMing Lei } 256885067747SMing Lei 25691da177e4SLinus Torvalds /* 25701da177e4SLinus Torvalds * Process the deferred bios 25711da177e4SLinus Torvalds */ 2572ef208587SMikulas Patocka static void dm_wq_work(struct work_struct *work) 25731da177e4SLinus Torvalds { 25740c2915b8SMike Snitzer struct mapped_device *md = container_of(work, struct mapped_device, work); 25750c2915b8SMike Snitzer struct bio *bio; 2576ef208587SMikulas Patocka 25773b00b203SMikulas Patocka while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) { 2578022c2611SMikulas Patocka spin_lock_irq(&md->deferred_lock); 25790c2915b8SMike Snitzer bio = bio_list_pop(&md->deferred); 2580022c2611SMikulas Patocka spin_unlock_irq(&md->deferred_lock); 2581022c2611SMikulas Patocka 25820c2915b8SMike Snitzer if (!bio) 2583df12ee99SAlasdair G Kergon break; 258473d410c0SMilan Broz 25850c2915b8SMike Snitzer submit_bio_noacct(bio); 25860ca44fceSPingfan Liu cond_resched(); 2587e6ee8c0bSKiyoshi Ueda } 25881da177e4SLinus Torvalds } 25891da177e4SLinus Torvalds 25909a1fb464SMikulas Patocka static void dm_queue_flush(struct mapped_device *md) 2591304f3f6aSMilan Broz { 25923b00b203SMikulas Patocka clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 25934e857c58SPeter Zijlstra smp_mb__after_atomic(); 259453d5914fSMikulas Patocka queue_work(md->wq, &md->work); 2595304f3f6aSMilan Broz } 2596304f3f6aSMilan Broz 25971da177e4SLinus Torvalds /* 2598042d2a9bSAlasdair G Kergon * Swap in a new table, returning the old one for the caller to destroy. 25991da177e4SLinus Torvalds */ 2600042d2a9bSAlasdair G Kergon struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table) 26011da177e4SLinus Torvalds { 260287eb5b21SMike Christie struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL); 2603754c5fc7SMike Snitzer struct queue_limits limits; 2604042d2a9bSAlasdair G Kergon int r; 26051da177e4SLinus Torvalds 2606e61290a4SDaniel Walker mutex_lock(&md->suspend_lock); 26071da177e4SLinus Torvalds 26081da177e4SLinus Torvalds /* device must be suspended */ 26094f186f8bSKiyoshi Ueda if (!dm_suspended_md(md)) 261093c534aeSAlasdair G Kergon goto out; 26111da177e4SLinus Torvalds 26123ae70656SMike Snitzer /* 26133ae70656SMike Snitzer * If the new table has no data devices, retain the existing limits. 26143ae70656SMike Snitzer * This helps multipath with queue_if_no_path if all paths disappear, 26153ae70656SMike Snitzer * then new I/O is queued based on these limits, and then some paths 26163ae70656SMike Snitzer * reappear. 26173ae70656SMike Snitzer */ 26183ae70656SMike Snitzer if (dm_table_has_no_data_devices(table)) { 261983d5e5b0SMikulas Patocka live_map = dm_get_live_table_fast(md); 26203ae70656SMike Snitzer if (live_map) 26213ae70656SMike Snitzer limits = md->queue->limits; 262283d5e5b0SMikulas Patocka dm_put_live_table_fast(md); 26233ae70656SMike Snitzer } 26243ae70656SMike Snitzer 262587eb5b21SMike Christie if (!live_map) { 2626754c5fc7SMike Snitzer r = dm_calculate_queue_limits(table, &limits); 2627042d2a9bSAlasdair G Kergon if (r) { 2628042d2a9bSAlasdair G Kergon map = ERR_PTR(r); 2629754c5fc7SMike Snitzer goto out; 2630042d2a9bSAlasdair G Kergon } 263187eb5b21SMike Christie } 2632754c5fc7SMike Snitzer 2633042d2a9bSAlasdair G Kergon map = __bind(md, table, &limits); 263462e08243SMikulas Patocka dm_issue_global_event(); 26351da177e4SLinus Torvalds 263693c534aeSAlasdair G Kergon out: 2637e61290a4SDaniel Walker mutex_unlock(&md->suspend_lock); 2638042d2a9bSAlasdair G Kergon return map; 26391da177e4SLinus Torvalds } 26401da177e4SLinus Torvalds 26411da177e4SLinus Torvalds /* 26421da177e4SLinus Torvalds * Functions to lock and unlock any filesystem running on the 26431da177e4SLinus Torvalds * device. 26441da177e4SLinus Torvalds */ 26452ca3310eSAlasdair G Kergon static int lock_fs(struct mapped_device *md) 26461da177e4SLinus Torvalds { 2647e39e2e95SAlasdair G Kergon int r; 26481da177e4SLinus Torvalds 2649040f04bdSChristoph Hellwig WARN_ON(test_bit(DMF_FROZEN, &md->flags)); 2650dfbe03f6SAlasdair G Kergon 2651977115c0SChristoph Hellwig r = freeze_bdev(md->disk->part0); 2652040f04bdSChristoph Hellwig if (!r) 2653aa8d7c2fSAlasdair G Kergon set_bit(DMF_FROZEN, &md->flags); 2654040f04bdSChristoph Hellwig return r; 26551da177e4SLinus Torvalds } 26561da177e4SLinus Torvalds 26572ca3310eSAlasdair G Kergon static void unlock_fs(struct mapped_device *md) 26581da177e4SLinus Torvalds { 2659aa8d7c2fSAlasdair G Kergon if (!test_bit(DMF_FROZEN, &md->flags)) 2660aa8d7c2fSAlasdair G Kergon return; 2661977115c0SChristoph Hellwig thaw_bdev(md->disk->part0); 2662aa8d7c2fSAlasdair G Kergon clear_bit(DMF_FROZEN, &md->flags); 26631da177e4SLinus Torvalds } 26641da177e4SLinus Torvalds 26651da177e4SLinus Torvalds /* 2666b48633f8SBart Van Assche * @suspend_flags: DM_SUSPEND_LOCKFS_FLAG and/or DM_SUSPEND_NOFLUSH_FLAG 2667b48633f8SBart Van Assche * @task_state: e.g. TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE 2668b48633f8SBart Van Assche * @dmf_suspended_flag: DMF_SUSPENDED or DMF_SUSPENDED_INTERNALLY 2669b48633f8SBart Van Assche * 2670ffcc3936SMike Snitzer * If __dm_suspend returns 0, the device is completely quiescent 2671ffcc3936SMike Snitzer * now. There is no request-processing activity. All new requests 2672ffcc3936SMike Snitzer * are being added to md->deferred list. 2673cec47e3dSKiyoshi Ueda */ 2674ffcc3936SMike Snitzer static int __dm_suspend(struct mapped_device *md, struct dm_table *map, 267586a3238cSHeinz Mauelshagen unsigned int suspend_flags, unsigned int task_state, 2676eaf9a736SMike Snitzer int dmf_suspended_flag) 26771da177e4SLinus Torvalds { 2678ffcc3936SMike Snitzer bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG; 2679ffcc3936SMike Snitzer bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG; 2680ffcc3936SMike Snitzer int r; 2681cf222b37SAlasdair G Kergon 26825a8f1f80SBart Van Assche lockdep_assert_held(&md->suspend_lock); 26835a8f1f80SBart Van Assche 26842e93ccc1SKiyoshi Ueda /* 26852e93ccc1SKiyoshi Ueda * DMF_NOFLUSH_SUSPENDING must be set before presuspend. 26862e93ccc1SKiyoshi Ueda * This flag is cleared before dm_suspend returns. 26872e93ccc1SKiyoshi Ueda */ 26882e93ccc1SKiyoshi Ueda if (noflush) 26892e93ccc1SKiyoshi Ueda set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 269086331f39SBart Van Assche else 2691ac75b09fSMike Snitzer DMDEBUG("%s: suspending with flush", dm_device_name(md)); 26922e93ccc1SKiyoshi Ueda 2693d67ee213SMike Snitzer /* 2694d67ee213SMike Snitzer * This gets reverted if there's an error later and the targets 2695d67ee213SMike Snitzer * provide the .presuspend_undo hook. 2696d67ee213SMike Snitzer */ 26971da177e4SLinus Torvalds dm_table_presuspend_targets(map); 26981da177e4SLinus Torvalds 26992e93ccc1SKiyoshi Ueda /* 27009f518b27SKiyoshi Ueda * Flush I/O to the device. 27019f518b27SKiyoshi Ueda * Any I/O submitted after lock_fs() may not be flushed. 27029f518b27SKiyoshi Ueda * noflush takes precedence over do_lockfs. 27039f518b27SKiyoshi Ueda * (lock_fs() flushes I/Os and waits for them to complete.) 27042e93ccc1SKiyoshi Ueda */ 270532a926daSMikulas Patocka if (!noflush && do_lockfs) { 27062ca3310eSAlasdair G Kergon r = lock_fs(md); 2707d67ee213SMike Snitzer if (r) { 2708d67ee213SMike Snitzer dm_table_presuspend_undo_targets(map); 2709ffcc3936SMike Snitzer return r; 2710aa8d7c2fSAlasdair G Kergon } 2711d67ee213SMike Snitzer } 27121da177e4SLinus Torvalds 27131da177e4SLinus Torvalds /* 27143b00b203SMikulas Patocka * Here we must make sure that no processes are submitting requests 27153b00b203SMikulas Patocka * to target drivers i.e. no one may be executing 271696c9865cSMike Snitzer * dm_split_and_process_bio from dm_submit_bio. 27173b00b203SMikulas Patocka * 271896c9865cSMike Snitzer * To get all processes out of dm_split_and_process_bio in dm_submit_bio, 27193b00b203SMikulas Patocka * we take the write lock. To prevent any process from reentering 272096c9865cSMike Snitzer * dm_split_and_process_bio from dm_submit_bio and quiesce the thread 27210cede372SMike Snitzer * (dm_wq_work), we set DMF_BLOCK_IO_FOR_SUSPEND and call 27226a8736d1STejun Heo * flush_workqueue(md->wq). 27231da177e4SLinus Torvalds */ 27241eb787ecSAlasdair G Kergon set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 272541abc4e1SHannes Reinecke if (map) 272683d5e5b0SMikulas Patocka synchronize_srcu(&md->io_barrier); 27271da177e4SLinus Torvalds 2728d0bcb878SKiyoshi Ueda /* 272929e4013dSTejun Heo * Stop md->queue before flushing md->wq in case request-based 273029e4013dSTejun Heo * dm defers requests to md->wq from md->queue. 2731d0bcb878SKiyoshi Ueda */ 27326a23e05cSJens Axboe if (dm_request_based(md)) 2733eca7ee6dSMike Snitzer dm_stop_queue(md->queue); 2734cec47e3dSKiyoshi Ueda 2735d0bcb878SKiyoshi Ueda flush_workqueue(md->wq); 2736d0bcb878SKiyoshi Ueda 27371da177e4SLinus Torvalds /* 27383b00b203SMikulas Patocka * At this point no more requests are entering target request routines. 27393b00b203SMikulas Patocka * We call dm_wait_for_completion to wait for all existing requests 27403b00b203SMikulas Patocka * to finish. 27411da177e4SLinus Torvalds */ 2742b48633f8SBart Van Assche r = dm_wait_for_completion(md, task_state); 2743eaf9a736SMike Snitzer if (!r) 2744eaf9a736SMike Snitzer set_bit(dmf_suspended_flag, &md->flags); 27451da177e4SLinus Torvalds 27466d6f10dfSMilan Broz if (noflush) 2747022c2611SMikulas Patocka clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 274841abc4e1SHannes Reinecke if (map) 274983d5e5b0SMikulas Patocka synchronize_srcu(&md->io_barrier); 27502e93ccc1SKiyoshi Ueda 27511da177e4SLinus Torvalds /* were we interrupted ? */ 275246125c1cSMilan Broz if (r < 0) { 27539a1fb464SMikulas Patocka dm_queue_flush(md); 275473d410c0SMilan Broz 2755cec47e3dSKiyoshi Ueda if (dm_request_based(md)) 2756eca7ee6dSMike Snitzer dm_start_queue(md->queue); 2757cec47e3dSKiyoshi Ueda 27582ca3310eSAlasdair G Kergon unlock_fs(md); 2759d67ee213SMike Snitzer dm_table_presuspend_undo_targets(map); 2760ffcc3936SMike Snitzer /* pushback list is already flushed, so skip flush */ 2761ffcc3936SMike Snitzer } 2762ffcc3936SMike Snitzer 2763ffcc3936SMike Snitzer return r; 27642ca3310eSAlasdair G Kergon } 27652ca3310eSAlasdair G Kergon 27663b00b203SMikulas Patocka /* 2767ffcc3936SMike Snitzer * We need to be able to change a mapping table under a mounted 2768ffcc3936SMike Snitzer * filesystem. For example we might want to move some data in 2769ffcc3936SMike Snitzer * the background. Before the table can be swapped with 2770ffcc3936SMike Snitzer * dm_bind_table, dm_suspend must be called to flush any in 2771ffcc3936SMike Snitzer * flight bios and ensure that any further io gets deferred. 27723b00b203SMikulas Patocka */ 2773ffcc3936SMike Snitzer /* 2774ffcc3936SMike Snitzer * Suspend mechanism in request-based dm. 2775ffcc3936SMike Snitzer * 2776ffcc3936SMike Snitzer * 1. Flush all I/Os by lock_fs() if needed. 2777ffcc3936SMike Snitzer * 2. Stop dispatching any I/O by stopping the request_queue. 2778ffcc3936SMike Snitzer * 3. Wait for all in-flight I/Os to be completed or requeued. 2779ffcc3936SMike Snitzer * 2780ffcc3936SMike Snitzer * To abort suspend, start the request_queue. 2781ffcc3936SMike Snitzer */ 278286a3238cSHeinz Mauelshagen int dm_suspend(struct mapped_device *md, unsigned int suspend_flags) 2783ffcc3936SMike Snitzer { 2784ffcc3936SMike Snitzer struct dm_table *map = NULL; 2785ffcc3936SMike Snitzer int r = 0; 2786ffcc3936SMike Snitzer 2787ffcc3936SMike Snitzer retry: 2788ffcc3936SMike Snitzer mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); 2789ffcc3936SMike Snitzer 2790ffcc3936SMike Snitzer if (dm_suspended_md(md)) { 2791ffcc3936SMike Snitzer r = -EINVAL; 2792ffcc3936SMike Snitzer goto out_unlock; 2793ffcc3936SMike Snitzer } 2794ffcc3936SMike Snitzer 2795ffcc3936SMike Snitzer if (dm_suspended_internally_md(md)) { 2796ffcc3936SMike Snitzer /* already internally suspended, wait for internal resume */ 2797ffcc3936SMike Snitzer mutex_unlock(&md->suspend_lock); 2798ffcc3936SMike Snitzer r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); 2799ffcc3936SMike Snitzer if (r) 2800ffcc3936SMike Snitzer return r; 2801ffcc3936SMike Snitzer goto retry; 2802ffcc3936SMike Snitzer } 2803ffcc3936SMike Snitzer 2804a12f5d48SEric Dumazet map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 28052760904dSLi Lingfeng if (!map) { 28062760904dSLi Lingfeng /* avoid deadlock with fs/namespace.c:do_mount() */ 28072760904dSLi Lingfeng suspend_flags &= ~DM_SUSPEND_LOCKFS_FLAG; 28082760904dSLi Lingfeng } 2809ffcc3936SMike Snitzer 2810eaf9a736SMike Snitzer r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED); 2811ffcc3936SMike Snitzer if (r) 2812ffcc3936SMike Snitzer goto out_unlock; 28133b00b203SMikulas Patocka 28145df96f2bSMikulas Patocka set_bit(DMF_POST_SUSPENDING, &md->flags); 28154d4471cbSKiyoshi Ueda dm_table_postsuspend_targets(map); 28165df96f2bSMikulas Patocka clear_bit(DMF_POST_SUSPENDING, &md->flags); 28174d4471cbSKiyoshi Ueda 2818d287483dSAlasdair G Kergon out_unlock: 2819e61290a4SDaniel Walker mutex_unlock(&md->suspend_lock); 2820cf222b37SAlasdair G Kergon return r; 28211da177e4SLinus Torvalds } 28221da177e4SLinus Torvalds 2823ffcc3936SMike Snitzer static int __dm_resume(struct mapped_device *md, struct dm_table *map) 28241da177e4SLinus Torvalds { 2825ffcc3936SMike Snitzer if (map) { 2826ffcc3936SMike Snitzer int r = dm_table_resume_targets(map); 2827b30f1607SHeinz Mauelshagen 28288757b776SMilan Broz if (r) 2829ffcc3936SMike Snitzer return r; 2830ffcc3936SMike Snitzer } 28312ca3310eSAlasdair G Kergon 28329a1fb464SMikulas Patocka dm_queue_flush(md); 28332ca3310eSAlasdair G Kergon 2834cec47e3dSKiyoshi Ueda /* 2835cec47e3dSKiyoshi Ueda * Flushing deferred I/Os must be done after targets are resumed 2836cec47e3dSKiyoshi Ueda * so that mapping of targets can work correctly. 2837cec47e3dSKiyoshi Ueda * Request-based dm is queueing the deferred I/Os in its request_queue. 2838cec47e3dSKiyoshi Ueda */ 2839cec47e3dSKiyoshi Ueda if (dm_request_based(md)) 2840eca7ee6dSMike Snitzer dm_start_queue(md->queue); 2841cec47e3dSKiyoshi Ueda 28422ca3310eSAlasdair G Kergon unlock_fs(md); 28432ca3310eSAlasdair G Kergon 2844ffcc3936SMike Snitzer return 0; 2845ffcc3936SMike Snitzer } 2846ffcc3936SMike Snitzer 2847ffcc3936SMike Snitzer int dm_resume(struct mapped_device *md) 2848ffcc3936SMike Snitzer { 28498dc23658SMinfei Huang int r; 2850ffcc3936SMike Snitzer struct dm_table *map = NULL; 2851ffcc3936SMike Snitzer 2852ffcc3936SMike Snitzer retry: 28538dc23658SMinfei Huang r = -EINVAL; 2854ffcc3936SMike Snitzer mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); 2855ffcc3936SMike Snitzer 2856ffcc3936SMike Snitzer if (!dm_suspended_md(md)) 2857ffcc3936SMike Snitzer goto out; 2858ffcc3936SMike Snitzer 2859ffcc3936SMike Snitzer if (dm_suspended_internally_md(md)) { 2860ffcc3936SMike Snitzer /* already internally suspended, wait for internal resume */ 2861ffcc3936SMike Snitzer mutex_unlock(&md->suspend_lock); 2862ffcc3936SMike Snitzer r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); 2863ffcc3936SMike Snitzer if (r) 2864ffcc3936SMike Snitzer return r; 2865ffcc3936SMike Snitzer goto retry; 2866ffcc3936SMike Snitzer } 2867ffcc3936SMike Snitzer 2868a12f5d48SEric Dumazet map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2869ffcc3936SMike Snitzer if (!map || !dm_table_get_size(map)) 2870ffcc3936SMike Snitzer goto out; 2871ffcc3936SMike Snitzer 2872ffcc3936SMike Snitzer r = __dm_resume(md, map); 2873ffcc3936SMike Snitzer if (r) 2874ffcc3936SMike Snitzer goto out; 2875ffcc3936SMike Snitzer 28762ca3310eSAlasdair G Kergon clear_bit(DMF_SUSPENDED, &md->flags); 2877cf222b37SAlasdair G Kergon out: 2878e61290a4SDaniel Walker mutex_unlock(&md->suspend_lock); 28792ca3310eSAlasdair G Kergon 2880cf222b37SAlasdair G Kergon return r; 28811da177e4SLinus Torvalds } 28821da177e4SLinus Torvalds 2883fd2ed4d2SMikulas Patocka /* 2884fd2ed4d2SMikulas Patocka * Internal suspend/resume works like userspace-driven suspend. It waits 2885fd2ed4d2SMikulas Patocka * until all bios finish and prevents issuing new bios to the target drivers. 2886fd2ed4d2SMikulas Patocka * It may be used only from the kernel. 2887fd2ed4d2SMikulas Patocka */ 2888fd2ed4d2SMikulas Patocka 288986a3238cSHeinz Mauelshagen static void __dm_internal_suspend(struct mapped_device *md, unsigned int suspend_flags) 2890ffcc3936SMike Snitzer { 2891ffcc3936SMike Snitzer struct dm_table *map = NULL; 2892ffcc3936SMike Snitzer 28931ea0654eSBart Van Assche lockdep_assert_held(&md->suspend_lock); 28941ea0654eSBart Van Assche 289596b26c8cSMikulas Patocka if (md->internal_suspend_count++) 2896ffcc3936SMike Snitzer return; /* nested internal suspend */ 2897ffcc3936SMike Snitzer 2898ffcc3936SMike Snitzer if (dm_suspended_md(md)) { 2899ffcc3936SMike Snitzer set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 2900ffcc3936SMike Snitzer return; /* nest suspend */ 2901ffcc3936SMike Snitzer } 2902ffcc3936SMike Snitzer 2903a12f5d48SEric Dumazet map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2904ffcc3936SMike Snitzer 2905ffcc3936SMike Snitzer /* 2906ffcc3936SMike Snitzer * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is 2907ffcc3936SMike Snitzer * supported. Properly supporting a TASK_INTERRUPTIBLE internal suspend 2908ffcc3936SMike Snitzer * would require changing .presuspend to return an error -- avoid this 2909ffcc3936SMike Snitzer * until there is a need for more elaborate variants of internal suspend. 2910ffcc3936SMike Snitzer */ 2911eaf9a736SMike Snitzer (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE, 2912eaf9a736SMike Snitzer DMF_SUSPENDED_INTERNALLY); 2913ffcc3936SMike Snitzer 29145df96f2bSMikulas Patocka set_bit(DMF_POST_SUSPENDING, &md->flags); 2915ffcc3936SMike Snitzer dm_table_postsuspend_targets(map); 29165df96f2bSMikulas Patocka clear_bit(DMF_POST_SUSPENDING, &md->flags); 2917ffcc3936SMike Snitzer } 2918ffcc3936SMike Snitzer 2919ffcc3936SMike Snitzer static void __dm_internal_resume(struct mapped_device *md) 2920ffcc3936SMike Snitzer { 2921*15a3fc5cSMikulas Patocka int r; 2922*15a3fc5cSMikulas Patocka struct dm_table *map; 2923*15a3fc5cSMikulas Patocka 292496b26c8cSMikulas Patocka BUG_ON(!md->internal_suspend_count); 292596b26c8cSMikulas Patocka 292696b26c8cSMikulas Patocka if (--md->internal_suspend_count) 2927ffcc3936SMike Snitzer return; /* resume from nested internal suspend */ 2928ffcc3936SMike Snitzer 2929ffcc3936SMike Snitzer if (dm_suspended_md(md)) 2930ffcc3936SMike Snitzer goto done; /* resume from nested suspend */ 2931ffcc3936SMike Snitzer 2932*15a3fc5cSMikulas Patocka map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2933*15a3fc5cSMikulas Patocka r = __dm_resume(md, map); 2934*15a3fc5cSMikulas Patocka if (r) { 2935ffcc3936SMike Snitzer /* 2936*15a3fc5cSMikulas Patocka * If a preresume method of some target failed, we are in a 2937*15a3fc5cSMikulas Patocka * tricky situation. We can't return an error to the caller. We 2938*15a3fc5cSMikulas Patocka * can't fake success because then the "resume" and 2939*15a3fc5cSMikulas Patocka * "postsuspend" methods would not be paired correctly, and it 2940*15a3fc5cSMikulas Patocka * would break various targets, for example it would cause list 2941*15a3fc5cSMikulas Patocka * corruption in the "origin" target. 2942*15a3fc5cSMikulas Patocka * 2943*15a3fc5cSMikulas Patocka * So, we fake normal suspend here, to make sure that the 2944*15a3fc5cSMikulas Patocka * "resume" and "postsuspend" methods will be paired correctly. 2945ffcc3936SMike Snitzer */ 2946*15a3fc5cSMikulas Patocka DMERR("Preresume method failed: %d", r); 2947*15a3fc5cSMikulas Patocka set_bit(DMF_SUSPENDED, &md->flags); 2948*15a3fc5cSMikulas Patocka } 2949ffcc3936SMike Snitzer done: 2950ffcc3936SMike Snitzer clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 2951ffcc3936SMike Snitzer smp_mb__after_atomic(); 2952ffcc3936SMike Snitzer wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY); 2953ffcc3936SMike Snitzer } 2954ffcc3936SMike Snitzer 2955ffcc3936SMike Snitzer void dm_internal_suspend_noflush(struct mapped_device *md) 2956fd2ed4d2SMikulas Patocka { 2957fd2ed4d2SMikulas Patocka mutex_lock(&md->suspend_lock); 2958ffcc3936SMike Snitzer __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG); 2959ffcc3936SMike Snitzer mutex_unlock(&md->suspend_lock); 2960ffcc3936SMike Snitzer } 2961ffcc3936SMike Snitzer EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush); 2962ffcc3936SMike Snitzer 2963ffcc3936SMike Snitzer void dm_internal_resume(struct mapped_device *md) 2964ffcc3936SMike Snitzer { 2965ffcc3936SMike Snitzer mutex_lock(&md->suspend_lock); 2966ffcc3936SMike Snitzer __dm_internal_resume(md); 2967ffcc3936SMike Snitzer mutex_unlock(&md->suspend_lock); 2968ffcc3936SMike Snitzer } 2969ffcc3936SMike Snitzer EXPORT_SYMBOL_GPL(dm_internal_resume); 2970ffcc3936SMike Snitzer 2971ffcc3936SMike Snitzer /* 2972ffcc3936SMike Snitzer * Fast variants of internal suspend/resume hold md->suspend_lock, 2973ffcc3936SMike Snitzer * which prevents interaction with userspace-driven suspend. 2974ffcc3936SMike Snitzer */ 2975ffcc3936SMike Snitzer 2976ffcc3936SMike Snitzer void dm_internal_suspend_fast(struct mapped_device *md) 2977ffcc3936SMike Snitzer { 2978ffcc3936SMike Snitzer mutex_lock(&md->suspend_lock); 2979ffcc3936SMike Snitzer if (dm_suspended_md(md) || dm_suspended_internally_md(md)) 2980fd2ed4d2SMikulas Patocka return; 2981fd2ed4d2SMikulas Patocka 2982fd2ed4d2SMikulas Patocka set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 2983fd2ed4d2SMikulas Patocka synchronize_srcu(&md->io_barrier); 2984fd2ed4d2SMikulas Patocka flush_workqueue(md->wq); 2985fd2ed4d2SMikulas Patocka dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE); 2986fd2ed4d2SMikulas Patocka } 2987b735fedeSMikulas Patocka EXPORT_SYMBOL_GPL(dm_internal_suspend_fast); 2988fd2ed4d2SMikulas Patocka 2989ffcc3936SMike Snitzer void dm_internal_resume_fast(struct mapped_device *md) 2990fd2ed4d2SMikulas Patocka { 2991ffcc3936SMike Snitzer if (dm_suspended_md(md) || dm_suspended_internally_md(md)) 2992fd2ed4d2SMikulas Patocka goto done; 2993fd2ed4d2SMikulas Patocka 2994fd2ed4d2SMikulas Patocka dm_queue_flush(md); 2995fd2ed4d2SMikulas Patocka 2996fd2ed4d2SMikulas Patocka done: 2997fd2ed4d2SMikulas Patocka mutex_unlock(&md->suspend_lock); 2998fd2ed4d2SMikulas Patocka } 2999b735fedeSMikulas Patocka EXPORT_SYMBOL_GPL(dm_internal_resume_fast); 3000fd2ed4d2SMikulas Patocka 3001a4a82ce3SHeinz Mauelshagen /* 3002a4a82ce3SHeinz Mauelshagen *--------------------------------------------------------------- 30031da177e4SLinus Torvalds * Event notification. 3004a4a82ce3SHeinz Mauelshagen *--------------------------------------------------------------- 3005a4a82ce3SHeinz Mauelshagen */ 30063abf85b5SPeter Rajnoha int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action, 300786a3238cSHeinz Mauelshagen unsigned int cookie, bool need_resize_uevent) 300869267a30SAlasdair G Kergon { 30096958c1c6SMikulas Patocka int r; 301086a3238cSHeinz Mauelshagen unsigned int noio_flag; 301160935eb2SMilan Broz char udev_cookie[DM_COOKIE_LENGTH]; 30127533afa1SMikulas Patocka char *envp[3] = { NULL, NULL, NULL }; 30137533afa1SMikulas Patocka char **envpp = envp; 30147533afa1SMikulas Patocka if (cookie) { 30157533afa1SMikulas Patocka snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u", 30167533afa1SMikulas Patocka DM_COOKIE_ENV_VAR_NAME, cookie); 30177533afa1SMikulas Patocka *envpp++ = udev_cookie; 30187533afa1SMikulas Patocka } 30197533afa1SMikulas Patocka if (need_resize_uevent) { 30207533afa1SMikulas Patocka *envpp++ = "RESIZE=1"; 30217533afa1SMikulas Patocka } 302260935eb2SMilan Broz 30236958c1c6SMikulas Patocka noio_flag = memalloc_noio_save(); 30246958c1c6SMikulas Patocka 30257533afa1SMikulas Patocka r = kobject_uevent_env(&disk_to_dev(md->disk)->kobj, action, envp); 30266958c1c6SMikulas Patocka 30276958c1c6SMikulas Patocka memalloc_noio_restore(noio_flag); 30286958c1c6SMikulas Patocka 30296958c1c6SMikulas Patocka return r; 303069267a30SAlasdair G Kergon } 303169267a30SAlasdair G Kergon 30327a8c3d3bSMike Anderson uint32_t dm_next_uevent_seq(struct mapped_device *md) 30337a8c3d3bSMike Anderson { 30347a8c3d3bSMike Anderson return atomic_add_return(1, &md->uevent_seq); 30357a8c3d3bSMike Anderson } 30367a8c3d3bSMike Anderson 30371da177e4SLinus Torvalds uint32_t dm_get_event_nr(struct mapped_device *md) 30381da177e4SLinus Torvalds { 30391da177e4SLinus Torvalds return atomic_read(&md->event_nr); 30401da177e4SLinus Torvalds } 30411da177e4SLinus Torvalds 30421da177e4SLinus Torvalds int dm_wait_event(struct mapped_device *md, int event_nr) 30431da177e4SLinus Torvalds { 30441da177e4SLinus Torvalds return wait_event_interruptible(md->eventq, 30451da177e4SLinus Torvalds (event_nr != atomic_read(&md->event_nr))); 30461da177e4SLinus Torvalds } 30471da177e4SLinus Torvalds 30487a8c3d3bSMike Anderson void dm_uevent_add(struct mapped_device *md, struct list_head *elist) 30497a8c3d3bSMike Anderson { 30507a8c3d3bSMike Anderson unsigned long flags; 30517a8c3d3bSMike Anderson 30527a8c3d3bSMike Anderson spin_lock_irqsave(&md->uevent_lock, flags); 30537a8c3d3bSMike Anderson list_add(elist, &md->uevent_list); 30547a8c3d3bSMike Anderson spin_unlock_irqrestore(&md->uevent_lock, flags); 30557a8c3d3bSMike Anderson } 30567a8c3d3bSMike Anderson 30571da177e4SLinus Torvalds /* 30581da177e4SLinus Torvalds * The gendisk is only valid as long as you have a reference 30591da177e4SLinus Torvalds * count on 'md'. 30601da177e4SLinus Torvalds */ 30611da177e4SLinus Torvalds struct gendisk *dm_disk(struct mapped_device *md) 30621da177e4SLinus Torvalds { 30631da177e4SLinus Torvalds return md->disk; 30641da177e4SLinus Torvalds } 306565ff5b7dSSami Tolvanen EXPORT_SYMBOL_GPL(dm_disk); 30661da177e4SLinus Torvalds 3067784aae73SMilan Broz struct kobject *dm_kobject(struct mapped_device *md) 3068784aae73SMilan Broz { 30692995fa78SMikulas Patocka return &md->kobj_holder.kobj; 3070784aae73SMilan Broz } 3071784aae73SMilan Broz 3072784aae73SMilan Broz struct mapped_device *dm_get_from_kobject(struct kobject *kobj) 3073784aae73SMilan Broz { 3074784aae73SMilan Broz struct mapped_device *md; 3075784aae73SMilan Broz 30762995fa78SMikulas Patocka md = container_of(kobj, struct mapped_device, kobj_holder.kobj); 3077784aae73SMilan Broz 3078b9a41d21SHou Tao spin_lock(&_minor_lock); 3079b9a41d21SHou Tao if (test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) { 3080b9a41d21SHou Tao md = NULL; 3081b9a41d21SHou Tao goto out; 3082b9a41d21SHou Tao } 3083784aae73SMilan Broz dm_get(md); 3084b9a41d21SHou Tao out: 3085b9a41d21SHou Tao spin_unlock(&_minor_lock); 3086b9a41d21SHou Tao 3087784aae73SMilan Broz return md; 3088784aae73SMilan Broz } 3089784aae73SMilan Broz 30904f186f8bSKiyoshi Ueda int dm_suspended_md(struct mapped_device *md) 30911da177e4SLinus Torvalds { 30921da177e4SLinus Torvalds return test_bit(DMF_SUSPENDED, &md->flags); 30931da177e4SLinus Torvalds } 30941da177e4SLinus Torvalds 30955df96f2bSMikulas Patocka static int dm_post_suspending_md(struct mapped_device *md) 30965df96f2bSMikulas Patocka { 30975df96f2bSMikulas Patocka return test_bit(DMF_POST_SUSPENDING, &md->flags); 30985df96f2bSMikulas Patocka } 30995df96f2bSMikulas Patocka 3100ffcc3936SMike Snitzer int dm_suspended_internally_md(struct mapped_device *md) 3101ffcc3936SMike Snitzer { 3102ffcc3936SMike Snitzer return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 3103ffcc3936SMike Snitzer } 3104ffcc3936SMike Snitzer 31052c140a24SMikulas Patocka int dm_test_deferred_remove_flag(struct mapped_device *md) 31062c140a24SMikulas Patocka { 31072c140a24SMikulas Patocka return test_bit(DMF_DEFERRED_REMOVE, &md->flags); 31082c140a24SMikulas Patocka } 31092c140a24SMikulas Patocka 311064dbce58SKiyoshi Ueda int dm_suspended(struct dm_target *ti) 311164dbce58SKiyoshi Ueda { 311233bd6f06SMike Snitzer return dm_suspended_md(ti->table->md); 311364dbce58SKiyoshi Ueda } 311464dbce58SKiyoshi Ueda EXPORT_SYMBOL_GPL(dm_suspended); 311564dbce58SKiyoshi Ueda 31165df96f2bSMikulas Patocka int dm_post_suspending(struct dm_target *ti) 31175df96f2bSMikulas Patocka { 311833bd6f06SMike Snitzer return dm_post_suspending_md(ti->table->md); 31195df96f2bSMikulas Patocka } 31205df96f2bSMikulas Patocka EXPORT_SYMBOL_GPL(dm_post_suspending); 31215df96f2bSMikulas Patocka 31222e93ccc1SKiyoshi Ueda int dm_noflush_suspending(struct dm_target *ti) 31232e93ccc1SKiyoshi Ueda { 312433bd6f06SMike Snitzer return __noflush_suspending(ti->table->md); 31252e93ccc1SKiyoshi Ueda } 31262e93ccc1SKiyoshi Ueda EXPORT_SYMBOL_GPL(dm_noflush_suspending); 31272e93ccc1SKiyoshi Ueda 3128e6ee8c0bSKiyoshi Ueda void dm_free_md_mempools(struct dm_md_mempools *pools) 3129e6ee8c0bSKiyoshi Ueda { 3130e6ee8c0bSKiyoshi Ueda if (!pools) 3131e6ee8c0bSKiyoshi Ueda return; 3132e6ee8c0bSKiyoshi Ueda 31336f1c819cSKent Overstreet bioset_exit(&pools->bs); 31346f1c819cSKent Overstreet bioset_exit(&pools->io_bs); 3135e6ee8c0bSKiyoshi Ueda 3136e6ee8c0bSKiyoshi Ueda kfree(pools); 3137e6ee8c0bSKiyoshi Ueda } 3138e6ee8c0bSKiyoshi Ueda 31399c72bad1SChristoph Hellwig struct dm_pr { 31409c72bad1SChristoph Hellwig u64 old_key; 31419c72bad1SChristoph Hellwig u64 new_key; 31429c72bad1SChristoph Hellwig u32 flags; 3143c6adada5SMike Christie bool abort; 31449c72bad1SChristoph Hellwig bool fail_early; 31458dd87f3cSMike Christie int ret; 314670151087SMike Christie enum pr_type type; 31478a8da082SMike Christie struct pr_keys *read_keys; 31488a8da082SMike Christie struct pr_held_reservation *rsv; 31499c72bad1SChristoph Hellwig }; 31509c72bad1SChristoph Hellwig 31519c72bad1SChristoph Hellwig static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn, 31528dd87f3cSMike Christie struct dm_pr *pr) 31539c72bad1SChristoph Hellwig { 31549c72bad1SChristoph Hellwig struct mapped_device *md = bdev->bd_disk->private_data; 31559c72bad1SChristoph Hellwig struct dm_table *table; 31569c72bad1SChristoph Hellwig struct dm_target *ti; 31579c72bad1SChristoph Hellwig int ret = -ENOTTY, srcu_idx; 31589c72bad1SChristoph Hellwig 31599c72bad1SChristoph Hellwig table = dm_get_live_table(md, &srcu_idx); 31609c72bad1SChristoph Hellwig if (!table || !dm_table_get_size(table)) 31619c72bad1SChristoph Hellwig goto out; 31629c72bad1SChristoph Hellwig 31639c72bad1SChristoph Hellwig /* We only support devices that have a single target */ 31642aec377aSMike Snitzer if (table->num_targets != 1) 31659c72bad1SChristoph Hellwig goto out; 31669c72bad1SChristoph Hellwig ti = dm_table_get_target(table, 0); 31679c72bad1SChristoph Hellwig 3168e120a5f1SMike Snitzer if (dm_suspended_md(md)) { 3169e120a5f1SMike Snitzer ret = -EAGAIN; 3170e120a5f1SMike Snitzer goto out; 3171e120a5f1SMike Snitzer } 3172e120a5f1SMike Snitzer 31739c72bad1SChristoph Hellwig ret = -EINVAL; 31749c72bad1SChristoph Hellwig if (!ti->type->iterate_devices) 31759c72bad1SChristoph Hellwig goto out; 31769c72bad1SChristoph Hellwig 31778dd87f3cSMike Christie ti->type->iterate_devices(ti, fn, pr); 31788dd87f3cSMike Christie ret = 0; 31799c72bad1SChristoph Hellwig out: 31809c72bad1SChristoph Hellwig dm_put_live_table(md, srcu_idx); 31819c72bad1SChristoph Hellwig return ret; 31829c72bad1SChristoph Hellwig } 31839c72bad1SChristoph Hellwig 31849c72bad1SChristoph Hellwig /* 31859c72bad1SChristoph Hellwig * For register / unregister we need to manually call out to every path. 31869c72bad1SChristoph Hellwig */ 31879c72bad1SChristoph Hellwig static int __dm_pr_register(struct dm_target *ti, struct dm_dev *dev, 31889c72bad1SChristoph Hellwig sector_t start, sector_t len, void *data) 31899c72bad1SChristoph Hellwig { 31909c72bad1SChristoph Hellwig struct dm_pr *pr = data; 31919c72bad1SChristoph Hellwig const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops; 31928dd87f3cSMike Christie int ret; 31939c72bad1SChristoph Hellwig 31948dd87f3cSMike Christie if (!ops || !ops->pr_register) { 31958dd87f3cSMike Christie pr->ret = -EOPNOTSUPP; 31968dd87f3cSMike Christie return -1; 31978dd87f3cSMike Christie } 31988dd87f3cSMike Christie 31998dd87f3cSMike Christie ret = ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags); 32008dd87f3cSMike Christie if (!ret) 32018dd87f3cSMike Christie return 0; 32028dd87f3cSMike Christie 32038dd87f3cSMike Christie if (!pr->ret) 32048dd87f3cSMike Christie pr->ret = ret; 32058dd87f3cSMike Christie 32068dd87f3cSMike Christie if (pr->fail_early) 32078dd87f3cSMike Christie return -1; 32088dd87f3cSMike Christie 32098dd87f3cSMike Christie return 0; 32109c72bad1SChristoph Hellwig } 32119c72bad1SChristoph Hellwig 321271cdb697SChristoph Hellwig static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key, 321371cdb697SChristoph Hellwig u32 flags) 321471cdb697SChristoph Hellwig { 32159c72bad1SChristoph Hellwig struct dm_pr pr = { 32169c72bad1SChristoph Hellwig .old_key = old_key, 32179c72bad1SChristoph Hellwig .new_key = new_key, 32189c72bad1SChristoph Hellwig .flags = flags, 32199c72bad1SChristoph Hellwig .fail_early = true, 32208dd87f3cSMike Christie .ret = 0, 32219c72bad1SChristoph Hellwig }; 32229c72bad1SChristoph Hellwig int ret; 322371cdb697SChristoph Hellwig 32249c72bad1SChristoph Hellwig ret = dm_call_pr(bdev, __dm_pr_register, &pr); 32258dd87f3cSMike Christie if (ret) { 32268dd87f3cSMike Christie /* Didn't even get to register a path */ 32278dd87f3cSMike Christie return ret; 32288dd87f3cSMike Christie } 32298dd87f3cSMike Christie 32308dd87f3cSMike Christie if (!pr.ret) 32318dd87f3cSMike Christie return 0; 32328dd87f3cSMike Christie ret = pr.ret; 32338dd87f3cSMike Christie 32348dd87f3cSMike Christie if (!new_key) 32358dd87f3cSMike Christie return ret; 32368dd87f3cSMike Christie 32379c72bad1SChristoph Hellwig /* unregister all paths if we failed to register any path */ 32389c72bad1SChristoph Hellwig pr.old_key = new_key; 32399c72bad1SChristoph Hellwig pr.new_key = 0; 32409c72bad1SChristoph Hellwig pr.flags = 0; 32419c72bad1SChristoph Hellwig pr.fail_early = false; 32428dd87f3cSMike Christie (void) dm_call_pr(bdev, __dm_pr_register, &pr); 32439c72bad1SChristoph Hellwig return ret; 324471cdb697SChristoph Hellwig } 3245956a4025SMike Snitzer 324670151087SMike Christie 324770151087SMike Christie static int __dm_pr_reserve(struct dm_target *ti, struct dm_dev *dev, 324870151087SMike Christie sector_t start, sector_t len, void *data) 324970151087SMike Christie { 325070151087SMike Christie struct dm_pr *pr = data; 325170151087SMike Christie const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops; 325270151087SMike Christie 325370151087SMike Christie if (!ops || !ops->pr_reserve) { 325470151087SMike Christie pr->ret = -EOPNOTSUPP; 325570151087SMike Christie return -1; 325670151087SMike Christie } 325770151087SMike Christie 325870151087SMike Christie pr->ret = ops->pr_reserve(dev->bdev, pr->old_key, pr->type, pr->flags); 325970151087SMike Christie if (!pr->ret) 326070151087SMike Christie return -1; 326170151087SMike Christie 326270151087SMike Christie return 0; 326371cdb697SChristoph Hellwig } 326471cdb697SChristoph Hellwig 326571cdb697SChristoph Hellwig static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type, 326671cdb697SChristoph Hellwig u32 flags) 326771cdb697SChristoph Hellwig { 326870151087SMike Christie struct dm_pr pr = { 326970151087SMike Christie .old_key = key, 327070151087SMike Christie .flags = flags, 327170151087SMike Christie .type = type, 327270151087SMike Christie .fail_early = false, 327370151087SMike Christie .ret = 0, 327470151087SMike Christie }; 327570151087SMike Christie int ret; 327671cdb697SChristoph Hellwig 327770151087SMike Christie ret = dm_call_pr(bdev, __dm_pr_reserve, &pr); 327870151087SMike Christie if (ret) 327970151087SMike Christie return ret; 328071cdb697SChristoph Hellwig 328170151087SMike Christie return pr.ret; 328271cdb697SChristoph Hellwig } 328371cdb697SChristoph Hellwig 328408a3c338SMike Christie /* 328508a3c338SMike Christie * If there is a non-All Registrants type of reservation, the release must be 328608a3c338SMike Christie * sent down the holding path. For the cases where there is no reservation or 328708a3c338SMike Christie * the path is not the holder the device will also return success, so we must 328808a3c338SMike Christie * try each path to make sure we got the correct path. 328908a3c338SMike Christie */ 329008a3c338SMike Christie static int __dm_pr_release(struct dm_target *ti, struct dm_dev *dev, 329108a3c338SMike Christie sector_t start, sector_t len, void *data) 329208a3c338SMike Christie { 329308a3c338SMike Christie struct dm_pr *pr = data; 329408a3c338SMike Christie const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops; 329508a3c338SMike Christie 329608a3c338SMike Christie if (!ops || !ops->pr_release) { 329708a3c338SMike Christie pr->ret = -EOPNOTSUPP; 329808a3c338SMike Christie return -1; 329908a3c338SMike Christie } 330008a3c338SMike Christie 330108a3c338SMike Christie pr->ret = ops->pr_release(dev->bdev, pr->old_key, pr->type); 330208a3c338SMike Christie if (pr->ret) 330308a3c338SMike Christie return -1; 330408a3c338SMike Christie 330508a3c338SMike Christie return 0; 330671cdb697SChristoph Hellwig } 330771cdb697SChristoph Hellwig 330871cdb697SChristoph Hellwig static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type) 330971cdb697SChristoph Hellwig { 331008a3c338SMike Christie struct dm_pr pr = { 331108a3c338SMike Christie .old_key = key, 331208a3c338SMike Christie .type = type, 331308a3c338SMike Christie .fail_early = false, 331408a3c338SMike Christie }; 331508a3c338SMike Christie int ret; 331671cdb697SChristoph Hellwig 331708a3c338SMike Christie ret = dm_call_pr(bdev, __dm_pr_release, &pr); 331808a3c338SMike Christie if (ret) 331908a3c338SMike Christie return ret; 332071cdb697SChristoph Hellwig 332108a3c338SMike Christie return pr.ret; 332271cdb697SChristoph Hellwig } 332371cdb697SChristoph Hellwig 3324c6adada5SMike Christie static int __dm_pr_preempt(struct dm_target *ti, struct dm_dev *dev, 3325c6adada5SMike Christie sector_t start, sector_t len, void *data) 3326c6adada5SMike Christie { 3327c6adada5SMike Christie struct dm_pr *pr = data; 3328c6adada5SMike Christie const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops; 3329c6adada5SMike Christie 3330c6adada5SMike Christie if (!ops || !ops->pr_preempt) { 3331c6adada5SMike Christie pr->ret = -EOPNOTSUPP; 3332c6adada5SMike Christie return -1; 3333c6adada5SMike Christie } 3334c6adada5SMike Christie 3335c6adada5SMike Christie pr->ret = ops->pr_preempt(dev->bdev, pr->old_key, pr->new_key, pr->type, 3336c6adada5SMike Christie pr->abort); 3337c6adada5SMike Christie if (!pr->ret) 3338c6adada5SMike Christie return -1; 3339c6adada5SMike Christie 3340c6adada5SMike Christie return 0; 334171cdb697SChristoph Hellwig } 334271cdb697SChristoph Hellwig 334371cdb697SChristoph Hellwig static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key, 334471cdb697SChristoph Hellwig enum pr_type type, bool abort) 334571cdb697SChristoph Hellwig { 3346c6adada5SMike Christie struct dm_pr pr = { 3347c6adada5SMike Christie .new_key = new_key, 3348c6adada5SMike Christie .old_key = old_key, 3349c6adada5SMike Christie .type = type, 3350c6adada5SMike Christie .fail_early = false, 3351c6adada5SMike Christie }; 3352c6adada5SMike Christie int ret; 335371cdb697SChristoph Hellwig 3354c6adada5SMike Christie ret = dm_call_pr(bdev, __dm_pr_preempt, &pr); 3355c6adada5SMike Christie if (ret) 3356c6adada5SMike Christie return ret; 335771cdb697SChristoph Hellwig 3358c6adada5SMike Christie return pr.ret; 335971cdb697SChristoph Hellwig } 336071cdb697SChristoph Hellwig 336171cdb697SChristoph Hellwig static int dm_pr_clear(struct block_device *bdev, u64 key) 336271cdb697SChristoph Hellwig { 336371cdb697SChristoph Hellwig struct mapped_device *md = bdev->bd_disk->private_data; 336471cdb697SChristoph Hellwig const struct pr_ops *ops; 3365971888c4SMike Snitzer int r, srcu_idx; 336671cdb697SChristoph Hellwig 33675bd5e8d8SMike Snitzer r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 336871cdb697SChristoph Hellwig if (r < 0) 3369971888c4SMike Snitzer goto out; 337071cdb697SChristoph Hellwig 337171cdb697SChristoph Hellwig ops = bdev->bd_disk->fops->pr_ops; 337271cdb697SChristoph Hellwig if (ops && ops->pr_clear) 337371cdb697SChristoph Hellwig r = ops->pr_clear(bdev, key); 337471cdb697SChristoph Hellwig else 337571cdb697SChristoph Hellwig r = -EOPNOTSUPP; 3376971888c4SMike Snitzer out: 3377971888c4SMike Snitzer dm_unprepare_ioctl(md, srcu_idx); 337871cdb697SChristoph Hellwig return r; 337971cdb697SChristoph Hellwig } 338071cdb697SChristoph Hellwig 33818a8da082SMike Christie static int __dm_pr_read_keys(struct dm_target *ti, struct dm_dev *dev, 33828a8da082SMike Christie sector_t start, sector_t len, void *data) 33838a8da082SMike Christie { 33848a8da082SMike Christie struct dm_pr *pr = data; 33858a8da082SMike Christie const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops; 33868a8da082SMike Christie 33878a8da082SMike Christie if (!ops || !ops->pr_read_keys) { 33888a8da082SMike Christie pr->ret = -EOPNOTSUPP; 33898a8da082SMike Christie return -1; 33908a8da082SMike Christie } 33918a8da082SMike Christie 33928a8da082SMike Christie pr->ret = ops->pr_read_keys(dev->bdev, pr->read_keys); 33938a8da082SMike Christie if (!pr->ret) 33948a8da082SMike Christie return -1; 33958a8da082SMike Christie 33968a8da082SMike Christie return 0; 33978a8da082SMike Christie } 33988a8da082SMike Christie 33998a8da082SMike Christie static int dm_pr_read_keys(struct block_device *bdev, struct pr_keys *keys) 34008a8da082SMike Christie { 34018a8da082SMike Christie struct dm_pr pr = { 34028a8da082SMike Christie .read_keys = keys, 34038a8da082SMike Christie }; 34048a8da082SMike Christie int ret; 34058a8da082SMike Christie 34068a8da082SMike Christie ret = dm_call_pr(bdev, __dm_pr_read_keys, &pr); 34078a8da082SMike Christie if (ret) 34088a8da082SMike Christie return ret; 34098a8da082SMike Christie 34108a8da082SMike Christie return pr.ret; 34118a8da082SMike Christie } 34128a8da082SMike Christie 34138a8da082SMike Christie static int __dm_pr_read_reservation(struct dm_target *ti, struct dm_dev *dev, 34148a8da082SMike Christie sector_t start, sector_t len, void *data) 34158a8da082SMike Christie { 34168a8da082SMike Christie struct dm_pr *pr = data; 34178a8da082SMike Christie const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops; 34188a8da082SMike Christie 34198a8da082SMike Christie if (!ops || !ops->pr_read_reservation) { 34208a8da082SMike Christie pr->ret = -EOPNOTSUPP; 34218a8da082SMike Christie return -1; 34228a8da082SMike Christie } 34238a8da082SMike Christie 34248a8da082SMike Christie pr->ret = ops->pr_read_reservation(dev->bdev, pr->rsv); 34258a8da082SMike Christie if (!pr->ret) 34268a8da082SMike Christie return -1; 34278a8da082SMike Christie 34288a8da082SMike Christie return 0; 34298a8da082SMike Christie } 34308a8da082SMike Christie 34318a8da082SMike Christie static int dm_pr_read_reservation(struct block_device *bdev, 34328a8da082SMike Christie struct pr_held_reservation *rsv) 34338a8da082SMike Christie { 34348a8da082SMike Christie struct dm_pr pr = { 34358a8da082SMike Christie .rsv = rsv, 34368a8da082SMike Christie }; 34378a8da082SMike Christie int ret; 34388a8da082SMike Christie 34398a8da082SMike Christie ret = dm_call_pr(bdev, __dm_pr_read_reservation, &pr); 34408a8da082SMike Christie if (ret) 34418a8da082SMike Christie return ret; 34428a8da082SMike Christie 34438a8da082SMike Christie return pr.ret; 34448a8da082SMike Christie } 34458a8da082SMike Christie 344671cdb697SChristoph Hellwig static const struct pr_ops dm_pr_ops = { 344771cdb697SChristoph Hellwig .pr_register = dm_pr_register, 344871cdb697SChristoph Hellwig .pr_reserve = dm_pr_reserve, 344971cdb697SChristoph Hellwig .pr_release = dm_pr_release, 345071cdb697SChristoph Hellwig .pr_preempt = dm_pr_preempt, 345171cdb697SChristoph Hellwig .pr_clear = dm_pr_clear, 34528a8da082SMike Christie .pr_read_keys = dm_pr_read_keys, 34538a8da082SMike Christie .pr_read_reservation = dm_pr_read_reservation, 345471cdb697SChristoph Hellwig }; 345571cdb697SChristoph Hellwig 345683d5cde4SAlexey Dobriyan static const struct block_device_operations dm_blk_dops = { 3457c62b37d9SChristoph Hellwig .submit_bio = dm_submit_bio, 3458b99fdcdcSMing Lei .poll_bio = dm_poll_bio, 34591da177e4SLinus Torvalds .open = dm_blk_open, 34601da177e4SLinus Torvalds .release = dm_blk_close, 3461aa129a22SMilan Broz .ioctl = dm_blk_ioctl, 34623ac51e74SDarrick J. Wong .getgeo = dm_blk_getgeo, 3463e76239a3SChristoph Hellwig .report_zones = dm_blk_report_zones, 346471cdb697SChristoph Hellwig .pr_ops = &dm_pr_ops, 34651da177e4SLinus Torvalds .owner = THIS_MODULE 34661da177e4SLinus Torvalds }; 34671da177e4SLinus Torvalds 3468681cc5e8SMike Snitzer static const struct block_device_operations dm_rq_blk_dops = { 3469681cc5e8SMike Snitzer .open = dm_blk_open, 3470681cc5e8SMike Snitzer .release = dm_blk_close, 3471681cc5e8SMike Snitzer .ioctl = dm_blk_ioctl, 3472681cc5e8SMike Snitzer .getgeo = dm_blk_getgeo, 3473681cc5e8SMike Snitzer .pr_ops = &dm_pr_ops, 3474681cc5e8SMike Snitzer .owner = THIS_MODULE 3475681cc5e8SMike Snitzer }; 3476681cc5e8SMike Snitzer 3477f26c5719SDan Williams static const struct dax_operations dm_dax_ops = { 3478f26c5719SDan Williams .direct_access = dm_dax_direct_access, 3479cdf6cdcdSVivek Goyal .zero_page_range = dm_dax_zero_page_range, 3480047218ecSJane Chu .recovery_write = dm_dax_recovery_write, 3481f26c5719SDan Williams }; 3482f26c5719SDan Williams 34831da177e4SLinus Torvalds /* 34841da177e4SLinus Torvalds * module hooks 34851da177e4SLinus Torvalds */ 34861da177e4SLinus Torvalds module_init(dm_init); 34871da177e4SLinus Torvalds module_exit(dm_exit); 34881da177e4SLinus Torvalds 34891da177e4SLinus Torvalds module_param(major, uint, 0); 34901da177e4SLinus Torvalds MODULE_PARM_DESC(major, "The major number of the device mapper"); 3491f4790826SMike Snitzer 34926a808034SHeinz Mauelshagen module_param(reserved_bio_based_ios, uint, 0644); 3493e8603136SMike Snitzer MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools"); 3494e8603136SMike Snitzer 34956a808034SHeinz Mauelshagen module_param(dm_numa_node, int, 0644); 3496115485e8SMike Snitzer MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations"); 3497115485e8SMike Snitzer 34986a808034SHeinz Mauelshagen module_param(swap_bios, int, 0644); 3499a666e5c0SMikulas Patocka MODULE_PARM_DESC(swap_bios, "Maximum allowed inflight swap IOs"); 3500a666e5c0SMikulas Patocka 35011da177e4SLinus Torvalds MODULE_DESCRIPTION(DM_NAME " driver"); 35021da177e4SLinus Torvalds MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>"); 35031da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 3504