11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * Copyright (C) 2001, 2002 Sistina Software (UK) Limited. 3784aae73SMilan Broz * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. 41da177e4SLinus Torvalds * 51da177e4SLinus Torvalds * This file is released under the GPL. 61da177e4SLinus Torvalds */ 71da177e4SLinus Torvalds 84cc96131SMike Snitzer #include "dm-core.h" 94cc96131SMike Snitzer #include "dm-rq.h" 1051e5b2bdSMike Anderson #include "dm-uevent.h" 1191ccbbacSTushar Sugandhi #include "dm-ima.h" 121da177e4SLinus Torvalds 131da177e4SLinus Torvalds #include <linux/init.h> 141da177e4SLinus Torvalds #include <linux/module.h> 1548c9c27bSArjan van de Ven #include <linux/mutex.h> 166958c1c6SMikulas Patocka #include <linux/sched/mm.h> 17174cd4b1SIngo Molnar #include <linux/sched/signal.h> 181da177e4SLinus Torvalds #include <linux/blkpg.h> 191da177e4SLinus Torvalds #include <linux/bio.h> 201da177e4SLinus Torvalds #include <linux/mempool.h> 21f26c5719SDan Williams #include <linux/dax.h> 221da177e4SLinus Torvalds #include <linux/slab.h> 231da177e4SLinus Torvalds #include <linux/idr.h> 247e026c8cSDan Williams #include <linux/uio.h> 253ac51e74SDarrick J. Wong #include <linux/hdreg.h> 263f77316dSKiyoshi Ueda #include <linux/delay.h> 27ffcc3936SMike Snitzer #include <linux/wait.h> 2871cdb697SChristoph Hellwig #include <linux/pr.h> 29b0b4d7c6SElena Reshetova #include <linux/refcount.h> 30c6a564ffSChristoph Hellwig #include <linux/part_stat.h> 31a892c8d5SSatya Tangirala #include <linux/blk-crypto.h> 321e8d44bdSEric Biggers #include <linux/blk-crypto-profile.h> 3355782138SLi Zefan 3472d94861SAlasdair G Kergon #define DM_MSG_PREFIX "core" 3572d94861SAlasdair G Kergon 3660935eb2SMilan Broz /* 3760935eb2SMilan Broz * Cookies are numeric values sent with CHANGE and REMOVE 3860935eb2SMilan Broz * uevents while resuming, removing or renaming the device. 3960935eb2SMilan Broz */ 4060935eb2SMilan Broz #define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE" 4160935eb2SMilan Broz #define DM_COOKIE_LENGTH 24 4260935eb2SMilan Broz 431da177e4SLinus Torvalds static const char *_name = DM_NAME; 441da177e4SLinus Torvalds 451da177e4SLinus Torvalds static unsigned int major = 0; 461da177e4SLinus Torvalds static unsigned int _major = 0; 471da177e4SLinus Torvalds 48d15b774cSAlasdair G Kergon static DEFINE_IDR(_minor_idr); 49d15b774cSAlasdair G Kergon 50f32c10b0SJeff Mahoney static DEFINE_SPINLOCK(_minor_lock); 512c140a24SMikulas Patocka 522c140a24SMikulas Patocka static void do_deferred_remove(struct work_struct *w); 532c140a24SMikulas Patocka 542c140a24SMikulas Patocka static DECLARE_WORK(deferred_remove_work, do_deferred_remove); 552c140a24SMikulas Patocka 56acfe0ad7SMikulas Patocka static struct workqueue_struct *deferred_remove_workqueue; 57acfe0ad7SMikulas Patocka 5893e6442cSMikulas Patocka atomic_t dm_global_event_nr = ATOMIC_INIT(0); 5993e6442cSMikulas Patocka DECLARE_WAIT_QUEUE_HEAD(dm_global_eventq); 6093e6442cSMikulas Patocka 6162e08243SMikulas Patocka void dm_issue_global_event(void) 6262e08243SMikulas Patocka { 6362e08243SMikulas Patocka atomic_inc(&dm_global_event_nr); 6462e08243SMikulas Patocka wake_up(&dm_global_eventq); 6562e08243SMikulas Patocka } 6662e08243SMikulas Patocka 671da177e4SLinus Torvalds /* 6864f52b0eSMike Snitzer * One of these is allocated (on-stack) per original bio. 691da177e4SLinus Torvalds */ 7064f52b0eSMike Snitzer struct clone_info { 7164f52b0eSMike Snitzer struct dm_table *map; 7264f52b0eSMike Snitzer struct bio *bio; 7364f52b0eSMike Snitzer struct dm_io *io; 7464f52b0eSMike Snitzer sector_t sector; 7564f52b0eSMike Snitzer unsigned sector_count; 7664f52b0eSMike Snitzer }; 7764f52b0eSMike Snitzer 7862f26317SJeffle Xu #define DM_TARGET_IO_BIO_OFFSET (offsetof(struct dm_target_io, clone)) 7962f26317SJeffle Xu #define DM_IO_BIO_OFFSET \ 8062f26317SJeffle Xu (offsetof(struct dm_target_io, clone) + offsetof(struct dm_io, tio)) 8162f26317SJeffle Xu 826c23f0bdSChristoph Hellwig static inline struct dm_target_io *clone_to_tio(struct bio *clone) 836c23f0bdSChristoph Hellwig { 846c23f0bdSChristoph Hellwig return container_of(clone, struct dm_target_io, clone); 856c23f0bdSChristoph Hellwig } 866c23f0bdSChristoph Hellwig 8764f52b0eSMike Snitzer void *dm_per_bio_data(struct bio *bio, size_t data_size) 8864f52b0eSMike Snitzer { 896c23f0bdSChristoph Hellwig if (!clone_to_tio(bio)->inside_dm_io) 9062f26317SJeffle Xu return (char *)bio - DM_TARGET_IO_BIO_OFFSET - data_size; 9162f26317SJeffle Xu return (char *)bio - DM_IO_BIO_OFFSET - data_size; 9264f52b0eSMike Snitzer } 9364f52b0eSMike Snitzer EXPORT_SYMBOL_GPL(dm_per_bio_data); 9464f52b0eSMike Snitzer 9564f52b0eSMike Snitzer struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size) 9664f52b0eSMike Snitzer { 9764f52b0eSMike Snitzer struct dm_io *io = (struct dm_io *)((char *)data + data_size); 9864f52b0eSMike Snitzer if (io->magic == DM_IO_MAGIC) 9962f26317SJeffle Xu return (struct bio *)((char *)io + DM_IO_BIO_OFFSET); 10064f52b0eSMike Snitzer BUG_ON(io->magic != DM_TIO_MAGIC); 10162f26317SJeffle Xu return (struct bio *)((char *)io + DM_TARGET_IO_BIO_OFFSET); 10264f52b0eSMike Snitzer } 10364f52b0eSMike Snitzer EXPORT_SYMBOL_GPL(dm_bio_from_per_bio_data); 10464f52b0eSMike Snitzer 10564f52b0eSMike Snitzer unsigned dm_bio_get_target_bio_nr(const struct bio *bio) 10664f52b0eSMike Snitzer { 10764f52b0eSMike Snitzer return container_of(bio, struct dm_target_io, clone)->target_bio_nr; 10864f52b0eSMike Snitzer } 10964f52b0eSMike Snitzer EXPORT_SYMBOL_GPL(dm_bio_get_target_bio_nr); 11064f52b0eSMike Snitzer 111ba61fdd1SJeff Mahoney #define MINOR_ALLOCED ((void *)-1) 112ba61fdd1SJeff Mahoney 113115485e8SMike Snitzer #define DM_NUMA_NODE NUMA_NO_NODE 114115485e8SMike Snitzer static int dm_numa_node = DM_NUMA_NODE; 115faad87dfSMike Snitzer 116a666e5c0SMikulas Patocka #define DEFAULT_SWAP_BIOS (8 * 1048576 / PAGE_SIZE) 117a666e5c0SMikulas Patocka static int swap_bios = DEFAULT_SWAP_BIOS; 118a666e5c0SMikulas Patocka static int get_swap_bios(void) 119a666e5c0SMikulas Patocka { 120a666e5c0SMikulas Patocka int latch = READ_ONCE(swap_bios); 121a666e5c0SMikulas Patocka if (unlikely(latch <= 0)) 122a666e5c0SMikulas Patocka latch = DEFAULT_SWAP_BIOS; 123a666e5c0SMikulas Patocka return latch; 124a666e5c0SMikulas Patocka } 125a666e5c0SMikulas Patocka 126e6ee8c0bSKiyoshi Ueda /* 127e6ee8c0bSKiyoshi Ueda * For mempools pre-allocation at the table loading time. 128e6ee8c0bSKiyoshi Ueda */ 129e6ee8c0bSKiyoshi Ueda struct dm_md_mempools { 1306f1c819cSKent Overstreet struct bio_set bs; 1316f1c819cSKent Overstreet struct bio_set io_bs; 132e6ee8c0bSKiyoshi Ueda }; 133e6ee8c0bSKiyoshi Ueda 13486f1152bSBenjamin Marzinski struct table_device { 13586f1152bSBenjamin Marzinski struct list_head list; 136b0b4d7c6SElena Reshetova refcount_t count; 13786f1152bSBenjamin Marzinski struct dm_dev dm_dev; 13886f1152bSBenjamin Marzinski }; 13986f1152bSBenjamin Marzinski 140f4790826SMike Snitzer /* 141e8603136SMike Snitzer * Bio-based DM's mempools' reserved IOs set by the user. 142e8603136SMike Snitzer */ 1434cc96131SMike Snitzer #define RESERVED_BIO_BASED_IOS 16 144e8603136SMike Snitzer static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS; 145e8603136SMike Snitzer 146115485e8SMike Snitzer static int __dm_get_module_param_int(int *module_param, int min, int max) 147115485e8SMike Snitzer { 1486aa7de05SMark Rutland int param = READ_ONCE(*module_param); 149115485e8SMike Snitzer int modified_param = 0; 150115485e8SMike Snitzer bool modified = true; 151115485e8SMike Snitzer 152115485e8SMike Snitzer if (param < min) 153115485e8SMike Snitzer modified_param = min; 154115485e8SMike Snitzer else if (param > max) 155115485e8SMike Snitzer modified_param = max; 156115485e8SMike Snitzer else 157115485e8SMike Snitzer modified = false; 158115485e8SMike Snitzer 159115485e8SMike Snitzer if (modified) { 160115485e8SMike Snitzer (void)cmpxchg(module_param, param, modified_param); 161115485e8SMike Snitzer param = modified_param; 162115485e8SMike Snitzer } 163115485e8SMike Snitzer 164115485e8SMike Snitzer return param; 165115485e8SMike Snitzer } 166115485e8SMike Snitzer 1674cc96131SMike Snitzer unsigned __dm_get_module_param(unsigned *module_param, 168f4790826SMike Snitzer unsigned def, unsigned max) 169f4790826SMike Snitzer { 1706aa7de05SMark Rutland unsigned param = READ_ONCE(*module_param); 17109c2d531SMike Snitzer unsigned modified_param = 0; 172f4790826SMike Snitzer 17309c2d531SMike Snitzer if (!param) 17409c2d531SMike Snitzer modified_param = def; 17509c2d531SMike Snitzer else if (param > max) 17609c2d531SMike Snitzer modified_param = max; 177f4790826SMike Snitzer 17809c2d531SMike Snitzer if (modified_param) { 17909c2d531SMike Snitzer (void)cmpxchg(module_param, param, modified_param); 18009c2d531SMike Snitzer param = modified_param; 181f4790826SMike Snitzer } 182f4790826SMike Snitzer 18309c2d531SMike Snitzer return param; 184f4790826SMike Snitzer } 185f4790826SMike Snitzer 186e8603136SMike Snitzer unsigned dm_get_reserved_bio_based_ios(void) 187e8603136SMike Snitzer { 18809c2d531SMike Snitzer return __dm_get_module_param(&reserved_bio_based_ios, 1894cc96131SMike Snitzer RESERVED_BIO_BASED_IOS, DM_RESERVED_MAX_IOS); 190e8603136SMike Snitzer } 191e8603136SMike Snitzer EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios); 192e8603136SMike Snitzer 193115485e8SMike Snitzer static unsigned dm_get_numa_node(void) 194115485e8SMike Snitzer { 195115485e8SMike Snitzer return __dm_get_module_param_int(&dm_numa_node, 196115485e8SMike Snitzer DM_NUMA_NODE, num_online_nodes() - 1); 197115485e8SMike Snitzer } 198115485e8SMike Snitzer 1991da177e4SLinus Torvalds static int __init local_init(void) 2001da177e4SLinus Torvalds { 201e689fbabSMike Snitzer int r; 2021ae49ea2SMike Snitzer 20351e5b2bdSMike Anderson r = dm_uevent_init(); 20451157b4aSKiyoshi Ueda if (r) 205e689fbabSMike Snitzer return r; 20651e5b2bdSMike Anderson 207acfe0ad7SMikulas Patocka deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1); 208acfe0ad7SMikulas Patocka if (!deferred_remove_workqueue) { 209acfe0ad7SMikulas Patocka r = -ENOMEM; 210acfe0ad7SMikulas Patocka goto out_uevent_exit; 211acfe0ad7SMikulas Patocka } 212acfe0ad7SMikulas Patocka 2131da177e4SLinus Torvalds _major = major; 2141da177e4SLinus Torvalds r = register_blkdev(_major, _name); 21551157b4aSKiyoshi Ueda if (r < 0) 216acfe0ad7SMikulas Patocka goto out_free_workqueue; 2171da177e4SLinus Torvalds 2181da177e4SLinus Torvalds if (!_major) 2191da177e4SLinus Torvalds _major = r; 2201da177e4SLinus Torvalds 2211da177e4SLinus Torvalds return 0; 22251157b4aSKiyoshi Ueda 223acfe0ad7SMikulas Patocka out_free_workqueue: 224acfe0ad7SMikulas Patocka destroy_workqueue(deferred_remove_workqueue); 22551157b4aSKiyoshi Ueda out_uevent_exit: 22651157b4aSKiyoshi Ueda dm_uevent_exit(); 22751157b4aSKiyoshi Ueda 22851157b4aSKiyoshi Ueda return r; 2291da177e4SLinus Torvalds } 2301da177e4SLinus Torvalds 2311da177e4SLinus Torvalds static void local_exit(void) 2321da177e4SLinus Torvalds { 2332c140a24SMikulas Patocka flush_scheduled_work(); 234acfe0ad7SMikulas Patocka destroy_workqueue(deferred_remove_workqueue); 2352c140a24SMikulas Patocka 23600d59405SAkinobu Mita unregister_blkdev(_major, _name); 23751e5b2bdSMike Anderson dm_uevent_exit(); 2381da177e4SLinus Torvalds 2391da177e4SLinus Torvalds _major = 0; 2401da177e4SLinus Torvalds 2411da177e4SLinus Torvalds DMINFO("cleaned up"); 2421da177e4SLinus Torvalds } 2431da177e4SLinus Torvalds 244b9249e55SAlasdair G Kergon static int (*_inits[])(void) __initdata = { 2451da177e4SLinus Torvalds local_init, 2461da177e4SLinus Torvalds dm_target_init, 2471da177e4SLinus Torvalds dm_linear_init, 2481da177e4SLinus Torvalds dm_stripe_init, 249952b3557SMikulas Patocka dm_io_init, 250945fa4d2SMikulas Patocka dm_kcopyd_init, 2511da177e4SLinus Torvalds dm_interface_init, 252fd2ed4d2SMikulas Patocka dm_statistics_init, 2531da177e4SLinus Torvalds }; 2541da177e4SLinus Torvalds 255b9249e55SAlasdair G Kergon static void (*_exits[])(void) = { 2561da177e4SLinus Torvalds local_exit, 2571da177e4SLinus Torvalds dm_target_exit, 2581da177e4SLinus Torvalds dm_linear_exit, 2591da177e4SLinus Torvalds dm_stripe_exit, 260952b3557SMikulas Patocka dm_io_exit, 261945fa4d2SMikulas Patocka dm_kcopyd_exit, 2621da177e4SLinus Torvalds dm_interface_exit, 263fd2ed4d2SMikulas Patocka dm_statistics_exit, 2641da177e4SLinus Torvalds }; 2651da177e4SLinus Torvalds 2661da177e4SLinus Torvalds static int __init dm_init(void) 2671da177e4SLinus Torvalds { 2681da177e4SLinus Torvalds const int count = ARRAY_SIZE(_inits); 2691da177e4SLinus Torvalds int r, i; 2701da177e4SLinus Torvalds 271f1cd6cb2STushar Sugandhi #if (IS_ENABLED(CONFIG_IMA) && !IS_ENABLED(CONFIG_IMA_DISABLE_HTABLE)) 272f1cd6cb2STushar Sugandhi DMWARN("CONFIG_IMA_DISABLE_HTABLE is disabled." 273f1cd6cb2STushar Sugandhi " Duplicate IMA measurements will not be recorded in the IMA log."); 274f1cd6cb2STushar Sugandhi #endif 275f1cd6cb2STushar Sugandhi 2761da177e4SLinus Torvalds for (i = 0; i < count; i++) { 2771da177e4SLinus Torvalds r = _inits[i](); 2781da177e4SLinus Torvalds if (r) 2791da177e4SLinus Torvalds goto bad; 2801da177e4SLinus Torvalds } 2811da177e4SLinus Torvalds 2821da177e4SLinus Torvalds return 0; 2831da177e4SLinus Torvalds bad: 2841da177e4SLinus Torvalds while (i--) 2851da177e4SLinus Torvalds _exits[i](); 2861da177e4SLinus Torvalds 2871da177e4SLinus Torvalds return r; 2881da177e4SLinus Torvalds } 2891da177e4SLinus Torvalds 2901da177e4SLinus Torvalds static void __exit dm_exit(void) 2911da177e4SLinus Torvalds { 2921da177e4SLinus Torvalds int i = ARRAY_SIZE(_exits); 2931da177e4SLinus Torvalds 2941da177e4SLinus Torvalds while (i--) 2951da177e4SLinus Torvalds _exits[i](); 296d15b774cSAlasdair G Kergon 297d15b774cSAlasdair G Kergon /* 298d15b774cSAlasdair G Kergon * Should be empty by this point. 299d15b774cSAlasdair G Kergon */ 300d15b774cSAlasdair G Kergon idr_destroy(&_minor_idr); 3011da177e4SLinus Torvalds } 3021da177e4SLinus Torvalds 3031da177e4SLinus Torvalds /* 3041da177e4SLinus Torvalds * Block device functions 3051da177e4SLinus Torvalds */ 306432a212cSMike Anderson int dm_deleting_md(struct mapped_device *md) 307432a212cSMike Anderson { 308432a212cSMike Anderson return test_bit(DMF_DELETING, &md->flags); 309432a212cSMike Anderson } 310432a212cSMike Anderson 311fe5f9f2cSAl Viro static int dm_blk_open(struct block_device *bdev, fmode_t mode) 3121da177e4SLinus Torvalds { 3131da177e4SLinus Torvalds struct mapped_device *md; 3141da177e4SLinus Torvalds 315fba9f90eSJeff Mahoney spin_lock(&_minor_lock); 316fba9f90eSJeff Mahoney 317fe5f9f2cSAl Viro md = bdev->bd_disk->private_data; 318fba9f90eSJeff Mahoney if (!md) 319fba9f90eSJeff Mahoney goto out; 320fba9f90eSJeff Mahoney 3215c6bd75dSAlasdair G Kergon if (test_bit(DMF_FREEING, &md->flags) || 322432a212cSMike Anderson dm_deleting_md(md)) { 323fba9f90eSJeff Mahoney md = NULL; 324fba9f90eSJeff Mahoney goto out; 325fba9f90eSJeff Mahoney } 326fba9f90eSJeff Mahoney 3271da177e4SLinus Torvalds dm_get(md); 3285c6bd75dSAlasdair G Kergon atomic_inc(&md->open_count); 329fba9f90eSJeff Mahoney out: 330fba9f90eSJeff Mahoney spin_unlock(&_minor_lock); 331fba9f90eSJeff Mahoney 332fba9f90eSJeff Mahoney return md ? 0 : -ENXIO; 3331da177e4SLinus Torvalds } 3341da177e4SLinus Torvalds 335db2a144bSAl Viro static void dm_blk_close(struct gendisk *disk, fmode_t mode) 3361da177e4SLinus Torvalds { 33763a4f065SMike Snitzer struct mapped_device *md; 3386e9624b8SArnd Bergmann 3394a1aeb98SMilan Broz spin_lock(&_minor_lock); 3404a1aeb98SMilan Broz 34163a4f065SMike Snitzer md = disk->private_data; 34263a4f065SMike Snitzer if (WARN_ON(!md)) 34363a4f065SMike Snitzer goto out; 34463a4f065SMike Snitzer 3452c140a24SMikulas Patocka if (atomic_dec_and_test(&md->open_count) && 3462c140a24SMikulas Patocka (test_bit(DMF_DEFERRED_REMOVE, &md->flags))) 347acfe0ad7SMikulas Patocka queue_work(deferred_remove_workqueue, &deferred_remove_work); 3482c140a24SMikulas Patocka 3491da177e4SLinus Torvalds dm_put(md); 35063a4f065SMike Snitzer out: 3514a1aeb98SMilan Broz spin_unlock(&_minor_lock); 3521da177e4SLinus Torvalds } 3531da177e4SLinus Torvalds 3545c6bd75dSAlasdair G Kergon int dm_open_count(struct mapped_device *md) 3555c6bd75dSAlasdair G Kergon { 3565c6bd75dSAlasdair G Kergon return atomic_read(&md->open_count); 3575c6bd75dSAlasdair G Kergon } 3585c6bd75dSAlasdair G Kergon 3595c6bd75dSAlasdair G Kergon /* 3605c6bd75dSAlasdair G Kergon * Guarantees nothing is using the device before it's deleted. 3615c6bd75dSAlasdair G Kergon */ 3622c140a24SMikulas Patocka int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred) 3635c6bd75dSAlasdair G Kergon { 3645c6bd75dSAlasdair G Kergon int r = 0; 3655c6bd75dSAlasdair G Kergon 3665c6bd75dSAlasdair G Kergon spin_lock(&_minor_lock); 3675c6bd75dSAlasdair G Kergon 3682c140a24SMikulas Patocka if (dm_open_count(md)) { 3695c6bd75dSAlasdair G Kergon r = -EBUSY; 3702c140a24SMikulas Patocka if (mark_deferred) 3712c140a24SMikulas Patocka set_bit(DMF_DEFERRED_REMOVE, &md->flags); 3722c140a24SMikulas Patocka } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags)) 3732c140a24SMikulas Patocka r = -EEXIST; 3745c6bd75dSAlasdair G Kergon else 3755c6bd75dSAlasdair G Kergon set_bit(DMF_DELETING, &md->flags); 3765c6bd75dSAlasdair G Kergon 3775c6bd75dSAlasdair G Kergon spin_unlock(&_minor_lock); 3785c6bd75dSAlasdair G Kergon 3795c6bd75dSAlasdair G Kergon return r; 3805c6bd75dSAlasdair G Kergon } 3815c6bd75dSAlasdair G Kergon 3822c140a24SMikulas Patocka int dm_cancel_deferred_remove(struct mapped_device *md) 3832c140a24SMikulas Patocka { 3842c140a24SMikulas Patocka int r = 0; 3852c140a24SMikulas Patocka 3862c140a24SMikulas Patocka spin_lock(&_minor_lock); 3872c140a24SMikulas Patocka 3882c140a24SMikulas Patocka if (test_bit(DMF_DELETING, &md->flags)) 3892c140a24SMikulas Patocka r = -EBUSY; 3902c140a24SMikulas Patocka else 3912c140a24SMikulas Patocka clear_bit(DMF_DEFERRED_REMOVE, &md->flags); 3922c140a24SMikulas Patocka 3932c140a24SMikulas Patocka spin_unlock(&_minor_lock); 3942c140a24SMikulas Patocka 3952c140a24SMikulas Patocka return r; 3962c140a24SMikulas Patocka } 3972c140a24SMikulas Patocka 3982c140a24SMikulas Patocka static void do_deferred_remove(struct work_struct *w) 3992c140a24SMikulas Patocka { 4002c140a24SMikulas Patocka dm_deferred_remove(); 4012c140a24SMikulas Patocka } 4022c140a24SMikulas Patocka 4033ac51e74SDarrick J. Wong static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo) 4043ac51e74SDarrick J. Wong { 4053ac51e74SDarrick J. Wong struct mapped_device *md = bdev->bd_disk->private_data; 4063ac51e74SDarrick J. Wong 4073ac51e74SDarrick J. Wong return dm_get_geometry(md, geo); 4083ac51e74SDarrick J. Wong } 4093ac51e74SDarrick J. Wong 410971888c4SMike Snitzer static int dm_prepare_ioctl(struct mapped_device *md, int *srcu_idx, 4115bd5e8d8SMike Snitzer struct block_device **bdev) 412aa129a22SMilan Broz { 41366482026SMike Snitzer struct dm_target *tgt; 4146c182cd8SHannes Reinecke struct dm_table *map; 415971888c4SMike Snitzer int r; 416aa129a22SMilan Broz 4176c182cd8SHannes Reinecke retry: 418e56f81e0SChristoph Hellwig r = -ENOTTY; 419971888c4SMike Snitzer map = dm_get_live_table(md, srcu_idx); 420aa129a22SMilan Broz if (!map || !dm_table_get_size(map)) 421971888c4SMike Snitzer return r; 422aa129a22SMilan Broz 423aa129a22SMilan Broz /* We only support devices that have a single target */ 424aa129a22SMilan Broz if (dm_table_get_num_targets(map) != 1) 425971888c4SMike Snitzer return r; 426aa129a22SMilan Broz 42766482026SMike Snitzer tgt = dm_table_get_target(map, 0); 42866482026SMike Snitzer if (!tgt->type->prepare_ioctl) 429e56f81e0SChristoph Hellwig return r; 430aa129a22SMilan Broz 431971888c4SMike Snitzer if (dm_suspended_md(md)) 432971888c4SMike Snitzer return -EAGAIN; 433971888c4SMike Snitzer 4345bd5e8d8SMike Snitzer r = tgt->type->prepare_ioctl(tgt, bdev); 4355bbbfdf6SJunichi Nomura if (r == -ENOTCONN && !fatal_signal_pending(current)) { 436971888c4SMike Snitzer dm_put_live_table(md, *srcu_idx); 4376c182cd8SHannes Reinecke msleep(10); 4386c182cd8SHannes Reinecke goto retry; 4396c182cd8SHannes Reinecke } 440971888c4SMike Snitzer 441e56f81e0SChristoph Hellwig return r; 442e56f81e0SChristoph Hellwig } 4436c182cd8SHannes Reinecke 444971888c4SMike Snitzer static void dm_unprepare_ioctl(struct mapped_device *md, int srcu_idx) 445971888c4SMike Snitzer { 446971888c4SMike Snitzer dm_put_live_table(md, srcu_idx); 447971888c4SMike Snitzer } 448971888c4SMike Snitzer 449e56f81e0SChristoph Hellwig static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode, 450e56f81e0SChristoph Hellwig unsigned int cmd, unsigned long arg) 451e56f81e0SChristoph Hellwig { 452e56f81e0SChristoph Hellwig struct mapped_device *md = bdev->bd_disk->private_data; 453971888c4SMike Snitzer int r, srcu_idx; 454e56f81e0SChristoph Hellwig 4555bd5e8d8SMike Snitzer r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 456e56f81e0SChristoph Hellwig if (r < 0) 457971888c4SMike Snitzer goto out; 458e56f81e0SChristoph Hellwig 459e56f81e0SChristoph Hellwig if (r > 0) { 460e56f81e0SChristoph Hellwig /* 461e980f623SChristoph Hellwig * Target determined this ioctl is being issued against a 462e980f623SChristoph Hellwig * subset of the parent bdev; require extra privileges. 463e56f81e0SChristoph Hellwig */ 464e980f623SChristoph Hellwig if (!capable(CAP_SYS_RAWIO)) { 4650378c625SMike Snitzer DMDEBUG_LIMIT( 466e980f623SChristoph Hellwig "%s: sending ioctl %x to DM device without required privilege.", 467e980f623SChristoph Hellwig current->comm, cmd); 468e980f623SChristoph Hellwig r = -ENOIOCTLCMD; 469e56f81e0SChristoph Hellwig goto out; 470e56f81e0SChristoph Hellwig } 471e980f623SChristoph Hellwig } 472e56f81e0SChristoph Hellwig 473a7cb3d2fSChristoph Hellwig if (!bdev->bd_disk->fops->ioctl) 474a7cb3d2fSChristoph Hellwig r = -ENOTTY; 475a7cb3d2fSChristoph Hellwig else 476a7cb3d2fSChristoph Hellwig r = bdev->bd_disk->fops->ioctl(bdev, mode, cmd, arg); 477e56f81e0SChristoph Hellwig out: 478971888c4SMike Snitzer dm_unprepare_ioctl(md, srcu_idx); 479aa129a22SMilan Broz return r; 480aa129a22SMilan Broz } 481aa129a22SMilan Broz 4827465d7acSMike Snitzer u64 dm_start_time_ns_from_clone(struct bio *bio) 4837465d7acSMike Snitzer { 4846c23f0bdSChristoph Hellwig return jiffies_to_nsecs(clone_to_tio(bio)->io->start_time); 4857465d7acSMike Snitzer } 4867465d7acSMike Snitzer EXPORT_SYMBOL_GPL(dm_start_time_ns_from_clone); 4877465d7acSMike Snitzer 4888d394bc4SMike Snitzer static bool bio_is_flush_with_data(struct bio *bio) 4897465d7acSMike Snitzer { 4908d394bc4SMike Snitzer return ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size); 4917465d7acSMike Snitzer } 4927465d7acSMike Snitzer 4938d394bc4SMike Snitzer static void dm_io_acct(bool end, struct mapped_device *md, struct bio *bio, 494d208b894SJiazi Li unsigned long start_time, struct dm_stats_aux *stats_aux) 4957465d7acSMike Snitzer { 4968d394bc4SMike Snitzer bool is_flush_with_data; 4978d394bc4SMike Snitzer unsigned int bi_size; 4987465d7acSMike Snitzer 4998d394bc4SMike Snitzer /* If REQ_PREFLUSH set save any payload but do not account it */ 5008d394bc4SMike Snitzer is_flush_with_data = bio_is_flush_with_data(bio); 5018d394bc4SMike Snitzer if (is_flush_with_data) { 5028d394bc4SMike Snitzer bi_size = bio->bi_iter.bi_size; 5038d394bc4SMike Snitzer bio->bi_iter.bi_size = 0; 5048d394bc4SMike Snitzer } 5058d394bc4SMike Snitzer 5068d394bc4SMike Snitzer if (!end) 5078d394bc4SMike Snitzer bio_start_io_acct_time(bio, start_time); 5088d394bc4SMike Snitzer else 509d208b894SJiazi Li bio_end_io_acct(bio, start_time); 5107465d7acSMike Snitzer 5117465d7acSMike Snitzer if (unlikely(dm_stats_used(&md->stats))) 5127465d7acSMike Snitzer dm_stats_account_io(&md->stats, bio_data_dir(bio), 5137465d7acSMike Snitzer bio->bi_iter.bi_sector, bio_sectors(bio), 5148d394bc4SMike Snitzer end, start_time, stats_aux); 5158d394bc4SMike Snitzer 5168d394bc4SMike Snitzer /* Restore bio's payload so it does get accounted upon requeue */ 5178d394bc4SMike Snitzer if (is_flush_with_data) 5188d394bc4SMike Snitzer bio->bi_iter.bi_size = bi_size; 5198d394bc4SMike Snitzer } 5208d394bc4SMike Snitzer 5218d394bc4SMike Snitzer static void start_io_acct(struct dm_io *io) 5228d394bc4SMike Snitzer { 5238d394bc4SMike Snitzer dm_io_acct(false, io->md, io->orig_bio, io->start_time, &io->stats_aux); 5248d394bc4SMike Snitzer } 5258d394bc4SMike Snitzer 5268d394bc4SMike Snitzer static void end_io_acct(struct mapped_device *md, struct bio *bio, 5278d394bc4SMike Snitzer unsigned long start_time, struct dm_stats_aux *stats_aux) 5288d394bc4SMike Snitzer { 5298d394bc4SMike Snitzer dm_io_acct(true, md, bio, start_time, stats_aux); 5307465d7acSMike Snitzer } 531978e51baSMike Snitzer 532978e51baSMike Snitzer static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio) 5331da177e4SLinus Torvalds { 53464f52b0eSMike Snitzer struct dm_io *io; 53564f52b0eSMike Snitzer struct dm_target_io *tio; 53664f52b0eSMike Snitzer struct bio *clone; 53764f52b0eSMike Snitzer 538abfc426dSChristoph Hellwig clone = bio_alloc_clone(bio->bi_bdev, bio, GFP_NOIO, &md->io_bs); 53964f52b0eSMike Snitzer 5406c23f0bdSChristoph Hellwig tio = clone_to_tio(clone); 54164f52b0eSMike Snitzer tio->inside_dm_io = true; 54264f52b0eSMike Snitzer tio->io = NULL; 54364f52b0eSMike Snitzer 54464f52b0eSMike Snitzer io = container_of(tio, struct dm_io, tio); 54564f52b0eSMike Snitzer io->magic = DM_IO_MAGIC; 546978e51baSMike Snitzer io->status = 0; 547978e51baSMike Snitzer atomic_set(&io->io_count, 1); 5489f6dc633SMike Snitzer this_cpu_inc(*md->pending_io); 549978e51baSMike Snitzer io->orig_bio = bio; 550978e51baSMike Snitzer io->md = md; 551978e51baSMike Snitzer spin_lock_init(&io->endio_lock); 552978e51baSMike Snitzer 553b879f915SMike Snitzer io->start_time = jiffies; 55464f52b0eSMike Snitzer 555*0cdb90f0SMike Snitzer dm_stats_record_start(&md->stats, &io->stats_aux); 556*0cdb90f0SMike Snitzer 55764f52b0eSMike Snitzer return io; 5581da177e4SLinus Torvalds } 5591da177e4SLinus Torvalds 560028867acSAlasdair G Kergon static void free_io(struct mapped_device *md, struct dm_io *io) 5611da177e4SLinus Torvalds { 56264f52b0eSMike Snitzer bio_put(&io->tio.clone); 56364f52b0eSMike Snitzer } 56464f52b0eSMike Snitzer 5651d1068ceSChristoph Hellwig static struct bio *alloc_tio(struct clone_info *ci, struct dm_target *ti, 566dc8e2021SChristoph Hellwig unsigned target_bio_nr, unsigned *len, gfp_t gfp_mask) 56764f52b0eSMike Snitzer { 56864f52b0eSMike Snitzer struct dm_target_io *tio; 56964f52b0eSMike Snitzer 57064f52b0eSMike Snitzer if (!ci->io->tio.io) { 57164f52b0eSMike Snitzer /* the dm_target_io embedded in ci->io is available */ 57264f52b0eSMike Snitzer tio = &ci->io->tio; 57364f52b0eSMike Snitzer } else { 574abfc426dSChristoph Hellwig struct bio *clone = bio_alloc_clone(ci->bio->bi_bdev, ci->bio, 575abfc426dSChristoph Hellwig gfp_mask, &ci->io->md->bs); 57664f52b0eSMike Snitzer if (!clone) 57764f52b0eSMike Snitzer return NULL; 57864f52b0eSMike Snitzer 5796c23f0bdSChristoph Hellwig tio = clone_to_tio(clone); 58064f52b0eSMike Snitzer tio->inside_dm_io = false; 58164f52b0eSMike Snitzer } 58256b4b5abSChristoph Hellwig 58364f52b0eSMike Snitzer tio->magic = DM_TIO_MAGIC; 58464f52b0eSMike Snitzer tio->io = ci->io; 58564f52b0eSMike Snitzer tio->ti = ti; 58664f52b0eSMike Snitzer tio->target_bio_nr = target_bio_nr; 587dc8e2021SChristoph Hellwig tio->len_ptr = len; 58864f52b0eSMike Snitzer 5891d1068ceSChristoph Hellwig return &tio->clone; 5901da177e4SLinus Torvalds } 5911da177e4SLinus Torvalds 5921d1068ceSChristoph Hellwig static void free_tio(struct bio *clone) 5931da177e4SLinus Torvalds { 5941d1068ceSChristoph Hellwig if (clone_to_tio(clone)->inside_dm_io) 59564f52b0eSMike Snitzer return; 5961d1068ceSChristoph Hellwig bio_put(clone); 5971da177e4SLinus Torvalds } 5981da177e4SLinus Torvalds 5991da177e4SLinus Torvalds /* 6001da177e4SLinus Torvalds * Add the bio to the list of deferred io. 6011da177e4SLinus Torvalds */ 60292c63902SMikulas Patocka static void queue_io(struct mapped_device *md, struct bio *bio) 6031da177e4SLinus Torvalds { 60405447420SKiyoshi Ueda unsigned long flags; 6051da177e4SLinus Torvalds 60605447420SKiyoshi Ueda spin_lock_irqsave(&md->deferred_lock, flags); 6071da177e4SLinus Torvalds bio_list_add(&md->deferred, bio); 60805447420SKiyoshi Ueda spin_unlock_irqrestore(&md->deferred_lock, flags); 60992c63902SMikulas Patocka queue_work(md->wq, &md->work); 6101da177e4SLinus Torvalds } 6111da177e4SLinus Torvalds 6121da177e4SLinus Torvalds /* 6131da177e4SLinus Torvalds * Everyone (including functions in this file), should use this 6141da177e4SLinus Torvalds * function to access the md->map field, and make sure they call 61583d5e5b0SMikulas Patocka * dm_put_live_table() when finished. 6161da177e4SLinus Torvalds */ 61783d5e5b0SMikulas Patocka struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier) 6181da177e4SLinus Torvalds { 61983d5e5b0SMikulas Patocka *srcu_idx = srcu_read_lock(&md->io_barrier); 6201da177e4SLinus Torvalds 62183d5e5b0SMikulas Patocka return srcu_dereference(md->map, &md->io_barrier); 62283d5e5b0SMikulas Patocka } 6231da177e4SLinus Torvalds 62483d5e5b0SMikulas Patocka void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier) 62583d5e5b0SMikulas Patocka { 62683d5e5b0SMikulas Patocka srcu_read_unlock(&md->io_barrier, srcu_idx); 62783d5e5b0SMikulas Patocka } 62883d5e5b0SMikulas Patocka 62983d5e5b0SMikulas Patocka void dm_sync_table(struct mapped_device *md) 63083d5e5b0SMikulas Patocka { 63183d5e5b0SMikulas Patocka synchronize_srcu(&md->io_barrier); 63283d5e5b0SMikulas Patocka synchronize_rcu_expedited(); 63383d5e5b0SMikulas Patocka } 63483d5e5b0SMikulas Patocka 63583d5e5b0SMikulas Patocka /* 63683d5e5b0SMikulas Patocka * A fast alternative to dm_get_live_table/dm_put_live_table. 63783d5e5b0SMikulas Patocka * The caller must not block between these two functions. 63883d5e5b0SMikulas Patocka */ 63983d5e5b0SMikulas Patocka static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU) 64083d5e5b0SMikulas Patocka { 64183d5e5b0SMikulas Patocka rcu_read_lock(); 64283d5e5b0SMikulas Patocka return rcu_dereference(md->map); 64383d5e5b0SMikulas Patocka } 64483d5e5b0SMikulas Patocka 64583d5e5b0SMikulas Patocka static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU) 64683d5e5b0SMikulas Patocka { 64783d5e5b0SMikulas Patocka rcu_read_unlock(); 6481da177e4SLinus Torvalds } 6491da177e4SLinus Torvalds 650971888c4SMike Snitzer static char *_dm_claim_ptr = "I belong to device-mapper"; 651971888c4SMike Snitzer 6523ac51e74SDarrick J. Wong /* 65386f1152bSBenjamin Marzinski * Open a table device so we can use it as a map destination. 65486f1152bSBenjamin Marzinski */ 65586f1152bSBenjamin Marzinski static int open_table_device(struct table_device *td, dev_t dev, 65686f1152bSBenjamin Marzinski struct mapped_device *md) 65786f1152bSBenjamin Marzinski { 65886f1152bSBenjamin Marzinski struct block_device *bdev; 659cd913c76SChristoph Hellwig u64 part_off; 66086f1152bSBenjamin Marzinski int r; 66186f1152bSBenjamin Marzinski 66286f1152bSBenjamin Marzinski BUG_ON(td->dm_dev.bdev); 66386f1152bSBenjamin Marzinski 664519049afSMike Snitzer bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _dm_claim_ptr); 66586f1152bSBenjamin Marzinski if (IS_ERR(bdev)) 66686f1152bSBenjamin Marzinski return PTR_ERR(bdev); 66786f1152bSBenjamin Marzinski 66886f1152bSBenjamin Marzinski r = bd_link_disk_holder(bdev, dm_disk(md)); 66986f1152bSBenjamin Marzinski if (r) { 67086f1152bSBenjamin Marzinski blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL); 67186f1152bSBenjamin Marzinski return r; 67286f1152bSBenjamin Marzinski } 67386f1152bSBenjamin Marzinski 67486f1152bSBenjamin Marzinski td->dm_dev.bdev = bdev; 675cd913c76SChristoph Hellwig td->dm_dev.dax_dev = fs_dax_get_by_bdev(bdev, &part_off); 67686f1152bSBenjamin Marzinski return 0; 67786f1152bSBenjamin Marzinski } 67886f1152bSBenjamin Marzinski 67986f1152bSBenjamin Marzinski /* 68086f1152bSBenjamin Marzinski * Close a table device that we've been using. 68186f1152bSBenjamin Marzinski */ 68286f1152bSBenjamin Marzinski static void close_table_device(struct table_device *td, struct mapped_device *md) 68386f1152bSBenjamin Marzinski { 68486f1152bSBenjamin Marzinski if (!td->dm_dev.bdev) 68586f1152bSBenjamin Marzinski return; 68686f1152bSBenjamin Marzinski 68786f1152bSBenjamin Marzinski bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md)); 68886f1152bSBenjamin Marzinski blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL); 689817bf402SDan Williams put_dax(td->dm_dev.dax_dev); 69086f1152bSBenjamin Marzinski td->dm_dev.bdev = NULL; 691817bf402SDan Williams td->dm_dev.dax_dev = NULL; 69286f1152bSBenjamin Marzinski } 69386f1152bSBenjamin Marzinski 69486f1152bSBenjamin Marzinski static struct table_device *find_table_device(struct list_head *l, dev_t dev, 6958454fca4SSheetal Singala fmode_t mode) 6968454fca4SSheetal Singala { 69786f1152bSBenjamin Marzinski struct table_device *td; 69886f1152bSBenjamin Marzinski 69986f1152bSBenjamin Marzinski list_for_each_entry(td, l, list) 70086f1152bSBenjamin Marzinski if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode) 70186f1152bSBenjamin Marzinski return td; 70286f1152bSBenjamin Marzinski 70386f1152bSBenjamin Marzinski return NULL; 70486f1152bSBenjamin Marzinski } 70586f1152bSBenjamin Marzinski 70686f1152bSBenjamin Marzinski int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode, 7078454fca4SSheetal Singala struct dm_dev **result) 7088454fca4SSheetal Singala { 70986f1152bSBenjamin Marzinski int r; 71086f1152bSBenjamin Marzinski struct table_device *td; 71186f1152bSBenjamin Marzinski 71286f1152bSBenjamin Marzinski mutex_lock(&md->table_devices_lock); 71386f1152bSBenjamin Marzinski td = find_table_device(&md->table_devices, dev, mode); 71486f1152bSBenjamin Marzinski if (!td) { 715115485e8SMike Snitzer td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id); 71686f1152bSBenjamin Marzinski if (!td) { 71786f1152bSBenjamin Marzinski mutex_unlock(&md->table_devices_lock); 71886f1152bSBenjamin Marzinski return -ENOMEM; 71986f1152bSBenjamin Marzinski } 72086f1152bSBenjamin Marzinski 72186f1152bSBenjamin Marzinski td->dm_dev.mode = mode; 72286f1152bSBenjamin Marzinski td->dm_dev.bdev = NULL; 72386f1152bSBenjamin Marzinski 72486f1152bSBenjamin Marzinski if ((r = open_table_device(td, dev, md))) { 72586f1152bSBenjamin Marzinski mutex_unlock(&md->table_devices_lock); 72686f1152bSBenjamin Marzinski kfree(td); 72786f1152bSBenjamin Marzinski return r; 72886f1152bSBenjamin Marzinski } 72986f1152bSBenjamin Marzinski 73086f1152bSBenjamin Marzinski format_dev_t(td->dm_dev.name, dev); 73186f1152bSBenjamin Marzinski 732b0b4d7c6SElena Reshetova refcount_set(&td->count, 1); 73386f1152bSBenjamin Marzinski list_add(&td->list, &md->table_devices); 734b0b4d7c6SElena Reshetova } else { 735b0b4d7c6SElena Reshetova refcount_inc(&td->count); 73686f1152bSBenjamin Marzinski } 73786f1152bSBenjamin Marzinski mutex_unlock(&md->table_devices_lock); 73886f1152bSBenjamin Marzinski 73986f1152bSBenjamin Marzinski *result = &td->dm_dev; 74086f1152bSBenjamin Marzinski return 0; 74186f1152bSBenjamin Marzinski } 74286f1152bSBenjamin Marzinski 74386f1152bSBenjamin Marzinski void dm_put_table_device(struct mapped_device *md, struct dm_dev *d) 74486f1152bSBenjamin Marzinski { 74586f1152bSBenjamin Marzinski struct table_device *td = container_of(d, struct table_device, dm_dev); 74686f1152bSBenjamin Marzinski 74786f1152bSBenjamin Marzinski mutex_lock(&md->table_devices_lock); 748b0b4d7c6SElena Reshetova if (refcount_dec_and_test(&td->count)) { 74986f1152bSBenjamin Marzinski close_table_device(td, md); 75086f1152bSBenjamin Marzinski list_del(&td->list); 75186f1152bSBenjamin Marzinski kfree(td); 75286f1152bSBenjamin Marzinski } 75386f1152bSBenjamin Marzinski mutex_unlock(&md->table_devices_lock); 75486f1152bSBenjamin Marzinski } 75586f1152bSBenjamin Marzinski 75686f1152bSBenjamin Marzinski static void free_table_devices(struct list_head *devices) 75786f1152bSBenjamin Marzinski { 75886f1152bSBenjamin Marzinski struct list_head *tmp, *next; 75986f1152bSBenjamin Marzinski 76086f1152bSBenjamin Marzinski list_for_each_safe(tmp, next, devices) { 76186f1152bSBenjamin Marzinski struct table_device *td = list_entry(tmp, struct table_device, list); 76286f1152bSBenjamin Marzinski 76386f1152bSBenjamin Marzinski DMWARN("dm_destroy: %s still exists with %d references", 764b0b4d7c6SElena Reshetova td->dm_dev.name, refcount_read(&td->count)); 76586f1152bSBenjamin Marzinski kfree(td); 76686f1152bSBenjamin Marzinski } 76786f1152bSBenjamin Marzinski } 76886f1152bSBenjamin Marzinski 76986f1152bSBenjamin Marzinski /* 7703ac51e74SDarrick J. Wong * Get the geometry associated with a dm device 7713ac51e74SDarrick J. Wong */ 7723ac51e74SDarrick J. Wong int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo) 7733ac51e74SDarrick J. Wong { 7743ac51e74SDarrick J. Wong *geo = md->geometry; 7753ac51e74SDarrick J. Wong 7763ac51e74SDarrick J. Wong return 0; 7773ac51e74SDarrick J. Wong } 7783ac51e74SDarrick J. Wong 7793ac51e74SDarrick J. Wong /* 7803ac51e74SDarrick J. Wong * Set the geometry of a device. 7813ac51e74SDarrick J. Wong */ 7823ac51e74SDarrick J. Wong int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo) 7833ac51e74SDarrick J. Wong { 7843ac51e74SDarrick J. Wong sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors; 7853ac51e74SDarrick J. Wong 7863ac51e74SDarrick J. Wong if (geo->start > sz) { 7873ac51e74SDarrick J. Wong DMWARN("Start sector is beyond the geometry limits."); 7883ac51e74SDarrick J. Wong return -EINVAL; 7893ac51e74SDarrick J. Wong } 7903ac51e74SDarrick J. Wong 7913ac51e74SDarrick J. Wong md->geometry = *geo; 7923ac51e74SDarrick J. Wong 7933ac51e74SDarrick J. Wong return 0; 7943ac51e74SDarrick J. Wong } 7953ac51e74SDarrick J. Wong 7962e93ccc1SKiyoshi Ueda static int __noflush_suspending(struct mapped_device *md) 7972e93ccc1SKiyoshi Ueda { 7982e93ccc1SKiyoshi Ueda return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 7992e93ccc1SKiyoshi Ueda } 8002e93ccc1SKiyoshi Ueda 8011da177e4SLinus Torvalds /* 8021da177e4SLinus Torvalds * Decrements the number of outstanding ios that a bio has been 8031da177e4SLinus Torvalds * cloned into, completing the original io if necc. 8041da177e4SLinus Torvalds */ 805e2118b3cSDamien Le Moal void dm_io_dec_pending(struct dm_io *io, blk_status_t error) 8061da177e4SLinus Torvalds { 8072e93ccc1SKiyoshi Ueda unsigned long flags; 8084e4cbee9SChristoph Hellwig blk_status_t io_error; 809b35f8caaSMilan Broz struct bio *bio; 810b35f8caaSMilan Broz struct mapped_device *md = io->md; 811d208b894SJiazi Li unsigned long start_time = 0; 812d208b894SJiazi Li struct dm_stats_aux stats_aux; 8132e93ccc1SKiyoshi Ueda 8142e93ccc1SKiyoshi Ueda /* Push-back supersedes any I/O errors */ 815f88fb981SKiyoshi Ueda if (unlikely(error)) { 816f88fb981SKiyoshi Ueda spin_lock_irqsave(&io->endio_lock, flags); 817745dc570SMike Snitzer if (!(io->status == BLK_STS_DM_REQUEUE && __noflush_suspending(md))) 8184e4cbee9SChristoph Hellwig io->status = error; 819f88fb981SKiyoshi Ueda spin_unlock_irqrestore(&io->endio_lock, flags); 820f88fb981SKiyoshi Ueda } 8211da177e4SLinus Torvalds 8221da177e4SLinus Torvalds if (atomic_dec_and_test(&io->io_count)) { 823bf14e2b2SDamien Le Moal bio = io->orig_bio; 8244e4cbee9SChristoph Hellwig if (io->status == BLK_STS_DM_REQUEUE) { 8252e93ccc1SKiyoshi Ueda /* 8262e93ccc1SKiyoshi Ueda * Target requested pushing back the I/O. 8272e93ccc1SKiyoshi Ueda */ 828022c2611SMikulas Patocka spin_lock_irqsave(&md->deferred_lock, flags); 829bf14e2b2SDamien Le Moal if (__noflush_suspending(md) && 830bf14e2b2SDamien Le Moal !WARN_ON_ONCE(dm_is_zone_write(md, bio))) { 831745dc570SMike Snitzer /* NOTE early return due to BLK_STS_DM_REQUEUE below */ 832bf14e2b2SDamien Le Moal bio_list_add_head(&md->deferred, bio); 833bf14e2b2SDamien Le Moal } else { 834bf14e2b2SDamien Le Moal /* 835bf14e2b2SDamien Le Moal * noflush suspend was interrupted or this is 836bf14e2b2SDamien Le Moal * a write to a zoned target. 837bf14e2b2SDamien Le Moal */ 8384e4cbee9SChristoph Hellwig io->status = BLK_STS_IOERR; 839bf14e2b2SDamien Le Moal } 840022c2611SMikulas Patocka spin_unlock_irqrestore(&md->deferred_lock, flags); 8412e93ccc1SKiyoshi Ueda } 8422e93ccc1SKiyoshi Ueda 8434e4cbee9SChristoph Hellwig io_error = io->status; 844d208b894SJiazi Li start_time = io->start_time; 845d208b894SJiazi Li stats_aux = io->stats_aux; 846a97f925aSMikulas Patocka free_io(md, io); 847d208b894SJiazi Li end_io_acct(md, bio, start_time, &stats_aux); 8489f6dc633SMike Snitzer smp_wmb(); 8499f6dc633SMike Snitzer this_cpu_dec(*md->pending_io); 8509f6dc633SMike Snitzer 8519f6dc633SMike Snitzer /* nudge anyone waiting on suspend queue */ 8529f6dc633SMike Snitzer if (unlikely(wq_has_sleeper(&md->wait))) 8539f6dc633SMike Snitzer wake_up(&md->wait); 8541da177e4SLinus Torvalds 8554e4cbee9SChristoph Hellwig if (io_error == BLK_STS_DM_REQUEUE) 8566a8736d1STejun Heo return; 8576a8736d1STejun Heo 8588d394bc4SMike Snitzer if (bio_is_flush_with_data(bio)) { 8591da177e4SLinus Torvalds /* 8606a8736d1STejun Heo * Preflush done for flush with data, reissue 86128a8f0d3SMike Christie * without REQ_PREFLUSH. 862af7e466aSMikulas Patocka */ 8631eff9d32SJens Axboe bio->bi_opf &= ~REQ_PREFLUSH; 8646a8736d1STejun Heo queue_io(md, bio); 865af7e466aSMikulas Patocka } else { 866b372d360SMike Snitzer /* done with normal IO or empty flush */ 8678dd601faSNeilBrown if (io_error) 8684e4cbee9SChristoph Hellwig bio->bi_status = io_error; 8694246a0b6SChristoph Hellwig bio_endio(bio); 8702e93ccc1SKiyoshi Ueda } 8711da177e4SLinus Torvalds } 872af7e466aSMikulas Patocka } 8731da177e4SLinus Torvalds 874bcb44433SMike Snitzer void disable_discard(struct mapped_device *md) 875bcb44433SMike Snitzer { 876bcb44433SMike Snitzer struct queue_limits *limits = dm_get_queue_limits(md); 877bcb44433SMike Snitzer 878bcb44433SMike Snitzer /* device doesn't really support DISCARD, disable it */ 879bcb44433SMike Snitzer limits->max_discard_sectors = 0; 880bcb44433SMike Snitzer blk_queue_flag_clear(QUEUE_FLAG_DISCARD, md->queue); 881bcb44433SMike Snitzer } 882bcb44433SMike Snitzer 8834cc96131SMike Snitzer void disable_write_same(struct mapped_device *md) 8847eee4ae2SMike Snitzer { 8857eee4ae2SMike Snitzer struct queue_limits *limits = dm_get_queue_limits(md); 8867eee4ae2SMike Snitzer 8877eee4ae2SMike Snitzer /* device doesn't really support WRITE SAME, disable it */ 8887eee4ae2SMike Snitzer limits->max_write_same_sectors = 0; 8897eee4ae2SMike Snitzer } 8907eee4ae2SMike Snitzer 891ac62d620SChristoph Hellwig void disable_write_zeroes(struct mapped_device *md) 892ac62d620SChristoph Hellwig { 893ac62d620SChristoph Hellwig struct queue_limits *limits = dm_get_queue_limits(md); 894ac62d620SChristoph Hellwig 895ac62d620SChristoph Hellwig /* device doesn't really support WRITE ZEROES, disable it */ 896ac62d620SChristoph Hellwig limits->max_write_zeroes_sectors = 0; 897ac62d620SChristoph Hellwig } 898ac62d620SChristoph Hellwig 899a666e5c0SMikulas Patocka static bool swap_bios_limit(struct dm_target *ti, struct bio *bio) 900a666e5c0SMikulas Patocka { 901a666e5c0SMikulas Patocka return unlikely((bio->bi_opf & REQ_SWAP) != 0) && unlikely(ti->limit_swap_bios); 902a666e5c0SMikulas Patocka } 903a666e5c0SMikulas Patocka 9044246a0b6SChristoph Hellwig static void clone_endio(struct bio *bio) 9051da177e4SLinus Torvalds { 9064e4cbee9SChristoph Hellwig blk_status_t error = bio->bi_status; 9076c23f0bdSChristoph Hellwig struct dm_target_io *tio = clone_to_tio(bio); 908b35f8caaSMilan Broz struct dm_io *io = tio->io; 9099faf400fSStefan Bader struct mapped_device *md = tio->io->md; 9101da177e4SLinus Torvalds dm_endio_fn endio = tio->ti->type->end_io; 911309dca30SChristoph Hellwig struct request_queue *q = bio->bi_bdev->bd_disk->queue; 9121da177e4SLinus Torvalds 9139c37de29SMike Snitzer if (unlikely(error == BLK_STS_TARGET)) { 914bcb44433SMike Snitzer if (bio_op(bio) == REQ_OP_DISCARD && 915309dca30SChristoph Hellwig !q->limits.max_discard_sectors) 916bcb44433SMike Snitzer disable_discard(md); 917bcb44433SMike Snitzer else if (bio_op(bio) == REQ_OP_WRITE_SAME && 918309dca30SChristoph Hellwig !q->limits.max_write_same_sectors) 9197eee4ae2SMike Snitzer disable_write_same(md); 920bcb44433SMike Snitzer else if (bio_op(bio) == REQ_OP_WRITE_ZEROES && 921309dca30SChristoph Hellwig !q->limits.max_write_zeroes_sectors) 922ac62d620SChristoph Hellwig disable_write_zeroes(md); 923ac62d620SChristoph Hellwig } 9247eee4ae2SMike Snitzer 925bb37d772SDamien Le Moal if (blk_queue_is_zoned(q)) 926bb37d772SDamien Le Moal dm_zone_endio(io, bio); 927415c79e1SJohannes Thumshirn 9281be56909SChristoph Hellwig if (endio) { 9294e4cbee9SChristoph Hellwig int r = endio(tio->ti, bio, &error); 9301be56909SChristoph Hellwig switch (r) { 9311be56909SChristoph Hellwig case DM_ENDIO_REQUEUE: 932bf14e2b2SDamien Le Moal /* 933bf14e2b2SDamien Le Moal * Requeuing writes to a sequential zone of a zoned 934bf14e2b2SDamien Le Moal * target will break the sequential write pattern: 935bf14e2b2SDamien Le Moal * fail such IO. 936bf14e2b2SDamien Le Moal */ 937bf14e2b2SDamien Le Moal if (WARN_ON_ONCE(dm_is_zone_write(md, bio))) 938bf14e2b2SDamien Le Moal error = BLK_STS_IOERR; 939bf14e2b2SDamien Le Moal else 9404e4cbee9SChristoph Hellwig error = BLK_STS_DM_REQUEUE; 941df561f66SGustavo A. R. Silva fallthrough; 9421be56909SChristoph Hellwig case DM_ENDIO_DONE: 9431be56909SChristoph Hellwig break; 9441be56909SChristoph Hellwig case DM_ENDIO_INCOMPLETE: 9451be56909SChristoph Hellwig /* The target will handle the io */ 9461be56909SChristoph Hellwig return; 9471be56909SChristoph Hellwig default: 9481be56909SChristoph Hellwig DMWARN("unimplemented target endio return value: %d", r); 9491be56909SChristoph Hellwig BUG(); 9501be56909SChristoph Hellwig } 9511be56909SChristoph Hellwig } 9521be56909SChristoph Hellwig 953a666e5c0SMikulas Patocka if (unlikely(swap_bios_limit(tio->ti, bio))) { 954a666e5c0SMikulas Patocka struct mapped_device *md = io->md; 955a666e5c0SMikulas Patocka up(&md->swap_bios_semaphore); 956a666e5c0SMikulas Patocka } 957a666e5c0SMikulas Patocka 9581d1068ceSChristoph Hellwig free_tio(bio); 959e2118b3cSDamien Le Moal dm_io_dec_pending(io, error); 9601da177e4SLinus Torvalds } 9611da177e4SLinus Torvalds 96278d8e58aSMike Snitzer /* 96356a67df7SMike Snitzer * Return maximum size of I/O possible at the supplied sector up to the current 96456a67df7SMike Snitzer * target boundary. 96556a67df7SMike Snitzer */ 9663720281dSMike Snitzer static inline sector_t max_io_len_target_boundary(struct dm_target *ti, 9673720281dSMike Snitzer sector_t target_offset) 9681da177e4SLinus Torvalds { 96956a67df7SMike Snitzer return ti->len - target_offset; 97056a67df7SMike Snitzer } 97156a67df7SMike Snitzer 9723720281dSMike Snitzer static sector_t max_io_len(struct dm_target *ti, sector_t sector) 97356a67df7SMike Snitzer { 9743720281dSMike Snitzer sector_t target_offset = dm_target_offset(ti, sector); 9753720281dSMike Snitzer sector_t len = max_io_len_target_boundary(ti, target_offset); 9765091cdecSMike Snitzer sector_t max_len; 9771da177e4SLinus Torvalds 9781da177e4SLinus Torvalds /* 9793ee16db3SMike Snitzer * Does the target need to split IO even further? 9803ee16db3SMike Snitzer * - varied (per target) IO splitting is a tenet of DM; this 9813ee16db3SMike Snitzer * explains why stacked chunk_sectors based splitting via 9823ee16db3SMike Snitzer * blk_max_size_offset() isn't possible here. So pass in 9833ee16db3SMike Snitzer * ti->max_io_len to override stacked chunk_sectors. 9841da177e4SLinus Torvalds */ 9853ee16db3SMike Snitzer if (ti->max_io_len) { 98633bd6f06SMike Snitzer max_len = blk_max_size_offset(ti->table->md->queue, 9873ee16db3SMike Snitzer target_offset, ti->max_io_len); 988542f9038SMike Snitzer if (len > max_len) 989542f9038SMike Snitzer len = max_len; 9903ee16db3SMike Snitzer } 9911da177e4SLinus Torvalds 9921da177e4SLinus Torvalds return len; 9931da177e4SLinus Torvalds } 9941da177e4SLinus Torvalds 995542f9038SMike Snitzer int dm_set_target_max_io_len(struct dm_target *ti, sector_t len) 996542f9038SMike Snitzer { 997542f9038SMike Snitzer if (len > UINT_MAX) { 998542f9038SMike Snitzer DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)", 999542f9038SMike Snitzer (unsigned long long)len, UINT_MAX); 1000542f9038SMike Snitzer ti->error = "Maximum size of target IO is too large"; 1001542f9038SMike Snitzer return -EINVAL; 1002542f9038SMike Snitzer } 1003542f9038SMike Snitzer 100475ae1936SMikulas Patocka ti->max_io_len = (uint32_t) len; 1005542f9038SMike Snitzer 1006542f9038SMike Snitzer return 0; 1007542f9038SMike Snitzer } 1008542f9038SMike Snitzer EXPORT_SYMBOL_GPL(dm_set_target_max_io_len); 1009542f9038SMike Snitzer 1010f26c5719SDan Williams static struct dm_target *dm_dax_get_live_target(struct mapped_device *md, 1011f26c5719SDan Williams sector_t sector, int *srcu_idx) 10123d97c829SMike Snitzer __acquires(md->io_barrier) 1013545ed20eSToshi Kani { 1014545ed20eSToshi Kani struct dm_table *map; 1015545ed20eSToshi Kani struct dm_target *ti; 1016545ed20eSToshi Kani 1017f26c5719SDan Williams map = dm_get_live_table(md, srcu_idx); 1018545ed20eSToshi Kani if (!map) 1019f26c5719SDan Williams return NULL; 1020545ed20eSToshi Kani 1021545ed20eSToshi Kani ti = dm_table_find_target(map, sector); 1022123d87d5SMikulas Patocka if (!ti) 1023f26c5719SDan Williams return NULL; 1024f26c5719SDan Williams 1025f26c5719SDan Williams return ti; 1026f26c5719SDan Williams } 1027f26c5719SDan Williams 1028f26c5719SDan Williams static long dm_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, 1029f26c5719SDan Williams long nr_pages, void **kaddr, pfn_t *pfn) 1030f26c5719SDan Williams { 1031f26c5719SDan Williams struct mapped_device *md = dax_get_private(dax_dev); 1032f26c5719SDan Williams sector_t sector = pgoff * PAGE_SECTORS; 1033f26c5719SDan Williams struct dm_target *ti; 1034f26c5719SDan Williams long len, ret = -EIO; 1035f26c5719SDan Williams int srcu_idx; 1036f26c5719SDan Williams 1037f26c5719SDan Williams ti = dm_dax_get_live_target(md, sector, &srcu_idx); 1038f26c5719SDan Williams 1039f26c5719SDan Williams if (!ti) 1040545ed20eSToshi Kani goto out; 1041f26c5719SDan Williams if (!ti->type->direct_access) 1042f26c5719SDan Williams goto out; 10433720281dSMike Snitzer len = max_io_len(ti, sector) / PAGE_SECTORS; 1044f26c5719SDan Williams if (len < 1) 1045f26c5719SDan Williams goto out; 1046f26c5719SDan Williams nr_pages = min(len, nr_pages); 1047817bf402SDan Williams ret = ti->type->direct_access(ti, pgoff, nr_pages, kaddr, pfn); 1048817bf402SDan Williams 1049545ed20eSToshi Kani out: 1050545ed20eSToshi Kani dm_put_live_table(md, srcu_idx); 1051f26c5719SDan Williams 1052f26c5719SDan Williams return ret; 1053545ed20eSToshi Kani } 1054545ed20eSToshi Kani 1055cdf6cdcdSVivek Goyal static int dm_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, 1056cdf6cdcdSVivek Goyal size_t nr_pages) 1057cdf6cdcdSVivek Goyal { 1058cdf6cdcdSVivek Goyal struct mapped_device *md = dax_get_private(dax_dev); 1059cdf6cdcdSVivek Goyal sector_t sector = pgoff * PAGE_SECTORS; 1060cdf6cdcdSVivek Goyal struct dm_target *ti; 1061cdf6cdcdSVivek Goyal int ret = -EIO; 1062cdf6cdcdSVivek Goyal int srcu_idx; 1063cdf6cdcdSVivek Goyal 1064cdf6cdcdSVivek Goyal ti = dm_dax_get_live_target(md, sector, &srcu_idx); 1065cdf6cdcdSVivek Goyal 1066cdf6cdcdSVivek Goyal if (!ti) 1067cdf6cdcdSVivek Goyal goto out; 1068cdf6cdcdSVivek Goyal if (WARN_ON(!ti->type->dax_zero_page_range)) { 1069cdf6cdcdSVivek Goyal /* 1070cdf6cdcdSVivek Goyal * ->zero_page_range() is mandatory dax operation. If we are 1071cdf6cdcdSVivek Goyal * here, something is wrong. 1072cdf6cdcdSVivek Goyal */ 1073cdf6cdcdSVivek Goyal goto out; 1074cdf6cdcdSVivek Goyal } 1075cdf6cdcdSVivek Goyal ret = ti->type->dax_zero_page_range(ti, pgoff, nr_pages); 1076cdf6cdcdSVivek Goyal out: 1077cdf6cdcdSVivek Goyal dm_put_live_table(md, srcu_idx); 1078cdf6cdcdSVivek Goyal 1079cdf6cdcdSVivek Goyal return ret; 1080cdf6cdcdSVivek Goyal } 1081cdf6cdcdSVivek Goyal 10821dd40c3eSMikulas Patocka /* 10831dd40c3eSMikulas Patocka * A target may call dm_accept_partial_bio only from the map routine. It is 10846842d264SDamien Le Moal * allowed for all bio types except REQ_PREFLUSH, REQ_OP_ZONE_* zone management 10856842d264SDamien Le Moal * operations and REQ_OP_ZONE_APPEND (zone append writes). 10861dd40c3eSMikulas Patocka * 10871dd40c3eSMikulas Patocka * dm_accept_partial_bio informs the dm that the target only wants to process 10881dd40c3eSMikulas Patocka * additional n_sectors sectors of the bio and the rest of the data should be 10891dd40c3eSMikulas Patocka * sent in a next bio. 10901dd40c3eSMikulas Patocka * 10911dd40c3eSMikulas Patocka * A diagram that explains the arithmetics: 10921dd40c3eSMikulas Patocka * +--------------------+---------------+-------+ 10931dd40c3eSMikulas Patocka * | 1 | 2 | 3 | 10941dd40c3eSMikulas Patocka * +--------------------+---------------+-------+ 10951dd40c3eSMikulas Patocka * 10961dd40c3eSMikulas Patocka * <-------------- *tio->len_ptr ---------------> 10971dd40c3eSMikulas Patocka * <------- bi_size -------> 10981dd40c3eSMikulas Patocka * <-- n_sectors --> 10991dd40c3eSMikulas Patocka * 11001dd40c3eSMikulas Patocka * Region 1 was already iterated over with bio_advance or similar function. 11011dd40c3eSMikulas Patocka * (it may be empty if the target doesn't use bio_advance) 11021dd40c3eSMikulas Patocka * Region 2 is the remaining bio size that the target wants to process. 11031dd40c3eSMikulas Patocka * (it may be empty if region 1 is non-empty, although there is no reason 11041dd40c3eSMikulas Patocka * to make it empty) 11051dd40c3eSMikulas Patocka * The target requires that region 3 is to be sent in the next bio. 11061dd40c3eSMikulas Patocka * 11071dd40c3eSMikulas Patocka * If the target wants to receive multiple copies of the bio (via num_*bios, etc), 11081dd40c3eSMikulas Patocka * the partially processed part (the sum of regions 1+2) must be the same for all 11091dd40c3eSMikulas Patocka * copies of the bio. 11101dd40c3eSMikulas Patocka */ 11111dd40c3eSMikulas Patocka void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors) 11121dd40c3eSMikulas Patocka { 11136c23f0bdSChristoph Hellwig struct dm_target_io *tio = clone_to_tio(bio); 11141dd40c3eSMikulas Patocka unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT; 11156842d264SDamien Le Moal 11161eff9d32SJens Axboe BUG_ON(bio->bi_opf & REQ_PREFLUSH); 11176842d264SDamien Le Moal BUG_ON(op_is_zone_mgmt(bio_op(bio))); 11186842d264SDamien Le Moal BUG_ON(bio_op(bio) == REQ_OP_ZONE_APPEND); 11191dd40c3eSMikulas Patocka BUG_ON(bi_size > *tio->len_ptr); 11201dd40c3eSMikulas Patocka BUG_ON(n_sectors > bi_size); 11216842d264SDamien Le Moal 11221dd40c3eSMikulas Patocka *tio->len_ptr -= bi_size - n_sectors; 11231dd40c3eSMikulas Patocka bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT; 11241dd40c3eSMikulas Patocka } 11251dd40c3eSMikulas Patocka EXPORT_SYMBOL_GPL(dm_accept_partial_bio); 11261dd40c3eSMikulas Patocka 1127a666e5c0SMikulas Patocka static noinline void __set_swap_bios_limit(struct mapped_device *md, int latch) 1128a666e5c0SMikulas Patocka { 1129a666e5c0SMikulas Patocka mutex_lock(&md->swap_bios_lock); 1130a666e5c0SMikulas Patocka while (latch < md->swap_bios) { 1131a666e5c0SMikulas Patocka cond_resched(); 1132a666e5c0SMikulas Patocka down(&md->swap_bios_semaphore); 1133a666e5c0SMikulas Patocka md->swap_bios--; 1134a666e5c0SMikulas Patocka } 1135a666e5c0SMikulas Patocka while (latch > md->swap_bios) { 1136a666e5c0SMikulas Patocka cond_resched(); 1137a666e5c0SMikulas Patocka up(&md->swap_bios_semaphore); 1138a666e5c0SMikulas Patocka md->swap_bios++; 1139a666e5c0SMikulas Patocka } 1140a666e5c0SMikulas Patocka mutex_unlock(&md->swap_bios_lock); 1141a666e5c0SMikulas Patocka } 1142a666e5c0SMikulas Patocka 11431561b396SChristoph Hellwig static void __map_bio(struct bio *clone) 11441da177e4SLinus Torvalds { 11451561b396SChristoph Hellwig struct dm_target_io *tio = clone_to_tio(clone); 11461da177e4SLinus Torvalds int r; 11472056a782SJens Axboe sector_t sector; 114864f52b0eSMike Snitzer struct dm_io *io = tio->io; 1149bd2a49b8SAlasdair G Kergon struct dm_target *ti = tio->ti; 11501da177e4SLinus Torvalds 11511da177e4SLinus Torvalds clone->bi_end_io = clone_endio; 11521da177e4SLinus Torvalds 11531da177e4SLinus Torvalds /* 11541da177e4SLinus Torvalds * Map the clone. If r == 0 we don't need to do 11551da177e4SLinus Torvalds * anything, the target has assumed ownership of 11561da177e4SLinus Torvalds * this io. 11571da177e4SLinus Torvalds */ 1158e2118b3cSDamien Le Moal dm_io_inc_pending(io); 11594f024f37SKent Overstreet sector = clone->bi_iter.bi_sector; 1160d67a5f4bSMikulas Patocka 1161a666e5c0SMikulas Patocka if (unlikely(swap_bios_limit(ti, clone))) { 1162a666e5c0SMikulas Patocka struct mapped_device *md = io->md; 1163a666e5c0SMikulas Patocka int latch = get_swap_bios(); 1164a666e5c0SMikulas Patocka if (unlikely(latch != md->swap_bios)) 1165a666e5c0SMikulas Patocka __set_swap_bios_limit(md, latch); 1166a666e5c0SMikulas Patocka down(&md->swap_bios_semaphore); 1167a666e5c0SMikulas Patocka } 1168a666e5c0SMikulas Patocka 1169bb37d772SDamien Le Moal /* 1170bb37d772SDamien Le Moal * Check if the IO needs a special mapping due to zone append emulation 1171bb37d772SDamien Le Moal * on zoned target. In this case, dm_zone_map_bio() calls the target 1172bb37d772SDamien Le Moal * map operation. 1173bb37d772SDamien Le Moal */ 1174bb37d772SDamien Le Moal if (dm_emulate_zone_append(io->md)) 1175bb37d772SDamien Le Moal r = dm_zone_map_bio(tio); 1176bb37d772SDamien Le Moal else 11777de3ee57SMikulas Patocka r = ti->type->map(ti, clone); 1178bb37d772SDamien Le Moal 1179846785e6SChristoph Hellwig switch (r) { 1180846785e6SChristoph Hellwig case DM_MAPIO_SUBMITTED: 1181846785e6SChristoph Hellwig break; 1182846785e6SChristoph Hellwig case DM_MAPIO_REMAPPED: 11831da177e4SLinus Torvalds /* the bio has been remapped so dispatch it */ 11841c02fca6SChristoph Hellwig trace_block_bio_remap(clone, bio_dev(io->orig_bio), sector); 11853e08773cSChristoph Hellwig submit_bio_noacct(clone); 1186846785e6SChristoph Hellwig break; 1187846785e6SChristoph Hellwig case DM_MAPIO_KILL: 1188a666e5c0SMikulas Patocka if (unlikely(swap_bios_limit(ti, clone))) { 1189a666e5c0SMikulas Patocka struct mapped_device *md = io->md; 1190a666e5c0SMikulas Patocka up(&md->swap_bios_semaphore); 1191a666e5c0SMikulas Patocka } 11921d1068ceSChristoph Hellwig free_tio(clone); 1193e2118b3cSDamien Le Moal dm_io_dec_pending(io, BLK_STS_IOERR); 11944e4cbee9SChristoph Hellwig break; 1195846785e6SChristoph Hellwig case DM_MAPIO_REQUEUE: 1196a666e5c0SMikulas Patocka if (unlikely(swap_bios_limit(ti, clone))) { 1197a666e5c0SMikulas Patocka struct mapped_device *md = io->md; 1198a666e5c0SMikulas Patocka up(&md->swap_bios_semaphore); 1199a666e5c0SMikulas Patocka } 12001d1068ceSChristoph Hellwig free_tio(clone); 1201e2118b3cSDamien Le Moal dm_io_dec_pending(io, BLK_STS_DM_REQUEUE); 1202846785e6SChristoph Hellwig break; 1203846785e6SChristoph Hellwig default: 120445cbcd79SKiyoshi Ueda DMWARN("unimplemented target map return value: %d", r); 120545cbcd79SKiyoshi Ueda BUG(); 12061da177e4SLinus Torvalds } 12071da177e4SLinus Torvalds } 12081da177e4SLinus Torvalds 1209e0d6609aSMikulas Patocka static void bio_setup_sector(struct bio *bio, sector_t sector, unsigned len) 1210bd2a49b8SAlasdair G Kergon { 12114f024f37SKent Overstreet bio->bi_iter.bi_sector = sector; 12124f024f37SKent Overstreet bio->bi_iter.bi_size = to_bytes(len); 12131da177e4SLinus Torvalds } 12141da177e4SLinus Torvalds 12151da177e4SLinus Torvalds /* 12161da177e4SLinus Torvalds * Creates a bio that consists of range of complete bvecs. 12171da177e4SLinus Torvalds */ 1218b1bee792SChristoph Hellwig static int __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti, 1219b1bee792SChristoph Hellwig sector_t sector, unsigned *len) 12201da177e4SLinus Torvalds { 1221b1bee792SChristoph Hellwig struct bio *bio = ci->bio, *clone; 12221da177e4SLinus Torvalds 12231d1068ceSChristoph Hellwig clone = alloc_tio(ci, ti, 0, len, GFP_NOIO); 1224fa8db494SMike Snitzer bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector)); 1225b1bee792SChristoph Hellwig clone->bi_iter.bi_size = to_bytes(*len); 1226fa8db494SMike Snitzer 1227fa8db494SMike Snitzer if (bio_integrity(bio)) 1228fa8db494SMike Snitzer bio_integrity_trim(clone); 1229c80914e8SMike Snitzer 12301561b396SChristoph Hellwig __map_bio(clone); 1231c80914e8SMike Snitzer return 0; 12321da177e4SLinus Torvalds } 12331da177e4SLinus Torvalds 1234318716ddSMike Snitzer static void alloc_multiple_bios(struct bio_list *blist, struct clone_info *ci, 1235dc8e2021SChristoph Hellwig struct dm_target *ti, unsigned num_bios, 1236dc8e2021SChristoph Hellwig unsigned *len) 1237f9ab94ceSMikulas Patocka { 12381d1068ceSChristoph Hellwig struct bio *bio; 1239318716ddSMike Snitzer int try; 1240dba14160SMikulas Patocka 1241318716ddSMike Snitzer for (try = 0; try < 2; try++) { 1242318716ddSMike Snitzer int bio_nr; 1243318716ddSMike Snitzer 1244318716ddSMike Snitzer if (try) 1245bc02cdbeSMike Snitzer mutex_lock(&ci->io->md->table_devices_lock); 1246318716ddSMike Snitzer for (bio_nr = 0; bio_nr < num_bios; bio_nr++) { 12471d1068ceSChristoph Hellwig bio = alloc_tio(ci, ti, bio_nr, len, 1248dc8e2021SChristoph Hellwig try ? GFP_NOIO : GFP_NOWAIT); 12491d1068ceSChristoph Hellwig if (!bio) 1250318716ddSMike Snitzer break; 1251318716ddSMike Snitzer 12521d1068ceSChristoph Hellwig bio_list_add(blist, bio); 1253318716ddSMike Snitzer } 1254318716ddSMike Snitzer if (try) 1255bc02cdbeSMike Snitzer mutex_unlock(&ci->io->md->table_devices_lock); 1256318716ddSMike Snitzer if (bio_nr == num_bios) 1257318716ddSMike Snitzer return; 1258318716ddSMike Snitzer 12596c23f0bdSChristoph Hellwig while ((bio = bio_list_pop(blist))) 12601d1068ceSChristoph Hellwig free_tio(bio); 1261318716ddSMike Snitzer } 1262318716ddSMike Snitzer } 1263318716ddSMike Snitzer 12648eabf5d0SChristoph Hellwig static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti, 12658eabf5d0SChristoph Hellwig unsigned num_bios, unsigned *len) 12669015df24SAlasdair G Kergon { 12678eabf5d0SChristoph Hellwig struct bio_list blist = BIO_EMPTY_LIST; 12688eabf5d0SChristoph Hellwig struct bio *clone; 12698eabf5d0SChristoph Hellwig 1270891fced6SChristoph Hellwig switch (num_bios) { 1271891fced6SChristoph Hellwig case 0: 1272891fced6SChristoph Hellwig break; 1273891fced6SChristoph Hellwig case 1: 1274891fced6SChristoph Hellwig clone = alloc_tio(ci, ti, 0, len, GFP_NOIO); 1275891fced6SChristoph Hellwig if (len) 1276891fced6SChristoph Hellwig bio_setup_sector(clone, ci->sector, *len); 1277891fced6SChristoph Hellwig __map_bio(clone); 1278891fced6SChristoph Hellwig break; 1279891fced6SChristoph Hellwig default: 1280dc8e2021SChristoph Hellwig alloc_multiple_bios(&blist, ci, ti, num_bios, len); 12818eabf5d0SChristoph Hellwig while ((clone = bio_list_pop(&blist))) { 1282bd2a49b8SAlasdair G Kergon if (len) 12831dd40c3eSMikulas Patocka bio_setup_sector(clone, ci->sector, *len); 12841561b396SChristoph Hellwig __map_bio(clone); 1285f9ab94ceSMikulas Patocka } 1286891fced6SChristoph Hellwig break; 1287891fced6SChristoph Hellwig } 128806a426ceSMike Snitzer } 128906a426ceSMike Snitzer 129014fe594dSAlasdair G Kergon static int __send_empty_flush(struct clone_info *ci) 1291f9ab94ceSMikulas Patocka { 129206a426ceSMike Snitzer unsigned target_nr = 0; 1293f9ab94ceSMikulas Patocka struct dm_target *ti; 1294828678b8SMike Snitzer struct bio flush_bio; 1295828678b8SMike Snitzer 1296828678b8SMike Snitzer /* 1297828678b8SMike Snitzer * Use an on-stack bio for this, it's safe since we don't 1298828678b8SMike Snitzer * need to reference it after submit. It's just used as 1299828678b8SMike Snitzer * the basis for the clone(s). 1300828678b8SMike Snitzer */ 130149add496SChristoph Hellwig bio_init(&flush_bio, ci->io->md->disk->part0, NULL, 0, 130249add496SChristoph Hellwig REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC); 130347d95102SChristoph Hellwig 1304828678b8SMike Snitzer ci->bio = &flush_bio; 1305828678b8SMike Snitzer ci->sector_count = 0; 1306f9ab94ceSMikulas Patocka 1307b372d360SMike Snitzer BUG_ON(bio_has_data(ci->bio)); 1308f9ab94ceSMikulas Patocka while ((ti = dm_table_get_target(ci->map, target_nr++))) 13091dd40c3eSMikulas Patocka __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL); 1310828678b8SMike Snitzer 1311828678b8SMike Snitzer bio_uninit(ci->bio); 1312f9ab94ceSMikulas Patocka return 0; 1313f9ab94ceSMikulas Patocka } 1314f9ab94ceSMikulas Patocka 13153d7f4562SMike Snitzer static int __send_changing_extent_only(struct clone_info *ci, struct dm_target *ti, 131661697a6aSMike Snitzer unsigned num_bios) 13175ae89a87SMike Snitzer { 131851b86f9aSMichael Lass unsigned len; 13195ae89a87SMike Snitzer 13205ae89a87SMike Snitzer /* 132123508a96SMike Snitzer * Even though the device advertised support for this type of 132223508a96SMike Snitzer * request, that does not mean every target supports it, and 1323936688d7SMike Snitzer * reconfiguration might also have changed that since the 13245ae89a87SMike Snitzer * check was performed. 13255ae89a87SMike Snitzer */ 132655a62eefSAlasdair G Kergon if (!num_bios) 13275ae89a87SMike Snitzer return -EOPNOTSUPP; 13285ae89a87SMike Snitzer 13293720281dSMike Snitzer len = min_t(sector_t, ci->sector_count, 13303720281dSMike Snitzer max_io_len_target_boundary(ti, dm_target_offset(ti, ci->sector))); 133151b86f9aSMichael Lass 13321dd40c3eSMikulas Patocka __send_duplicate_bios(ci, ti, num_bios, &len); 13335ae89a87SMike Snitzer 1334a79245b3SMike Snitzer ci->sector += len; 13353d7f4562SMike Snitzer ci->sector_count -= len; 13365ae89a87SMike Snitzer 13375ae89a87SMike Snitzer return 0; 13385ae89a87SMike Snitzer } 13395ae89a87SMike Snitzer 1340568c73a3SMike Snitzer static bool is_abnormal_io(struct bio *bio) 1341568c73a3SMike Snitzer { 1342568c73a3SMike Snitzer bool r = false; 1343568c73a3SMike Snitzer 1344568c73a3SMike Snitzer switch (bio_op(bio)) { 1345568c73a3SMike Snitzer case REQ_OP_DISCARD: 1346568c73a3SMike Snitzer case REQ_OP_SECURE_ERASE: 1347568c73a3SMike Snitzer case REQ_OP_WRITE_SAME: 1348568c73a3SMike Snitzer case REQ_OP_WRITE_ZEROES: 1349568c73a3SMike Snitzer r = true; 1350568c73a3SMike Snitzer break; 1351568c73a3SMike Snitzer } 1352568c73a3SMike Snitzer 1353568c73a3SMike Snitzer return r; 1354568c73a3SMike Snitzer } 1355568c73a3SMike Snitzer 13560519c71eSMike Snitzer static bool __process_abnormal_io(struct clone_info *ci, struct dm_target *ti, 13570519c71eSMike Snitzer int *result) 13580519c71eSMike Snitzer { 13590519c71eSMike Snitzer struct bio *bio = ci->bio; 13609679b5a7SMike Snitzer unsigned num_bios = 0; 13610519c71eSMike Snitzer 13629679b5a7SMike Snitzer switch (bio_op(bio)) { 13639679b5a7SMike Snitzer case REQ_OP_DISCARD: 13649679b5a7SMike Snitzer num_bios = ti->num_discard_bios; 13659679b5a7SMike Snitzer break; 13669679b5a7SMike Snitzer case REQ_OP_SECURE_ERASE: 13679679b5a7SMike Snitzer num_bios = ti->num_secure_erase_bios; 13689679b5a7SMike Snitzer break; 13699679b5a7SMike Snitzer case REQ_OP_WRITE_SAME: 13709679b5a7SMike Snitzer num_bios = ti->num_write_same_bios; 13719679b5a7SMike Snitzer break; 13729679b5a7SMike Snitzer case REQ_OP_WRITE_ZEROES: 13739679b5a7SMike Snitzer num_bios = ti->num_write_zeroes_bios; 13749679b5a7SMike Snitzer break; 13759679b5a7SMike Snitzer default: 13760519c71eSMike Snitzer return false; 13779679b5a7SMike Snitzer } 13780519c71eSMike Snitzer 13799679b5a7SMike Snitzer *result = __send_changing_extent_only(ci, ti, num_bios); 13800519c71eSMike Snitzer return true; 13810519c71eSMike Snitzer } 13820519c71eSMike Snitzer 1383e4c93811SAlasdair G Kergon /* 1384e4c93811SAlasdair G Kergon * Select the correct strategy for processing a non-flush bio. 1385e4c93811SAlasdair G Kergon */ 1386e4c93811SAlasdair G Kergon static int __split_and_process_non_flush(struct clone_info *ci) 1387e4c93811SAlasdair G Kergon { 1388e4c93811SAlasdair G Kergon struct dm_target *ti; 13891c3b13e6SKent Overstreet unsigned len; 1390c80914e8SMike Snitzer int r; 1391e4c93811SAlasdair G Kergon 1392e4c93811SAlasdair G Kergon ti = dm_table_find_target(ci->map, ci->sector); 1393123d87d5SMikulas Patocka if (!ti) 1394e4c93811SAlasdair G Kergon return -EIO; 1395e4c93811SAlasdair G Kergon 1396568c73a3SMike Snitzer if (__process_abnormal_io(ci, ti, &r)) 13970519c71eSMike Snitzer return r; 13983d7f4562SMike Snitzer 13993720281dSMike Snitzer len = min_t(sector_t, max_io_len(ti, ci->sector), ci->sector_count); 1400e4c93811SAlasdair G Kergon 1401c80914e8SMike Snitzer r = __clone_and_map_data_bio(ci, ti, ci->sector, &len); 1402c80914e8SMike Snitzer if (r < 0) 1403c80914e8SMike Snitzer return r; 1404e4c93811SAlasdair G Kergon 1405e4c93811SAlasdair G Kergon ci->sector += len; 1406e4c93811SAlasdair G Kergon ci->sector_count -= len; 1407e4c93811SAlasdair G Kergon 1408e4c93811SAlasdair G Kergon return 0; 1409e4c93811SAlasdair G Kergon } 1410e4c93811SAlasdair G Kergon 1411978e51baSMike Snitzer static void init_clone_info(struct clone_info *ci, struct mapped_device *md, 1412978e51baSMike Snitzer struct dm_table *map, struct bio *bio) 1413978e51baSMike Snitzer { 1414978e51baSMike Snitzer ci->map = map; 1415978e51baSMike Snitzer ci->io = alloc_io(md, bio); 1416978e51baSMike Snitzer ci->sector = bio->bi_iter.bi_sector; 1417978e51baSMike Snitzer } 1418978e51baSMike Snitzer 1419e4c93811SAlasdair G Kergon /* 142014fe594dSAlasdair G Kergon * Entry point to split a bio into clones and submit them to the targets. 14211da177e4SLinus Torvalds */ 14223e08773cSChristoph Hellwig static void __split_and_process_bio(struct mapped_device *md, 142383d5e5b0SMikulas Patocka struct dm_table *map, struct bio *bio) 14241da177e4SLinus Torvalds { 14251da177e4SLinus Torvalds struct clone_info ci; 1426512875bdSJun'ichi Nomura int error = 0; 14271da177e4SLinus Torvalds 1428978e51baSMike Snitzer init_clone_info(&ci, md, map, bio); 1429bd2a49b8SAlasdair G Kergon 14301eff9d32SJens Axboe if (bio->bi_opf & REQ_PREFLUSH) { 143114fe594dSAlasdair G Kergon error = __send_empty_flush(&ci); 1432e2118b3cSDamien Le Moal /* dm_io_dec_pending submits any data associated with flush */ 14332e2d6f7eSAjay Joshi } else if (op_is_zone_mgmt(bio_op(bio))) { 1434a4aa5e56SDamien Le Moal ci.bio = bio; 1435a4aa5e56SDamien Le Moal ci.sector_count = 0; 1436a4aa5e56SDamien Le Moal error = __split_and_process_non_flush(&ci); 1437b372d360SMike Snitzer } else { 14386a8736d1STejun Heo ci.bio = bio; 14391da177e4SLinus Torvalds ci.sector_count = bio_sectors(bio); 144014fe594dSAlasdair G Kergon error = __split_and_process_non_flush(&ci); 1441985eabdcSJeffle Xu if (ci.sector_count && !error) { 144218a25da8SNeilBrown /* 1443ed00aabdSChristoph Hellwig * Remainder must be passed to submit_bio_noacct() 144418a25da8SNeilBrown * so that it gets handled *after* bios already submitted 144518a25da8SNeilBrown * have been completely processed. 144618a25da8SNeilBrown * We take a clone of the original to store in 1447745dc570SMike Snitzer * ci.io->orig_bio to be used by end_io_acct() and 144818a25da8SNeilBrown * for dec_pending to use for completion handling. 144918a25da8SNeilBrown */ 1450f21c601aSMike Snitzer struct bio *b = bio_split(bio, bio_sectors(bio) - ci.sector_count, 1451f21c601aSMike Snitzer GFP_NOIO, &md->queue->bio_split); 1452745dc570SMike Snitzer ci.io->orig_bio = b; 1453a1e1cb72SMike Snitzer 145418a25da8SNeilBrown bio_chain(b, bio); 1455eb6f7f7cSChristoph Hellwig trace_block_split(b, bio->bi_iter.bi_sector); 14563e08773cSChristoph Hellwig submit_bio_noacct(bio); 145718a25da8SNeilBrown } 1458d87f4c14STejun Heo } 1459b879f915SMike Snitzer start_io_acct(ci.io); 14601da177e4SLinus Torvalds 14611da177e4SLinus Torvalds /* drop the extra reference count */ 1462e2118b3cSDamien Le Moal dm_io_dec_pending(ci.io, errno_to_blk_status(error)); 14631da177e4SLinus Torvalds } 14641da177e4SLinus Torvalds 14653e08773cSChristoph Hellwig static void dm_submit_bio(struct bio *bio) 14661da177e4SLinus Torvalds { 1467309dca30SChristoph Hellwig struct mapped_device *md = bio->bi_bdev->bd_disk->private_data; 146883d5e5b0SMikulas Patocka int srcu_idx; 146983d5e5b0SMikulas Patocka struct dm_table *map; 14701da177e4SLinus Torvalds 147183d5e5b0SMikulas Patocka map = dm_get_live_table(md, &srcu_idx); 1472b2abdb1bSMike Snitzer if (unlikely(!map)) { 1473b2abdb1bSMike Snitzer DMERR_LIMIT("%s: mapping table unavailable, erroring io", 1474b2abdb1bSMike Snitzer dm_device_name(md)); 14756a8736d1STejun Heo bio_io_error(bio); 1476b2abdb1bSMike Snitzer goto out; 14771da177e4SLinus Torvalds } 147892c63902SMikulas Patocka 1479b2abdb1bSMike Snitzer /* If suspended, queue this IO for later */ 14801da177e4SLinus Torvalds if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) { 14816abc4946SKonstantin Khlebnikov if (bio->bi_opf & REQ_NOWAIT) 14826abc4946SKonstantin Khlebnikov bio_wouldblock_error(bio); 1483b2abdb1bSMike Snitzer else if (bio->bi_opf & REQ_RAHEAD) 14841da177e4SLinus Torvalds bio_io_error(bio); 1485b2abdb1bSMike Snitzer else 1486b2abdb1bSMike Snitzer queue_io(md, bio); 1487b2abdb1bSMike Snitzer goto out; 14881da177e4SLinus Torvalds } 14891da177e4SLinus Torvalds 1490b2abdb1bSMike Snitzer /* 1491b2abdb1bSMike Snitzer * Use blk_queue_split() for abnormal IO (e.g. discard, writesame, etc) 1492b2abdb1bSMike Snitzer * otherwise associated queue_limits won't be imposed. 1493b2abdb1bSMike Snitzer */ 1494b2abdb1bSMike Snitzer if (is_abnormal_io(bio)) 1495b2abdb1bSMike Snitzer blk_queue_split(&bio); 1496978e51baSMike Snitzer 14973e08773cSChristoph Hellwig __split_and_process_bio(md, map, bio); 1498b2abdb1bSMike Snitzer out: 149983d5e5b0SMikulas Patocka dm_put_live_table(md, srcu_idx); 1500978e51baSMike Snitzer } 1501978e51baSMike Snitzer 15021da177e4SLinus Torvalds /*----------------------------------------------------------------- 15031da177e4SLinus Torvalds * An IDR is used to keep track of allocated minor numbers. 15041da177e4SLinus Torvalds *---------------------------------------------------------------*/ 15052b06cfffSAlasdair G Kergon static void free_minor(int minor) 15061da177e4SLinus Torvalds { 1507f32c10b0SJeff Mahoney spin_lock(&_minor_lock); 15081da177e4SLinus Torvalds idr_remove(&_minor_idr, minor); 1509f32c10b0SJeff Mahoney spin_unlock(&_minor_lock); 15101da177e4SLinus Torvalds } 15111da177e4SLinus Torvalds 15121da177e4SLinus Torvalds /* 15131da177e4SLinus Torvalds * See if the device with a specific minor # is free. 15141da177e4SLinus Torvalds */ 1515cf13ab8eSFrederik Deweerdt static int specific_minor(int minor) 15161da177e4SLinus Torvalds { 1517c9d76be6STejun Heo int r; 15181da177e4SLinus Torvalds 15191da177e4SLinus Torvalds if (minor >= (1 << MINORBITS)) 15201da177e4SLinus Torvalds return -EINVAL; 15211da177e4SLinus Torvalds 1522c9d76be6STejun Heo idr_preload(GFP_KERNEL); 1523f32c10b0SJeff Mahoney spin_lock(&_minor_lock); 15241da177e4SLinus Torvalds 1525c9d76be6STejun Heo r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT); 15261da177e4SLinus Torvalds 1527f32c10b0SJeff Mahoney spin_unlock(&_minor_lock); 1528c9d76be6STejun Heo idr_preload_end(); 1529c9d76be6STejun Heo if (r < 0) 1530c9d76be6STejun Heo return r == -ENOSPC ? -EBUSY : r; 1531c9d76be6STejun Heo return 0; 15321da177e4SLinus Torvalds } 15331da177e4SLinus Torvalds 1534cf13ab8eSFrederik Deweerdt static int next_free_minor(int *minor) 15351da177e4SLinus Torvalds { 1536c9d76be6STejun Heo int r; 15371da177e4SLinus Torvalds 1538c9d76be6STejun Heo idr_preload(GFP_KERNEL); 1539f32c10b0SJeff Mahoney spin_lock(&_minor_lock); 15401da177e4SLinus Torvalds 1541c9d76be6STejun Heo r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT); 15421da177e4SLinus Torvalds 1543f32c10b0SJeff Mahoney spin_unlock(&_minor_lock); 1544c9d76be6STejun Heo idr_preload_end(); 1545c9d76be6STejun Heo if (r < 0) 15461da177e4SLinus Torvalds return r; 1547c9d76be6STejun Heo *minor = r; 1548c9d76be6STejun Heo return 0; 15491da177e4SLinus Torvalds } 15501da177e4SLinus Torvalds 155183d5cde4SAlexey Dobriyan static const struct block_device_operations dm_blk_dops; 1552681cc5e8SMike Snitzer static const struct block_device_operations dm_rq_blk_dops; 1553f26c5719SDan Williams static const struct dax_operations dm_dax_ops; 15541da177e4SLinus Torvalds 155553d5914fSMikulas Patocka static void dm_wq_work(struct work_struct *work); 155653d5914fSMikulas Patocka 1557aa6ce87aSSatya Tangirala #ifdef CONFIG_BLK_INLINE_ENCRYPTION 1558cb77cb5aSEric Biggers static void dm_queue_destroy_crypto_profile(struct request_queue *q) 1559aa6ce87aSSatya Tangirala { 1560cb77cb5aSEric Biggers dm_destroy_crypto_profile(q->crypto_profile); 1561aa6ce87aSSatya Tangirala } 1562aa6ce87aSSatya Tangirala 1563aa6ce87aSSatya Tangirala #else /* CONFIG_BLK_INLINE_ENCRYPTION */ 1564aa6ce87aSSatya Tangirala 1565cb77cb5aSEric Biggers static inline void dm_queue_destroy_crypto_profile(struct request_queue *q) 1566aa6ce87aSSatya Tangirala { 1567aa6ce87aSSatya Tangirala } 1568aa6ce87aSSatya Tangirala #endif /* !CONFIG_BLK_INLINE_ENCRYPTION */ 1569aa6ce87aSSatya Tangirala 15700f20972fSMike Snitzer static void cleanup_mapped_device(struct mapped_device *md) 15710f20972fSMike Snitzer { 15720f20972fSMike Snitzer if (md->wq) 15730f20972fSMike Snitzer destroy_workqueue(md->wq); 15746f1c819cSKent Overstreet bioset_exit(&md->bs); 15756f1c819cSKent Overstreet bioset_exit(&md->io_bs); 15760f20972fSMike Snitzer 1577f26c5719SDan Williams if (md->dax_dev) { 1578fb08a190SChristoph Hellwig dax_remove_host(md->disk); 1579f26c5719SDan Williams kill_dax(md->dax_dev); 1580f26c5719SDan Williams put_dax(md->dax_dev); 1581f26c5719SDan Williams md->dax_dev = NULL; 1582f26c5719SDan Williams } 1583f26c5719SDan Williams 15840f20972fSMike Snitzer if (md->disk) { 15850f20972fSMike Snitzer spin_lock(&_minor_lock); 15860f20972fSMike Snitzer md->disk->private_data = NULL; 15870f20972fSMike Snitzer spin_unlock(&_minor_lock); 158889f871afSChristoph Hellwig if (dm_get_md_type(md) != DM_TYPE_NONE) { 158989f871afSChristoph Hellwig dm_sysfs_exit(md); 15900f20972fSMike Snitzer del_gendisk(md->disk); 159189f871afSChristoph Hellwig } 1592cb77cb5aSEric Biggers dm_queue_destroy_crypto_profile(md->queue); 159374fe6ba9SChristoph Hellwig blk_cleanup_disk(md->disk); 159474a2b6ecSChristoph Hellwig } 15950f20972fSMike Snitzer 15969f6dc633SMike Snitzer if (md->pending_io) { 15979f6dc633SMike Snitzer free_percpu(md->pending_io); 15989f6dc633SMike Snitzer md->pending_io = NULL; 15999f6dc633SMike Snitzer } 16009f6dc633SMike Snitzer 1601d09960b0STahsin Erdogan cleanup_srcu_struct(&md->io_barrier); 1602d09960b0STahsin Erdogan 1603d5ffebddSMike Snitzer mutex_destroy(&md->suspend_lock); 1604d5ffebddSMike Snitzer mutex_destroy(&md->type_lock); 1605d5ffebddSMike Snitzer mutex_destroy(&md->table_devices_lock); 1606a666e5c0SMikulas Patocka mutex_destroy(&md->swap_bios_lock); 1607d5ffebddSMike Snitzer 16084cc96131SMike Snitzer dm_mq_cleanup_mapped_device(md); 1609bb37d772SDamien Le Moal dm_cleanup_zoned_dev(md); 16100f20972fSMike Snitzer } 16110f20972fSMike Snitzer 16121da177e4SLinus Torvalds /* 16131da177e4SLinus Torvalds * Allocate and initialise a blank device with a given minor. 16141da177e4SLinus Torvalds */ 16152b06cfffSAlasdair G Kergon static struct mapped_device *alloc_dev(int minor) 16161da177e4SLinus Torvalds { 1617115485e8SMike Snitzer int r, numa_node_id = dm_get_numa_node(); 1618115485e8SMike Snitzer struct mapped_device *md; 1619ba61fdd1SJeff Mahoney void *old_md; 16201da177e4SLinus Torvalds 1621856eb091SMikulas Patocka md = kvzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id); 16221da177e4SLinus Torvalds if (!md) { 16231da177e4SLinus Torvalds DMWARN("unable to allocate device, out of memory."); 16241da177e4SLinus Torvalds return NULL; 16251da177e4SLinus Torvalds } 16261da177e4SLinus Torvalds 162710da4f79SJeff Mahoney if (!try_module_get(THIS_MODULE)) 16286ed7ade8SMilan Broz goto bad_module_get; 162910da4f79SJeff Mahoney 16301da177e4SLinus Torvalds /* get a minor number for the dev */ 16312b06cfffSAlasdair G Kergon if (minor == DM_ANY_MINOR) 1632cf13ab8eSFrederik Deweerdt r = next_free_minor(&minor); 16332b06cfffSAlasdair G Kergon else 1634cf13ab8eSFrederik Deweerdt r = specific_minor(minor); 16351da177e4SLinus Torvalds if (r < 0) 16366ed7ade8SMilan Broz goto bad_minor; 16371da177e4SLinus Torvalds 163883d5e5b0SMikulas Patocka r = init_srcu_struct(&md->io_barrier); 163983d5e5b0SMikulas Patocka if (r < 0) 164083d5e5b0SMikulas Patocka goto bad_io_barrier; 164183d5e5b0SMikulas Patocka 1642115485e8SMike Snitzer md->numa_node_id = numa_node_id; 1643591ddcfcSMike Snitzer md->init_tio_pdu = false; 1644a5664dadSMike Snitzer md->type = DM_TYPE_NONE; 1645e61290a4SDaniel Walker mutex_init(&md->suspend_lock); 1646a5664dadSMike Snitzer mutex_init(&md->type_lock); 164786f1152bSBenjamin Marzinski mutex_init(&md->table_devices_lock); 1648022c2611SMikulas Patocka spin_lock_init(&md->deferred_lock); 16491da177e4SLinus Torvalds atomic_set(&md->holders, 1); 16505c6bd75dSAlasdair G Kergon atomic_set(&md->open_count, 0); 16511da177e4SLinus Torvalds atomic_set(&md->event_nr, 0); 16527a8c3d3bSMike Anderson atomic_set(&md->uevent_seq, 0); 16537a8c3d3bSMike Anderson INIT_LIST_HEAD(&md->uevent_list); 165486f1152bSBenjamin Marzinski INIT_LIST_HEAD(&md->table_devices); 16557a8c3d3bSMike Anderson spin_lock_init(&md->uevent_lock); 16561da177e4SLinus Torvalds 165747ace7e0SMike Snitzer /* 1658c62b37d9SChristoph Hellwig * default to bio-based until DM table is loaded and md->type 1659c62b37d9SChristoph Hellwig * established. If request-based table is loaded: blk-mq will 1660c62b37d9SChristoph Hellwig * override accordingly. 166147ace7e0SMike Snitzer */ 166274fe6ba9SChristoph Hellwig md->disk = blk_alloc_disk(md->numa_node_id); 16631da177e4SLinus Torvalds if (!md->disk) 16640f20972fSMike Snitzer goto bad; 166574fe6ba9SChristoph Hellwig md->queue = md->disk->queue; 16661da177e4SLinus Torvalds 1667f0b04115SJeff Mahoney init_waitqueue_head(&md->wait); 166853d5914fSMikulas Patocka INIT_WORK(&md->work, dm_wq_work); 1669f0b04115SJeff Mahoney init_waitqueue_head(&md->eventq); 16702995fa78SMikulas Patocka init_completion(&md->kobj_holder.completion); 1671f0b04115SJeff Mahoney 1672a666e5c0SMikulas Patocka md->swap_bios = get_swap_bios(); 1673a666e5c0SMikulas Patocka sema_init(&md->swap_bios_semaphore, md->swap_bios); 1674a666e5c0SMikulas Patocka mutex_init(&md->swap_bios_lock); 1675a666e5c0SMikulas Patocka 16761da177e4SLinus Torvalds md->disk->major = _major; 16771da177e4SLinus Torvalds md->disk->first_minor = minor; 167874fe6ba9SChristoph Hellwig md->disk->minors = 1; 16791ebe2e5fSChristoph Hellwig md->disk->flags |= GENHD_FL_NO_PART; 16801da177e4SLinus Torvalds md->disk->fops = &dm_blk_dops; 16811da177e4SLinus Torvalds md->disk->queue = md->queue; 16821da177e4SLinus Torvalds md->disk->private_data = md; 16831da177e4SLinus Torvalds sprintf(md->disk->disk_name, "dm-%d", minor); 1684f26c5719SDan Williams 16855d2a228bSChristoph Hellwig if (IS_ENABLED(CONFIG_FS_DAX)) { 168630c6828aSChristoph Hellwig md->dax_dev = alloc_dax(md, &dm_dax_ops); 1687d7519392SChristoph Hellwig if (IS_ERR(md->dax_dev)) { 1688d7519392SChristoph Hellwig md->dax_dev = NULL; 1689f26c5719SDan Williams goto bad; 1690976431b0SDan Williams } 16917ac5360cSChristoph Hellwig set_dax_nocache(md->dax_dev); 16927ac5360cSChristoph Hellwig set_dax_nomc(md->dax_dev); 1693fb08a190SChristoph Hellwig if (dax_add_host(md->dax_dev, md->disk)) 1694f26c5719SDan Williams goto bad; 1695f26c5719SDan Williams } 16961da177e4SLinus Torvalds 16977e51f257SMike Anderson format_dev_t(md->name, MKDEV(_major, minor)); 16981da177e4SLinus Torvalds 1699c7c879eeSMichał Mirosław md->wq = alloc_workqueue("kdmflush/%s", WQ_MEM_RECLAIM, 0, md->name); 1700304f3f6aSMilan Broz if (!md->wq) 17010f20972fSMike Snitzer goto bad; 1702304f3f6aSMilan Broz 17039f6dc633SMike Snitzer md->pending_io = alloc_percpu(unsigned long); 17049f6dc633SMike Snitzer if (!md->pending_io) 17059f6dc633SMike Snitzer goto bad; 17069f6dc633SMike Snitzer 1707fd2ed4d2SMikulas Patocka dm_stats_init(&md->stats); 1708fd2ed4d2SMikulas Patocka 1709ba61fdd1SJeff Mahoney /* Populate the mapping, nobody knows we exist yet */ 1710f32c10b0SJeff Mahoney spin_lock(&_minor_lock); 1711ba61fdd1SJeff Mahoney old_md = idr_replace(&_minor_idr, md, minor); 1712f32c10b0SJeff Mahoney spin_unlock(&_minor_lock); 1713ba61fdd1SJeff Mahoney 1714ba61fdd1SJeff Mahoney BUG_ON(old_md != MINOR_ALLOCED); 1715ba61fdd1SJeff Mahoney 17161da177e4SLinus Torvalds return md; 17171da177e4SLinus Torvalds 17180f20972fSMike Snitzer bad: 17190f20972fSMike Snitzer cleanup_mapped_device(md); 172083d5e5b0SMikulas Patocka bad_io_barrier: 17211da177e4SLinus Torvalds free_minor(minor); 17226ed7ade8SMilan Broz bad_minor: 172310da4f79SJeff Mahoney module_put(THIS_MODULE); 17246ed7ade8SMilan Broz bad_module_get: 1725856eb091SMikulas Patocka kvfree(md); 17261da177e4SLinus Torvalds return NULL; 17271da177e4SLinus Torvalds } 17281da177e4SLinus Torvalds 1729ae9da83fSJun'ichi Nomura static void unlock_fs(struct mapped_device *md); 1730ae9da83fSJun'ichi Nomura 17311da177e4SLinus Torvalds static void free_dev(struct mapped_device *md) 17321da177e4SLinus Torvalds { 1733f331c029STejun Heo int minor = MINOR(disk_devt(md->disk)); 173463d94e48SJun'ichi Nomura 1735ae9da83fSJun'ichi Nomura unlock_fs(md); 17362eb6e1e3SKeith Busch 17370f20972fSMike Snitzer cleanup_mapped_device(md); 17380f20972fSMike Snitzer 17390f20972fSMike Snitzer free_table_devices(&md->table_devices); 17400f20972fSMike Snitzer dm_stats_cleanup(&md->stats); 174163a4f065SMike Snitzer free_minor(minor); 174263a4f065SMike Snitzer 174310da4f79SJeff Mahoney module_put(THIS_MODULE); 1744856eb091SMikulas Patocka kvfree(md); 17451da177e4SLinus Torvalds } 17461da177e4SLinus Torvalds 17472a2a4c51SJens Axboe static int __bind_mempools(struct mapped_device *md, struct dm_table *t) 1748e6ee8c0bSKiyoshi Ueda { 1749c0820cf5SMikulas Patocka struct dm_md_mempools *p = dm_table_get_md_mempools(t); 17502a2a4c51SJens Axboe int ret = 0; 1751e6ee8c0bSKiyoshi Ueda 1752545ed20eSToshi Kani if (dm_table_bio_based(t)) { 1753c0820cf5SMikulas Patocka /* 175464f52b0eSMike Snitzer * The md may already have mempools that need changing. 175564f52b0eSMike Snitzer * If so, reload bioset because front_pad may have changed 175616245bdcSJun'ichi Nomura * because a different table was loaded. 1757c0820cf5SMikulas Patocka */ 17586f1c819cSKent Overstreet bioset_exit(&md->bs); 17596f1c819cSKent Overstreet bioset_exit(&md->io_bs); 17600776aa0eSMike Snitzer 17616f1c819cSKent Overstreet } else if (bioset_initialized(&md->bs)) { 1762cbc4e3c1SMike Snitzer /* 17634e6e36c3SMike Snitzer * There's no need to reload with request-based dm 17644e6e36c3SMike Snitzer * because the size of front_pad doesn't change. 17654e6e36c3SMike Snitzer * Note for future: If you are to reload bioset, 17664e6e36c3SMike Snitzer * prep-ed requests in the queue may refer 17674e6e36c3SMike Snitzer * to bio from the old bioset, so you must walk 17684e6e36c3SMike Snitzer * through the queue to unprep. 1769cbc4e3c1SMike Snitzer */ 1770cbc4e3c1SMike Snitzer goto out; 1771cbc4e3c1SMike Snitzer } 1772cbc4e3c1SMike Snitzer 17736f1c819cSKent Overstreet BUG_ON(!p || 17746f1c819cSKent Overstreet bioset_initialized(&md->bs) || 17756f1c819cSKent Overstreet bioset_initialized(&md->io_bs)); 1776e6ee8c0bSKiyoshi Ueda 17772a2a4c51SJens Axboe ret = bioset_init_from_src(&md->bs, &p->bs); 17782a2a4c51SJens Axboe if (ret) 17792a2a4c51SJens Axboe goto out; 17802a2a4c51SJens Axboe ret = bioset_init_from_src(&md->io_bs, &p->io_bs); 17812a2a4c51SJens Axboe if (ret) 17822a2a4c51SJens Axboe bioset_exit(&md->bs); 1783e6ee8c0bSKiyoshi Ueda out: 178402233342SMike Snitzer /* mempool bind completed, no longer need any mempools in the table */ 1785e6ee8c0bSKiyoshi Ueda dm_table_free_md_mempools(t); 17862a2a4c51SJens Axboe return ret; 1787e6ee8c0bSKiyoshi Ueda } 1788e6ee8c0bSKiyoshi Ueda 17891da177e4SLinus Torvalds /* 17901da177e4SLinus Torvalds * Bind a table to the device. 17911da177e4SLinus Torvalds */ 17921da177e4SLinus Torvalds static void event_callback(void *context) 17931da177e4SLinus Torvalds { 17947a8c3d3bSMike Anderson unsigned long flags; 17957a8c3d3bSMike Anderson LIST_HEAD(uevents); 17961da177e4SLinus Torvalds struct mapped_device *md = (struct mapped_device *) context; 17971da177e4SLinus Torvalds 17987a8c3d3bSMike Anderson spin_lock_irqsave(&md->uevent_lock, flags); 17997a8c3d3bSMike Anderson list_splice_init(&md->uevent_list, &uevents); 18007a8c3d3bSMike Anderson spin_unlock_irqrestore(&md->uevent_lock, flags); 18017a8c3d3bSMike Anderson 1802ed9e1982STejun Heo dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj); 18037a8c3d3bSMike Anderson 18041da177e4SLinus Torvalds atomic_inc(&md->event_nr); 18051da177e4SLinus Torvalds wake_up(&md->eventq); 180662e08243SMikulas Patocka dm_issue_global_event(); 18071da177e4SLinus Torvalds } 18081da177e4SLinus Torvalds 1809c217649bSMike Snitzer /* 1810042d2a9bSAlasdair G Kergon * Returns old map, which caller must destroy. 1811042d2a9bSAlasdair G Kergon */ 1812042d2a9bSAlasdair G Kergon static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t, 1813754c5fc7SMike Snitzer struct queue_limits *limits) 18141da177e4SLinus Torvalds { 1815042d2a9bSAlasdair G Kergon struct dm_table *old_map; 1816165125e1SJens Axboe struct request_queue *q = md->queue; 1817978e51baSMike Snitzer bool request_based = dm_table_request_based(t); 18181da177e4SLinus Torvalds sector_t size; 18192a2a4c51SJens Axboe int ret; 18201da177e4SLinus Torvalds 18215a8f1f80SBart Van Assche lockdep_assert_held(&md->suspend_lock); 18225a8f1f80SBart Van Assche 18231da177e4SLinus Torvalds size = dm_table_get_size(t); 18243ac51e74SDarrick J. Wong 18253ac51e74SDarrick J. Wong /* 18263ac51e74SDarrick J. Wong * Wipe any geometry if the size of the table changed. 18273ac51e74SDarrick J. Wong */ 1828fd2ed4d2SMikulas Patocka if (size != dm_get_size(md)) 18293ac51e74SDarrick J. Wong memset(&md->geometry, 0, sizeof(md->geometry)); 18303ac51e74SDarrick J. Wong 18315424a0b8SMikulas Patocka if (!get_capacity(md->disk)) 18325424a0b8SMikulas Patocka set_capacity(md->disk, size); 18335424a0b8SMikulas Patocka else 1834f64d9b2eSChristoph Hellwig set_capacity_and_notify(md->disk, size); 18351da177e4SLinus Torvalds 1836cf222b37SAlasdair G Kergon dm_table_event_callback(t, event_callback, md); 18372ca3310eSAlasdair G Kergon 18389c37de29SMike Snitzer if (request_based) { 183916f12266SMike Snitzer /* 18409c37de29SMike Snitzer * Leverage the fact that request-based DM targets are 18419c37de29SMike Snitzer * immutable singletons - used to optimize dm_mq_queue_rq. 184216f12266SMike Snitzer */ 184316f12266SMike Snitzer md->immutable_target = dm_table_get_immutable_target(t); 184416f12266SMike Snitzer } 1845e6ee8c0bSKiyoshi Ueda 18462a2a4c51SJens Axboe ret = __bind_mempools(md, t); 18472a2a4c51SJens Axboe if (ret) { 18482a2a4c51SJens Axboe old_map = ERR_PTR(ret); 18492a2a4c51SJens Axboe goto out; 18502a2a4c51SJens Axboe } 1851e6ee8c0bSKiyoshi Ueda 1852bb37d772SDamien Le Moal ret = dm_table_set_restrictions(t, q, limits); 1853bb37d772SDamien Le Moal if (ret) { 1854bb37d772SDamien Le Moal old_map = ERR_PTR(ret); 1855bb37d772SDamien Le Moal goto out; 1856bb37d772SDamien Le Moal } 1857bb37d772SDamien Le Moal 1858a12f5d48SEric Dumazet old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 18591d3aa6f6SMike Snitzer rcu_assign_pointer(md->map, (void *)t); 186036a0456fSAlasdair G Kergon md->immutable_target_type = dm_table_get_immutable_target_type(t); 186136a0456fSAlasdair G Kergon 186241abc4e1SHannes Reinecke if (old_map) 186383d5e5b0SMikulas Patocka dm_sync_table(md); 18642ca3310eSAlasdair G Kergon 18652a2a4c51SJens Axboe out: 1866042d2a9bSAlasdair G Kergon return old_map; 18671da177e4SLinus Torvalds } 18681da177e4SLinus Torvalds 1869a7940155SAlasdair G Kergon /* 1870a7940155SAlasdair G Kergon * Returns unbound table for the caller to free. 1871a7940155SAlasdair G Kergon */ 1872a7940155SAlasdair G Kergon static struct dm_table *__unbind(struct mapped_device *md) 18731da177e4SLinus Torvalds { 1874a12f5d48SEric Dumazet struct dm_table *map = rcu_dereference_protected(md->map, 1); 18751da177e4SLinus Torvalds 18761da177e4SLinus Torvalds if (!map) 1877a7940155SAlasdair G Kergon return NULL; 18781da177e4SLinus Torvalds 18791da177e4SLinus Torvalds dm_table_event_callback(map, NULL, NULL); 18809cdb8520SMonam Agarwal RCU_INIT_POINTER(md->map, NULL); 188183d5e5b0SMikulas Patocka dm_sync_table(md); 1882a7940155SAlasdair G Kergon 1883a7940155SAlasdair G Kergon return map; 18841da177e4SLinus Torvalds } 18851da177e4SLinus Torvalds 18861da177e4SLinus Torvalds /* 18871da177e4SLinus Torvalds * Constructor for a new device. 18881da177e4SLinus Torvalds */ 18892b06cfffSAlasdair G Kergon int dm_create(int minor, struct mapped_device **result) 18901da177e4SLinus Torvalds { 18911da177e4SLinus Torvalds struct mapped_device *md; 18921da177e4SLinus Torvalds 18932b06cfffSAlasdair G Kergon md = alloc_dev(minor); 18941da177e4SLinus Torvalds if (!md) 18951da177e4SLinus Torvalds return -ENXIO; 18961da177e4SLinus Torvalds 189791ccbbacSTushar Sugandhi dm_ima_reset_data(md); 189891ccbbacSTushar Sugandhi 18991da177e4SLinus Torvalds *result = md; 19001da177e4SLinus Torvalds return 0; 19011da177e4SLinus Torvalds } 19021da177e4SLinus Torvalds 1903a5664dadSMike Snitzer /* 1904a5664dadSMike Snitzer * Functions to manage md->type. 1905a5664dadSMike Snitzer * All are required to hold md->type_lock. 1906a5664dadSMike Snitzer */ 1907a5664dadSMike Snitzer void dm_lock_md_type(struct mapped_device *md) 1908a5664dadSMike Snitzer { 1909a5664dadSMike Snitzer mutex_lock(&md->type_lock); 1910a5664dadSMike Snitzer } 1911a5664dadSMike Snitzer 1912a5664dadSMike Snitzer void dm_unlock_md_type(struct mapped_device *md) 1913a5664dadSMike Snitzer { 1914a5664dadSMike Snitzer mutex_unlock(&md->type_lock); 1915a5664dadSMike Snitzer } 1916a5664dadSMike Snitzer 19177e0d574fSBart Van Assche void dm_set_md_type(struct mapped_device *md, enum dm_queue_mode type) 1918a5664dadSMike Snitzer { 191900c4fc3bSMike Snitzer BUG_ON(!mutex_is_locked(&md->type_lock)); 1920a5664dadSMike Snitzer md->type = type; 1921a5664dadSMike Snitzer } 1922a5664dadSMike Snitzer 19237e0d574fSBart Van Assche enum dm_queue_mode dm_get_md_type(struct mapped_device *md) 1924a5664dadSMike Snitzer { 1925a5664dadSMike Snitzer return md->type; 1926a5664dadSMike Snitzer } 1927a5664dadSMike Snitzer 192836a0456fSAlasdair G Kergon struct target_type *dm_get_immutable_target_type(struct mapped_device *md) 192936a0456fSAlasdair G Kergon { 193036a0456fSAlasdair G Kergon return md->immutable_target_type; 193136a0456fSAlasdair G Kergon } 193236a0456fSAlasdair G Kergon 19334a0b4ddfSMike Snitzer /* 1934f84cb8a4SMike Snitzer * The queue_limits are only valid as long as you have a reference 1935f84cb8a4SMike Snitzer * count on 'md'. 1936f84cb8a4SMike Snitzer */ 1937f84cb8a4SMike Snitzer struct queue_limits *dm_get_queue_limits(struct mapped_device *md) 1938f84cb8a4SMike Snitzer { 1939f84cb8a4SMike Snitzer BUG_ON(!atomic_read(&md->holders)); 1940f84cb8a4SMike Snitzer return &md->queue->limits; 1941f84cb8a4SMike Snitzer } 1942f84cb8a4SMike Snitzer EXPORT_SYMBOL_GPL(dm_get_queue_limits); 1943f84cb8a4SMike Snitzer 19444a0b4ddfSMike Snitzer /* 19454a0b4ddfSMike Snitzer * Setup the DM device's queue based on md's type 19464a0b4ddfSMike Snitzer */ 1947591ddcfcSMike Snitzer int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t) 19484a0b4ddfSMike Snitzer { 1949ba305859SChristoph Hellwig enum dm_queue_mode type = dm_table_get_type(t); 1950c100ec49SMike Snitzer struct queue_limits limits; 1951ba305859SChristoph Hellwig int r; 1952bfebd1cdSMike Snitzer 1953545ed20eSToshi Kani switch (type) { 1954bfebd1cdSMike Snitzer case DM_TYPE_REQUEST_BASED: 1955681cc5e8SMike Snitzer md->disk->fops = &dm_rq_blk_dops; 1956e83068a5SMike Snitzer r = dm_mq_init_request_queue(md, t); 1957bfebd1cdSMike Snitzer if (r) { 1958681cc5e8SMike Snitzer DMERR("Cannot initialize queue for request-based dm mapped device"); 1959bfebd1cdSMike Snitzer return r; 1960bfebd1cdSMike Snitzer } 1961bfebd1cdSMike Snitzer break; 1962bfebd1cdSMike Snitzer case DM_TYPE_BIO_BASED: 1963545ed20eSToshi Kani case DM_TYPE_DAX_BIO_BASED: 1964bfebd1cdSMike Snitzer break; 19657e0d574fSBart Van Assche case DM_TYPE_NONE: 19667e0d574fSBart Van Assche WARN_ON_ONCE(true); 19677e0d574fSBart Van Assche break; 1968ff36ab34SMike Snitzer } 19694a0b4ddfSMike Snitzer 1970c100ec49SMike Snitzer r = dm_calculate_queue_limits(t, &limits); 1971c100ec49SMike Snitzer if (r) { 1972c100ec49SMike Snitzer DMERR("Cannot calculate initial queue limits"); 1973c100ec49SMike Snitzer return r; 1974c100ec49SMike Snitzer } 1975bb37d772SDamien Le Moal r = dm_table_set_restrictions(t, md->queue, &limits); 1976bb37d772SDamien Le Moal if (r) 1977bb37d772SDamien Le Moal return r; 197889f871afSChristoph Hellwig 1979e7089f65SLuis Chamberlain r = add_disk(md->disk); 1980e7089f65SLuis Chamberlain if (r) 1981e7089f65SLuis Chamberlain return r; 198289f871afSChristoph Hellwig 198389f871afSChristoph Hellwig r = dm_sysfs_init(md); 198489f871afSChristoph Hellwig if (r) { 198589f871afSChristoph Hellwig del_gendisk(md->disk); 198689f871afSChristoph Hellwig return r; 198789f871afSChristoph Hellwig } 1988ba305859SChristoph Hellwig md->type = type; 19894a0b4ddfSMike Snitzer return 0; 19904a0b4ddfSMike Snitzer } 19914a0b4ddfSMike Snitzer 19922bec1f4aSMikulas Patocka struct mapped_device *dm_get_md(dev_t dev) 19931da177e4SLinus Torvalds { 19941da177e4SLinus Torvalds struct mapped_device *md; 19951da177e4SLinus Torvalds unsigned minor = MINOR(dev); 19961da177e4SLinus Torvalds 19971da177e4SLinus Torvalds if (MAJOR(dev) != _major || minor >= (1 << MINORBITS)) 19981da177e4SLinus Torvalds return NULL; 19991da177e4SLinus Torvalds 2000f32c10b0SJeff Mahoney spin_lock(&_minor_lock); 20011da177e4SLinus Torvalds 20021da177e4SLinus Torvalds md = idr_find(&_minor_idr, minor); 200349de5769SMike Snitzer if (!md || md == MINOR_ALLOCED || (MINOR(disk_devt(dm_disk(md))) != minor) || 200449de5769SMike Snitzer test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) { 2005637842cfSDavid Teigland md = NULL; 2006fba9f90eSJeff Mahoney goto out; 2007fba9f90eSJeff Mahoney } 20082bec1f4aSMikulas Patocka dm_get(md); 2009fba9f90eSJeff Mahoney out: 2010f32c10b0SJeff Mahoney spin_unlock(&_minor_lock); 20111da177e4SLinus Torvalds 2012637842cfSDavid Teigland return md; 2013637842cfSDavid Teigland } 20143cf2e4baSAlasdair G Kergon EXPORT_SYMBOL_GPL(dm_get_md); 2015d229a958SDavid Teigland 20169ade92a9SAlasdair G Kergon void *dm_get_mdptr(struct mapped_device *md) 2017637842cfSDavid Teigland { 20189ade92a9SAlasdair G Kergon return md->interface_ptr; 20191da177e4SLinus Torvalds } 20201da177e4SLinus Torvalds 20211da177e4SLinus Torvalds void dm_set_mdptr(struct mapped_device *md, void *ptr) 20221da177e4SLinus Torvalds { 20231da177e4SLinus Torvalds md->interface_ptr = ptr; 20241da177e4SLinus Torvalds } 20251da177e4SLinus Torvalds 20261da177e4SLinus Torvalds void dm_get(struct mapped_device *md) 20271da177e4SLinus Torvalds { 20281da177e4SLinus Torvalds atomic_inc(&md->holders); 20293f77316dSKiyoshi Ueda BUG_ON(test_bit(DMF_FREEING, &md->flags)); 20301da177e4SLinus Torvalds } 20311da177e4SLinus Torvalds 203209ee96b2SMikulas Patocka int dm_hold(struct mapped_device *md) 203309ee96b2SMikulas Patocka { 203409ee96b2SMikulas Patocka spin_lock(&_minor_lock); 203509ee96b2SMikulas Patocka if (test_bit(DMF_FREEING, &md->flags)) { 203609ee96b2SMikulas Patocka spin_unlock(&_minor_lock); 203709ee96b2SMikulas Patocka return -EBUSY; 203809ee96b2SMikulas Patocka } 203909ee96b2SMikulas Patocka dm_get(md); 204009ee96b2SMikulas Patocka spin_unlock(&_minor_lock); 204109ee96b2SMikulas Patocka return 0; 204209ee96b2SMikulas Patocka } 204309ee96b2SMikulas Patocka EXPORT_SYMBOL_GPL(dm_hold); 204409ee96b2SMikulas Patocka 204572d94861SAlasdair G Kergon const char *dm_device_name(struct mapped_device *md) 204672d94861SAlasdair G Kergon { 204772d94861SAlasdair G Kergon return md->name; 204872d94861SAlasdair G Kergon } 204972d94861SAlasdair G Kergon EXPORT_SYMBOL_GPL(dm_device_name); 205072d94861SAlasdair G Kergon 20513f77316dSKiyoshi Ueda static void __dm_destroy(struct mapped_device *md, bool wait) 20521da177e4SLinus Torvalds { 20531134e5aeSMike Anderson struct dm_table *map; 205483d5e5b0SMikulas Patocka int srcu_idx; 20551da177e4SLinus Torvalds 20563f77316dSKiyoshi Ueda might_sleep(); 2057fba9f90eSJeff Mahoney 205863a4f065SMike Snitzer spin_lock(&_minor_lock); 20593f77316dSKiyoshi Ueda idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md)))); 2060fba9f90eSJeff Mahoney set_bit(DMF_FREEING, &md->flags); 2061f32c10b0SJeff Mahoney spin_unlock(&_minor_lock); 20623f77316dSKiyoshi Ueda 2063c12c9a3cSMike Snitzer blk_set_queue_dying(md->queue); 20643b785fbcSBart Van Assche 2065ab7c7bb6SMikulas Patocka /* 2066ab7c7bb6SMikulas Patocka * Take suspend_lock so that presuspend and postsuspend methods 2067ab7c7bb6SMikulas Patocka * do not race with internal suspend. 2068ab7c7bb6SMikulas Patocka */ 2069ab7c7bb6SMikulas Patocka mutex_lock(&md->suspend_lock); 20702a708cffSJunichi Nomura map = dm_get_live_table(md, &srcu_idx); 20714f186f8bSKiyoshi Ueda if (!dm_suspended_md(md)) { 20721da177e4SLinus Torvalds dm_table_presuspend_targets(map); 2073adc0daadSMikulas Patocka set_bit(DMF_SUSPENDED, &md->flags); 20745df96f2bSMikulas Patocka set_bit(DMF_POST_SUSPENDING, &md->flags); 20751da177e4SLinus Torvalds dm_table_postsuspend_targets(map); 20761da177e4SLinus Torvalds } 207783d5e5b0SMikulas Patocka /* dm_put_live_table must be before msleep, otherwise deadlock is possible */ 207883d5e5b0SMikulas Patocka dm_put_live_table(md, srcu_idx); 20792a708cffSJunichi Nomura mutex_unlock(&md->suspend_lock); 208083d5e5b0SMikulas Patocka 20813f77316dSKiyoshi Ueda /* 20823f77316dSKiyoshi Ueda * Rare, but there may be I/O requests still going to complete, 20833f77316dSKiyoshi Ueda * for example. Wait for all references to disappear. 20843f77316dSKiyoshi Ueda * No one should increment the reference count of the mapped_device, 20853f77316dSKiyoshi Ueda * after the mapped_device state becomes DMF_FREEING. 20863f77316dSKiyoshi Ueda */ 20873f77316dSKiyoshi Ueda if (wait) 20883f77316dSKiyoshi Ueda while (atomic_read(&md->holders)) 20893f77316dSKiyoshi Ueda msleep(1); 20903f77316dSKiyoshi Ueda else if (atomic_read(&md->holders)) 20913f77316dSKiyoshi Ueda DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)", 20923f77316dSKiyoshi Ueda dm_device_name(md), atomic_read(&md->holders)); 20933f77316dSKiyoshi Ueda 2094a7940155SAlasdair G Kergon dm_table_destroy(__unbind(md)); 20951da177e4SLinus Torvalds free_dev(md); 20961da177e4SLinus Torvalds } 20973f77316dSKiyoshi Ueda 20983f77316dSKiyoshi Ueda void dm_destroy(struct mapped_device *md) 20993f77316dSKiyoshi Ueda { 21003f77316dSKiyoshi Ueda __dm_destroy(md, true); 21013f77316dSKiyoshi Ueda } 21023f77316dSKiyoshi Ueda 21033f77316dSKiyoshi Ueda void dm_destroy_immediate(struct mapped_device *md) 21043f77316dSKiyoshi Ueda { 21053f77316dSKiyoshi Ueda __dm_destroy(md, false); 21063f77316dSKiyoshi Ueda } 21073f77316dSKiyoshi Ueda 21083f77316dSKiyoshi Ueda void dm_put(struct mapped_device *md) 21093f77316dSKiyoshi Ueda { 21103f77316dSKiyoshi Ueda atomic_dec(&md->holders); 21111da177e4SLinus Torvalds } 211279eb885cSEdward Goggin EXPORT_SYMBOL_GPL(dm_put); 21131da177e4SLinus Torvalds 21149f6dc633SMike Snitzer static bool dm_in_flight_bios(struct mapped_device *md) 211585067747SMing Lei { 211685067747SMing Lei int cpu; 21179f6dc633SMike Snitzer unsigned long sum = 0; 211885067747SMing Lei 21199f6dc633SMike Snitzer for_each_possible_cpu(cpu) 21209f6dc633SMike Snitzer sum += *per_cpu_ptr(md->pending_io, cpu); 212185067747SMing Lei 212285067747SMing Lei return sum != 0; 212385067747SMing Lei } 212485067747SMing Lei 21252f064a59SPeter Zijlstra static int dm_wait_for_bios_completion(struct mapped_device *md, unsigned int task_state) 212646125c1cSMilan Broz { 212746125c1cSMilan Broz int r = 0; 21289f4c3f87SBart Van Assche DEFINE_WAIT(wait); 212946125c1cSMilan Broz 213085067747SMing Lei while (true) { 21319f4c3f87SBart Van Assche prepare_to_wait(&md->wait, &wait, task_state); 213246125c1cSMilan Broz 21339f6dc633SMike Snitzer if (!dm_in_flight_bios(md)) 213446125c1cSMilan Broz break; 213546125c1cSMilan Broz 2136e3fabdfdSBart Van Assche if (signal_pending_state(task_state, current)) { 213746125c1cSMilan Broz r = -EINTR; 213846125c1cSMilan Broz break; 213946125c1cSMilan Broz } 214046125c1cSMilan Broz 214146125c1cSMilan Broz io_schedule(); 214246125c1cSMilan Broz } 21439f4c3f87SBart Van Assche finish_wait(&md->wait, &wait); 2144b44ebeb0SMikulas Patocka 21459f6dc633SMike Snitzer smp_rmb(); 21469f6dc633SMike Snitzer 214746125c1cSMilan Broz return r; 214846125c1cSMilan Broz } 214946125c1cSMilan Broz 21502f064a59SPeter Zijlstra static int dm_wait_for_completion(struct mapped_device *md, unsigned int task_state) 215185067747SMing Lei { 215285067747SMing Lei int r = 0; 215385067747SMing Lei 215485067747SMing Lei if (!queue_is_mq(md->queue)) 215585067747SMing Lei return dm_wait_for_bios_completion(md, task_state); 215685067747SMing Lei 215785067747SMing Lei while (true) { 215885067747SMing Lei if (!blk_mq_queue_inflight(md->queue)) 215985067747SMing Lei break; 216085067747SMing Lei 216185067747SMing Lei if (signal_pending_state(task_state, current)) { 216285067747SMing Lei r = -EINTR; 216385067747SMing Lei break; 216485067747SMing Lei } 216585067747SMing Lei 216685067747SMing Lei msleep(5); 216785067747SMing Lei } 216885067747SMing Lei 216985067747SMing Lei return r; 217085067747SMing Lei } 217185067747SMing Lei 21721da177e4SLinus Torvalds /* 21731da177e4SLinus Torvalds * Process the deferred bios 21741da177e4SLinus Torvalds */ 2175ef208587SMikulas Patocka static void dm_wq_work(struct work_struct *work) 21761da177e4SLinus Torvalds { 21770c2915b8SMike Snitzer struct mapped_device *md = container_of(work, struct mapped_device, work); 21780c2915b8SMike Snitzer struct bio *bio; 2179ef208587SMikulas Patocka 21803b00b203SMikulas Patocka while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) { 2181022c2611SMikulas Patocka spin_lock_irq(&md->deferred_lock); 21820c2915b8SMike Snitzer bio = bio_list_pop(&md->deferred); 2183022c2611SMikulas Patocka spin_unlock_irq(&md->deferred_lock); 2184022c2611SMikulas Patocka 21850c2915b8SMike Snitzer if (!bio) 2186df12ee99SAlasdair G Kergon break; 218773d410c0SMilan Broz 21880c2915b8SMike Snitzer submit_bio_noacct(bio); 2189e6ee8c0bSKiyoshi Ueda } 21901da177e4SLinus Torvalds } 21911da177e4SLinus Torvalds 21929a1fb464SMikulas Patocka static void dm_queue_flush(struct mapped_device *md) 2193304f3f6aSMilan Broz { 21943b00b203SMikulas Patocka clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 21954e857c58SPeter Zijlstra smp_mb__after_atomic(); 219653d5914fSMikulas Patocka queue_work(md->wq, &md->work); 2197304f3f6aSMilan Broz } 2198304f3f6aSMilan Broz 21991da177e4SLinus Torvalds /* 2200042d2a9bSAlasdair G Kergon * Swap in a new table, returning the old one for the caller to destroy. 22011da177e4SLinus Torvalds */ 2202042d2a9bSAlasdair G Kergon struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table) 22031da177e4SLinus Torvalds { 220487eb5b21SMike Christie struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL); 2205754c5fc7SMike Snitzer struct queue_limits limits; 2206042d2a9bSAlasdair G Kergon int r; 22071da177e4SLinus Torvalds 2208e61290a4SDaniel Walker mutex_lock(&md->suspend_lock); 22091da177e4SLinus Torvalds 22101da177e4SLinus Torvalds /* device must be suspended */ 22114f186f8bSKiyoshi Ueda if (!dm_suspended_md(md)) 221293c534aeSAlasdair G Kergon goto out; 22131da177e4SLinus Torvalds 22143ae70656SMike Snitzer /* 22153ae70656SMike Snitzer * If the new table has no data devices, retain the existing limits. 22163ae70656SMike Snitzer * This helps multipath with queue_if_no_path if all paths disappear, 22173ae70656SMike Snitzer * then new I/O is queued based on these limits, and then some paths 22183ae70656SMike Snitzer * reappear. 22193ae70656SMike Snitzer */ 22203ae70656SMike Snitzer if (dm_table_has_no_data_devices(table)) { 222183d5e5b0SMikulas Patocka live_map = dm_get_live_table_fast(md); 22223ae70656SMike Snitzer if (live_map) 22233ae70656SMike Snitzer limits = md->queue->limits; 222483d5e5b0SMikulas Patocka dm_put_live_table_fast(md); 22253ae70656SMike Snitzer } 22263ae70656SMike Snitzer 222787eb5b21SMike Christie if (!live_map) { 2228754c5fc7SMike Snitzer r = dm_calculate_queue_limits(table, &limits); 2229042d2a9bSAlasdair G Kergon if (r) { 2230042d2a9bSAlasdair G Kergon map = ERR_PTR(r); 2231754c5fc7SMike Snitzer goto out; 2232042d2a9bSAlasdair G Kergon } 223387eb5b21SMike Christie } 2234754c5fc7SMike Snitzer 2235042d2a9bSAlasdair G Kergon map = __bind(md, table, &limits); 223662e08243SMikulas Patocka dm_issue_global_event(); 22371da177e4SLinus Torvalds 223893c534aeSAlasdair G Kergon out: 2239e61290a4SDaniel Walker mutex_unlock(&md->suspend_lock); 2240042d2a9bSAlasdair G Kergon return map; 22411da177e4SLinus Torvalds } 22421da177e4SLinus Torvalds 22431da177e4SLinus Torvalds /* 22441da177e4SLinus Torvalds * Functions to lock and unlock any filesystem running on the 22451da177e4SLinus Torvalds * device. 22461da177e4SLinus Torvalds */ 22472ca3310eSAlasdair G Kergon static int lock_fs(struct mapped_device *md) 22481da177e4SLinus Torvalds { 2249e39e2e95SAlasdair G Kergon int r; 22501da177e4SLinus Torvalds 2251040f04bdSChristoph Hellwig WARN_ON(test_bit(DMF_FROZEN, &md->flags)); 2252dfbe03f6SAlasdair G Kergon 2253977115c0SChristoph Hellwig r = freeze_bdev(md->disk->part0); 2254040f04bdSChristoph Hellwig if (!r) 2255aa8d7c2fSAlasdair G Kergon set_bit(DMF_FROZEN, &md->flags); 2256040f04bdSChristoph Hellwig return r; 22571da177e4SLinus Torvalds } 22581da177e4SLinus Torvalds 22592ca3310eSAlasdair G Kergon static void unlock_fs(struct mapped_device *md) 22601da177e4SLinus Torvalds { 2261aa8d7c2fSAlasdair G Kergon if (!test_bit(DMF_FROZEN, &md->flags)) 2262aa8d7c2fSAlasdair G Kergon return; 2263977115c0SChristoph Hellwig thaw_bdev(md->disk->part0); 2264aa8d7c2fSAlasdair G Kergon clear_bit(DMF_FROZEN, &md->flags); 22651da177e4SLinus Torvalds } 22661da177e4SLinus Torvalds 22671da177e4SLinus Torvalds /* 2268b48633f8SBart Van Assche * @suspend_flags: DM_SUSPEND_LOCKFS_FLAG and/or DM_SUSPEND_NOFLUSH_FLAG 2269b48633f8SBart Van Assche * @task_state: e.g. TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE 2270b48633f8SBart Van Assche * @dmf_suspended_flag: DMF_SUSPENDED or DMF_SUSPENDED_INTERNALLY 2271b48633f8SBart Van Assche * 2272ffcc3936SMike Snitzer * If __dm_suspend returns 0, the device is completely quiescent 2273ffcc3936SMike Snitzer * now. There is no request-processing activity. All new requests 2274ffcc3936SMike Snitzer * are being added to md->deferred list. 2275cec47e3dSKiyoshi Ueda */ 2276ffcc3936SMike Snitzer static int __dm_suspend(struct mapped_device *md, struct dm_table *map, 22772f064a59SPeter Zijlstra unsigned suspend_flags, unsigned int task_state, 2278eaf9a736SMike Snitzer int dmf_suspended_flag) 22791da177e4SLinus Torvalds { 2280ffcc3936SMike Snitzer bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG; 2281ffcc3936SMike Snitzer bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG; 2282ffcc3936SMike Snitzer int r; 2283cf222b37SAlasdair G Kergon 22845a8f1f80SBart Van Assche lockdep_assert_held(&md->suspend_lock); 22855a8f1f80SBart Van Assche 22862e93ccc1SKiyoshi Ueda /* 22872e93ccc1SKiyoshi Ueda * DMF_NOFLUSH_SUSPENDING must be set before presuspend. 22882e93ccc1SKiyoshi Ueda * This flag is cleared before dm_suspend returns. 22892e93ccc1SKiyoshi Ueda */ 22902e93ccc1SKiyoshi Ueda if (noflush) 22912e93ccc1SKiyoshi Ueda set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 229286331f39SBart Van Assche else 2293ac75b09fSMike Snitzer DMDEBUG("%s: suspending with flush", dm_device_name(md)); 22942e93ccc1SKiyoshi Ueda 2295d67ee213SMike Snitzer /* 2296d67ee213SMike Snitzer * This gets reverted if there's an error later and the targets 2297d67ee213SMike Snitzer * provide the .presuspend_undo hook. 2298d67ee213SMike Snitzer */ 22991da177e4SLinus Torvalds dm_table_presuspend_targets(map); 23001da177e4SLinus Torvalds 23012e93ccc1SKiyoshi Ueda /* 23029f518b27SKiyoshi Ueda * Flush I/O to the device. 23039f518b27SKiyoshi Ueda * Any I/O submitted after lock_fs() may not be flushed. 23049f518b27SKiyoshi Ueda * noflush takes precedence over do_lockfs. 23059f518b27SKiyoshi Ueda * (lock_fs() flushes I/Os and waits for them to complete.) 23062e93ccc1SKiyoshi Ueda */ 230732a926daSMikulas Patocka if (!noflush && do_lockfs) { 23082ca3310eSAlasdair G Kergon r = lock_fs(md); 2309d67ee213SMike Snitzer if (r) { 2310d67ee213SMike Snitzer dm_table_presuspend_undo_targets(map); 2311ffcc3936SMike Snitzer return r; 2312aa8d7c2fSAlasdair G Kergon } 2313d67ee213SMike Snitzer } 23141da177e4SLinus Torvalds 23151da177e4SLinus Torvalds /* 23163b00b203SMikulas Patocka * Here we must make sure that no processes are submitting requests 23173b00b203SMikulas Patocka * to target drivers i.e. no one may be executing 23180cede372SMike Snitzer * __split_and_process_bio from dm_submit_bio. 23193b00b203SMikulas Patocka * 23200cede372SMike Snitzer * To get all processes out of __split_and_process_bio in dm_submit_bio, 23213b00b203SMikulas Patocka * we take the write lock. To prevent any process from reentering 23220cede372SMike Snitzer * __split_and_process_bio from dm_submit_bio and quiesce the thread 23230cede372SMike Snitzer * (dm_wq_work), we set DMF_BLOCK_IO_FOR_SUSPEND and call 23246a8736d1STejun Heo * flush_workqueue(md->wq). 23251da177e4SLinus Torvalds */ 23261eb787ecSAlasdair G Kergon set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 232741abc4e1SHannes Reinecke if (map) 232883d5e5b0SMikulas Patocka synchronize_srcu(&md->io_barrier); 23291da177e4SLinus Torvalds 2330d0bcb878SKiyoshi Ueda /* 233129e4013dSTejun Heo * Stop md->queue before flushing md->wq in case request-based 233229e4013dSTejun Heo * dm defers requests to md->wq from md->queue. 2333d0bcb878SKiyoshi Ueda */ 23346a23e05cSJens Axboe if (dm_request_based(md)) 2335eca7ee6dSMike Snitzer dm_stop_queue(md->queue); 2336cec47e3dSKiyoshi Ueda 2337d0bcb878SKiyoshi Ueda flush_workqueue(md->wq); 2338d0bcb878SKiyoshi Ueda 23391da177e4SLinus Torvalds /* 23403b00b203SMikulas Patocka * At this point no more requests are entering target request routines. 23413b00b203SMikulas Patocka * We call dm_wait_for_completion to wait for all existing requests 23423b00b203SMikulas Patocka * to finish. 23431da177e4SLinus Torvalds */ 2344b48633f8SBart Van Assche r = dm_wait_for_completion(md, task_state); 2345eaf9a736SMike Snitzer if (!r) 2346eaf9a736SMike Snitzer set_bit(dmf_suspended_flag, &md->flags); 23471da177e4SLinus Torvalds 23486d6f10dfSMilan Broz if (noflush) 2349022c2611SMikulas Patocka clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 235041abc4e1SHannes Reinecke if (map) 235183d5e5b0SMikulas Patocka synchronize_srcu(&md->io_barrier); 23522e93ccc1SKiyoshi Ueda 23531da177e4SLinus Torvalds /* were we interrupted ? */ 235446125c1cSMilan Broz if (r < 0) { 23559a1fb464SMikulas Patocka dm_queue_flush(md); 235673d410c0SMilan Broz 2357cec47e3dSKiyoshi Ueda if (dm_request_based(md)) 2358eca7ee6dSMike Snitzer dm_start_queue(md->queue); 2359cec47e3dSKiyoshi Ueda 23602ca3310eSAlasdair G Kergon unlock_fs(md); 2361d67ee213SMike Snitzer dm_table_presuspend_undo_targets(map); 2362ffcc3936SMike Snitzer /* pushback list is already flushed, so skip flush */ 2363ffcc3936SMike Snitzer } 2364ffcc3936SMike Snitzer 2365ffcc3936SMike Snitzer return r; 23662ca3310eSAlasdair G Kergon } 23672ca3310eSAlasdair G Kergon 23683b00b203SMikulas Patocka /* 2369ffcc3936SMike Snitzer * We need to be able to change a mapping table under a mounted 2370ffcc3936SMike Snitzer * filesystem. For example we might want to move some data in 2371ffcc3936SMike Snitzer * the background. Before the table can be swapped with 2372ffcc3936SMike Snitzer * dm_bind_table, dm_suspend must be called to flush any in 2373ffcc3936SMike Snitzer * flight bios and ensure that any further io gets deferred. 23743b00b203SMikulas Patocka */ 2375ffcc3936SMike Snitzer /* 2376ffcc3936SMike Snitzer * Suspend mechanism in request-based dm. 2377ffcc3936SMike Snitzer * 2378ffcc3936SMike Snitzer * 1. Flush all I/Os by lock_fs() if needed. 2379ffcc3936SMike Snitzer * 2. Stop dispatching any I/O by stopping the request_queue. 2380ffcc3936SMike Snitzer * 3. Wait for all in-flight I/Os to be completed or requeued. 2381ffcc3936SMike Snitzer * 2382ffcc3936SMike Snitzer * To abort suspend, start the request_queue. 2383ffcc3936SMike Snitzer */ 2384ffcc3936SMike Snitzer int dm_suspend(struct mapped_device *md, unsigned suspend_flags) 2385ffcc3936SMike Snitzer { 2386ffcc3936SMike Snitzer struct dm_table *map = NULL; 2387ffcc3936SMike Snitzer int r = 0; 2388ffcc3936SMike Snitzer 2389ffcc3936SMike Snitzer retry: 2390ffcc3936SMike Snitzer mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); 2391ffcc3936SMike Snitzer 2392ffcc3936SMike Snitzer if (dm_suspended_md(md)) { 2393ffcc3936SMike Snitzer r = -EINVAL; 2394ffcc3936SMike Snitzer goto out_unlock; 2395ffcc3936SMike Snitzer } 2396ffcc3936SMike Snitzer 2397ffcc3936SMike Snitzer if (dm_suspended_internally_md(md)) { 2398ffcc3936SMike Snitzer /* already internally suspended, wait for internal resume */ 2399ffcc3936SMike Snitzer mutex_unlock(&md->suspend_lock); 2400ffcc3936SMike Snitzer r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); 2401ffcc3936SMike Snitzer if (r) 2402ffcc3936SMike Snitzer return r; 2403ffcc3936SMike Snitzer goto retry; 2404ffcc3936SMike Snitzer } 2405ffcc3936SMike Snitzer 2406a12f5d48SEric Dumazet map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2407ffcc3936SMike Snitzer 2408eaf9a736SMike Snitzer r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED); 2409ffcc3936SMike Snitzer if (r) 2410ffcc3936SMike Snitzer goto out_unlock; 24113b00b203SMikulas Patocka 24125df96f2bSMikulas Patocka set_bit(DMF_POST_SUSPENDING, &md->flags); 24134d4471cbSKiyoshi Ueda dm_table_postsuspend_targets(map); 24145df96f2bSMikulas Patocka clear_bit(DMF_POST_SUSPENDING, &md->flags); 24154d4471cbSKiyoshi Ueda 2416d287483dSAlasdair G Kergon out_unlock: 2417e61290a4SDaniel Walker mutex_unlock(&md->suspend_lock); 2418cf222b37SAlasdair G Kergon return r; 24191da177e4SLinus Torvalds } 24201da177e4SLinus Torvalds 2421ffcc3936SMike Snitzer static int __dm_resume(struct mapped_device *md, struct dm_table *map) 24221da177e4SLinus Torvalds { 2423ffcc3936SMike Snitzer if (map) { 2424ffcc3936SMike Snitzer int r = dm_table_resume_targets(map); 24258757b776SMilan Broz if (r) 2426ffcc3936SMike Snitzer return r; 2427ffcc3936SMike Snitzer } 24282ca3310eSAlasdair G Kergon 24299a1fb464SMikulas Patocka dm_queue_flush(md); 24302ca3310eSAlasdair G Kergon 2431cec47e3dSKiyoshi Ueda /* 2432cec47e3dSKiyoshi Ueda * Flushing deferred I/Os must be done after targets are resumed 2433cec47e3dSKiyoshi Ueda * so that mapping of targets can work correctly. 2434cec47e3dSKiyoshi Ueda * Request-based dm is queueing the deferred I/Os in its request_queue. 2435cec47e3dSKiyoshi Ueda */ 2436cec47e3dSKiyoshi Ueda if (dm_request_based(md)) 2437eca7ee6dSMike Snitzer dm_start_queue(md->queue); 2438cec47e3dSKiyoshi Ueda 24392ca3310eSAlasdair G Kergon unlock_fs(md); 24402ca3310eSAlasdair G Kergon 2441ffcc3936SMike Snitzer return 0; 2442ffcc3936SMike Snitzer } 2443ffcc3936SMike Snitzer 2444ffcc3936SMike Snitzer int dm_resume(struct mapped_device *md) 2445ffcc3936SMike Snitzer { 24468dc23658SMinfei Huang int r; 2447ffcc3936SMike Snitzer struct dm_table *map = NULL; 2448ffcc3936SMike Snitzer 2449ffcc3936SMike Snitzer retry: 24508dc23658SMinfei Huang r = -EINVAL; 2451ffcc3936SMike Snitzer mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); 2452ffcc3936SMike Snitzer 2453ffcc3936SMike Snitzer if (!dm_suspended_md(md)) 2454ffcc3936SMike Snitzer goto out; 2455ffcc3936SMike Snitzer 2456ffcc3936SMike Snitzer if (dm_suspended_internally_md(md)) { 2457ffcc3936SMike Snitzer /* already internally suspended, wait for internal resume */ 2458ffcc3936SMike Snitzer mutex_unlock(&md->suspend_lock); 2459ffcc3936SMike Snitzer r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); 2460ffcc3936SMike Snitzer if (r) 2461ffcc3936SMike Snitzer return r; 2462ffcc3936SMike Snitzer goto retry; 2463ffcc3936SMike Snitzer } 2464ffcc3936SMike Snitzer 2465a12f5d48SEric Dumazet map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2466ffcc3936SMike Snitzer if (!map || !dm_table_get_size(map)) 2467ffcc3936SMike Snitzer goto out; 2468ffcc3936SMike Snitzer 2469ffcc3936SMike Snitzer r = __dm_resume(md, map); 2470ffcc3936SMike Snitzer if (r) 2471ffcc3936SMike Snitzer goto out; 2472ffcc3936SMike Snitzer 24732ca3310eSAlasdair G Kergon clear_bit(DMF_SUSPENDED, &md->flags); 2474cf222b37SAlasdair G Kergon out: 2475e61290a4SDaniel Walker mutex_unlock(&md->suspend_lock); 24762ca3310eSAlasdair G Kergon 2477cf222b37SAlasdair G Kergon return r; 24781da177e4SLinus Torvalds } 24791da177e4SLinus Torvalds 2480fd2ed4d2SMikulas Patocka /* 2481fd2ed4d2SMikulas Patocka * Internal suspend/resume works like userspace-driven suspend. It waits 2482fd2ed4d2SMikulas Patocka * until all bios finish and prevents issuing new bios to the target drivers. 2483fd2ed4d2SMikulas Patocka * It may be used only from the kernel. 2484fd2ed4d2SMikulas Patocka */ 2485fd2ed4d2SMikulas Patocka 2486ffcc3936SMike Snitzer static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags) 2487ffcc3936SMike Snitzer { 2488ffcc3936SMike Snitzer struct dm_table *map = NULL; 2489ffcc3936SMike Snitzer 24901ea0654eSBart Van Assche lockdep_assert_held(&md->suspend_lock); 24911ea0654eSBart Van Assche 249296b26c8cSMikulas Patocka if (md->internal_suspend_count++) 2493ffcc3936SMike Snitzer return; /* nested internal suspend */ 2494ffcc3936SMike Snitzer 2495ffcc3936SMike Snitzer if (dm_suspended_md(md)) { 2496ffcc3936SMike Snitzer set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 2497ffcc3936SMike Snitzer return; /* nest suspend */ 2498ffcc3936SMike Snitzer } 2499ffcc3936SMike Snitzer 2500a12f5d48SEric Dumazet map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2501ffcc3936SMike Snitzer 2502ffcc3936SMike Snitzer /* 2503ffcc3936SMike Snitzer * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is 2504ffcc3936SMike Snitzer * supported. Properly supporting a TASK_INTERRUPTIBLE internal suspend 2505ffcc3936SMike Snitzer * would require changing .presuspend to return an error -- avoid this 2506ffcc3936SMike Snitzer * until there is a need for more elaborate variants of internal suspend. 2507ffcc3936SMike Snitzer */ 2508eaf9a736SMike Snitzer (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE, 2509eaf9a736SMike Snitzer DMF_SUSPENDED_INTERNALLY); 2510ffcc3936SMike Snitzer 25115df96f2bSMikulas Patocka set_bit(DMF_POST_SUSPENDING, &md->flags); 2512ffcc3936SMike Snitzer dm_table_postsuspend_targets(map); 25135df96f2bSMikulas Patocka clear_bit(DMF_POST_SUSPENDING, &md->flags); 2514ffcc3936SMike Snitzer } 2515ffcc3936SMike Snitzer 2516ffcc3936SMike Snitzer static void __dm_internal_resume(struct mapped_device *md) 2517ffcc3936SMike Snitzer { 251896b26c8cSMikulas Patocka BUG_ON(!md->internal_suspend_count); 251996b26c8cSMikulas Patocka 252096b26c8cSMikulas Patocka if (--md->internal_suspend_count) 2521ffcc3936SMike Snitzer return; /* resume from nested internal suspend */ 2522ffcc3936SMike Snitzer 2523ffcc3936SMike Snitzer if (dm_suspended_md(md)) 2524ffcc3936SMike Snitzer goto done; /* resume from nested suspend */ 2525ffcc3936SMike Snitzer 2526ffcc3936SMike Snitzer /* 2527ffcc3936SMike Snitzer * NOTE: existing callers don't need to call dm_table_resume_targets 2528ffcc3936SMike Snitzer * (which may fail -- so best to avoid it for now by passing NULL map) 2529ffcc3936SMike Snitzer */ 2530ffcc3936SMike Snitzer (void) __dm_resume(md, NULL); 2531ffcc3936SMike Snitzer 2532ffcc3936SMike Snitzer done: 2533ffcc3936SMike Snitzer clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 2534ffcc3936SMike Snitzer smp_mb__after_atomic(); 2535ffcc3936SMike Snitzer wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY); 2536ffcc3936SMike Snitzer } 2537ffcc3936SMike Snitzer 2538ffcc3936SMike Snitzer void dm_internal_suspend_noflush(struct mapped_device *md) 2539fd2ed4d2SMikulas Patocka { 2540fd2ed4d2SMikulas Patocka mutex_lock(&md->suspend_lock); 2541ffcc3936SMike Snitzer __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG); 2542ffcc3936SMike Snitzer mutex_unlock(&md->suspend_lock); 2543ffcc3936SMike Snitzer } 2544ffcc3936SMike Snitzer EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush); 2545ffcc3936SMike Snitzer 2546ffcc3936SMike Snitzer void dm_internal_resume(struct mapped_device *md) 2547ffcc3936SMike Snitzer { 2548ffcc3936SMike Snitzer mutex_lock(&md->suspend_lock); 2549ffcc3936SMike Snitzer __dm_internal_resume(md); 2550ffcc3936SMike Snitzer mutex_unlock(&md->suspend_lock); 2551ffcc3936SMike Snitzer } 2552ffcc3936SMike Snitzer EXPORT_SYMBOL_GPL(dm_internal_resume); 2553ffcc3936SMike Snitzer 2554ffcc3936SMike Snitzer /* 2555ffcc3936SMike Snitzer * Fast variants of internal suspend/resume hold md->suspend_lock, 2556ffcc3936SMike Snitzer * which prevents interaction with userspace-driven suspend. 2557ffcc3936SMike Snitzer */ 2558ffcc3936SMike Snitzer 2559ffcc3936SMike Snitzer void dm_internal_suspend_fast(struct mapped_device *md) 2560ffcc3936SMike Snitzer { 2561ffcc3936SMike Snitzer mutex_lock(&md->suspend_lock); 2562ffcc3936SMike Snitzer if (dm_suspended_md(md) || dm_suspended_internally_md(md)) 2563fd2ed4d2SMikulas Patocka return; 2564fd2ed4d2SMikulas Patocka 2565fd2ed4d2SMikulas Patocka set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 2566fd2ed4d2SMikulas Patocka synchronize_srcu(&md->io_barrier); 2567fd2ed4d2SMikulas Patocka flush_workqueue(md->wq); 2568fd2ed4d2SMikulas Patocka dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE); 2569fd2ed4d2SMikulas Patocka } 2570b735fedeSMikulas Patocka EXPORT_SYMBOL_GPL(dm_internal_suspend_fast); 2571fd2ed4d2SMikulas Patocka 2572ffcc3936SMike Snitzer void dm_internal_resume_fast(struct mapped_device *md) 2573fd2ed4d2SMikulas Patocka { 2574ffcc3936SMike Snitzer if (dm_suspended_md(md) || dm_suspended_internally_md(md)) 2575fd2ed4d2SMikulas Patocka goto done; 2576fd2ed4d2SMikulas Patocka 2577fd2ed4d2SMikulas Patocka dm_queue_flush(md); 2578fd2ed4d2SMikulas Patocka 2579fd2ed4d2SMikulas Patocka done: 2580fd2ed4d2SMikulas Patocka mutex_unlock(&md->suspend_lock); 2581fd2ed4d2SMikulas Patocka } 2582b735fedeSMikulas Patocka EXPORT_SYMBOL_GPL(dm_internal_resume_fast); 2583fd2ed4d2SMikulas Patocka 25841da177e4SLinus Torvalds /*----------------------------------------------------------------- 25851da177e4SLinus Torvalds * Event notification. 25861da177e4SLinus Torvalds *---------------------------------------------------------------*/ 25873abf85b5SPeter Rajnoha int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action, 258860935eb2SMilan Broz unsigned cookie) 258969267a30SAlasdair G Kergon { 25906958c1c6SMikulas Patocka int r; 25916958c1c6SMikulas Patocka unsigned noio_flag; 259260935eb2SMilan Broz char udev_cookie[DM_COOKIE_LENGTH]; 259360935eb2SMilan Broz char *envp[] = { udev_cookie, NULL }; 259460935eb2SMilan Broz 25956958c1c6SMikulas Patocka noio_flag = memalloc_noio_save(); 25966958c1c6SMikulas Patocka 259760935eb2SMilan Broz if (!cookie) 25986958c1c6SMikulas Patocka r = kobject_uevent(&disk_to_dev(md->disk)->kobj, action); 259960935eb2SMilan Broz else { 260060935eb2SMilan Broz snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u", 260160935eb2SMilan Broz DM_COOKIE_ENV_VAR_NAME, cookie); 26026958c1c6SMikulas Patocka r = kobject_uevent_env(&disk_to_dev(md->disk)->kobj, 26033abf85b5SPeter Rajnoha action, envp); 260460935eb2SMilan Broz } 26056958c1c6SMikulas Patocka 26066958c1c6SMikulas Patocka memalloc_noio_restore(noio_flag); 26076958c1c6SMikulas Patocka 26086958c1c6SMikulas Patocka return r; 260969267a30SAlasdair G Kergon } 261069267a30SAlasdair G Kergon 26117a8c3d3bSMike Anderson uint32_t dm_next_uevent_seq(struct mapped_device *md) 26127a8c3d3bSMike Anderson { 26137a8c3d3bSMike Anderson return atomic_add_return(1, &md->uevent_seq); 26147a8c3d3bSMike Anderson } 26157a8c3d3bSMike Anderson 26161da177e4SLinus Torvalds uint32_t dm_get_event_nr(struct mapped_device *md) 26171da177e4SLinus Torvalds { 26181da177e4SLinus Torvalds return atomic_read(&md->event_nr); 26191da177e4SLinus Torvalds } 26201da177e4SLinus Torvalds 26211da177e4SLinus Torvalds int dm_wait_event(struct mapped_device *md, int event_nr) 26221da177e4SLinus Torvalds { 26231da177e4SLinus Torvalds return wait_event_interruptible(md->eventq, 26241da177e4SLinus Torvalds (event_nr != atomic_read(&md->event_nr))); 26251da177e4SLinus Torvalds } 26261da177e4SLinus Torvalds 26277a8c3d3bSMike Anderson void dm_uevent_add(struct mapped_device *md, struct list_head *elist) 26287a8c3d3bSMike Anderson { 26297a8c3d3bSMike Anderson unsigned long flags; 26307a8c3d3bSMike Anderson 26317a8c3d3bSMike Anderson spin_lock_irqsave(&md->uevent_lock, flags); 26327a8c3d3bSMike Anderson list_add(elist, &md->uevent_list); 26337a8c3d3bSMike Anderson spin_unlock_irqrestore(&md->uevent_lock, flags); 26347a8c3d3bSMike Anderson } 26357a8c3d3bSMike Anderson 26361da177e4SLinus Torvalds /* 26371da177e4SLinus Torvalds * The gendisk is only valid as long as you have a reference 26381da177e4SLinus Torvalds * count on 'md'. 26391da177e4SLinus Torvalds */ 26401da177e4SLinus Torvalds struct gendisk *dm_disk(struct mapped_device *md) 26411da177e4SLinus Torvalds { 26421da177e4SLinus Torvalds return md->disk; 26431da177e4SLinus Torvalds } 264465ff5b7dSSami Tolvanen EXPORT_SYMBOL_GPL(dm_disk); 26451da177e4SLinus Torvalds 2646784aae73SMilan Broz struct kobject *dm_kobject(struct mapped_device *md) 2647784aae73SMilan Broz { 26482995fa78SMikulas Patocka return &md->kobj_holder.kobj; 2649784aae73SMilan Broz } 2650784aae73SMilan Broz 2651784aae73SMilan Broz struct mapped_device *dm_get_from_kobject(struct kobject *kobj) 2652784aae73SMilan Broz { 2653784aae73SMilan Broz struct mapped_device *md; 2654784aae73SMilan Broz 26552995fa78SMikulas Patocka md = container_of(kobj, struct mapped_device, kobj_holder.kobj); 2656784aae73SMilan Broz 2657b9a41d21SHou Tao spin_lock(&_minor_lock); 2658b9a41d21SHou Tao if (test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) { 2659b9a41d21SHou Tao md = NULL; 2660b9a41d21SHou Tao goto out; 2661b9a41d21SHou Tao } 2662784aae73SMilan Broz dm_get(md); 2663b9a41d21SHou Tao out: 2664b9a41d21SHou Tao spin_unlock(&_minor_lock); 2665b9a41d21SHou Tao 2666784aae73SMilan Broz return md; 2667784aae73SMilan Broz } 2668784aae73SMilan Broz 26694f186f8bSKiyoshi Ueda int dm_suspended_md(struct mapped_device *md) 26701da177e4SLinus Torvalds { 26711da177e4SLinus Torvalds return test_bit(DMF_SUSPENDED, &md->flags); 26721da177e4SLinus Torvalds } 26731da177e4SLinus Torvalds 26745df96f2bSMikulas Patocka static int dm_post_suspending_md(struct mapped_device *md) 26755df96f2bSMikulas Patocka { 26765df96f2bSMikulas Patocka return test_bit(DMF_POST_SUSPENDING, &md->flags); 26775df96f2bSMikulas Patocka } 26785df96f2bSMikulas Patocka 2679ffcc3936SMike Snitzer int dm_suspended_internally_md(struct mapped_device *md) 2680ffcc3936SMike Snitzer { 2681ffcc3936SMike Snitzer return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 2682ffcc3936SMike Snitzer } 2683ffcc3936SMike Snitzer 26842c140a24SMikulas Patocka int dm_test_deferred_remove_flag(struct mapped_device *md) 26852c140a24SMikulas Patocka { 26862c140a24SMikulas Patocka return test_bit(DMF_DEFERRED_REMOVE, &md->flags); 26872c140a24SMikulas Patocka } 26882c140a24SMikulas Patocka 268964dbce58SKiyoshi Ueda int dm_suspended(struct dm_target *ti) 269064dbce58SKiyoshi Ueda { 269133bd6f06SMike Snitzer return dm_suspended_md(ti->table->md); 269264dbce58SKiyoshi Ueda } 269364dbce58SKiyoshi Ueda EXPORT_SYMBOL_GPL(dm_suspended); 269464dbce58SKiyoshi Ueda 26955df96f2bSMikulas Patocka int dm_post_suspending(struct dm_target *ti) 26965df96f2bSMikulas Patocka { 269733bd6f06SMike Snitzer return dm_post_suspending_md(ti->table->md); 26985df96f2bSMikulas Patocka } 26995df96f2bSMikulas Patocka EXPORT_SYMBOL_GPL(dm_post_suspending); 27005df96f2bSMikulas Patocka 27012e93ccc1SKiyoshi Ueda int dm_noflush_suspending(struct dm_target *ti) 27022e93ccc1SKiyoshi Ueda { 270333bd6f06SMike Snitzer return __noflush_suspending(ti->table->md); 27042e93ccc1SKiyoshi Ueda } 27052e93ccc1SKiyoshi Ueda EXPORT_SYMBOL_GPL(dm_noflush_suspending); 27062e93ccc1SKiyoshi Ueda 27077e0d574fSBart Van Assche struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_queue_mode type, 27080776aa0eSMike Snitzer unsigned integrity, unsigned per_io_data_size, 27090776aa0eSMike Snitzer unsigned min_pool_size) 2710e6ee8c0bSKiyoshi Ueda { 2711115485e8SMike Snitzer struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id); 271278d8e58aSMike Snitzer unsigned int pool_size = 0; 271364f52b0eSMike Snitzer unsigned int front_pad, io_front_pad; 27146f1c819cSKent Overstreet int ret; 2715e6ee8c0bSKiyoshi Ueda 2716e6ee8c0bSKiyoshi Ueda if (!pools) 27174e6e36c3SMike Snitzer return NULL; 2718e6ee8c0bSKiyoshi Ueda 271978d8e58aSMike Snitzer switch (type) { 272078d8e58aSMike Snitzer case DM_TYPE_BIO_BASED: 2721545ed20eSToshi Kani case DM_TYPE_DAX_BIO_BASED: 27220776aa0eSMike Snitzer pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size); 272362f26317SJeffle Xu front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + DM_TARGET_IO_BIO_OFFSET; 272462f26317SJeffle Xu io_front_pad = roundup(per_io_data_size, __alignof__(struct dm_io)) + DM_IO_BIO_OFFSET; 27256f1c819cSKent Overstreet ret = bioset_init(&pools->io_bs, pool_size, io_front_pad, 0); 27266f1c819cSKent Overstreet if (ret) 272764f52b0eSMike Snitzer goto out; 27286f1c819cSKent Overstreet if (integrity && bioset_integrity_create(&pools->io_bs, pool_size)) 2729eb8db831SChristoph Hellwig goto out; 273078d8e58aSMike Snitzer break; 273178d8e58aSMike Snitzer case DM_TYPE_REQUEST_BASED: 27320776aa0eSMike Snitzer pool_size = max(dm_get_reserved_rq_based_ios(), min_pool_size); 273378d8e58aSMike Snitzer front_pad = offsetof(struct dm_rq_clone_bio_info, clone); 2734591ddcfcSMike Snitzer /* per_io_data_size is used for blk-mq pdu at queue allocation */ 273578d8e58aSMike Snitzer break; 273678d8e58aSMike Snitzer default: 273778d8e58aSMike Snitzer BUG(); 273878d8e58aSMike Snitzer } 273978d8e58aSMike Snitzer 27406f1c819cSKent Overstreet ret = bioset_init(&pools->bs, pool_size, front_pad, 0); 27416f1c819cSKent Overstreet if (ret) 27425f015204SJun'ichi Nomura goto out; 2743e6ee8c0bSKiyoshi Ueda 27446f1c819cSKent Overstreet if (integrity && bioset_integrity_create(&pools->bs, pool_size)) 27455f015204SJun'ichi Nomura goto out; 2746a91a2785SMartin K. Petersen 2747e6ee8c0bSKiyoshi Ueda return pools; 274878d8e58aSMike Snitzer 27495f015204SJun'ichi Nomura out: 27505f015204SJun'ichi Nomura dm_free_md_mempools(pools); 2751e6ee8c0bSKiyoshi Ueda 27524e6e36c3SMike Snitzer return NULL; 2753e6ee8c0bSKiyoshi Ueda } 2754e6ee8c0bSKiyoshi Ueda 2755e6ee8c0bSKiyoshi Ueda void dm_free_md_mempools(struct dm_md_mempools *pools) 2756e6ee8c0bSKiyoshi Ueda { 2757e6ee8c0bSKiyoshi Ueda if (!pools) 2758e6ee8c0bSKiyoshi Ueda return; 2759e6ee8c0bSKiyoshi Ueda 27606f1c819cSKent Overstreet bioset_exit(&pools->bs); 27616f1c819cSKent Overstreet bioset_exit(&pools->io_bs); 2762e6ee8c0bSKiyoshi Ueda 2763e6ee8c0bSKiyoshi Ueda kfree(pools); 2764e6ee8c0bSKiyoshi Ueda } 2765e6ee8c0bSKiyoshi Ueda 27669c72bad1SChristoph Hellwig struct dm_pr { 27679c72bad1SChristoph Hellwig u64 old_key; 27689c72bad1SChristoph Hellwig u64 new_key; 27699c72bad1SChristoph Hellwig u32 flags; 27709c72bad1SChristoph Hellwig bool fail_early; 27719c72bad1SChristoph Hellwig }; 27729c72bad1SChristoph Hellwig 27739c72bad1SChristoph Hellwig static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn, 27749c72bad1SChristoph Hellwig void *data) 27759c72bad1SChristoph Hellwig { 27769c72bad1SChristoph Hellwig struct mapped_device *md = bdev->bd_disk->private_data; 27779c72bad1SChristoph Hellwig struct dm_table *table; 27789c72bad1SChristoph Hellwig struct dm_target *ti; 27799c72bad1SChristoph Hellwig int ret = -ENOTTY, srcu_idx; 27809c72bad1SChristoph Hellwig 27819c72bad1SChristoph Hellwig table = dm_get_live_table(md, &srcu_idx); 27829c72bad1SChristoph Hellwig if (!table || !dm_table_get_size(table)) 27839c72bad1SChristoph Hellwig goto out; 27849c72bad1SChristoph Hellwig 27859c72bad1SChristoph Hellwig /* We only support devices that have a single target */ 27869c72bad1SChristoph Hellwig if (dm_table_get_num_targets(table) != 1) 27879c72bad1SChristoph Hellwig goto out; 27889c72bad1SChristoph Hellwig ti = dm_table_get_target(table, 0); 27899c72bad1SChristoph Hellwig 27909c72bad1SChristoph Hellwig ret = -EINVAL; 27919c72bad1SChristoph Hellwig if (!ti->type->iterate_devices) 27929c72bad1SChristoph Hellwig goto out; 27939c72bad1SChristoph Hellwig 27949c72bad1SChristoph Hellwig ret = ti->type->iterate_devices(ti, fn, data); 27959c72bad1SChristoph Hellwig out: 27969c72bad1SChristoph Hellwig dm_put_live_table(md, srcu_idx); 27979c72bad1SChristoph Hellwig return ret; 27989c72bad1SChristoph Hellwig } 27999c72bad1SChristoph Hellwig 28009c72bad1SChristoph Hellwig /* 28019c72bad1SChristoph Hellwig * For register / unregister we need to manually call out to every path. 28029c72bad1SChristoph Hellwig */ 28039c72bad1SChristoph Hellwig static int __dm_pr_register(struct dm_target *ti, struct dm_dev *dev, 28049c72bad1SChristoph Hellwig sector_t start, sector_t len, void *data) 28059c72bad1SChristoph Hellwig { 28069c72bad1SChristoph Hellwig struct dm_pr *pr = data; 28079c72bad1SChristoph Hellwig const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops; 28089c72bad1SChristoph Hellwig 28099c72bad1SChristoph Hellwig if (!ops || !ops->pr_register) 28109c72bad1SChristoph Hellwig return -EOPNOTSUPP; 28119c72bad1SChristoph Hellwig return ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags); 28129c72bad1SChristoph Hellwig } 28139c72bad1SChristoph Hellwig 281471cdb697SChristoph Hellwig static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key, 281571cdb697SChristoph Hellwig u32 flags) 281671cdb697SChristoph Hellwig { 28179c72bad1SChristoph Hellwig struct dm_pr pr = { 28189c72bad1SChristoph Hellwig .old_key = old_key, 28199c72bad1SChristoph Hellwig .new_key = new_key, 28209c72bad1SChristoph Hellwig .flags = flags, 28219c72bad1SChristoph Hellwig .fail_early = true, 28229c72bad1SChristoph Hellwig }; 28239c72bad1SChristoph Hellwig int ret; 282471cdb697SChristoph Hellwig 28259c72bad1SChristoph Hellwig ret = dm_call_pr(bdev, __dm_pr_register, &pr); 28269c72bad1SChristoph Hellwig if (ret && new_key) { 28279c72bad1SChristoph Hellwig /* unregister all paths if we failed to register any path */ 28289c72bad1SChristoph Hellwig pr.old_key = new_key; 28299c72bad1SChristoph Hellwig pr.new_key = 0; 28309c72bad1SChristoph Hellwig pr.flags = 0; 28319c72bad1SChristoph Hellwig pr.fail_early = false; 28329c72bad1SChristoph Hellwig dm_call_pr(bdev, __dm_pr_register, &pr); 28339c72bad1SChristoph Hellwig } 283471cdb697SChristoph Hellwig 28359c72bad1SChristoph Hellwig return ret; 283671cdb697SChristoph Hellwig } 283771cdb697SChristoph Hellwig 283871cdb697SChristoph Hellwig static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type, 283971cdb697SChristoph Hellwig u32 flags) 284071cdb697SChristoph Hellwig { 284171cdb697SChristoph Hellwig struct mapped_device *md = bdev->bd_disk->private_data; 284271cdb697SChristoph Hellwig const struct pr_ops *ops; 2843971888c4SMike Snitzer int r, srcu_idx; 284471cdb697SChristoph Hellwig 28455bd5e8d8SMike Snitzer r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 284671cdb697SChristoph Hellwig if (r < 0) 2847971888c4SMike Snitzer goto out; 284871cdb697SChristoph Hellwig 284971cdb697SChristoph Hellwig ops = bdev->bd_disk->fops->pr_ops; 285071cdb697SChristoph Hellwig if (ops && ops->pr_reserve) 285171cdb697SChristoph Hellwig r = ops->pr_reserve(bdev, key, type, flags); 285271cdb697SChristoph Hellwig else 285371cdb697SChristoph Hellwig r = -EOPNOTSUPP; 2854971888c4SMike Snitzer out: 2855971888c4SMike Snitzer dm_unprepare_ioctl(md, srcu_idx); 285671cdb697SChristoph Hellwig return r; 285771cdb697SChristoph Hellwig } 285871cdb697SChristoph Hellwig 285971cdb697SChristoph Hellwig static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type) 286071cdb697SChristoph Hellwig { 286171cdb697SChristoph Hellwig struct mapped_device *md = bdev->bd_disk->private_data; 286271cdb697SChristoph Hellwig const struct pr_ops *ops; 2863971888c4SMike Snitzer int r, srcu_idx; 286471cdb697SChristoph Hellwig 28655bd5e8d8SMike Snitzer r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 286671cdb697SChristoph Hellwig if (r < 0) 2867971888c4SMike Snitzer goto out; 286871cdb697SChristoph Hellwig 286971cdb697SChristoph Hellwig ops = bdev->bd_disk->fops->pr_ops; 287071cdb697SChristoph Hellwig if (ops && ops->pr_release) 287171cdb697SChristoph Hellwig r = ops->pr_release(bdev, key, type); 287271cdb697SChristoph Hellwig else 287371cdb697SChristoph Hellwig r = -EOPNOTSUPP; 2874971888c4SMike Snitzer out: 2875971888c4SMike Snitzer dm_unprepare_ioctl(md, srcu_idx); 287671cdb697SChristoph Hellwig return r; 287771cdb697SChristoph Hellwig } 287871cdb697SChristoph Hellwig 287971cdb697SChristoph Hellwig static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key, 288071cdb697SChristoph Hellwig enum pr_type type, bool abort) 288171cdb697SChristoph Hellwig { 288271cdb697SChristoph Hellwig struct mapped_device *md = bdev->bd_disk->private_data; 288371cdb697SChristoph Hellwig const struct pr_ops *ops; 2884971888c4SMike Snitzer int r, srcu_idx; 288571cdb697SChristoph Hellwig 28865bd5e8d8SMike Snitzer r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 288771cdb697SChristoph Hellwig if (r < 0) 2888971888c4SMike Snitzer goto out; 288971cdb697SChristoph Hellwig 289071cdb697SChristoph Hellwig ops = bdev->bd_disk->fops->pr_ops; 289171cdb697SChristoph Hellwig if (ops && ops->pr_preempt) 289271cdb697SChristoph Hellwig r = ops->pr_preempt(bdev, old_key, new_key, type, abort); 289371cdb697SChristoph Hellwig else 289471cdb697SChristoph Hellwig r = -EOPNOTSUPP; 2895971888c4SMike Snitzer out: 2896971888c4SMike Snitzer dm_unprepare_ioctl(md, srcu_idx); 289771cdb697SChristoph Hellwig return r; 289871cdb697SChristoph Hellwig } 289971cdb697SChristoph Hellwig 290071cdb697SChristoph Hellwig static int dm_pr_clear(struct block_device *bdev, u64 key) 290171cdb697SChristoph Hellwig { 290271cdb697SChristoph Hellwig struct mapped_device *md = bdev->bd_disk->private_data; 290371cdb697SChristoph Hellwig const struct pr_ops *ops; 2904971888c4SMike Snitzer int r, srcu_idx; 290571cdb697SChristoph Hellwig 29065bd5e8d8SMike Snitzer r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 290771cdb697SChristoph Hellwig if (r < 0) 2908971888c4SMike Snitzer goto out; 290971cdb697SChristoph Hellwig 291071cdb697SChristoph Hellwig ops = bdev->bd_disk->fops->pr_ops; 291171cdb697SChristoph Hellwig if (ops && ops->pr_clear) 291271cdb697SChristoph Hellwig r = ops->pr_clear(bdev, key); 291371cdb697SChristoph Hellwig else 291471cdb697SChristoph Hellwig r = -EOPNOTSUPP; 2915971888c4SMike Snitzer out: 2916971888c4SMike Snitzer dm_unprepare_ioctl(md, srcu_idx); 291771cdb697SChristoph Hellwig return r; 291871cdb697SChristoph Hellwig } 291971cdb697SChristoph Hellwig 292071cdb697SChristoph Hellwig static const struct pr_ops dm_pr_ops = { 292171cdb697SChristoph Hellwig .pr_register = dm_pr_register, 292271cdb697SChristoph Hellwig .pr_reserve = dm_pr_reserve, 292371cdb697SChristoph Hellwig .pr_release = dm_pr_release, 292471cdb697SChristoph Hellwig .pr_preempt = dm_pr_preempt, 292571cdb697SChristoph Hellwig .pr_clear = dm_pr_clear, 292671cdb697SChristoph Hellwig }; 292771cdb697SChristoph Hellwig 292883d5cde4SAlexey Dobriyan static const struct block_device_operations dm_blk_dops = { 2929c62b37d9SChristoph Hellwig .submit_bio = dm_submit_bio, 29301da177e4SLinus Torvalds .open = dm_blk_open, 29311da177e4SLinus Torvalds .release = dm_blk_close, 2932aa129a22SMilan Broz .ioctl = dm_blk_ioctl, 29333ac51e74SDarrick J. Wong .getgeo = dm_blk_getgeo, 2934e76239a3SChristoph Hellwig .report_zones = dm_blk_report_zones, 293571cdb697SChristoph Hellwig .pr_ops = &dm_pr_ops, 29361da177e4SLinus Torvalds .owner = THIS_MODULE 29371da177e4SLinus Torvalds }; 29381da177e4SLinus Torvalds 2939681cc5e8SMike Snitzer static const struct block_device_operations dm_rq_blk_dops = { 2940681cc5e8SMike Snitzer .open = dm_blk_open, 2941681cc5e8SMike Snitzer .release = dm_blk_close, 2942681cc5e8SMike Snitzer .ioctl = dm_blk_ioctl, 2943681cc5e8SMike Snitzer .getgeo = dm_blk_getgeo, 2944681cc5e8SMike Snitzer .pr_ops = &dm_pr_ops, 2945681cc5e8SMike Snitzer .owner = THIS_MODULE 2946681cc5e8SMike Snitzer }; 2947681cc5e8SMike Snitzer 2948f26c5719SDan Williams static const struct dax_operations dm_dax_ops = { 2949f26c5719SDan Williams .direct_access = dm_dax_direct_access, 2950cdf6cdcdSVivek Goyal .zero_page_range = dm_dax_zero_page_range, 2951f26c5719SDan Williams }; 2952f26c5719SDan Williams 29531da177e4SLinus Torvalds /* 29541da177e4SLinus Torvalds * module hooks 29551da177e4SLinus Torvalds */ 29561da177e4SLinus Torvalds module_init(dm_init); 29571da177e4SLinus Torvalds module_exit(dm_exit); 29581da177e4SLinus Torvalds 29591da177e4SLinus Torvalds module_param(major, uint, 0); 29601da177e4SLinus Torvalds MODULE_PARM_DESC(major, "The major number of the device mapper"); 2961f4790826SMike Snitzer 2962e8603136SMike Snitzer module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR); 2963e8603136SMike Snitzer MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools"); 2964e8603136SMike Snitzer 2965115485e8SMike Snitzer module_param(dm_numa_node, int, S_IRUGO | S_IWUSR); 2966115485e8SMike Snitzer MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations"); 2967115485e8SMike Snitzer 2968a666e5c0SMikulas Patocka module_param(swap_bios, int, S_IRUGO | S_IWUSR); 2969a666e5c0SMikulas Patocka MODULE_PARM_DESC(swap_bios, "Maximum allowed inflight swap IOs"); 2970a666e5c0SMikulas Patocka 29711da177e4SLinus Torvalds MODULE_DESCRIPTION(DM_NAME " driver"); 29721da177e4SLinus Torvalds MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>"); 29731da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 2974