11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * Copyright (C) 2001, 2002 Sistina Software (UK) Limited. 3784aae73SMilan Broz * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. 41da177e4SLinus Torvalds * 51da177e4SLinus Torvalds * This file is released under the GPL. 61da177e4SLinus Torvalds */ 71da177e4SLinus Torvalds 81da177e4SLinus Torvalds #include "dm.h" 951e5b2bdSMike Anderson #include "dm-uevent.h" 101da177e4SLinus Torvalds 111da177e4SLinus Torvalds #include <linux/init.h> 121da177e4SLinus Torvalds #include <linux/module.h> 1348c9c27bSArjan van de Ven #include <linux/mutex.h> 141da177e4SLinus Torvalds #include <linux/moduleparam.h> 151da177e4SLinus Torvalds #include <linux/blkpg.h> 161da177e4SLinus Torvalds #include <linux/bio.h> 171da177e4SLinus Torvalds #include <linux/mempool.h> 181da177e4SLinus Torvalds #include <linux/slab.h> 191da177e4SLinus Torvalds #include <linux/idr.h> 203ac51e74SDarrick J. Wong #include <linux/hdreg.h> 213f77316dSKiyoshi Ueda #include <linux/delay.h> 22ffcc3936SMike Snitzer #include <linux/wait.h> 232eb6e1e3SKeith Busch #include <linux/kthread.h> 240ce65797SMike Snitzer #include <linux/ktime.h> 25de3ec86dSMike Snitzer #include <linux/elevator.h> /* for rq_end_sector() */ 26bfebd1cdSMike Snitzer #include <linux/blk-mq.h> 2755782138SLi Zefan 2855782138SLi Zefan #include <trace/events/block.h> 291da177e4SLinus Torvalds 3072d94861SAlasdair G Kergon #define DM_MSG_PREFIX "core" 3172d94861SAlasdair G Kergon 3271a16736SNamhyung Kim #ifdef CONFIG_PRINTK 3371a16736SNamhyung Kim /* 3471a16736SNamhyung Kim * ratelimit state to be used in DMXXX_LIMIT(). 3571a16736SNamhyung Kim */ 3671a16736SNamhyung Kim DEFINE_RATELIMIT_STATE(dm_ratelimit_state, 3771a16736SNamhyung Kim DEFAULT_RATELIMIT_INTERVAL, 3871a16736SNamhyung Kim DEFAULT_RATELIMIT_BURST); 3971a16736SNamhyung Kim EXPORT_SYMBOL(dm_ratelimit_state); 4071a16736SNamhyung Kim #endif 4171a16736SNamhyung Kim 4260935eb2SMilan Broz /* 4360935eb2SMilan Broz * Cookies are numeric values sent with CHANGE and REMOVE 4460935eb2SMilan Broz * uevents while resuming, removing or renaming the device. 4560935eb2SMilan Broz */ 4660935eb2SMilan Broz #define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE" 4760935eb2SMilan Broz #define DM_COOKIE_LENGTH 24 4860935eb2SMilan Broz 491da177e4SLinus Torvalds static const char *_name = DM_NAME; 501da177e4SLinus Torvalds 511da177e4SLinus Torvalds static unsigned int major = 0; 521da177e4SLinus Torvalds static unsigned int _major = 0; 531da177e4SLinus Torvalds 54d15b774cSAlasdair G Kergon static DEFINE_IDR(_minor_idr); 55d15b774cSAlasdair G Kergon 56f32c10b0SJeff Mahoney static DEFINE_SPINLOCK(_minor_lock); 572c140a24SMikulas Patocka 582c140a24SMikulas Patocka static void do_deferred_remove(struct work_struct *w); 592c140a24SMikulas Patocka 602c140a24SMikulas Patocka static DECLARE_WORK(deferred_remove_work, do_deferred_remove); 612c140a24SMikulas Patocka 62acfe0ad7SMikulas Patocka static struct workqueue_struct *deferred_remove_workqueue; 63acfe0ad7SMikulas Patocka 641da177e4SLinus Torvalds /* 658fbf26adSKiyoshi Ueda * For bio-based dm. 661da177e4SLinus Torvalds * One of these is allocated per bio. 671da177e4SLinus Torvalds */ 681da177e4SLinus Torvalds struct dm_io { 691da177e4SLinus Torvalds struct mapped_device *md; 701da177e4SLinus Torvalds int error; 711da177e4SLinus Torvalds atomic_t io_count; 726ae2fa67SRichard Kennedy struct bio *bio; 733eaf840eSJun'ichi "Nick" Nomura unsigned long start_time; 74f88fb981SKiyoshi Ueda spinlock_t endio_lock; 75fd2ed4d2SMikulas Patocka struct dm_stats_aux stats_aux; 761da177e4SLinus Torvalds }; 771da177e4SLinus Torvalds 781da177e4SLinus Torvalds /* 798fbf26adSKiyoshi Ueda * For request-based dm. 808fbf26adSKiyoshi Ueda * One of these is allocated per request. 818fbf26adSKiyoshi Ueda */ 828fbf26adSKiyoshi Ueda struct dm_rq_target_io { 838fbf26adSKiyoshi Ueda struct mapped_device *md; 848fbf26adSKiyoshi Ueda struct dm_target *ti; 851ae49ea2SMike Snitzer struct request *orig, *clone; 862eb6e1e3SKeith Busch struct kthread_work work; 878fbf26adSKiyoshi Ueda int error; 888fbf26adSKiyoshi Ueda union map_info info; 898fbf26adSKiyoshi Ueda }; 908fbf26adSKiyoshi Ueda 918fbf26adSKiyoshi Ueda /* 9294818742SKent Overstreet * For request-based dm - the bio clones we allocate are embedded in these 9394818742SKent Overstreet * structs. 9494818742SKent Overstreet * 9594818742SKent Overstreet * We allocate these with bio_alloc_bioset, using the front_pad parameter when 9694818742SKent Overstreet * the bioset is created - this means the bio has to come at the end of the 9794818742SKent Overstreet * struct. 988fbf26adSKiyoshi Ueda */ 998fbf26adSKiyoshi Ueda struct dm_rq_clone_bio_info { 1008fbf26adSKiyoshi Ueda struct bio *orig; 101cec47e3dSKiyoshi Ueda struct dm_rq_target_io *tio; 10294818742SKent Overstreet struct bio clone; 1038fbf26adSKiyoshi Ueda }; 1048fbf26adSKiyoshi Ueda 105cec47e3dSKiyoshi Ueda union map_info *dm_get_rq_mapinfo(struct request *rq) 106cec47e3dSKiyoshi Ueda { 107cec47e3dSKiyoshi Ueda if (rq && rq->end_io_data) 108cec47e3dSKiyoshi Ueda return &((struct dm_rq_target_io *)rq->end_io_data)->info; 109cec47e3dSKiyoshi Ueda return NULL; 110cec47e3dSKiyoshi Ueda } 111cec47e3dSKiyoshi Ueda EXPORT_SYMBOL_GPL(dm_get_rq_mapinfo); 112cec47e3dSKiyoshi Ueda 113ba61fdd1SJeff Mahoney #define MINOR_ALLOCED ((void *)-1) 114ba61fdd1SJeff Mahoney 1151da177e4SLinus Torvalds /* 1161da177e4SLinus Torvalds * Bits for the md->flags field. 1171da177e4SLinus Torvalds */ 1181eb787ecSAlasdair G Kergon #define DMF_BLOCK_IO_FOR_SUSPEND 0 1191da177e4SLinus Torvalds #define DMF_SUSPENDED 1 120aa8d7c2fSAlasdair G Kergon #define DMF_FROZEN 2 121fba9f90eSJeff Mahoney #define DMF_FREEING 3 1225c6bd75dSAlasdair G Kergon #define DMF_DELETING 4 1232e93ccc1SKiyoshi Ueda #define DMF_NOFLUSH_SUSPENDING 5 124d5b9dd04SMikulas Patocka #define DMF_MERGE_IS_OPTIONAL 6 1252c140a24SMikulas Patocka #define DMF_DEFERRED_REMOVE 7 126ffcc3936SMike Snitzer #define DMF_SUSPENDED_INTERNALLY 8 1271da177e4SLinus Torvalds 128304f3f6aSMilan Broz /* 12983d5e5b0SMikulas Patocka * A dummy definition to make RCU happy. 13083d5e5b0SMikulas Patocka * struct dm_table should never be dereferenced in this file. 13183d5e5b0SMikulas Patocka */ 13283d5e5b0SMikulas Patocka struct dm_table { 13383d5e5b0SMikulas Patocka int undefined__; 13483d5e5b0SMikulas Patocka }; 13583d5e5b0SMikulas Patocka 13683d5e5b0SMikulas Patocka /* 137304f3f6aSMilan Broz * Work processed by per-device workqueue. 138304f3f6aSMilan Broz */ 1391da177e4SLinus Torvalds struct mapped_device { 14083d5e5b0SMikulas Patocka struct srcu_struct io_barrier; 141e61290a4SDaniel Walker struct mutex suspend_lock; 1421da177e4SLinus Torvalds atomic_t holders; 1435c6bd75dSAlasdair G Kergon atomic_t open_count; 1441da177e4SLinus Torvalds 1452a7faeb1SMikulas Patocka /* 1462a7faeb1SMikulas Patocka * The current mapping. 1472a7faeb1SMikulas Patocka * Use dm_get_live_table{_fast} or take suspend_lock for 1482a7faeb1SMikulas Patocka * dereference. 1492a7faeb1SMikulas Patocka */ 1506fa99520SPranith Kumar struct dm_table __rcu *map; 1512a7faeb1SMikulas Patocka 15286f1152bSBenjamin Marzinski struct list_head table_devices; 15386f1152bSBenjamin Marzinski struct mutex table_devices_lock; 15486f1152bSBenjamin Marzinski 1551da177e4SLinus Torvalds unsigned long flags; 1561da177e4SLinus Torvalds 157165125e1SJens Axboe struct request_queue *queue; 158a5664dadSMike Snitzer unsigned type; 1594a0b4ddfSMike Snitzer /* Protect queue and type against concurrent access. */ 160a5664dadSMike Snitzer struct mutex type_lock; 161a5664dadSMike Snitzer 16236a0456fSAlasdair G Kergon struct target_type *immutable_target_type; 16336a0456fSAlasdair G Kergon 1641da177e4SLinus Torvalds struct gendisk *disk; 1657e51f257SMike Anderson char name[16]; 1661da177e4SLinus Torvalds 1671da177e4SLinus Torvalds void *interface_ptr; 1681da177e4SLinus Torvalds 1691da177e4SLinus Torvalds /* 1701da177e4SLinus Torvalds * A list of ios that arrived while we were suspended. 1711da177e4SLinus Torvalds */ 172316d315bSNikanth Karthikesan atomic_t pending[2]; 1731da177e4SLinus Torvalds wait_queue_head_t wait; 17453d5914fSMikulas Patocka struct work_struct work; 1751da177e4SLinus Torvalds struct bio_list deferred; 176022c2611SMikulas Patocka spinlock_t deferred_lock; 1771da177e4SLinus Torvalds 1781da177e4SLinus Torvalds /* 17929e4013dSTejun Heo * Processing queue (flush) 180304f3f6aSMilan Broz */ 181304f3f6aSMilan Broz struct workqueue_struct *wq; 182304f3f6aSMilan Broz 183304f3f6aSMilan Broz /* 1841da177e4SLinus Torvalds * io objects are allocated from here. 1851da177e4SLinus Torvalds */ 1861da177e4SLinus Torvalds mempool_t *io_pool; 1871ae49ea2SMike Snitzer mempool_t *rq_pool; 1881da177e4SLinus Torvalds 1899faf400fSStefan Bader struct bio_set *bs; 1909faf400fSStefan Bader 1911da177e4SLinus Torvalds /* 1921da177e4SLinus Torvalds * Event handling. 1931da177e4SLinus Torvalds */ 1941da177e4SLinus Torvalds atomic_t event_nr; 1951da177e4SLinus Torvalds wait_queue_head_t eventq; 1967a8c3d3bSMike Anderson atomic_t uevent_seq; 1977a8c3d3bSMike Anderson struct list_head uevent_list; 1987a8c3d3bSMike Anderson spinlock_t uevent_lock; /* Protect access to uevent_list */ 1991da177e4SLinus Torvalds 2001da177e4SLinus Torvalds /* 2011da177e4SLinus Torvalds * freeze/thaw support require holding onto a super block 2021da177e4SLinus Torvalds */ 2031da177e4SLinus Torvalds struct super_block *frozen_sb; 204db8fef4fSMikulas Patocka struct block_device *bdev; 2053ac51e74SDarrick J. Wong 2063ac51e74SDarrick J. Wong /* forced geometry settings */ 2073ac51e74SDarrick J. Wong struct hd_geometry geometry; 208784aae73SMilan Broz 2092995fa78SMikulas Patocka /* kobject and completion */ 2102995fa78SMikulas Patocka struct dm_kobject_holder kobj_holder; 211be35f486SMikulas Patocka 212d87f4c14STejun Heo /* zero-length flush that will be cloned and submitted to targets */ 213d87f4c14STejun Heo struct bio flush_bio; 214fd2ed4d2SMikulas Patocka 21596b26c8cSMikulas Patocka /* the number of internal suspends */ 21696b26c8cSMikulas Patocka unsigned internal_suspend_count; 21796b26c8cSMikulas Patocka 218fd2ed4d2SMikulas Patocka struct dm_stats stats; 2192eb6e1e3SKeith Busch 2202eb6e1e3SKeith Busch struct kthread_worker kworker; 2212eb6e1e3SKeith Busch struct task_struct *kworker_task; 222de3ec86dSMike Snitzer 223de3ec86dSMike Snitzer /* for request-based merge heuristic in dm_request_fn() */ 2240ce65797SMike Snitzer unsigned seq_rq_merge_deadline_usecs; 225de3ec86dSMike Snitzer int last_rq_rw; 2260ce65797SMike Snitzer sector_t last_rq_pos; 2270ce65797SMike Snitzer ktime_t last_rq_start_time; 228bfebd1cdSMike Snitzer 229bfebd1cdSMike Snitzer /* for blk-mq request-based DM support */ 230bfebd1cdSMike Snitzer struct blk_mq_tag_set tag_set; 23117e149b8SMike Snitzer bool use_blk_mq; 2321da177e4SLinus Torvalds }; 2331da177e4SLinus Torvalds 23417e149b8SMike Snitzer #ifdef CONFIG_DM_MQ_DEFAULT 23517e149b8SMike Snitzer static bool use_blk_mq = true; 23617e149b8SMike Snitzer #else 23717e149b8SMike Snitzer static bool use_blk_mq = false; 23817e149b8SMike Snitzer #endif 23917e149b8SMike Snitzer 24017e149b8SMike Snitzer bool dm_use_blk_mq(struct mapped_device *md) 24117e149b8SMike Snitzer { 24217e149b8SMike Snitzer return md->use_blk_mq; 24317e149b8SMike Snitzer } 24417e149b8SMike Snitzer 245e6ee8c0bSKiyoshi Ueda /* 246e6ee8c0bSKiyoshi Ueda * For mempools pre-allocation at the table loading time. 247e6ee8c0bSKiyoshi Ueda */ 248e6ee8c0bSKiyoshi Ueda struct dm_md_mempools { 249e6ee8c0bSKiyoshi Ueda mempool_t *io_pool; 2501ae49ea2SMike Snitzer mempool_t *rq_pool; 251e6ee8c0bSKiyoshi Ueda struct bio_set *bs; 252e6ee8c0bSKiyoshi Ueda }; 253e6ee8c0bSKiyoshi Ueda 25486f1152bSBenjamin Marzinski struct table_device { 25586f1152bSBenjamin Marzinski struct list_head list; 25686f1152bSBenjamin Marzinski atomic_t count; 25786f1152bSBenjamin Marzinski struct dm_dev dm_dev; 25886f1152bSBenjamin Marzinski }; 25986f1152bSBenjamin Marzinski 2606cfa5857SMike Snitzer #define RESERVED_BIO_BASED_IOS 16 2616cfa5857SMike Snitzer #define RESERVED_REQUEST_BASED_IOS 256 262f4790826SMike Snitzer #define RESERVED_MAX_IOS 1024 263e18b890bSChristoph Lameter static struct kmem_cache *_io_cache; 2648fbf26adSKiyoshi Ueda static struct kmem_cache *_rq_tio_cache; 2651ae49ea2SMike Snitzer static struct kmem_cache *_rq_cache; 26694818742SKent Overstreet 267f4790826SMike Snitzer /* 268e8603136SMike Snitzer * Bio-based DM's mempools' reserved IOs set by the user. 269e8603136SMike Snitzer */ 270e8603136SMike Snitzer static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS; 271e8603136SMike Snitzer 272e8603136SMike Snitzer /* 273f4790826SMike Snitzer * Request-based DM's mempools' reserved IOs set by the user. 274f4790826SMike Snitzer */ 275f4790826SMike Snitzer static unsigned reserved_rq_based_ios = RESERVED_REQUEST_BASED_IOS; 276f4790826SMike Snitzer 27709c2d531SMike Snitzer static unsigned __dm_get_module_param(unsigned *module_param, 278f4790826SMike Snitzer unsigned def, unsigned max) 279f4790826SMike Snitzer { 28009c2d531SMike Snitzer unsigned param = ACCESS_ONCE(*module_param); 28109c2d531SMike Snitzer unsigned modified_param = 0; 282f4790826SMike Snitzer 28309c2d531SMike Snitzer if (!param) 28409c2d531SMike Snitzer modified_param = def; 28509c2d531SMike Snitzer else if (param > max) 28609c2d531SMike Snitzer modified_param = max; 287f4790826SMike Snitzer 28809c2d531SMike Snitzer if (modified_param) { 28909c2d531SMike Snitzer (void)cmpxchg(module_param, param, modified_param); 29009c2d531SMike Snitzer param = modified_param; 291f4790826SMike Snitzer } 292f4790826SMike Snitzer 29309c2d531SMike Snitzer return param; 294f4790826SMike Snitzer } 295f4790826SMike Snitzer 296e8603136SMike Snitzer unsigned dm_get_reserved_bio_based_ios(void) 297e8603136SMike Snitzer { 29809c2d531SMike Snitzer return __dm_get_module_param(&reserved_bio_based_ios, 299e8603136SMike Snitzer RESERVED_BIO_BASED_IOS, RESERVED_MAX_IOS); 300e8603136SMike Snitzer } 301e8603136SMike Snitzer EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios); 302e8603136SMike Snitzer 303f4790826SMike Snitzer unsigned dm_get_reserved_rq_based_ios(void) 304f4790826SMike Snitzer { 30509c2d531SMike Snitzer return __dm_get_module_param(&reserved_rq_based_ios, 306f4790826SMike Snitzer RESERVED_REQUEST_BASED_IOS, RESERVED_MAX_IOS); 307f4790826SMike Snitzer } 308f4790826SMike Snitzer EXPORT_SYMBOL_GPL(dm_get_reserved_rq_based_ios); 309f4790826SMike Snitzer 3101da177e4SLinus Torvalds static int __init local_init(void) 3111da177e4SLinus Torvalds { 31251157b4aSKiyoshi Ueda int r = -ENOMEM; 3131da177e4SLinus Torvalds 3141da177e4SLinus Torvalds /* allocate a slab for the dm_ios */ 315028867acSAlasdair G Kergon _io_cache = KMEM_CACHE(dm_io, 0); 3161da177e4SLinus Torvalds if (!_io_cache) 31751157b4aSKiyoshi Ueda return r; 3181da177e4SLinus Torvalds 3198fbf26adSKiyoshi Ueda _rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0); 3208fbf26adSKiyoshi Ueda if (!_rq_tio_cache) 321dba14160SMikulas Patocka goto out_free_io_cache; 3228fbf26adSKiyoshi Ueda 3231ae49ea2SMike Snitzer _rq_cache = kmem_cache_create("dm_clone_request", sizeof(struct request), 3241ae49ea2SMike Snitzer __alignof__(struct request), 0, NULL); 3251ae49ea2SMike Snitzer if (!_rq_cache) 3261ae49ea2SMike Snitzer goto out_free_rq_tio_cache; 3271ae49ea2SMike Snitzer 32851e5b2bdSMike Anderson r = dm_uevent_init(); 32951157b4aSKiyoshi Ueda if (r) 3301ae49ea2SMike Snitzer goto out_free_rq_cache; 33151e5b2bdSMike Anderson 332acfe0ad7SMikulas Patocka deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1); 333acfe0ad7SMikulas Patocka if (!deferred_remove_workqueue) { 334acfe0ad7SMikulas Patocka r = -ENOMEM; 335acfe0ad7SMikulas Patocka goto out_uevent_exit; 336acfe0ad7SMikulas Patocka } 337acfe0ad7SMikulas Patocka 3381da177e4SLinus Torvalds _major = major; 3391da177e4SLinus Torvalds r = register_blkdev(_major, _name); 34051157b4aSKiyoshi Ueda if (r < 0) 341acfe0ad7SMikulas Patocka goto out_free_workqueue; 3421da177e4SLinus Torvalds 3431da177e4SLinus Torvalds if (!_major) 3441da177e4SLinus Torvalds _major = r; 3451da177e4SLinus Torvalds 3461da177e4SLinus Torvalds return 0; 34751157b4aSKiyoshi Ueda 348acfe0ad7SMikulas Patocka out_free_workqueue: 349acfe0ad7SMikulas Patocka destroy_workqueue(deferred_remove_workqueue); 35051157b4aSKiyoshi Ueda out_uevent_exit: 35151157b4aSKiyoshi Ueda dm_uevent_exit(); 3521ae49ea2SMike Snitzer out_free_rq_cache: 3531ae49ea2SMike Snitzer kmem_cache_destroy(_rq_cache); 3548fbf26adSKiyoshi Ueda out_free_rq_tio_cache: 3558fbf26adSKiyoshi Ueda kmem_cache_destroy(_rq_tio_cache); 35651157b4aSKiyoshi Ueda out_free_io_cache: 35751157b4aSKiyoshi Ueda kmem_cache_destroy(_io_cache); 35851157b4aSKiyoshi Ueda 35951157b4aSKiyoshi Ueda return r; 3601da177e4SLinus Torvalds } 3611da177e4SLinus Torvalds 3621da177e4SLinus Torvalds static void local_exit(void) 3631da177e4SLinus Torvalds { 3642c140a24SMikulas Patocka flush_scheduled_work(); 365acfe0ad7SMikulas Patocka destroy_workqueue(deferred_remove_workqueue); 3662c140a24SMikulas Patocka 3671ae49ea2SMike Snitzer kmem_cache_destroy(_rq_cache); 3688fbf26adSKiyoshi Ueda kmem_cache_destroy(_rq_tio_cache); 3691da177e4SLinus Torvalds kmem_cache_destroy(_io_cache); 37000d59405SAkinobu Mita unregister_blkdev(_major, _name); 37151e5b2bdSMike Anderson dm_uevent_exit(); 3721da177e4SLinus Torvalds 3731da177e4SLinus Torvalds _major = 0; 3741da177e4SLinus Torvalds 3751da177e4SLinus Torvalds DMINFO("cleaned up"); 3761da177e4SLinus Torvalds } 3771da177e4SLinus Torvalds 378b9249e55SAlasdair G Kergon static int (*_inits[])(void) __initdata = { 3791da177e4SLinus Torvalds local_init, 3801da177e4SLinus Torvalds dm_target_init, 3811da177e4SLinus Torvalds dm_linear_init, 3821da177e4SLinus Torvalds dm_stripe_init, 383952b3557SMikulas Patocka dm_io_init, 384945fa4d2SMikulas Patocka dm_kcopyd_init, 3851da177e4SLinus Torvalds dm_interface_init, 386fd2ed4d2SMikulas Patocka dm_statistics_init, 3871da177e4SLinus Torvalds }; 3881da177e4SLinus Torvalds 389b9249e55SAlasdair G Kergon static void (*_exits[])(void) = { 3901da177e4SLinus Torvalds local_exit, 3911da177e4SLinus Torvalds dm_target_exit, 3921da177e4SLinus Torvalds dm_linear_exit, 3931da177e4SLinus Torvalds dm_stripe_exit, 394952b3557SMikulas Patocka dm_io_exit, 395945fa4d2SMikulas Patocka dm_kcopyd_exit, 3961da177e4SLinus Torvalds dm_interface_exit, 397fd2ed4d2SMikulas Patocka dm_statistics_exit, 3981da177e4SLinus Torvalds }; 3991da177e4SLinus Torvalds 4001da177e4SLinus Torvalds static int __init dm_init(void) 4011da177e4SLinus Torvalds { 4021da177e4SLinus Torvalds const int count = ARRAY_SIZE(_inits); 4031da177e4SLinus Torvalds 4041da177e4SLinus Torvalds int r, i; 4051da177e4SLinus Torvalds 4061da177e4SLinus Torvalds for (i = 0; i < count; i++) { 4071da177e4SLinus Torvalds r = _inits[i](); 4081da177e4SLinus Torvalds if (r) 4091da177e4SLinus Torvalds goto bad; 4101da177e4SLinus Torvalds } 4111da177e4SLinus Torvalds 4121da177e4SLinus Torvalds return 0; 4131da177e4SLinus Torvalds 4141da177e4SLinus Torvalds bad: 4151da177e4SLinus Torvalds while (i--) 4161da177e4SLinus Torvalds _exits[i](); 4171da177e4SLinus Torvalds 4181da177e4SLinus Torvalds return r; 4191da177e4SLinus Torvalds } 4201da177e4SLinus Torvalds 4211da177e4SLinus Torvalds static void __exit dm_exit(void) 4221da177e4SLinus Torvalds { 4231da177e4SLinus Torvalds int i = ARRAY_SIZE(_exits); 4241da177e4SLinus Torvalds 4251da177e4SLinus Torvalds while (i--) 4261da177e4SLinus Torvalds _exits[i](); 427d15b774cSAlasdair G Kergon 428d15b774cSAlasdair G Kergon /* 429d15b774cSAlasdair G Kergon * Should be empty by this point. 430d15b774cSAlasdair G Kergon */ 431d15b774cSAlasdair G Kergon idr_destroy(&_minor_idr); 4321da177e4SLinus Torvalds } 4331da177e4SLinus Torvalds 4341da177e4SLinus Torvalds /* 4351da177e4SLinus Torvalds * Block device functions 4361da177e4SLinus Torvalds */ 437432a212cSMike Anderson int dm_deleting_md(struct mapped_device *md) 438432a212cSMike Anderson { 439432a212cSMike Anderson return test_bit(DMF_DELETING, &md->flags); 440432a212cSMike Anderson } 441432a212cSMike Anderson 442fe5f9f2cSAl Viro static int dm_blk_open(struct block_device *bdev, fmode_t mode) 4431da177e4SLinus Torvalds { 4441da177e4SLinus Torvalds struct mapped_device *md; 4451da177e4SLinus Torvalds 446fba9f90eSJeff Mahoney spin_lock(&_minor_lock); 447fba9f90eSJeff Mahoney 448fe5f9f2cSAl Viro md = bdev->bd_disk->private_data; 449fba9f90eSJeff Mahoney if (!md) 450fba9f90eSJeff Mahoney goto out; 451fba9f90eSJeff Mahoney 4525c6bd75dSAlasdair G Kergon if (test_bit(DMF_FREEING, &md->flags) || 453432a212cSMike Anderson dm_deleting_md(md)) { 454fba9f90eSJeff Mahoney md = NULL; 455fba9f90eSJeff Mahoney goto out; 456fba9f90eSJeff Mahoney } 457fba9f90eSJeff Mahoney 4581da177e4SLinus Torvalds dm_get(md); 4595c6bd75dSAlasdair G Kergon atomic_inc(&md->open_count); 460fba9f90eSJeff Mahoney out: 461fba9f90eSJeff Mahoney spin_unlock(&_minor_lock); 462fba9f90eSJeff Mahoney 463fba9f90eSJeff Mahoney return md ? 0 : -ENXIO; 4641da177e4SLinus Torvalds } 4651da177e4SLinus Torvalds 466db2a144bSAl Viro static void dm_blk_close(struct gendisk *disk, fmode_t mode) 4671da177e4SLinus Torvalds { 46863a4f065SMike Snitzer struct mapped_device *md; 4696e9624b8SArnd Bergmann 4704a1aeb98SMilan Broz spin_lock(&_minor_lock); 4714a1aeb98SMilan Broz 47263a4f065SMike Snitzer md = disk->private_data; 47363a4f065SMike Snitzer if (WARN_ON(!md)) 47463a4f065SMike Snitzer goto out; 47563a4f065SMike Snitzer 4762c140a24SMikulas Patocka if (atomic_dec_and_test(&md->open_count) && 4772c140a24SMikulas Patocka (test_bit(DMF_DEFERRED_REMOVE, &md->flags))) 478acfe0ad7SMikulas Patocka queue_work(deferred_remove_workqueue, &deferred_remove_work); 4792c140a24SMikulas Patocka 4801da177e4SLinus Torvalds dm_put(md); 48163a4f065SMike Snitzer out: 4824a1aeb98SMilan Broz spin_unlock(&_minor_lock); 4831da177e4SLinus Torvalds } 4841da177e4SLinus Torvalds 4855c6bd75dSAlasdair G Kergon int dm_open_count(struct mapped_device *md) 4865c6bd75dSAlasdair G Kergon { 4875c6bd75dSAlasdair G Kergon return atomic_read(&md->open_count); 4885c6bd75dSAlasdair G Kergon } 4895c6bd75dSAlasdair G Kergon 4905c6bd75dSAlasdair G Kergon /* 4915c6bd75dSAlasdair G Kergon * Guarantees nothing is using the device before it's deleted. 4925c6bd75dSAlasdair G Kergon */ 4932c140a24SMikulas Patocka int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred) 4945c6bd75dSAlasdair G Kergon { 4955c6bd75dSAlasdair G Kergon int r = 0; 4965c6bd75dSAlasdair G Kergon 4975c6bd75dSAlasdair G Kergon spin_lock(&_minor_lock); 4985c6bd75dSAlasdair G Kergon 4992c140a24SMikulas Patocka if (dm_open_count(md)) { 5005c6bd75dSAlasdair G Kergon r = -EBUSY; 5012c140a24SMikulas Patocka if (mark_deferred) 5022c140a24SMikulas Patocka set_bit(DMF_DEFERRED_REMOVE, &md->flags); 5032c140a24SMikulas Patocka } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags)) 5042c140a24SMikulas Patocka r = -EEXIST; 5055c6bd75dSAlasdair G Kergon else 5065c6bd75dSAlasdair G Kergon set_bit(DMF_DELETING, &md->flags); 5075c6bd75dSAlasdair G Kergon 5085c6bd75dSAlasdair G Kergon spin_unlock(&_minor_lock); 5095c6bd75dSAlasdair G Kergon 5105c6bd75dSAlasdair G Kergon return r; 5115c6bd75dSAlasdair G Kergon } 5125c6bd75dSAlasdair G Kergon 5132c140a24SMikulas Patocka int dm_cancel_deferred_remove(struct mapped_device *md) 5142c140a24SMikulas Patocka { 5152c140a24SMikulas Patocka int r = 0; 5162c140a24SMikulas Patocka 5172c140a24SMikulas Patocka spin_lock(&_minor_lock); 5182c140a24SMikulas Patocka 5192c140a24SMikulas Patocka if (test_bit(DMF_DELETING, &md->flags)) 5202c140a24SMikulas Patocka r = -EBUSY; 5212c140a24SMikulas Patocka else 5222c140a24SMikulas Patocka clear_bit(DMF_DEFERRED_REMOVE, &md->flags); 5232c140a24SMikulas Patocka 5242c140a24SMikulas Patocka spin_unlock(&_minor_lock); 5252c140a24SMikulas Patocka 5262c140a24SMikulas Patocka return r; 5272c140a24SMikulas Patocka } 5282c140a24SMikulas Patocka 5292c140a24SMikulas Patocka static void do_deferred_remove(struct work_struct *w) 5302c140a24SMikulas Patocka { 5312c140a24SMikulas Patocka dm_deferred_remove(); 5322c140a24SMikulas Patocka } 5332c140a24SMikulas Patocka 534fd2ed4d2SMikulas Patocka sector_t dm_get_size(struct mapped_device *md) 535fd2ed4d2SMikulas Patocka { 536fd2ed4d2SMikulas Patocka return get_capacity(md->disk); 537fd2ed4d2SMikulas Patocka } 538fd2ed4d2SMikulas Patocka 5399974fa2cSMike Snitzer struct request_queue *dm_get_md_queue(struct mapped_device *md) 5409974fa2cSMike Snitzer { 5419974fa2cSMike Snitzer return md->queue; 5429974fa2cSMike Snitzer } 5439974fa2cSMike Snitzer 544fd2ed4d2SMikulas Patocka struct dm_stats *dm_get_stats(struct mapped_device *md) 545fd2ed4d2SMikulas Patocka { 546fd2ed4d2SMikulas Patocka return &md->stats; 547fd2ed4d2SMikulas Patocka } 548fd2ed4d2SMikulas Patocka 5493ac51e74SDarrick J. Wong static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo) 5503ac51e74SDarrick J. Wong { 5513ac51e74SDarrick J. Wong struct mapped_device *md = bdev->bd_disk->private_data; 5523ac51e74SDarrick J. Wong 5533ac51e74SDarrick J. Wong return dm_get_geometry(md, geo); 5543ac51e74SDarrick J. Wong } 5553ac51e74SDarrick J. Wong 556fe5f9f2cSAl Viro static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode, 557aa129a22SMilan Broz unsigned int cmd, unsigned long arg) 558aa129a22SMilan Broz { 559fe5f9f2cSAl Viro struct mapped_device *md = bdev->bd_disk->private_data; 56083d5e5b0SMikulas Patocka int srcu_idx; 5616c182cd8SHannes Reinecke struct dm_table *map; 562aa129a22SMilan Broz struct dm_target *tgt; 563aa129a22SMilan Broz int r = -ENOTTY; 564aa129a22SMilan Broz 5656c182cd8SHannes Reinecke retry: 56683d5e5b0SMikulas Patocka map = dm_get_live_table(md, &srcu_idx); 56783d5e5b0SMikulas Patocka 568aa129a22SMilan Broz if (!map || !dm_table_get_size(map)) 569aa129a22SMilan Broz goto out; 570aa129a22SMilan Broz 571aa129a22SMilan Broz /* We only support devices that have a single target */ 572aa129a22SMilan Broz if (dm_table_get_num_targets(map) != 1) 573aa129a22SMilan Broz goto out; 574aa129a22SMilan Broz 575aa129a22SMilan Broz tgt = dm_table_get_target(map, 0); 5764d341d82SMike Snitzer if (!tgt->type->ioctl) 5774d341d82SMike Snitzer goto out; 578aa129a22SMilan Broz 5794f186f8bSKiyoshi Ueda if (dm_suspended_md(md)) { 580aa129a22SMilan Broz r = -EAGAIN; 581aa129a22SMilan Broz goto out; 582aa129a22SMilan Broz } 583aa129a22SMilan Broz 584647b3d00SAl Viro r = tgt->type->ioctl(tgt, cmd, arg); 585aa129a22SMilan Broz 586aa129a22SMilan Broz out: 58783d5e5b0SMikulas Patocka dm_put_live_table(md, srcu_idx); 588aa129a22SMilan Broz 5896c182cd8SHannes Reinecke if (r == -ENOTCONN) { 5906c182cd8SHannes Reinecke msleep(10); 5916c182cd8SHannes Reinecke goto retry; 5926c182cd8SHannes Reinecke } 5936c182cd8SHannes Reinecke 594aa129a22SMilan Broz return r; 595aa129a22SMilan Broz } 596aa129a22SMilan Broz 597028867acSAlasdair G Kergon static struct dm_io *alloc_io(struct mapped_device *md) 5981da177e4SLinus Torvalds { 5991da177e4SLinus Torvalds return mempool_alloc(md->io_pool, GFP_NOIO); 6001da177e4SLinus Torvalds } 6011da177e4SLinus Torvalds 602028867acSAlasdair G Kergon static void free_io(struct mapped_device *md, struct dm_io *io) 6031da177e4SLinus Torvalds { 6041da177e4SLinus Torvalds mempool_free(io, md->io_pool); 6051da177e4SLinus Torvalds } 6061da177e4SLinus Torvalds 607028867acSAlasdair G Kergon static void free_tio(struct mapped_device *md, struct dm_target_io *tio) 6081da177e4SLinus Torvalds { 609dba14160SMikulas Patocka bio_put(&tio->clone); 6101da177e4SLinus Torvalds } 6111da177e4SLinus Torvalds 61208885643SKiyoshi Ueda static struct dm_rq_target_io *alloc_rq_tio(struct mapped_device *md, 61308885643SKiyoshi Ueda gfp_t gfp_mask) 614cec47e3dSKiyoshi Ueda { 6155f015204SJun'ichi Nomura return mempool_alloc(md->io_pool, gfp_mask); 616cec47e3dSKiyoshi Ueda } 617cec47e3dSKiyoshi Ueda 618cec47e3dSKiyoshi Ueda static void free_rq_tio(struct dm_rq_target_io *tio) 619cec47e3dSKiyoshi Ueda { 6205f015204SJun'ichi Nomura mempool_free(tio, tio->md->io_pool); 621cec47e3dSKiyoshi Ueda } 622cec47e3dSKiyoshi Ueda 6231ae49ea2SMike Snitzer static struct request *alloc_clone_request(struct mapped_device *md, 6241ae49ea2SMike Snitzer gfp_t gfp_mask) 6251ae49ea2SMike Snitzer { 6261ae49ea2SMike Snitzer return mempool_alloc(md->rq_pool, gfp_mask); 6271ae49ea2SMike Snitzer } 6281ae49ea2SMike Snitzer 6291ae49ea2SMike Snitzer static void free_clone_request(struct mapped_device *md, struct request *rq) 6301ae49ea2SMike Snitzer { 6311ae49ea2SMike Snitzer mempool_free(rq, md->rq_pool); 6321ae49ea2SMike Snitzer } 6331ae49ea2SMike Snitzer 63490abb8c4SKiyoshi Ueda static int md_in_flight(struct mapped_device *md) 63590abb8c4SKiyoshi Ueda { 63690abb8c4SKiyoshi Ueda return atomic_read(&md->pending[READ]) + 63790abb8c4SKiyoshi Ueda atomic_read(&md->pending[WRITE]); 63890abb8c4SKiyoshi Ueda } 63990abb8c4SKiyoshi Ueda 6403eaf840eSJun'ichi "Nick" Nomura static void start_io_acct(struct dm_io *io) 6413eaf840eSJun'ichi "Nick" Nomura { 6423eaf840eSJun'ichi "Nick" Nomura struct mapped_device *md = io->md; 643fd2ed4d2SMikulas Patocka struct bio *bio = io->bio; 644c9959059STejun Heo int cpu; 645fd2ed4d2SMikulas Patocka int rw = bio_data_dir(bio); 6463eaf840eSJun'ichi "Nick" Nomura 6473eaf840eSJun'ichi "Nick" Nomura io->start_time = jiffies; 6483eaf840eSJun'ichi "Nick" Nomura 649074a7acaSTejun Heo cpu = part_stat_lock(); 650074a7acaSTejun Heo part_round_stats(cpu, &dm_disk(md)->part0); 651074a7acaSTejun Heo part_stat_unlock(); 6521e9bb880SShaohua Li atomic_set(&dm_disk(md)->part0.in_flight[rw], 6531e9bb880SShaohua Li atomic_inc_return(&md->pending[rw])); 654fd2ed4d2SMikulas Patocka 655fd2ed4d2SMikulas Patocka if (unlikely(dm_stats_used(&md->stats))) 6564f024f37SKent Overstreet dm_stats_account_io(&md->stats, bio->bi_rw, bio->bi_iter.bi_sector, 657fd2ed4d2SMikulas Patocka bio_sectors(bio), false, 0, &io->stats_aux); 6583eaf840eSJun'ichi "Nick" Nomura } 6593eaf840eSJun'ichi "Nick" Nomura 660d221d2e7SMikulas Patocka static void end_io_acct(struct dm_io *io) 6613eaf840eSJun'ichi "Nick" Nomura { 6623eaf840eSJun'ichi "Nick" Nomura struct mapped_device *md = io->md; 6633eaf840eSJun'ichi "Nick" Nomura struct bio *bio = io->bio; 6643eaf840eSJun'ichi "Nick" Nomura unsigned long duration = jiffies - io->start_time; 66518c0b223SGu Zheng int pending; 6663eaf840eSJun'ichi "Nick" Nomura int rw = bio_data_dir(bio); 6673eaf840eSJun'ichi "Nick" Nomura 66818c0b223SGu Zheng generic_end_io_acct(rw, &dm_disk(md)->part0, io->start_time); 6693eaf840eSJun'ichi "Nick" Nomura 670fd2ed4d2SMikulas Patocka if (unlikely(dm_stats_used(&md->stats))) 6714f024f37SKent Overstreet dm_stats_account_io(&md->stats, bio->bi_rw, bio->bi_iter.bi_sector, 672fd2ed4d2SMikulas Patocka bio_sectors(bio), true, duration, &io->stats_aux); 673fd2ed4d2SMikulas Patocka 674af7e466aSMikulas Patocka /* 675af7e466aSMikulas Patocka * After this is decremented the bio must not be touched if it is 676d87f4c14STejun Heo * a flush. 677af7e466aSMikulas Patocka */ 6781e9bb880SShaohua Li pending = atomic_dec_return(&md->pending[rw]); 6791e9bb880SShaohua Li atomic_set(&dm_disk(md)->part0.in_flight[rw], pending); 680316d315bSNikanth Karthikesan pending += atomic_read(&md->pending[rw^0x1]); 6813eaf840eSJun'ichi "Nick" Nomura 682d221d2e7SMikulas Patocka /* nudge anyone waiting on suspend queue */ 683d221d2e7SMikulas Patocka if (!pending) 684d221d2e7SMikulas Patocka wake_up(&md->wait); 6853eaf840eSJun'ichi "Nick" Nomura } 6863eaf840eSJun'ichi "Nick" Nomura 6871da177e4SLinus Torvalds /* 6881da177e4SLinus Torvalds * Add the bio to the list of deferred io. 6891da177e4SLinus Torvalds */ 69092c63902SMikulas Patocka static void queue_io(struct mapped_device *md, struct bio *bio) 6911da177e4SLinus Torvalds { 69205447420SKiyoshi Ueda unsigned long flags; 6931da177e4SLinus Torvalds 69405447420SKiyoshi Ueda spin_lock_irqsave(&md->deferred_lock, flags); 6951da177e4SLinus Torvalds bio_list_add(&md->deferred, bio); 69605447420SKiyoshi Ueda spin_unlock_irqrestore(&md->deferred_lock, flags); 69792c63902SMikulas Patocka queue_work(md->wq, &md->work); 6981da177e4SLinus Torvalds } 6991da177e4SLinus Torvalds 7001da177e4SLinus Torvalds /* 7011da177e4SLinus Torvalds * Everyone (including functions in this file), should use this 7021da177e4SLinus Torvalds * function to access the md->map field, and make sure they call 70383d5e5b0SMikulas Patocka * dm_put_live_table() when finished. 7041da177e4SLinus Torvalds */ 70583d5e5b0SMikulas Patocka struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier) 7061da177e4SLinus Torvalds { 70783d5e5b0SMikulas Patocka *srcu_idx = srcu_read_lock(&md->io_barrier); 7081da177e4SLinus Torvalds 70983d5e5b0SMikulas Patocka return srcu_dereference(md->map, &md->io_barrier); 71083d5e5b0SMikulas Patocka } 7111da177e4SLinus Torvalds 71283d5e5b0SMikulas Patocka void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier) 71383d5e5b0SMikulas Patocka { 71483d5e5b0SMikulas Patocka srcu_read_unlock(&md->io_barrier, srcu_idx); 71583d5e5b0SMikulas Patocka } 71683d5e5b0SMikulas Patocka 71783d5e5b0SMikulas Patocka void dm_sync_table(struct mapped_device *md) 71883d5e5b0SMikulas Patocka { 71983d5e5b0SMikulas Patocka synchronize_srcu(&md->io_barrier); 72083d5e5b0SMikulas Patocka synchronize_rcu_expedited(); 72183d5e5b0SMikulas Patocka } 72283d5e5b0SMikulas Patocka 72383d5e5b0SMikulas Patocka /* 72483d5e5b0SMikulas Patocka * A fast alternative to dm_get_live_table/dm_put_live_table. 72583d5e5b0SMikulas Patocka * The caller must not block between these two functions. 72683d5e5b0SMikulas Patocka */ 72783d5e5b0SMikulas Patocka static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU) 72883d5e5b0SMikulas Patocka { 72983d5e5b0SMikulas Patocka rcu_read_lock(); 73083d5e5b0SMikulas Patocka return rcu_dereference(md->map); 73183d5e5b0SMikulas Patocka } 73283d5e5b0SMikulas Patocka 73383d5e5b0SMikulas Patocka static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU) 73483d5e5b0SMikulas Patocka { 73583d5e5b0SMikulas Patocka rcu_read_unlock(); 7361da177e4SLinus Torvalds } 7371da177e4SLinus Torvalds 7383ac51e74SDarrick J. Wong /* 73986f1152bSBenjamin Marzinski * Open a table device so we can use it as a map destination. 74086f1152bSBenjamin Marzinski */ 74186f1152bSBenjamin Marzinski static int open_table_device(struct table_device *td, dev_t dev, 74286f1152bSBenjamin Marzinski struct mapped_device *md) 74386f1152bSBenjamin Marzinski { 74486f1152bSBenjamin Marzinski static char *_claim_ptr = "I belong to device-mapper"; 74586f1152bSBenjamin Marzinski struct block_device *bdev; 74686f1152bSBenjamin Marzinski 74786f1152bSBenjamin Marzinski int r; 74886f1152bSBenjamin Marzinski 74986f1152bSBenjamin Marzinski BUG_ON(td->dm_dev.bdev); 75086f1152bSBenjamin Marzinski 75186f1152bSBenjamin Marzinski bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _claim_ptr); 75286f1152bSBenjamin Marzinski if (IS_ERR(bdev)) 75386f1152bSBenjamin Marzinski return PTR_ERR(bdev); 75486f1152bSBenjamin Marzinski 75586f1152bSBenjamin Marzinski r = bd_link_disk_holder(bdev, dm_disk(md)); 75686f1152bSBenjamin Marzinski if (r) { 75786f1152bSBenjamin Marzinski blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL); 75886f1152bSBenjamin Marzinski return r; 75986f1152bSBenjamin Marzinski } 76086f1152bSBenjamin Marzinski 76186f1152bSBenjamin Marzinski td->dm_dev.bdev = bdev; 76286f1152bSBenjamin Marzinski return 0; 76386f1152bSBenjamin Marzinski } 76486f1152bSBenjamin Marzinski 76586f1152bSBenjamin Marzinski /* 76686f1152bSBenjamin Marzinski * Close a table device that we've been using. 76786f1152bSBenjamin Marzinski */ 76886f1152bSBenjamin Marzinski static void close_table_device(struct table_device *td, struct mapped_device *md) 76986f1152bSBenjamin Marzinski { 77086f1152bSBenjamin Marzinski if (!td->dm_dev.bdev) 77186f1152bSBenjamin Marzinski return; 77286f1152bSBenjamin Marzinski 77386f1152bSBenjamin Marzinski bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md)); 77486f1152bSBenjamin Marzinski blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL); 77586f1152bSBenjamin Marzinski td->dm_dev.bdev = NULL; 77686f1152bSBenjamin Marzinski } 77786f1152bSBenjamin Marzinski 77886f1152bSBenjamin Marzinski static struct table_device *find_table_device(struct list_head *l, dev_t dev, 77986f1152bSBenjamin Marzinski fmode_t mode) { 78086f1152bSBenjamin Marzinski struct table_device *td; 78186f1152bSBenjamin Marzinski 78286f1152bSBenjamin Marzinski list_for_each_entry(td, l, list) 78386f1152bSBenjamin Marzinski if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode) 78486f1152bSBenjamin Marzinski return td; 78586f1152bSBenjamin Marzinski 78686f1152bSBenjamin Marzinski return NULL; 78786f1152bSBenjamin Marzinski } 78886f1152bSBenjamin Marzinski 78986f1152bSBenjamin Marzinski int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode, 79086f1152bSBenjamin Marzinski struct dm_dev **result) { 79186f1152bSBenjamin Marzinski int r; 79286f1152bSBenjamin Marzinski struct table_device *td; 79386f1152bSBenjamin Marzinski 79486f1152bSBenjamin Marzinski mutex_lock(&md->table_devices_lock); 79586f1152bSBenjamin Marzinski td = find_table_device(&md->table_devices, dev, mode); 79686f1152bSBenjamin Marzinski if (!td) { 79786f1152bSBenjamin Marzinski td = kmalloc(sizeof(*td), GFP_KERNEL); 79886f1152bSBenjamin Marzinski if (!td) { 79986f1152bSBenjamin Marzinski mutex_unlock(&md->table_devices_lock); 80086f1152bSBenjamin Marzinski return -ENOMEM; 80186f1152bSBenjamin Marzinski } 80286f1152bSBenjamin Marzinski 80386f1152bSBenjamin Marzinski td->dm_dev.mode = mode; 80486f1152bSBenjamin Marzinski td->dm_dev.bdev = NULL; 80586f1152bSBenjamin Marzinski 80686f1152bSBenjamin Marzinski if ((r = open_table_device(td, dev, md))) { 80786f1152bSBenjamin Marzinski mutex_unlock(&md->table_devices_lock); 80886f1152bSBenjamin Marzinski kfree(td); 80986f1152bSBenjamin Marzinski return r; 81086f1152bSBenjamin Marzinski } 81186f1152bSBenjamin Marzinski 81286f1152bSBenjamin Marzinski format_dev_t(td->dm_dev.name, dev); 81386f1152bSBenjamin Marzinski 81486f1152bSBenjamin Marzinski atomic_set(&td->count, 0); 81586f1152bSBenjamin Marzinski list_add(&td->list, &md->table_devices); 81686f1152bSBenjamin Marzinski } 81786f1152bSBenjamin Marzinski atomic_inc(&td->count); 81886f1152bSBenjamin Marzinski mutex_unlock(&md->table_devices_lock); 81986f1152bSBenjamin Marzinski 82086f1152bSBenjamin Marzinski *result = &td->dm_dev; 82186f1152bSBenjamin Marzinski return 0; 82286f1152bSBenjamin Marzinski } 82386f1152bSBenjamin Marzinski EXPORT_SYMBOL_GPL(dm_get_table_device); 82486f1152bSBenjamin Marzinski 82586f1152bSBenjamin Marzinski void dm_put_table_device(struct mapped_device *md, struct dm_dev *d) 82686f1152bSBenjamin Marzinski { 82786f1152bSBenjamin Marzinski struct table_device *td = container_of(d, struct table_device, dm_dev); 82886f1152bSBenjamin Marzinski 82986f1152bSBenjamin Marzinski mutex_lock(&md->table_devices_lock); 83086f1152bSBenjamin Marzinski if (atomic_dec_and_test(&td->count)) { 83186f1152bSBenjamin Marzinski close_table_device(td, md); 83286f1152bSBenjamin Marzinski list_del(&td->list); 83386f1152bSBenjamin Marzinski kfree(td); 83486f1152bSBenjamin Marzinski } 83586f1152bSBenjamin Marzinski mutex_unlock(&md->table_devices_lock); 83686f1152bSBenjamin Marzinski } 83786f1152bSBenjamin Marzinski EXPORT_SYMBOL(dm_put_table_device); 83886f1152bSBenjamin Marzinski 83986f1152bSBenjamin Marzinski static void free_table_devices(struct list_head *devices) 84086f1152bSBenjamin Marzinski { 84186f1152bSBenjamin Marzinski struct list_head *tmp, *next; 84286f1152bSBenjamin Marzinski 84386f1152bSBenjamin Marzinski list_for_each_safe(tmp, next, devices) { 84486f1152bSBenjamin Marzinski struct table_device *td = list_entry(tmp, struct table_device, list); 84586f1152bSBenjamin Marzinski 84686f1152bSBenjamin Marzinski DMWARN("dm_destroy: %s still exists with %d references", 84786f1152bSBenjamin Marzinski td->dm_dev.name, atomic_read(&td->count)); 84886f1152bSBenjamin Marzinski kfree(td); 84986f1152bSBenjamin Marzinski } 85086f1152bSBenjamin Marzinski } 85186f1152bSBenjamin Marzinski 85286f1152bSBenjamin Marzinski /* 8533ac51e74SDarrick J. Wong * Get the geometry associated with a dm device 8543ac51e74SDarrick J. Wong */ 8553ac51e74SDarrick J. Wong int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo) 8563ac51e74SDarrick J. Wong { 8573ac51e74SDarrick J. Wong *geo = md->geometry; 8583ac51e74SDarrick J. Wong 8593ac51e74SDarrick J. Wong return 0; 8603ac51e74SDarrick J. Wong } 8613ac51e74SDarrick J. Wong 8623ac51e74SDarrick J. Wong /* 8633ac51e74SDarrick J. Wong * Set the geometry of a device. 8643ac51e74SDarrick J. Wong */ 8653ac51e74SDarrick J. Wong int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo) 8663ac51e74SDarrick J. Wong { 8673ac51e74SDarrick J. Wong sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors; 8683ac51e74SDarrick J. Wong 8693ac51e74SDarrick J. Wong if (geo->start > sz) { 8703ac51e74SDarrick J. Wong DMWARN("Start sector is beyond the geometry limits."); 8713ac51e74SDarrick J. Wong return -EINVAL; 8723ac51e74SDarrick J. Wong } 8733ac51e74SDarrick J. Wong 8743ac51e74SDarrick J. Wong md->geometry = *geo; 8753ac51e74SDarrick J. Wong 8763ac51e74SDarrick J. Wong return 0; 8773ac51e74SDarrick J. Wong } 8783ac51e74SDarrick J. Wong 8791da177e4SLinus Torvalds /*----------------------------------------------------------------- 8801da177e4SLinus Torvalds * CRUD START: 8811da177e4SLinus Torvalds * A more elegant soln is in the works that uses the queue 8821da177e4SLinus Torvalds * merge fn, unfortunately there are a couple of changes to 8831da177e4SLinus Torvalds * the block layer that I want to make for this. So in the 8841da177e4SLinus Torvalds * interests of getting something for people to use I give 8851da177e4SLinus Torvalds * you this clearly demarcated crap. 8861da177e4SLinus Torvalds *---------------------------------------------------------------*/ 8871da177e4SLinus Torvalds 8882e93ccc1SKiyoshi Ueda static int __noflush_suspending(struct mapped_device *md) 8892e93ccc1SKiyoshi Ueda { 8902e93ccc1SKiyoshi Ueda return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 8912e93ccc1SKiyoshi Ueda } 8922e93ccc1SKiyoshi Ueda 8931da177e4SLinus Torvalds /* 8941da177e4SLinus Torvalds * Decrements the number of outstanding ios that a bio has been 8951da177e4SLinus Torvalds * cloned into, completing the original io if necc. 8961da177e4SLinus Torvalds */ 897858119e1SArjan van de Ven static void dec_pending(struct dm_io *io, int error) 8981da177e4SLinus Torvalds { 8992e93ccc1SKiyoshi Ueda unsigned long flags; 900b35f8caaSMilan Broz int io_error; 901b35f8caaSMilan Broz struct bio *bio; 902b35f8caaSMilan Broz struct mapped_device *md = io->md; 9032e93ccc1SKiyoshi Ueda 9042e93ccc1SKiyoshi Ueda /* Push-back supersedes any I/O errors */ 905f88fb981SKiyoshi Ueda if (unlikely(error)) { 906f88fb981SKiyoshi Ueda spin_lock_irqsave(&io->endio_lock, flags); 907f88fb981SKiyoshi Ueda if (!(io->error > 0 && __noflush_suspending(md))) 9081da177e4SLinus Torvalds io->error = error; 909f88fb981SKiyoshi Ueda spin_unlock_irqrestore(&io->endio_lock, flags); 910f88fb981SKiyoshi Ueda } 9111da177e4SLinus Torvalds 9121da177e4SLinus Torvalds if (atomic_dec_and_test(&io->io_count)) { 9132e93ccc1SKiyoshi Ueda if (io->error == DM_ENDIO_REQUEUE) { 9142e93ccc1SKiyoshi Ueda /* 9152e93ccc1SKiyoshi Ueda * Target requested pushing back the I/O. 9162e93ccc1SKiyoshi Ueda */ 917022c2611SMikulas Patocka spin_lock_irqsave(&md->deferred_lock, flags); 9186a8736d1STejun Heo if (__noflush_suspending(md)) 9196a8736d1STejun Heo bio_list_add_head(&md->deferred, io->bio); 9206a8736d1STejun Heo else 9212e93ccc1SKiyoshi Ueda /* noflush suspend was interrupted. */ 9222e93ccc1SKiyoshi Ueda io->error = -EIO; 923022c2611SMikulas Patocka spin_unlock_irqrestore(&md->deferred_lock, flags); 9242e93ccc1SKiyoshi Ueda } 9252e93ccc1SKiyoshi Ueda 926b35f8caaSMilan Broz io_error = io->error; 927b35f8caaSMilan Broz bio = io->bio; 928af7e466aSMikulas Patocka end_io_acct(io); 929a97f925aSMikulas Patocka free_io(md, io); 9301da177e4SLinus Torvalds 9316a8736d1STejun Heo if (io_error == DM_ENDIO_REQUEUE) 9326a8736d1STejun Heo return; 9336a8736d1STejun Heo 9344f024f37SKent Overstreet if ((bio->bi_rw & REQ_FLUSH) && bio->bi_iter.bi_size) { 9351da177e4SLinus Torvalds /* 9366a8736d1STejun Heo * Preflush done for flush with data, reissue 9376a8736d1STejun Heo * without REQ_FLUSH. 9381da177e4SLinus Torvalds */ 9396a8736d1STejun Heo bio->bi_rw &= ~REQ_FLUSH; 9406a8736d1STejun Heo queue_io(md, bio); 9415f3ea37cSArnaldo Carvalho de Melo } else { 942b372d360SMike Snitzer /* done with normal IO or empty flush */ 9430a82a8d1SLinus Torvalds trace_block_bio_complete(md->queue, bio, io_error); 944b35f8caaSMilan Broz bio_endio(bio, io_error); 9452e93ccc1SKiyoshi Ueda } 9461da177e4SLinus Torvalds } 947af7e466aSMikulas Patocka } 9481da177e4SLinus Torvalds 9497eee4ae2SMike Snitzer static void disable_write_same(struct mapped_device *md) 9507eee4ae2SMike Snitzer { 9517eee4ae2SMike Snitzer struct queue_limits *limits = dm_get_queue_limits(md); 9527eee4ae2SMike Snitzer 9537eee4ae2SMike Snitzer /* device doesn't really support WRITE SAME, disable it */ 9547eee4ae2SMike Snitzer limits->max_write_same_sectors = 0; 9557eee4ae2SMike Snitzer } 9567eee4ae2SMike Snitzer 9576712ecf8SNeilBrown static void clone_endio(struct bio *bio, int error) 9581da177e4SLinus Torvalds { 9595164beceSzhendong chen int r = error; 960bfc6d41cSMikulas Patocka struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone); 961b35f8caaSMilan Broz struct dm_io *io = tio->io; 9629faf400fSStefan Bader struct mapped_device *md = tio->io->md; 9631da177e4SLinus Torvalds dm_endio_fn endio = tio->ti->type->end_io; 9641da177e4SLinus Torvalds 9651da177e4SLinus Torvalds if (!bio_flagged(bio, BIO_UPTODATE) && !error) 9661da177e4SLinus Torvalds error = -EIO; 9671da177e4SLinus Torvalds 9681da177e4SLinus Torvalds if (endio) { 9697de3ee57SMikulas Patocka r = endio(tio->ti, bio, error); 9702e93ccc1SKiyoshi Ueda if (r < 0 || r == DM_ENDIO_REQUEUE) 9712e93ccc1SKiyoshi Ueda /* 9722e93ccc1SKiyoshi Ueda * error and requeue request are handled 9732e93ccc1SKiyoshi Ueda * in dec_pending(). 9742e93ccc1SKiyoshi Ueda */ 9751da177e4SLinus Torvalds error = r; 97645cbcd79SKiyoshi Ueda else if (r == DM_ENDIO_INCOMPLETE) 97745cbcd79SKiyoshi Ueda /* The target will handle the io */ 9786712ecf8SNeilBrown return; 97945cbcd79SKiyoshi Ueda else if (r) { 98045cbcd79SKiyoshi Ueda DMWARN("unimplemented target endio return value: %d", r); 98145cbcd79SKiyoshi Ueda BUG(); 98245cbcd79SKiyoshi Ueda } 9831da177e4SLinus Torvalds } 9841da177e4SLinus Torvalds 9857eee4ae2SMike Snitzer if (unlikely(r == -EREMOTEIO && (bio->bi_rw & REQ_WRITE_SAME) && 9867eee4ae2SMike Snitzer !bdev_get_queue(bio->bi_bdev)->limits.max_write_same_sectors)) 9877eee4ae2SMike Snitzer disable_write_same(md); 9887eee4ae2SMike Snitzer 9899faf400fSStefan Bader free_tio(md, tio); 990b35f8caaSMilan Broz dec_pending(io, error); 9911da177e4SLinus Torvalds } 9921da177e4SLinus Torvalds 993bfebd1cdSMike Snitzer static struct dm_rq_target_io *tio_from_request(struct request *rq) 994bfebd1cdSMike Snitzer { 995bfebd1cdSMike Snitzer return (rq->q->mq_ops ? blk_mq_rq_to_pdu(rq) : rq->special); 996bfebd1cdSMike Snitzer } 997bfebd1cdSMike Snitzer 998cec47e3dSKiyoshi Ueda /* 999cec47e3dSKiyoshi Ueda * Don't touch any member of the md after calling this function because 1000cec47e3dSKiyoshi Ueda * the md may be freed in dm_put() at the end of this function. 1001cec47e3dSKiyoshi Ueda * Or do dm_get() before calling this function and dm_put() later. 1002cec47e3dSKiyoshi Ueda */ 1003466d89a6SKeith Busch static void rq_completed(struct mapped_device *md, int rw, bool run_queue) 1004cec47e3dSKiyoshi Ueda { 10059a0e609eSMike Snitzer int nr_requests_pending; 10069a0e609eSMike Snitzer 1007b4324feeSKiyoshi Ueda atomic_dec(&md->pending[rw]); 1008cec47e3dSKiyoshi Ueda 1009cec47e3dSKiyoshi Ueda /* nudge anyone waiting on suspend queue */ 10109a0e609eSMike Snitzer nr_requests_pending = md_in_flight(md); 10119a0e609eSMike Snitzer if (!nr_requests_pending) 1012cec47e3dSKiyoshi Ueda wake_up(&md->wait); 1013cec47e3dSKiyoshi Ueda 1014a8c32a5cSJens Axboe /* 1015a8c32a5cSJens Axboe * Run this off this callpath, as drivers could invoke end_io while 1016a8c32a5cSJens Axboe * inside their request_fn (and holding the queue lock). Calling 1017a8c32a5cSJens Axboe * back into ->request_fn() could deadlock attempting to grab the 1018a8c32a5cSJens Axboe * queue lock again. 1019a8c32a5cSJens Axboe */ 10209a0e609eSMike Snitzer if (run_queue) { 1021bfebd1cdSMike Snitzer if (md->queue->mq_ops) 1022bfebd1cdSMike Snitzer blk_mq_run_hw_queues(md->queue, true); 1023bfebd1cdSMike Snitzer else if (!nr_requests_pending || 10249a0e609eSMike Snitzer (nr_requests_pending >= md->queue->nr_congestion_on)) 1025a8c32a5cSJens Axboe blk_run_queue_async(md->queue); 10269a0e609eSMike Snitzer } 1027cec47e3dSKiyoshi Ueda 1028cec47e3dSKiyoshi Ueda /* 1029cec47e3dSKiyoshi Ueda * dm_put() must be at the end of this function. See the comment above 1030cec47e3dSKiyoshi Ueda */ 1031cec47e3dSKiyoshi Ueda dm_put(md); 1032cec47e3dSKiyoshi Ueda } 1033cec47e3dSKiyoshi Ueda 1034aa6df8ddSMike Snitzer static void free_rq_clone(struct request *clone, bool must_be_mapped) 1035a77e28c7SKiyoshi Ueda { 1036a77e28c7SKiyoshi Ueda struct dm_rq_target_io *tio = clone->end_io_data; 1037bfebd1cdSMike Snitzer struct mapped_device *md = tio->md; 1038a77e28c7SKiyoshi Ueda 1039aa6df8ddSMike Snitzer WARN_ON_ONCE(must_be_mapped && !clone->q); 1040aa6df8ddSMike Snitzer 1041aa6df8ddSMike Snitzer if (md->type == DM_TYPE_MQ_REQUEST_BASED) 1042aa6df8ddSMike Snitzer /* stacked on blk-mq queue(s) */ 1043e5863d9aSMike Snitzer tio->ti->type->release_clone_rq(clone); 104402233342SMike Snitzer else if (!md->queue->mq_ops) 104502233342SMike Snitzer /* request_fn queue stacked on request_fn queue(s) */ 1046bfebd1cdSMike Snitzer free_clone_request(md, clone); 1047aa6df8ddSMike Snitzer /* 1048aa6df8ddSMike Snitzer * NOTE: for the blk-mq queue stacked on request_fn queue(s) case: 1049aa6df8ddSMike Snitzer * no need to call free_clone_request() because we leverage blk-mq by 1050aa6df8ddSMike Snitzer * allocating the clone at the end of the blk-mq pdu (see: clone_rq) 1051aa6df8ddSMike Snitzer */ 1052bfebd1cdSMike Snitzer 1053bfebd1cdSMike Snitzer if (!md->queue->mq_ops) 1054a77e28c7SKiyoshi Ueda free_rq_tio(tio); 1055a77e28c7SKiyoshi Ueda } 1056a77e28c7SKiyoshi Ueda 1057980691e5SKiyoshi Ueda /* 1058980691e5SKiyoshi Ueda * Complete the clone and the original request. 1059466d89a6SKeith Busch * Must be called without clone's queue lock held, 1060466d89a6SKeith Busch * see end_clone_request() for more details. 1061980691e5SKiyoshi Ueda */ 1062980691e5SKiyoshi Ueda static void dm_end_request(struct request *clone, int error) 1063980691e5SKiyoshi Ueda { 1064980691e5SKiyoshi Ueda int rw = rq_data_dir(clone); 1065980691e5SKiyoshi Ueda struct dm_rq_target_io *tio = clone->end_io_data; 1066980691e5SKiyoshi Ueda struct mapped_device *md = tio->md; 1067980691e5SKiyoshi Ueda struct request *rq = tio->orig; 1068980691e5SKiyoshi Ueda 106929e4013dSTejun Heo if (rq->cmd_type == REQ_TYPE_BLOCK_PC) { 1070980691e5SKiyoshi Ueda rq->errors = clone->errors; 1071980691e5SKiyoshi Ueda rq->resid_len = clone->resid_len; 1072980691e5SKiyoshi Ueda 1073980691e5SKiyoshi Ueda if (rq->sense) 1074980691e5SKiyoshi Ueda /* 1075980691e5SKiyoshi Ueda * We are using the sense buffer of the original 1076980691e5SKiyoshi Ueda * request. 1077980691e5SKiyoshi Ueda * So setting the length of the sense data is enough. 1078980691e5SKiyoshi Ueda */ 1079980691e5SKiyoshi Ueda rq->sense_len = clone->sense_len; 1080980691e5SKiyoshi Ueda } 1081980691e5SKiyoshi Ueda 1082aa6df8ddSMike Snitzer free_rq_clone(clone, true); 1083bfebd1cdSMike Snitzer if (!rq->q->mq_ops) 1084980691e5SKiyoshi Ueda blk_end_request_all(rq, error); 1085bfebd1cdSMike Snitzer else 1086bfebd1cdSMike Snitzer blk_mq_end_request(rq, error); 108729e4013dSTejun Heo rq_completed(md, rw, true); 1088980691e5SKiyoshi Ueda } 1089980691e5SKiyoshi Ueda 1090cec47e3dSKiyoshi Ueda static void dm_unprep_request(struct request *rq) 1091cec47e3dSKiyoshi Ueda { 1092bfebd1cdSMike Snitzer struct dm_rq_target_io *tio = tio_from_request(rq); 1093466d89a6SKeith Busch struct request *clone = tio->clone; 1094cec47e3dSKiyoshi Ueda 1095bfebd1cdSMike Snitzer if (!rq->q->mq_ops) { 1096cec47e3dSKiyoshi Ueda rq->special = NULL; 1097cec47e3dSKiyoshi Ueda rq->cmd_flags &= ~REQ_DONTPREP; 1098bfebd1cdSMike Snitzer } 1099cec47e3dSKiyoshi Ueda 1100e5863d9aSMike Snitzer if (clone) 1101aa6df8ddSMike Snitzer free_rq_clone(clone, false); 1102cec47e3dSKiyoshi Ueda } 1103cec47e3dSKiyoshi Ueda 1104cec47e3dSKiyoshi Ueda /* 1105cec47e3dSKiyoshi Ueda * Requeue the original request of a clone. 1106cec47e3dSKiyoshi Ueda */ 1107bfebd1cdSMike Snitzer static void old_requeue_request(struct request *rq) 1108cec47e3dSKiyoshi Ueda { 1109cec47e3dSKiyoshi Ueda struct request_queue *q = rq->q; 1110cec47e3dSKiyoshi Ueda unsigned long flags; 1111cec47e3dSKiyoshi Ueda 1112cec47e3dSKiyoshi Ueda spin_lock_irqsave(q->queue_lock, flags); 1113cec47e3dSKiyoshi Ueda blk_requeue_request(q, rq); 1114cec47e3dSKiyoshi Ueda spin_unlock_irqrestore(q->queue_lock, flags); 1115bfebd1cdSMike Snitzer } 1116bfebd1cdSMike Snitzer 1117bfebd1cdSMike Snitzer static void dm_requeue_unmapped_original_request(struct mapped_device *md, 1118bfebd1cdSMike Snitzer struct request *rq) 1119bfebd1cdSMike Snitzer { 1120bfebd1cdSMike Snitzer int rw = rq_data_dir(rq); 1121bfebd1cdSMike Snitzer 1122bfebd1cdSMike Snitzer dm_unprep_request(rq); 1123bfebd1cdSMike Snitzer 1124bfebd1cdSMike Snitzer if (!rq->q->mq_ops) 1125bfebd1cdSMike Snitzer old_requeue_request(rq); 1126bfebd1cdSMike Snitzer else { 1127bfebd1cdSMike Snitzer blk_mq_requeue_request(rq); 1128bfebd1cdSMike Snitzer blk_mq_kick_requeue_list(rq->q); 1129bfebd1cdSMike Snitzer } 1130cec47e3dSKiyoshi Ueda 1131466d89a6SKeith Busch rq_completed(md, rw, false); 1132cec47e3dSKiyoshi Ueda } 1133466d89a6SKeith Busch 1134466d89a6SKeith Busch static void dm_requeue_unmapped_request(struct request *clone) 1135466d89a6SKeith Busch { 1136466d89a6SKeith Busch struct dm_rq_target_io *tio = clone->end_io_data; 1137466d89a6SKeith Busch 1138466d89a6SKeith Busch dm_requeue_unmapped_original_request(tio->md, tio->orig); 1139cec47e3dSKiyoshi Ueda } 1140cec47e3dSKiyoshi Ueda 1141bfebd1cdSMike Snitzer static void old_stop_queue(struct request_queue *q) 1142cec47e3dSKiyoshi Ueda { 1143bfebd1cdSMike Snitzer unsigned long flags; 1144bfebd1cdSMike Snitzer 1145bfebd1cdSMike Snitzer if (blk_queue_stopped(q)) 1146bfebd1cdSMike Snitzer return; 1147bfebd1cdSMike Snitzer 1148bfebd1cdSMike Snitzer spin_lock_irqsave(q->queue_lock, flags); 1149cec47e3dSKiyoshi Ueda blk_stop_queue(q); 1150bfebd1cdSMike Snitzer spin_unlock_irqrestore(q->queue_lock, flags); 1151cec47e3dSKiyoshi Ueda } 1152cec47e3dSKiyoshi Ueda 1153cec47e3dSKiyoshi Ueda static void stop_queue(struct request_queue *q) 1154cec47e3dSKiyoshi Ueda { 1155bfebd1cdSMike Snitzer if (!q->mq_ops) 1156bfebd1cdSMike Snitzer old_stop_queue(q); 1157bfebd1cdSMike Snitzer else 1158bfebd1cdSMike Snitzer blk_mq_stop_hw_queues(q); 1159bfebd1cdSMike Snitzer } 1160bfebd1cdSMike Snitzer 1161bfebd1cdSMike Snitzer static void old_start_queue(struct request_queue *q) 1162bfebd1cdSMike Snitzer { 1163cec47e3dSKiyoshi Ueda unsigned long flags; 1164cec47e3dSKiyoshi Ueda 1165cec47e3dSKiyoshi Ueda spin_lock_irqsave(q->queue_lock, flags); 1166cec47e3dSKiyoshi Ueda if (blk_queue_stopped(q)) 1167cec47e3dSKiyoshi Ueda blk_start_queue(q); 1168bfebd1cdSMike Snitzer spin_unlock_irqrestore(q->queue_lock, flags); 1169cec47e3dSKiyoshi Ueda } 1170cec47e3dSKiyoshi Ueda 1171cec47e3dSKiyoshi Ueda static void start_queue(struct request_queue *q) 1172cec47e3dSKiyoshi Ueda { 1173bfebd1cdSMike Snitzer if (!q->mq_ops) 1174bfebd1cdSMike Snitzer old_start_queue(q); 1175bfebd1cdSMike Snitzer else 1176bfebd1cdSMike Snitzer blk_mq_start_stopped_hw_queues(q, true); 1177cec47e3dSKiyoshi Ueda } 1178cec47e3dSKiyoshi Ueda 117911a68244SKiyoshi Ueda static void dm_done(struct request *clone, int error, bool mapped) 118011a68244SKiyoshi Ueda { 118111a68244SKiyoshi Ueda int r = error; 118211a68244SKiyoshi Ueda struct dm_rq_target_io *tio = clone->end_io_data; 1183ba1cbad9SMike Snitzer dm_request_endio_fn rq_end_io = NULL; 1184ba1cbad9SMike Snitzer 1185ba1cbad9SMike Snitzer if (tio->ti) { 1186ba1cbad9SMike Snitzer rq_end_io = tio->ti->type->rq_end_io; 118711a68244SKiyoshi Ueda 118811a68244SKiyoshi Ueda if (mapped && rq_end_io) 118911a68244SKiyoshi Ueda r = rq_end_io(tio->ti, clone, error, &tio->info); 1190ba1cbad9SMike Snitzer } 119111a68244SKiyoshi Ueda 11927eee4ae2SMike Snitzer if (unlikely(r == -EREMOTEIO && (clone->cmd_flags & REQ_WRITE_SAME) && 11937eee4ae2SMike Snitzer !clone->q->limits.max_write_same_sectors)) 11947eee4ae2SMike Snitzer disable_write_same(tio->md); 11957eee4ae2SMike Snitzer 119611a68244SKiyoshi Ueda if (r <= 0) 119711a68244SKiyoshi Ueda /* The target wants to complete the I/O */ 119811a68244SKiyoshi Ueda dm_end_request(clone, r); 119911a68244SKiyoshi Ueda else if (r == DM_ENDIO_INCOMPLETE) 120011a68244SKiyoshi Ueda /* The target will handle the I/O */ 120111a68244SKiyoshi Ueda return; 120211a68244SKiyoshi Ueda else if (r == DM_ENDIO_REQUEUE) 120311a68244SKiyoshi Ueda /* The target wants to requeue the I/O */ 120411a68244SKiyoshi Ueda dm_requeue_unmapped_request(clone); 120511a68244SKiyoshi Ueda else { 120611a68244SKiyoshi Ueda DMWARN("unimplemented target endio return value: %d", r); 120711a68244SKiyoshi Ueda BUG(); 120811a68244SKiyoshi Ueda } 120911a68244SKiyoshi Ueda } 121011a68244SKiyoshi Ueda 1211cec47e3dSKiyoshi Ueda /* 1212cec47e3dSKiyoshi Ueda * Request completion handler for request-based dm 1213cec47e3dSKiyoshi Ueda */ 1214cec47e3dSKiyoshi Ueda static void dm_softirq_done(struct request *rq) 1215cec47e3dSKiyoshi Ueda { 121611a68244SKiyoshi Ueda bool mapped = true; 1217bfebd1cdSMike Snitzer struct dm_rq_target_io *tio = tio_from_request(rq); 1218466d89a6SKeith Busch struct request *clone = tio->clone; 1219bfebd1cdSMike Snitzer int rw; 1220cec47e3dSKiyoshi Ueda 1221e5863d9aSMike Snitzer if (!clone) { 1222bfebd1cdSMike Snitzer rw = rq_data_dir(rq); 1223bfebd1cdSMike Snitzer if (!rq->q->mq_ops) { 1224e5863d9aSMike Snitzer blk_end_request_all(rq, tio->error); 1225bfebd1cdSMike Snitzer rq_completed(tio->md, rw, false); 1226e5863d9aSMike Snitzer free_rq_tio(tio); 1227bfebd1cdSMike Snitzer } else { 1228bfebd1cdSMike Snitzer blk_mq_end_request(rq, tio->error); 1229bfebd1cdSMike Snitzer rq_completed(tio->md, rw, false); 1230bfebd1cdSMike Snitzer } 1231e5863d9aSMike Snitzer return; 1232e5863d9aSMike Snitzer } 1233cec47e3dSKiyoshi Ueda 123411a68244SKiyoshi Ueda if (rq->cmd_flags & REQ_FAILED) 123511a68244SKiyoshi Ueda mapped = false; 1236cec47e3dSKiyoshi Ueda 123711a68244SKiyoshi Ueda dm_done(clone, tio->error, mapped); 1238cec47e3dSKiyoshi Ueda } 1239cec47e3dSKiyoshi Ueda 1240cec47e3dSKiyoshi Ueda /* 1241cec47e3dSKiyoshi Ueda * Complete the clone and the original request with the error status 1242cec47e3dSKiyoshi Ueda * through softirq context. 1243cec47e3dSKiyoshi Ueda */ 1244466d89a6SKeith Busch static void dm_complete_request(struct request *rq, int error) 1245cec47e3dSKiyoshi Ueda { 1246bfebd1cdSMike Snitzer struct dm_rq_target_io *tio = tio_from_request(rq); 1247cec47e3dSKiyoshi Ueda 1248cec47e3dSKiyoshi Ueda tio->error = error; 1249cec47e3dSKiyoshi Ueda blk_complete_request(rq); 1250cec47e3dSKiyoshi Ueda } 1251cec47e3dSKiyoshi Ueda 1252cec47e3dSKiyoshi Ueda /* 1253cec47e3dSKiyoshi Ueda * Complete the not-mapped clone and the original request with the error status 1254cec47e3dSKiyoshi Ueda * through softirq context. 1255cec47e3dSKiyoshi Ueda * Target's rq_end_io() function isn't called. 1256e5863d9aSMike Snitzer * This may be used when the target's map_rq() or clone_and_map_rq() functions fail. 1257cec47e3dSKiyoshi Ueda */ 1258466d89a6SKeith Busch static void dm_kill_unmapped_request(struct request *rq, int error) 1259cec47e3dSKiyoshi Ueda { 1260cec47e3dSKiyoshi Ueda rq->cmd_flags |= REQ_FAILED; 1261466d89a6SKeith Busch dm_complete_request(rq, error); 1262cec47e3dSKiyoshi Ueda } 1263cec47e3dSKiyoshi Ueda 1264cec47e3dSKiyoshi Ueda /* 1265bfebd1cdSMike Snitzer * Called with the clone's queue lock held (for non-blk-mq) 1266cec47e3dSKiyoshi Ueda */ 1267cec47e3dSKiyoshi Ueda static void end_clone_request(struct request *clone, int error) 1268cec47e3dSKiyoshi Ueda { 1269466d89a6SKeith Busch struct dm_rq_target_io *tio = clone->end_io_data; 1270466d89a6SKeith Busch 1271e5863d9aSMike Snitzer if (!clone->q->mq_ops) { 1272cec47e3dSKiyoshi Ueda /* 1273cec47e3dSKiyoshi Ueda * For just cleaning up the information of the queue in which 1274cec47e3dSKiyoshi Ueda * the clone was dispatched. 1275e5863d9aSMike Snitzer * The clone is *NOT* freed actually here because it is alloced 1276e5863d9aSMike Snitzer * from dm own mempool (REQ_ALLOCED isn't set). 1277cec47e3dSKiyoshi Ueda */ 1278cec47e3dSKiyoshi Ueda __blk_put_request(clone->q, clone); 1279e5863d9aSMike Snitzer } 1280cec47e3dSKiyoshi Ueda 1281cec47e3dSKiyoshi Ueda /* 1282cec47e3dSKiyoshi Ueda * Actual request completion is done in a softirq context which doesn't 1283466d89a6SKeith Busch * hold the clone's queue lock. Otherwise, deadlock could occur because: 1284cec47e3dSKiyoshi Ueda * - another request may be submitted by the upper level driver 1285cec47e3dSKiyoshi Ueda * of the stacking during the completion 1286cec47e3dSKiyoshi Ueda * - the submission which requires queue lock may be done 1287466d89a6SKeith Busch * against this clone's queue 1288cec47e3dSKiyoshi Ueda */ 1289466d89a6SKeith Busch dm_complete_request(tio->orig, error); 1290cec47e3dSKiyoshi Ueda } 1291cec47e3dSKiyoshi Ueda 129256a67df7SMike Snitzer /* 129356a67df7SMike Snitzer * Return maximum size of I/O possible at the supplied sector up to the current 129456a67df7SMike Snitzer * target boundary. 129556a67df7SMike Snitzer */ 129656a67df7SMike Snitzer static sector_t max_io_len_target_boundary(sector_t sector, struct dm_target *ti) 12971da177e4SLinus Torvalds { 129856a67df7SMike Snitzer sector_t target_offset = dm_target_offset(ti, sector); 129956a67df7SMike Snitzer 130056a67df7SMike Snitzer return ti->len - target_offset; 130156a67df7SMike Snitzer } 130256a67df7SMike Snitzer 130356a67df7SMike Snitzer static sector_t max_io_len(sector_t sector, struct dm_target *ti) 130456a67df7SMike Snitzer { 130556a67df7SMike Snitzer sector_t len = max_io_len_target_boundary(sector, ti); 1306542f9038SMike Snitzer sector_t offset, max_len; 13071da177e4SLinus Torvalds 13081da177e4SLinus Torvalds /* 13091da177e4SLinus Torvalds * Does the target need to split even further? 13101da177e4SLinus Torvalds */ 1311542f9038SMike Snitzer if (ti->max_io_len) { 1312542f9038SMike Snitzer offset = dm_target_offset(ti, sector); 1313542f9038SMike Snitzer if (unlikely(ti->max_io_len & (ti->max_io_len - 1))) 1314542f9038SMike Snitzer max_len = sector_div(offset, ti->max_io_len); 1315542f9038SMike Snitzer else 1316542f9038SMike Snitzer max_len = offset & (ti->max_io_len - 1); 1317542f9038SMike Snitzer max_len = ti->max_io_len - max_len; 1318542f9038SMike Snitzer 1319542f9038SMike Snitzer if (len > max_len) 1320542f9038SMike Snitzer len = max_len; 13211da177e4SLinus Torvalds } 13221da177e4SLinus Torvalds 13231da177e4SLinus Torvalds return len; 13241da177e4SLinus Torvalds } 13251da177e4SLinus Torvalds 1326542f9038SMike Snitzer int dm_set_target_max_io_len(struct dm_target *ti, sector_t len) 1327542f9038SMike Snitzer { 1328542f9038SMike Snitzer if (len > UINT_MAX) { 1329542f9038SMike Snitzer DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)", 1330542f9038SMike Snitzer (unsigned long long)len, UINT_MAX); 1331542f9038SMike Snitzer ti->error = "Maximum size of target IO is too large"; 1332542f9038SMike Snitzer return -EINVAL; 1333542f9038SMike Snitzer } 1334542f9038SMike Snitzer 1335542f9038SMike Snitzer ti->max_io_len = (uint32_t) len; 1336542f9038SMike Snitzer 1337542f9038SMike Snitzer return 0; 1338542f9038SMike Snitzer } 1339542f9038SMike Snitzer EXPORT_SYMBOL_GPL(dm_set_target_max_io_len); 1340542f9038SMike Snitzer 13411dd40c3eSMikulas Patocka /* 13421dd40c3eSMikulas Patocka * A target may call dm_accept_partial_bio only from the map routine. It is 13431dd40c3eSMikulas Patocka * allowed for all bio types except REQ_FLUSH. 13441dd40c3eSMikulas Patocka * 13451dd40c3eSMikulas Patocka * dm_accept_partial_bio informs the dm that the target only wants to process 13461dd40c3eSMikulas Patocka * additional n_sectors sectors of the bio and the rest of the data should be 13471dd40c3eSMikulas Patocka * sent in a next bio. 13481dd40c3eSMikulas Patocka * 13491dd40c3eSMikulas Patocka * A diagram that explains the arithmetics: 13501dd40c3eSMikulas Patocka * +--------------------+---------------+-------+ 13511dd40c3eSMikulas Patocka * | 1 | 2 | 3 | 13521dd40c3eSMikulas Patocka * +--------------------+---------------+-------+ 13531dd40c3eSMikulas Patocka * 13541dd40c3eSMikulas Patocka * <-------------- *tio->len_ptr ---------------> 13551dd40c3eSMikulas Patocka * <------- bi_size -------> 13561dd40c3eSMikulas Patocka * <-- n_sectors --> 13571dd40c3eSMikulas Patocka * 13581dd40c3eSMikulas Patocka * Region 1 was already iterated over with bio_advance or similar function. 13591dd40c3eSMikulas Patocka * (it may be empty if the target doesn't use bio_advance) 13601dd40c3eSMikulas Patocka * Region 2 is the remaining bio size that the target wants to process. 13611dd40c3eSMikulas Patocka * (it may be empty if region 1 is non-empty, although there is no reason 13621dd40c3eSMikulas Patocka * to make it empty) 13631dd40c3eSMikulas Patocka * The target requires that region 3 is to be sent in the next bio. 13641dd40c3eSMikulas Patocka * 13651dd40c3eSMikulas Patocka * If the target wants to receive multiple copies of the bio (via num_*bios, etc), 13661dd40c3eSMikulas Patocka * the partially processed part (the sum of regions 1+2) must be the same for all 13671dd40c3eSMikulas Patocka * copies of the bio. 13681dd40c3eSMikulas Patocka */ 13691dd40c3eSMikulas Patocka void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors) 13701dd40c3eSMikulas Patocka { 13711dd40c3eSMikulas Patocka struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone); 13721dd40c3eSMikulas Patocka unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT; 13731dd40c3eSMikulas Patocka BUG_ON(bio->bi_rw & REQ_FLUSH); 13741dd40c3eSMikulas Patocka BUG_ON(bi_size > *tio->len_ptr); 13751dd40c3eSMikulas Patocka BUG_ON(n_sectors > bi_size); 13761dd40c3eSMikulas Patocka *tio->len_ptr -= bi_size - n_sectors; 13771dd40c3eSMikulas Patocka bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT; 13781dd40c3eSMikulas Patocka } 13791dd40c3eSMikulas Patocka EXPORT_SYMBOL_GPL(dm_accept_partial_bio); 13801dd40c3eSMikulas Patocka 1381bd2a49b8SAlasdair G Kergon static void __map_bio(struct dm_target_io *tio) 13821da177e4SLinus Torvalds { 13831da177e4SLinus Torvalds int r; 13842056a782SJens Axboe sector_t sector; 13859faf400fSStefan Bader struct mapped_device *md; 1386dba14160SMikulas Patocka struct bio *clone = &tio->clone; 1387bd2a49b8SAlasdair G Kergon struct dm_target *ti = tio->ti; 13881da177e4SLinus Torvalds 13891da177e4SLinus Torvalds clone->bi_end_io = clone_endio; 13901da177e4SLinus Torvalds 13911da177e4SLinus Torvalds /* 13921da177e4SLinus Torvalds * Map the clone. If r == 0 we don't need to do 13931da177e4SLinus Torvalds * anything, the target has assumed ownership of 13941da177e4SLinus Torvalds * this io. 13951da177e4SLinus Torvalds */ 13961da177e4SLinus Torvalds atomic_inc(&tio->io->io_count); 13974f024f37SKent Overstreet sector = clone->bi_iter.bi_sector; 13987de3ee57SMikulas Patocka r = ti->type->map(ti, clone); 139945cbcd79SKiyoshi Ueda if (r == DM_MAPIO_REMAPPED) { 14001da177e4SLinus Torvalds /* the bio has been remapped so dispatch it */ 14012056a782SJens Axboe 1402d07335e5SMike Snitzer trace_block_bio_remap(bdev_get_queue(clone->bi_bdev), clone, 140322a7c31aSAlan D. Brunelle tio->io->bio->bi_bdev->bd_dev, sector); 14042056a782SJens Axboe 14051da177e4SLinus Torvalds generic_make_request(clone); 14062e93ccc1SKiyoshi Ueda } else if (r < 0 || r == DM_MAPIO_REQUEUE) { 14072e93ccc1SKiyoshi Ueda /* error the io and bail out, or requeue it if needed */ 14089faf400fSStefan Bader md = tio->io->md; 14099faf400fSStefan Bader dec_pending(tio->io, r); 14109faf400fSStefan Bader free_tio(md, tio); 141145cbcd79SKiyoshi Ueda } else if (r) { 141245cbcd79SKiyoshi Ueda DMWARN("unimplemented target map return value: %d", r); 141345cbcd79SKiyoshi Ueda BUG(); 14141da177e4SLinus Torvalds } 14151da177e4SLinus Torvalds } 14161da177e4SLinus Torvalds 14171da177e4SLinus Torvalds struct clone_info { 14181da177e4SLinus Torvalds struct mapped_device *md; 14191da177e4SLinus Torvalds struct dm_table *map; 14201da177e4SLinus Torvalds struct bio *bio; 14211da177e4SLinus Torvalds struct dm_io *io; 14221da177e4SLinus Torvalds sector_t sector; 1423e0d6609aSMikulas Patocka unsigned sector_count; 14241da177e4SLinus Torvalds }; 14251da177e4SLinus Torvalds 1426e0d6609aSMikulas Patocka static void bio_setup_sector(struct bio *bio, sector_t sector, unsigned len) 1427bd2a49b8SAlasdair G Kergon { 14284f024f37SKent Overstreet bio->bi_iter.bi_sector = sector; 14294f024f37SKent Overstreet bio->bi_iter.bi_size = to_bytes(len); 14301da177e4SLinus Torvalds } 14311da177e4SLinus Torvalds 14321da177e4SLinus Torvalds /* 14331da177e4SLinus Torvalds * Creates a bio that consists of range of complete bvecs. 14341da177e4SLinus Torvalds */ 1435dba14160SMikulas Patocka static void clone_bio(struct dm_target_io *tio, struct bio *bio, 14361c3b13e6SKent Overstreet sector_t sector, unsigned len) 14371da177e4SLinus Torvalds { 1438dba14160SMikulas Patocka struct bio *clone = &tio->clone; 14391da177e4SLinus Torvalds 14401c3b13e6SKent Overstreet __bio_clone_fast(clone, bio); 14419c47008dSMartin K. Petersen 14421c3b13e6SKent Overstreet if (bio_integrity(bio)) 14431c3b13e6SKent Overstreet bio_integrity_clone(clone, bio, GFP_NOIO); 14441c3b13e6SKent Overstreet 14451c3b13e6SKent Overstreet bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector)); 14461c3b13e6SKent Overstreet clone->bi_iter.bi_size = to_bytes(len); 14471c3b13e6SKent Overstreet 14481c3b13e6SKent Overstreet if (bio_integrity(bio)) 14491c3b13e6SKent Overstreet bio_integrity_trim(clone, 0, len); 14501da177e4SLinus Torvalds } 14511da177e4SLinus Torvalds 14529015df24SAlasdair G Kergon static struct dm_target_io *alloc_tio(struct clone_info *ci, 145399778273SJunichi Nomura struct dm_target *ti, 145455a62eefSAlasdair G Kergon unsigned target_bio_nr) 1455f9ab94ceSMikulas Patocka { 1456dba14160SMikulas Patocka struct dm_target_io *tio; 1457dba14160SMikulas Patocka struct bio *clone; 1458dba14160SMikulas Patocka 145999778273SJunichi Nomura clone = bio_alloc_bioset(GFP_NOIO, 0, ci->md->bs); 1460dba14160SMikulas Patocka tio = container_of(clone, struct dm_target_io, clone); 1461f9ab94ceSMikulas Patocka 1462f9ab94ceSMikulas Patocka tio->io = ci->io; 1463f9ab94ceSMikulas Patocka tio->ti = ti; 146455a62eefSAlasdair G Kergon tio->target_bio_nr = target_bio_nr; 14659015df24SAlasdair G Kergon 14669015df24SAlasdair G Kergon return tio; 14679015df24SAlasdair G Kergon } 14689015df24SAlasdair G Kergon 146914fe594dSAlasdair G Kergon static void __clone_and_map_simple_bio(struct clone_info *ci, 147014fe594dSAlasdair G Kergon struct dm_target *ti, 14711dd40c3eSMikulas Patocka unsigned target_bio_nr, unsigned *len) 14729015df24SAlasdair G Kergon { 147399778273SJunichi Nomura struct dm_target_io *tio = alloc_tio(ci, ti, target_bio_nr); 1474dba14160SMikulas Patocka struct bio *clone = &tio->clone; 14759015df24SAlasdair G Kergon 14761dd40c3eSMikulas Patocka tio->len_ptr = len; 14771dd40c3eSMikulas Patocka 14781c3b13e6SKent Overstreet __bio_clone_fast(clone, ci->bio); 1479bd2a49b8SAlasdair G Kergon if (len) 14801dd40c3eSMikulas Patocka bio_setup_sector(clone, ci->sector, *len); 1481f9ab94ceSMikulas Patocka 1482bd2a49b8SAlasdair G Kergon __map_bio(tio); 1483f9ab94ceSMikulas Patocka } 1484f9ab94ceSMikulas Patocka 148514fe594dSAlasdair G Kergon static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti, 14861dd40c3eSMikulas Patocka unsigned num_bios, unsigned *len) 148706a426ceSMike Snitzer { 148855a62eefSAlasdair G Kergon unsigned target_bio_nr; 148906a426ceSMike Snitzer 149055a62eefSAlasdair G Kergon for (target_bio_nr = 0; target_bio_nr < num_bios; target_bio_nr++) 149114fe594dSAlasdair G Kergon __clone_and_map_simple_bio(ci, ti, target_bio_nr, len); 149206a426ceSMike Snitzer } 149306a426ceSMike Snitzer 149414fe594dSAlasdair G Kergon static int __send_empty_flush(struct clone_info *ci) 1495f9ab94ceSMikulas Patocka { 149606a426ceSMike Snitzer unsigned target_nr = 0; 1497f9ab94ceSMikulas Patocka struct dm_target *ti; 1498f9ab94ceSMikulas Patocka 1499b372d360SMike Snitzer BUG_ON(bio_has_data(ci->bio)); 1500f9ab94ceSMikulas Patocka while ((ti = dm_table_get_target(ci->map, target_nr++))) 15011dd40c3eSMikulas Patocka __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL); 1502f9ab94ceSMikulas Patocka 1503f9ab94ceSMikulas Patocka return 0; 1504f9ab94ceSMikulas Patocka } 1505f9ab94ceSMikulas Patocka 1506e4c93811SAlasdair G Kergon static void __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti, 15071dd40c3eSMikulas Patocka sector_t sector, unsigned *len) 15085ae89a87SMike Snitzer { 1509dba14160SMikulas Patocka struct bio *bio = ci->bio; 15105ae89a87SMike Snitzer struct dm_target_io *tio; 1511b0d8ed4dSAlasdair G Kergon unsigned target_bio_nr; 1512b0d8ed4dSAlasdair G Kergon unsigned num_target_bios = 1; 15135ae89a87SMike Snitzer 1514b0d8ed4dSAlasdair G Kergon /* 1515b0d8ed4dSAlasdair G Kergon * Does the target want to receive duplicate copies of the bio? 1516b0d8ed4dSAlasdair G Kergon */ 1517b0d8ed4dSAlasdair G Kergon if (bio_data_dir(bio) == WRITE && ti->num_write_bios) 1518b0d8ed4dSAlasdair G Kergon num_target_bios = ti->num_write_bios(ti, bio); 1519e4c93811SAlasdair G Kergon 1520b0d8ed4dSAlasdair G Kergon for (target_bio_nr = 0; target_bio_nr < num_target_bios; target_bio_nr++) { 152199778273SJunichi Nomura tio = alloc_tio(ci, ti, target_bio_nr); 15221dd40c3eSMikulas Patocka tio->len_ptr = len; 15231dd40c3eSMikulas Patocka clone_bio(tio, bio, sector, *len); 1524bd2a49b8SAlasdair G Kergon __map_bio(tio); 15255ae89a87SMike Snitzer } 1526b0d8ed4dSAlasdair G Kergon } 15275ae89a87SMike Snitzer 152855a62eefSAlasdair G Kergon typedef unsigned (*get_num_bios_fn)(struct dm_target *ti); 152923508a96SMike Snitzer 153055a62eefSAlasdair G Kergon static unsigned get_num_discard_bios(struct dm_target *ti) 153123508a96SMike Snitzer { 153255a62eefSAlasdair G Kergon return ti->num_discard_bios; 153323508a96SMike Snitzer } 153423508a96SMike Snitzer 153555a62eefSAlasdair G Kergon static unsigned get_num_write_same_bios(struct dm_target *ti) 153623508a96SMike Snitzer { 153755a62eefSAlasdair G Kergon return ti->num_write_same_bios; 153823508a96SMike Snitzer } 153923508a96SMike Snitzer 154023508a96SMike Snitzer typedef bool (*is_split_required_fn)(struct dm_target *ti); 154123508a96SMike Snitzer 154223508a96SMike Snitzer static bool is_split_required_for_discard(struct dm_target *ti) 154323508a96SMike Snitzer { 154455a62eefSAlasdair G Kergon return ti->split_discard_bios; 154523508a96SMike Snitzer } 154623508a96SMike Snitzer 154714fe594dSAlasdair G Kergon static int __send_changing_extent_only(struct clone_info *ci, 154855a62eefSAlasdair G Kergon get_num_bios_fn get_num_bios, 154923508a96SMike Snitzer is_split_required_fn is_split_required) 15505ae89a87SMike Snitzer { 15515ae89a87SMike Snitzer struct dm_target *ti; 1552e0d6609aSMikulas Patocka unsigned len; 155355a62eefSAlasdair G Kergon unsigned num_bios; 15545ae89a87SMike Snitzer 1555a79245b3SMike Snitzer do { 15565ae89a87SMike Snitzer ti = dm_table_find_target(ci->map, ci->sector); 15575ae89a87SMike Snitzer if (!dm_target_is_valid(ti)) 15585ae89a87SMike Snitzer return -EIO; 15595ae89a87SMike Snitzer 15605ae89a87SMike Snitzer /* 156123508a96SMike Snitzer * Even though the device advertised support for this type of 156223508a96SMike Snitzer * request, that does not mean every target supports it, and 1563936688d7SMike Snitzer * reconfiguration might also have changed that since the 15645ae89a87SMike Snitzer * check was performed. 15655ae89a87SMike Snitzer */ 156655a62eefSAlasdair G Kergon num_bios = get_num_bios ? get_num_bios(ti) : 0; 156755a62eefSAlasdair G Kergon if (!num_bios) 15685ae89a87SMike Snitzer return -EOPNOTSUPP; 15695ae89a87SMike Snitzer 157023508a96SMike Snitzer if (is_split_required && !is_split_required(ti)) 1571e0d6609aSMikulas Patocka len = min((sector_t)ci->sector_count, max_io_len_target_boundary(ci->sector, ti)); 15727acf0277SMikulas Patocka else 1573e0d6609aSMikulas Patocka len = min((sector_t)ci->sector_count, max_io_len(ci->sector, ti)); 15745ae89a87SMike Snitzer 15751dd40c3eSMikulas Patocka __send_duplicate_bios(ci, ti, num_bios, &len); 15765ae89a87SMike Snitzer 1577a79245b3SMike Snitzer ci->sector += len; 1578a79245b3SMike Snitzer } while (ci->sector_count -= len); 15795ae89a87SMike Snitzer 15805ae89a87SMike Snitzer return 0; 15815ae89a87SMike Snitzer } 15825ae89a87SMike Snitzer 158314fe594dSAlasdair G Kergon static int __send_discard(struct clone_info *ci) 158423508a96SMike Snitzer { 158514fe594dSAlasdair G Kergon return __send_changing_extent_only(ci, get_num_discard_bios, 158623508a96SMike Snitzer is_split_required_for_discard); 158723508a96SMike Snitzer } 158823508a96SMike Snitzer 158914fe594dSAlasdair G Kergon static int __send_write_same(struct clone_info *ci) 159023508a96SMike Snitzer { 159114fe594dSAlasdair G Kergon return __send_changing_extent_only(ci, get_num_write_same_bios, NULL); 159223508a96SMike Snitzer } 159323508a96SMike Snitzer 1594e4c93811SAlasdair G Kergon /* 1595e4c93811SAlasdair G Kergon * Select the correct strategy for processing a non-flush bio. 1596e4c93811SAlasdair G Kergon */ 1597e4c93811SAlasdair G Kergon static int __split_and_process_non_flush(struct clone_info *ci) 1598e4c93811SAlasdair G Kergon { 1599e4c93811SAlasdair G Kergon struct bio *bio = ci->bio; 1600e4c93811SAlasdair G Kergon struct dm_target *ti; 16011c3b13e6SKent Overstreet unsigned len; 1602e4c93811SAlasdair G Kergon 1603e4c93811SAlasdair G Kergon if (unlikely(bio->bi_rw & REQ_DISCARD)) 1604e4c93811SAlasdair G Kergon return __send_discard(ci); 1605e4c93811SAlasdair G Kergon else if (unlikely(bio->bi_rw & REQ_WRITE_SAME)) 1606e4c93811SAlasdair G Kergon return __send_write_same(ci); 1607e4c93811SAlasdair G Kergon 1608e4c93811SAlasdair G Kergon ti = dm_table_find_target(ci->map, ci->sector); 1609e4c93811SAlasdair G Kergon if (!dm_target_is_valid(ti)) 1610e4c93811SAlasdair G Kergon return -EIO; 1611e4c93811SAlasdair G Kergon 16121c3b13e6SKent Overstreet len = min_t(sector_t, max_io_len(ci->sector, ti), ci->sector_count); 1613e4c93811SAlasdair G Kergon 16141dd40c3eSMikulas Patocka __clone_and_map_data_bio(ci, ti, ci->sector, &len); 1615e4c93811SAlasdair G Kergon 1616e4c93811SAlasdair G Kergon ci->sector += len; 1617e4c93811SAlasdair G Kergon ci->sector_count -= len; 1618e4c93811SAlasdair G Kergon 1619e4c93811SAlasdair G Kergon return 0; 1620e4c93811SAlasdair G Kergon } 1621e4c93811SAlasdair G Kergon 1622e4c93811SAlasdair G Kergon /* 162314fe594dSAlasdair G Kergon * Entry point to split a bio into clones and submit them to the targets. 16241da177e4SLinus Torvalds */ 162583d5e5b0SMikulas Patocka static void __split_and_process_bio(struct mapped_device *md, 162683d5e5b0SMikulas Patocka struct dm_table *map, struct bio *bio) 16271da177e4SLinus Torvalds { 16281da177e4SLinus Torvalds struct clone_info ci; 1629512875bdSJun'ichi Nomura int error = 0; 16301da177e4SLinus Torvalds 163183d5e5b0SMikulas Patocka if (unlikely(!map)) { 1632f0b9a450SMikulas Patocka bio_io_error(bio); 1633f0b9a450SMikulas Patocka return; 1634f0b9a450SMikulas Patocka } 1635692d0eb9SMikulas Patocka 163683d5e5b0SMikulas Patocka ci.map = map; 16371da177e4SLinus Torvalds ci.md = md; 16381da177e4SLinus Torvalds ci.io = alloc_io(md); 16391da177e4SLinus Torvalds ci.io->error = 0; 16401da177e4SLinus Torvalds atomic_set(&ci.io->io_count, 1); 16411da177e4SLinus Torvalds ci.io->bio = bio; 16421da177e4SLinus Torvalds ci.io->md = md; 1643f88fb981SKiyoshi Ueda spin_lock_init(&ci.io->endio_lock); 16444f024f37SKent Overstreet ci.sector = bio->bi_iter.bi_sector; 16451da177e4SLinus Torvalds 16463eaf840eSJun'ichi "Nick" Nomura start_io_acct(ci.io); 1647bd2a49b8SAlasdair G Kergon 1648b372d360SMike Snitzer if (bio->bi_rw & REQ_FLUSH) { 1649b372d360SMike Snitzer ci.bio = &ci.md->flush_bio; 1650b372d360SMike Snitzer ci.sector_count = 0; 165114fe594dSAlasdair G Kergon error = __send_empty_flush(&ci); 1652b372d360SMike Snitzer /* dec_pending submits any data associated with flush */ 1653b372d360SMike Snitzer } else { 16546a8736d1STejun Heo ci.bio = bio; 1655f6fccb12SMilan Broz ci.sector_count = bio_sectors(bio); 1656512875bdSJun'ichi Nomura while (ci.sector_count && !error) 165714fe594dSAlasdair G Kergon error = __split_and_process_non_flush(&ci); 1658d87f4c14STejun Heo } 16591da177e4SLinus Torvalds 16601da177e4SLinus Torvalds /* drop the extra reference count */ 1661512875bdSJun'ichi Nomura dec_pending(ci.io, error); 16629e4e5f87SMilan Broz } 16639e4e5f87SMilan Broz /*----------------------------------------------------------------- 16641da177e4SLinus Torvalds * CRUD END 16651da177e4SLinus Torvalds *---------------------------------------------------------------*/ 16661da177e4SLinus Torvalds 16671da177e4SLinus Torvalds static int dm_merge_bvec(struct request_queue *q, 16681da177e4SLinus Torvalds struct bvec_merge_data *bvm, 1669f6fccb12SMilan Broz struct bio_vec *biovec) 1670f6fccb12SMilan Broz { 1671f6fccb12SMilan Broz struct mapped_device *md = q->queuedata; 167283d5e5b0SMikulas Patocka struct dm_table *map = dm_get_live_table_fast(md); 1673f6fccb12SMilan Broz struct dm_target *ti; 1674f6fccb12SMilan Broz sector_t max_sectors; 1675f6fccb12SMilan Broz int max_size = 0; 1676f6fccb12SMilan Broz 1677f6fccb12SMilan Broz if (unlikely(!map)) 1678f6fccb12SMilan Broz goto out; 1679f6fccb12SMilan Broz 1680f6fccb12SMilan Broz ti = dm_table_find_target(map, bvm->bi_sector); 1681f6fccb12SMilan Broz if (!dm_target_is_valid(ti)) 168283d5e5b0SMikulas Patocka goto out; 1683f6fccb12SMilan Broz 1684f6fccb12SMilan Broz /* 1685f6fccb12SMilan Broz * Find maximum amount of I/O that won't need splitting 1686f6fccb12SMilan Broz */ 168756a67df7SMike Snitzer max_sectors = min(max_io_len(bvm->bi_sector, ti), 1688148e51baSMike Snitzer (sector_t) queue_max_sectors(q)); 1689f6fccb12SMilan Broz max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size; 1690148e51baSMike Snitzer if (unlikely(max_size < 0)) /* this shouldn't _ever_ happen */ 1691f6fccb12SMilan Broz max_size = 0; 1692f6fccb12SMilan Broz 1693f6fccb12SMilan Broz /* 1694f6fccb12SMilan Broz * merge_bvec_fn() returns number of bytes 1695f6fccb12SMilan Broz * it can accept at this offset 1696f6fccb12SMilan Broz * max is precomputed maximal io size 1697f6fccb12SMilan Broz */ 1698f6fccb12SMilan Broz if (max_size && ti->type->merge) 1699f6fccb12SMilan Broz max_size = ti->type->merge(ti, bvm, biovec, max_size); 17008cbeb67aSMikulas Patocka /* 17018cbeb67aSMikulas Patocka * If the target doesn't support merge method and some of the devices 1702148e51baSMike Snitzer * provided their merge_bvec method (we know this by looking for the 1703148e51baSMike Snitzer * max_hw_sectors that dm_set_device_limits may set), then we can't 1704148e51baSMike Snitzer * allow bios with multiple vector entries. So always set max_size 1705148e51baSMike Snitzer * to 0, and the code below allows just one page. 17068cbeb67aSMikulas Patocka */ 17078cbeb67aSMikulas Patocka else if (queue_max_hw_sectors(q) <= PAGE_SIZE >> 9) 17088cbeb67aSMikulas Patocka max_size = 0; 1709f6fccb12SMilan Broz 17105037108aSMikulas Patocka out: 171183d5e5b0SMikulas Patocka dm_put_live_table_fast(md); 1712f6fccb12SMilan Broz /* 1713f6fccb12SMilan Broz * Always allow an entire first page 1714f6fccb12SMilan Broz */ 1715f6fccb12SMilan Broz if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT)) 1716f6fccb12SMilan Broz max_size = biovec->bv_len; 1717f6fccb12SMilan Broz 1718f6fccb12SMilan Broz return max_size; 1719f6fccb12SMilan Broz } 1720f6fccb12SMilan Broz 17211da177e4SLinus Torvalds /* 17221da177e4SLinus Torvalds * The request function that just remaps the bio built up by 17231da177e4SLinus Torvalds * dm_merge_bvec. 17241da177e4SLinus Torvalds */ 1725ff36ab34SMike Snitzer static void dm_make_request(struct request_queue *q, struct bio *bio) 17261da177e4SLinus Torvalds { 172712f03a49SKevin Corry int rw = bio_data_dir(bio); 17281da177e4SLinus Torvalds struct mapped_device *md = q->queuedata; 172983d5e5b0SMikulas Patocka int srcu_idx; 173083d5e5b0SMikulas Patocka struct dm_table *map; 17311da177e4SLinus Torvalds 173283d5e5b0SMikulas Patocka map = dm_get_live_table(md, &srcu_idx); 17331da177e4SLinus Torvalds 173418c0b223SGu Zheng generic_start_io_acct(rw, bio_sectors(bio), &dm_disk(md)->part0); 173512f03a49SKevin Corry 17366a8736d1STejun Heo /* if we're suspended, we have to queue this io for later */ 17376a8736d1STejun Heo if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) { 173883d5e5b0SMikulas Patocka dm_put_live_table(md, srcu_idx); 17391da177e4SLinus Torvalds 17406a8736d1STejun Heo if (bio_rw(bio) != READA) 174192c63902SMikulas Patocka queue_io(md, bio); 17426a8736d1STejun Heo else 17436a8736d1STejun Heo bio_io_error(bio); 17445a7bbad2SChristoph Hellwig return; 17451da177e4SLinus Torvalds } 17461da177e4SLinus Torvalds 174783d5e5b0SMikulas Patocka __split_and_process_bio(md, map, bio); 174883d5e5b0SMikulas Patocka dm_put_live_table(md, srcu_idx); 17495a7bbad2SChristoph Hellwig return; 1750cec47e3dSKiyoshi Ueda } 1751cec47e3dSKiyoshi Ueda 1752fd2ed4d2SMikulas Patocka int dm_request_based(struct mapped_device *md) 1753cec47e3dSKiyoshi Ueda { 1754cec47e3dSKiyoshi Ueda return blk_queue_stackable(md->queue); 1755cec47e3dSKiyoshi Ueda } 1756cec47e3dSKiyoshi Ueda 1757466d89a6SKeith Busch static void dm_dispatch_clone_request(struct request *clone, struct request *rq) 1758cec47e3dSKiyoshi Ueda { 1759cec47e3dSKiyoshi Ueda int r; 1760cec47e3dSKiyoshi Ueda 1761466d89a6SKeith Busch if (blk_queue_io_stat(clone->q)) 1762466d89a6SKeith Busch clone->cmd_flags |= REQ_IO_STAT; 1763cec47e3dSKiyoshi Ueda 1764466d89a6SKeith Busch clone->start_time = jiffies; 1765466d89a6SKeith Busch r = blk_insert_cloned_request(clone->q, clone); 1766cec47e3dSKiyoshi Ueda if (r) 1767466d89a6SKeith Busch /* must complete clone in terms of original request */ 1768cec47e3dSKiyoshi Ueda dm_complete_request(rq, r); 1769cec47e3dSKiyoshi Ueda } 1770cec47e3dSKiyoshi Ueda 1771*5f1b670dSChristoph Hellwig static void setup_clone(struct request *clone, struct request *rq, 1772*5f1b670dSChristoph Hellwig struct dm_rq_target_io *tio) 1773cec47e3dSKiyoshi Ueda { 1774*5f1b670dSChristoph Hellwig blk_rq_prep_clone(clone, rq); 1775cec47e3dSKiyoshi Ueda clone->end_io = end_clone_request; 1776cec47e3dSKiyoshi Ueda clone->end_io_data = tio; 17771ae49ea2SMike Snitzer tio->clone = clone; 1778cec47e3dSKiyoshi Ueda } 1779cec47e3dSKiyoshi Ueda 17806facdaffSKiyoshi Ueda static struct request *clone_rq(struct request *rq, struct mapped_device *md, 17811ae49ea2SMike Snitzer struct dm_rq_target_io *tio, gfp_t gfp_mask) 17826facdaffSKiyoshi Ueda { 178302233342SMike Snitzer /* 178402233342SMike Snitzer * Do not allocate a clone if tio->clone was already set 178502233342SMike Snitzer * (see: dm_mq_queue_rq). 178602233342SMike Snitzer */ 178702233342SMike Snitzer bool alloc_clone = !tio->clone; 178802233342SMike Snitzer struct request *clone; 17891ae49ea2SMike Snitzer 179002233342SMike Snitzer if (alloc_clone) { 179102233342SMike Snitzer clone = alloc_clone_request(md, gfp_mask); 17921ae49ea2SMike Snitzer if (!clone) 17931ae49ea2SMike Snitzer return NULL; 179402233342SMike Snitzer } else 179502233342SMike Snitzer clone = tio->clone; 17961ae49ea2SMike Snitzer 17971ae49ea2SMike Snitzer blk_rq_init(NULL, clone); 1798*5f1b670dSChristoph Hellwig setup_clone(clone, rq, tio); 17991ae49ea2SMike Snitzer 18001ae49ea2SMike Snitzer return clone; 18011ae49ea2SMike Snitzer } 18021ae49ea2SMike Snitzer 18032eb6e1e3SKeith Busch static void map_tio_request(struct kthread_work *work); 18042eb6e1e3SKeith Busch 1805bfebd1cdSMike Snitzer static void init_tio(struct dm_rq_target_io *tio, struct request *rq, 1806bfebd1cdSMike Snitzer struct mapped_device *md) 1807bfebd1cdSMike Snitzer { 1808bfebd1cdSMike Snitzer tio->md = md; 1809bfebd1cdSMike Snitzer tio->ti = NULL; 1810bfebd1cdSMike Snitzer tio->clone = NULL; 1811bfebd1cdSMike Snitzer tio->orig = rq; 1812bfebd1cdSMike Snitzer tio->error = 0; 1813bfebd1cdSMike Snitzer memset(&tio->info, 0, sizeof(tio->info)); 181402233342SMike Snitzer if (md->kworker_task) 1815bfebd1cdSMike Snitzer init_kthread_work(&tio->work, map_tio_request); 1816bfebd1cdSMike Snitzer } 1817bfebd1cdSMike Snitzer 1818466d89a6SKeith Busch static struct dm_rq_target_io *prep_tio(struct request *rq, 1819466d89a6SKeith Busch struct mapped_device *md, gfp_t gfp_mask) 18206facdaffSKiyoshi Ueda { 18216facdaffSKiyoshi Ueda struct dm_rq_target_io *tio; 1822e5863d9aSMike Snitzer int srcu_idx; 1823e5863d9aSMike Snitzer struct dm_table *table; 18246facdaffSKiyoshi Ueda 18256facdaffSKiyoshi Ueda tio = alloc_rq_tio(md, gfp_mask); 18266facdaffSKiyoshi Ueda if (!tio) 18276facdaffSKiyoshi Ueda return NULL; 18286facdaffSKiyoshi Ueda 1829bfebd1cdSMike Snitzer init_tio(tio, rq, md); 18306facdaffSKiyoshi Ueda 1831e5863d9aSMike Snitzer table = dm_get_live_table(md, &srcu_idx); 1832e5863d9aSMike Snitzer if (!dm_table_mq_request_based(table)) { 1833466d89a6SKeith Busch if (!clone_rq(rq, md, tio, gfp_mask)) { 1834e5863d9aSMike Snitzer dm_put_live_table(md, srcu_idx); 18356facdaffSKiyoshi Ueda free_rq_tio(tio); 18366facdaffSKiyoshi Ueda return NULL; 18376facdaffSKiyoshi Ueda } 1838e5863d9aSMike Snitzer } 1839e5863d9aSMike Snitzer dm_put_live_table(md, srcu_idx); 18406facdaffSKiyoshi Ueda 1841466d89a6SKeith Busch return tio; 18426facdaffSKiyoshi Ueda } 18436facdaffSKiyoshi Ueda 1844cec47e3dSKiyoshi Ueda /* 1845cec47e3dSKiyoshi Ueda * Called with the queue lock held. 1846cec47e3dSKiyoshi Ueda */ 1847cec47e3dSKiyoshi Ueda static int dm_prep_fn(struct request_queue *q, struct request *rq) 1848cec47e3dSKiyoshi Ueda { 1849cec47e3dSKiyoshi Ueda struct mapped_device *md = q->queuedata; 1850466d89a6SKeith Busch struct dm_rq_target_io *tio; 1851cec47e3dSKiyoshi Ueda 1852cec47e3dSKiyoshi Ueda if (unlikely(rq->special)) { 1853cec47e3dSKiyoshi Ueda DMWARN("Already has something in rq->special."); 1854cec47e3dSKiyoshi Ueda return BLKPREP_KILL; 1855cec47e3dSKiyoshi Ueda } 1856cec47e3dSKiyoshi Ueda 1857466d89a6SKeith Busch tio = prep_tio(rq, md, GFP_ATOMIC); 1858466d89a6SKeith Busch if (!tio) 1859cec47e3dSKiyoshi Ueda return BLKPREP_DEFER; 1860cec47e3dSKiyoshi Ueda 1861466d89a6SKeith Busch rq->special = tio; 1862cec47e3dSKiyoshi Ueda rq->cmd_flags |= REQ_DONTPREP; 1863cec47e3dSKiyoshi Ueda 1864cec47e3dSKiyoshi Ueda return BLKPREP_OK; 1865cec47e3dSKiyoshi Ueda } 1866cec47e3dSKiyoshi Ueda 18679eef87daSKiyoshi Ueda /* 18689eef87daSKiyoshi Ueda * Returns: 1869e5863d9aSMike Snitzer * 0 : the request has been processed 1870e5863d9aSMike Snitzer * DM_MAPIO_REQUEUE : the original request needs to be requeued 1871e5863d9aSMike Snitzer * < 0 : the request was completed due to failure 18729eef87daSKiyoshi Ueda */ 1873bfebd1cdSMike Snitzer static int map_request(struct dm_rq_target_io *tio, struct request *rq, 1874cec47e3dSKiyoshi Ueda struct mapped_device *md) 1875cec47e3dSKiyoshi Ueda { 1876e5863d9aSMike Snitzer int r; 1877bfebd1cdSMike Snitzer struct dm_target *ti = tio->ti; 1878e5863d9aSMike Snitzer struct request *clone = NULL; 1879cec47e3dSKiyoshi Ueda 1880e5863d9aSMike Snitzer if (tio->clone) { 1881e5863d9aSMike Snitzer clone = tio->clone; 1882cec47e3dSKiyoshi Ueda r = ti->type->map_rq(ti, clone, &tio->info); 1883e5863d9aSMike Snitzer } else { 1884e5863d9aSMike Snitzer r = ti->type->clone_and_map_rq(ti, rq, &tio->info, &clone); 1885e5863d9aSMike Snitzer if (r < 0) { 1886e5863d9aSMike Snitzer /* The target wants to complete the I/O */ 1887e5863d9aSMike Snitzer dm_kill_unmapped_request(rq, r); 1888e5863d9aSMike Snitzer return r; 1889e5863d9aSMike Snitzer } 1890e5863d9aSMike Snitzer if (IS_ERR(clone)) 1891e5863d9aSMike Snitzer return DM_MAPIO_REQUEUE; 1892*5f1b670dSChristoph Hellwig setup_clone(clone, rq, tio); 1893e5863d9aSMike Snitzer } 1894e5863d9aSMike Snitzer 1895cec47e3dSKiyoshi Ueda switch (r) { 1896cec47e3dSKiyoshi Ueda case DM_MAPIO_SUBMITTED: 1897cec47e3dSKiyoshi Ueda /* The target has taken the I/O to submit by itself later */ 1898cec47e3dSKiyoshi Ueda break; 1899cec47e3dSKiyoshi Ueda case DM_MAPIO_REMAPPED: 1900cec47e3dSKiyoshi Ueda /* The target has remapped the I/O so dispatch it */ 19016db4ccd6SJun'ichi Nomura trace_block_rq_remap(clone->q, clone, disk_devt(dm_disk(md)), 1902466d89a6SKeith Busch blk_rq_pos(rq)); 1903466d89a6SKeith Busch dm_dispatch_clone_request(clone, rq); 1904cec47e3dSKiyoshi Ueda break; 1905cec47e3dSKiyoshi Ueda case DM_MAPIO_REQUEUE: 1906cec47e3dSKiyoshi Ueda /* The target wants to requeue the I/O */ 1907cec47e3dSKiyoshi Ueda dm_requeue_unmapped_request(clone); 1908cec47e3dSKiyoshi Ueda break; 1909cec47e3dSKiyoshi Ueda default: 1910cec47e3dSKiyoshi Ueda if (r > 0) { 1911cec47e3dSKiyoshi Ueda DMWARN("unimplemented target map return value: %d", r); 1912cec47e3dSKiyoshi Ueda BUG(); 1913cec47e3dSKiyoshi Ueda } 1914cec47e3dSKiyoshi Ueda 1915cec47e3dSKiyoshi Ueda /* The target wants to complete the I/O */ 1916466d89a6SKeith Busch dm_kill_unmapped_request(rq, r); 1917e5863d9aSMike Snitzer return r; 1918cec47e3dSKiyoshi Ueda } 19199eef87daSKiyoshi Ueda 1920e5863d9aSMike Snitzer return 0; 1921cec47e3dSKiyoshi Ueda } 1922cec47e3dSKiyoshi Ueda 19232eb6e1e3SKeith Busch static void map_tio_request(struct kthread_work *work) 1924ba1cbad9SMike Snitzer { 19252eb6e1e3SKeith Busch struct dm_rq_target_io *tio = container_of(work, struct dm_rq_target_io, work); 1926e5863d9aSMike Snitzer struct request *rq = tio->orig; 1927e5863d9aSMike Snitzer struct mapped_device *md = tio->md; 1928ba1cbad9SMike Snitzer 1929bfebd1cdSMike Snitzer if (map_request(tio, rq, md) == DM_MAPIO_REQUEUE) 1930e5863d9aSMike Snitzer dm_requeue_unmapped_original_request(md, rq); 19312eb6e1e3SKeith Busch } 19322eb6e1e3SKeith Busch 1933466d89a6SKeith Busch static void dm_start_request(struct mapped_device *md, struct request *orig) 1934ba1cbad9SMike Snitzer { 1935bfebd1cdSMike Snitzer if (!orig->q->mq_ops) 1936ba1cbad9SMike Snitzer blk_start_request(orig); 1937bfebd1cdSMike Snitzer else 1938bfebd1cdSMike Snitzer blk_mq_start_request(orig); 1939466d89a6SKeith Busch atomic_inc(&md->pending[rq_data_dir(orig)]); 1940ba1cbad9SMike Snitzer 19410ce65797SMike Snitzer if (md->seq_rq_merge_deadline_usecs) { 1942de3ec86dSMike Snitzer md->last_rq_pos = rq_end_sector(orig); 1943de3ec86dSMike Snitzer md->last_rq_rw = rq_data_dir(orig); 19440ce65797SMike Snitzer md->last_rq_start_time = ktime_get(); 19450ce65797SMike Snitzer } 1946de3ec86dSMike Snitzer 1947ba1cbad9SMike Snitzer /* 1948ba1cbad9SMike Snitzer * Hold the md reference here for the in-flight I/O. 1949ba1cbad9SMike Snitzer * We can't rely on the reference count by device opener, 1950ba1cbad9SMike Snitzer * because the device may be closed during the request completion 1951ba1cbad9SMike Snitzer * when all bios are completed. 1952ba1cbad9SMike Snitzer * See the comment in rq_completed() too. 1953ba1cbad9SMike Snitzer */ 1954ba1cbad9SMike Snitzer dm_get(md); 1955ba1cbad9SMike Snitzer } 1956ba1cbad9SMike Snitzer 19570ce65797SMike Snitzer #define MAX_SEQ_RQ_MERGE_DEADLINE_USECS 100000 19580ce65797SMike Snitzer 19590ce65797SMike Snitzer ssize_t dm_attr_rq_based_seq_io_merge_deadline_show(struct mapped_device *md, char *buf) 19600ce65797SMike Snitzer { 19610ce65797SMike Snitzer return sprintf(buf, "%u\n", md->seq_rq_merge_deadline_usecs); 19620ce65797SMike Snitzer } 19630ce65797SMike Snitzer 19640ce65797SMike Snitzer ssize_t dm_attr_rq_based_seq_io_merge_deadline_store(struct mapped_device *md, 19650ce65797SMike Snitzer const char *buf, size_t count) 19660ce65797SMike Snitzer { 19670ce65797SMike Snitzer unsigned deadline; 19680ce65797SMike Snitzer 196917e149b8SMike Snitzer if (!dm_request_based(md) || md->use_blk_mq) 19700ce65797SMike Snitzer return count; 19710ce65797SMike Snitzer 19720ce65797SMike Snitzer if (kstrtouint(buf, 10, &deadline)) 19730ce65797SMike Snitzer return -EINVAL; 19740ce65797SMike Snitzer 19750ce65797SMike Snitzer if (deadline > MAX_SEQ_RQ_MERGE_DEADLINE_USECS) 19760ce65797SMike Snitzer deadline = MAX_SEQ_RQ_MERGE_DEADLINE_USECS; 19770ce65797SMike Snitzer 19780ce65797SMike Snitzer md->seq_rq_merge_deadline_usecs = deadline; 19790ce65797SMike Snitzer 19800ce65797SMike Snitzer return count; 19810ce65797SMike Snitzer } 19820ce65797SMike Snitzer 19830ce65797SMike Snitzer static bool dm_request_peeked_before_merge_deadline(struct mapped_device *md) 19840ce65797SMike Snitzer { 19850ce65797SMike Snitzer ktime_t kt_deadline; 19860ce65797SMike Snitzer 19870ce65797SMike Snitzer if (!md->seq_rq_merge_deadline_usecs) 19880ce65797SMike Snitzer return false; 19890ce65797SMike Snitzer 19900ce65797SMike Snitzer kt_deadline = ns_to_ktime((u64)md->seq_rq_merge_deadline_usecs * NSEC_PER_USEC); 19910ce65797SMike Snitzer kt_deadline = ktime_add_safe(md->last_rq_start_time, kt_deadline); 19920ce65797SMike Snitzer 19930ce65797SMike Snitzer return !ktime_after(ktime_get(), kt_deadline); 19940ce65797SMike Snitzer } 19950ce65797SMike Snitzer 1996cec47e3dSKiyoshi Ueda /* 1997cec47e3dSKiyoshi Ueda * q->request_fn for request-based dm. 1998cec47e3dSKiyoshi Ueda * Called with the queue lock held. 1999cec47e3dSKiyoshi Ueda */ 2000cec47e3dSKiyoshi Ueda static void dm_request_fn(struct request_queue *q) 2001cec47e3dSKiyoshi Ueda { 2002cec47e3dSKiyoshi Ueda struct mapped_device *md = q->queuedata; 200383d5e5b0SMikulas Patocka int srcu_idx; 200483d5e5b0SMikulas Patocka struct dm_table *map = dm_get_live_table(md, &srcu_idx); 2005cec47e3dSKiyoshi Ueda struct dm_target *ti; 2006466d89a6SKeith Busch struct request *rq; 20072eb6e1e3SKeith Busch struct dm_rq_target_io *tio; 200829e4013dSTejun Heo sector_t pos; 2009cec47e3dSKiyoshi Ueda 2010cec47e3dSKiyoshi Ueda /* 2011b4324feeSKiyoshi Ueda * For suspend, check blk_queue_stopped() and increment 2012b4324feeSKiyoshi Ueda * ->pending within a single queue_lock not to increment the 2013b4324feeSKiyoshi Ueda * number of in-flight I/Os after the queue is stopped in 2014b4324feeSKiyoshi Ueda * dm_suspend(). 2015cec47e3dSKiyoshi Ueda */ 20167eaceaccSJens Axboe while (!blk_queue_stopped(q)) { 2017cec47e3dSKiyoshi Ueda rq = blk_peek_request(q); 2018cec47e3dSKiyoshi Ueda if (!rq) 20199d1deb83SMike Snitzer goto out; 2020cec47e3dSKiyoshi Ueda 202129e4013dSTejun Heo /* always use block 0 to find the target for flushes for now */ 202229e4013dSTejun Heo pos = 0; 202329e4013dSTejun Heo if (!(rq->cmd_flags & REQ_FLUSH)) 202429e4013dSTejun Heo pos = blk_rq_pos(rq); 2025d0bcb878SKiyoshi Ueda 202629e4013dSTejun Heo ti = dm_table_find_target(map, pos); 2027ba1cbad9SMike Snitzer if (!dm_target_is_valid(ti)) { 2028ba1cbad9SMike Snitzer /* 2029466d89a6SKeith Busch * Must perform setup, that rq_completed() requires, 2030ba1cbad9SMike Snitzer * before calling dm_kill_unmapped_request 2031ba1cbad9SMike Snitzer */ 2032ba1cbad9SMike Snitzer DMERR_LIMIT("request attempted access beyond the end of device"); 2033466d89a6SKeith Busch dm_start_request(md, rq); 2034466d89a6SKeith Busch dm_kill_unmapped_request(rq, -EIO); 2035ba1cbad9SMike Snitzer continue; 2036ba1cbad9SMike Snitzer } 203729e4013dSTejun Heo 20380ce65797SMike Snitzer if (dm_request_peeked_before_merge_deadline(md) && 20390ce65797SMike Snitzer md_in_flight(md) && rq->bio && rq->bio->bi_vcnt == 1 && 2040de3ec86dSMike Snitzer md->last_rq_pos == pos && md->last_rq_rw == rq_data_dir(rq)) 2041de3ec86dSMike Snitzer goto delay_and_out; 2042de3ec86dSMike Snitzer 2043cec47e3dSKiyoshi Ueda if (ti->type->busy && ti->type->busy(ti)) 20447eaceaccSJens Axboe goto delay_and_out; 2045cec47e3dSKiyoshi Ueda 2046466d89a6SKeith Busch dm_start_request(md, rq); 2047b4324feeSKiyoshi Ueda 2048bfebd1cdSMike Snitzer tio = tio_from_request(rq); 20492eb6e1e3SKeith Busch /* Establish tio->ti before queuing work (map_tio_request) */ 20502eb6e1e3SKeith Busch tio->ti = ti; 20512eb6e1e3SKeith Busch queue_kthread_work(&md->kworker, &tio->work); 2052052189a2SKiyoshi Ueda BUG_ON(!irqs_disabled()); 2053cec47e3dSKiyoshi Ueda } 2054cec47e3dSKiyoshi Ueda 2055cec47e3dSKiyoshi Ueda goto out; 2056cec47e3dSKiyoshi Ueda 20577eaceaccSJens Axboe delay_and_out: 2058d548b34bSMike Snitzer blk_delay_queue(q, HZ / 100); 2059cec47e3dSKiyoshi Ueda out: 206083d5e5b0SMikulas Patocka dm_put_live_table(md, srcu_idx); 2061cec47e3dSKiyoshi Ueda } 2062cec47e3dSKiyoshi Ueda 20631da177e4SLinus Torvalds static int dm_any_congested(void *congested_data, int bdi_bits) 20641da177e4SLinus Torvalds { 20658a57dfc6SChandra Seetharaman int r = bdi_bits; 20668a57dfc6SChandra Seetharaman struct mapped_device *md = congested_data; 20678a57dfc6SChandra Seetharaman struct dm_table *map; 20681da177e4SLinus Torvalds 20691eb787ecSAlasdair G Kergon if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) { 207083d5e5b0SMikulas Patocka map = dm_get_live_table_fast(md); 20718a57dfc6SChandra Seetharaman if (map) { 2072cec47e3dSKiyoshi Ueda /* 2073cec47e3dSKiyoshi Ueda * Request-based dm cares about only own queue for 2074cec47e3dSKiyoshi Ueda * the query about congestion status of request_queue 2075cec47e3dSKiyoshi Ueda */ 2076cec47e3dSKiyoshi Ueda if (dm_request_based(md)) 2077cec47e3dSKiyoshi Ueda r = md->queue->backing_dev_info.state & 2078cec47e3dSKiyoshi Ueda bdi_bits; 2079cec47e3dSKiyoshi Ueda else 20801da177e4SLinus Torvalds r = dm_table_any_congested(map, bdi_bits); 20818a57dfc6SChandra Seetharaman } 208283d5e5b0SMikulas Patocka dm_put_live_table_fast(md); 20838a57dfc6SChandra Seetharaman } 20848a57dfc6SChandra Seetharaman 20851da177e4SLinus Torvalds return r; 20861da177e4SLinus Torvalds } 20871da177e4SLinus Torvalds 20881da177e4SLinus Torvalds /*----------------------------------------------------------------- 20891da177e4SLinus Torvalds * An IDR is used to keep track of allocated minor numbers. 20901da177e4SLinus Torvalds *---------------------------------------------------------------*/ 20912b06cfffSAlasdair G Kergon static void free_minor(int minor) 20921da177e4SLinus Torvalds { 2093f32c10b0SJeff Mahoney spin_lock(&_minor_lock); 20941da177e4SLinus Torvalds idr_remove(&_minor_idr, minor); 2095f32c10b0SJeff Mahoney spin_unlock(&_minor_lock); 20961da177e4SLinus Torvalds } 20971da177e4SLinus Torvalds 20981da177e4SLinus Torvalds /* 20991da177e4SLinus Torvalds * See if the device with a specific minor # is free. 21001da177e4SLinus Torvalds */ 2101cf13ab8eSFrederik Deweerdt static int specific_minor(int minor) 21021da177e4SLinus Torvalds { 2103c9d76be6STejun Heo int r; 21041da177e4SLinus Torvalds 21051da177e4SLinus Torvalds if (minor >= (1 << MINORBITS)) 21061da177e4SLinus Torvalds return -EINVAL; 21071da177e4SLinus Torvalds 2108c9d76be6STejun Heo idr_preload(GFP_KERNEL); 2109f32c10b0SJeff Mahoney spin_lock(&_minor_lock); 21101da177e4SLinus Torvalds 2111c9d76be6STejun Heo r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT); 21121da177e4SLinus Torvalds 2113f32c10b0SJeff Mahoney spin_unlock(&_minor_lock); 2114c9d76be6STejun Heo idr_preload_end(); 2115c9d76be6STejun Heo if (r < 0) 2116c9d76be6STejun Heo return r == -ENOSPC ? -EBUSY : r; 2117c9d76be6STejun Heo return 0; 21181da177e4SLinus Torvalds } 21191da177e4SLinus Torvalds 2120cf13ab8eSFrederik Deweerdt static int next_free_minor(int *minor) 21211da177e4SLinus Torvalds { 2122c9d76be6STejun Heo int r; 21231da177e4SLinus Torvalds 2124c9d76be6STejun Heo idr_preload(GFP_KERNEL); 2125f32c10b0SJeff Mahoney spin_lock(&_minor_lock); 21261da177e4SLinus Torvalds 2127c9d76be6STejun Heo r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT); 21281da177e4SLinus Torvalds 2129f32c10b0SJeff Mahoney spin_unlock(&_minor_lock); 2130c9d76be6STejun Heo idr_preload_end(); 2131c9d76be6STejun Heo if (r < 0) 21321da177e4SLinus Torvalds return r; 2133c9d76be6STejun Heo *minor = r; 2134c9d76be6STejun Heo return 0; 21351da177e4SLinus Torvalds } 21361da177e4SLinus Torvalds 213783d5cde4SAlexey Dobriyan static const struct block_device_operations dm_blk_dops; 21381da177e4SLinus Torvalds 213953d5914fSMikulas Patocka static void dm_wq_work(struct work_struct *work); 214053d5914fSMikulas Patocka 21414a0b4ddfSMike Snitzer static void dm_init_md_queue(struct mapped_device *md) 21424a0b4ddfSMike Snitzer { 21434a0b4ddfSMike Snitzer /* 21444a0b4ddfSMike Snitzer * Request-based dm devices cannot be stacked on top of bio-based dm 2145bfebd1cdSMike Snitzer * devices. The type of this dm device may not have been decided yet. 21464a0b4ddfSMike Snitzer * The type is decided at the first table loading time. 21474a0b4ddfSMike Snitzer * To prevent problematic device stacking, clear the queue flag 21484a0b4ddfSMike Snitzer * for request stacking support until then. 21494a0b4ddfSMike Snitzer * 21504a0b4ddfSMike Snitzer * This queue is new, so no concurrency on the queue_flags. 21514a0b4ddfSMike Snitzer */ 21524a0b4ddfSMike Snitzer queue_flag_clear_unlocked(QUEUE_FLAG_STACKABLE, md->queue); 2153bfebd1cdSMike Snitzer } 21544a0b4ddfSMike Snitzer 2155bfebd1cdSMike Snitzer static void dm_init_old_md_queue(struct mapped_device *md) 2156bfebd1cdSMike Snitzer { 215717e149b8SMike Snitzer md->use_blk_mq = false; 2158bfebd1cdSMike Snitzer dm_init_md_queue(md); 2159bfebd1cdSMike Snitzer 2160bfebd1cdSMike Snitzer /* 2161bfebd1cdSMike Snitzer * Initialize aspects of queue that aren't relevant for blk-mq 2162bfebd1cdSMike Snitzer */ 21634a0b4ddfSMike Snitzer md->queue->queuedata = md; 21644a0b4ddfSMike Snitzer md->queue->backing_dev_info.congested_fn = dm_any_congested; 21654a0b4ddfSMike Snitzer md->queue->backing_dev_info.congested_data = md; 2166ff36ab34SMike Snitzer 21674a0b4ddfSMike Snitzer blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY); 21684a0b4ddfSMike Snitzer } 21694a0b4ddfSMike Snitzer 21701da177e4SLinus Torvalds /* 21711da177e4SLinus Torvalds * Allocate and initialise a blank device with a given minor. 21721da177e4SLinus Torvalds */ 21732b06cfffSAlasdair G Kergon static struct mapped_device *alloc_dev(int minor) 21741da177e4SLinus Torvalds { 21751da177e4SLinus Torvalds int r; 2176cf13ab8eSFrederik Deweerdt struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL); 2177ba61fdd1SJeff Mahoney void *old_md; 21781da177e4SLinus Torvalds 21791da177e4SLinus Torvalds if (!md) { 21801da177e4SLinus Torvalds DMWARN("unable to allocate device, out of memory."); 21811da177e4SLinus Torvalds return NULL; 21821da177e4SLinus Torvalds } 21831da177e4SLinus Torvalds 218410da4f79SJeff Mahoney if (!try_module_get(THIS_MODULE)) 21856ed7ade8SMilan Broz goto bad_module_get; 218610da4f79SJeff Mahoney 21871da177e4SLinus Torvalds /* get a minor number for the dev */ 21882b06cfffSAlasdair G Kergon if (minor == DM_ANY_MINOR) 2189cf13ab8eSFrederik Deweerdt r = next_free_minor(&minor); 21902b06cfffSAlasdair G Kergon else 2191cf13ab8eSFrederik Deweerdt r = specific_minor(minor); 21921da177e4SLinus Torvalds if (r < 0) 21936ed7ade8SMilan Broz goto bad_minor; 21941da177e4SLinus Torvalds 219583d5e5b0SMikulas Patocka r = init_srcu_struct(&md->io_barrier); 219683d5e5b0SMikulas Patocka if (r < 0) 219783d5e5b0SMikulas Patocka goto bad_io_barrier; 219883d5e5b0SMikulas Patocka 219917e149b8SMike Snitzer md->use_blk_mq = use_blk_mq; 2200a5664dadSMike Snitzer md->type = DM_TYPE_NONE; 2201e61290a4SDaniel Walker mutex_init(&md->suspend_lock); 2202a5664dadSMike Snitzer mutex_init(&md->type_lock); 220386f1152bSBenjamin Marzinski mutex_init(&md->table_devices_lock); 2204022c2611SMikulas Patocka spin_lock_init(&md->deferred_lock); 22051da177e4SLinus Torvalds atomic_set(&md->holders, 1); 22065c6bd75dSAlasdair G Kergon atomic_set(&md->open_count, 0); 22071da177e4SLinus Torvalds atomic_set(&md->event_nr, 0); 22087a8c3d3bSMike Anderson atomic_set(&md->uevent_seq, 0); 22097a8c3d3bSMike Anderson INIT_LIST_HEAD(&md->uevent_list); 221086f1152bSBenjamin Marzinski INIT_LIST_HEAD(&md->table_devices); 22117a8c3d3bSMike Anderson spin_lock_init(&md->uevent_lock); 22121da177e4SLinus Torvalds 22134a0b4ddfSMike Snitzer md->queue = blk_alloc_queue(GFP_KERNEL); 22141da177e4SLinus Torvalds if (!md->queue) 22156ed7ade8SMilan Broz goto bad_queue; 22161da177e4SLinus Torvalds 22174a0b4ddfSMike Snitzer dm_init_md_queue(md); 22189faf400fSStefan Bader 22191da177e4SLinus Torvalds md->disk = alloc_disk(1); 22201da177e4SLinus Torvalds if (!md->disk) 22216ed7ade8SMilan Broz goto bad_disk; 22221da177e4SLinus Torvalds 2223316d315bSNikanth Karthikesan atomic_set(&md->pending[0], 0); 2224316d315bSNikanth Karthikesan atomic_set(&md->pending[1], 0); 2225f0b04115SJeff Mahoney init_waitqueue_head(&md->wait); 222653d5914fSMikulas Patocka INIT_WORK(&md->work, dm_wq_work); 2227f0b04115SJeff Mahoney init_waitqueue_head(&md->eventq); 22282995fa78SMikulas Patocka init_completion(&md->kobj_holder.completion); 22292eb6e1e3SKeith Busch md->kworker_task = NULL; 2230f0b04115SJeff Mahoney 22311da177e4SLinus Torvalds md->disk->major = _major; 22321da177e4SLinus Torvalds md->disk->first_minor = minor; 22331da177e4SLinus Torvalds md->disk->fops = &dm_blk_dops; 22341da177e4SLinus Torvalds md->disk->queue = md->queue; 22351da177e4SLinus Torvalds md->disk->private_data = md; 22361da177e4SLinus Torvalds sprintf(md->disk->disk_name, "dm-%d", minor); 22371da177e4SLinus Torvalds add_disk(md->disk); 22387e51f257SMike Anderson format_dev_t(md->name, MKDEV(_major, minor)); 22391da177e4SLinus Torvalds 2240670368a8STejun Heo md->wq = alloc_workqueue("kdmflush", WQ_MEM_RECLAIM, 0); 2241304f3f6aSMilan Broz if (!md->wq) 2242304f3f6aSMilan Broz goto bad_thread; 2243304f3f6aSMilan Broz 224432a926daSMikulas Patocka md->bdev = bdget_disk(md->disk, 0); 224532a926daSMikulas Patocka if (!md->bdev) 224632a926daSMikulas Patocka goto bad_bdev; 224732a926daSMikulas Patocka 22486a8736d1STejun Heo bio_init(&md->flush_bio); 22496a8736d1STejun Heo md->flush_bio.bi_bdev = md->bdev; 22506a8736d1STejun Heo md->flush_bio.bi_rw = WRITE_FLUSH; 22516a8736d1STejun Heo 2252fd2ed4d2SMikulas Patocka dm_stats_init(&md->stats); 2253fd2ed4d2SMikulas Patocka 2254ba61fdd1SJeff Mahoney /* Populate the mapping, nobody knows we exist yet */ 2255f32c10b0SJeff Mahoney spin_lock(&_minor_lock); 2256ba61fdd1SJeff Mahoney old_md = idr_replace(&_minor_idr, md, minor); 2257f32c10b0SJeff Mahoney spin_unlock(&_minor_lock); 2258ba61fdd1SJeff Mahoney 2259ba61fdd1SJeff Mahoney BUG_ON(old_md != MINOR_ALLOCED); 2260ba61fdd1SJeff Mahoney 22611da177e4SLinus Torvalds return md; 22621da177e4SLinus Torvalds 226332a926daSMikulas Patocka bad_bdev: 226432a926daSMikulas Patocka destroy_workqueue(md->wq); 2265304f3f6aSMilan Broz bad_thread: 226603022c54SZdenek Kabelac del_gendisk(md->disk); 2267304f3f6aSMilan Broz put_disk(md->disk); 22686ed7ade8SMilan Broz bad_disk: 22691312f40eSAl Viro blk_cleanup_queue(md->queue); 22706ed7ade8SMilan Broz bad_queue: 227183d5e5b0SMikulas Patocka cleanup_srcu_struct(&md->io_barrier); 227283d5e5b0SMikulas Patocka bad_io_barrier: 22731da177e4SLinus Torvalds free_minor(minor); 22746ed7ade8SMilan Broz bad_minor: 227510da4f79SJeff Mahoney module_put(THIS_MODULE); 22766ed7ade8SMilan Broz bad_module_get: 22771da177e4SLinus Torvalds kfree(md); 22781da177e4SLinus Torvalds return NULL; 22791da177e4SLinus Torvalds } 22801da177e4SLinus Torvalds 2281ae9da83fSJun'ichi Nomura static void unlock_fs(struct mapped_device *md); 2282ae9da83fSJun'ichi Nomura 22831da177e4SLinus Torvalds static void free_dev(struct mapped_device *md) 22841da177e4SLinus Torvalds { 2285f331c029STejun Heo int minor = MINOR(disk_devt(md->disk)); 228663d94e48SJun'ichi Nomura 2287ae9da83fSJun'ichi Nomura unlock_fs(md); 2288304f3f6aSMilan Broz destroy_workqueue(md->wq); 22892eb6e1e3SKeith Busch 22902eb6e1e3SKeith Busch if (md->kworker_task) 22912eb6e1e3SKeith Busch kthread_stop(md->kworker_task); 2292e6ee8c0bSKiyoshi Ueda if (md->io_pool) 22931da177e4SLinus Torvalds mempool_destroy(md->io_pool); 22941ae49ea2SMike Snitzer if (md->rq_pool) 22951ae49ea2SMike Snitzer mempool_destroy(md->rq_pool); 2296e6ee8c0bSKiyoshi Ueda if (md->bs) 22979faf400fSStefan Bader bioset_free(md->bs); 229863a4f065SMike Snitzer 229983d5e5b0SMikulas Patocka cleanup_srcu_struct(&md->io_barrier); 230086f1152bSBenjamin Marzinski free_table_devices(&md->table_devices); 230163a4f065SMike Snitzer dm_stats_cleanup(&md->stats); 2302fba9f90eSJeff Mahoney 2303fba9f90eSJeff Mahoney spin_lock(&_minor_lock); 2304fba9f90eSJeff Mahoney md->disk->private_data = NULL; 2305fba9f90eSJeff Mahoney spin_unlock(&_minor_lock); 230663a4f065SMike Snitzer if (blk_get_integrity(md->disk)) 230763a4f065SMike Snitzer blk_integrity_unregister(md->disk); 230863a4f065SMike Snitzer del_gendisk(md->disk); 23091da177e4SLinus Torvalds put_disk(md->disk); 23101312f40eSAl Viro blk_cleanup_queue(md->queue); 231117e149b8SMike Snitzer if (md->use_blk_mq) 2312bfebd1cdSMike Snitzer blk_mq_free_tag_set(&md->tag_set); 231363a4f065SMike Snitzer bdput(md->bdev); 231463a4f065SMike Snitzer free_minor(minor); 231563a4f065SMike Snitzer 231610da4f79SJeff Mahoney module_put(THIS_MODULE); 23171da177e4SLinus Torvalds kfree(md); 23181da177e4SLinus Torvalds } 23191da177e4SLinus Torvalds 2320e6ee8c0bSKiyoshi Ueda static void __bind_mempools(struct mapped_device *md, struct dm_table *t) 2321e6ee8c0bSKiyoshi Ueda { 2322c0820cf5SMikulas Patocka struct dm_md_mempools *p = dm_table_get_md_mempools(t); 2323e6ee8c0bSKiyoshi Ueda 232417e149b8SMike Snitzer if (md->bs) { 232516245bdcSJun'ichi Nomura /* The md already has necessary mempools. */ 232616245bdcSJun'ichi Nomura if (dm_table_get_type(t) == DM_TYPE_BIO_BASED) { 2327c0820cf5SMikulas Patocka /* 232816245bdcSJun'ichi Nomura * Reload bioset because front_pad may have changed 232916245bdcSJun'ichi Nomura * because a different table was loaded. 2330c0820cf5SMikulas Patocka */ 2331c0820cf5SMikulas Patocka bioset_free(md->bs); 2332c0820cf5SMikulas Patocka md->bs = p->bs; 2333c0820cf5SMikulas Patocka p->bs = NULL; 2334466d89a6SKeith Busch } 233516245bdcSJun'ichi Nomura /* 233616245bdcSJun'ichi Nomura * There's no need to reload with request-based dm 233716245bdcSJun'ichi Nomura * because the size of front_pad doesn't change. 233816245bdcSJun'ichi Nomura * Note for future: If you are to reload bioset, 233916245bdcSJun'ichi Nomura * prep-ed requests in the queue may refer 234016245bdcSJun'ichi Nomura * to bio from the old bioset, so you must walk 234116245bdcSJun'ichi Nomura * through the queue to unprep. 234216245bdcSJun'ichi Nomura */ 2343e6ee8c0bSKiyoshi Ueda goto out; 2344c0820cf5SMikulas Patocka } 2345e6ee8c0bSKiyoshi Ueda 2346e6ee8c0bSKiyoshi Ueda md->io_pool = p->io_pool; 2347e6ee8c0bSKiyoshi Ueda p->io_pool = NULL; 23481ae49ea2SMike Snitzer md->rq_pool = p->rq_pool; 23491ae49ea2SMike Snitzer p->rq_pool = NULL; 2350e6ee8c0bSKiyoshi Ueda md->bs = p->bs; 2351e6ee8c0bSKiyoshi Ueda p->bs = NULL; 2352e6ee8c0bSKiyoshi Ueda 2353e6ee8c0bSKiyoshi Ueda out: 235402233342SMike Snitzer /* mempool bind completed, no longer need any mempools in the table */ 2355e6ee8c0bSKiyoshi Ueda dm_table_free_md_mempools(t); 2356e6ee8c0bSKiyoshi Ueda } 2357e6ee8c0bSKiyoshi Ueda 23581da177e4SLinus Torvalds /* 23591da177e4SLinus Torvalds * Bind a table to the device. 23601da177e4SLinus Torvalds */ 23611da177e4SLinus Torvalds static void event_callback(void *context) 23621da177e4SLinus Torvalds { 23637a8c3d3bSMike Anderson unsigned long flags; 23647a8c3d3bSMike Anderson LIST_HEAD(uevents); 23651da177e4SLinus Torvalds struct mapped_device *md = (struct mapped_device *) context; 23661da177e4SLinus Torvalds 23677a8c3d3bSMike Anderson spin_lock_irqsave(&md->uevent_lock, flags); 23687a8c3d3bSMike Anderson list_splice_init(&md->uevent_list, &uevents); 23697a8c3d3bSMike Anderson spin_unlock_irqrestore(&md->uevent_lock, flags); 23707a8c3d3bSMike Anderson 2371ed9e1982STejun Heo dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj); 23727a8c3d3bSMike Anderson 23731da177e4SLinus Torvalds atomic_inc(&md->event_nr); 23741da177e4SLinus Torvalds wake_up(&md->eventq); 23751da177e4SLinus Torvalds } 23761da177e4SLinus Torvalds 2377c217649bSMike Snitzer /* 2378c217649bSMike Snitzer * Protected by md->suspend_lock obtained by dm_swap_table(). 2379c217649bSMike Snitzer */ 23804e90188bSAlasdair G Kergon static void __set_size(struct mapped_device *md, sector_t size) 23811da177e4SLinus Torvalds { 23824e90188bSAlasdair G Kergon set_capacity(md->disk, size); 23831da177e4SLinus Torvalds 2384db8fef4fSMikulas Patocka i_size_write(md->bdev->bd_inode, (loff_t)size << SECTOR_SHIFT); 23851da177e4SLinus Torvalds } 23861da177e4SLinus Torvalds 2387042d2a9bSAlasdair G Kergon /* 2388d5b9dd04SMikulas Patocka * Return 1 if the queue has a compulsory merge_bvec_fn function. 2389d5b9dd04SMikulas Patocka * 2390d5b9dd04SMikulas Patocka * If this function returns 0, then the device is either a non-dm 2391d5b9dd04SMikulas Patocka * device without a merge_bvec_fn, or it is a dm device that is 2392d5b9dd04SMikulas Patocka * able to split any bios it receives that are too big. 2393d5b9dd04SMikulas Patocka */ 2394d5b9dd04SMikulas Patocka int dm_queue_merge_is_compulsory(struct request_queue *q) 2395d5b9dd04SMikulas Patocka { 2396d5b9dd04SMikulas Patocka struct mapped_device *dev_md; 2397d5b9dd04SMikulas Patocka 2398d5b9dd04SMikulas Patocka if (!q->merge_bvec_fn) 2399d5b9dd04SMikulas Patocka return 0; 2400d5b9dd04SMikulas Patocka 2401ff36ab34SMike Snitzer if (q->make_request_fn == dm_make_request) { 2402d5b9dd04SMikulas Patocka dev_md = q->queuedata; 2403d5b9dd04SMikulas Patocka if (test_bit(DMF_MERGE_IS_OPTIONAL, &dev_md->flags)) 2404d5b9dd04SMikulas Patocka return 0; 2405d5b9dd04SMikulas Patocka } 2406d5b9dd04SMikulas Patocka 2407d5b9dd04SMikulas Patocka return 1; 2408d5b9dd04SMikulas Patocka } 2409d5b9dd04SMikulas Patocka 2410d5b9dd04SMikulas Patocka static int dm_device_merge_is_compulsory(struct dm_target *ti, 2411d5b9dd04SMikulas Patocka struct dm_dev *dev, sector_t start, 2412d5b9dd04SMikulas Patocka sector_t len, void *data) 2413d5b9dd04SMikulas Patocka { 2414d5b9dd04SMikulas Patocka struct block_device *bdev = dev->bdev; 2415d5b9dd04SMikulas Patocka struct request_queue *q = bdev_get_queue(bdev); 2416d5b9dd04SMikulas Patocka 2417d5b9dd04SMikulas Patocka return dm_queue_merge_is_compulsory(q); 2418d5b9dd04SMikulas Patocka } 2419d5b9dd04SMikulas Patocka 2420d5b9dd04SMikulas Patocka /* 2421d5b9dd04SMikulas Patocka * Return 1 if it is acceptable to ignore merge_bvec_fn based 2422d5b9dd04SMikulas Patocka * on the properties of the underlying devices. 2423d5b9dd04SMikulas Patocka */ 2424d5b9dd04SMikulas Patocka static int dm_table_merge_is_optional(struct dm_table *table) 2425d5b9dd04SMikulas Patocka { 2426d5b9dd04SMikulas Patocka unsigned i = 0; 2427d5b9dd04SMikulas Patocka struct dm_target *ti; 2428d5b9dd04SMikulas Patocka 2429d5b9dd04SMikulas Patocka while (i < dm_table_get_num_targets(table)) { 2430d5b9dd04SMikulas Patocka ti = dm_table_get_target(table, i++); 2431d5b9dd04SMikulas Patocka 2432d5b9dd04SMikulas Patocka if (ti->type->iterate_devices && 2433d5b9dd04SMikulas Patocka ti->type->iterate_devices(ti, dm_device_merge_is_compulsory, NULL)) 2434d5b9dd04SMikulas Patocka return 0; 2435d5b9dd04SMikulas Patocka } 2436d5b9dd04SMikulas Patocka 2437d5b9dd04SMikulas Patocka return 1; 2438d5b9dd04SMikulas Patocka } 2439d5b9dd04SMikulas Patocka 2440d5b9dd04SMikulas Patocka /* 2441042d2a9bSAlasdair G Kergon * Returns old map, which caller must destroy. 2442042d2a9bSAlasdair G Kergon */ 2443042d2a9bSAlasdair G Kergon static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t, 2444754c5fc7SMike Snitzer struct queue_limits *limits) 24451da177e4SLinus Torvalds { 2446042d2a9bSAlasdair G Kergon struct dm_table *old_map; 2447165125e1SJens Axboe struct request_queue *q = md->queue; 24481da177e4SLinus Torvalds sector_t size; 2449d5b9dd04SMikulas Patocka int merge_is_optional; 24501da177e4SLinus Torvalds 24511da177e4SLinus Torvalds size = dm_table_get_size(t); 24523ac51e74SDarrick J. Wong 24533ac51e74SDarrick J. Wong /* 24543ac51e74SDarrick J. Wong * Wipe any geometry if the size of the table changed. 24553ac51e74SDarrick J. Wong */ 2456fd2ed4d2SMikulas Patocka if (size != dm_get_size(md)) 24573ac51e74SDarrick J. Wong memset(&md->geometry, 0, sizeof(md->geometry)); 24583ac51e74SDarrick J. Wong 24594e90188bSAlasdair G Kergon __set_size(md, size); 24601da177e4SLinus Torvalds 2461cf222b37SAlasdair G Kergon dm_table_event_callback(t, event_callback, md); 24622ca3310eSAlasdair G Kergon 2463e6ee8c0bSKiyoshi Ueda /* 2464e6ee8c0bSKiyoshi Ueda * The queue hasn't been stopped yet, if the old table type wasn't 2465e6ee8c0bSKiyoshi Ueda * for request-based during suspension. So stop it to prevent 2466e6ee8c0bSKiyoshi Ueda * I/O mapping before resume. 2467e6ee8c0bSKiyoshi Ueda * This must be done before setting the queue restrictions, 2468e6ee8c0bSKiyoshi Ueda * because request-based dm may be run just after the setting. 2469e6ee8c0bSKiyoshi Ueda */ 2470bfebd1cdSMike Snitzer if (dm_table_request_based(t)) 2471e6ee8c0bSKiyoshi Ueda stop_queue(q); 2472e6ee8c0bSKiyoshi Ueda 2473e6ee8c0bSKiyoshi Ueda __bind_mempools(md, t); 2474e6ee8c0bSKiyoshi Ueda 2475d5b9dd04SMikulas Patocka merge_is_optional = dm_table_merge_is_optional(t); 2476d5b9dd04SMikulas Patocka 2477a12f5d48SEric Dumazet old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 247883d5e5b0SMikulas Patocka rcu_assign_pointer(md->map, t); 247936a0456fSAlasdair G Kergon md->immutable_target_type = dm_table_get_immutable_target_type(t); 248036a0456fSAlasdair G Kergon 2481754c5fc7SMike Snitzer dm_table_set_restrictions(t, q, limits); 2482d5b9dd04SMikulas Patocka if (merge_is_optional) 2483d5b9dd04SMikulas Patocka set_bit(DMF_MERGE_IS_OPTIONAL, &md->flags); 2484d5b9dd04SMikulas Patocka else 2485d5b9dd04SMikulas Patocka clear_bit(DMF_MERGE_IS_OPTIONAL, &md->flags); 248641abc4e1SHannes Reinecke if (old_map) 248783d5e5b0SMikulas Patocka dm_sync_table(md); 24882ca3310eSAlasdair G Kergon 2489042d2a9bSAlasdair G Kergon return old_map; 24901da177e4SLinus Torvalds } 24911da177e4SLinus Torvalds 2492a7940155SAlasdair G Kergon /* 2493a7940155SAlasdair G Kergon * Returns unbound table for the caller to free. 2494a7940155SAlasdair G Kergon */ 2495a7940155SAlasdair G Kergon static struct dm_table *__unbind(struct mapped_device *md) 24961da177e4SLinus Torvalds { 2497a12f5d48SEric Dumazet struct dm_table *map = rcu_dereference_protected(md->map, 1); 24981da177e4SLinus Torvalds 24991da177e4SLinus Torvalds if (!map) 2500a7940155SAlasdair G Kergon return NULL; 25011da177e4SLinus Torvalds 25021da177e4SLinus Torvalds dm_table_event_callback(map, NULL, NULL); 25039cdb8520SMonam Agarwal RCU_INIT_POINTER(md->map, NULL); 250483d5e5b0SMikulas Patocka dm_sync_table(md); 2505a7940155SAlasdair G Kergon 2506a7940155SAlasdair G Kergon return map; 25071da177e4SLinus Torvalds } 25081da177e4SLinus Torvalds 25091da177e4SLinus Torvalds /* 25101da177e4SLinus Torvalds * Constructor for a new device. 25111da177e4SLinus Torvalds */ 25122b06cfffSAlasdair G Kergon int dm_create(int minor, struct mapped_device **result) 25131da177e4SLinus Torvalds { 25141da177e4SLinus Torvalds struct mapped_device *md; 25151da177e4SLinus Torvalds 25162b06cfffSAlasdair G Kergon md = alloc_dev(minor); 25171da177e4SLinus Torvalds if (!md) 25181da177e4SLinus Torvalds return -ENXIO; 25191da177e4SLinus Torvalds 2520784aae73SMilan Broz dm_sysfs_init(md); 2521784aae73SMilan Broz 25221da177e4SLinus Torvalds *result = md; 25231da177e4SLinus Torvalds return 0; 25241da177e4SLinus Torvalds } 25251da177e4SLinus Torvalds 2526a5664dadSMike Snitzer /* 2527a5664dadSMike Snitzer * Functions to manage md->type. 2528a5664dadSMike Snitzer * All are required to hold md->type_lock. 2529a5664dadSMike Snitzer */ 2530a5664dadSMike Snitzer void dm_lock_md_type(struct mapped_device *md) 2531a5664dadSMike Snitzer { 2532a5664dadSMike Snitzer mutex_lock(&md->type_lock); 2533a5664dadSMike Snitzer } 2534a5664dadSMike Snitzer 2535a5664dadSMike Snitzer void dm_unlock_md_type(struct mapped_device *md) 2536a5664dadSMike Snitzer { 2537a5664dadSMike Snitzer mutex_unlock(&md->type_lock); 2538a5664dadSMike Snitzer } 2539a5664dadSMike Snitzer 2540a5664dadSMike Snitzer void dm_set_md_type(struct mapped_device *md, unsigned type) 2541a5664dadSMike Snitzer { 254200c4fc3bSMike Snitzer BUG_ON(!mutex_is_locked(&md->type_lock)); 2543a5664dadSMike Snitzer md->type = type; 2544a5664dadSMike Snitzer } 2545a5664dadSMike Snitzer 2546a5664dadSMike Snitzer unsigned dm_get_md_type(struct mapped_device *md) 2547a5664dadSMike Snitzer { 254800c4fc3bSMike Snitzer BUG_ON(!mutex_is_locked(&md->type_lock)); 2549a5664dadSMike Snitzer return md->type; 2550a5664dadSMike Snitzer } 2551a5664dadSMike Snitzer 255236a0456fSAlasdair G Kergon struct target_type *dm_get_immutable_target_type(struct mapped_device *md) 255336a0456fSAlasdair G Kergon { 255436a0456fSAlasdair G Kergon return md->immutable_target_type; 255536a0456fSAlasdair G Kergon } 255636a0456fSAlasdair G Kergon 25574a0b4ddfSMike Snitzer /* 2558f84cb8a4SMike Snitzer * The queue_limits are only valid as long as you have a reference 2559f84cb8a4SMike Snitzer * count on 'md'. 2560f84cb8a4SMike Snitzer */ 2561f84cb8a4SMike Snitzer struct queue_limits *dm_get_queue_limits(struct mapped_device *md) 2562f84cb8a4SMike Snitzer { 2563f84cb8a4SMike Snitzer BUG_ON(!atomic_read(&md->holders)); 2564f84cb8a4SMike Snitzer return &md->queue->limits; 2565f84cb8a4SMike Snitzer } 2566f84cb8a4SMike Snitzer EXPORT_SYMBOL_GPL(dm_get_queue_limits); 2567f84cb8a4SMike Snitzer 2568bfebd1cdSMike Snitzer static void init_rq_based_worker_thread(struct mapped_device *md) 2569bfebd1cdSMike Snitzer { 2570bfebd1cdSMike Snitzer /* Initialize the request-based DM worker thread */ 2571bfebd1cdSMike Snitzer init_kthread_worker(&md->kworker); 2572bfebd1cdSMike Snitzer md->kworker_task = kthread_run(kthread_worker_fn, &md->kworker, 2573bfebd1cdSMike Snitzer "kdmwork-%s", dm_device_name(md)); 2574bfebd1cdSMike Snitzer } 2575bfebd1cdSMike Snitzer 2576f84cb8a4SMike Snitzer /* 25774a0b4ddfSMike Snitzer * Fully initialize a request-based queue (->elevator, ->request_fn, etc). 25784a0b4ddfSMike Snitzer */ 25794a0b4ddfSMike Snitzer static int dm_init_request_based_queue(struct mapped_device *md) 25804a0b4ddfSMike Snitzer { 25814a0b4ddfSMike Snitzer struct request_queue *q = NULL; 25824a0b4ddfSMike Snitzer 25834a0b4ddfSMike Snitzer /* Fully initialize the queue */ 25844a0b4ddfSMike Snitzer q = blk_init_allocated_queue(md->queue, dm_request_fn, NULL); 25854a0b4ddfSMike Snitzer if (!q) 2586bfebd1cdSMike Snitzer return -EINVAL; 25874a0b4ddfSMike Snitzer 25880ce65797SMike Snitzer /* disable dm_request_fn's merge heuristic by default */ 25890ce65797SMike Snitzer md->seq_rq_merge_deadline_usecs = 0; 25900ce65797SMike Snitzer 25914a0b4ddfSMike Snitzer md->queue = q; 2592bfebd1cdSMike Snitzer dm_init_old_md_queue(md); 25934a0b4ddfSMike Snitzer blk_queue_softirq_done(md->queue, dm_softirq_done); 25944a0b4ddfSMike Snitzer blk_queue_prep_rq(md->queue, dm_prep_fn); 25954a0b4ddfSMike Snitzer 2596bfebd1cdSMike Snitzer init_rq_based_worker_thread(md); 25972eb6e1e3SKeith Busch 25984a0b4ddfSMike Snitzer elv_register_queue(md->queue); 25994a0b4ddfSMike Snitzer 2600bfebd1cdSMike Snitzer return 0; 2601bfebd1cdSMike Snitzer } 2602bfebd1cdSMike Snitzer 2603bfebd1cdSMike Snitzer static int dm_mq_init_request(void *data, struct request *rq, 2604bfebd1cdSMike Snitzer unsigned int hctx_idx, unsigned int request_idx, 2605bfebd1cdSMike Snitzer unsigned int numa_node) 2606bfebd1cdSMike Snitzer { 2607bfebd1cdSMike Snitzer struct mapped_device *md = data; 2608bfebd1cdSMike Snitzer struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq); 2609bfebd1cdSMike Snitzer 2610bfebd1cdSMike Snitzer /* 2611bfebd1cdSMike Snitzer * Must initialize md member of tio, otherwise it won't 2612bfebd1cdSMike Snitzer * be available in dm_mq_queue_rq. 2613bfebd1cdSMike Snitzer */ 2614bfebd1cdSMike Snitzer tio->md = md; 2615bfebd1cdSMike Snitzer 2616bfebd1cdSMike Snitzer return 0; 2617bfebd1cdSMike Snitzer } 2618bfebd1cdSMike Snitzer 2619bfebd1cdSMike Snitzer static int dm_mq_queue_rq(struct blk_mq_hw_ctx *hctx, 2620bfebd1cdSMike Snitzer const struct blk_mq_queue_data *bd) 2621bfebd1cdSMike Snitzer { 2622bfebd1cdSMike Snitzer struct request *rq = bd->rq; 2623bfebd1cdSMike Snitzer struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq); 2624bfebd1cdSMike Snitzer struct mapped_device *md = tio->md; 2625bfebd1cdSMike Snitzer int srcu_idx; 2626bfebd1cdSMike Snitzer struct dm_table *map = dm_get_live_table(md, &srcu_idx); 2627bfebd1cdSMike Snitzer struct dm_target *ti; 2628bfebd1cdSMike Snitzer sector_t pos; 2629bfebd1cdSMike Snitzer 2630bfebd1cdSMike Snitzer /* always use block 0 to find the target for flushes for now */ 2631bfebd1cdSMike Snitzer pos = 0; 2632bfebd1cdSMike Snitzer if (!(rq->cmd_flags & REQ_FLUSH)) 2633bfebd1cdSMike Snitzer pos = blk_rq_pos(rq); 2634bfebd1cdSMike Snitzer 2635bfebd1cdSMike Snitzer ti = dm_table_find_target(map, pos); 2636bfebd1cdSMike Snitzer if (!dm_target_is_valid(ti)) { 2637bfebd1cdSMike Snitzer dm_put_live_table(md, srcu_idx); 2638bfebd1cdSMike Snitzer DMERR_LIMIT("request attempted access beyond the end of device"); 2639bfebd1cdSMike Snitzer /* 2640bfebd1cdSMike Snitzer * Must perform setup, that rq_completed() requires, 2641bfebd1cdSMike Snitzer * before returning BLK_MQ_RQ_QUEUE_ERROR 2642bfebd1cdSMike Snitzer */ 2643bfebd1cdSMike Snitzer dm_start_request(md, rq); 2644bfebd1cdSMike Snitzer return BLK_MQ_RQ_QUEUE_ERROR; 2645bfebd1cdSMike Snitzer } 2646bfebd1cdSMike Snitzer dm_put_live_table(md, srcu_idx); 2647bfebd1cdSMike Snitzer 2648bfebd1cdSMike Snitzer if (ti->type->busy && ti->type->busy(ti)) 2649bfebd1cdSMike Snitzer return BLK_MQ_RQ_QUEUE_BUSY; 2650bfebd1cdSMike Snitzer 2651bfebd1cdSMike Snitzer dm_start_request(md, rq); 2652bfebd1cdSMike Snitzer 2653bfebd1cdSMike Snitzer /* Init tio using md established in .init_request */ 2654bfebd1cdSMike Snitzer init_tio(tio, rq, md); 2655bfebd1cdSMike Snitzer 265602233342SMike Snitzer /* 265702233342SMike Snitzer * Establish tio->ti before queuing work (map_tio_request) 265802233342SMike Snitzer * or making direct call to map_request(). 265902233342SMike Snitzer */ 2660bfebd1cdSMike Snitzer tio->ti = ti; 266102233342SMike Snitzer 266202233342SMike Snitzer /* Clone the request if underlying devices aren't blk-mq */ 266302233342SMike Snitzer if (dm_table_get_type(map) == DM_TYPE_REQUEST_BASED) { 266402233342SMike Snitzer /* clone request is allocated at the end of the pdu */ 266502233342SMike Snitzer tio->clone = (void *)blk_mq_rq_to_pdu(rq) + sizeof(struct dm_rq_target_io); 266602233342SMike Snitzer if (!clone_rq(rq, md, tio, GFP_ATOMIC)) 266702233342SMike Snitzer return BLK_MQ_RQ_QUEUE_BUSY; 2668bfebd1cdSMike Snitzer queue_kthread_work(&md->kworker, &tio->work); 266902233342SMike Snitzer } else { 267002233342SMike Snitzer /* Direct call is fine since .queue_rq allows allocations */ 267102233342SMike Snitzer if (map_request(tio, rq, md) == DM_MAPIO_REQUEUE) 267202233342SMike Snitzer dm_requeue_unmapped_original_request(md, rq); 267302233342SMike Snitzer } 2674bfebd1cdSMike Snitzer 2675bfebd1cdSMike Snitzer return BLK_MQ_RQ_QUEUE_OK; 2676bfebd1cdSMike Snitzer } 2677bfebd1cdSMike Snitzer 2678bfebd1cdSMike Snitzer static struct blk_mq_ops dm_mq_ops = { 2679bfebd1cdSMike Snitzer .queue_rq = dm_mq_queue_rq, 2680bfebd1cdSMike Snitzer .map_queue = blk_mq_map_queue, 2681bfebd1cdSMike Snitzer .complete = dm_softirq_done, 2682bfebd1cdSMike Snitzer .init_request = dm_mq_init_request, 2683bfebd1cdSMike Snitzer }; 2684bfebd1cdSMike Snitzer 2685bfebd1cdSMike Snitzer static int dm_init_request_based_blk_mq_queue(struct mapped_device *md) 2686bfebd1cdSMike Snitzer { 268702233342SMike Snitzer unsigned md_type = dm_get_md_type(md); 2688bfebd1cdSMike Snitzer struct request_queue *q; 2689bfebd1cdSMike Snitzer int err; 2690bfebd1cdSMike Snitzer 2691bfebd1cdSMike Snitzer memset(&md->tag_set, 0, sizeof(md->tag_set)); 2692bfebd1cdSMike Snitzer md->tag_set.ops = &dm_mq_ops; 2693bfebd1cdSMike Snitzer md->tag_set.queue_depth = BLKDEV_MAX_RQ; 2694bfebd1cdSMike Snitzer md->tag_set.numa_node = NUMA_NO_NODE; 2695bfebd1cdSMike Snitzer md->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE; 2696bfebd1cdSMike Snitzer md->tag_set.nr_hw_queues = 1; 269702233342SMike Snitzer if (md_type == DM_TYPE_REQUEST_BASED) { 269802233342SMike Snitzer /* make the memory for non-blk-mq clone part of the pdu */ 269902233342SMike Snitzer md->tag_set.cmd_size = sizeof(struct dm_rq_target_io) + sizeof(struct request); 270002233342SMike Snitzer } else 2701bfebd1cdSMike Snitzer md->tag_set.cmd_size = sizeof(struct dm_rq_target_io); 2702bfebd1cdSMike Snitzer md->tag_set.driver_data = md; 2703bfebd1cdSMike Snitzer 2704bfebd1cdSMike Snitzer err = blk_mq_alloc_tag_set(&md->tag_set); 2705bfebd1cdSMike Snitzer if (err) 2706bfebd1cdSMike Snitzer return err; 2707bfebd1cdSMike Snitzer 2708bfebd1cdSMike Snitzer q = blk_mq_init_allocated_queue(&md->tag_set, md->queue); 2709bfebd1cdSMike Snitzer if (IS_ERR(q)) { 2710bfebd1cdSMike Snitzer err = PTR_ERR(q); 2711bfebd1cdSMike Snitzer goto out_tag_set; 2712bfebd1cdSMike Snitzer } 2713bfebd1cdSMike Snitzer md->queue = q; 2714bfebd1cdSMike Snitzer dm_init_md_queue(md); 2715bfebd1cdSMike Snitzer 2716bfebd1cdSMike Snitzer /* backfill 'mq' sysfs registration normally done in blk_register_queue */ 2717bfebd1cdSMike Snitzer blk_mq_register_disk(md->disk); 2718bfebd1cdSMike Snitzer 271902233342SMike Snitzer if (md_type == DM_TYPE_REQUEST_BASED) 2720bfebd1cdSMike Snitzer init_rq_based_worker_thread(md); 2721bfebd1cdSMike Snitzer 2722bfebd1cdSMike Snitzer return 0; 2723bfebd1cdSMike Snitzer 2724bfebd1cdSMike Snitzer out_tag_set: 2725bfebd1cdSMike Snitzer blk_mq_free_tag_set(&md->tag_set); 2726bfebd1cdSMike Snitzer return err; 27274a0b4ddfSMike Snitzer } 27284a0b4ddfSMike Snitzer 272917e149b8SMike Snitzer static unsigned filter_md_type(unsigned type, struct mapped_device *md) 273017e149b8SMike Snitzer { 273117e149b8SMike Snitzer if (type == DM_TYPE_BIO_BASED) 273217e149b8SMike Snitzer return type; 273317e149b8SMike Snitzer 273417e149b8SMike Snitzer return !md->use_blk_mq ? DM_TYPE_REQUEST_BASED : DM_TYPE_MQ_REQUEST_BASED; 273517e149b8SMike Snitzer } 273617e149b8SMike Snitzer 27374a0b4ddfSMike Snitzer /* 27384a0b4ddfSMike Snitzer * Setup the DM device's queue based on md's type 27394a0b4ddfSMike Snitzer */ 27404a0b4ddfSMike Snitzer int dm_setup_md_queue(struct mapped_device *md) 27414a0b4ddfSMike Snitzer { 2742bfebd1cdSMike Snitzer int r; 274317e149b8SMike Snitzer unsigned md_type = filter_md_type(dm_get_md_type(md), md); 2744bfebd1cdSMike Snitzer 2745bfebd1cdSMike Snitzer switch (md_type) { 2746bfebd1cdSMike Snitzer case DM_TYPE_REQUEST_BASED: 2747bfebd1cdSMike Snitzer r = dm_init_request_based_queue(md); 2748bfebd1cdSMike Snitzer if (r) { 27494a0b4ddfSMike Snitzer DMWARN("Cannot initialize queue for request-based mapped device"); 2750bfebd1cdSMike Snitzer return r; 27514a0b4ddfSMike Snitzer } 2752bfebd1cdSMike Snitzer break; 2753bfebd1cdSMike Snitzer case DM_TYPE_MQ_REQUEST_BASED: 2754bfebd1cdSMike Snitzer r = dm_init_request_based_blk_mq_queue(md); 2755bfebd1cdSMike Snitzer if (r) { 2756bfebd1cdSMike Snitzer DMWARN("Cannot initialize queue for request-based blk-mq mapped device"); 2757bfebd1cdSMike Snitzer return r; 2758bfebd1cdSMike Snitzer } 2759bfebd1cdSMike Snitzer break; 2760bfebd1cdSMike Snitzer case DM_TYPE_BIO_BASED: 2761bfebd1cdSMike Snitzer dm_init_old_md_queue(md); 2762ff36ab34SMike Snitzer blk_queue_make_request(md->queue, dm_make_request); 2763ff36ab34SMike Snitzer blk_queue_merge_bvec(md->queue, dm_merge_bvec); 2764bfebd1cdSMike Snitzer break; 2765ff36ab34SMike Snitzer } 27664a0b4ddfSMike Snitzer 27674a0b4ddfSMike Snitzer return 0; 27684a0b4ddfSMike Snitzer } 27694a0b4ddfSMike Snitzer 27702bec1f4aSMikulas Patocka struct mapped_device *dm_get_md(dev_t dev) 27711da177e4SLinus Torvalds { 27721da177e4SLinus Torvalds struct mapped_device *md; 27731da177e4SLinus Torvalds unsigned minor = MINOR(dev); 27741da177e4SLinus Torvalds 27751da177e4SLinus Torvalds if (MAJOR(dev) != _major || minor >= (1 << MINORBITS)) 27761da177e4SLinus Torvalds return NULL; 27771da177e4SLinus Torvalds 2778f32c10b0SJeff Mahoney spin_lock(&_minor_lock); 27791da177e4SLinus Torvalds 27801da177e4SLinus Torvalds md = idr_find(&_minor_idr, minor); 27812bec1f4aSMikulas Patocka if (md) { 27822bec1f4aSMikulas Patocka if ((md == MINOR_ALLOCED || 2783f331c029STejun Heo (MINOR(disk_devt(dm_disk(md))) != minor) || 2784abdc568bSKiyoshi Ueda dm_deleting_md(md) || 2785fba9f90eSJeff Mahoney test_bit(DMF_FREEING, &md->flags))) { 2786637842cfSDavid Teigland md = NULL; 2787fba9f90eSJeff Mahoney goto out; 2788fba9f90eSJeff Mahoney } 27892bec1f4aSMikulas Patocka dm_get(md); 27902bec1f4aSMikulas Patocka } 27911da177e4SLinus Torvalds 2792fba9f90eSJeff Mahoney out: 2793f32c10b0SJeff Mahoney spin_unlock(&_minor_lock); 27941da177e4SLinus Torvalds 2795637842cfSDavid Teigland return md; 2796637842cfSDavid Teigland } 27973cf2e4baSAlasdair G Kergon EXPORT_SYMBOL_GPL(dm_get_md); 2798d229a958SDavid Teigland 27999ade92a9SAlasdair G Kergon void *dm_get_mdptr(struct mapped_device *md) 2800637842cfSDavid Teigland { 28019ade92a9SAlasdair G Kergon return md->interface_ptr; 28021da177e4SLinus Torvalds } 28031da177e4SLinus Torvalds 28041da177e4SLinus Torvalds void dm_set_mdptr(struct mapped_device *md, void *ptr) 28051da177e4SLinus Torvalds { 28061da177e4SLinus Torvalds md->interface_ptr = ptr; 28071da177e4SLinus Torvalds } 28081da177e4SLinus Torvalds 28091da177e4SLinus Torvalds void dm_get(struct mapped_device *md) 28101da177e4SLinus Torvalds { 28111da177e4SLinus Torvalds atomic_inc(&md->holders); 28123f77316dSKiyoshi Ueda BUG_ON(test_bit(DMF_FREEING, &md->flags)); 28131da177e4SLinus Torvalds } 28141da177e4SLinus Torvalds 281509ee96b2SMikulas Patocka int dm_hold(struct mapped_device *md) 281609ee96b2SMikulas Patocka { 281709ee96b2SMikulas Patocka spin_lock(&_minor_lock); 281809ee96b2SMikulas Patocka if (test_bit(DMF_FREEING, &md->flags)) { 281909ee96b2SMikulas Patocka spin_unlock(&_minor_lock); 282009ee96b2SMikulas Patocka return -EBUSY; 282109ee96b2SMikulas Patocka } 282209ee96b2SMikulas Patocka dm_get(md); 282309ee96b2SMikulas Patocka spin_unlock(&_minor_lock); 282409ee96b2SMikulas Patocka return 0; 282509ee96b2SMikulas Patocka } 282609ee96b2SMikulas Patocka EXPORT_SYMBOL_GPL(dm_hold); 282709ee96b2SMikulas Patocka 282872d94861SAlasdair G Kergon const char *dm_device_name(struct mapped_device *md) 282972d94861SAlasdair G Kergon { 283072d94861SAlasdair G Kergon return md->name; 283172d94861SAlasdair G Kergon } 283272d94861SAlasdair G Kergon EXPORT_SYMBOL_GPL(dm_device_name); 283372d94861SAlasdair G Kergon 28343f77316dSKiyoshi Ueda static void __dm_destroy(struct mapped_device *md, bool wait) 28351da177e4SLinus Torvalds { 28361134e5aeSMike Anderson struct dm_table *map; 283783d5e5b0SMikulas Patocka int srcu_idx; 28381da177e4SLinus Torvalds 28393f77316dSKiyoshi Ueda might_sleep(); 2840fba9f90eSJeff Mahoney 284183d5e5b0SMikulas Patocka map = dm_get_live_table(md, &srcu_idx); 284263a4f065SMike Snitzer 284363a4f065SMike Snitzer spin_lock(&_minor_lock); 28443f77316dSKiyoshi Ueda idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md)))); 2845fba9f90eSJeff Mahoney set_bit(DMF_FREEING, &md->flags); 2846f32c10b0SJeff Mahoney spin_unlock(&_minor_lock); 28473f77316dSKiyoshi Ueda 284802233342SMike Snitzer if (dm_request_based(md) && md->kworker_task) 28492eb6e1e3SKeith Busch flush_kthread_worker(&md->kworker); 28502eb6e1e3SKeith Busch 2851ab7c7bb6SMikulas Patocka /* 2852ab7c7bb6SMikulas Patocka * Take suspend_lock so that presuspend and postsuspend methods 2853ab7c7bb6SMikulas Patocka * do not race with internal suspend. 2854ab7c7bb6SMikulas Patocka */ 2855ab7c7bb6SMikulas Patocka mutex_lock(&md->suspend_lock); 28564f186f8bSKiyoshi Ueda if (!dm_suspended_md(md)) { 28571da177e4SLinus Torvalds dm_table_presuspend_targets(map); 28581da177e4SLinus Torvalds dm_table_postsuspend_targets(map); 28591da177e4SLinus Torvalds } 2860ab7c7bb6SMikulas Patocka mutex_unlock(&md->suspend_lock); 28613f77316dSKiyoshi Ueda 286283d5e5b0SMikulas Patocka /* dm_put_live_table must be before msleep, otherwise deadlock is possible */ 286383d5e5b0SMikulas Patocka dm_put_live_table(md, srcu_idx); 286483d5e5b0SMikulas Patocka 28653f77316dSKiyoshi Ueda /* 28663f77316dSKiyoshi Ueda * Rare, but there may be I/O requests still going to complete, 28673f77316dSKiyoshi Ueda * for example. Wait for all references to disappear. 28683f77316dSKiyoshi Ueda * No one should increment the reference count of the mapped_device, 28693f77316dSKiyoshi Ueda * after the mapped_device state becomes DMF_FREEING. 28703f77316dSKiyoshi Ueda */ 28713f77316dSKiyoshi Ueda if (wait) 28723f77316dSKiyoshi Ueda while (atomic_read(&md->holders)) 28733f77316dSKiyoshi Ueda msleep(1); 28743f77316dSKiyoshi Ueda else if (atomic_read(&md->holders)) 28753f77316dSKiyoshi Ueda DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)", 28763f77316dSKiyoshi Ueda dm_device_name(md), atomic_read(&md->holders)); 28773f77316dSKiyoshi Ueda 2878784aae73SMilan Broz dm_sysfs_exit(md); 2879a7940155SAlasdair G Kergon dm_table_destroy(__unbind(md)); 28801da177e4SLinus Torvalds free_dev(md); 28811da177e4SLinus Torvalds } 28823f77316dSKiyoshi Ueda 28833f77316dSKiyoshi Ueda void dm_destroy(struct mapped_device *md) 28843f77316dSKiyoshi Ueda { 28853f77316dSKiyoshi Ueda __dm_destroy(md, true); 28863f77316dSKiyoshi Ueda } 28873f77316dSKiyoshi Ueda 28883f77316dSKiyoshi Ueda void dm_destroy_immediate(struct mapped_device *md) 28893f77316dSKiyoshi Ueda { 28903f77316dSKiyoshi Ueda __dm_destroy(md, false); 28913f77316dSKiyoshi Ueda } 28923f77316dSKiyoshi Ueda 28933f77316dSKiyoshi Ueda void dm_put(struct mapped_device *md) 28943f77316dSKiyoshi Ueda { 28953f77316dSKiyoshi Ueda atomic_dec(&md->holders); 28961da177e4SLinus Torvalds } 289779eb885cSEdward Goggin EXPORT_SYMBOL_GPL(dm_put); 28981da177e4SLinus Torvalds 2899401600dfSMikulas Patocka static int dm_wait_for_completion(struct mapped_device *md, int interruptible) 290046125c1cSMilan Broz { 290146125c1cSMilan Broz int r = 0; 2902b44ebeb0SMikulas Patocka DECLARE_WAITQUEUE(wait, current); 2903b44ebeb0SMikulas Patocka 2904b44ebeb0SMikulas Patocka add_wait_queue(&md->wait, &wait); 290546125c1cSMilan Broz 290646125c1cSMilan Broz while (1) { 2907401600dfSMikulas Patocka set_current_state(interruptible); 290846125c1cSMilan Broz 2909b4324feeSKiyoshi Ueda if (!md_in_flight(md)) 291046125c1cSMilan Broz break; 291146125c1cSMilan Broz 2912401600dfSMikulas Patocka if (interruptible == TASK_INTERRUPTIBLE && 2913401600dfSMikulas Patocka signal_pending(current)) { 291446125c1cSMilan Broz r = -EINTR; 291546125c1cSMilan Broz break; 291646125c1cSMilan Broz } 291746125c1cSMilan Broz 291846125c1cSMilan Broz io_schedule(); 291946125c1cSMilan Broz } 292046125c1cSMilan Broz set_current_state(TASK_RUNNING); 292146125c1cSMilan Broz 2922b44ebeb0SMikulas Patocka remove_wait_queue(&md->wait, &wait); 2923b44ebeb0SMikulas Patocka 292446125c1cSMilan Broz return r; 292546125c1cSMilan Broz } 292646125c1cSMilan Broz 29271da177e4SLinus Torvalds /* 29281da177e4SLinus Torvalds * Process the deferred bios 29291da177e4SLinus Torvalds */ 2930ef208587SMikulas Patocka static void dm_wq_work(struct work_struct *work) 29311da177e4SLinus Torvalds { 2932ef208587SMikulas Patocka struct mapped_device *md = container_of(work, struct mapped_device, 2933ef208587SMikulas Patocka work); 29346d6f10dfSMilan Broz struct bio *c; 293583d5e5b0SMikulas Patocka int srcu_idx; 293683d5e5b0SMikulas Patocka struct dm_table *map; 29371da177e4SLinus Torvalds 293883d5e5b0SMikulas Patocka map = dm_get_live_table(md, &srcu_idx); 2939ef208587SMikulas Patocka 29403b00b203SMikulas Patocka while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) { 2941022c2611SMikulas Patocka spin_lock_irq(&md->deferred_lock); 2942022c2611SMikulas Patocka c = bio_list_pop(&md->deferred); 2943022c2611SMikulas Patocka spin_unlock_irq(&md->deferred_lock); 2944022c2611SMikulas Patocka 29456a8736d1STejun Heo if (!c) 2946df12ee99SAlasdair G Kergon break; 294773d410c0SMilan Broz 2948e6ee8c0bSKiyoshi Ueda if (dm_request_based(md)) 2949e6ee8c0bSKiyoshi Ueda generic_make_request(c); 2950af7e466aSMikulas Patocka else 295183d5e5b0SMikulas Patocka __split_and_process_bio(md, map, c); 2952e6ee8c0bSKiyoshi Ueda } 29533b00b203SMikulas Patocka 295483d5e5b0SMikulas Patocka dm_put_live_table(md, srcu_idx); 29551da177e4SLinus Torvalds } 29561da177e4SLinus Torvalds 29579a1fb464SMikulas Patocka static void dm_queue_flush(struct mapped_device *md) 2958304f3f6aSMilan Broz { 29593b00b203SMikulas Patocka clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 29604e857c58SPeter Zijlstra smp_mb__after_atomic(); 296153d5914fSMikulas Patocka queue_work(md->wq, &md->work); 2962304f3f6aSMilan Broz } 2963304f3f6aSMilan Broz 29641da177e4SLinus Torvalds /* 2965042d2a9bSAlasdair G Kergon * Swap in a new table, returning the old one for the caller to destroy. 29661da177e4SLinus Torvalds */ 2967042d2a9bSAlasdair G Kergon struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table) 29681da177e4SLinus Torvalds { 296987eb5b21SMike Christie struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL); 2970754c5fc7SMike Snitzer struct queue_limits limits; 2971042d2a9bSAlasdair G Kergon int r; 29721da177e4SLinus Torvalds 2973e61290a4SDaniel Walker mutex_lock(&md->suspend_lock); 29741da177e4SLinus Torvalds 29751da177e4SLinus Torvalds /* device must be suspended */ 29764f186f8bSKiyoshi Ueda if (!dm_suspended_md(md)) 297793c534aeSAlasdair G Kergon goto out; 29781da177e4SLinus Torvalds 29793ae70656SMike Snitzer /* 29803ae70656SMike Snitzer * If the new table has no data devices, retain the existing limits. 29813ae70656SMike Snitzer * This helps multipath with queue_if_no_path if all paths disappear, 29823ae70656SMike Snitzer * then new I/O is queued based on these limits, and then some paths 29833ae70656SMike Snitzer * reappear. 29843ae70656SMike Snitzer */ 29853ae70656SMike Snitzer if (dm_table_has_no_data_devices(table)) { 298683d5e5b0SMikulas Patocka live_map = dm_get_live_table_fast(md); 29873ae70656SMike Snitzer if (live_map) 29883ae70656SMike Snitzer limits = md->queue->limits; 298983d5e5b0SMikulas Patocka dm_put_live_table_fast(md); 29903ae70656SMike Snitzer } 29913ae70656SMike Snitzer 299287eb5b21SMike Christie if (!live_map) { 2993754c5fc7SMike Snitzer r = dm_calculate_queue_limits(table, &limits); 2994042d2a9bSAlasdair G Kergon if (r) { 2995042d2a9bSAlasdair G Kergon map = ERR_PTR(r); 2996754c5fc7SMike Snitzer goto out; 2997042d2a9bSAlasdair G Kergon } 299887eb5b21SMike Christie } 2999754c5fc7SMike Snitzer 3000042d2a9bSAlasdair G Kergon map = __bind(md, table, &limits); 30011da177e4SLinus Torvalds 300293c534aeSAlasdair G Kergon out: 3003e61290a4SDaniel Walker mutex_unlock(&md->suspend_lock); 3004042d2a9bSAlasdair G Kergon return map; 30051da177e4SLinus Torvalds } 30061da177e4SLinus Torvalds 30071da177e4SLinus Torvalds /* 30081da177e4SLinus Torvalds * Functions to lock and unlock any filesystem running on the 30091da177e4SLinus Torvalds * device. 30101da177e4SLinus Torvalds */ 30112ca3310eSAlasdair G Kergon static int lock_fs(struct mapped_device *md) 30121da177e4SLinus Torvalds { 3013e39e2e95SAlasdair G Kergon int r; 30141da177e4SLinus Torvalds 30151da177e4SLinus Torvalds WARN_ON(md->frozen_sb); 3016dfbe03f6SAlasdair G Kergon 3017db8fef4fSMikulas Patocka md->frozen_sb = freeze_bdev(md->bdev); 3018dfbe03f6SAlasdair G Kergon if (IS_ERR(md->frozen_sb)) { 3019cf222b37SAlasdair G Kergon r = PTR_ERR(md->frozen_sb); 3020e39e2e95SAlasdair G Kergon md->frozen_sb = NULL; 3021e39e2e95SAlasdair G Kergon return r; 3022dfbe03f6SAlasdair G Kergon } 3023dfbe03f6SAlasdair G Kergon 3024aa8d7c2fSAlasdair G Kergon set_bit(DMF_FROZEN, &md->flags); 3025aa8d7c2fSAlasdair G Kergon 30261da177e4SLinus Torvalds return 0; 30271da177e4SLinus Torvalds } 30281da177e4SLinus Torvalds 30292ca3310eSAlasdair G Kergon static void unlock_fs(struct mapped_device *md) 30301da177e4SLinus Torvalds { 3031aa8d7c2fSAlasdair G Kergon if (!test_bit(DMF_FROZEN, &md->flags)) 3032aa8d7c2fSAlasdair G Kergon return; 3033aa8d7c2fSAlasdair G Kergon 3034db8fef4fSMikulas Patocka thaw_bdev(md->bdev, md->frozen_sb); 30351da177e4SLinus Torvalds md->frozen_sb = NULL; 3036aa8d7c2fSAlasdair G Kergon clear_bit(DMF_FROZEN, &md->flags); 30371da177e4SLinus Torvalds } 30381da177e4SLinus Torvalds 30391da177e4SLinus Torvalds /* 3040ffcc3936SMike Snitzer * If __dm_suspend returns 0, the device is completely quiescent 3041ffcc3936SMike Snitzer * now. There is no request-processing activity. All new requests 3042ffcc3936SMike Snitzer * are being added to md->deferred list. 3043cec47e3dSKiyoshi Ueda * 3044ffcc3936SMike Snitzer * Caller must hold md->suspend_lock 3045cec47e3dSKiyoshi Ueda */ 3046ffcc3936SMike Snitzer static int __dm_suspend(struct mapped_device *md, struct dm_table *map, 3047ffcc3936SMike Snitzer unsigned suspend_flags, int interruptible) 30481da177e4SLinus Torvalds { 3049ffcc3936SMike Snitzer bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG; 3050ffcc3936SMike Snitzer bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG; 3051ffcc3936SMike Snitzer int r; 3052cf222b37SAlasdair G Kergon 30532e93ccc1SKiyoshi Ueda /* 30542e93ccc1SKiyoshi Ueda * DMF_NOFLUSH_SUSPENDING must be set before presuspend. 30552e93ccc1SKiyoshi Ueda * This flag is cleared before dm_suspend returns. 30562e93ccc1SKiyoshi Ueda */ 30572e93ccc1SKiyoshi Ueda if (noflush) 30582e93ccc1SKiyoshi Ueda set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 30592e93ccc1SKiyoshi Ueda 3060d67ee213SMike Snitzer /* 3061d67ee213SMike Snitzer * This gets reverted if there's an error later and the targets 3062d67ee213SMike Snitzer * provide the .presuspend_undo hook. 3063d67ee213SMike Snitzer */ 30641da177e4SLinus Torvalds dm_table_presuspend_targets(map); 30651da177e4SLinus Torvalds 30662e93ccc1SKiyoshi Ueda /* 30679f518b27SKiyoshi Ueda * Flush I/O to the device. 30689f518b27SKiyoshi Ueda * Any I/O submitted after lock_fs() may not be flushed. 30699f518b27SKiyoshi Ueda * noflush takes precedence over do_lockfs. 30709f518b27SKiyoshi Ueda * (lock_fs() flushes I/Os and waits for them to complete.) 30712e93ccc1SKiyoshi Ueda */ 307232a926daSMikulas Patocka if (!noflush && do_lockfs) { 30732ca3310eSAlasdair G Kergon r = lock_fs(md); 3074d67ee213SMike Snitzer if (r) { 3075d67ee213SMike Snitzer dm_table_presuspend_undo_targets(map); 3076ffcc3936SMike Snitzer return r; 3077aa8d7c2fSAlasdair G Kergon } 3078d67ee213SMike Snitzer } 30791da177e4SLinus Torvalds 30801da177e4SLinus Torvalds /* 30813b00b203SMikulas Patocka * Here we must make sure that no processes are submitting requests 30823b00b203SMikulas Patocka * to target drivers i.e. no one may be executing 30833b00b203SMikulas Patocka * __split_and_process_bio. This is called from dm_request and 30843b00b203SMikulas Patocka * dm_wq_work. 30853b00b203SMikulas Patocka * 30863b00b203SMikulas Patocka * To get all processes out of __split_and_process_bio in dm_request, 30873b00b203SMikulas Patocka * we take the write lock. To prevent any process from reentering 30886a8736d1STejun Heo * __split_and_process_bio from dm_request and quiesce the thread 30896a8736d1STejun Heo * (dm_wq_work), we set BMF_BLOCK_IO_FOR_SUSPEND and call 30906a8736d1STejun Heo * flush_workqueue(md->wq). 30911da177e4SLinus Torvalds */ 30921eb787ecSAlasdair G Kergon set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 309341abc4e1SHannes Reinecke if (map) 309483d5e5b0SMikulas Patocka synchronize_srcu(&md->io_barrier); 30951da177e4SLinus Torvalds 3096d0bcb878SKiyoshi Ueda /* 309729e4013dSTejun Heo * Stop md->queue before flushing md->wq in case request-based 309829e4013dSTejun Heo * dm defers requests to md->wq from md->queue. 3099d0bcb878SKiyoshi Ueda */ 31002eb6e1e3SKeith Busch if (dm_request_based(md)) { 31019f518b27SKiyoshi Ueda stop_queue(md->queue); 310202233342SMike Snitzer if (md->kworker_task) 31032eb6e1e3SKeith Busch flush_kthread_worker(&md->kworker); 31042eb6e1e3SKeith Busch } 3105cec47e3dSKiyoshi Ueda 3106d0bcb878SKiyoshi Ueda flush_workqueue(md->wq); 3107d0bcb878SKiyoshi Ueda 31081da177e4SLinus Torvalds /* 31093b00b203SMikulas Patocka * At this point no more requests are entering target request routines. 31103b00b203SMikulas Patocka * We call dm_wait_for_completion to wait for all existing requests 31113b00b203SMikulas Patocka * to finish. 31121da177e4SLinus Torvalds */ 3113ffcc3936SMike Snitzer r = dm_wait_for_completion(md, interruptible); 31141da177e4SLinus Torvalds 31156d6f10dfSMilan Broz if (noflush) 3116022c2611SMikulas Patocka clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 311741abc4e1SHannes Reinecke if (map) 311883d5e5b0SMikulas Patocka synchronize_srcu(&md->io_barrier); 31192e93ccc1SKiyoshi Ueda 31201da177e4SLinus Torvalds /* were we interrupted ? */ 312146125c1cSMilan Broz if (r < 0) { 31229a1fb464SMikulas Patocka dm_queue_flush(md); 312373d410c0SMilan Broz 3124cec47e3dSKiyoshi Ueda if (dm_request_based(md)) 31259f518b27SKiyoshi Ueda start_queue(md->queue); 3126cec47e3dSKiyoshi Ueda 31272ca3310eSAlasdair G Kergon unlock_fs(md); 3128d67ee213SMike Snitzer dm_table_presuspend_undo_targets(map); 3129ffcc3936SMike Snitzer /* pushback list is already flushed, so skip flush */ 3130ffcc3936SMike Snitzer } 3131ffcc3936SMike Snitzer 3132ffcc3936SMike Snitzer return r; 31332ca3310eSAlasdair G Kergon } 31342ca3310eSAlasdair G Kergon 31353b00b203SMikulas Patocka /* 3136ffcc3936SMike Snitzer * We need to be able to change a mapping table under a mounted 3137ffcc3936SMike Snitzer * filesystem. For example we might want to move some data in 3138ffcc3936SMike Snitzer * the background. Before the table can be swapped with 3139ffcc3936SMike Snitzer * dm_bind_table, dm_suspend must be called to flush any in 3140ffcc3936SMike Snitzer * flight bios and ensure that any further io gets deferred. 31413b00b203SMikulas Patocka */ 3142ffcc3936SMike Snitzer /* 3143ffcc3936SMike Snitzer * Suspend mechanism in request-based dm. 3144ffcc3936SMike Snitzer * 3145ffcc3936SMike Snitzer * 1. Flush all I/Os by lock_fs() if needed. 3146ffcc3936SMike Snitzer * 2. Stop dispatching any I/O by stopping the request_queue. 3147ffcc3936SMike Snitzer * 3. Wait for all in-flight I/Os to be completed or requeued. 3148ffcc3936SMike Snitzer * 3149ffcc3936SMike Snitzer * To abort suspend, start the request_queue. 3150ffcc3936SMike Snitzer */ 3151ffcc3936SMike Snitzer int dm_suspend(struct mapped_device *md, unsigned suspend_flags) 3152ffcc3936SMike Snitzer { 3153ffcc3936SMike Snitzer struct dm_table *map = NULL; 3154ffcc3936SMike Snitzer int r = 0; 3155ffcc3936SMike Snitzer 3156ffcc3936SMike Snitzer retry: 3157ffcc3936SMike Snitzer mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); 3158ffcc3936SMike Snitzer 3159ffcc3936SMike Snitzer if (dm_suspended_md(md)) { 3160ffcc3936SMike Snitzer r = -EINVAL; 3161ffcc3936SMike Snitzer goto out_unlock; 3162ffcc3936SMike Snitzer } 3163ffcc3936SMike Snitzer 3164ffcc3936SMike Snitzer if (dm_suspended_internally_md(md)) { 3165ffcc3936SMike Snitzer /* already internally suspended, wait for internal resume */ 3166ffcc3936SMike Snitzer mutex_unlock(&md->suspend_lock); 3167ffcc3936SMike Snitzer r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); 3168ffcc3936SMike Snitzer if (r) 3169ffcc3936SMike Snitzer return r; 3170ffcc3936SMike Snitzer goto retry; 3171ffcc3936SMike Snitzer } 3172ffcc3936SMike Snitzer 3173a12f5d48SEric Dumazet map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 3174ffcc3936SMike Snitzer 3175ffcc3936SMike Snitzer r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE); 3176ffcc3936SMike Snitzer if (r) 3177ffcc3936SMike Snitzer goto out_unlock; 31783b00b203SMikulas Patocka 31791da177e4SLinus Torvalds set_bit(DMF_SUSPENDED, &md->flags); 31801da177e4SLinus Torvalds 31814d4471cbSKiyoshi Ueda dm_table_postsuspend_targets(map); 31824d4471cbSKiyoshi Ueda 3183d287483dSAlasdair G Kergon out_unlock: 3184e61290a4SDaniel Walker mutex_unlock(&md->suspend_lock); 3185cf222b37SAlasdair G Kergon return r; 31861da177e4SLinus Torvalds } 31871da177e4SLinus Torvalds 3188ffcc3936SMike Snitzer static int __dm_resume(struct mapped_device *md, struct dm_table *map) 31891da177e4SLinus Torvalds { 3190ffcc3936SMike Snitzer if (map) { 3191ffcc3936SMike Snitzer int r = dm_table_resume_targets(map); 31928757b776SMilan Broz if (r) 3193ffcc3936SMike Snitzer return r; 3194ffcc3936SMike Snitzer } 31952ca3310eSAlasdair G Kergon 31969a1fb464SMikulas Patocka dm_queue_flush(md); 31972ca3310eSAlasdair G Kergon 3198cec47e3dSKiyoshi Ueda /* 3199cec47e3dSKiyoshi Ueda * Flushing deferred I/Os must be done after targets are resumed 3200cec47e3dSKiyoshi Ueda * so that mapping of targets can work correctly. 3201cec47e3dSKiyoshi Ueda * Request-based dm is queueing the deferred I/Os in its request_queue. 3202cec47e3dSKiyoshi Ueda */ 3203cec47e3dSKiyoshi Ueda if (dm_request_based(md)) 3204cec47e3dSKiyoshi Ueda start_queue(md->queue); 3205cec47e3dSKiyoshi Ueda 32062ca3310eSAlasdair G Kergon unlock_fs(md); 32072ca3310eSAlasdair G Kergon 3208ffcc3936SMike Snitzer return 0; 3209ffcc3936SMike Snitzer } 3210ffcc3936SMike Snitzer 3211ffcc3936SMike Snitzer int dm_resume(struct mapped_device *md) 3212ffcc3936SMike Snitzer { 3213ffcc3936SMike Snitzer int r = -EINVAL; 3214ffcc3936SMike Snitzer struct dm_table *map = NULL; 3215ffcc3936SMike Snitzer 3216ffcc3936SMike Snitzer retry: 3217ffcc3936SMike Snitzer mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); 3218ffcc3936SMike Snitzer 3219ffcc3936SMike Snitzer if (!dm_suspended_md(md)) 3220ffcc3936SMike Snitzer goto out; 3221ffcc3936SMike Snitzer 3222ffcc3936SMike Snitzer if (dm_suspended_internally_md(md)) { 3223ffcc3936SMike Snitzer /* already internally suspended, wait for internal resume */ 3224ffcc3936SMike Snitzer mutex_unlock(&md->suspend_lock); 3225ffcc3936SMike Snitzer r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); 3226ffcc3936SMike Snitzer if (r) 3227ffcc3936SMike Snitzer return r; 3228ffcc3936SMike Snitzer goto retry; 3229ffcc3936SMike Snitzer } 3230ffcc3936SMike Snitzer 3231a12f5d48SEric Dumazet map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 3232ffcc3936SMike Snitzer if (!map || !dm_table_get_size(map)) 3233ffcc3936SMike Snitzer goto out; 3234ffcc3936SMike Snitzer 3235ffcc3936SMike Snitzer r = __dm_resume(md, map); 3236ffcc3936SMike Snitzer if (r) 3237ffcc3936SMike Snitzer goto out; 3238ffcc3936SMike Snitzer 32392ca3310eSAlasdair G Kergon clear_bit(DMF_SUSPENDED, &md->flags); 32402ca3310eSAlasdair G Kergon 3241cf222b37SAlasdair G Kergon r = 0; 3242cf222b37SAlasdair G Kergon out: 3243e61290a4SDaniel Walker mutex_unlock(&md->suspend_lock); 32442ca3310eSAlasdair G Kergon 3245cf222b37SAlasdair G Kergon return r; 32461da177e4SLinus Torvalds } 32471da177e4SLinus Torvalds 3248fd2ed4d2SMikulas Patocka /* 3249fd2ed4d2SMikulas Patocka * Internal suspend/resume works like userspace-driven suspend. It waits 3250fd2ed4d2SMikulas Patocka * until all bios finish and prevents issuing new bios to the target drivers. 3251fd2ed4d2SMikulas Patocka * It may be used only from the kernel. 3252fd2ed4d2SMikulas Patocka */ 3253fd2ed4d2SMikulas Patocka 3254ffcc3936SMike Snitzer static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags) 3255ffcc3936SMike Snitzer { 3256ffcc3936SMike Snitzer struct dm_table *map = NULL; 3257ffcc3936SMike Snitzer 325896b26c8cSMikulas Patocka if (md->internal_suspend_count++) 3259ffcc3936SMike Snitzer return; /* nested internal suspend */ 3260ffcc3936SMike Snitzer 3261ffcc3936SMike Snitzer if (dm_suspended_md(md)) { 3262ffcc3936SMike Snitzer set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 3263ffcc3936SMike Snitzer return; /* nest suspend */ 3264ffcc3936SMike Snitzer } 3265ffcc3936SMike Snitzer 3266a12f5d48SEric Dumazet map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 3267ffcc3936SMike Snitzer 3268ffcc3936SMike Snitzer /* 3269ffcc3936SMike Snitzer * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is 3270ffcc3936SMike Snitzer * supported. Properly supporting a TASK_INTERRUPTIBLE internal suspend 3271ffcc3936SMike Snitzer * would require changing .presuspend to return an error -- avoid this 3272ffcc3936SMike Snitzer * until there is a need for more elaborate variants of internal suspend. 3273ffcc3936SMike Snitzer */ 3274ffcc3936SMike Snitzer (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE); 3275ffcc3936SMike Snitzer 3276ffcc3936SMike Snitzer set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 3277ffcc3936SMike Snitzer 3278ffcc3936SMike Snitzer dm_table_postsuspend_targets(map); 3279ffcc3936SMike Snitzer } 3280ffcc3936SMike Snitzer 3281ffcc3936SMike Snitzer static void __dm_internal_resume(struct mapped_device *md) 3282ffcc3936SMike Snitzer { 328396b26c8cSMikulas Patocka BUG_ON(!md->internal_suspend_count); 328496b26c8cSMikulas Patocka 328596b26c8cSMikulas Patocka if (--md->internal_suspend_count) 3286ffcc3936SMike Snitzer return; /* resume from nested internal suspend */ 3287ffcc3936SMike Snitzer 3288ffcc3936SMike Snitzer if (dm_suspended_md(md)) 3289ffcc3936SMike Snitzer goto done; /* resume from nested suspend */ 3290ffcc3936SMike Snitzer 3291ffcc3936SMike Snitzer /* 3292ffcc3936SMike Snitzer * NOTE: existing callers don't need to call dm_table_resume_targets 3293ffcc3936SMike Snitzer * (which may fail -- so best to avoid it for now by passing NULL map) 3294ffcc3936SMike Snitzer */ 3295ffcc3936SMike Snitzer (void) __dm_resume(md, NULL); 3296ffcc3936SMike Snitzer 3297ffcc3936SMike Snitzer done: 3298ffcc3936SMike Snitzer clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 3299ffcc3936SMike Snitzer smp_mb__after_atomic(); 3300ffcc3936SMike Snitzer wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY); 3301ffcc3936SMike Snitzer } 3302ffcc3936SMike Snitzer 3303ffcc3936SMike Snitzer void dm_internal_suspend_noflush(struct mapped_device *md) 3304fd2ed4d2SMikulas Patocka { 3305fd2ed4d2SMikulas Patocka mutex_lock(&md->suspend_lock); 3306ffcc3936SMike Snitzer __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG); 3307ffcc3936SMike Snitzer mutex_unlock(&md->suspend_lock); 3308ffcc3936SMike Snitzer } 3309ffcc3936SMike Snitzer EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush); 3310ffcc3936SMike Snitzer 3311ffcc3936SMike Snitzer void dm_internal_resume(struct mapped_device *md) 3312ffcc3936SMike Snitzer { 3313ffcc3936SMike Snitzer mutex_lock(&md->suspend_lock); 3314ffcc3936SMike Snitzer __dm_internal_resume(md); 3315ffcc3936SMike Snitzer mutex_unlock(&md->suspend_lock); 3316ffcc3936SMike Snitzer } 3317ffcc3936SMike Snitzer EXPORT_SYMBOL_GPL(dm_internal_resume); 3318ffcc3936SMike Snitzer 3319ffcc3936SMike Snitzer /* 3320ffcc3936SMike Snitzer * Fast variants of internal suspend/resume hold md->suspend_lock, 3321ffcc3936SMike Snitzer * which prevents interaction with userspace-driven suspend. 3322ffcc3936SMike Snitzer */ 3323ffcc3936SMike Snitzer 3324ffcc3936SMike Snitzer void dm_internal_suspend_fast(struct mapped_device *md) 3325ffcc3936SMike Snitzer { 3326ffcc3936SMike Snitzer mutex_lock(&md->suspend_lock); 3327ffcc3936SMike Snitzer if (dm_suspended_md(md) || dm_suspended_internally_md(md)) 3328fd2ed4d2SMikulas Patocka return; 3329fd2ed4d2SMikulas Patocka 3330fd2ed4d2SMikulas Patocka set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 3331fd2ed4d2SMikulas Patocka synchronize_srcu(&md->io_barrier); 3332fd2ed4d2SMikulas Patocka flush_workqueue(md->wq); 3333fd2ed4d2SMikulas Patocka dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE); 3334fd2ed4d2SMikulas Patocka } 3335b735fedeSMikulas Patocka EXPORT_SYMBOL_GPL(dm_internal_suspend_fast); 3336fd2ed4d2SMikulas Patocka 3337ffcc3936SMike Snitzer void dm_internal_resume_fast(struct mapped_device *md) 3338fd2ed4d2SMikulas Patocka { 3339ffcc3936SMike Snitzer if (dm_suspended_md(md) || dm_suspended_internally_md(md)) 3340fd2ed4d2SMikulas Patocka goto done; 3341fd2ed4d2SMikulas Patocka 3342fd2ed4d2SMikulas Patocka dm_queue_flush(md); 3343fd2ed4d2SMikulas Patocka 3344fd2ed4d2SMikulas Patocka done: 3345fd2ed4d2SMikulas Patocka mutex_unlock(&md->suspend_lock); 3346fd2ed4d2SMikulas Patocka } 3347b735fedeSMikulas Patocka EXPORT_SYMBOL_GPL(dm_internal_resume_fast); 3348fd2ed4d2SMikulas Patocka 33491da177e4SLinus Torvalds /*----------------------------------------------------------------- 33501da177e4SLinus Torvalds * Event notification. 33511da177e4SLinus Torvalds *---------------------------------------------------------------*/ 33523abf85b5SPeter Rajnoha int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action, 335360935eb2SMilan Broz unsigned cookie) 335469267a30SAlasdair G Kergon { 335560935eb2SMilan Broz char udev_cookie[DM_COOKIE_LENGTH]; 335660935eb2SMilan Broz char *envp[] = { udev_cookie, NULL }; 335760935eb2SMilan Broz 335860935eb2SMilan Broz if (!cookie) 33593abf85b5SPeter Rajnoha return kobject_uevent(&disk_to_dev(md->disk)->kobj, action); 336060935eb2SMilan Broz else { 336160935eb2SMilan Broz snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u", 336260935eb2SMilan Broz DM_COOKIE_ENV_VAR_NAME, cookie); 33633abf85b5SPeter Rajnoha return kobject_uevent_env(&disk_to_dev(md->disk)->kobj, 33643abf85b5SPeter Rajnoha action, envp); 336560935eb2SMilan Broz } 336669267a30SAlasdair G Kergon } 336769267a30SAlasdair G Kergon 33687a8c3d3bSMike Anderson uint32_t dm_next_uevent_seq(struct mapped_device *md) 33697a8c3d3bSMike Anderson { 33707a8c3d3bSMike Anderson return atomic_add_return(1, &md->uevent_seq); 33717a8c3d3bSMike Anderson } 33727a8c3d3bSMike Anderson 33731da177e4SLinus Torvalds uint32_t dm_get_event_nr(struct mapped_device *md) 33741da177e4SLinus Torvalds { 33751da177e4SLinus Torvalds return atomic_read(&md->event_nr); 33761da177e4SLinus Torvalds } 33771da177e4SLinus Torvalds 33781da177e4SLinus Torvalds int dm_wait_event(struct mapped_device *md, int event_nr) 33791da177e4SLinus Torvalds { 33801da177e4SLinus Torvalds return wait_event_interruptible(md->eventq, 33811da177e4SLinus Torvalds (event_nr != atomic_read(&md->event_nr))); 33821da177e4SLinus Torvalds } 33831da177e4SLinus Torvalds 33847a8c3d3bSMike Anderson void dm_uevent_add(struct mapped_device *md, struct list_head *elist) 33857a8c3d3bSMike Anderson { 33867a8c3d3bSMike Anderson unsigned long flags; 33877a8c3d3bSMike Anderson 33887a8c3d3bSMike Anderson spin_lock_irqsave(&md->uevent_lock, flags); 33897a8c3d3bSMike Anderson list_add(elist, &md->uevent_list); 33907a8c3d3bSMike Anderson spin_unlock_irqrestore(&md->uevent_lock, flags); 33917a8c3d3bSMike Anderson } 33927a8c3d3bSMike Anderson 33931da177e4SLinus Torvalds /* 33941da177e4SLinus Torvalds * The gendisk is only valid as long as you have a reference 33951da177e4SLinus Torvalds * count on 'md'. 33961da177e4SLinus Torvalds */ 33971da177e4SLinus Torvalds struct gendisk *dm_disk(struct mapped_device *md) 33981da177e4SLinus Torvalds { 33991da177e4SLinus Torvalds return md->disk; 34001da177e4SLinus Torvalds } 340165ff5b7dSSami Tolvanen EXPORT_SYMBOL_GPL(dm_disk); 34021da177e4SLinus Torvalds 3403784aae73SMilan Broz struct kobject *dm_kobject(struct mapped_device *md) 3404784aae73SMilan Broz { 34052995fa78SMikulas Patocka return &md->kobj_holder.kobj; 3406784aae73SMilan Broz } 3407784aae73SMilan Broz 3408784aae73SMilan Broz struct mapped_device *dm_get_from_kobject(struct kobject *kobj) 3409784aae73SMilan Broz { 3410784aae73SMilan Broz struct mapped_device *md; 3411784aae73SMilan Broz 34122995fa78SMikulas Patocka md = container_of(kobj, struct mapped_device, kobj_holder.kobj); 3413784aae73SMilan Broz 34144d89b7b4SMilan Broz if (test_bit(DMF_FREEING, &md->flags) || 3415432a212cSMike Anderson dm_deleting_md(md)) 34164d89b7b4SMilan Broz return NULL; 34174d89b7b4SMilan Broz 3418784aae73SMilan Broz dm_get(md); 3419784aae73SMilan Broz return md; 3420784aae73SMilan Broz } 3421784aae73SMilan Broz 34224f186f8bSKiyoshi Ueda int dm_suspended_md(struct mapped_device *md) 34231da177e4SLinus Torvalds { 34241da177e4SLinus Torvalds return test_bit(DMF_SUSPENDED, &md->flags); 34251da177e4SLinus Torvalds } 34261da177e4SLinus Torvalds 3427ffcc3936SMike Snitzer int dm_suspended_internally_md(struct mapped_device *md) 3428ffcc3936SMike Snitzer { 3429ffcc3936SMike Snitzer return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 3430ffcc3936SMike Snitzer } 3431ffcc3936SMike Snitzer 34322c140a24SMikulas Patocka int dm_test_deferred_remove_flag(struct mapped_device *md) 34332c140a24SMikulas Patocka { 34342c140a24SMikulas Patocka return test_bit(DMF_DEFERRED_REMOVE, &md->flags); 34352c140a24SMikulas Patocka } 34362c140a24SMikulas Patocka 343764dbce58SKiyoshi Ueda int dm_suspended(struct dm_target *ti) 343864dbce58SKiyoshi Ueda { 3439ecdb2e25SKiyoshi Ueda return dm_suspended_md(dm_table_get_md(ti->table)); 344064dbce58SKiyoshi Ueda } 344164dbce58SKiyoshi Ueda EXPORT_SYMBOL_GPL(dm_suspended); 344264dbce58SKiyoshi Ueda 34432e93ccc1SKiyoshi Ueda int dm_noflush_suspending(struct dm_target *ti) 34442e93ccc1SKiyoshi Ueda { 3445ecdb2e25SKiyoshi Ueda return __noflush_suspending(dm_table_get_md(ti->table)); 34462e93ccc1SKiyoshi Ueda } 34472e93ccc1SKiyoshi Ueda EXPORT_SYMBOL_GPL(dm_noflush_suspending); 34482e93ccc1SKiyoshi Ueda 3449*5f1b670dSChristoph Hellwig struct dm_md_mempools *dm_alloc_bio_mempools(unsigned integrity, 3450*5f1b670dSChristoph Hellwig unsigned per_bio_data_size) 3451e6ee8c0bSKiyoshi Ueda { 3452*5f1b670dSChristoph Hellwig struct dm_md_mempools *pools; 3453*5f1b670dSChristoph Hellwig unsigned int pool_size = dm_get_reserved_bio_based_ios(); 34545f015204SJun'ichi Nomura unsigned int front_pad; 3455e6ee8c0bSKiyoshi Ueda 3456*5f1b670dSChristoph Hellwig pools = kzalloc(sizeof(*pools), GFP_KERNEL); 3457e6ee8c0bSKiyoshi Ueda if (!pools) 3458e6ee8c0bSKiyoshi Ueda return NULL; 3459e6ee8c0bSKiyoshi Ueda 3460*5f1b670dSChristoph Hellwig front_pad = roundup(per_bio_data_size, __alignof__(struct dm_target_io)) + 3461*5f1b670dSChristoph Hellwig offsetof(struct dm_target_io, clone); 346217e149b8SMike Snitzer 3463*5f1b670dSChristoph Hellwig pools->io_pool = mempool_create_slab_pool(pool_size, _io_cache); 3464e6ee8c0bSKiyoshi Ueda if (!pools->io_pool) 34655f015204SJun'ichi Nomura goto out; 3466e6ee8c0bSKiyoshi Ueda 34673d8aab2dSJunichi Nomura pools->bs = bioset_create_nobvec(pool_size, front_pad); 3468e6ee8c0bSKiyoshi Ueda if (!pools->bs) 34695f015204SJun'ichi Nomura goto out; 3470e6ee8c0bSKiyoshi Ueda 3471a91a2785SMartin K. Petersen if (integrity && bioset_integrity_create(pools->bs, pool_size)) 34725f015204SJun'ichi Nomura goto out; 3473a91a2785SMartin K. Petersen 3474e6ee8c0bSKiyoshi Ueda return pools; 34755f015204SJun'ichi Nomura out: 34765f015204SJun'ichi Nomura dm_free_md_mempools(pools); 3477*5f1b670dSChristoph Hellwig return NULL; 3478*5f1b670dSChristoph Hellwig } 3479e6ee8c0bSKiyoshi Ueda 3480*5f1b670dSChristoph Hellwig struct dm_md_mempools *dm_alloc_rq_mempools(struct mapped_device *md, 3481*5f1b670dSChristoph Hellwig unsigned type) 3482*5f1b670dSChristoph Hellwig { 3483*5f1b670dSChristoph Hellwig unsigned int pool_size = dm_get_reserved_rq_based_ios(); 3484*5f1b670dSChristoph Hellwig struct dm_md_mempools *pools; 3485*5f1b670dSChristoph Hellwig 3486*5f1b670dSChristoph Hellwig pools = kzalloc(sizeof(*pools), GFP_KERNEL); 3487*5f1b670dSChristoph Hellwig if (!pools) 3488*5f1b670dSChristoph Hellwig return NULL; 3489*5f1b670dSChristoph Hellwig 3490*5f1b670dSChristoph Hellwig if (filter_md_type(type, md) == DM_TYPE_REQUEST_BASED) { 3491*5f1b670dSChristoph Hellwig pools->rq_pool = mempool_create_slab_pool(pool_size, _rq_cache); 3492*5f1b670dSChristoph Hellwig if (!pools->rq_pool) 3493*5f1b670dSChristoph Hellwig goto out; 3494*5f1b670dSChristoph Hellwig } 3495*5f1b670dSChristoph Hellwig 3496*5f1b670dSChristoph Hellwig pools->io_pool = mempool_create_slab_pool(pool_size, _rq_tio_cache); 3497*5f1b670dSChristoph Hellwig if (!pools->io_pool) 3498*5f1b670dSChristoph Hellwig goto out; 3499*5f1b670dSChristoph Hellwig 3500*5f1b670dSChristoph Hellwig return pools; 3501*5f1b670dSChristoph Hellwig out: 3502*5f1b670dSChristoph Hellwig dm_free_md_mempools(pools); 3503e6ee8c0bSKiyoshi Ueda return NULL; 3504e6ee8c0bSKiyoshi Ueda } 3505e6ee8c0bSKiyoshi Ueda 3506e6ee8c0bSKiyoshi Ueda void dm_free_md_mempools(struct dm_md_mempools *pools) 3507e6ee8c0bSKiyoshi Ueda { 3508e6ee8c0bSKiyoshi Ueda if (!pools) 3509e6ee8c0bSKiyoshi Ueda return; 3510e6ee8c0bSKiyoshi Ueda 3511e6ee8c0bSKiyoshi Ueda if (pools->io_pool) 3512e6ee8c0bSKiyoshi Ueda mempool_destroy(pools->io_pool); 3513e6ee8c0bSKiyoshi Ueda 35141ae49ea2SMike Snitzer if (pools->rq_pool) 35151ae49ea2SMike Snitzer mempool_destroy(pools->rq_pool); 35161ae49ea2SMike Snitzer 3517e6ee8c0bSKiyoshi Ueda if (pools->bs) 3518e6ee8c0bSKiyoshi Ueda bioset_free(pools->bs); 3519e6ee8c0bSKiyoshi Ueda 3520e6ee8c0bSKiyoshi Ueda kfree(pools); 3521e6ee8c0bSKiyoshi Ueda } 3522e6ee8c0bSKiyoshi Ueda 352383d5cde4SAlexey Dobriyan static const struct block_device_operations dm_blk_dops = { 35241da177e4SLinus Torvalds .open = dm_blk_open, 35251da177e4SLinus Torvalds .release = dm_blk_close, 3526aa129a22SMilan Broz .ioctl = dm_blk_ioctl, 35273ac51e74SDarrick J. Wong .getgeo = dm_blk_getgeo, 35281da177e4SLinus Torvalds .owner = THIS_MODULE 35291da177e4SLinus Torvalds }; 35301da177e4SLinus Torvalds 35311da177e4SLinus Torvalds /* 35321da177e4SLinus Torvalds * module hooks 35331da177e4SLinus Torvalds */ 35341da177e4SLinus Torvalds module_init(dm_init); 35351da177e4SLinus Torvalds module_exit(dm_exit); 35361da177e4SLinus Torvalds 35371da177e4SLinus Torvalds module_param(major, uint, 0); 35381da177e4SLinus Torvalds MODULE_PARM_DESC(major, "The major number of the device mapper"); 3539f4790826SMike Snitzer 3540e8603136SMike Snitzer module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR); 3541e8603136SMike Snitzer MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools"); 3542e8603136SMike Snitzer 3543f4790826SMike Snitzer module_param(reserved_rq_based_ios, uint, S_IRUGO | S_IWUSR); 3544f4790826SMike Snitzer MODULE_PARM_DESC(reserved_rq_based_ios, "Reserved IOs in request-based mempools"); 3545f4790826SMike Snitzer 354617e149b8SMike Snitzer module_param(use_blk_mq, bool, S_IRUGO | S_IWUSR); 354717e149b8SMike Snitzer MODULE_PARM_DESC(use_blk_mq, "Use block multiqueue for request-based DM devices"); 354817e149b8SMike Snitzer 35491da177e4SLinus Torvalds MODULE_DESCRIPTION(DM_NAME " driver"); 35501da177e4SLinus Torvalds MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>"); 35511da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 3552