xref: /openbmc/linux/drivers/md/dm.c (revision 5f1b670d0bef508a5554d92525f5f6d00d640b38)
11da177e4SLinus Torvalds /*
21da177e4SLinus Torvalds  * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3784aae73SMilan Broz  * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
41da177e4SLinus Torvalds  *
51da177e4SLinus Torvalds  * This file is released under the GPL.
61da177e4SLinus Torvalds  */
71da177e4SLinus Torvalds 
81da177e4SLinus Torvalds #include "dm.h"
951e5b2bdSMike Anderson #include "dm-uevent.h"
101da177e4SLinus Torvalds 
111da177e4SLinus Torvalds #include <linux/init.h>
121da177e4SLinus Torvalds #include <linux/module.h>
1348c9c27bSArjan van de Ven #include <linux/mutex.h>
141da177e4SLinus Torvalds #include <linux/moduleparam.h>
151da177e4SLinus Torvalds #include <linux/blkpg.h>
161da177e4SLinus Torvalds #include <linux/bio.h>
171da177e4SLinus Torvalds #include <linux/mempool.h>
181da177e4SLinus Torvalds #include <linux/slab.h>
191da177e4SLinus Torvalds #include <linux/idr.h>
203ac51e74SDarrick J. Wong #include <linux/hdreg.h>
213f77316dSKiyoshi Ueda #include <linux/delay.h>
22ffcc3936SMike Snitzer #include <linux/wait.h>
232eb6e1e3SKeith Busch #include <linux/kthread.h>
240ce65797SMike Snitzer #include <linux/ktime.h>
25de3ec86dSMike Snitzer #include <linux/elevator.h> /* for rq_end_sector() */
26bfebd1cdSMike Snitzer #include <linux/blk-mq.h>
2755782138SLi Zefan 
2855782138SLi Zefan #include <trace/events/block.h>
291da177e4SLinus Torvalds 
3072d94861SAlasdair G Kergon #define DM_MSG_PREFIX "core"
3172d94861SAlasdair G Kergon 
3271a16736SNamhyung Kim #ifdef CONFIG_PRINTK
3371a16736SNamhyung Kim /*
3471a16736SNamhyung Kim  * ratelimit state to be used in DMXXX_LIMIT().
3571a16736SNamhyung Kim  */
3671a16736SNamhyung Kim DEFINE_RATELIMIT_STATE(dm_ratelimit_state,
3771a16736SNamhyung Kim 		       DEFAULT_RATELIMIT_INTERVAL,
3871a16736SNamhyung Kim 		       DEFAULT_RATELIMIT_BURST);
3971a16736SNamhyung Kim EXPORT_SYMBOL(dm_ratelimit_state);
4071a16736SNamhyung Kim #endif
4171a16736SNamhyung Kim 
4260935eb2SMilan Broz /*
4360935eb2SMilan Broz  * Cookies are numeric values sent with CHANGE and REMOVE
4460935eb2SMilan Broz  * uevents while resuming, removing or renaming the device.
4560935eb2SMilan Broz  */
4660935eb2SMilan Broz #define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE"
4760935eb2SMilan Broz #define DM_COOKIE_LENGTH 24
4860935eb2SMilan Broz 
491da177e4SLinus Torvalds static const char *_name = DM_NAME;
501da177e4SLinus Torvalds 
511da177e4SLinus Torvalds static unsigned int major = 0;
521da177e4SLinus Torvalds static unsigned int _major = 0;
531da177e4SLinus Torvalds 
54d15b774cSAlasdair G Kergon static DEFINE_IDR(_minor_idr);
55d15b774cSAlasdair G Kergon 
56f32c10b0SJeff Mahoney static DEFINE_SPINLOCK(_minor_lock);
572c140a24SMikulas Patocka 
582c140a24SMikulas Patocka static void do_deferred_remove(struct work_struct *w);
592c140a24SMikulas Patocka 
602c140a24SMikulas Patocka static DECLARE_WORK(deferred_remove_work, do_deferred_remove);
612c140a24SMikulas Patocka 
62acfe0ad7SMikulas Patocka static struct workqueue_struct *deferred_remove_workqueue;
63acfe0ad7SMikulas Patocka 
641da177e4SLinus Torvalds /*
658fbf26adSKiyoshi Ueda  * For bio-based dm.
661da177e4SLinus Torvalds  * One of these is allocated per bio.
671da177e4SLinus Torvalds  */
681da177e4SLinus Torvalds struct dm_io {
691da177e4SLinus Torvalds 	struct mapped_device *md;
701da177e4SLinus Torvalds 	int error;
711da177e4SLinus Torvalds 	atomic_t io_count;
726ae2fa67SRichard Kennedy 	struct bio *bio;
733eaf840eSJun'ichi "Nick" Nomura 	unsigned long start_time;
74f88fb981SKiyoshi Ueda 	spinlock_t endio_lock;
75fd2ed4d2SMikulas Patocka 	struct dm_stats_aux stats_aux;
761da177e4SLinus Torvalds };
771da177e4SLinus Torvalds 
781da177e4SLinus Torvalds /*
798fbf26adSKiyoshi Ueda  * For request-based dm.
808fbf26adSKiyoshi Ueda  * One of these is allocated per request.
818fbf26adSKiyoshi Ueda  */
828fbf26adSKiyoshi Ueda struct dm_rq_target_io {
838fbf26adSKiyoshi Ueda 	struct mapped_device *md;
848fbf26adSKiyoshi Ueda 	struct dm_target *ti;
851ae49ea2SMike Snitzer 	struct request *orig, *clone;
862eb6e1e3SKeith Busch 	struct kthread_work work;
878fbf26adSKiyoshi Ueda 	int error;
888fbf26adSKiyoshi Ueda 	union map_info info;
898fbf26adSKiyoshi Ueda };
908fbf26adSKiyoshi Ueda 
918fbf26adSKiyoshi Ueda /*
9294818742SKent Overstreet  * For request-based dm - the bio clones we allocate are embedded in these
9394818742SKent Overstreet  * structs.
9494818742SKent Overstreet  *
9594818742SKent Overstreet  * We allocate these with bio_alloc_bioset, using the front_pad parameter when
9694818742SKent Overstreet  * the bioset is created - this means the bio has to come at the end of the
9794818742SKent Overstreet  * struct.
988fbf26adSKiyoshi Ueda  */
998fbf26adSKiyoshi Ueda struct dm_rq_clone_bio_info {
1008fbf26adSKiyoshi Ueda 	struct bio *orig;
101cec47e3dSKiyoshi Ueda 	struct dm_rq_target_io *tio;
10294818742SKent Overstreet 	struct bio clone;
1038fbf26adSKiyoshi Ueda };
1048fbf26adSKiyoshi Ueda 
105cec47e3dSKiyoshi Ueda union map_info *dm_get_rq_mapinfo(struct request *rq)
106cec47e3dSKiyoshi Ueda {
107cec47e3dSKiyoshi Ueda 	if (rq && rq->end_io_data)
108cec47e3dSKiyoshi Ueda 		return &((struct dm_rq_target_io *)rq->end_io_data)->info;
109cec47e3dSKiyoshi Ueda 	return NULL;
110cec47e3dSKiyoshi Ueda }
111cec47e3dSKiyoshi Ueda EXPORT_SYMBOL_GPL(dm_get_rq_mapinfo);
112cec47e3dSKiyoshi Ueda 
113ba61fdd1SJeff Mahoney #define MINOR_ALLOCED ((void *)-1)
114ba61fdd1SJeff Mahoney 
1151da177e4SLinus Torvalds /*
1161da177e4SLinus Torvalds  * Bits for the md->flags field.
1171da177e4SLinus Torvalds  */
1181eb787ecSAlasdair G Kergon #define DMF_BLOCK_IO_FOR_SUSPEND 0
1191da177e4SLinus Torvalds #define DMF_SUSPENDED 1
120aa8d7c2fSAlasdair G Kergon #define DMF_FROZEN 2
121fba9f90eSJeff Mahoney #define DMF_FREEING 3
1225c6bd75dSAlasdair G Kergon #define DMF_DELETING 4
1232e93ccc1SKiyoshi Ueda #define DMF_NOFLUSH_SUSPENDING 5
124d5b9dd04SMikulas Patocka #define DMF_MERGE_IS_OPTIONAL 6
1252c140a24SMikulas Patocka #define DMF_DEFERRED_REMOVE 7
126ffcc3936SMike Snitzer #define DMF_SUSPENDED_INTERNALLY 8
1271da177e4SLinus Torvalds 
128304f3f6aSMilan Broz /*
12983d5e5b0SMikulas Patocka  * A dummy definition to make RCU happy.
13083d5e5b0SMikulas Patocka  * struct dm_table should never be dereferenced in this file.
13183d5e5b0SMikulas Patocka  */
13283d5e5b0SMikulas Patocka struct dm_table {
13383d5e5b0SMikulas Patocka 	int undefined__;
13483d5e5b0SMikulas Patocka };
13583d5e5b0SMikulas Patocka 
13683d5e5b0SMikulas Patocka /*
137304f3f6aSMilan Broz  * Work processed by per-device workqueue.
138304f3f6aSMilan Broz  */
1391da177e4SLinus Torvalds struct mapped_device {
14083d5e5b0SMikulas Patocka 	struct srcu_struct io_barrier;
141e61290a4SDaniel Walker 	struct mutex suspend_lock;
1421da177e4SLinus Torvalds 	atomic_t holders;
1435c6bd75dSAlasdair G Kergon 	atomic_t open_count;
1441da177e4SLinus Torvalds 
1452a7faeb1SMikulas Patocka 	/*
1462a7faeb1SMikulas Patocka 	 * The current mapping.
1472a7faeb1SMikulas Patocka 	 * Use dm_get_live_table{_fast} or take suspend_lock for
1482a7faeb1SMikulas Patocka 	 * dereference.
1492a7faeb1SMikulas Patocka 	 */
1506fa99520SPranith Kumar 	struct dm_table __rcu *map;
1512a7faeb1SMikulas Patocka 
15286f1152bSBenjamin Marzinski 	struct list_head table_devices;
15386f1152bSBenjamin Marzinski 	struct mutex table_devices_lock;
15486f1152bSBenjamin Marzinski 
1551da177e4SLinus Torvalds 	unsigned long flags;
1561da177e4SLinus Torvalds 
157165125e1SJens Axboe 	struct request_queue *queue;
158a5664dadSMike Snitzer 	unsigned type;
1594a0b4ddfSMike Snitzer 	/* Protect queue and type against concurrent access. */
160a5664dadSMike Snitzer 	struct mutex type_lock;
161a5664dadSMike Snitzer 
16236a0456fSAlasdair G Kergon 	struct target_type *immutable_target_type;
16336a0456fSAlasdair G Kergon 
1641da177e4SLinus Torvalds 	struct gendisk *disk;
1657e51f257SMike Anderson 	char name[16];
1661da177e4SLinus Torvalds 
1671da177e4SLinus Torvalds 	void *interface_ptr;
1681da177e4SLinus Torvalds 
1691da177e4SLinus Torvalds 	/*
1701da177e4SLinus Torvalds 	 * A list of ios that arrived while we were suspended.
1711da177e4SLinus Torvalds 	 */
172316d315bSNikanth Karthikesan 	atomic_t pending[2];
1731da177e4SLinus Torvalds 	wait_queue_head_t wait;
17453d5914fSMikulas Patocka 	struct work_struct work;
1751da177e4SLinus Torvalds 	struct bio_list deferred;
176022c2611SMikulas Patocka 	spinlock_t deferred_lock;
1771da177e4SLinus Torvalds 
1781da177e4SLinus Torvalds 	/*
17929e4013dSTejun Heo 	 * Processing queue (flush)
180304f3f6aSMilan Broz 	 */
181304f3f6aSMilan Broz 	struct workqueue_struct *wq;
182304f3f6aSMilan Broz 
183304f3f6aSMilan Broz 	/*
1841da177e4SLinus Torvalds 	 * io objects are allocated from here.
1851da177e4SLinus Torvalds 	 */
1861da177e4SLinus Torvalds 	mempool_t *io_pool;
1871ae49ea2SMike Snitzer 	mempool_t *rq_pool;
1881da177e4SLinus Torvalds 
1899faf400fSStefan Bader 	struct bio_set *bs;
1909faf400fSStefan Bader 
1911da177e4SLinus Torvalds 	/*
1921da177e4SLinus Torvalds 	 * Event handling.
1931da177e4SLinus Torvalds 	 */
1941da177e4SLinus Torvalds 	atomic_t event_nr;
1951da177e4SLinus Torvalds 	wait_queue_head_t eventq;
1967a8c3d3bSMike Anderson 	atomic_t uevent_seq;
1977a8c3d3bSMike Anderson 	struct list_head uevent_list;
1987a8c3d3bSMike Anderson 	spinlock_t uevent_lock; /* Protect access to uevent_list */
1991da177e4SLinus Torvalds 
2001da177e4SLinus Torvalds 	/*
2011da177e4SLinus Torvalds 	 * freeze/thaw support require holding onto a super block
2021da177e4SLinus Torvalds 	 */
2031da177e4SLinus Torvalds 	struct super_block *frozen_sb;
204db8fef4fSMikulas Patocka 	struct block_device *bdev;
2053ac51e74SDarrick J. Wong 
2063ac51e74SDarrick J. Wong 	/* forced geometry settings */
2073ac51e74SDarrick J. Wong 	struct hd_geometry geometry;
208784aae73SMilan Broz 
2092995fa78SMikulas Patocka 	/* kobject and completion */
2102995fa78SMikulas Patocka 	struct dm_kobject_holder kobj_holder;
211be35f486SMikulas Patocka 
212d87f4c14STejun Heo 	/* zero-length flush that will be cloned and submitted to targets */
213d87f4c14STejun Heo 	struct bio flush_bio;
214fd2ed4d2SMikulas Patocka 
21596b26c8cSMikulas Patocka 	/* the number of internal suspends */
21696b26c8cSMikulas Patocka 	unsigned internal_suspend_count;
21796b26c8cSMikulas Patocka 
218fd2ed4d2SMikulas Patocka 	struct dm_stats stats;
2192eb6e1e3SKeith Busch 
2202eb6e1e3SKeith Busch 	struct kthread_worker kworker;
2212eb6e1e3SKeith Busch 	struct task_struct *kworker_task;
222de3ec86dSMike Snitzer 
223de3ec86dSMike Snitzer 	/* for request-based merge heuristic in dm_request_fn() */
2240ce65797SMike Snitzer 	unsigned seq_rq_merge_deadline_usecs;
225de3ec86dSMike Snitzer 	int last_rq_rw;
2260ce65797SMike Snitzer 	sector_t last_rq_pos;
2270ce65797SMike Snitzer 	ktime_t last_rq_start_time;
228bfebd1cdSMike Snitzer 
229bfebd1cdSMike Snitzer 	/* for blk-mq request-based DM support */
230bfebd1cdSMike Snitzer 	struct blk_mq_tag_set tag_set;
23117e149b8SMike Snitzer 	bool use_blk_mq;
2321da177e4SLinus Torvalds };
2331da177e4SLinus Torvalds 
23417e149b8SMike Snitzer #ifdef CONFIG_DM_MQ_DEFAULT
23517e149b8SMike Snitzer static bool use_blk_mq = true;
23617e149b8SMike Snitzer #else
23717e149b8SMike Snitzer static bool use_blk_mq = false;
23817e149b8SMike Snitzer #endif
23917e149b8SMike Snitzer 
24017e149b8SMike Snitzer bool dm_use_blk_mq(struct mapped_device *md)
24117e149b8SMike Snitzer {
24217e149b8SMike Snitzer 	return md->use_blk_mq;
24317e149b8SMike Snitzer }
24417e149b8SMike Snitzer 
245e6ee8c0bSKiyoshi Ueda /*
246e6ee8c0bSKiyoshi Ueda  * For mempools pre-allocation at the table loading time.
247e6ee8c0bSKiyoshi Ueda  */
248e6ee8c0bSKiyoshi Ueda struct dm_md_mempools {
249e6ee8c0bSKiyoshi Ueda 	mempool_t *io_pool;
2501ae49ea2SMike Snitzer 	mempool_t *rq_pool;
251e6ee8c0bSKiyoshi Ueda 	struct bio_set *bs;
252e6ee8c0bSKiyoshi Ueda };
253e6ee8c0bSKiyoshi Ueda 
25486f1152bSBenjamin Marzinski struct table_device {
25586f1152bSBenjamin Marzinski 	struct list_head list;
25686f1152bSBenjamin Marzinski 	atomic_t count;
25786f1152bSBenjamin Marzinski 	struct dm_dev dm_dev;
25886f1152bSBenjamin Marzinski };
25986f1152bSBenjamin Marzinski 
2606cfa5857SMike Snitzer #define RESERVED_BIO_BASED_IOS		16
2616cfa5857SMike Snitzer #define RESERVED_REQUEST_BASED_IOS	256
262f4790826SMike Snitzer #define RESERVED_MAX_IOS		1024
263e18b890bSChristoph Lameter static struct kmem_cache *_io_cache;
2648fbf26adSKiyoshi Ueda static struct kmem_cache *_rq_tio_cache;
2651ae49ea2SMike Snitzer static struct kmem_cache *_rq_cache;
26694818742SKent Overstreet 
267f4790826SMike Snitzer /*
268e8603136SMike Snitzer  * Bio-based DM's mempools' reserved IOs set by the user.
269e8603136SMike Snitzer  */
270e8603136SMike Snitzer static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS;
271e8603136SMike Snitzer 
272e8603136SMike Snitzer /*
273f4790826SMike Snitzer  * Request-based DM's mempools' reserved IOs set by the user.
274f4790826SMike Snitzer  */
275f4790826SMike Snitzer static unsigned reserved_rq_based_ios = RESERVED_REQUEST_BASED_IOS;
276f4790826SMike Snitzer 
27709c2d531SMike Snitzer static unsigned __dm_get_module_param(unsigned *module_param,
278f4790826SMike Snitzer 				      unsigned def, unsigned max)
279f4790826SMike Snitzer {
28009c2d531SMike Snitzer 	unsigned param = ACCESS_ONCE(*module_param);
28109c2d531SMike Snitzer 	unsigned modified_param = 0;
282f4790826SMike Snitzer 
28309c2d531SMike Snitzer 	if (!param)
28409c2d531SMike Snitzer 		modified_param = def;
28509c2d531SMike Snitzer 	else if (param > max)
28609c2d531SMike Snitzer 		modified_param = max;
287f4790826SMike Snitzer 
28809c2d531SMike Snitzer 	if (modified_param) {
28909c2d531SMike Snitzer 		(void)cmpxchg(module_param, param, modified_param);
29009c2d531SMike Snitzer 		param = modified_param;
291f4790826SMike Snitzer 	}
292f4790826SMike Snitzer 
29309c2d531SMike Snitzer 	return param;
294f4790826SMike Snitzer }
295f4790826SMike Snitzer 
296e8603136SMike Snitzer unsigned dm_get_reserved_bio_based_ios(void)
297e8603136SMike Snitzer {
29809c2d531SMike Snitzer 	return __dm_get_module_param(&reserved_bio_based_ios,
299e8603136SMike Snitzer 				     RESERVED_BIO_BASED_IOS, RESERVED_MAX_IOS);
300e8603136SMike Snitzer }
301e8603136SMike Snitzer EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios);
302e8603136SMike Snitzer 
303f4790826SMike Snitzer unsigned dm_get_reserved_rq_based_ios(void)
304f4790826SMike Snitzer {
30509c2d531SMike Snitzer 	return __dm_get_module_param(&reserved_rq_based_ios,
306f4790826SMike Snitzer 				     RESERVED_REQUEST_BASED_IOS, RESERVED_MAX_IOS);
307f4790826SMike Snitzer }
308f4790826SMike Snitzer EXPORT_SYMBOL_GPL(dm_get_reserved_rq_based_ios);
309f4790826SMike Snitzer 
3101da177e4SLinus Torvalds static int __init local_init(void)
3111da177e4SLinus Torvalds {
31251157b4aSKiyoshi Ueda 	int r = -ENOMEM;
3131da177e4SLinus Torvalds 
3141da177e4SLinus Torvalds 	/* allocate a slab for the dm_ios */
315028867acSAlasdair G Kergon 	_io_cache = KMEM_CACHE(dm_io, 0);
3161da177e4SLinus Torvalds 	if (!_io_cache)
31751157b4aSKiyoshi Ueda 		return r;
3181da177e4SLinus Torvalds 
3198fbf26adSKiyoshi Ueda 	_rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0);
3208fbf26adSKiyoshi Ueda 	if (!_rq_tio_cache)
321dba14160SMikulas Patocka 		goto out_free_io_cache;
3228fbf26adSKiyoshi Ueda 
3231ae49ea2SMike Snitzer 	_rq_cache = kmem_cache_create("dm_clone_request", sizeof(struct request),
3241ae49ea2SMike Snitzer 				      __alignof__(struct request), 0, NULL);
3251ae49ea2SMike Snitzer 	if (!_rq_cache)
3261ae49ea2SMike Snitzer 		goto out_free_rq_tio_cache;
3271ae49ea2SMike Snitzer 
32851e5b2bdSMike Anderson 	r = dm_uevent_init();
32951157b4aSKiyoshi Ueda 	if (r)
3301ae49ea2SMike Snitzer 		goto out_free_rq_cache;
33151e5b2bdSMike Anderson 
332acfe0ad7SMikulas Patocka 	deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1);
333acfe0ad7SMikulas Patocka 	if (!deferred_remove_workqueue) {
334acfe0ad7SMikulas Patocka 		r = -ENOMEM;
335acfe0ad7SMikulas Patocka 		goto out_uevent_exit;
336acfe0ad7SMikulas Patocka 	}
337acfe0ad7SMikulas Patocka 
3381da177e4SLinus Torvalds 	_major = major;
3391da177e4SLinus Torvalds 	r = register_blkdev(_major, _name);
34051157b4aSKiyoshi Ueda 	if (r < 0)
341acfe0ad7SMikulas Patocka 		goto out_free_workqueue;
3421da177e4SLinus Torvalds 
3431da177e4SLinus Torvalds 	if (!_major)
3441da177e4SLinus Torvalds 		_major = r;
3451da177e4SLinus Torvalds 
3461da177e4SLinus Torvalds 	return 0;
34751157b4aSKiyoshi Ueda 
348acfe0ad7SMikulas Patocka out_free_workqueue:
349acfe0ad7SMikulas Patocka 	destroy_workqueue(deferred_remove_workqueue);
35051157b4aSKiyoshi Ueda out_uevent_exit:
35151157b4aSKiyoshi Ueda 	dm_uevent_exit();
3521ae49ea2SMike Snitzer out_free_rq_cache:
3531ae49ea2SMike Snitzer 	kmem_cache_destroy(_rq_cache);
3548fbf26adSKiyoshi Ueda out_free_rq_tio_cache:
3558fbf26adSKiyoshi Ueda 	kmem_cache_destroy(_rq_tio_cache);
35651157b4aSKiyoshi Ueda out_free_io_cache:
35751157b4aSKiyoshi Ueda 	kmem_cache_destroy(_io_cache);
35851157b4aSKiyoshi Ueda 
35951157b4aSKiyoshi Ueda 	return r;
3601da177e4SLinus Torvalds }
3611da177e4SLinus Torvalds 
3621da177e4SLinus Torvalds static void local_exit(void)
3631da177e4SLinus Torvalds {
3642c140a24SMikulas Patocka 	flush_scheduled_work();
365acfe0ad7SMikulas Patocka 	destroy_workqueue(deferred_remove_workqueue);
3662c140a24SMikulas Patocka 
3671ae49ea2SMike Snitzer 	kmem_cache_destroy(_rq_cache);
3688fbf26adSKiyoshi Ueda 	kmem_cache_destroy(_rq_tio_cache);
3691da177e4SLinus Torvalds 	kmem_cache_destroy(_io_cache);
37000d59405SAkinobu Mita 	unregister_blkdev(_major, _name);
37151e5b2bdSMike Anderson 	dm_uevent_exit();
3721da177e4SLinus Torvalds 
3731da177e4SLinus Torvalds 	_major = 0;
3741da177e4SLinus Torvalds 
3751da177e4SLinus Torvalds 	DMINFO("cleaned up");
3761da177e4SLinus Torvalds }
3771da177e4SLinus Torvalds 
378b9249e55SAlasdair G Kergon static int (*_inits[])(void) __initdata = {
3791da177e4SLinus Torvalds 	local_init,
3801da177e4SLinus Torvalds 	dm_target_init,
3811da177e4SLinus Torvalds 	dm_linear_init,
3821da177e4SLinus Torvalds 	dm_stripe_init,
383952b3557SMikulas Patocka 	dm_io_init,
384945fa4d2SMikulas Patocka 	dm_kcopyd_init,
3851da177e4SLinus Torvalds 	dm_interface_init,
386fd2ed4d2SMikulas Patocka 	dm_statistics_init,
3871da177e4SLinus Torvalds };
3881da177e4SLinus Torvalds 
389b9249e55SAlasdair G Kergon static void (*_exits[])(void) = {
3901da177e4SLinus Torvalds 	local_exit,
3911da177e4SLinus Torvalds 	dm_target_exit,
3921da177e4SLinus Torvalds 	dm_linear_exit,
3931da177e4SLinus Torvalds 	dm_stripe_exit,
394952b3557SMikulas Patocka 	dm_io_exit,
395945fa4d2SMikulas Patocka 	dm_kcopyd_exit,
3961da177e4SLinus Torvalds 	dm_interface_exit,
397fd2ed4d2SMikulas Patocka 	dm_statistics_exit,
3981da177e4SLinus Torvalds };
3991da177e4SLinus Torvalds 
4001da177e4SLinus Torvalds static int __init dm_init(void)
4011da177e4SLinus Torvalds {
4021da177e4SLinus Torvalds 	const int count = ARRAY_SIZE(_inits);
4031da177e4SLinus Torvalds 
4041da177e4SLinus Torvalds 	int r, i;
4051da177e4SLinus Torvalds 
4061da177e4SLinus Torvalds 	for (i = 0; i < count; i++) {
4071da177e4SLinus Torvalds 		r = _inits[i]();
4081da177e4SLinus Torvalds 		if (r)
4091da177e4SLinus Torvalds 			goto bad;
4101da177e4SLinus Torvalds 	}
4111da177e4SLinus Torvalds 
4121da177e4SLinus Torvalds 	return 0;
4131da177e4SLinus Torvalds 
4141da177e4SLinus Torvalds       bad:
4151da177e4SLinus Torvalds 	while (i--)
4161da177e4SLinus Torvalds 		_exits[i]();
4171da177e4SLinus Torvalds 
4181da177e4SLinus Torvalds 	return r;
4191da177e4SLinus Torvalds }
4201da177e4SLinus Torvalds 
4211da177e4SLinus Torvalds static void __exit dm_exit(void)
4221da177e4SLinus Torvalds {
4231da177e4SLinus Torvalds 	int i = ARRAY_SIZE(_exits);
4241da177e4SLinus Torvalds 
4251da177e4SLinus Torvalds 	while (i--)
4261da177e4SLinus Torvalds 		_exits[i]();
427d15b774cSAlasdair G Kergon 
428d15b774cSAlasdair G Kergon 	/*
429d15b774cSAlasdair G Kergon 	 * Should be empty by this point.
430d15b774cSAlasdair G Kergon 	 */
431d15b774cSAlasdair G Kergon 	idr_destroy(&_minor_idr);
4321da177e4SLinus Torvalds }
4331da177e4SLinus Torvalds 
4341da177e4SLinus Torvalds /*
4351da177e4SLinus Torvalds  * Block device functions
4361da177e4SLinus Torvalds  */
437432a212cSMike Anderson int dm_deleting_md(struct mapped_device *md)
438432a212cSMike Anderson {
439432a212cSMike Anderson 	return test_bit(DMF_DELETING, &md->flags);
440432a212cSMike Anderson }
441432a212cSMike Anderson 
442fe5f9f2cSAl Viro static int dm_blk_open(struct block_device *bdev, fmode_t mode)
4431da177e4SLinus Torvalds {
4441da177e4SLinus Torvalds 	struct mapped_device *md;
4451da177e4SLinus Torvalds 
446fba9f90eSJeff Mahoney 	spin_lock(&_minor_lock);
447fba9f90eSJeff Mahoney 
448fe5f9f2cSAl Viro 	md = bdev->bd_disk->private_data;
449fba9f90eSJeff Mahoney 	if (!md)
450fba9f90eSJeff Mahoney 		goto out;
451fba9f90eSJeff Mahoney 
4525c6bd75dSAlasdair G Kergon 	if (test_bit(DMF_FREEING, &md->flags) ||
453432a212cSMike Anderson 	    dm_deleting_md(md)) {
454fba9f90eSJeff Mahoney 		md = NULL;
455fba9f90eSJeff Mahoney 		goto out;
456fba9f90eSJeff Mahoney 	}
457fba9f90eSJeff Mahoney 
4581da177e4SLinus Torvalds 	dm_get(md);
4595c6bd75dSAlasdair G Kergon 	atomic_inc(&md->open_count);
460fba9f90eSJeff Mahoney out:
461fba9f90eSJeff Mahoney 	spin_unlock(&_minor_lock);
462fba9f90eSJeff Mahoney 
463fba9f90eSJeff Mahoney 	return md ? 0 : -ENXIO;
4641da177e4SLinus Torvalds }
4651da177e4SLinus Torvalds 
466db2a144bSAl Viro static void dm_blk_close(struct gendisk *disk, fmode_t mode)
4671da177e4SLinus Torvalds {
46863a4f065SMike Snitzer 	struct mapped_device *md;
4696e9624b8SArnd Bergmann 
4704a1aeb98SMilan Broz 	spin_lock(&_minor_lock);
4714a1aeb98SMilan Broz 
47263a4f065SMike Snitzer 	md = disk->private_data;
47363a4f065SMike Snitzer 	if (WARN_ON(!md))
47463a4f065SMike Snitzer 		goto out;
47563a4f065SMike Snitzer 
4762c140a24SMikulas Patocka 	if (atomic_dec_and_test(&md->open_count) &&
4772c140a24SMikulas Patocka 	    (test_bit(DMF_DEFERRED_REMOVE, &md->flags)))
478acfe0ad7SMikulas Patocka 		queue_work(deferred_remove_workqueue, &deferred_remove_work);
4792c140a24SMikulas Patocka 
4801da177e4SLinus Torvalds 	dm_put(md);
48163a4f065SMike Snitzer out:
4824a1aeb98SMilan Broz 	spin_unlock(&_minor_lock);
4831da177e4SLinus Torvalds }
4841da177e4SLinus Torvalds 
4855c6bd75dSAlasdair G Kergon int dm_open_count(struct mapped_device *md)
4865c6bd75dSAlasdair G Kergon {
4875c6bd75dSAlasdair G Kergon 	return atomic_read(&md->open_count);
4885c6bd75dSAlasdair G Kergon }
4895c6bd75dSAlasdair G Kergon 
4905c6bd75dSAlasdair G Kergon /*
4915c6bd75dSAlasdair G Kergon  * Guarantees nothing is using the device before it's deleted.
4925c6bd75dSAlasdair G Kergon  */
4932c140a24SMikulas Patocka int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred)
4945c6bd75dSAlasdair G Kergon {
4955c6bd75dSAlasdair G Kergon 	int r = 0;
4965c6bd75dSAlasdair G Kergon 
4975c6bd75dSAlasdair G Kergon 	spin_lock(&_minor_lock);
4985c6bd75dSAlasdair G Kergon 
4992c140a24SMikulas Patocka 	if (dm_open_count(md)) {
5005c6bd75dSAlasdair G Kergon 		r = -EBUSY;
5012c140a24SMikulas Patocka 		if (mark_deferred)
5022c140a24SMikulas Patocka 			set_bit(DMF_DEFERRED_REMOVE, &md->flags);
5032c140a24SMikulas Patocka 	} else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags))
5042c140a24SMikulas Patocka 		r = -EEXIST;
5055c6bd75dSAlasdair G Kergon 	else
5065c6bd75dSAlasdair G Kergon 		set_bit(DMF_DELETING, &md->flags);
5075c6bd75dSAlasdair G Kergon 
5085c6bd75dSAlasdair G Kergon 	spin_unlock(&_minor_lock);
5095c6bd75dSAlasdair G Kergon 
5105c6bd75dSAlasdair G Kergon 	return r;
5115c6bd75dSAlasdair G Kergon }
5125c6bd75dSAlasdair G Kergon 
5132c140a24SMikulas Patocka int dm_cancel_deferred_remove(struct mapped_device *md)
5142c140a24SMikulas Patocka {
5152c140a24SMikulas Patocka 	int r = 0;
5162c140a24SMikulas Patocka 
5172c140a24SMikulas Patocka 	spin_lock(&_minor_lock);
5182c140a24SMikulas Patocka 
5192c140a24SMikulas Patocka 	if (test_bit(DMF_DELETING, &md->flags))
5202c140a24SMikulas Patocka 		r = -EBUSY;
5212c140a24SMikulas Patocka 	else
5222c140a24SMikulas Patocka 		clear_bit(DMF_DEFERRED_REMOVE, &md->flags);
5232c140a24SMikulas Patocka 
5242c140a24SMikulas Patocka 	spin_unlock(&_minor_lock);
5252c140a24SMikulas Patocka 
5262c140a24SMikulas Patocka 	return r;
5272c140a24SMikulas Patocka }
5282c140a24SMikulas Patocka 
5292c140a24SMikulas Patocka static void do_deferred_remove(struct work_struct *w)
5302c140a24SMikulas Patocka {
5312c140a24SMikulas Patocka 	dm_deferred_remove();
5322c140a24SMikulas Patocka }
5332c140a24SMikulas Patocka 
534fd2ed4d2SMikulas Patocka sector_t dm_get_size(struct mapped_device *md)
535fd2ed4d2SMikulas Patocka {
536fd2ed4d2SMikulas Patocka 	return get_capacity(md->disk);
537fd2ed4d2SMikulas Patocka }
538fd2ed4d2SMikulas Patocka 
5399974fa2cSMike Snitzer struct request_queue *dm_get_md_queue(struct mapped_device *md)
5409974fa2cSMike Snitzer {
5419974fa2cSMike Snitzer 	return md->queue;
5429974fa2cSMike Snitzer }
5439974fa2cSMike Snitzer 
544fd2ed4d2SMikulas Patocka struct dm_stats *dm_get_stats(struct mapped_device *md)
545fd2ed4d2SMikulas Patocka {
546fd2ed4d2SMikulas Patocka 	return &md->stats;
547fd2ed4d2SMikulas Patocka }
548fd2ed4d2SMikulas Patocka 
5493ac51e74SDarrick J. Wong static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
5503ac51e74SDarrick J. Wong {
5513ac51e74SDarrick J. Wong 	struct mapped_device *md = bdev->bd_disk->private_data;
5523ac51e74SDarrick J. Wong 
5533ac51e74SDarrick J. Wong 	return dm_get_geometry(md, geo);
5543ac51e74SDarrick J. Wong }
5553ac51e74SDarrick J. Wong 
556fe5f9f2cSAl Viro static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
557aa129a22SMilan Broz 			unsigned int cmd, unsigned long arg)
558aa129a22SMilan Broz {
559fe5f9f2cSAl Viro 	struct mapped_device *md = bdev->bd_disk->private_data;
56083d5e5b0SMikulas Patocka 	int srcu_idx;
5616c182cd8SHannes Reinecke 	struct dm_table *map;
562aa129a22SMilan Broz 	struct dm_target *tgt;
563aa129a22SMilan Broz 	int r = -ENOTTY;
564aa129a22SMilan Broz 
5656c182cd8SHannes Reinecke retry:
56683d5e5b0SMikulas Patocka 	map = dm_get_live_table(md, &srcu_idx);
56783d5e5b0SMikulas Patocka 
568aa129a22SMilan Broz 	if (!map || !dm_table_get_size(map))
569aa129a22SMilan Broz 		goto out;
570aa129a22SMilan Broz 
571aa129a22SMilan Broz 	/* We only support devices that have a single target */
572aa129a22SMilan Broz 	if (dm_table_get_num_targets(map) != 1)
573aa129a22SMilan Broz 		goto out;
574aa129a22SMilan Broz 
575aa129a22SMilan Broz 	tgt = dm_table_get_target(map, 0);
5764d341d82SMike Snitzer 	if (!tgt->type->ioctl)
5774d341d82SMike Snitzer 		goto out;
578aa129a22SMilan Broz 
5794f186f8bSKiyoshi Ueda 	if (dm_suspended_md(md)) {
580aa129a22SMilan Broz 		r = -EAGAIN;
581aa129a22SMilan Broz 		goto out;
582aa129a22SMilan Broz 	}
583aa129a22SMilan Broz 
584647b3d00SAl Viro 	r = tgt->type->ioctl(tgt, cmd, arg);
585aa129a22SMilan Broz 
586aa129a22SMilan Broz out:
58783d5e5b0SMikulas Patocka 	dm_put_live_table(md, srcu_idx);
588aa129a22SMilan Broz 
5896c182cd8SHannes Reinecke 	if (r == -ENOTCONN) {
5906c182cd8SHannes Reinecke 		msleep(10);
5916c182cd8SHannes Reinecke 		goto retry;
5926c182cd8SHannes Reinecke 	}
5936c182cd8SHannes Reinecke 
594aa129a22SMilan Broz 	return r;
595aa129a22SMilan Broz }
596aa129a22SMilan Broz 
597028867acSAlasdair G Kergon static struct dm_io *alloc_io(struct mapped_device *md)
5981da177e4SLinus Torvalds {
5991da177e4SLinus Torvalds 	return mempool_alloc(md->io_pool, GFP_NOIO);
6001da177e4SLinus Torvalds }
6011da177e4SLinus Torvalds 
602028867acSAlasdair G Kergon static void free_io(struct mapped_device *md, struct dm_io *io)
6031da177e4SLinus Torvalds {
6041da177e4SLinus Torvalds 	mempool_free(io, md->io_pool);
6051da177e4SLinus Torvalds }
6061da177e4SLinus Torvalds 
607028867acSAlasdair G Kergon static void free_tio(struct mapped_device *md, struct dm_target_io *tio)
6081da177e4SLinus Torvalds {
609dba14160SMikulas Patocka 	bio_put(&tio->clone);
6101da177e4SLinus Torvalds }
6111da177e4SLinus Torvalds 
61208885643SKiyoshi Ueda static struct dm_rq_target_io *alloc_rq_tio(struct mapped_device *md,
61308885643SKiyoshi Ueda 					    gfp_t gfp_mask)
614cec47e3dSKiyoshi Ueda {
6155f015204SJun'ichi Nomura 	return mempool_alloc(md->io_pool, gfp_mask);
616cec47e3dSKiyoshi Ueda }
617cec47e3dSKiyoshi Ueda 
618cec47e3dSKiyoshi Ueda static void free_rq_tio(struct dm_rq_target_io *tio)
619cec47e3dSKiyoshi Ueda {
6205f015204SJun'ichi Nomura 	mempool_free(tio, tio->md->io_pool);
621cec47e3dSKiyoshi Ueda }
622cec47e3dSKiyoshi Ueda 
6231ae49ea2SMike Snitzer static struct request *alloc_clone_request(struct mapped_device *md,
6241ae49ea2SMike Snitzer 					   gfp_t gfp_mask)
6251ae49ea2SMike Snitzer {
6261ae49ea2SMike Snitzer 	return mempool_alloc(md->rq_pool, gfp_mask);
6271ae49ea2SMike Snitzer }
6281ae49ea2SMike Snitzer 
6291ae49ea2SMike Snitzer static void free_clone_request(struct mapped_device *md, struct request *rq)
6301ae49ea2SMike Snitzer {
6311ae49ea2SMike Snitzer 	mempool_free(rq, md->rq_pool);
6321ae49ea2SMike Snitzer }
6331ae49ea2SMike Snitzer 
63490abb8c4SKiyoshi Ueda static int md_in_flight(struct mapped_device *md)
63590abb8c4SKiyoshi Ueda {
63690abb8c4SKiyoshi Ueda 	return atomic_read(&md->pending[READ]) +
63790abb8c4SKiyoshi Ueda 	       atomic_read(&md->pending[WRITE]);
63890abb8c4SKiyoshi Ueda }
63990abb8c4SKiyoshi Ueda 
6403eaf840eSJun'ichi "Nick" Nomura static void start_io_acct(struct dm_io *io)
6413eaf840eSJun'ichi "Nick" Nomura {
6423eaf840eSJun'ichi "Nick" Nomura 	struct mapped_device *md = io->md;
643fd2ed4d2SMikulas Patocka 	struct bio *bio = io->bio;
644c9959059STejun Heo 	int cpu;
645fd2ed4d2SMikulas Patocka 	int rw = bio_data_dir(bio);
6463eaf840eSJun'ichi "Nick" Nomura 
6473eaf840eSJun'ichi "Nick" Nomura 	io->start_time = jiffies;
6483eaf840eSJun'ichi "Nick" Nomura 
649074a7acaSTejun Heo 	cpu = part_stat_lock();
650074a7acaSTejun Heo 	part_round_stats(cpu, &dm_disk(md)->part0);
651074a7acaSTejun Heo 	part_stat_unlock();
6521e9bb880SShaohua Li 	atomic_set(&dm_disk(md)->part0.in_flight[rw],
6531e9bb880SShaohua Li 		atomic_inc_return(&md->pending[rw]));
654fd2ed4d2SMikulas Patocka 
655fd2ed4d2SMikulas Patocka 	if (unlikely(dm_stats_used(&md->stats)))
6564f024f37SKent Overstreet 		dm_stats_account_io(&md->stats, bio->bi_rw, bio->bi_iter.bi_sector,
657fd2ed4d2SMikulas Patocka 				    bio_sectors(bio), false, 0, &io->stats_aux);
6583eaf840eSJun'ichi "Nick" Nomura }
6593eaf840eSJun'ichi "Nick" Nomura 
660d221d2e7SMikulas Patocka static void end_io_acct(struct dm_io *io)
6613eaf840eSJun'ichi "Nick" Nomura {
6623eaf840eSJun'ichi "Nick" Nomura 	struct mapped_device *md = io->md;
6633eaf840eSJun'ichi "Nick" Nomura 	struct bio *bio = io->bio;
6643eaf840eSJun'ichi "Nick" Nomura 	unsigned long duration = jiffies - io->start_time;
66518c0b223SGu Zheng 	int pending;
6663eaf840eSJun'ichi "Nick" Nomura 	int rw = bio_data_dir(bio);
6673eaf840eSJun'ichi "Nick" Nomura 
66818c0b223SGu Zheng 	generic_end_io_acct(rw, &dm_disk(md)->part0, io->start_time);
6693eaf840eSJun'ichi "Nick" Nomura 
670fd2ed4d2SMikulas Patocka 	if (unlikely(dm_stats_used(&md->stats)))
6714f024f37SKent Overstreet 		dm_stats_account_io(&md->stats, bio->bi_rw, bio->bi_iter.bi_sector,
672fd2ed4d2SMikulas Patocka 				    bio_sectors(bio), true, duration, &io->stats_aux);
673fd2ed4d2SMikulas Patocka 
674af7e466aSMikulas Patocka 	/*
675af7e466aSMikulas Patocka 	 * After this is decremented the bio must not be touched if it is
676d87f4c14STejun Heo 	 * a flush.
677af7e466aSMikulas Patocka 	 */
6781e9bb880SShaohua Li 	pending = atomic_dec_return(&md->pending[rw]);
6791e9bb880SShaohua Li 	atomic_set(&dm_disk(md)->part0.in_flight[rw], pending);
680316d315bSNikanth Karthikesan 	pending += atomic_read(&md->pending[rw^0x1]);
6813eaf840eSJun'ichi "Nick" Nomura 
682d221d2e7SMikulas Patocka 	/* nudge anyone waiting on suspend queue */
683d221d2e7SMikulas Patocka 	if (!pending)
684d221d2e7SMikulas Patocka 		wake_up(&md->wait);
6853eaf840eSJun'ichi "Nick" Nomura }
6863eaf840eSJun'ichi "Nick" Nomura 
6871da177e4SLinus Torvalds /*
6881da177e4SLinus Torvalds  * Add the bio to the list of deferred io.
6891da177e4SLinus Torvalds  */
69092c63902SMikulas Patocka static void queue_io(struct mapped_device *md, struct bio *bio)
6911da177e4SLinus Torvalds {
69205447420SKiyoshi Ueda 	unsigned long flags;
6931da177e4SLinus Torvalds 
69405447420SKiyoshi Ueda 	spin_lock_irqsave(&md->deferred_lock, flags);
6951da177e4SLinus Torvalds 	bio_list_add(&md->deferred, bio);
69605447420SKiyoshi Ueda 	spin_unlock_irqrestore(&md->deferred_lock, flags);
69792c63902SMikulas Patocka 	queue_work(md->wq, &md->work);
6981da177e4SLinus Torvalds }
6991da177e4SLinus Torvalds 
7001da177e4SLinus Torvalds /*
7011da177e4SLinus Torvalds  * Everyone (including functions in this file), should use this
7021da177e4SLinus Torvalds  * function to access the md->map field, and make sure they call
70383d5e5b0SMikulas Patocka  * dm_put_live_table() when finished.
7041da177e4SLinus Torvalds  */
70583d5e5b0SMikulas Patocka struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier)
7061da177e4SLinus Torvalds {
70783d5e5b0SMikulas Patocka 	*srcu_idx = srcu_read_lock(&md->io_barrier);
7081da177e4SLinus Torvalds 
70983d5e5b0SMikulas Patocka 	return srcu_dereference(md->map, &md->io_barrier);
71083d5e5b0SMikulas Patocka }
7111da177e4SLinus Torvalds 
71283d5e5b0SMikulas Patocka void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier)
71383d5e5b0SMikulas Patocka {
71483d5e5b0SMikulas Patocka 	srcu_read_unlock(&md->io_barrier, srcu_idx);
71583d5e5b0SMikulas Patocka }
71683d5e5b0SMikulas Patocka 
71783d5e5b0SMikulas Patocka void dm_sync_table(struct mapped_device *md)
71883d5e5b0SMikulas Patocka {
71983d5e5b0SMikulas Patocka 	synchronize_srcu(&md->io_barrier);
72083d5e5b0SMikulas Patocka 	synchronize_rcu_expedited();
72183d5e5b0SMikulas Patocka }
72283d5e5b0SMikulas Patocka 
72383d5e5b0SMikulas Patocka /*
72483d5e5b0SMikulas Patocka  * A fast alternative to dm_get_live_table/dm_put_live_table.
72583d5e5b0SMikulas Patocka  * The caller must not block between these two functions.
72683d5e5b0SMikulas Patocka  */
72783d5e5b0SMikulas Patocka static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU)
72883d5e5b0SMikulas Patocka {
72983d5e5b0SMikulas Patocka 	rcu_read_lock();
73083d5e5b0SMikulas Patocka 	return rcu_dereference(md->map);
73183d5e5b0SMikulas Patocka }
73283d5e5b0SMikulas Patocka 
73383d5e5b0SMikulas Patocka static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU)
73483d5e5b0SMikulas Patocka {
73583d5e5b0SMikulas Patocka 	rcu_read_unlock();
7361da177e4SLinus Torvalds }
7371da177e4SLinus Torvalds 
7383ac51e74SDarrick J. Wong /*
73986f1152bSBenjamin Marzinski  * Open a table device so we can use it as a map destination.
74086f1152bSBenjamin Marzinski  */
74186f1152bSBenjamin Marzinski static int open_table_device(struct table_device *td, dev_t dev,
74286f1152bSBenjamin Marzinski 			     struct mapped_device *md)
74386f1152bSBenjamin Marzinski {
74486f1152bSBenjamin Marzinski 	static char *_claim_ptr = "I belong to device-mapper";
74586f1152bSBenjamin Marzinski 	struct block_device *bdev;
74686f1152bSBenjamin Marzinski 
74786f1152bSBenjamin Marzinski 	int r;
74886f1152bSBenjamin Marzinski 
74986f1152bSBenjamin Marzinski 	BUG_ON(td->dm_dev.bdev);
75086f1152bSBenjamin Marzinski 
75186f1152bSBenjamin Marzinski 	bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _claim_ptr);
75286f1152bSBenjamin Marzinski 	if (IS_ERR(bdev))
75386f1152bSBenjamin Marzinski 		return PTR_ERR(bdev);
75486f1152bSBenjamin Marzinski 
75586f1152bSBenjamin Marzinski 	r = bd_link_disk_holder(bdev, dm_disk(md));
75686f1152bSBenjamin Marzinski 	if (r) {
75786f1152bSBenjamin Marzinski 		blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL);
75886f1152bSBenjamin Marzinski 		return r;
75986f1152bSBenjamin Marzinski 	}
76086f1152bSBenjamin Marzinski 
76186f1152bSBenjamin Marzinski 	td->dm_dev.bdev = bdev;
76286f1152bSBenjamin Marzinski 	return 0;
76386f1152bSBenjamin Marzinski }
76486f1152bSBenjamin Marzinski 
76586f1152bSBenjamin Marzinski /*
76686f1152bSBenjamin Marzinski  * Close a table device that we've been using.
76786f1152bSBenjamin Marzinski  */
76886f1152bSBenjamin Marzinski static void close_table_device(struct table_device *td, struct mapped_device *md)
76986f1152bSBenjamin Marzinski {
77086f1152bSBenjamin Marzinski 	if (!td->dm_dev.bdev)
77186f1152bSBenjamin Marzinski 		return;
77286f1152bSBenjamin Marzinski 
77386f1152bSBenjamin Marzinski 	bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md));
77486f1152bSBenjamin Marzinski 	blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL);
77586f1152bSBenjamin Marzinski 	td->dm_dev.bdev = NULL;
77686f1152bSBenjamin Marzinski }
77786f1152bSBenjamin Marzinski 
77886f1152bSBenjamin Marzinski static struct table_device *find_table_device(struct list_head *l, dev_t dev,
77986f1152bSBenjamin Marzinski 					      fmode_t mode) {
78086f1152bSBenjamin Marzinski 	struct table_device *td;
78186f1152bSBenjamin Marzinski 
78286f1152bSBenjamin Marzinski 	list_for_each_entry(td, l, list)
78386f1152bSBenjamin Marzinski 		if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode)
78486f1152bSBenjamin Marzinski 			return td;
78586f1152bSBenjamin Marzinski 
78686f1152bSBenjamin Marzinski 	return NULL;
78786f1152bSBenjamin Marzinski }
78886f1152bSBenjamin Marzinski 
78986f1152bSBenjamin Marzinski int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode,
79086f1152bSBenjamin Marzinski 			struct dm_dev **result) {
79186f1152bSBenjamin Marzinski 	int r;
79286f1152bSBenjamin Marzinski 	struct table_device *td;
79386f1152bSBenjamin Marzinski 
79486f1152bSBenjamin Marzinski 	mutex_lock(&md->table_devices_lock);
79586f1152bSBenjamin Marzinski 	td = find_table_device(&md->table_devices, dev, mode);
79686f1152bSBenjamin Marzinski 	if (!td) {
79786f1152bSBenjamin Marzinski 		td = kmalloc(sizeof(*td), GFP_KERNEL);
79886f1152bSBenjamin Marzinski 		if (!td) {
79986f1152bSBenjamin Marzinski 			mutex_unlock(&md->table_devices_lock);
80086f1152bSBenjamin Marzinski 			return -ENOMEM;
80186f1152bSBenjamin Marzinski 		}
80286f1152bSBenjamin Marzinski 
80386f1152bSBenjamin Marzinski 		td->dm_dev.mode = mode;
80486f1152bSBenjamin Marzinski 		td->dm_dev.bdev = NULL;
80586f1152bSBenjamin Marzinski 
80686f1152bSBenjamin Marzinski 		if ((r = open_table_device(td, dev, md))) {
80786f1152bSBenjamin Marzinski 			mutex_unlock(&md->table_devices_lock);
80886f1152bSBenjamin Marzinski 			kfree(td);
80986f1152bSBenjamin Marzinski 			return r;
81086f1152bSBenjamin Marzinski 		}
81186f1152bSBenjamin Marzinski 
81286f1152bSBenjamin Marzinski 		format_dev_t(td->dm_dev.name, dev);
81386f1152bSBenjamin Marzinski 
81486f1152bSBenjamin Marzinski 		atomic_set(&td->count, 0);
81586f1152bSBenjamin Marzinski 		list_add(&td->list, &md->table_devices);
81686f1152bSBenjamin Marzinski 	}
81786f1152bSBenjamin Marzinski 	atomic_inc(&td->count);
81886f1152bSBenjamin Marzinski 	mutex_unlock(&md->table_devices_lock);
81986f1152bSBenjamin Marzinski 
82086f1152bSBenjamin Marzinski 	*result = &td->dm_dev;
82186f1152bSBenjamin Marzinski 	return 0;
82286f1152bSBenjamin Marzinski }
82386f1152bSBenjamin Marzinski EXPORT_SYMBOL_GPL(dm_get_table_device);
82486f1152bSBenjamin Marzinski 
82586f1152bSBenjamin Marzinski void dm_put_table_device(struct mapped_device *md, struct dm_dev *d)
82686f1152bSBenjamin Marzinski {
82786f1152bSBenjamin Marzinski 	struct table_device *td = container_of(d, struct table_device, dm_dev);
82886f1152bSBenjamin Marzinski 
82986f1152bSBenjamin Marzinski 	mutex_lock(&md->table_devices_lock);
83086f1152bSBenjamin Marzinski 	if (atomic_dec_and_test(&td->count)) {
83186f1152bSBenjamin Marzinski 		close_table_device(td, md);
83286f1152bSBenjamin Marzinski 		list_del(&td->list);
83386f1152bSBenjamin Marzinski 		kfree(td);
83486f1152bSBenjamin Marzinski 	}
83586f1152bSBenjamin Marzinski 	mutex_unlock(&md->table_devices_lock);
83686f1152bSBenjamin Marzinski }
83786f1152bSBenjamin Marzinski EXPORT_SYMBOL(dm_put_table_device);
83886f1152bSBenjamin Marzinski 
83986f1152bSBenjamin Marzinski static void free_table_devices(struct list_head *devices)
84086f1152bSBenjamin Marzinski {
84186f1152bSBenjamin Marzinski 	struct list_head *tmp, *next;
84286f1152bSBenjamin Marzinski 
84386f1152bSBenjamin Marzinski 	list_for_each_safe(tmp, next, devices) {
84486f1152bSBenjamin Marzinski 		struct table_device *td = list_entry(tmp, struct table_device, list);
84586f1152bSBenjamin Marzinski 
84686f1152bSBenjamin Marzinski 		DMWARN("dm_destroy: %s still exists with %d references",
84786f1152bSBenjamin Marzinski 		       td->dm_dev.name, atomic_read(&td->count));
84886f1152bSBenjamin Marzinski 		kfree(td);
84986f1152bSBenjamin Marzinski 	}
85086f1152bSBenjamin Marzinski }
85186f1152bSBenjamin Marzinski 
85286f1152bSBenjamin Marzinski /*
8533ac51e74SDarrick J. Wong  * Get the geometry associated with a dm device
8543ac51e74SDarrick J. Wong  */
8553ac51e74SDarrick J. Wong int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
8563ac51e74SDarrick J. Wong {
8573ac51e74SDarrick J. Wong 	*geo = md->geometry;
8583ac51e74SDarrick J. Wong 
8593ac51e74SDarrick J. Wong 	return 0;
8603ac51e74SDarrick J. Wong }
8613ac51e74SDarrick J. Wong 
8623ac51e74SDarrick J. Wong /*
8633ac51e74SDarrick J. Wong  * Set the geometry of a device.
8643ac51e74SDarrick J. Wong  */
8653ac51e74SDarrick J. Wong int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
8663ac51e74SDarrick J. Wong {
8673ac51e74SDarrick J. Wong 	sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
8683ac51e74SDarrick J. Wong 
8693ac51e74SDarrick J. Wong 	if (geo->start > sz) {
8703ac51e74SDarrick J. Wong 		DMWARN("Start sector is beyond the geometry limits.");
8713ac51e74SDarrick J. Wong 		return -EINVAL;
8723ac51e74SDarrick J. Wong 	}
8733ac51e74SDarrick J. Wong 
8743ac51e74SDarrick J. Wong 	md->geometry = *geo;
8753ac51e74SDarrick J. Wong 
8763ac51e74SDarrick J. Wong 	return 0;
8773ac51e74SDarrick J. Wong }
8783ac51e74SDarrick J. Wong 
8791da177e4SLinus Torvalds /*-----------------------------------------------------------------
8801da177e4SLinus Torvalds  * CRUD START:
8811da177e4SLinus Torvalds  *   A more elegant soln is in the works that uses the queue
8821da177e4SLinus Torvalds  *   merge fn, unfortunately there are a couple of changes to
8831da177e4SLinus Torvalds  *   the block layer that I want to make for this.  So in the
8841da177e4SLinus Torvalds  *   interests of getting something for people to use I give
8851da177e4SLinus Torvalds  *   you this clearly demarcated crap.
8861da177e4SLinus Torvalds  *---------------------------------------------------------------*/
8871da177e4SLinus Torvalds 
8882e93ccc1SKiyoshi Ueda static int __noflush_suspending(struct mapped_device *md)
8892e93ccc1SKiyoshi Ueda {
8902e93ccc1SKiyoshi Ueda 	return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
8912e93ccc1SKiyoshi Ueda }
8922e93ccc1SKiyoshi Ueda 
8931da177e4SLinus Torvalds /*
8941da177e4SLinus Torvalds  * Decrements the number of outstanding ios that a bio has been
8951da177e4SLinus Torvalds  * cloned into, completing the original io if necc.
8961da177e4SLinus Torvalds  */
897858119e1SArjan van de Ven static void dec_pending(struct dm_io *io, int error)
8981da177e4SLinus Torvalds {
8992e93ccc1SKiyoshi Ueda 	unsigned long flags;
900b35f8caaSMilan Broz 	int io_error;
901b35f8caaSMilan Broz 	struct bio *bio;
902b35f8caaSMilan Broz 	struct mapped_device *md = io->md;
9032e93ccc1SKiyoshi Ueda 
9042e93ccc1SKiyoshi Ueda 	/* Push-back supersedes any I/O errors */
905f88fb981SKiyoshi Ueda 	if (unlikely(error)) {
906f88fb981SKiyoshi Ueda 		spin_lock_irqsave(&io->endio_lock, flags);
907f88fb981SKiyoshi Ueda 		if (!(io->error > 0 && __noflush_suspending(md)))
9081da177e4SLinus Torvalds 			io->error = error;
909f88fb981SKiyoshi Ueda 		spin_unlock_irqrestore(&io->endio_lock, flags);
910f88fb981SKiyoshi Ueda 	}
9111da177e4SLinus Torvalds 
9121da177e4SLinus Torvalds 	if (atomic_dec_and_test(&io->io_count)) {
9132e93ccc1SKiyoshi Ueda 		if (io->error == DM_ENDIO_REQUEUE) {
9142e93ccc1SKiyoshi Ueda 			/*
9152e93ccc1SKiyoshi Ueda 			 * Target requested pushing back the I/O.
9162e93ccc1SKiyoshi Ueda 			 */
917022c2611SMikulas Patocka 			spin_lock_irqsave(&md->deferred_lock, flags);
9186a8736d1STejun Heo 			if (__noflush_suspending(md))
9196a8736d1STejun Heo 				bio_list_add_head(&md->deferred, io->bio);
9206a8736d1STejun Heo 			else
9212e93ccc1SKiyoshi Ueda 				/* noflush suspend was interrupted. */
9222e93ccc1SKiyoshi Ueda 				io->error = -EIO;
923022c2611SMikulas Patocka 			spin_unlock_irqrestore(&md->deferred_lock, flags);
9242e93ccc1SKiyoshi Ueda 		}
9252e93ccc1SKiyoshi Ueda 
926b35f8caaSMilan Broz 		io_error = io->error;
927b35f8caaSMilan Broz 		bio = io->bio;
928af7e466aSMikulas Patocka 		end_io_acct(io);
929a97f925aSMikulas Patocka 		free_io(md, io);
9301da177e4SLinus Torvalds 
9316a8736d1STejun Heo 		if (io_error == DM_ENDIO_REQUEUE)
9326a8736d1STejun Heo 			return;
9336a8736d1STejun Heo 
9344f024f37SKent Overstreet 		if ((bio->bi_rw & REQ_FLUSH) && bio->bi_iter.bi_size) {
9351da177e4SLinus Torvalds 			/*
9366a8736d1STejun Heo 			 * Preflush done for flush with data, reissue
9376a8736d1STejun Heo 			 * without REQ_FLUSH.
9381da177e4SLinus Torvalds 			 */
9396a8736d1STejun Heo 			bio->bi_rw &= ~REQ_FLUSH;
9406a8736d1STejun Heo 			queue_io(md, bio);
9415f3ea37cSArnaldo Carvalho de Melo 		} else {
942b372d360SMike Snitzer 			/* done with normal IO or empty flush */
9430a82a8d1SLinus Torvalds 			trace_block_bio_complete(md->queue, bio, io_error);
944b35f8caaSMilan Broz 			bio_endio(bio, io_error);
9452e93ccc1SKiyoshi Ueda 		}
9461da177e4SLinus Torvalds 	}
947af7e466aSMikulas Patocka }
9481da177e4SLinus Torvalds 
9497eee4ae2SMike Snitzer static void disable_write_same(struct mapped_device *md)
9507eee4ae2SMike Snitzer {
9517eee4ae2SMike Snitzer 	struct queue_limits *limits = dm_get_queue_limits(md);
9527eee4ae2SMike Snitzer 
9537eee4ae2SMike Snitzer 	/* device doesn't really support WRITE SAME, disable it */
9547eee4ae2SMike Snitzer 	limits->max_write_same_sectors = 0;
9557eee4ae2SMike Snitzer }
9567eee4ae2SMike Snitzer 
9576712ecf8SNeilBrown static void clone_endio(struct bio *bio, int error)
9581da177e4SLinus Torvalds {
9595164beceSzhendong chen 	int r = error;
960bfc6d41cSMikulas Patocka 	struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
961b35f8caaSMilan Broz 	struct dm_io *io = tio->io;
9629faf400fSStefan Bader 	struct mapped_device *md = tio->io->md;
9631da177e4SLinus Torvalds 	dm_endio_fn endio = tio->ti->type->end_io;
9641da177e4SLinus Torvalds 
9651da177e4SLinus Torvalds 	if (!bio_flagged(bio, BIO_UPTODATE) && !error)
9661da177e4SLinus Torvalds 		error = -EIO;
9671da177e4SLinus Torvalds 
9681da177e4SLinus Torvalds 	if (endio) {
9697de3ee57SMikulas Patocka 		r = endio(tio->ti, bio, error);
9702e93ccc1SKiyoshi Ueda 		if (r < 0 || r == DM_ENDIO_REQUEUE)
9712e93ccc1SKiyoshi Ueda 			/*
9722e93ccc1SKiyoshi Ueda 			 * error and requeue request are handled
9732e93ccc1SKiyoshi Ueda 			 * in dec_pending().
9742e93ccc1SKiyoshi Ueda 			 */
9751da177e4SLinus Torvalds 			error = r;
97645cbcd79SKiyoshi Ueda 		else if (r == DM_ENDIO_INCOMPLETE)
97745cbcd79SKiyoshi Ueda 			/* The target will handle the io */
9786712ecf8SNeilBrown 			return;
97945cbcd79SKiyoshi Ueda 		else if (r) {
98045cbcd79SKiyoshi Ueda 			DMWARN("unimplemented target endio return value: %d", r);
98145cbcd79SKiyoshi Ueda 			BUG();
98245cbcd79SKiyoshi Ueda 		}
9831da177e4SLinus Torvalds 	}
9841da177e4SLinus Torvalds 
9857eee4ae2SMike Snitzer 	if (unlikely(r == -EREMOTEIO && (bio->bi_rw & REQ_WRITE_SAME) &&
9867eee4ae2SMike Snitzer 		     !bdev_get_queue(bio->bi_bdev)->limits.max_write_same_sectors))
9877eee4ae2SMike Snitzer 		disable_write_same(md);
9887eee4ae2SMike Snitzer 
9899faf400fSStefan Bader 	free_tio(md, tio);
990b35f8caaSMilan Broz 	dec_pending(io, error);
9911da177e4SLinus Torvalds }
9921da177e4SLinus Torvalds 
993bfebd1cdSMike Snitzer static struct dm_rq_target_io *tio_from_request(struct request *rq)
994bfebd1cdSMike Snitzer {
995bfebd1cdSMike Snitzer 	return (rq->q->mq_ops ? blk_mq_rq_to_pdu(rq) : rq->special);
996bfebd1cdSMike Snitzer }
997bfebd1cdSMike Snitzer 
998cec47e3dSKiyoshi Ueda /*
999cec47e3dSKiyoshi Ueda  * Don't touch any member of the md after calling this function because
1000cec47e3dSKiyoshi Ueda  * the md may be freed in dm_put() at the end of this function.
1001cec47e3dSKiyoshi Ueda  * Or do dm_get() before calling this function and dm_put() later.
1002cec47e3dSKiyoshi Ueda  */
1003466d89a6SKeith Busch static void rq_completed(struct mapped_device *md, int rw, bool run_queue)
1004cec47e3dSKiyoshi Ueda {
10059a0e609eSMike Snitzer 	int nr_requests_pending;
10069a0e609eSMike Snitzer 
1007b4324feeSKiyoshi Ueda 	atomic_dec(&md->pending[rw]);
1008cec47e3dSKiyoshi Ueda 
1009cec47e3dSKiyoshi Ueda 	/* nudge anyone waiting on suspend queue */
10109a0e609eSMike Snitzer 	nr_requests_pending = md_in_flight(md);
10119a0e609eSMike Snitzer 	if (!nr_requests_pending)
1012cec47e3dSKiyoshi Ueda 		wake_up(&md->wait);
1013cec47e3dSKiyoshi Ueda 
1014a8c32a5cSJens Axboe 	/*
1015a8c32a5cSJens Axboe 	 * Run this off this callpath, as drivers could invoke end_io while
1016a8c32a5cSJens Axboe 	 * inside their request_fn (and holding the queue lock). Calling
1017a8c32a5cSJens Axboe 	 * back into ->request_fn() could deadlock attempting to grab the
1018a8c32a5cSJens Axboe 	 * queue lock again.
1019a8c32a5cSJens Axboe 	 */
10209a0e609eSMike Snitzer 	if (run_queue) {
1021bfebd1cdSMike Snitzer 		if (md->queue->mq_ops)
1022bfebd1cdSMike Snitzer 			blk_mq_run_hw_queues(md->queue, true);
1023bfebd1cdSMike Snitzer 		else if (!nr_requests_pending ||
10249a0e609eSMike Snitzer 			 (nr_requests_pending >= md->queue->nr_congestion_on))
1025a8c32a5cSJens Axboe 			blk_run_queue_async(md->queue);
10269a0e609eSMike Snitzer 	}
1027cec47e3dSKiyoshi Ueda 
1028cec47e3dSKiyoshi Ueda 	/*
1029cec47e3dSKiyoshi Ueda 	 * dm_put() must be at the end of this function. See the comment above
1030cec47e3dSKiyoshi Ueda 	 */
1031cec47e3dSKiyoshi Ueda 	dm_put(md);
1032cec47e3dSKiyoshi Ueda }
1033cec47e3dSKiyoshi Ueda 
1034aa6df8ddSMike Snitzer static void free_rq_clone(struct request *clone, bool must_be_mapped)
1035a77e28c7SKiyoshi Ueda {
1036a77e28c7SKiyoshi Ueda 	struct dm_rq_target_io *tio = clone->end_io_data;
1037bfebd1cdSMike Snitzer 	struct mapped_device *md = tio->md;
1038a77e28c7SKiyoshi Ueda 
1039aa6df8ddSMike Snitzer 	WARN_ON_ONCE(must_be_mapped && !clone->q);
1040aa6df8ddSMike Snitzer 
1041aa6df8ddSMike Snitzer 	if (md->type == DM_TYPE_MQ_REQUEST_BASED)
1042aa6df8ddSMike Snitzer 		/* stacked on blk-mq queue(s) */
1043e5863d9aSMike Snitzer 		tio->ti->type->release_clone_rq(clone);
104402233342SMike Snitzer 	else if (!md->queue->mq_ops)
104502233342SMike Snitzer 		/* request_fn queue stacked on request_fn queue(s) */
1046bfebd1cdSMike Snitzer 		free_clone_request(md, clone);
1047aa6df8ddSMike Snitzer 	/*
1048aa6df8ddSMike Snitzer 	 * NOTE: for the blk-mq queue stacked on request_fn queue(s) case:
1049aa6df8ddSMike Snitzer 	 * no need to call free_clone_request() because we leverage blk-mq by
1050aa6df8ddSMike Snitzer 	 * allocating the clone at the end of the blk-mq pdu (see: clone_rq)
1051aa6df8ddSMike Snitzer 	 */
1052bfebd1cdSMike Snitzer 
1053bfebd1cdSMike Snitzer 	if (!md->queue->mq_ops)
1054a77e28c7SKiyoshi Ueda 		free_rq_tio(tio);
1055a77e28c7SKiyoshi Ueda }
1056a77e28c7SKiyoshi Ueda 
1057980691e5SKiyoshi Ueda /*
1058980691e5SKiyoshi Ueda  * Complete the clone and the original request.
1059466d89a6SKeith Busch  * Must be called without clone's queue lock held,
1060466d89a6SKeith Busch  * see end_clone_request() for more details.
1061980691e5SKiyoshi Ueda  */
1062980691e5SKiyoshi Ueda static void dm_end_request(struct request *clone, int error)
1063980691e5SKiyoshi Ueda {
1064980691e5SKiyoshi Ueda 	int rw = rq_data_dir(clone);
1065980691e5SKiyoshi Ueda 	struct dm_rq_target_io *tio = clone->end_io_data;
1066980691e5SKiyoshi Ueda 	struct mapped_device *md = tio->md;
1067980691e5SKiyoshi Ueda 	struct request *rq = tio->orig;
1068980691e5SKiyoshi Ueda 
106929e4013dSTejun Heo 	if (rq->cmd_type == REQ_TYPE_BLOCK_PC) {
1070980691e5SKiyoshi Ueda 		rq->errors = clone->errors;
1071980691e5SKiyoshi Ueda 		rq->resid_len = clone->resid_len;
1072980691e5SKiyoshi Ueda 
1073980691e5SKiyoshi Ueda 		if (rq->sense)
1074980691e5SKiyoshi Ueda 			/*
1075980691e5SKiyoshi Ueda 			 * We are using the sense buffer of the original
1076980691e5SKiyoshi Ueda 			 * request.
1077980691e5SKiyoshi Ueda 			 * So setting the length of the sense data is enough.
1078980691e5SKiyoshi Ueda 			 */
1079980691e5SKiyoshi Ueda 			rq->sense_len = clone->sense_len;
1080980691e5SKiyoshi Ueda 	}
1081980691e5SKiyoshi Ueda 
1082aa6df8ddSMike Snitzer 	free_rq_clone(clone, true);
1083bfebd1cdSMike Snitzer 	if (!rq->q->mq_ops)
1084980691e5SKiyoshi Ueda 		blk_end_request_all(rq, error);
1085bfebd1cdSMike Snitzer 	else
1086bfebd1cdSMike Snitzer 		blk_mq_end_request(rq, error);
108729e4013dSTejun Heo 	rq_completed(md, rw, true);
1088980691e5SKiyoshi Ueda }
1089980691e5SKiyoshi Ueda 
1090cec47e3dSKiyoshi Ueda static void dm_unprep_request(struct request *rq)
1091cec47e3dSKiyoshi Ueda {
1092bfebd1cdSMike Snitzer 	struct dm_rq_target_io *tio = tio_from_request(rq);
1093466d89a6SKeith Busch 	struct request *clone = tio->clone;
1094cec47e3dSKiyoshi Ueda 
1095bfebd1cdSMike Snitzer 	if (!rq->q->mq_ops) {
1096cec47e3dSKiyoshi Ueda 		rq->special = NULL;
1097cec47e3dSKiyoshi Ueda 		rq->cmd_flags &= ~REQ_DONTPREP;
1098bfebd1cdSMike Snitzer 	}
1099cec47e3dSKiyoshi Ueda 
1100e5863d9aSMike Snitzer 	if (clone)
1101aa6df8ddSMike Snitzer 		free_rq_clone(clone, false);
1102cec47e3dSKiyoshi Ueda }
1103cec47e3dSKiyoshi Ueda 
1104cec47e3dSKiyoshi Ueda /*
1105cec47e3dSKiyoshi Ueda  * Requeue the original request of a clone.
1106cec47e3dSKiyoshi Ueda  */
1107bfebd1cdSMike Snitzer static void old_requeue_request(struct request *rq)
1108cec47e3dSKiyoshi Ueda {
1109cec47e3dSKiyoshi Ueda 	struct request_queue *q = rq->q;
1110cec47e3dSKiyoshi Ueda 	unsigned long flags;
1111cec47e3dSKiyoshi Ueda 
1112cec47e3dSKiyoshi Ueda 	spin_lock_irqsave(q->queue_lock, flags);
1113cec47e3dSKiyoshi Ueda 	blk_requeue_request(q, rq);
1114cec47e3dSKiyoshi Ueda 	spin_unlock_irqrestore(q->queue_lock, flags);
1115bfebd1cdSMike Snitzer }
1116bfebd1cdSMike Snitzer 
1117bfebd1cdSMike Snitzer static void dm_requeue_unmapped_original_request(struct mapped_device *md,
1118bfebd1cdSMike Snitzer 						 struct request *rq)
1119bfebd1cdSMike Snitzer {
1120bfebd1cdSMike Snitzer 	int rw = rq_data_dir(rq);
1121bfebd1cdSMike Snitzer 
1122bfebd1cdSMike Snitzer 	dm_unprep_request(rq);
1123bfebd1cdSMike Snitzer 
1124bfebd1cdSMike Snitzer 	if (!rq->q->mq_ops)
1125bfebd1cdSMike Snitzer 		old_requeue_request(rq);
1126bfebd1cdSMike Snitzer 	else {
1127bfebd1cdSMike Snitzer 		blk_mq_requeue_request(rq);
1128bfebd1cdSMike Snitzer 		blk_mq_kick_requeue_list(rq->q);
1129bfebd1cdSMike Snitzer 	}
1130cec47e3dSKiyoshi Ueda 
1131466d89a6SKeith Busch 	rq_completed(md, rw, false);
1132cec47e3dSKiyoshi Ueda }
1133466d89a6SKeith Busch 
1134466d89a6SKeith Busch static void dm_requeue_unmapped_request(struct request *clone)
1135466d89a6SKeith Busch {
1136466d89a6SKeith Busch 	struct dm_rq_target_io *tio = clone->end_io_data;
1137466d89a6SKeith Busch 
1138466d89a6SKeith Busch 	dm_requeue_unmapped_original_request(tio->md, tio->orig);
1139cec47e3dSKiyoshi Ueda }
1140cec47e3dSKiyoshi Ueda 
1141bfebd1cdSMike Snitzer static void old_stop_queue(struct request_queue *q)
1142cec47e3dSKiyoshi Ueda {
1143bfebd1cdSMike Snitzer 	unsigned long flags;
1144bfebd1cdSMike Snitzer 
1145bfebd1cdSMike Snitzer 	if (blk_queue_stopped(q))
1146bfebd1cdSMike Snitzer 		return;
1147bfebd1cdSMike Snitzer 
1148bfebd1cdSMike Snitzer 	spin_lock_irqsave(q->queue_lock, flags);
1149cec47e3dSKiyoshi Ueda 	blk_stop_queue(q);
1150bfebd1cdSMike Snitzer 	spin_unlock_irqrestore(q->queue_lock, flags);
1151cec47e3dSKiyoshi Ueda }
1152cec47e3dSKiyoshi Ueda 
1153cec47e3dSKiyoshi Ueda static void stop_queue(struct request_queue *q)
1154cec47e3dSKiyoshi Ueda {
1155bfebd1cdSMike Snitzer 	if (!q->mq_ops)
1156bfebd1cdSMike Snitzer 		old_stop_queue(q);
1157bfebd1cdSMike Snitzer 	else
1158bfebd1cdSMike Snitzer 		blk_mq_stop_hw_queues(q);
1159bfebd1cdSMike Snitzer }
1160bfebd1cdSMike Snitzer 
1161bfebd1cdSMike Snitzer static void old_start_queue(struct request_queue *q)
1162bfebd1cdSMike Snitzer {
1163cec47e3dSKiyoshi Ueda 	unsigned long flags;
1164cec47e3dSKiyoshi Ueda 
1165cec47e3dSKiyoshi Ueda 	spin_lock_irqsave(q->queue_lock, flags);
1166cec47e3dSKiyoshi Ueda 	if (blk_queue_stopped(q))
1167cec47e3dSKiyoshi Ueda 		blk_start_queue(q);
1168bfebd1cdSMike Snitzer 	spin_unlock_irqrestore(q->queue_lock, flags);
1169cec47e3dSKiyoshi Ueda }
1170cec47e3dSKiyoshi Ueda 
1171cec47e3dSKiyoshi Ueda static void start_queue(struct request_queue *q)
1172cec47e3dSKiyoshi Ueda {
1173bfebd1cdSMike Snitzer 	if (!q->mq_ops)
1174bfebd1cdSMike Snitzer 		old_start_queue(q);
1175bfebd1cdSMike Snitzer 	else
1176bfebd1cdSMike Snitzer 		blk_mq_start_stopped_hw_queues(q, true);
1177cec47e3dSKiyoshi Ueda }
1178cec47e3dSKiyoshi Ueda 
117911a68244SKiyoshi Ueda static void dm_done(struct request *clone, int error, bool mapped)
118011a68244SKiyoshi Ueda {
118111a68244SKiyoshi Ueda 	int r = error;
118211a68244SKiyoshi Ueda 	struct dm_rq_target_io *tio = clone->end_io_data;
1183ba1cbad9SMike Snitzer 	dm_request_endio_fn rq_end_io = NULL;
1184ba1cbad9SMike Snitzer 
1185ba1cbad9SMike Snitzer 	if (tio->ti) {
1186ba1cbad9SMike Snitzer 		rq_end_io = tio->ti->type->rq_end_io;
118711a68244SKiyoshi Ueda 
118811a68244SKiyoshi Ueda 		if (mapped && rq_end_io)
118911a68244SKiyoshi Ueda 			r = rq_end_io(tio->ti, clone, error, &tio->info);
1190ba1cbad9SMike Snitzer 	}
119111a68244SKiyoshi Ueda 
11927eee4ae2SMike Snitzer 	if (unlikely(r == -EREMOTEIO && (clone->cmd_flags & REQ_WRITE_SAME) &&
11937eee4ae2SMike Snitzer 		     !clone->q->limits.max_write_same_sectors))
11947eee4ae2SMike Snitzer 		disable_write_same(tio->md);
11957eee4ae2SMike Snitzer 
119611a68244SKiyoshi Ueda 	if (r <= 0)
119711a68244SKiyoshi Ueda 		/* The target wants to complete the I/O */
119811a68244SKiyoshi Ueda 		dm_end_request(clone, r);
119911a68244SKiyoshi Ueda 	else if (r == DM_ENDIO_INCOMPLETE)
120011a68244SKiyoshi Ueda 		/* The target will handle the I/O */
120111a68244SKiyoshi Ueda 		return;
120211a68244SKiyoshi Ueda 	else if (r == DM_ENDIO_REQUEUE)
120311a68244SKiyoshi Ueda 		/* The target wants to requeue the I/O */
120411a68244SKiyoshi Ueda 		dm_requeue_unmapped_request(clone);
120511a68244SKiyoshi Ueda 	else {
120611a68244SKiyoshi Ueda 		DMWARN("unimplemented target endio return value: %d", r);
120711a68244SKiyoshi Ueda 		BUG();
120811a68244SKiyoshi Ueda 	}
120911a68244SKiyoshi Ueda }
121011a68244SKiyoshi Ueda 
1211cec47e3dSKiyoshi Ueda /*
1212cec47e3dSKiyoshi Ueda  * Request completion handler for request-based dm
1213cec47e3dSKiyoshi Ueda  */
1214cec47e3dSKiyoshi Ueda static void dm_softirq_done(struct request *rq)
1215cec47e3dSKiyoshi Ueda {
121611a68244SKiyoshi Ueda 	bool mapped = true;
1217bfebd1cdSMike Snitzer 	struct dm_rq_target_io *tio = tio_from_request(rq);
1218466d89a6SKeith Busch 	struct request *clone = tio->clone;
1219bfebd1cdSMike Snitzer 	int rw;
1220cec47e3dSKiyoshi Ueda 
1221e5863d9aSMike Snitzer 	if (!clone) {
1222bfebd1cdSMike Snitzer 		rw = rq_data_dir(rq);
1223bfebd1cdSMike Snitzer 		if (!rq->q->mq_ops) {
1224e5863d9aSMike Snitzer 			blk_end_request_all(rq, tio->error);
1225bfebd1cdSMike Snitzer 			rq_completed(tio->md, rw, false);
1226e5863d9aSMike Snitzer 			free_rq_tio(tio);
1227bfebd1cdSMike Snitzer 		} else {
1228bfebd1cdSMike Snitzer 			blk_mq_end_request(rq, tio->error);
1229bfebd1cdSMike Snitzer 			rq_completed(tio->md, rw, false);
1230bfebd1cdSMike Snitzer 		}
1231e5863d9aSMike Snitzer 		return;
1232e5863d9aSMike Snitzer 	}
1233cec47e3dSKiyoshi Ueda 
123411a68244SKiyoshi Ueda 	if (rq->cmd_flags & REQ_FAILED)
123511a68244SKiyoshi Ueda 		mapped = false;
1236cec47e3dSKiyoshi Ueda 
123711a68244SKiyoshi Ueda 	dm_done(clone, tio->error, mapped);
1238cec47e3dSKiyoshi Ueda }
1239cec47e3dSKiyoshi Ueda 
1240cec47e3dSKiyoshi Ueda /*
1241cec47e3dSKiyoshi Ueda  * Complete the clone and the original request with the error status
1242cec47e3dSKiyoshi Ueda  * through softirq context.
1243cec47e3dSKiyoshi Ueda  */
1244466d89a6SKeith Busch static void dm_complete_request(struct request *rq, int error)
1245cec47e3dSKiyoshi Ueda {
1246bfebd1cdSMike Snitzer 	struct dm_rq_target_io *tio = tio_from_request(rq);
1247cec47e3dSKiyoshi Ueda 
1248cec47e3dSKiyoshi Ueda 	tio->error = error;
1249cec47e3dSKiyoshi Ueda 	blk_complete_request(rq);
1250cec47e3dSKiyoshi Ueda }
1251cec47e3dSKiyoshi Ueda 
1252cec47e3dSKiyoshi Ueda /*
1253cec47e3dSKiyoshi Ueda  * Complete the not-mapped clone and the original request with the error status
1254cec47e3dSKiyoshi Ueda  * through softirq context.
1255cec47e3dSKiyoshi Ueda  * Target's rq_end_io() function isn't called.
1256e5863d9aSMike Snitzer  * This may be used when the target's map_rq() or clone_and_map_rq() functions fail.
1257cec47e3dSKiyoshi Ueda  */
1258466d89a6SKeith Busch static void dm_kill_unmapped_request(struct request *rq, int error)
1259cec47e3dSKiyoshi Ueda {
1260cec47e3dSKiyoshi Ueda 	rq->cmd_flags |= REQ_FAILED;
1261466d89a6SKeith Busch 	dm_complete_request(rq, error);
1262cec47e3dSKiyoshi Ueda }
1263cec47e3dSKiyoshi Ueda 
1264cec47e3dSKiyoshi Ueda /*
1265bfebd1cdSMike Snitzer  * Called with the clone's queue lock held (for non-blk-mq)
1266cec47e3dSKiyoshi Ueda  */
1267cec47e3dSKiyoshi Ueda static void end_clone_request(struct request *clone, int error)
1268cec47e3dSKiyoshi Ueda {
1269466d89a6SKeith Busch 	struct dm_rq_target_io *tio = clone->end_io_data;
1270466d89a6SKeith Busch 
1271e5863d9aSMike Snitzer 	if (!clone->q->mq_ops) {
1272cec47e3dSKiyoshi Ueda 		/*
1273cec47e3dSKiyoshi Ueda 		 * For just cleaning up the information of the queue in which
1274cec47e3dSKiyoshi Ueda 		 * the clone was dispatched.
1275e5863d9aSMike Snitzer 		 * The clone is *NOT* freed actually here because it is alloced
1276e5863d9aSMike Snitzer 		 * from dm own mempool (REQ_ALLOCED isn't set).
1277cec47e3dSKiyoshi Ueda 		 */
1278cec47e3dSKiyoshi Ueda 		__blk_put_request(clone->q, clone);
1279e5863d9aSMike Snitzer 	}
1280cec47e3dSKiyoshi Ueda 
1281cec47e3dSKiyoshi Ueda 	/*
1282cec47e3dSKiyoshi Ueda 	 * Actual request completion is done in a softirq context which doesn't
1283466d89a6SKeith Busch 	 * hold the clone's queue lock.  Otherwise, deadlock could occur because:
1284cec47e3dSKiyoshi Ueda 	 *     - another request may be submitted by the upper level driver
1285cec47e3dSKiyoshi Ueda 	 *       of the stacking during the completion
1286cec47e3dSKiyoshi Ueda 	 *     - the submission which requires queue lock may be done
1287466d89a6SKeith Busch 	 *       against this clone's queue
1288cec47e3dSKiyoshi Ueda 	 */
1289466d89a6SKeith Busch 	dm_complete_request(tio->orig, error);
1290cec47e3dSKiyoshi Ueda }
1291cec47e3dSKiyoshi Ueda 
129256a67df7SMike Snitzer /*
129356a67df7SMike Snitzer  * Return maximum size of I/O possible at the supplied sector up to the current
129456a67df7SMike Snitzer  * target boundary.
129556a67df7SMike Snitzer  */
129656a67df7SMike Snitzer static sector_t max_io_len_target_boundary(sector_t sector, struct dm_target *ti)
12971da177e4SLinus Torvalds {
129856a67df7SMike Snitzer 	sector_t target_offset = dm_target_offset(ti, sector);
129956a67df7SMike Snitzer 
130056a67df7SMike Snitzer 	return ti->len - target_offset;
130156a67df7SMike Snitzer }
130256a67df7SMike Snitzer 
130356a67df7SMike Snitzer static sector_t max_io_len(sector_t sector, struct dm_target *ti)
130456a67df7SMike Snitzer {
130556a67df7SMike Snitzer 	sector_t len = max_io_len_target_boundary(sector, ti);
1306542f9038SMike Snitzer 	sector_t offset, max_len;
13071da177e4SLinus Torvalds 
13081da177e4SLinus Torvalds 	/*
13091da177e4SLinus Torvalds 	 * Does the target need to split even further?
13101da177e4SLinus Torvalds 	 */
1311542f9038SMike Snitzer 	if (ti->max_io_len) {
1312542f9038SMike Snitzer 		offset = dm_target_offset(ti, sector);
1313542f9038SMike Snitzer 		if (unlikely(ti->max_io_len & (ti->max_io_len - 1)))
1314542f9038SMike Snitzer 			max_len = sector_div(offset, ti->max_io_len);
1315542f9038SMike Snitzer 		else
1316542f9038SMike Snitzer 			max_len = offset & (ti->max_io_len - 1);
1317542f9038SMike Snitzer 		max_len = ti->max_io_len - max_len;
1318542f9038SMike Snitzer 
1319542f9038SMike Snitzer 		if (len > max_len)
1320542f9038SMike Snitzer 			len = max_len;
13211da177e4SLinus Torvalds 	}
13221da177e4SLinus Torvalds 
13231da177e4SLinus Torvalds 	return len;
13241da177e4SLinus Torvalds }
13251da177e4SLinus Torvalds 
1326542f9038SMike Snitzer int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
1327542f9038SMike Snitzer {
1328542f9038SMike Snitzer 	if (len > UINT_MAX) {
1329542f9038SMike Snitzer 		DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)",
1330542f9038SMike Snitzer 		      (unsigned long long)len, UINT_MAX);
1331542f9038SMike Snitzer 		ti->error = "Maximum size of target IO is too large";
1332542f9038SMike Snitzer 		return -EINVAL;
1333542f9038SMike Snitzer 	}
1334542f9038SMike Snitzer 
1335542f9038SMike Snitzer 	ti->max_io_len = (uint32_t) len;
1336542f9038SMike Snitzer 
1337542f9038SMike Snitzer 	return 0;
1338542f9038SMike Snitzer }
1339542f9038SMike Snitzer EXPORT_SYMBOL_GPL(dm_set_target_max_io_len);
1340542f9038SMike Snitzer 
13411dd40c3eSMikulas Patocka /*
13421dd40c3eSMikulas Patocka  * A target may call dm_accept_partial_bio only from the map routine.  It is
13431dd40c3eSMikulas Patocka  * allowed for all bio types except REQ_FLUSH.
13441dd40c3eSMikulas Patocka  *
13451dd40c3eSMikulas Patocka  * dm_accept_partial_bio informs the dm that the target only wants to process
13461dd40c3eSMikulas Patocka  * additional n_sectors sectors of the bio and the rest of the data should be
13471dd40c3eSMikulas Patocka  * sent in a next bio.
13481dd40c3eSMikulas Patocka  *
13491dd40c3eSMikulas Patocka  * A diagram that explains the arithmetics:
13501dd40c3eSMikulas Patocka  * +--------------------+---------------+-------+
13511dd40c3eSMikulas Patocka  * |         1          |       2       |   3   |
13521dd40c3eSMikulas Patocka  * +--------------------+---------------+-------+
13531dd40c3eSMikulas Patocka  *
13541dd40c3eSMikulas Patocka  * <-------------- *tio->len_ptr --------------->
13551dd40c3eSMikulas Patocka  *                      <------- bi_size ------->
13561dd40c3eSMikulas Patocka  *                      <-- n_sectors -->
13571dd40c3eSMikulas Patocka  *
13581dd40c3eSMikulas Patocka  * Region 1 was already iterated over with bio_advance or similar function.
13591dd40c3eSMikulas Patocka  *	(it may be empty if the target doesn't use bio_advance)
13601dd40c3eSMikulas Patocka  * Region 2 is the remaining bio size that the target wants to process.
13611dd40c3eSMikulas Patocka  *	(it may be empty if region 1 is non-empty, although there is no reason
13621dd40c3eSMikulas Patocka  *	 to make it empty)
13631dd40c3eSMikulas Patocka  * The target requires that region 3 is to be sent in the next bio.
13641dd40c3eSMikulas Patocka  *
13651dd40c3eSMikulas Patocka  * If the target wants to receive multiple copies of the bio (via num_*bios, etc),
13661dd40c3eSMikulas Patocka  * the partially processed part (the sum of regions 1+2) must be the same for all
13671dd40c3eSMikulas Patocka  * copies of the bio.
13681dd40c3eSMikulas Patocka  */
13691dd40c3eSMikulas Patocka void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors)
13701dd40c3eSMikulas Patocka {
13711dd40c3eSMikulas Patocka 	struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
13721dd40c3eSMikulas Patocka 	unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT;
13731dd40c3eSMikulas Patocka 	BUG_ON(bio->bi_rw & REQ_FLUSH);
13741dd40c3eSMikulas Patocka 	BUG_ON(bi_size > *tio->len_ptr);
13751dd40c3eSMikulas Patocka 	BUG_ON(n_sectors > bi_size);
13761dd40c3eSMikulas Patocka 	*tio->len_ptr -= bi_size - n_sectors;
13771dd40c3eSMikulas Patocka 	bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT;
13781dd40c3eSMikulas Patocka }
13791dd40c3eSMikulas Patocka EXPORT_SYMBOL_GPL(dm_accept_partial_bio);
13801dd40c3eSMikulas Patocka 
1381bd2a49b8SAlasdair G Kergon static void __map_bio(struct dm_target_io *tio)
13821da177e4SLinus Torvalds {
13831da177e4SLinus Torvalds 	int r;
13842056a782SJens Axboe 	sector_t sector;
13859faf400fSStefan Bader 	struct mapped_device *md;
1386dba14160SMikulas Patocka 	struct bio *clone = &tio->clone;
1387bd2a49b8SAlasdair G Kergon 	struct dm_target *ti = tio->ti;
13881da177e4SLinus Torvalds 
13891da177e4SLinus Torvalds 	clone->bi_end_io = clone_endio;
13901da177e4SLinus Torvalds 
13911da177e4SLinus Torvalds 	/*
13921da177e4SLinus Torvalds 	 * Map the clone.  If r == 0 we don't need to do
13931da177e4SLinus Torvalds 	 * anything, the target has assumed ownership of
13941da177e4SLinus Torvalds 	 * this io.
13951da177e4SLinus Torvalds 	 */
13961da177e4SLinus Torvalds 	atomic_inc(&tio->io->io_count);
13974f024f37SKent Overstreet 	sector = clone->bi_iter.bi_sector;
13987de3ee57SMikulas Patocka 	r = ti->type->map(ti, clone);
139945cbcd79SKiyoshi Ueda 	if (r == DM_MAPIO_REMAPPED) {
14001da177e4SLinus Torvalds 		/* the bio has been remapped so dispatch it */
14012056a782SJens Axboe 
1402d07335e5SMike Snitzer 		trace_block_bio_remap(bdev_get_queue(clone->bi_bdev), clone,
140322a7c31aSAlan D. Brunelle 				      tio->io->bio->bi_bdev->bd_dev, sector);
14042056a782SJens Axboe 
14051da177e4SLinus Torvalds 		generic_make_request(clone);
14062e93ccc1SKiyoshi Ueda 	} else if (r < 0 || r == DM_MAPIO_REQUEUE) {
14072e93ccc1SKiyoshi Ueda 		/* error the io and bail out, or requeue it if needed */
14089faf400fSStefan Bader 		md = tio->io->md;
14099faf400fSStefan Bader 		dec_pending(tio->io, r);
14109faf400fSStefan Bader 		free_tio(md, tio);
141145cbcd79SKiyoshi Ueda 	} else if (r) {
141245cbcd79SKiyoshi Ueda 		DMWARN("unimplemented target map return value: %d", r);
141345cbcd79SKiyoshi Ueda 		BUG();
14141da177e4SLinus Torvalds 	}
14151da177e4SLinus Torvalds }
14161da177e4SLinus Torvalds 
14171da177e4SLinus Torvalds struct clone_info {
14181da177e4SLinus Torvalds 	struct mapped_device *md;
14191da177e4SLinus Torvalds 	struct dm_table *map;
14201da177e4SLinus Torvalds 	struct bio *bio;
14211da177e4SLinus Torvalds 	struct dm_io *io;
14221da177e4SLinus Torvalds 	sector_t sector;
1423e0d6609aSMikulas Patocka 	unsigned sector_count;
14241da177e4SLinus Torvalds };
14251da177e4SLinus Torvalds 
1426e0d6609aSMikulas Patocka static void bio_setup_sector(struct bio *bio, sector_t sector, unsigned len)
1427bd2a49b8SAlasdair G Kergon {
14284f024f37SKent Overstreet 	bio->bi_iter.bi_sector = sector;
14294f024f37SKent Overstreet 	bio->bi_iter.bi_size = to_bytes(len);
14301da177e4SLinus Torvalds }
14311da177e4SLinus Torvalds 
14321da177e4SLinus Torvalds /*
14331da177e4SLinus Torvalds  * Creates a bio that consists of range of complete bvecs.
14341da177e4SLinus Torvalds  */
1435dba14160SMikulas Patocka static void clone_bio(struct dm_target_io *tio, struct bio *bio,
14361c3b13e6SKent Overstreet 		      sector_t sector, unsigned len)
14371da177e4SLinus Torvalds {
1438dba14160SMikulas Patocka 	struct bio *clone = &tio->clone;
14391da177e4SLinus Torvalds 
14401c3b13e6SKent Overstreet 	__bio_clone_fast(clone, bio);
14419c47008dSMartin K. Petersen 
14421c3b13e6SKent Overstreet 	if (bio_integrity(bio))
14431c3b13e6SKent Overstreet 		bio_integrity_clone(clone, bio, GFP_NOIO);
14441c3b13e6SKent Overstreet 
14451c3b13e6SKent Overstreet 	bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector));
14461c3b13e6SKent Overstreet 	clone->bi_iter.bi_size = to_bytes(len);
14471c3b13e6SKent Overstreet 
14481c3b13e6SKent Overstreet 	if (bio_integrity(bio))
14491c3b13e6SKent Overstreet 		bio_integrity_trim(clone, 0, len);
14501da177e4SLinus Torvalds }
14511da177e4SLinus Torvalds 
14529015df24SAlasdair G Kergon static struct dm_target_io *alloc_tio(struct clone_info *ci,
145399778273SJunichi Nomura 				      struct dm_target *ti,
145455a62eefSAlasdair G Kergon 				      unsigned target_bio_nr)
1455f9ab94ceSMikulas Patocka {
1456dba14160SMikulas Patocka 	struct dm_target_io *tio;
1457dba14160SMikulas Patocka 	struct bio *clone;
1458dba14160SMikulas Patocka 
145999778273SJunichi Nomura 	clone = bio_alloc_bioset(GFP_NOIO, 0, ci->md->bs);
1460dba14160SMikulas Patocka 	tio = container_of(clone, struct dm_target_io, clone);
1461f9ab94ceSMikulas Patocka 
1462f9ab94ceSMikulas Patocka 	tio->io = ci->io;
1463f9ab94ceSMikulas Patocka 	tio->ti = ti;
146455a62eefSAlasdair G Kergon 	tio->target_bio_nr = target_bio_nr;
14659015df24SAlasdair G Kergon 
14669015df24SAlasdair G Kergon 	return tio;
14679015df24SAlasdair G Kergon }
14689015df24SAlasdair G Kergon 
146914fe594dSAlasdair G Kergon static void __clone_and_map_simple_bio(struct clone_info *ci,
147014fe594dSAlasdair G Kergon 				       struct dm_target *ti,
14711dd40c3eSMikulas Patocka 				       unsigned target_bio_nr, unsigned *len)
14729015df24SAlasdair G Kergon {
147399778273SJunichi Nomura 	struct dm_target_io *tio = alloc_tio(ci, ti, target_bio_nr);
1474dba14160SMikulas Patocka 	struct bio *clone = &tio->clone;
14759015df24SAlasdair G Kergon 
14761dd40c3eSMikulas Patocka 	tio->len_ptr = len;
14771dd40c3eSMikulas Patocka 
14781c3b13e6SKent Overstreet 	__bio_clone_fast(clone, ci->bio);
1479bd2a49b8SAlasdair G Kergon 	if (len)
14801dd40c3eSMikulas Patocka 		bio_setup_sector(clone, ci->sector, *len);
1481f9ab94ceSMikulas Patocka 
1482bd2a49b8SAlasdair G Kergon 	__map_bio(tio);
1483f9ab94ceSMikulas Patocka }
1484f9ab94ceSMikulas Patocka 
148514fe594dSAlasdair G Kergon static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti,
14861dd40c3eSMikulas Patocka 				  unsigned num_bios, unsigned *len)
148706a426ceSMike Snitzer {
148855a62eefSAlasdair G Kergon 	unsigned target_bio_nr;
148906a426ceSMike Snitzer 
149055a62eefSAlasdair G Kergon 	for (target_bio_nr = 0; target_bio_nr < num_bios; target_bio_nr++)
149114fe594dSAlasdair G Kergon 		__clone_and_map_simple_bio(ci, ti, target_bio_nr, len);
149206a426ceSMike Snitzer }
149306a426ceSMike Snitzer 
149414fe594dSAlasdair G Kergon static int __send_empty_flush(struct clone_info *ci)
1495f9ab94ceSMikulas Patocka {
149606a426ceSMike Snitzer 	unsigned target_nr = 0;
1497f9ab94ceSMikulas Patocka 	struct dm_target *ti;
1498f9ab94ceSMikulas Patocka 
1499b372d360SMike Snitzer 	BUG_ON(bio_has_data(ci->bio));
1500f9ab94ceSMikulas Patocka 	while ((ti = dm_table_get_target(ci->map, target_nr++)))
15011dd40c3eSMikulas Patocka 		__send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL);
1502f9ab94ceSMikulas Patocka 
1503f9ab94ceSMikulas Patocka 	return 0;
1504f9ab94ceSMikulas Patocka }
1505f9ab94ceSMikulas Patocka 
1506e4c93811SAlasdair G Kergon static void __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti,
15071dd40c3eSMikulas Patocka 				     sector_t sector, unsigned *len)
15085ae89a87SMike Snitzer {
1509dba14160SMikulas Patocka 	struct bio *bio = ci->bio;
15105ae89a87SMike Snitzer 	struct dm_target_io *tio;
1511b0d8ed4dSAlasdair G Kergon 	unsigned target_bio_nr;
1512b0d8ed4dSAlasdair G Kergon 	unsigned num_target_bios = 1;
15135ae89a87SMike Snitzer 
1514b0d8ed4dSAlasdair G Kergon 	/*
1515b0d8ed4dSAlasdair G Kergon 	 * Does the target want to receive duplicate copies of the bio?
1516b0d8ed4dSAlasdair G Kergon 	 */
1517b0d8ed4dSAlasdair G Kergon 	if (bio_data_dir(bio) == WRITE && ti->num_write_bios)
1518b0d8ed4dSAlasdair G Kergon 		num_target_bios = ti->num_write_bios(ti, bio);
1519e4c93811SAlasdair G Kergon 
1520b0d8ed4dSAlasdair G Kergon 	for (target_bio_nr = 0; target_bio_nr < num_target_bios; target_bio_nr++) {
152199778273SJunichi Nomura 		tio = alloc_tio(ci, ti, target_bio_nr);
15221dd40c3eSMikulas Patocka 		tio->len_ptr = len;
15231dd40c3eSMikulas Patocka 		clone_bio(tio, bio, sector, *len);
1524bd2a49b8SAlasdair G Kergon 		__map_bio(tio);
15255ae89a87SMike Snitzer 	}
1526b0d8ed4dSAlasdair G Kergon }
15275ae89a87SMike Snitzer 
152855a62eefSAlasdair G Kergon typedef unsigned (*get_num_bios_fn)(struct dm_target *ti);
152923508a96SMike Snitzer 
153055a62eefSAlasdair G Kergon static unsigned get_num_discard_bios(struct dm_target *ti)
153123508a96SMike Snitzer {
153255a62eefSAlasdair G Kergon 	return ti->num_discard_bios;
153323508a96SMike Snitzer }
153423508a96SMike Snitzer 
153555a62eefSAlasdair G Kergon static unsigned get_num_write_same_bios(struct dm_target *ti)
153623508a96SMike Snitzer {
153755a62eefSAlasdair G Kergon 	return ti->num_write_same_bios;
153823508a96SMike Snitzer }
153923508a96SMike Snitzer 
154023508a96SMike Snitzer typedef bool (*is_split_required_fn)(struct dm_target *ti);
154123508a96SMike Snitzer 
154223508a96SMike Snitzer static bool is_split_required_for_discard(struct dm_target *ti)
154323508a96SMike Snitzer {
154455a62eefSAlasdair G Kergon 	return ti->split_discard_bios;
154523508a96SMike Snitzer }
154623508a96SMike Snitzer 
154714fe594dSAlasdair G Kergon static int __send_changing_extent_only(struct clone_info *ci,
154855a62eefSAlasdair G Kergon 				       get_num_bios_fn get_num_bios,
154923508a96SMike Snitzer 				       is_split_required_fn is_split_required)
15505ae89a87SMike Snitzer {
15515ae89a87SMike Snitzer 	struct dm_target *ti;
1552e0d6609aSMikulas Patocka 	unsigned len;
155355a62eefSAlasdair G Kergon 	unsigned num_bios;
15545ae89a87SMike Snitzer 
1555a79245b3SMike Snitzer 	do {
15565ae89a87SMike Snitzer 		ti = dm_table_find_target(ci->map, ci->sector);
15575ae89a87SMike Snitzer 		if (!dm_target_is_valid(ti))
15585ae89a87SMike Snitzer 			return -EIO;
15595ae89a87SMike Snitzer 
15605ae89a87SMike Snitzer 		/*
156123508a96SMike Snitzer 		 * Even though the device advertised support for this type of
156223508a96SMike Snitzer 		 * request, that does not mean every target supports it, and
1563936688d7SMike Snitzer 		 * reconfiguration might also have changed that since the
15645ae89a87SMike Snitzer 		 * check was performed.
15655ae89a87SMike Snitzer 		 */
156655a62eefSAlasdair G Kergon 		num_bios = get_num_bios ? get_num_bios(ti) : 0;
156755a62eefSAlasdair G Kergon 		if (!num_bios)
15685ae89a87SMike Snitzer 			return -EOPNOTSUPP;
15695ae89a87SMike Snitzer 
157023508a96SMike Snitzer 		if (is_split_required && !is_split_required(ti))
1571e0d6609aSMikulas Patocka 			len = min((sector_t)ci->sector_count, max_io_len_target_boundary(ci->sector, ti));
15727acf0277SMikulas Patocka 		else
1573e0d6609aSMikulas Patocka 			len = min((sector_t)ci->sector_count, max_io_len(ci->sector, ti));
15745ae89a87SMike Snitzer 
15751dd40c3eSMikulas Patocka 		__send_duplicate_bios(ci, ti, num_bios, &len);
15765ae89a87SMike Snitzer 
1577a79245b3SMike Snitzer 		ci->sector += len;
1578a79245b3SMike Snitzer 	} while (ci->sector_count -= len);
15795ae89a87SMike Snitzer 
15805ae89a87SMike Snitzer 	return 0;
15815ae89a87SMike Snitzer }
15825ae89a87SMike Snitzer 
158314fe594dSAlasdair G Kergon static int __send_discard(struct clone_info *ci)
158423508a96SMike Snitzer {
158514fe594dSAlasdair G Kergon 	return __send_changing_extent_only(ci, get_num_discard_bios,
158623508a96SMike Snitzer 					   is_split_required_for_discard);
158723508a96SMike Snitzer }
158823508a96SMike Snitzer 
158914fe594dSAlasdair G Kergon static int __send_write_same(struct clone_info *ci)
159023508a96SMike Snitzer {
159114fe594dSAlasdair G Kergon 	return __send_changing_extent_only(ci, get_num_write_same_bios, NULL);
159223508a96SMike Snitzer }
159323508a96SMike Snitzer 
1594e4c93811SAlasdair G Kergon /*
1595e4c93811SAlasdair G Kergon  * Select the correct strategy for processing a non-flush bio.
1596e4c93811SAlasdair G Kergon  */
1597e4c93811SAlasdair G Kergon static int __split_and_process_non_flush(struct clone_info *ci)
1598e4c93811SAlasdair G Kergon {
1599e4c93811SAlasdair G Kergon 	struct bio *bio = ci->bio;
1600e4c93811SAlasdair G Kergon 	struct dm_target *ti;
16011c3b13e6SKent Overstreet 	unsigned len;
1602e4c93811SAlasdair G Kergon 
1603e4c93811SAlasdair G Kergon 	if (unlikely(bio->bi_rw & REQ_DISCARD))
1604e4c93811SAlasdair G Kergon 		return __send_discard(ci);
1605e4c93811SAlasdair G Kergon 	else if (unlikely(bio->bi_rw & REQ_WRITE_SAME))
1606e4c93811SAlasdair G Kergon 		return __send_write_same(ci);
1607e4c93811SAlasdair G Kergon 
1608e4c93811SAlasdair G Kergon 	ti = dm_table_find_target(ci->map, ci->sector);
1609e4c93811SAlasdair G Kergon 	if (!dm_target_is_valid(ti))
1610e4c93811SAlasdair G Kergon 		return -EIO;
1611e4c93811SAlasdair G Kergon 
16121c3b13e6SKent Overstreet 	len = min_t(sector_t, max_io_len(ci->sector, ti), ci->sector_count);
1613e4c93811SAlasdair G Kergon 
16141dd40c3eSMikulas Patocka 	__clone_and_map_data_bio(ci, ti, ci->sector, &len);
1615e4c93811SAlasdair G Kergon 
1616e4c93811SAlasdair G Kergon 	ci->sector += len;
1617e4c93811SAlasdair G Kergon 	ci->sector_count -= len;
1618e4c93811SAlasdair G Kergon 
1619e4c93811SAlasdair G Kergon 	return 0;
1620e4c93811SAlasdair G Kergon }
1621e4c93811SAlasdair G Kergon 
1622e4c93811SAlasdair G Kergon /*
162314fe594dSAlasdair G Kergon  * Entry point to split a bio into clones and submit them to the targets.
16241da177e4SLinus Torvalds  */
162583d5e5b0SMikulas Patocka static void __split_and_process_bio(struct mapped_device *md,
162683d5e5b0SMikulas Patocka 				    struct dm_table *map, struct bio *bio)
16271da177e4SLinus Torvalds {
16281da177e4SLinus Torvalds 	struct clone_info ci;
1629512875bdSJun'ichi Nomura 	int error = 0;
16301da177e4SLinus Torvalds 
163183d5e5b0SMikulas Patocka 	if (unlikely(!map)) {
1632f0b9a450SMikulas Patocka 		bio_io_error(bio);
1633f0b9a450SMikulas Patocka 		return;
1634f0b9a450SMikulas Patocka 	}
1635692d0eb9SMikulas Patocka 
163683d5e5b0SMikulas Patocka 	ci.map = map;
16371da177e4SLinus Torvalds 	ci.md = md;
16381da177e4SLinus Torvalds 	ci.io = alloc_io(md);
16391da177e4SLinus Torvalds 	ci.io->error = 0;
16401da177e4SLinus Torvalds 	atomic_set(&ci.io->io_count, 1);
16411da177e4SLinus Torvalds 	ci.io->bio = bio;
16421da177e4SLinus Torvalds 	ci.io->md = md;
1643f88fb981SKiyoshi Ueda 	spin_lock_init(&ci.io->endio_lock);
16444f024f37SKent Overstreet 	ci.sector = bio->bi_iter.bi_sector;
16451da177e4SLinus Torvalds 
16463eaf840eSJun'ichi "Nick" Nomura 	start_io_acct(ci.io);
1647bd2a49b8SAlasdair G Kergon 
1648b372d360SMike Snitzer 	if (bio->bi_rw & REQ_FLUSH) {
1649b372d360SMike Snitzer 		ci.bio = &ci.md->flush_bio;
1650b372d360SMike Snitzer 		ci.sector_count = 0;
165114fe594dSAlasdair G Kergon 		error = __send_empty_flush(&ci);
1652b372d360SMike Snitzer 		/* dec_pending submits any data associated with flush */
1653b372d360SMike Snitzer 	} else {
16546a8736d1STejun Heo 		ci.bio = bio;
1655f6fccb12SMilan Broz 		ci.sector_count = bio_sectors(bio);
1656512875bdSJun'ichi Nomura 		while (ci.sector_count && !error)
165714fe594dSAlasdair G Kergon 			error = __split_and_process_non_flush(&ci);
1658d87f4c14STejun Heo 	}
16591da177e4SLinus Torvalds 
16601da177e4SLinus Torvalds 	/* drop the extra reference count */
1661512875bdSJun'ichi Nomura 	dec_pending(ci.io, error);
16629e4e5f87SMilan Broz }
16639e4e5f87SMilan Broz /*-----------------------------------------------------------------
16641da177e4SLinus Torvalds  * CRUD END
16651da177e4SLinus Torvalds  *---------------------------------------------------------------*/
16661da177e4SLinus Torvalds 
16671da177e4SLinus Torvalds static int dm_merge_bvec(struct request_queue *q,
16681da177e4SLinus Torvalds 			 struct bvec_merge_data *bvm,
1669f6fccb12SMilan Broz 			 struct bio_vec *biovec)
1670f6fccb12SMilan Broz {
1671f6fccb12SMilan Broz 	struct mapped_device *md = q->queuedata;
167283d5e5b0SMikulas Patocka 	struct dm_table *map = dm_get_live_table_fast(md);
1673f6fccb12SMilan Broz 	struct dm_target *ti;
1674f6fccb12SMilan Broz 	sector_t max_sectors;
1675f6fccb12SMilan Broz 	int max_size = 0;
1676f6fccb12SMilan Broz 
1677f6fccb12SMilan Broz 	if (unlikely(!map))
1678f6fccb12SMilan Broz 		goto out;
1679f6fccb12SMilan Broz 
1680f6fccb12SMilan Broz 	ti = dm_table_find_target(map, bvm->bi_sector);
1681f6fccb12SMilan Broz 	if (!dm_target_is_valid(ti))
168283d5e5b0SMikulas Patocka 		goto out;
1683f6fccb12SMilan Broz 
1684f6fccb12SMilan Broz 	/*
1685f6fccb12SMilan Broz 	 * Find maximum amount of I/O that won't need splitting
1686f6fccb12SMilan Broz 	 */
168756a67df7SMike Snitzer 	max_sectors = min(max_io_len(bvm->bi_sector, ti),
1688148e51baSMike Snitzer 			  (sector_t) queue_max_sectors(q));
1689f6fccb12SMilan Broz 	max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size;
1690148e51baSMike Snitzer 	if (unlikely(max_size < 0)) /* this shouldn't _ever_ happen */
1691f6fccb12SMilan Broz 		max_size = 0;
1692f6fccb12SMilan Broz 
1693f6fccb12SMilan Broz 	/*
1694f6fccb12SMilan Broz 	 * merge_bvec_fn() returns number of bytes
1695f6fccb12SMilan Broz 	 * it can accept at this offset
1696f6fccb12SMilan Broz 	 * max is precomputed maximal io size
1697f6fccb12SMilan Broz 	 */
1698f6fccb12SMilan Broz 	if (max_size && ti->type->merge)
1699f6fccb12SMilan Broz 		max_size = ti->type->merge(ti, bvm, biovec, max_size);
17008cbeb67aSMikulas Patocka 	/*
17018cbeb67aSMikulas Patocka 	 * If the target doesn't support merge method and some of the devices
1702148e51baSMike Snitzer 	 * provided their merge_bvec method (we know this by looking for the
1703148e51baSMike Snitzer 	 * max_hw_sectors that dm_set_device_limits may set), then we can't
1704148e51baSMike Snitzer 	 * allow bios with multiple vector entries.  So always set max_size
1705148e51baSMike Snitzer 	 * to 0, and the code below allows just one page.
17068cbeb67aSMikulas Patocka 	 */
17078cbeb67aSMikulas Patocka 	else if (queue_max_hw_sectors(q) <= PAGE_SIZE >> 9)
17088cbeb67aSMikulas Patocka 		max_size = 0;
1709f6fccb12SMilan Broz 
17105037108aSMikulas Patocka out:
171183d5e5b0SMikulas Patocka 	dm_put_live_table_fast(md);
1712f6fccb12SMilan Broz 	/*
1713f6fccb12SMilan Broz 	 * Always allow an entire first page
1714f6fccb12SMilan Broz 	 */
1715f6fccb12SMilan Broz 	if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT))
1716f6fccb12SMilan Broz 		max_size = biovec->bv_len;
1717f6fccb12SMilan Broz 
1718f6fccb12SMilan Broz 	return max_size;
1719f6fccb12SMilan Broz }
1720f6fccb12SMilan Broz 
17211da177e4SLinus Torvalds /*
17221da177e4SLinus Torvalds  * The request function that just remaps the bio built up by
17231da177e4SLinus Torvalds  * dm_merge_bvec.
17241da177e4SLinus Torvalds  */
1725ff36ab34SMike Snitzer static void dm_make_request(struct request_queue *q, struct bio *bio)
17261da177e4SLinus Torvalds {
172712f03a49SKevin Corry 	int rw = bio_data_dir(bio);
17281da177e4SLinus Torvalds 	struct mapped_device *md = q->queuedata;
172983d5e5b0SMikulas Patocka 	int srcu_idx;
173083d5e5b0SMikulas Patocka 	struct dm_table *map;
17311da177e4SLinus Torvalds 
173283d5e5b0SMikulas Patocka 	map = dm_get_live_table(md, &srcu_idx);
17331da177e4SLinus Torvalds 
173418c0b223SGu Zheng 	generic_start_io_acct(rw, bio_sectors(bio), &dm_disk(md)->part0);
173512f03a49SKevin Corry 
17366a8736d1STejun Heo 	/* if we're suspended, we have to queue this io for later */
17376a8736d1STejun Heo 	if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) {
173883d5e5b0SMikulas Patocka 		dm_put_live_table(md, srcu_idx);
17391da177e4SLinus Torvalds 
17406a8736d1STejun Heo 		if (bio_rw(bio) != READA)
174192c63902SMikulas Patocka 			queue_io(md, bio);
17426a8736d1STejun Heo 		else
17436a8736d1STejun Heo 			bio_io_error(bio);
17445a7bbad2SChristoph Hellwig 		return;
17451da177e4SLinus Torvalds 	}
17461da177e4SLinus Torvalds 
174783d5e5b0SMikulas Patocka 	__split_and_process_bio(md, map, bio);
174883d5e5b0SMikulas Patocka 	dm_put_live_table(md, srcu_idx);
17495a7bbad2SChristoph Hellwig 	return;
1750cec47e3dSKiyoshi Ueda }
1751cec47e3dSKiyoshi Ueda 
1752fd2ed4d2SMikulas Patocka int dm_request_based(struct mapped_device *md)
1753cec47e3dSKiyoshi Ueda {
1754cec47e3dSKiyoshi Ueda 	return blk_queue_stackable(md->queue);
1755cec47e3dSKiyoshi Ueda }
1756cec47e3dSKiyoshi Ueda 
1757466d89a6SKeith Busch static void dm_dispatch_clone_request(struct request *clone, struct request *rq)
1758cec47e3dSKiyoshi Ueda {
1759cec47e3dSKiyoshi Ueda 	int r;
1760cec47e3dSKiyoshi Ueda 
1761466d89a6SKeith Busch 	if (blk_queue_io_stat(clone->q))
1762466d89a6SKeith Busch 		clone->cmd_flags |= REQ_IO_STAT;
1763cec47e3dSKiyoshi Ueda 
1764466d89a6SKeith Busch 	clone->start_time = jiffies;
1765466d89a6SKeith Busch 	r = blk_insert_cloned_request(clone->q, clone);
1766cec47e3dSKiyoshi Ueda 	if (r)
1767466d89a6SKeith Busch 		/* must complete clone in terms of original request */
1768cec47e3dSKiyoshi Ueda 		dm_complete_request(rq, r);
1769cec47e3dSKiyoshi Ueda }
1770cec47e3dSKiyoshi Ueda 
1771*5f1b670dSChristoph Hellwig static void setup_clone(struct request *clone, struct request *rq,
1772*5f1b670dSChristoph Hellwig 		        struct dm_rq_target_io *tio)
1773cec47e3dSKiyoshi Ueda {
1774*5f1b670dSChristoph Hellwig 	blk_rq_prep_clone(clone, rq);
1775cec47e3dSKiyoshi Ueda 	clone->end_io = end_clone_request;
1776cec47e3dSKiyoshi Ueda 	clone->end_io_data = tio;
17771ae49ea2SMike Snitzer 	tio->clone = clone;
1778cec47e3dSKiyoshi Ueda }
1779cec47e3dSKiyoshi Ueda 
17806facdaffSKiyoshi Ueda static struct request *clone_rq(struct request *rq, struct mapped_device *md,
17811ae49ea2SMike Snitzer 				struct dm_rq_target_io *tio, gfp_t gfp_mask)
17826facdaffSKiyoshi Ueda {
178302233342SMike Snitzer 	/*
178402233342SMike Snitzer 	 * Do not allocate a clone if tio->clone was already set
178502233342SMike Snitzer 	 * (see: dm_mq_queue_rq).
178602233342SMike Snitzer 	 */
178702233342SMike Snitzer 	bool alloc_clone = !tio->clone;
178802233342SMike Snitzer 	struct request *clone;
17891ae49ea2SMike Snitzer 
179002233342SMike Snitzer 	if (alloc_clone) {
179102233342SMike Snitzer 		clone = alloc_clone_request(md, gfp_mask);
17921ae49ea2SMike Snitzer 		if (!clone)
17931ae49ea2SMike Snitzer 			return NULL;
179402233342SMike Snitzer 	} else
179502233342SMike Snitzer 		clone = tio->clone;
17961ae49ea2SMike Snitzer 
17971ae49ea2SMike Snitzer 	blk_rq_init(NULL, clone);
1798*5f1b670dSChristoph Hellwig 	setup_clone(clone, rq, tio);
17991ae49ea2SMike Snitzer 
18001ae49ea2SMike Snitzer 	return clone;
18011ae49ea2SMike Snitzer }
18021ae49ea2SMike Snitzer 
18032eb6e1e3SKeith Busch static void map_tio_request(struct kthread_work *work);
18042eb6e1e3SKeith Busch 
1805bfebd1cdSMike Snitzer static void init_tio(struct dm_rq_target_io *tio, struct request *rq,
1806bfebd1cdSMike Snitzer 		     struct mapped_device *md)
1807bfebd1cdSMike Snitzer {
1808bfebd1cdSMike Snitzer 	tio->md = md;
1809bfebd1cdSMike Snitzer 	tio->ti = NULL;
1810bfebd1cdSMike Snitzer 	tio->clone = NULL;
1811bfebd1cdSMike Snitzer 	tio->orig = rq;
1812bfebd1cdSMike Snitzer 	tio->error = 0;
1813bfebd1cdSMike Snitzer 	memset(&tio->info, 0, sizeof(tio->info));
181402233342SMike Snitzer 	if (md->kworker_task)
1815bfebd1cdSMike Snitzer 		init_kthread_work(&tio->work, map_tio_request);
1816bfebd1cdSMike Snitzer }
1817bfebd1cdSMike Snitzer 
1818466d89a6SKeith Busch static struct dm_rq_target_io *prep_tio(struct request *rq,
1819466d89a6SKeith Busch 					struct mapped_device *md, gfp_t gfp_mask)
18206facdaffSKiyoshi Ueda {
18216facdaffSKiyoshi Ueda 	struct dm_rq_target_io *tio;
1822e5863d9aSMike Snitzer 	int srcu_idx;
1823e5863d9aSMike Snitzer 	struct dm_table *table;
18246facdaffSKiyoshi Ueda 
18256facdaffSKiyoshi Ueda 	tio = alloc_rq_tio(md, gfp_mask);
18266facdaffSKiyoshi Ueda 	if (!tio)
18276facdaffSKiyoshi Ueda 		return NULL;
18286facdaffSKiyoshi Ueda 
1829bfebd1cdSMike Snitzer 	init_tio(tio, rq, md);
18306facdaffSKiyoshi Ueda 
1831e5863d9aSMike Snitzer 	table = dm_get_live_table(md, &srcu_idx);
1832e5863d9aSMike Snitzer 	if (!dm_table_mq_request_based(table)) {
1833466d89a6SKeith Busch 		if (!clone_rq(rq, md, tio, gfp_mask)) {
1834e5863d9aSMike Snitzer 			dm_put_live_table(md, srcu_idx);
18356facdaffSKiyoshi Ueda 			free_rq_tio(tio);
18366facdaffSKiyoshi Ueda 			return NULL;
18376facdaffSKiyoshi Ueda 		}
1838e5863d9aSMike Snitzer 	}
1839e5863d9aSMike Snitzer 	dm_put_live_table(md, srcu_idx);
18406facdaffSKiyoshi Ueda 
1841466d89a6SKeith Busch 	return tio;
18426facdaffSKiyoshi Ueda }
18436facdaffSKiyoshi Ueda 
1844cec47e3dSKiyoshi Ueda /*
1845cec47e3dSKiyoshi Ueda  * Called with the queue lock held.
1846cec47e3dSKiyoshi Ueda  */
1847cec47e3dSKiyoshi Ueda static int dm_prep_fn(struct request_queue *q, struct request *rq)
1848cec47e3dSKiyoshi Ueda {
1849cec47e3dSKiyoshi Ueda 	struct mapped_device *md = q->queuedata;
1850466d89a6SKeith Busch 	struct dm_rq_target_io *tio;
1851cec47e3dSKiyoshi Ueda 
1852cec47e3dSKiyoshi Ueda 	if (unlikely(rq->special)) {
1853cec47e3dSKiyoshi Ueda 		DMWARN("Already has something in rq->special.");
1854cec47e3dSKiyoshi Ueda 		return BLKPREP_KILL;
1855cec47e3dSKiyoshi Ueda 	}
1856cec47e3dSKiyoshi Ueda 
1857466d89a6SKeith Busch 	tio = prep_tio(rq, md, GFP_ATOMIC);
1858466d89a6SKeith Busch 	if (!tio)
1859cec47e3dSKiyoshi Ueda 		return BLKPREP_DEFER;
1860cec47e3dSKiyoshi Ueda 
1861466d89a6SKeith Busch 	rq->special = tio;
1862cec47e3dSKiyoshi Ueda 	rq->cmd_flags |= REQ_DONTPREP;
1863cec47e3dSKiyoshi Ueda 
1864cec47e3dSKiyoshi Ueda 	return BLKPREP_OK;
1865cec47e3dSKiyoshi Ueda }
1866cec47e3dSKiyoshi Ueda 
18679eef87daSKiyoshi Ueda /*
18689eef87daSKiyoshi Ueda  * Returns:
1869e5863d9aSMike Snitzer  * 0                : the request has been processed
1870e5863d9aSMike Snitzer  * DM_MAPIO_REQUEUE : the original request needs to be requeued
1871e5863d9aSMike Snitzer  * < 0              : the request was completed due to failure
18729eef87daSKiyoshi Ueda  */
1873bfebd1cdSMike Snitzer static int map_request(struct dm_rq_target_io *tio, struct request *rq,
1874cec47e3dSKiyoshi Ueda 		       struct mapped_device *md)
1875cec47e3dSKiyoshi Ueda {
1876e5863d9aSMike Snitzer 	int r;
1877bfebd1cdSMike Snitzer 	struct dm_target *ti = tio->ti;
1878e5863d9aSMike Snitzer 	struct request *clone = NULL;
1879cec47e3dSKiyoshi Ueda 
1880e5863d9aSMike Snitzer 	if (tio->clone) {
1881e5863d9aSMike Snitzer 		clone = tio->clone;
1882cec47e3dSKiyoshi Ueda 		r = ti->type->map_rq(ti, clone, &tio->info);
1883e5863d9aSMike Snitzer 	} else {
1884e5863d9aSMike Snitzer 		r = ti->type->clone_and_map_rq(ti, rq, &tio->info, &clone);
1885e5863d9aSMike Snitzer 		if (r < 0) {
1886e5863d9aSMike Snitzer 			/* The target wants to complete the I/O */
1887e5863d9aSMike Snitzer 			dm_kill_unmapped_request(rq, r);
1888e5863d9aSMike Snitzer 			return r;
1889e5863d9aSMike Snitzer 		}
1890e5863d9aSMike Snitzer 		if (IS_ERR(clone))
1891e5863d9aSMike Snitzer 			return DM_MAPIO_REQUEUE;
1892*5f1b670dSChristoph Hellwig 		setup_clone(clone, rq, tio);
1893e5863d9aSMike Snitzer 	}
1894e5863d9aSMike Snitzer 
1895cec47e3dSKiyoshi Ueda 	switch (r) {
1896cec47e3dSKiyoshi Ueda 	case DM_MAPIO_SUBMITTED:
1897cec47e3dSKiyoshi Ueda 		/* The target has taken the I/O to submit by itself later */
1898cec47e3dSKiyoshi Ueda 		break;
1899cec47e3dSKiyoshi Ueda 	case DM_MAPIO_REMAPPED:
1900cec47e3dSKiyoshi Ueda 		/* The target has remapped the I/O so dispatch it */
19016db4ccd6SJun'ichi Nomura 		trace_block_rq_remap(clone->q, clone, disk_devt(dm_disk(md)),
1902466d89a6SKeith Busch 				     blk_rq_pos(rq));
1903466d89a6SKeith Busch 		dm_dispatch_clone_request(clone, rq);
1904cec47e3dSKiyoshi Ueda 		break;
1905cec47e3dSKiyoshi Ueda 	case DM_MAPIO_REQUEUE:
1906cec47e3dSKiyoshi Ueda 		/* The target wants to requeue the I/O */
1907cec47e3dSKiyoshi Ueda 		dm_requeue_unmapped_request(clone);
1908cec47e3dSKiyoshi Ueda 		break;
1909cec47e3dSKiyoshi Ueda 	default:
1910cec47e3dSKiyoshi Ueda 		if (r > 0) {
1911cec47e3dSKiyoshi Ueda 			DMWARN("unimplemented target map return value: %d", r);
1912cec47e3dSKiyoshi Ueda 			BUG();
1913cec47e3dSKiyoshi Ueda 		}
1914cec47e3dSKiyoshi Ueda 
1915cec47e3dSKiyoshi Ueda 		/* The target wants to complete the I/O */
1916466d89a6SKeith Busch 		dm_kill_unmapped_request(rq, r);
1917e5863d9aSMike Snitzer 		return r;
1918cec47e3dSKiyoshi Ueda 	}
19199eef87daSKiyoshi Ueda 
1920e5863d9aSMike Snitzer 	return 0;
1921cec47e3dSKiyoshi Ueda }
1922cec47e3dSKiyoshi Ueda 
19232eb6e1e3SKeith Busch static void map_tio_request(struct kthread_work *work)
1924ba1cbad9SMike Snitzer {
19252eb6e1e3SKeith Busch 	struct dm_rq_target_io *tio = container_of(work, struct dm_rq_target_io, work);
1926e5863d9aSMike Snitzer 	struct request *rq = tio->orig;
1927e5863d9aSMike Snitzer 	struct mapped_device *md = tio->md;
1928ba1cbad9SMike Snitzer 
1929bfebd1cdSMike Snitzer 	if (map_request(tio, rq, md) == DM_MAPIO_REQUEUE)
1930e5863d9aSMike Snitzer 		dm_requeue_unmapped_original_request(md, rq);
19312eb6e1e3SKeith Busch }
19322eb6e1e3SKeith Busch 
1933466d89a6SKeith Busch static void dm_start_request(struct mapped_device *md, struct request *orig)
1934ba1cbad9SMike Snitzer {
1935bfebd1cdSMike Snitzer 	if (!orig->q->mq_ops)
1936ba1cbad9SMike Snitzer 		blk_start_request(orig);
1937bfebd1cdSMike Snitzer 	else
1938bfebd1cdSMike Snitzer 		blk_mq_start_request(orig);
1939466d89a6SKeith Busch 	atomic_inc(&md->pending[rq_data_dir(orig)]);
1940ba1cbad9SMike Snitzer 
19410ce65797SMike Snitzer 	if (md->seq_rq_merge_deadline_usecs) {
1942de3ec86dSMike Snitzer 		md->last_rq_pos = rq_end_sector(orig);
1943de3ec86dSMike Snitzer 		md->last_rq_rw = rq_data_dir(orig);
19440ce65797SMike Snitzer 		md->last_rq_start_time = ktime_get();
19450ce65797SMike Snitzer 	}
1946de3ec86dSMike Snitzer 
1947ba1cbad9SMike Snitzer 	/*
1948ba1cbad9SMike Snitzer 	 * Hold the md reference here for the in-flight I/O.
1949ba1cbad9SMike Snitzer 	 * We can't rely on the reference count by device opener,
1950ba1cbad9SMike Snitzer 	 * because the device may be closed during the request completion
1951ba1cbad9SMike Snitzer 	 * when all bios are completed.
1952ba1cbad9SMike Snitzer 	 * See the comment in rq_completed() too.
1953ba1cbad9SMike Snitzer 	 */
1954ba1cbad9SMike Snitzer 	dm_get(md);
1955ba1cbad9SMike Snitzer }
1956ba1cbad9SMike Snitzer 
19570ce65797SMike Snitzer #define MAX_SEQ_RQ_MERGE_DEADLINE_USECS 100000
19580ce65797SMike Snitzer 
19590ce65797SMike Snitzer ssize_t dm_attr_rq_based_seq_io_merge_deadline_show(struct mapped_device *md, char *buf)
19600ce65797SMike Snitzer {
19610ce65797SMike Snitzer 	return sprintf(buf, "%u\n", md->seq_rq_merge_deadline_usecs);
19620ce65797SMike Snitzer }
19630ce65797SMike Snitzer 
19640ce65797SMike Snitzer ssize_t dm_attr_rq_based_seq_io_merge_deadline_store(struct mapped_device *md,
19650ce65797SMike Snitzer 						     const char *buf, size_t count)
19660ce65797SMike Snitzer {
19670ce65797SMike Snitzer 	unsigned deadline;
19680ce65797SMike Snitzer 
196917e149b8SMike Snitzer 	if (!dm_request_based(md) || md->use_blk_mq)
19700ce65797SMike Snitzer 		return count;
19710ce65797SMike Snitzer 
19720ce65797SMike Snitzer 	if (kstrtouint(buf, 10, &deadline))
19730ce65797SMike Snitzer 		return -EINVAL;
19740ce65797SMike Snitzer 
19750ce65797SMike Snitzer 	if (deadline > MAX_SEQ_RQ_MERGE_DEADLINE_USECS)
19760ce65797SMike Snitzer 		deadline = MAX_SEQ_RQ_MERGE_DEADLINE_USECS;
19770ce65797SMike Snitzer 
19780ce65797SMike Snitzer 	md->seq_rq_merge_deadline_usecs = deadline;
19790ce65797SMike Snitzer 
19800ce65797SMike Snitzer 	return count;
19810ce65797SMike Snitzer }
19820ce65797SMike Snitzer 
19830ce65797SMike Snitzer static bool dm_request_peeked_before_merge_deadline(struct mapped_device *md)
19840ce65797SMike Snitzer {
19850ce65797SMike Snitzer 	ktime_t kt_deadline;
19860ce65797SMike Snitzer 
19870ce65797SMike Snitzer 	if (!md->seq_rq_merge_deadline_usecs)
19880ce65797SMike Snitzer 		return false;
19890ce65797SMike Snitzer 
19900ce65797SMike Snitzer 	kt_deadline = ns_to_ktime((u64)md->seq_rq_merge_deadline_usecs * NSEC_PER_USEC);
19910ce65797SMike Snitzer 	kt_deadline = ktime_add_safe(md->last_rq_start_time, kt_deadline);
19920ce65797SMike Snitzer 
19930ce65797SMike Snitzer 	return !ktime_after(ktime_get(), kt_deadline);
19940ce65797SMike Snitzer }
19950ce65797SMike Snitzer 
1996cec47e3dSKiyoshi Ueda /*
1997cec47e3dSKiyoshi Ueda  * q->request_fn for request-based dm.
1998cec47e3dSKiyoshi Ueda  * Called with the queue lock held.
1999cec47e3dSKiyoshi Ueda  */
2000cec47e3dSKiyoshi Ueda static void dm_request_fn(struct request_queue *q)
2001cec47e3dSKiyoshi Ueda {
2002cec47e3dSKiyoshi Ueda 	struct mapped_device *md = q->queuedata;
200383d5e5b0SMikulas Patocka 	int srcu_idx;
200483d5e5b0SMikulas Patocka 	struct dm_table *map = dm_get_live_table(md, &srcu_idx);
2005cec47e3dSKiyoshi Ueda 	struct dm_target *ti;
2006466d89a6SKeith Busch 	struct request *rq;
20072eb6e1e3SKeith Busch 	struct dm_rq_target_io *tio;
200829e4013dSTejun Heo 	sector_t pos;
2009cec47e3dSKiyoshi Ueda 
2010cec47e3dSKiyoshi Ueda 	/*
2011b4324feeSKiyoshi Ueda 	 * For suspend, check blk_queue_stopped() and increment
2012b4324feeSKiyoshi Ueda 	 * ->pending within a single queue_lock not to increment the
2013b4324feeSKiyoshi Ueda 	 * number of in-flight I/Os after the queue is stopped in
2014b4324feeSKiyoshi Ueda 	 * dm_suspend().
2015cec47e3dSKiyoshi Ueda 	 */
20167eaceaccSJens Axboe 	while (!blk_queue_stopped(q)) {
2017cec47e3dSKiyoshi Ueda 		rq = blk_peek_request(q);
2018cec47e3dSKiyoshi Ueda 		if (!rq)
20199d1deb83SMike Snitzer 			goto out;
2020cec47e3dSKiyoshi Ueda 
202129e4013dSTejun Heo 		/* always use block 0 to find the target for flushes for now */
202229e4013dSTejun Heo 		pos = 0;
202329e4013dSTejun Heo 		if (!(rq->cmd_flags & REQ_FLUSH))
202429e4013dSTejun Heo 			pos = blk_rq_pos(rq);
2025d0bcb878SKiyoshi Ueda 
202629e4013dSTejun Heo 		ti = dm_table_find_target(map, pos);
2027ba1cbad9SMike Snitzer 		if (!dm_target_is_valid(ti)) {
2028ba1cbad9SMike Snitzer 			/*
2029466d89a6SKeith Busch 			 * Must perform setup, that rq_completed() requires,
2030ba1cbad9SMike Snitzer 			 * before calling dm_kill_unmapped_request
2031ba1cbad9SMike Snitzer 			 */
2032ba1cbad9SMike Snitzer 			DMERR_LIMIT("request attempted access beyond the end of device");
2033466d89a6SKeith Busch 			dm_start_request(md, rq);
2034466d89a6SKeith Busch 			dm_kill_unmapped_request(rq, -EIO);
2035ba1cbad9SMike Snitzer 			continue;
2036ba1cbad9SMike Snitzer 		}
203729e4013dSTejun Heo 
20380ce65797SMike Snitzer 		if (dm_request_peeked_before_merge_deadline(md) &&
20390ce65797SMike Snitzer 		    md_in_flight(md) && rq->bio && rq->bio->bi_vcnt == 1 &&
2040de3ec86dSMike Snitzer 		    md->last_rq_pos == pos && md->last_rq_rw == rq_data_dir(rq))
2041de3ec86dSMike Snitzer 			goto delay_and_out;
2042de3ec86dSMike Snitzer 
2043cec47e3dSKiyoshi Ueda 		if (ti->type->busy && ti->type->busy(ti))
20447eaceaccSJens Axboe 			goto delay_and_out;
2045cec47e3dSKiyoshi Ueda 
2046466d89a6SKeith Busch 		dm_start_request(md, rq);
2047b4324feeSKiyoshi Ueda 
2048bfebd1cdSMike Snitzer 		tio = tio_from_request(rq);
20492eb6e1e3SKeith Busch 		/* Establish tio->ti before queuing work (map_tio_request) */
20502eb6e1e3SKeith Busch 		tio->ti = ti;
20512eb6e1e3SKeith Busch 		queue_kthread_work(&md->kworker, &tio->work);
2052052189a2SKiyoshi Ueda 		BUG_ON(!irqs_disabled());
2053cec47e3dSKiyoshi Ueda 	}
2054cec47e3dSKiyoshi Ueda 
2055cec47e3dSKiyoshi Ueda 	goto out;
2056cec47e3dSKiyoshi Ueda 
20577eaceaccSJens Axboe delay_and_out:
2058d548b34bSMike Snitzer 	blk_delay_queue(q, HZ / 100);
2059cec47e3dSKiyoshi Ueda out:
206083d5e5b0SMikulas Patocka 	dm_put_live_table(md, srcu_idx);
2061cec47e3dSKiyoshi Ueda }
2062cec47e3dSKiyoshi Ueda 
20631da177e4SLinus Torvalds static int dm_any_congested(void *congested_data, int bdi_bits)
20641da177e4SLinus Torvalds {
20658a57dfc6SChandra Seetharaman 	int r = bdi_bits;
20668a57dfc6SChandra Seetharaman 	struct mapped_device *md = congested_data;
20678a57dfc6SChandra Seetharaman 	struct dm_table *map;
20681da177e4SLinus Torvalds 
20691eb787ecSAlasdair G Kergon 	if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
207083d5e5b0SMikulas Patocka 		map = dm_get_live_table_fast(md);
20718a57dfc6SChandra Seetharaman 		if (map) {
2072cec47e3dSKiyoshi Ueda 			/*
2073cec47e3dSKiyoshi Ueda 			 * Request-based dm cares about only own queue for
2074cec47e3dSKiyoshi Ueda 			 * the query about congestion status of request_queue
2075cec47e3dSKiyoshi Ueda 			 */
2076cec47e3dSKiyoshi Ueda 			if (dm_request_based(md))
2077cec47e3dSKiyoshi Ueda 				r = md->queue->backing_dev_info.state &
2078cec47e3dSKiyoshi Ueda 				    bdi_bits;
2079cec47e3dSKiyoshi Ueda 			else
20801da177e4SLinus Torvalds 				r = dm_table_any_congested(map, bdi_bits);
20818a57dfc6SChandra Seetharaman 		}
208283d5e5b0SMikulas Patocka 		dm_put_live_table_fast(md);
20838a57dfc6SChandra Seetharaman 	}
20848a57dfc6SChandra Seetharaman 
20851da177e4SLinus Torvalds 	return r;
20861da177e4SLinus Torvalds }
20871da177e4SLinus Torvalds 
20881da177e4SLinus Torvalds /*-----------------------------------------------------------------
20891da177e4SLinus Torvalds  * An IDR is used to keep track of allocated minor numbers.
20901da177e4SLinus Torvalds  *---------------------------------------------------------------*/
20912b06cfffSAlasdair G Kergon static void free_minor(int minor)
20921da177e4SLinus Torvalds {
2093f32c10b0SJeff Mahoney 	spin_lock(&_minor_lock);
20941da177e4SLinus Torvalds 	idr_remove(&_minor_idr, minor);
2095f32c10b0SJeff Mahoney 	spin_unlock(&_minor_lock);
20961da177e4SLinus Torvalds }
20971da177e4SLinus Torvalds 
20981da177e4SLinus Torvalds /*
20991da177e4SLinus Torvalds  * See if the device with a specific minor # is free.
21001da177e4SLinus Torvalds  */
2101cf13ab8eSFrederik Deweerdt static int specific_minor(int minor)
21021da177e4SLinus Torvalds {
2103c9d76be6STejun Heo 	int r;
21041da177e4SLinus Torvalds 
21051da177e4SLinus Torvalds 	if (minor >= (1 << MINORBITS))
21061da177e4SLinus Torvalds 		return -EINVAL;
21071da177e4SLinus Torvalds 
2108c9d76be6STejun Heo 	idr_preload(GFP_KERNEL);
2109f32c10b0SJeff Mahoney 	spin_lock(&_minor_lock);
21101da177e4SLinus Torvalds 
2111c9d76be6STejun Heo 	r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT);
21121da177e4SLinus Torvalds 
2113f32c10b0SJeff Mahoney 	spin_unlock(&_minor_lock);
2114c9d76be6STejun Heo 	idr_preload_end();
2115c9d76be6STejun Heo 	if (r < 0)
2116c9d76be6STejun Heo 		return r == -ENOSPC ? -EBUSY : r;
2117c9d76be6STejun Heo 	return 0;
21181da177e4SLinus Torvalds }
21191da177e4SLinus Torvalds 
2120cf13ab8eSFrederik Deweerdt static int next_free_minor(int *minor)
21211da177e4SLinus Torvalds {
2122c9d76be6STejun Heo 	int r;
21231da177e4SLinus Torvalds 
2124c9d76be6STejun Heo 	idr_preload(GFP_KERNEL);
2125f32c10b0SJeff Mahoney 	spin_lock(&_minor_lock);
21261da177e4SLinus Torvalds 
2127c9d76be6STejun Heo 	r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT);
21281da177e4SLinus Torvalds 
2129f32c10b0SJeff Mahoney 	spin_unlock(&_minor_lock);
2130c9d76be6STejun Heo 	idr_preload_end();
2131c9d76be6STejun Heo 	if (r < 0)
21321da177e4SLinus Torvalds 		return r;
2133c9d76be6STejun Heo 	*minor = r;
2134c9d76be6STejun Heo 	return 0;
21351da177e4SLinus Torvalds }
21361da177e4SLinus Torvalds 
213783d5cde4SAlexey Dobriyan static const struct block_device_operations dm_blk_dops;
21381da177e4SLinus Torvalds 
213953d5914fSMikulas Patocka static void dm_wq_work(struct work_struct *work);
214053d5914fSMikulas Patocka 
21414a0b4ddfSMike Snitzer static void dm_init_md_queue(struct mapped_device *md)
21424a0b4ddfSMike Snitzer {
21434a0b4ddfSMike Snitzer 	/*
21444a0b4ddfSMike Snitzer 	 * Request-based dm devices cannot be stacked on top of bio-based dm
2145bfebd1cdSMike Snitzer 	 * devices.  The type of this dm device may not have been decided yet.
21464a0b4ddfSMike Snitzer 	 * The type is decided at the first table loading time.
21474a0b4ddfSMike Snitzer 	 * To prevent problematic device stacking, clear the queue flag
21484a0b4ddfSMike Snitzer 	 * for request stacking support until then.
21494a0b4ddfSMike Snitzer 	 *
21504a0b4ddfSMike Snitzer 	 * This queue is new, so no concurrency on the queue_flags.
21514a0b4ddfSMike Snitzer 	 */
21524a0b4ddfSMike Snitzer 	queue_flag_clear_unlocked(QUEUE_FLAG_STACKABLE, md->queue);
2153bfebd1cdSMike Snitzer }
21544a0b4ddfSMike Snitzer 
2155bfebd1cdSMike Snitzer static void dm_init_old_md_queue(struct mapped_device *md)
2156bfebd1cdSMike Snitzer {
215717e149b8SMike Snitzer 	md->use_blk_mq = false;
2158bfebd1cdSMike Snitzer 	dm_init_md_queue(md);
2159bfebd1cdSMike Snitzer 
2160bfebd1cdSMike Snitzer 	/*
2161bfebd1cdSMike Snitzer 	 * Initialize aspects of queue that aren't relevant for blk-mq
2162bfebd1cdSMike Snitzer 	 */
21634a0b4ddfSMike Snitzer 	md->queue->queuedata = md;
21644a0b4ddfSMike Snitzer 	md->queue->backing_dev_info.congested_fn = dm_any_congested;
21654a0b4ddfSMike Snitzer 	md->queue->backing_dev_info.congested_data = md;
2166ff36ab34SMike Snitzer 
21674a0b4ddfSMike Snitzer 	blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
21684a0b4ddfSMike Snitzer }
21694a0b4ddfSMike Snitzer 
21701da177e4SLinus Torvalds /*
21711da177e4SLinus Torvalds  * Allocate and initialise a blank device with a given minor.
21721da177e4SLinus Torvalds  */
21732b06cfffSAlasdair G Kergon static struct mapped_device *alloc_dev(int minor)
21741da177e4SLinus Torvalds {
21751da177e4SLinus Torvalds 	int r;
2176cf13ab8eSFrederik Deweerdt 	struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL);
2177ba61fdd1SJeff Mahoney 	void *old_md;
21781da177e4SLinus Torvalds 
21791da177e4SLinus Torvalds 	if (!md) {
21801da177e4SLinus Torvalds 		DMWARN("unable to allocate device, out of memory.");
21811da177e4SLinus Torvalds 		return NULL;
21821da177e4SLinus Torvalds 	}
21831da177e4SLinus Torvalds 
218410da4f79SJeff Mahoney 	if (!try_module_get(THIS_MODULE))
21856ed7ade8SMilan Broz 		goto bad_module_get;
218610da4f79SJeff Mahoney 
21871da177e4SLinus Torvalds 	/* get a minor number for the dev */
21882b06cfffSAlasdair G Kergon 	if (minor == DM_ANY_MINOR)
2189cf13ab8eSFrederik Deweerdt 		r = next_free_minor(&minor);
21902b06cfffSAlasdair G Kergon 	else
2191cf13ab8eSFrederik Deweerdt 		r = specific_minor(minor);
21921da177e4SLinus Torvalds 	if (r < 0)
21936ed7ade8SMilan Broz 		goto bad_minor;
21941da177e4SLinus Torvalds 
219583d5e5b0SMikulas Patocka 	r = init_srcu_struct(&md->io_barrier);
219683d5e5b0SMikulas Patocka 	if (r < 0)
219783d5e5b0SMikulas Patocka 		goto bad_io_barrier;
219883d5e5b0SMikulas Patocka 
219917e149b8SMike Snitzer 	md->use_blk_mq = use_blk_mq;
2200a5664dadSMike Snitzer 	md->type = DM_TYPE_NONE;
2201e61290a4SDaniel Walker 	mutex_init(&md->suspend_lock);
2202a5664dadSMike Snitzer 	mutex_init(&md->type_lock);
220386f1152bSBenjamin Marzinski 	mutex_init(&md->table_devices_lock);
2204022c2611SMikulas Patocka 	spin_lock_init(&md->deferred_lock);
22051da177e4SLinus Torvalds 	atomic_set(&md->holders, 1);
22065c6bd75dSAlasdair G Kergon 	atomic_set(&md->open_count, 0);
22071da177e4SLinus Torvalds 	atomic_set(&md->event_nr, 0);
22087a8c3d3bSMike Anderson 	atomic_set(&md->uevent_seq, 0);
22097a8c3d3bSMike Anderson 	INIT_LIST_HEAD(&md->uevent_list);
221086f1152bSBenjamin Marzinski 	INIT_LIST_HEAD(&md->table_devices);
22117a8c3d3bSMike Anderson 	spin_lock_init(&md->uevent_lock);
22121da177e4SLinus Torvalds 
22134a0b4ddfSMike Snitzer 	md->queue = blk_alloc_queue(GFP_KERNEL);
22141da177e4SLinus Torvalds 	if (!md->queue)
22156ed7ade8SMilan Broz 		goto bad_queue;
22161da177e4SLinus Torvalds 
22174a0b4ddfSMike Snitzer 	dm_init_md_queue(md);
22189faf400fSStefan Bader 
22191da177e4SLinus Torvalds 	md->disk = alloc_disk(1);
22201da177e4SLinus Torvalds 	if (!md->disk)
22216ed7ade8SMilan Broz 		goto bad_disk;
22221da177e4SLinus Torvalds 
2223316d315bSNikanth Karthikesan 	atomic_set(&md->pending[0], 0);
2224316d315bSNikanth Karthikesan 	atomic_set(&md->pending[1], 0);
2225f0b04115SJeff Mahoney 	init_waitqueue_head(&md->wait);
222653d5914fSMikulas Patocka 	INIT_WORK(&md->work, dm_wq_work);
2227f0b04115SJeff Mahoney 	init_waitqueue_head(&md->eventq);
22282995fa78SMikulas Patocka 	init_completion(&md->kobj_holder.completion);
22292eb6e1e3SKeith Busch 	md->kworker_task = NULL;
2230f0b04115SJeff Mahoney 
22311da177e4SLinus Torvalds 	md->disk->major = _major;
22321da177e4SLinus Torvalds 	md->disk->first_minor = minor;
22331da177e4SLinus Torvalds 	md->disk->fops = &dm_blk_dops;
22341da177e4SLinus Torvalds 	md->disk->queue = md->queue;
22351da177e4SLinus Torvalds 	md->disk->private_data = md;
22361da177e4SLinus Torvalds 	sprintf(md->disk->disk_name, "dm-%d", minor);
22371da177e4SLinus Torvalds 	add_disk(md->disk);
22387e51f257SMike Anderson 	format_dev_t(md->name, MKDEV(_major, minor));
22391da177e4SLinus Torvalds 
2240670368a8STejun Heo 	md->wq = alloc_workqueue("kdmflush", WQ_MEM_RECLAIM, 0);
2241304f3f6aSMilan Broz 	if (!md->wq)
2242304f3f6aSMilan Broz 		goto bad_thread;
2243304f3f6aSMilan Broz 
224432a926daSMikulas Patocka 	md->bdev = bdget_disk(md->disk, 0);
224532a926daSMikulas Patocka 	if (!md->bdev)
224632a926daSMikulas Patocka 		goto bad_bdev;
224732a926daSMikulas Patocka 
22486a8736d1STejun Heo 	bio_init(&md->flush_bio);
22496a8736d1STejun Heo 	md->flush_bio.bi_bdev = md->bdev;
22506a8736d1STejun Heo 	md->flush_bio.bi_rw = WRITE_FLUSH;
22516a8736d1STejun Heo 
2252fd2ed4d2SMikulas Patocka 	dm_stats_init(&md->stats);
2253fd2ed4d2SMikulas Patocka 
2254ba61fdd1SJeff Mahoney 	/* Populate the mapping, nobody knows we exist yet */
2255f32c10b0SJeff Mahoney 	spin_lock(&_minor_lock);
2256ba61fdd1SJeff Mahoney 	old_md = idr_replace(&_minor_idr, md, minor);
2257f32c10b0SJeff Mahoney 	spin_unlock(&_minor_lock);
2258ba61fdd1SJeff Mahoney 
2259ba61fdd1SJeff Mahoney 	BUG_ON(old_md != MINOR_ALLOCED);
2260ba61fdd1SJeff Mahoney 
22611da177e4SLinus Torvalds 	return md;
22621da177e4SLinus Torvalds 
226332a926daSMikulas Patocka bad_bdev:
226432a926daSMikulas Patocka 	destroy_workqueue(md->wq);
2265304f3f6aSMilan Broz bad_thread:
226603022c54SZdenek Kabelac 	del_gendisk(md->disk);
2267304f3f6aSMilan Broz 	put_disk(md->disk);
22686ed7ade8SMilan Broz bad_disk:
22691312f40eSAl Viro 	blk_cleanup_queue(md->queue);
22706ed7ade8SMilan Broz bad_queue:
227183d5e5b0SMikulas Patocka 	cleanup_srcu_struct(&md->io_barrier);
227283d5e5b0SMikulas Patocka bad_io_barrier:
22731da177e4SLinus Torvalds 	free_minor(minor);
22746ed7ade8SMilan Broz bad_minor:
227510da4f79SJeff Mahoney 	module_put(THIS_MODULE);
22766ed7ade8SMilan Broz bad_module_get:
22771da177e4SLinus Torvalds 	kfree(md);
22781da177e4SLinus Torvalds 	return NULL;
22791da177e4SLinus Torvalds }
22801da177e4SLinus Torvalds 
2281ae9da83fSJun'ichi Nomura static void unlock_fs(struct mapped_device *md);
2282ae9da83fSJun'ichi Nomura 
22831da177e4SLinus Torvalds static void free_dev(struct mapped_device *md)
22841da177e4SLinus Torvalds {
2285f331c029STejun Heo 	int minor = MINOR(disk_devt(md->disk));
228663d94e48SJun'ichi Nomura 
2287ae9da83fSJun'ichi Nomura 	unlock_fs(md);
2288304f3f6aSMilan Broz 	destroy_workqueue(md->wq);
22892eb6e1e3SKeith Busch 
22902eb6e1e3SKeith Busch 	if (md->kworker_task)
22912eb6e1e3SKeith Busch 		kthread_stop(md->kworker_task);
2292e6ee8c0bSKiyoshi Ueda 	if (md->io_pool)
22931da177e4SLinus Torvalds 		mempool_destroy(md->io_pool);
22941ae49ea2SMike Snitzer 	if (md->rq_pool)
22951ae49ea2SMike Snitzer 		mempool_destroy(md->rq_pool);
2296e6ee8c0bSKiyoshi Ueda 	if (md->bs)
22979faf400fSStefan Bader 		bioset_free(md->bs);
229863a4f065SMike Snitzer 
229983d5e5b0SMikulas Patocka 	cleanup_srcu_struct(&md->io_barrier);
230086f1152bSBenjamin Marzinski 	free_table_devices(&md->table_devices);
230163a4f065SMike Snitzer 	dm_stats_cleanup(&md->stats);
2302fba9f90eSJeff Mahoney 
2303fba9f90eSJeff Mahoney 	spin_lock(&_minor_lock);
2304fba9f90eSJeff Mahoney 	md->disk->private_data = NULL;
2305fba9f90eSJeff Mahoney 	spin_unlock(&_minor_lock);
230663a4f065SMike Snitzer 	if (blk_get_integrity(md->disk))
230763a4f065SMike Snitzer 		blk_integrity_unregister(md->disk);
230863a4f065SMike Snitzer 	del_gendisk(md->disk);
23091da177e4SLinus Torvalds 	put_disk(md->disk);
23101312f40eSAl Viro 	blk_cleanup_queue(md->queue);
231117e149b8SMike Snitzer 	if (md->use_blk_mq)
2312bfebd1cdSMike Snitzer 		blk_mq_free_tag_set(&md->tag_set);
231363a4f065SMike Snitzer 	bdput(md->bdev);
231463a4f065SMike Snitzer 	free_minor(minor);
231563a4f065SMike Snitzer 
231610da4f79SJeff Mahoney 	module_put(THIS_MODULE);
23171da177e4SLinus Torvalds 	kfree(md);
23181da177e4SLinus Torvalds }
23191da177e4SLinus Torvalds 
2320e6ee8c0bSKiyoshi Ueda static void __bind_mempools(struct mapped_device *md, struct dm_table *t)
2321e6ee8c0bSKiyoshi Ueda {
2322c0820cf5SMikulas Patocka 	struct dm_md_mempools *p = dm_table_get_md_mempools(t);
2323e6ee8c0bSKiyoshi Ueda 
232417e149b8SMike Snitzer 	if (md->bs) {
232516245bdcSJun'ichi Nomura 		/* The md already has necessary mempools. */
232616245bdcSJun'ichi Nomura 		if (dm_table_get_type(t) == DM_TYPE_BIO_BASED) {
2327c0820cf5SMikulas Patocka 			/*
232816245bdcSJun'ichi Nomura 			 * Reload bioset because front_pad may have changed
232916245bdcSJun'ichi Nomura 			 * because a different table was loaded.
2330c0820cf5SMikulas Patocka 			 */
2331c0820cf5SMikulas Patocka 			bioset_free(md->bs);
2332c0820cf5SMikulas Patocka 			md->bs = p->bs;
2333c0820cf5SMikulas Patocka 			p->bs = NULL;
2334466d89a6SKeith Busch 		}
233516245bdcSJun'ichi Nomura 		/*
233616245bdcSJun'ichi Nomura 		 * There's no need to reload with request-based dm
233716245bdcSJun'ichi Nomura 		 * because the size of front_pad doesn't change.
233816245bdcSJun'ichi Nomura 		 * Note for future: If you are to reload bioset,
233916245bdcSJun'ichi Nomura 		 * prep-ed requests in the queue may refer
234016245bdcSJun'ichi Nomura 		 * to bio from the old bioset, so you must walk
234116245bdcSJun'ichi Nomura 		 * through the queue to unprep.
234216245bdcSJun'ichi Nomura 		 */
2343e6ee8c0bSKiyoshi Ueda 		goto out;
2344c0820cf5SMikulas Patocka 	}
2345e6ee8c0bSKiyoshi Ueda 
2346e6ee8c0bSKiyoshi Ueda 	md->io_pool = p->io_pool;
2347e6ee8c0bSKiyoshi Ueda 	p->io_pool = NULL;
23481ae49ea2SMike Snitzer 	md->rq_pool = p->rq_pool;
23491ae49ea2SMike Snitzer 	p->rq_pool = NULL;
2350e6ee8c0bSKiyoshi Ueda 	md->bs = p->bs;
2351e6ee8c0bSKiyoshi Ueda 	p->bs = NULL;
2352e6ee8c0bSKiyoshi Ueda 
2353e6ee8c0bSKiyoshi Ueda out:
235402233342SMike Snitzer 	/* mempool bind completed, no longer need any mempools in the table */
2355e6ee8c0bSKiyoshi Ueda 	dm_table_free_md_mempools(t);
2356e6ee8c0bSKiyoshi Ueda }
2357e6ee8c0bSKiyoshi Ueda 
23581da177e4SLinus Torvalds /*
23591da177e4SLinus Torvalds  * Bind a table to the device.
23601da177e4SLinus Torvalds  */
23611da177e4SLinus Torvalds static void event_callback(void *context)
23621da177e4SLinus Torvalds {
23637a8c3d3bSMike Anderson 	unsigned long flags;
23647a8c3d3bSMike Anderson 	LIST_HEAD(uevents);
23651da177e4SLinus Torvalds 	struct mapped_device *md = (struct mapped_device *) context;
23661da177e4SLinus Torvalds 
23677a8c3d3bSMike Anderson 	spin_lock_irqsave(&md->uevent_lock, flags);
23687a8c3d3bSMike Anderson 	list_splice_init(&md->uevent_list, &uevents);
23697a8c3d3bSMike Anderson 	spin_unlock_irqrestore(&md->uevent_lock, flags);
23707a8c3d3bSMike Anderson 
2371ed9e1982STejun Heo 	dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
23727a8c3d3bSMike Anderson 
23731da177e4SLinus Torvalds 	atomic_inc(&md->event_nr);
23741da177e4SLinus Torvalds 	wake_up(&md->eventq);
23751da177e4SLinus Torvalds }
23761da177e4SLinus Torvalds 
2377c217649bSMike Snitzer /*
2378c217649bSMike Snitzer  * Protected by md->suspend_lock obtained by dm_swap_table().
2379c217649bSMike Snitzer  */
23804e90188bSAlasdair G Kergon static void __set_size(struct mapped_device *md, sector_t size)
23811da177e4SLinus Torvalds {
23824e90188bSAlasdair G Kergon 	set_capacity(md->disk, size);
23831da177e4SLinus Torvalds 
2384db8fef4fSMikulas Patocka 	i_size_write(md->bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
23851da177e4SLinus Torvalds }
23861da177e4SLinus Torvalds 
2387042d2a9bSAlasdair G Kergon /*
2388d5b9dd04SMikulas Patocka  * Return 1 if the queue has a compulsory merge_bvec_fn function.
2389d5b9dd04SMikulas Patocka  *
2390d5b9dd04SMikulas Patocka  * If this function returns 0, then the device is either a non-dm
2391d5b9dd04SMikulas Patocka  * device without a merge_bvec_fn, or it is a dm device that is
2392d5b9dd04SMikulas Patocka  * able to split any bios it receives that are too big.
2393d5b9dd04SMikulas Patocka  */
2394d5b9dd04SMikulas Patocka int dm_queue_merge_is_compulsory(struct request_queue *q)
2395d5b9dd04SMikulas Patocka {
2396d5b9dd04SMikulas Patocka 	struct mapped_device *dev_md;
2397d5b9dd04SMikulas Patocka 
2398d5b9dd04SMikulas Patocka 	if (!q->merge_bvec_fn)
2399d5b9dd04SMikulas Patocka 		return 0;
2400d5b9dd04SMikulas Patocka 
2401ff36ab34SMike Snitzer 	if (q->make_request_fn == dm_make_request) {
2402d5b9dd04SMikulas Patocka 		dev_md = q->queuedata;
2403d5b9dd04SMikulas Patocka 		if (test_bit(DMF_MERGE_IS_OPTIONAL, &dev_md->flags))
2404d5b9dd04SMikulas Patocka 			return 0;
2405d5b9dd04SMikulas Patocka 	}
2406d5b9dd04SMikulas Patocka 
2407d5b9dd04SMikulas Patocka 	return 1;
2408d5b9dd04SMikulas Patocka }
2409d5b9dd04SMikulas Patocka 
2410d5b9dd04SMikulas Patocka static int dm_device_merge_is_compulsory(struct dm_target *ti,
2411d5b9dd04SMikulas Patocka 					 struct dm_dev *dev, sector_t start,
2412d5b9dd04SMikulas Patocka 					 sector_t len, void *data)
2413d5b9dd04SMikulas Patocka {
2414d5b9dd04SMikulas Patocka 	struct block_device *bdev = dev->bdev;
2415d5b9dd04SMikulas Patocka 	struct request_queue *q = bdev_get_queue(bdev);
2416d5b9dd04SMikulas Patocka 
2417d5b9dd04SMikulas Patocka 	return dm_queue_merge_is_compulsory(q);
2418d5b9dd04SMikulas Patocka }
2419d5b9dd04SMikulas Patocka 
2420d5b9dd04SMikulas Patocka /*
2421d5b9dd04SMikulas Patocka  * Return 1 if it is acceptable to ignore merge_bvec_fn based
2422d5b9dd04SMikulas Patocka  * on the properties of the underlying devices.
2423d5b9dd04SMikulas Patocka  */
2424d5b9dd04SMikulas Patocka static int dm_table_merge_is_optional(struct dm_table *table)
2425d5b9dd04SMikulas Patocka {
2426d5b9dd04SMikulas Patocka 	unsigned i = 0;
2427d5b9dd04SMikulas Patocka 	struct dm_target *ti;
2428d5b9dd04SMikulas Patocka 
2429d5b9dd04SMikulas Patocka 	while (i < dm_table_get_num_targets(table)) {
2430d5b9dd04SMikulas Patocka 		ti = dm_table_get_target(table, i++);
2431d5b9dd04SMikulas Patocka 
2432d5b9dd04SMikulas Patocka 		if (ti->type->iterate_devices &&
2433d5b9dd04SMikulas Patocka 		    ti->type->iterate_devices(ti, dm_device_merge_is_compulsory, NULL))
2434d5b9dd04SMikulas Patocka 			return 0;
2435d5b9dd04SMikulas Patocka 	}
2436d5b9dd04SMikulas Patocka 
2437d5b9dd04SMikulas Patocka 	return 1;
2438d5b9dd04SMikulas Patocka }
2439d5b9dd04SMikulas Patocka 
2440d5b9dd04SMikulas Patocka /*
2441042d2a9bSAlasdair G Kergon  * Returns old map, which caller must destroy.
2442042d2a9bSAlasdair G Kergon  */
2443042d2a9bSAlasdair G Kergon static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
2444754c5fc7SMike Snitzer 			       struct queue_limits *limits)
24451da177e4SLinus Torvalds {
2446042d2a9bSAlasdair G Kergon 	struct dm_table *old_map;
2447165125e1SJens Axboe 	struct request_queue *q = md->queue;
24481da177e4SLinus Torvalds 	sector_t size;
2449d5b9dd04SMikulas Patocka 	int merge_is_optional;
24501da177e4SLinus Torvalds 
24511da177e4SLinus Torvalds 	size = dm_table_get_size(t);
24523ac51e74SDarrick J. Wong 
24533ac51e74SDarrick J. Wong 	/*
24543ac51e74SDarrick J. Wong 	 * Wipe any geometry if the size of the table changed.
24553ac51e74SDarrick J. Wong 	 */
2456fd2ed4d2SMikulas Patocka 	if (size != dm_get_size(md))
24573ac51e74SDarrick J. Wong 		memset(&md->geometry, 0, sizeof(md->geometry));
24583ac51e74SDarrick J. Wong 
24594e90188bSAlasdair G Kergon 	__set_size(md, size);
24601da177e4SLinus Torvalds 
2461cf222b37SAlasdair G Kergon 	dm_table_event_callback(t, event_callback, md);
24622ca3310eSAlasdair G Kergon 
2463e6ee8c0bSKiyoshi Ueda 	/*
2464e6ee8c0bSKiyoshi Ueda 	 * The queue hasn't been stopped yet, if the old table type wasn't
2465e6ee8c0bSKiyoshi Ueda 	 * for request-based during suspension.  So stop it to prevent
2466e6ee8c0bSKiyoshi Ueda 	 * I/O mapping before resume.
2467e6ee8c0bSKiyoshi Ueda 	 * This must be done before setting the queue restrictions,
2468e6ee8c0bSKiyoshi Ueda 	 * because request-based dm may be run just after the setting.
2469e6ee8c0bSKiyoshi Ueda 	 */
2470bfebd1cdSMike Snitzer 	if (dm_table_request_based(t))
2471e6ee8c0bSKiyoshi Ueda 		stop_queue(q);
2472e6ee8c0bSKiyoshi Ueda 
2473e6ee8c0bSKiyoshi Ueda 	__bind_mempools(md, t);
2474e6ee8c0bSKiyoshi Ueda 
2475d5b9dd04SMikulas Patocka 	merge_is_optional = dm_table_merge_is_optional(t);
2476d5b9dd04SMikulas Patocka 
2477a12f5d48SEric Dumazet 	old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
247883d5e5b0SMikulas Patocka 	rcu_assign_pointer(md->map, t);
247936a0456fSAlasdair G Kergon 	md->immutable_target_type = dm_table_get_immutable_target_type(t);
248036a0456fSAlasdair G Kergon 
2481754c5fc7SMike Snitzer 	dm_table_set_restrictions(t, q, limits);
2482d5b9dd04SMikulas Patocka 	if (merge_is_optional)
2483d5b9dd04SMikulas Patocka 		set_bit(DMF_MERGE_IS_OPTIONAL, &md->flags);
2484d5b9dd04SMikulas Patocka 	else
2485d5b9dd04SMikulas Patocka 		clear_bit(DMF_MERGE_IS_OPTIONAL, &md->flags);
248641abc4e1SHannes Reinecke 	if (old_map)
248783d5e5b0SMikulas Patocka 		dm_sync_table(md);
24882ca3310eSAlasdair G Kergon 
2489042d2a9bSAlasdair G Kergon 	return old_map;
24901da177e4SLinus Torvalds }
24911da177e4SLinus Torvalds 
2492a7940155SAlasdair G Kergon /*
2493a7940155SAlasdair G Kergon  * Returns unbound table for the caller to free.
2494a7940155SAlasdair G Kergon  */
2495a7940155SAlasdair G Kergon static struct dm_table *__unbind(struct mapped_device *md)
24961da177e4SLinus Torvalds {
2497a12f5d48SEric Dumazet 	struct dm_table *map = rcu_dereference_protected(md->map, 1);
24981da177e4SLinus Torvalds 
24991da177e4SLinus Torvalds 	if (!map)
2500a7940155SAlasdair G Kergon 		return NULL;
25011da177e4SLinus Torvalds 
25021da177e4SLinus Torvalds 	dm_table_event_callback(map, NULL, NULL);
25039cdb8520SMonam Agarwal 	RCU_INIT_POINTER(md->map, NULL);
250483d5e5b0SMikulas Patocka 	dm_sync_table(md);
2505a7940155SAlasdair G Kergon 
2506a7940155SAlasdair G Kergon 	return map;
25071da177e4SLinus Torvalds }
25081da177e4SLinus Torvalds 
25091da177e4SLinus Torvalds /*
25101da177e4SLinus Torvalds  * Constructor for a new device.
25111da177e4SLinus Torvalds  */
25122b06cfffSAlasdair G Kergon int dm_create(int minor, struct mapped_device **result)
25131da177e4SLinus Torvalds {
25141da177e4SLinus Torvalds 	struct mapped_device *md;
25151da177e4SLinus Torvalds 
25162b06cfffSAlasdair G Kergon 	md = alloc_dev(minor);
25171da177e4SLinus Torvalds 	if (!md)
25181da177e4SLinus Torvalds 		return -ENXIO;
25191da177e4SLinus Torvalds 
2520784aae73SMilan Broz 	dm_sysfs_init(md);
2521784aae73SMilan Broz 
25221da177e4SLinus Torvalds 	*result = md;
25231da177e4SLinus Torvalds 	return 0;
25241da177e4SLinus Torvalds }
25251da177e4SLinus Torvalds 
2526a5664dadSMike Snitzer /*
2527a5664dadSMike Snitzer  * Functions to manage md->type.
2528a5664dadSMike Snitzer  * All are required to hold md->type_lock.
2529a5664dadSMike Snitzer  */
2530a5664dadSMike Snitzer void dm_lock_md_type(struct mapped_device *md)
2531a5664dadSMike Snitzer {
2532a5664dadSMike Snitzer 	mutex_lock(&md->type_lock);
2533a5664dadSMike Snitzer }
2534a5664dadSMike Snitzer 
2535a5664dadSMike Snitzer void dm_unlock_md_type(struct mapped_device *md)
2536a5664dadSMike Snitzer {
2537a5664dadSMike Snitzer 	mutex_unlock(&md->type_lock);
2538a5664dadSMike Snitzer }
2539a5664dadSMike Snitzer 
2540a5664dadSMike Snitzer void dm_set_md_type(struct mapped_device *md, unsigned type)
2541a5664dadSMike Snitzer {
254200c4fc3bSMike Snitzer 	BUG_ON(!mutex_is_locked(&md->type_lock));
2543a5664dadSMike Snitzer 	md->type = type;
2544a5664dadSMike Snitzer }
2545a5664dadSMike Snitzer 
2546a5664dadSMike Snitzer unsigned dm_get_md_type(struct mapped_device *md)
2547a5664dadSMike Snitzer {
254800c4fc3bSMike Snitzer 	BUG_ON(!mutex_is_locked(&md->type_lock));
2549a5664dadSMike Snitzer 	return md->type;
2550a5664dadSMike Snitzer }
2551a5664dadSMike Snitzer 
255236a0456fSAlasdair G Kergon struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
255336a0456fSAlasdair G Kergon {
255436a0456fSAlasdair G Kergon 	return md->immutable_target_type;
255536a0456fSAlasdair G Kergon }
255636a0456fSAlasdair G Kergon 
25574a0b4ddfSMike Snitzer /*
2558f84cb8a4SMike Snitzer  * The queue_limits are only valid as long as you have a reference
2559f84cb8a4SMike Snitzer  * count on 'md'.
2560f84cb8a4SMike Snitzer  */
2561f84cb8a4SMike Snitzer struct queue_limits *dm_get_queue_limits(struct mapped_device *md)
2562f84cb8a4SMike Snitzer {
2563f84cb8a4SMike Snitzer 	BUG_ON(!atomic_read(&md->holders));
2564f84cb8a4SMike Snitzer 	return &md->queue->limits;
2565f84cb8a4SMike Snitzer }
2566f84cb8a4SMike Snitzer EXPORT_SYMBOL_GPL(dm_get_queue_limits);
2567f84cb8a4SMike Snitzer 
2568bfebd1cdSMike Snitzer static void init_rq_based_worker_thread(struct mapped_device *md)
2569bfebd1cdSMike Snitzer {
2570bfebd1cdSMike Snitzer 	/* Initialize the request-based DM worker thread */
2571bfebd1cdSMike Snitzer 	init_kthread_worker(&md->kworker);
2572bfebd1cdSMike Snitzer 	md->kworker_task = kthread_run(kthread_worker_fn, &md->kworker,
2573bfebd1cdSMike Snitzer 				       "kdmwork-%s", dm_device_name(md));
2574bfebd1cdSMike Snitzer }
2575bfebd1cdSMike Snitzer 
2576f84cb8a4SMike Snitzer /*
25774a0b4ddfSMike Snitzer  * Fully initialize a request-based queue (->elevator, ->request_fn, etc).
25784a0b4ddfSMike Snitzer  */
25794a0b4ddfSMike Snitzer static int dm_init_request_based_queue(struct mapped_device *md)
25804a0b4ddfSMike Snitzer {
25814a0b4ddfSMike Snitzer 	struct request_queue *q = NULL;
25824a0b4ddfSMike Snitzer 
25834a0b4ddfSMike Snitzer 	/* Fully initialize the queue */
25844a0b4ddfSMike Snitzer 	q = blk_init_allocated_queue(md->queue, dm_request_fn, NULL);
25854a0b4ddfSMike Snitzer 	if (!q)
2586bfebd1cdSMike Snitzer 		return -EINVAL;
25874a0b4ddfSMike Snitzer 
25880ce65797SMike Snitzer 	/* disable dm_request_fn's merge heuristic by default */
25890ce65797SMike Snitzer 	md->seq_rq_merge_deadline_usecs = 0;
25900ce65797SMike Snitzer 
25914a0b4ddfSMike Snitzer 	md->queue = q;
2592bfebd1cdSMike Snitzer 	dm_init_old_md_queue(md);
25934a0b4ddfSMike Snitzer 	blk_queue_softirq_done(md->queue, dm_softirq_done);
25944a0b4ddfSMike Snitzer 	blk_queue_prep_rq(md->queue, dm_prep_fn);
25954a0b4ddfSMike Snitzer 
2596bfebd1cdSMike Snitzer 	init_rq_based_worker_thread(md);
25972eb6e1e3SKeith Busch 
25984a0b4ddfSMike Snitzer 	elv_register_queue(md->queue);
25994a0b4ddfSMike Snitzer 
2600bfebd1cdSMike Snitzer 	return 0;
2601bfebd1cdSMike Snitzer }
2602bfebd1cdSMike Snitzer 
2603bfebd1cdSMike Snitzer static int dm_mq_init_request(void *data, struct request *rq,
2604bfebd1cdSMike Snitzer 			      unsigned int hctx_idx, unsigned int request_idx,
2605bfebd1cdSMike Snitzer 			      unsigned int numa_node)
2606bfebd1cdSMike Snitzer {
2607bfebd1cdSMike Snitzer 	struct mapped_device *md = data;
2608bfebd1cdSMike Snitzer 	struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq);
2609bfebd1cdSMike Snitzer 
2610bfebd1cdSMike Snitzer 	/*
2611bfebd1cdSMike Snitzer 	 * Must initialize md member of tio, otherwise it won't
2612bfebd1cdSMike Snitzer 	 * be available in dm_mq_queue_rq.
2613bfebd1cdSMike Snitzer 	 */
2614bfebd1cdSMike Snitzer 	tio->md = md;
2615bfebd1cdSMike Snitzer 
2616bfebd1cdSMike Snitzer 	return 0;
2617bfebd1cdSMike Snitzer }
2618bfebd1cdSMike Snitzer 
2619bfebd1cdSMike Snitzer static int dm_mq_queue_rq(struct blk_mq_hw_ctx *hctx,
2620bfebd1cdSMike Snitzer 			  const struct blk_mq_queue_data *bd)
2621bfebd1cdSMike Snitzer {
2622bfebd1cdSMike Snitzer 	struct request *rq = bd->rq;
2623bfebd1cdSMike Snitzer 	struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq);
2624bfebd1cdSMike Snitzer 	struct mapped_device *md = tio->md;
2625bfebd1cdSMike Snitzer 	int srcu_idx;
2626bfebd1cdSMike Snitzer 	struct dm_table *map = dm_get_live_table(md, &srcu_idx);
2627bfebd1cdSMike Snitzer 	struct dm_target *ti;
2628bfebd1cdSMike Snitzer 	sector_t pos;
2629bfebd1cdSMike Snitzer 
2630bfebd1cdSMike Snitzer 	/* always use block 0 to find the target for flushes for now */
2631bfebd1cdSMike Snitzer 	pos = 0;
2632bfebd1cdSMike Snitzer 	if (!(rq->cmd_flags & REQ_FLUSH))
2633bfebd1cdSMike Snitzer 		pos = blk_rq_pos(rq);
2634bfebd1cdSMike Snitzer 
2635bfebd1cdSMike Snitzer 	ti = dm_table_find_target(map, pos);
2636bfebd1cdSMike Snitzer 	if (!dm_target_is_valid(ti)) {
2637bfebd1cdSMike Snitzer 		dm_put_live_table(md, srcu_idx);
2638bfebd1cdSMike Snitzer 		DMERR_LIMIT("request attempted access beyond the end of device");
2639bfebd1cdSMike Snitzer 		/*
2640bfebd1cdSMike Snitzer 		 * Must perform setup, that rq_completed() requires,
2641bfebd1cdSMike Snitzer 		 * before returning BLK_MQ_RQ_QUEUE_ERROR
2642bfebd1cdSMike Snitzer 		 */
2643bfebd1cdSMike Snitzer 		dm_start_request(md, rq);
2644bfebd1cdSMike Snitzer 		return BLK_MQ_RQ_QUEUE_ERROR;
2645bfebd1cdSMike Snitzer 	}
2646bfebd1cdSMike Snitzer 	dm_put_live_table(md, srcu_idx);
2647bfebd1cdSMike Snitzer 
2648bfebd1cdSMike Snitzer 	if (ti->type->busy && ti->type->busy(ti))
2649bfebd1cdSMike Snitzer 		return BLK_MQ_RQ_QUEUE_BUSY;
2650bfebd1cdSMike Snitzer 
2651bfebd1cdSMike Snitzer 	dm_start_request(md, rq);
2652bfebd1cdSMike Snitzer 
2653bfebd1cdSMike Snitzer 	/* Init tio using md established in .init_request */
2654bfebd1cdSMike Snitzer 	init_tio(tio, rq, md);
2655bfebd1cdSMike Snitzer 
265602233342SMike Snitzer 	/*
265702233342SMike Snitzer 	 * Establish tio->ti before queuing work (map_tio_request)
265802233342SMike Snitzer 	 * or making direct call to map_request().
265902233342SMike Snitzer 	 */
2660bfebd1cdSMike Snitzer 	tio->ti = ti;
266102233342SMike Snitzer 
266202233342SMike Snitzer 	/* Clone the request if underlying devices aren't blk-mq */
266302233342SMike Snitzer 	if (dm_table_get_type(map) == DM_TYPE_REQUEST_BASED) {
266402233342SMike Snitzer 		/* clone request is allocated at the end of the pdu */
266502233342SMike Snitzer 		tio->clone = (void *)blk_mq_rq_to_pdu(rq) + sizeof(struct dm_rq_target_io);
266602233342SMike Snitzer 		if (!clone_rq(rq, md, tio, GFP_ATOMIC))
266702233342SMike Snitzer 			return BLK_MQ_RQ_QUEUE_BUSY;
2668bfebd1cdSMike Snitzer 		queue_kthread_work(&md->kworker, &tio->work);
266902233342SMike Snitzer 	} else {
267002233342SMike Snitzer 		/* Direct call is fine since .queue_rq allows allocations */
267102233342SMike Snitzer 		if (map_request(tio, rq, md) == DM_MAPIO_REQUEUE)
267202233342SMike Snitzer 			dm_requeue_unmapped_original_request(md, rq);
267302233342SMike Snitzer 	}
2674bfebd1cdSMike Snitzer 
2675bfebd1cdSMike Snitzer 	return BLK_MQ_RQ_QUEUE_OK;
2676bfebd1cdSMike Snitzer }
2677bfebd1cdSMike Snitzer 
2678bfebd1cdSMike Snitzer static struct blk_mq_ops dm_mq_ops = {
2679bfebd1cdSMike Snitzer 	.queue_rq = dm_mq_queue_rq,
2680bfebd1cdSMike Snitzer 	.map_queue = blk_mq_map_queue,
2681bfebd1cdSMike Snitzer 	.complete = dm_softirq_done,
2682bfebd1cdSMike Snitzer 	.init_request = dm_mq_init_request,
2683bfebd1cdSMike Snitzer };
2684bfebd1cdSMike Snitzer 
2685bfebd1cdSMike Snitzer static int dm_init_request_based_blk_mq_queue(struct mapped_device *md)
2686bfebd1cdSMike Snitzer {
268702233342SMike Snitzer 	unsigned md_type = dm_get_md_type(md);
2688bfebd1cdSMike Snitzer 	struct request_queue *q;
2689bfebd1cdSMike Snitzer 	int err;
2690bfebd1cdSMike Snitzer 
2691bfebd1cdSMike Snitzer 	memset(&md->tag_set, 0, sizeof(md->tag_set));
2692bfebd1cdSMike Snitzer 	md->tag_set.ops = &dm_mq_ops;
2693bfebd1cdSMike Snitzer 	md->tag_set.queue_depth = BLKDEV_MAX_RQ;
2694bfebd1cdSMike Snitzer 	md->tag_set.numa_node = NUMA_NO_NODE;
2695bfebd1cdSMike Snitzer 	md->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE;
2696bfebd1cdSMike Snitzer 	md->tag_set.nr_hw_queues = 1;
269702233342SMike Snitzer 	if (md_type == DM_TYPE_REQUEST_BASED) {
269802233342SMike Snitzer 		/* make the memory for non-blk-mq clone part of the pdu */
269902233342SMike Snitzer 		md->tag_set.cmd_size = sizeof(struct dm_rq_target_io) + sizeof(struct request);
270002233342SMike Snitzer 	} else
2701bfebd1cdSMike Snitzer 		md->tag_set.cmd_size = sizeof(struct dm_rq_target_io);
2702bfebd1cdSMike Snitzer 	md->tag_set.driver_data = md;
2703bfebd1cdSMike Snitzer 
2704bfebd1cdSMike Snitzer 	err = blk_mq_alloc_tag_set(&md->tag_set);
2705bfebd1cdSMike Snitzer 	if (err)
2706bfebd1cdSMike Snitzer 		return err;
2707bfebd1cdSMike Snitzer 
2708bfebd1cdSMike Snitzer 	q = blk_mq_init_allocated_queue(&md->tag_set, md->queue);
2709bfebd1cdSMike Snitzer 	if (IS_ERR(q)) {
2710bfebd1cdSMike Snitzer 		err = PTR_ERR(q);
2711bfebd1cdSMike Snitzer 		goto out_tag_set;
2712bfebd1cdSMike Snitzer 	}
2713bfebd1cdSMike Snitzer 	md->queue = q;
2714bfebd1cdSMike Snitzer 	dm_init_md_queue(md);
2715bfebd1cdSMike Snitzer 
2716bfebd1cdSMike Snitzer 	/* backfill 'mq' sysfs registration normally done in blk_register_queue */
2717bfebd1cdSMike Snitzer 	blk_mq_register_disk(md->disk);
2718bfebd1cdSMike Snitzer 
271902233342SMike Snitzer 	if (md_type == DM_TYPE_REQUEST_BASED)
2720bfebd1cdSMike Snitzer 		init_rq_based_worker_thread(md);
2721bfebd1cdSMike Snitzer 
2722bfebd1cdSMike Snitzer 	return 0;
2723bfebd1cdSMike Snitzer 
2724bfebd1cdSMike Snitzer out_tag_set:
2725bfebd1cdSMike Snitzer 	blk_mq_free_tag_set(&md->tag_set);
2726bfebd1cdSMike Snitzer 	return err;
27274a0b4ddfSMike Snitzer }
27284a0b4ddfSMike Snitzer 
272917e149b8SMike Snitzer static unsigned filter_md_type(unsigned type, struct mapped_device *md)
273017e149b8SMike Snitzer {
273117e149b8SMike Snitzer 	if (type == DM_TYPE_BIO_BASED)
273217e149b8SMike Snitzer 		return type;
273317e149b8SMike Snitzer 
273417e149b8SMike Snitzer 	return !md->use_blk_mq ? DM_TYPE_REQUEST_BASED : DM_TYPE_MQ_REQUEST_BASED;
273517e149b8SMike Snitzer }
273617e149b8SMike Snitzer 
27374a0b4ddfSMike Snitzer /*
27384a0b4ddfSMike Snitzer  * Setup the DM device's queue based on md's type
27394a0b4ddfSMike Snitzer  */
27404a0b4ddfSMike Snitzer int dm_setup_md_queue(struct mapped_device *md)
27414a0b4ddfSMike Snitzer {
2742bfebd1cdSMike Snitzer 	int r;
274317e149b8SMike Snitzer 	unsigned md_type = filter_md_type(dm_get_md_type(md), md);
2744bfebd1cdSMike Snitzer 
2745bfebd1cdSMike Snitzer 	switch (md_type) {
2746bfebd1cdSMike Snitzer 	case DM_TYPE_REQUEST_BASED:
2747bfebd1cdSMike Snitzer 		r = dm_init_request_based_queue(md);
2748bfebd1cdSMike Snitzer 		if (r) {
27494a0b4ddfSMike Snitzer 			DMWARN("Cannot initialize queue for request-based mapped device");
2750bfebd1cdSMike Snitzer 			return r;
27514a0b4ddfSMike Snitzer 		}
2752bfebd1cdSMike Snitzer 		break;
2753bfebd1cdSMike Snitzer 	case DM_TYPE_MQ_REQUEST_BASED:
2754bfebd1cdSMike Snitzer 		r = dm_init_request_based_blk_mq_queue(md);
2755bfebd1cdSMike Snitzer 		if (r) {
2756bfebd1cdSMike Snitzer 			DMWARN("Cannot initialize queue for request-based blk-mq mapped device");
2757bfebd1cdSMike Snitzer 			return r;
2758bfebd1cdSMike Snitzer 		}
2759bfebd1cdSMike Snitzer 		break;
2760bfebd1cdSMike Snitzer 	case DM_TYPE_BIO_BASED:
2761bfebd1cdSMike Snitzer 		dm_init_old_md_queue(md);
2762ff36ab34SMike Snitzer 		blk_queue_make_request(md->queue, dm_make_request);
2763ff36ab34SMike Snitzer 		blk_queue_merge_bvec(md->queue, dm_merge_bvec);
2764bfebd1cdSMike Snitzer 		break;
2765ff36ab34SMike Snitzer 	}
27664a0b4ddfSMike Snitzer 
27674a0b4ddfSMike Snitzer 	return 0;
27684a0b4ddfSMike Snitzer }
27694a0b4ddfSMike Snitzer 
27702bec1f4aSMikulas Patocka struct mapped_device *dm_get_md(dev_t dev)
27711da177e4SLinus Torvalds {
27721da177e4SLinus Torvalds 	struct mapped_device *md;
27731da177e4SLinus Torvalds 	unsigned minor = MINOR(dev);
27741da177e4SLinus Torvalds 
27751da177e4SLinus Torvalds 	if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
27761da177e4SLinus Torvalds 		return NULL;
27771da177e4SLinus Torvalds 
2778f32c10b0SJeff Mahoney 	spin_lock(&_minor_lock);
27791da177e4SLinus Torvalds 
27801da177e4SLinus Torvalds 	md = idr_find(&_minor_idr, minor);
27812bec1f4aSMikulas Patocka 	if (md) {
27822bec1f4aSMikulas Patocka 		if ((md == MINOR_ALLOCED ||
2783f331c029STejun Heo 		     (MINOR(disk_devt(dm_disk(md))) != minor) ||
2784abdc568bSKiyoshi Ueda 		     dm_deleting_md(md) ||
2785fba9f90eSJeff Mahoney 		     test_bit(DMF_FREEING, &md->flags))) {
2786637842cfSDavid Teigland 			md = NULL;
2787fba9f90eSJeff Mahoney 			goto out;
2788fba9f90eSJeff Mahoney 		}
27892bec1f4aSMikulas Patocka 		dm_get(md);
27902bec1f4aSMikulas Patocka 	}
27911da177e4SLinus Torvalds 
2792fba9f90eSJeff Mahoney out:
2793f32c10b0SJeff Mahoney 	spin_unlock(&_minor_lock);
27941da177e4SLinus Torvalds 
2795637842cfSDavid Teigland 	return md;
2796637842cfSDavid Teigland }
27973cf2e4baSAlasdair G Kergon EXPORT_SYMBOL_GPL(dm_get_md);
2798d229a958SDavid Teigland 
27999ade92a9SAlasdair G Kergon void *dm_get_mdptr(struct mapped_device *md)
2800637842cfSDavid Teigland {
28019ade92a9SAlasdair G Kergon 	return md->interface_ptr;
28021da177e4SLinus Torvalds }
28031da177e4SLinus Torvalds 
28041da177e4SLinus Torvalds void dm_set_mdptr(struct mapped_device *md, void *ptr)
28051da177e4SLinus Torvalds {
28061da177e4SLinus Torvalds 	md->interface_ptr = ptr;
28071da177e4SLinus Torvalds }
28081da177e4SLinus Torvalds 
28091da177e4SLinus Torvalds void dm_get(struct mapped_device *md)
28101da177e4SLinus Torvalds {
28111da177e4SLinus Torvalds 	atomic_inc(&md->holders);
28123f77316dSKiyoshi Ueda 	BUG_ON(test_bit(DMF_FREEING, &md->flags));
28131da177e4SLinus Torvalds }
28141da177e4SLinus Torvalds 
281509ee96b2SMikulas Patocka int dm_hold(struct mapped_device *md)
281609ee96b2SMikulas Patocka {
281709ee96b2SMikulas Patocka 	spin_lock(&_minor_lock);
281809ee96b2SMikulas Patocka 	if (test_bit(DMF_FREEING, &md->flags)) {
281909ee96b2SMikulas Patocka 		spin_unlock(&_minor_lock);
282009ee96b2SMikulas Patocka 		return -EBUSY;
282109ee96b2SMikulas Patocka 	}
282209ee96b2SMikulas Patocka 	dm_get(md);
282309ee96b2SMikulas Patocka 	spin_unlock(&_minor_lock);
282409ee96b2SMikulas Patocka 	return 0;
282509ee96b2SMikulas Patocka }
282609ee96b2SMikulas Patocka EXPORT_SYMBOL_GPL(dm_hold);
282709ee96b2SMikulas Patocka 
282872d94861SAlasdair G Kergon const char *dm_device_name(struct mapped_device *md)
282972d94861SAlasdair G Kergon {
283072d94861SAlasdair G Kergon 	return md->name;
283172d94861SAlasdair G Kergon }
283272d94861SAlasdair G Kergon EXPORT_SYMBOL_GPL(dm_device_name);
283372d94861SAlasdair G Kergon 
28343f77316dSKiyoshi Ueda static void __dm_destroy(struct mapped_device *md, bool wait)
28351da177e4SLinus Torvalds {
28361134e5aeSMike Anderson 	struct dm_table *map;
283783d5e5b0SMikulas Patocka 	int srcu_idx;
28381da177e4SLinus Torvalds 
28393f77316dSKiyoshi Ueda 	might_sleep();
2840fba9f90eSJeff Mahoney 
284183d5e5b0SMikulas Patocka 	map = dm_get_live_table(md, &srcu_idx);
284263a4f065SMike Snitzer 
284363a4f065SMike Snitzer 	spin_lock(&_minor_lock);
28443f77316dSKiyoshi Ueda 	idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));
2845fba9f90eSJeff Mahoney 	set_bit(DMF_FREEING, &md->flags);
2846f32c10b0SJeff Mahoney 	spin_unlock(&_minor_lock);
28473f77316dSKiyoshi Ueda 
284802233342SMike Snitzer 	if (dm_request_based(md) && md->kworker_task)
28492eb6e1e3SKeith Busch 		flush_kthread_worker(&md->kworker);
28502eb6e1e3SKeith Busch 
2851ab7c7bb6SMikulas Patocka 	/*
2852ab7c7bb6SMikulas Patocka 	 * Take suspend_lock so that presuspend and postsuspend methods
2853ab7c7bb6SMikulas Patocka 	 * do not race with internal suspend.
2854ab7c7bb6SMikulas Patocka 	 */
2855ab7c7bb6SMikulas Patocka 	mutex_lock(&md->suspend_lock);
28564f186f8bSKiyoshi Ueda 	if (!dm_suspended_md(md)) {
28571da177e4SLinus Torvalds 		dm_table_presuspend_targets(map);
28581da177e4SLinus Torvalds 		dm_table_postsuspend_targets(map);
28591da177e4SLinus Torvalds 	}
2860ab7c7bb6SMikulas Patocka 	mutex_unlock(&md->suspend_lock);
28613f77316dSKiyoshi Ueda 
286283d5e5b0SMikulas Patocka 	/* dm_put_live_table must be before msleep, otherwise deadlock is possible */
286383d5e5b0SMikulas Patocka 	dm_put_live_table(md, srcu_idx);
286483d5e5b0SMikulas Patocka 
28653f77316dSKiyoshi Ueda 	/*
28663f77316dSKiyoshi Ueda 	 * Rare, but there may be I/O requests still going to complete,
28673f77316dSKiyoshi Ueda 	 * for example.  Wait for all references to disappear.
28683f77316dSKiyoshi Ueda 	 * No one should increment the reference count of the mapped_device,
28693f77316dSKiyoshi Ueda 	 * after the mapped_device state becomes DMF_FREEING.
28703f77316dSKiyoshi Ueda 	 */
28713f77316dSKiyoshi Ueda 	if (wait)
28723f77316dSKiyoshi Ueda 		while (atomic_read(&md->holders))
28733f77316dSKiyoshi Ueda 			msleep(1);
28743f77316dSKiyoshi Ueda 	else if (atomic_read(&md->holders))
28753f77316dSKiyoshi Ueda 		DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)",
28763f77316dSKiyoshi Ueda 		       dm_device_name(md), atomic_read(&md->holders));
28773f77316dSKiyoshi Ueda 
2878784aae73SMilan Broz 	dm_sysfs_exit(md);
2879a7940155SAlasdair G Kergon 	dm_table_destroy(__unbind(md));
28801da177e4SLinus Torvalds 	free_dev(md);
28811da177e4SLinus Torvalds }
28823f77316dSKiyoshi Ueda 
28833f77316dSKiyoshi Ueda void dm_destroy(struct mapped_device *md)
28843f77316dSKiyoshi Ueda {
28853f77316dSKiyoshi Ueda 	__dm_destroy(md, true);
28863f77316dSKiyoshi Ueda }
28873f77316dSKiyoshi Ueda 
28883f77316dSKiyoshi Ueda void dm_destroy_immediate(struct mapped_device *md)
28893f77316dSKiyoshi Ueda {
28903f77316dSKiyoshi Ueda 	__dm_destroy(md, false);
28913f77316dSKiyoshi Ueda }
28923f77316dSKiyoshi Ueda 
28933f77316dSKiyoshi Ueda void dm_put(struct mapped_device *md)
28943f77316dSKiyoshi Ueda {
28953f77316dSKiyoshi Ueda 	atomic_dec(&md->holders);
28961da177e4SLinus Torvalds }
289779eb885cSEdward Goggin EXPORT_SYMBOL_GPL(dm_put);
28981da177e4SLinus Torvalds 
2899401600dfSMikulas Patocka static int dm_wait_for_completion(struct mapped_device *md, int interruptible)
290046125c1cSMilan Broz {
290146125c1cSMilan Broz 	int r = 0;
2902b44ebeb0SMikulas Patocka 	DECLARE_WAITQUEUE(wait, current);
2903b44ebeb0SMikulas Patocka 
2904b44ebeb0SMikulas Patocka 	add_wait_queue(&md->wait, &wait);
290546125c1cSMilan Broz 
290646125c1cSMilan Broz 	while (1) {
2907401600dfSMikulas Patocka 		set_current_state(interruptible);
290846125c1cSMilan Broz 
2909b4324feeSKiyoshi Ueda 		if (!md_in_flight(md))
291046125c1cSMilan Broz 			break;
291146125c1cSMilan Broz 
2912401600dfSMikulas Patocka 		if (interruptible == TASK_INTERRUPTIBLE &&
2913401600dfSMikulas Patocka 		    signal_pending(current)) {
291446125c1cSMilan Broz 			r = -EINTR;
291546125c1cSMilan Broz 			break;
291646125c1cSMilan Broz 		}
291746125c1cSMilan Broz 
291846125c1cSMilan Broz 		io_schedule();
291946125c1cSMilan Broz 	}
292046125c1cSMilan Broz 	set_current_state(TASK_RUNNING);
292146125c1cSMilan Broz 
2922b44ebeb0SMikulas Patocka 	remove_wait_queue(&md->wait, &wait);
2923b44ebeb0SMikulas Patocka 
292446125c1cSMilan Broz 	return r;
292546125c1cSMilan Broz }
292646125c1cSMilan Broz 
29271da177e4SLinus Torvalds /*
29281da177e4SLinus Torvalds  * Process the deferred bios
29291da177e4SLinus Torvalds  */
2930ef208587SMikulas Patocka static void dm_wq_work(struct work_struct *work)
29311da177e4SLinus Torvalds {
2932ef208587SMikulas Patocka 	struct mapped_device *md = container_of(work, struct mapped_device,
2933ef208587SMikulas Patocka 						work);
29346d6f10dfSMilan Broz 	struct bio *c;
293583d5e5b0SMikulas Patocka 	int srcu_idx;
293683d5e5b0SMikulas Patocka 	struct dm_table *map;
29371da177e4SLinus Torvalds 
293883d5e5b0SMikulas Patocka 	map = dm_get_live_table(md, &srcu_idx);
2939ef208587SMikulas Patocka 
29403b00b203SMikulas Patocka 	while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
2941022c2611SMikulas Patocka 		spin_lock_irq(&md->deferred_lock);
2942022c2611SMikulas Patocka 		c = bio_list_pop(&md->deferred);
2943022c2611SMikulas Patocka 		spin_unlock_irq(&md->deferred_lock);
2944022c2611SMikulas Patocka 
29456a8736d1STejun Heo 		if (!c)
2946df12ee99SAlasdair G Kergon 			break;
294773d410c0SMilan Broz 
2948e6ee8c0bSKiyoshi Ueda 		if (dm_request_based(md))
2949e6ee8c0bSKiyoshi Ueda 			generic_make_request(c);
2950af7e466aSMikulas Patocka 		else
295183d5e5b0SMikulas Patocka 			__split_and_process_bio(md, map, c);
2952e6ee8c0bSKiyoshi Ueda 	}
29533b00b203SMikulas Patocka 
295483d5e5b0SMikulas Patocka 	dm_put_live_table(md, srcu_idx);
29551da177e4SLinus Torvalds }
29561da177e4SLinus Torvalds 
29579a1fb464SMikulas Patocka static void dm_queue_flush(struct mapped_device *md)
2958304f3f6aSMilan Broz {
29593b00b203SMikulas Patocka 	clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
29604e857c58SPeter Zijlstra 	smp_mb__after_atomic();
296153d5914fSMikulas Patocka 	queue_work(md->wq, &md->work);
2962304f3f6aSMilan Broz }
2963304f3f6aSMilan Broz 
29641da177e4SLinus Torvalds /*
2965042d2a9bSAlasdair G Kergon  * Swap in a new table, returning the old one for the caller to destroy.
29661da177e4SLinus Torvalds  */
2967042d2a9bSAlasdair G Kergon struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table)
29681da177e4SLinus Torvalds {
296987eb5b21SMike Christie 	struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL);
2970754c5fc7SMike Snitzer 	struct queue_limits limits;
2971042d2a9bSAlasdair G Kergon 	int r;
29721da177e4SLinus Torvalds 
2973e61290a4SDaniel Walker 	mutex_lock(&md->suspend_lock);
29741da177e4SLinus Torvalds 
29751da177e4SLinus Torvalds 	/* device must be suspended */
29764f186f8bSKiyoshi Ueda 	if (!dm_suspended_md(md))
297793c534aeSAlasdair G Kergon 		goto out;
29781da177e4SLinus Torvalds 
29793ae70656SMike Snitzer 	/*
29803ae70656SMike Snitzer 	 * If the new table has no data devices, retain the existing limits.
29813ae70656SMike Snitzer 	 * This helps multipath with queue_if_no_path if all paths disappear,
29823ae70656SMike Snitzer 	 * then new I/O is queued based on these limits, and then some paths
29833ae70656SMike Snitzer 	 * reappear.
29843ae70656SMike Snitzer 	 */
29853ae70656SMike Snitzer 	if (dm_table_has_no_data_devices(table)) {
298683d5e5b0SMikulas Patocka 		live_map = dm_get_live_table_fast(md);
29873ae70656SMike Snitzer 		if (live_map)
29883ae70656SMike Snitzer 			limits = md->queue->limits;
298983d5e5b0SMikulas Patocka 		dm_put_live_table_fast(md);
29903ae70656SMike Snitzer 	}
29913ae70656SMike Snitzer 
299287eb5b21SMike Christie 	if (!live_map) {
2993754c5fc7SMike Snitzer 		r = dm_calculate_queue_limits(table, &limits);
2994042d2a9bSAlasdair G Kergon 		if (r) {
2995042d2a9bSAlasdair G Kergon 			map = ERR_PTR(r);
2996754c5fc7SMike Snitzer 			goto out;
2997042d2a9bSAlasdair G Kergon 		}
299887eb5b21SMike Christie 	}
2999754c5fc7SMike Snitzer 
3000042d2a9bSAlasdair G Kergon 	map = __bind(md, table, &limits);
30011da177e4SLinus Torvalds 
300293c534aeSAlasdair G Kergon out:
3003e61290a4SDaniel Walker 	mutex_unlock(&md->suspend_lock);
3004042d2a9bSAlasdair G Kergon 	return map;
30051da177e4SLinus Torvalds }
30061da177e4SLinus Torvalds 
30071da177e4SLinus Torvalds /*
30081da177e4SLinus Torvalds  * Functions to lock and unlock any filesystem running on the
30091da177e4SLinus Torvalds  * device.
30101da177e4SLinus Torvalds  */
30112ca3310eSAlasdair G Kergon static int lock_fs(struct mapped_device *md)
30121da177e4SLinus Torvalds {
3013e39e2e95SAlasdair G Kergon 	int r;
30141da177e4SLinus Torvalds 
30151da177e4SLinus Torvalds 	WARN_ON(md->frozen_sb);
3016dfbe03f6SAlasdair G Kergon 
3017db8fef4fSMikulas Patocka 	md->frozen_sb = freeze_bdev(md->bdev);
3018dfbe03f6SAlasdair G Kergon 	if (IS_ERR(md->frozen_sb)) {
3019cf222b37SAlasdair G Kergon 		r = PTR_ERR(md->frozen_sb);
3020e39e2e95SAlasdair G Kergon 		md->frozen_sb = NULL;
3021e39e2e95SAlasdair G Kergon 		return r;
3022dfbe03f6SAlasdair G Kergon 	}
3023dfbe03f6SAlasdair G Kergon 
3024aa8d7c2fSAlasdair G Kergon 	set_bit(DMF_FROZEN, &md->flags);
3025aa8d7c2fSAlasdair G Kergon 
30261da177e4SLinus Torvalds 	return 0;
30271da177e4SLinus Torvalds }
30281da177e4SLinus Torvalds 
30292ca3310eSAlasdair G Kergon static void unlock_fs(struct mapped_device *md)
30301da177e4SLinus Torvalds {
3031aa8d7c2fSAlasdair G Kergon 	if (!test_bit(DMF_FROZEN, &md->flags))
3032aa8d7c2fSAlasdair G Kergon 		return;
3033aa8d7c2fSAlasdair G Kergon 
3034db8fef4fSMikulas Patocka 	thaw_bdev(md->bdev, md->frozen_sb);
30351da177e4SLinus Torvalds 	md->frozen_sb = NULL;
3036aa8d7c2fSAlasdair G Kergon 	clear_bit(DMF_FROZEN, &md->flags);
30371da177e4SLinus Torvalds }
30381da177e4SLinus Torvalds 
30391da177e4SLinus Torvalds /*
3040ffcc3936SMike Snitzer  * If __dm_suspend returns 0, the device is completely quiescent
3041ffcc3936SMike Snitzer  * now. There is no request-processing activity. All new requests
3042ffcc3936SMike Snitzer  * are being added to md->deferred list.
3043cec47e3dSKiyoshi Ueda  *
3044ffcc3936SMike Snitzer  * Caller must hold md->suspend_lock
3045cec47e3dSKiyoshi Ueda  */
3046ffcc3936SMike Snitzer static int __dm_suspend(struct mapped_device *md, struct dm_table *map,
3047ffcc3936SMike Snitzer 			unsigned suspend_flags, int interruptible)
30481da177e4SLinus Torvalds {
3049ffcc3936SMike Snitzer 	bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG;
3050ffcc3936SMike Snitzer 	bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG;
3051ffcc3936SMike Snitzer 	int r;
3052cf222b37SAlasdair G Kergon 
30532e93ccc1SKiyoshi Ueda 	/*
30542e93ccc1SKiyoshi Ueda 	 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
30552e93ccc1SKiyoshi Ueda 	 * This flag is cleared before dm_suspend returns.
30562e93ccc1SKiyoshi Ueda 	 */
30572e93ccc1SKiyoshi Ueda 	if (noflush)
30582e93ccc1SKiyoshi Ueda 		set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
30592e93ccc1SKiyoshi Ueda 
3060d67ee213SMike Snitzer 	/*
3061d67ee213SMike Snitzer 	 * This gets reverted if there's an error later and the targets
3062d67ee213SMike Snitzer 	 * provide the .presuspend_undo hook.
3063d67ee213SMike Snitzer 	 */
30641da177e4SLinus Torvalds 	dm_table_presuspend_targets(map);
30651da177e4SLinus Torvalds 
30662e93ccc1SKiyoshi Ueda 	/*
30679f518b27SKiyoshi Ueda 	 * Flush I/O to the device.
30689f518b27SKiyoshi Ueda 	 * Any I/O submitted after lock_fs() may not be flushed.
30699f518b27SKiyoshi Ueda 	 * noflush takes precedence over do_lockfs.
30709f518b27SKiyoshi Ueda 	 * (lock_fs() flushes I/Os and waits for them to complete.)
30712e93ccc1SKiyoshi Ueda 	 */
307232a926daSMikulas Patocka 	if (!noflush && do_lockfs) {
30732ca3310eSAlasdair G Kergon 		r = lock_fs(md);
3074d67ee213SMike Snitzer 		if (r) {
3075d67ee213SMike Snitzer 			dm_table_presuspend_undo_targets(map);
3076ffcc3936SMike Snitzer 			return r;
3077aa8d7c2fSAlasdair G Kergon 		}
3078d67ee213SMike Snitzer 	}
30791da177e4SLinus Torvalds 
30801da177e4SLinus Torvalds 	/*
30813b00b203SMikulas Patocka 	 * Here we must make sure that no processes are submitting requests
30823b00b203SMikulas Patocka 	 * to target drivers i.e. no one may be executing
30833b00b203SMikulas Patocka 	 * __split_and_process_bio. This is called from dm_request and
30843b00b203SMikulas Patocka 	 * dm_wq_work.
30853b00b203SMikulas Patocka 	 *
30863b00b203SMikulas Patocka 	 * To get all processes out of __split_and_process_bio in dm_request,
30873b00b203SMikulas Patocka 	 * we take the write lock. To prevent any process from reentering
30886a8736d1STejun Heo 	 * __split_and_process_bio from dm_request and quiesce the thread
30896a8736d1STejun Heo 	 * (dm_wq_work), we set BMF_BLOCK_IO_FOR_SUSPEND and call
30906a8736d1STejun Heo 	 * flush_workqueue(md->wq).
30911da177e4SLinus Torvalds 	 */
30921eb787ecSAlasdair G Kergon 	set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
309341abc4e1SHannes Reinecke 	if (map)
309483d5e5b0SMikulas Patocka 		synchronize_srcu(&md->io_barrier);
30951da177e4SLinus Torvalds 
3096d0bcb878SKiyoshi Ueda 	/*
309729e4013dSTejun Heo 	 * Stop md->queue before flushing md->wq in case request-based
309829e4013dSTejun Heo 	 * dm defers requests to md->wq from md->queue.
3099d0bcb878SKiyoshi Ueda 	 */
31002eb6e1e3SKeith Busch 	if (dm_request_based(md)) {
31019f518b27SKiyoshi Ueda 		stop_queue(md->queue);
310202233342SMike Snitzer 		if (md->kworker_task)
31032eb6e1e3SKeith Busch 			flush_kthread_worker(&md->kworker);
31042eb6e1e3SKeith Busch 	}
3105cec47e3dSKiyoshi Ueda 
3106d0bcb878SKiyoshi Ueda 	flush_workqueue(md->wq);
3107d0bcb878SKiyoshi Ueda 
31081da177e4SLinus Torvalds 	/*
31093b00b203SMikulas Patocka 	 * At this point no more requests are entering target request routines.
31103b00b203SMikulas Patocka 	 * We call dm_wait_for_completion to wait for all existing requests
31113b00b203SMikulas Patocka 	 * to finish.
31121da177e4SLinus Torvalds 	 */
3113ffcc3936SMike Snitzer 	r = dm_wait_for_completion(md, interruptible);
31141da177e4SLinus Torvalds 
31156d6f10dfSMilan Broz 	if (noflush)
3116022c2611SMikulas Patocka 		clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
311741abc4e1SHannes Reinecke 	if (map)
311883d5e5b0SMikulas Patocka 		synchronize_srcu(&md->io_barrier);
31192e93ccc1SKiyoshi Ueda 
31201da177e4SLinus Torvalds 	/* were we interrupted ? */
312146125c1cSMilan Broz 	if (r < 0) {
31229a1fb464SMikulas Patocka 		dm_queue_flush(md);
312373d410c0SMilan Broz 
3124cec47e3dSKiyoshi Ueda 		if (dm_request_based(md))
31259f518b27SKiyoshi Ueda 			start_queue(md->queue);
3126cec47e3dSKiyoshi Ueda 
31272ca3310eSAlasdair G Kergon 		unlock_fs(md);
3128d67ee213SMike Snitzer 		dm_table_presuspend_undo_targets(map);
3129ffcc3936SMike Snitzer 		/* pushback list is already flushed, so skip flush */
3130ffcc3936SMike Snitzer 	}
3131ffcc3936SMike Snitzer 
3132ffcc3936SMike Snitzer 	return r;
31332ca3310eSAlasdair G Kergon }
31342ca3310eSAlasdair G Kergon 
31353b00b203SMikulas Patocka /*
3136ffcc3936SMike Snitzer  * We need to be able to change a mapping table under a mounted
3137ffcc3936SMike Snitzer  * filesystem.  For example we might want to move some data in
3138ffcc3936SMike Snitzer  * the background.  Before the table can be swapped with
3139ffcc3936SMike Snitzer  * dm_bind_table, dm_suspend must be called to flush any in
3140ffcc3936SMike Snitzer  * flight bios and ensure that any further io gets deferred.
31413b00b203SMikulas Patocka  */
3142ffcc3936SMike Snitzer /*
3143ffcc3936SMike Snitzer  * Suspend mechanism in request-based dm.
3144ffcc3936SMike Snitzer  *
3145ffcc3936SMike Snitzer  * 1. Flush all I/Os by lock_fs() if needed.
3146ffcc3936SMike Snitzer  * 2. Stop dispatching any I/O by stopping the request_queue.
3147ffcc3936SMike Snitzer  * 3. Wait for all in-flight I/Os to be completed or requeued.
3148ffcc3936SMike Snitzer  *
3149ffcc3936SMike Snitzer  * To abort suspend, start the request_queue.
3150ffcc3936SMike Snitzer  */
3151ffcc3936SMike Snitzer int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
3152ffcc3936SMike Snitzer {
3153ffcc3936SMike Snitzer 	struct dm_table *map = NULL;
3154ffcc3936SMike Snitzer 	int r = 0;
3155ffcc3936SMike Snitzer 
3156ffcc3936SMike Snitzer retry:
3157ffcc3936SMike Snitzer 	mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
3158ffcc3936SMike Snitzer 
3159ffcc3936SMike Snitzer 	if (dm_suspended_md(md)) {
3160ffcc3936SMike Snitzer 		r = -EINVAL;
3161ffcc3936SMike Snitzer 		goto out_unlock;
3162ffcc3936SMike Snitzer 	}
3163ffcc3936SMike Snitzer 
3164ffcc3936SMike Snitzer 	if (dm_suspended_internally_md(md)) {
3165ffcc3936SMike Snitzer 		/* already internally suspended, wait for internal resume */
3166ffcc3936SMike Snitzer 		mutex_unlock(&md->suspend_lock);
3167ffcc3936SMike Snitzer 		r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
3168ffcc3936SMike Snitzer 		if (r)
3169ffcc3936SMike Snitzer 			return r;
3170ffcc3936SMike Snitzer 		goto retry;
3171ffcc3936SMike Snitzer 	}
3172ffcc3936SMike Snitzer 
3173a12f5d48SEric Dumazet 	map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
3174ffcc3936SMike Snitzer 
3175ffcc3936SMike Snitzer 	r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE);
3176ffcc3936SMike Snitzer 	if (r)
3177ffcc3936SMike Snitzer 		goto out_unlock;
31783b00b203SMikulas Patocka 
31791da177e4SLinus Torvalds 	set_bit(DMF_SUSPENDED, &md->flags);
31801da177e4SLinus Torvalds 
31814d4471cbSKiyoshi Ueda 	dm_table_postsuspend_targets(map);
31824d4471cbSKiyoshi Ueda 
3183d287483dSAlasdair G Kergon out_unlock:
3184e61290a4SDaniel Walker 	mutex_unlock(&md->suspend_lock);
3185cf222b37SAlasdair G Kergon 	return r;
31861da177e4SLinus Torvalds }
31871da177e4SLinus Torvalds 
3188ffcc3936SMike Snitzer static int __dm_resume(struct mapped_device *md, struct dm_table *map)
31891da177e4SLinus Torvalds {
3190ffcc3936SMike Snitzer 	if (map) {
3191ffcc3936SMike Snitzer 		int r = dm_table_resume_targets(map);
31928757b776SMilan Broz 		if (r)
3193ffcc3936SMike Snitzer 			return r;
3194ffcc3936SMike Snitzer 	}
31952ca3310eSAlasdair G Kergon 
31969a1fb464SMikulas Patocka 	dm_queue_flush(md);
31972ca3310eSAlasdair G Kergon 
3198cec47e3dSKiyoshi Ueda 	/*
3199cec47e3dSKiyoshi Ueda 	 * Flushing deferred I/Os must be done after targets are resumed
3200cec47e3dSKiyoshi Ueda 	 * so that mapping of targets can work correctly.
3201cec47e3dSKiyoshi Ueda 	 * Request-based dm is queueing the deferred I/Os in its request_queue.
3202cec47e3dSKiyoshi Ueda 	 */
3203cec47e3dSKiyoshi Ueda 	if (dm_request_based(md))
3204cec47e3dSKiyoshi Ueda 		start_queue(md->queue);
3205cec47e3dSKiyoshi Ueda 
32062ca3310eSAlasdair G Kergon 	unlock_fs(md);
32072ca3310eSAlasdair G Kergon 
3208ffcc3936SMike Snitzer 	return 0;
3209ffcc3936SMike Snitzer }
3210ffcc3936SMike Snitzer 
3211ffcc3936SMike Snitzer int dm_resume(struct mapped_device *md)
3212ffcc3936SMike Snitzer {
3213ffcc3936SMike Snitzer 	int r = -EINVAL;
3214ffcc3936SMike Snitzer 	struct dm_table *map = NULL;
3215ffcc3936SMike Snitzer 
3216ffcc3936SMike Snitzer retry:
3217ffcc3936SMike Snitzer 	mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
3218ffcc3936SMike Snitzer 
3219ffcc3936SMike Snitzer 	if (!dm_suspended_md(md))
3220ffcc3936SMike Snitzer 		goto out;
3221ffcc3936SMike Snitzer 
3222ffcc3936SMike Snitzer 	if (dm_suspended_internally_md(md)) {
3223ffcc3936SMike Snitzer 		/* already internally suspended, wait for internal resume */
3224ffcc3936SMike Snitzer 		mutex_unlock(&md->suspend_lock);
3225ffcc3936SMike Snitzer 		r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
3226ffcc3936SMike Snitzer 		if (r)
3227ffcc3936SMike Snitzer 			return r;
3228ffcc3936SMike Snitzer 		goto retry;
3229ffcc3936SMike Snitzer 	}
3230ffcc3936SMike Snitzer 
3231a12f5d48SEric Dumazet 	map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
3232ffcc3936SMike Snitzer 	if (!map || !dm_table_get_size(map))
3233ffcc3936SMike Snitzer 		goto out;
3234ffcc3936SMike Snitzer 
3235ffcc3936SMike Snitzer 	r = __dm_resume(md, map);
3236ffcc3936SMike Snitzer 	if (r)
3237ffcc3936SMike Snitzer 		goto out;
3238ffcc3936SMike Snitzer 
32392ca3310eSAlasdair G Kergon 	clear_bit(DMF_SUSPENDED, &md->flags);
32402ca3310eSAlasdair G Kergon 
3241cf222b37SAlasdair G Kergon 	r = 0;
3242cf222b37SAlasdair G Kergon out:
3243e61290a4SDaniel Walker 	mutex_unlock(&md->suspend_lock);
32442ca3310eSAlasdair G Kergon 
3245cf222b37SAlasdair G Kergon 	return r;
32461da177e4SLinus Torvalds }
32471da177e4SLinus Torvalds 
3248fd2ed4d2SMikulas Patocka /*
3249fd2ed4d2SMikulas Patocka  * Internal suspend/resume works like userspace-driven suspend. It waits
3250fd2ed4d2SMikulas Patocka  * until all bios finish and prevents issuing new bios to the target drivers.
3251fd2ed4d2SMikulas Patocka  * It may be used only from the kernel.
3252fd2ed4d2SMikulas Patocka  */
3253fd2ed4d2SMikulas Patocka 
3254ffcc3936SMike Snitzer static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags)
3255ffcc3936SMike Snitzer {
3256ffcc3936SMike Snitzer 	struct dm_table *map = NULL;
3257ffcc3936SMike Snitzer 
325896b26c8cSMikulas Patocka 	if (md->internal_suspend_count++)
3259ffcc3936SMike Snitzer 		return; /* nested internal suspend */
3260ffcc3936SMike Snitzer 
3261ffcc3936SMike Snitzer 	if (dm_suspended_md(md)) {
3262ffcc3936SMike Snitzer 		set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
3263ffcc3936SMike Snitzer 		return; /* nest suspend */
3264ffcc3936SMike Snitzer 	}
3265ffcc3936SMike Snitzer 
3266a12f5d48SEric Dumazet 	map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
3267ffcc3936SMike Snitzer 
3268ffcc3936SMike Snitzer 	/*
3269ffcc3936SMike Snitzer 	 * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is
3270ffcc3936SMike Snitzer 	 * supported.  Properly supporting a TASK_INTERRUPTIBLE internal suspend
3271ffcc3936SMike Snitzer 	 * would require changing .presuspend to return an error -- avoid this
3272ffcc3936SMike Snitzer 	 * until there is a need for more elaborate variants of internal suspend.
3273ffcc3936SMike Snitzer 	 */
3274ffcc3936SMike Snitzer 	(void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE);
3275ffcc3936SMike Snitzer 
3276ffcc3936SMike Snitzer 	set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
3277ffcc3936SMike Snitzer 
3278ffcc3936SMike Snitzer 	dm_table_postsuspend_targets(map);
3279ffcc3936SMike Snitzer }
3280ffcc3936SMike Snitzer 
3281ffcc3936SMike Snitzer static void __dm_internal_resume(struct mapped_device *md)
3282ffcc3936SMike Snitzer {
328396b26c8cSMikulas Patocka 	BUG_ON(!md->internal_suspend_count);
328496b26c8cSMikulas Patocka 
328596b26c8cSMikulas Patocka 	if (--md->internal_suspend_count)
3286ffcc3936SMike Snitzer 		return; /* resume from nested internal suspend */
3287ffcc3936SMike Snitzer 
3288ffcc3936SMike Snitzer 	if (dm_suspended_md(md))
3289ffcc3936SMike Snitzer 		goto done; /* resume from nested suspend */
3290ffcc3936SMike Snitzer 
3291ffcc3936SMike Snitzer 	/*
3292ffcc3936SMike Snitzer 	 * NOTE: existing callers don't need to call dm_table_resume_targets
3293ffcc3936SMike Snitzer 	 * (which may fail -- so best to avoid it for now by passing NULL map)
3294ffcc3936SMike Snitzer 	 */
3295ffcc3936SMike Snitzer 	(void) __dm_resume(md, NULL);
3296ffcc3936SMike Snitzer 
3297ffcc3936SMike Snitzer done:
3298ffcc3936SMike Snitzer 	clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
3299ffcc3936SMike Snitzer 	smp_mb__after_atomic();
3300ffcc3936SMike Snitzer 	wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY);
3301ffcc3936SMike Snitzer }
3302ffcc3936SMike Snitzer 
3303ffcc3936SMike Snitzer void dm_internal_suspend_noflush(struct mapped_device *md)
3304fd2ed4d2SMikulas Patocka {
3305fd2ed4d2SMikulas Patocka 	mutex_lock(&md->suspend_lock);
3306ffcc3936SMike Snitzer 	__dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG);
3307ffcc3936SMike Snitzer 	mutex_unlock(&md->suspend_lock);
3308ffcc3936SMike Snitzer }
3309ffcc3936SMike Snitzer EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush);
3310ffcc3936SMike Snitzer 
3311ffcc3936SMike Snitzer void dm_internal_resume(struct mapped_device *md)
3312ffcc3936SMike Snitzer {
3313ffcc3936SMike Snitzer 	mutex_lock(&md->suspend_lock);
3314ffcc3936SMike Snitzer 	__dm_internal_resume(md);
3315ffcc3936SMike Snitzer 	mutex_unlock(&md->suspend_lock);
3316ffcc3936SMike Snitzer }
3317ffcc3936SMike Snitzer EXPORT_SYMBOL_GPL(dm_internal_resume);
3318ffcc3936SMike Snitzer 
3319ffcc3936SMike Snitzer /*
3320ffcc3936SMike Snitzer  * Fast variants of internal suspend/resume hold md->suspend_lock,
3321ffcc3936SMike Snitzer  * which prevents interaction with userspace-driven suspend.
3322ffcc3936SMike Snitzer  */
3323ffcc3936SMike Snitzer 
3324ffcc3936SMike Snitzer void dm_internal_suspend_fast(struct mapped_device *md)
3325ffcc3936SMike Snitzer {
3326ffcc3936SMike Snitzer 	mutex_lock(&md->suspend_lock);
3327ffcc3936SMike Snitzer 	if (dm_suspended_md(md) || dm_suspended_internally_md(md))
3328fd2ed4d2SMikulas Patocka 		return;
3329fd2ed4d2SMikulas Patocka 
3330fd2ed4d2SMikulas Patocka 	set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
3331fd2ed4d2SMikulas Patocka 	synchronize_srcu(&md->io_barrier);
3332fd2ed4d2SMikulas Patocka 	flush_workqueue(md->wq);
3333fd2ed4d2SMikulas Patocka 	dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
3334fd2ed4d2SMikulas Patocka }
3335b735fedeSMikulas Patocka EXPORT_SYMBOL_GPL(dm_internal_suspend_fast);
3336fd2ed4d2SMikulas Patocka 
3337ffcc3936SMike Snitzer void dm_internal_resume_fast(struct mapped_device *md)
3338fd2ed4d2SMikulas Patocka {
3339ffcc3936SMike Snitzer 	if (dm_suspended_md(md) || dm_suspended_internally_md(md))
3340fd2ed4d2SMikulas Patocka 		goto done;
3341fd2ed4d2SMikulas Patocka 
3342fd2ed4d2SMikulas Patocka 	dm_queue_flush(md);
3343fd2ed4d2SMikulas Patocka 
3344fd2ed4d2SMikulas Patocka done:
3345fd2ed4d2SMikulas Patocka 	mutex_unlock(&md->suspend_lock);
3346fd2ed4d2SMikulas Patocka }
3347b735fedeSMikulas Patocka EXPORT_SYMBOL_GPL(dm_internal_resume_fast);
3348fd2ed4d2SMikulas Patocka 
33491da177e4SLinus Torvalds /*-----------------------------------------------------------------
33501da177e4SLinus Torvalds  * Event notification.
33511da177e4SLinus Torvalds  *---------------------------------------------------------------*/
33523abf85b5SPeter Rajnoha int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action,
335360935eb2SMilan Broz 		       unsigned cookie)
335469267a30SAlasdair G Kergon {
335560935eb2SMilan Broz 	char udev_cookie[DM_COOKIE_LENGTH];
335660935eb2SMilan Broz 	char *envp[] = { udev_cookie, NULL };
335760935eb2SMilan Broz 
335860935eb2SMilan Broz 	if (!cookie)
33593abf85b5SPeter Rajnoha 		return kobject_uevent(&disk_to_dev(md->disk)->kobj, action);
336060935eb2SMilan Broz 	else {
336160935eb2SMilan Broz 		snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u",
336260935eb2SMilan Broz 			 DM_COOKIE_ENV_VAR_NAME, cookie);
33633abf85b5SPeter Rajnoha 		return kobject_uevent_env(&disk_to_dev(md->disk)->kobj,
33643abf85b5SPeter Rajnoha 					  action, envp);
336560935eb2SMilan Broz 	}
336669267a30SAlasdair G Kergon }
336769267a30SAlasdair G Kergon 
33687a8c3d3bSMike Anderson uint32_t dm_next_uevent_seq(struct mapped_device *md)
33697a8c3d3bSMike Anderson {
33707a8c3d3bSMike Anderson 	return atomic_add_return(1, &md->uevent_seq);
33717a8c3d3bSMike Anderson }
33727a8c3d3bSMike Anderson 
33731da177e4SLinus Torvalds uint32_t dm_get_event_nr(struct mapped_device *md)
33741da177e4SLinus Torvalds {
33751da177e4SLinus Torvalds 	return atomic_read(&md->event_nr);
33761da177e4SLinus Torvalds }
33771da177e4SLinus Torvalds 
33781da177e4SLinus Torvalds int dm_wait_event(struct mapped_device *md, int event_nr)
33791da177e4SLinus Torvalds {
33801da177e4SLinus Torvalds 	return wait_event_interruptible(md->eventq,
33811da177e4SLinus Torvalds 			(event_nr != atomic_read(&md->event_nr)));
33821da177e4SLinus Torvalds }
33831da177e4SLinus Torvalds 
33847a8c3d3bSMike Anderson void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
33857a8c3d3bSMike Anderson {
33867a8c3d3bSMike Anderson 	unsigned long flags;
33877a8c3d3bSMike Anderson 
33887a8c3d3bSMike Anderson 	spin_lock_irqsave(&md->uevent_lock, flags);
33897a8c3d3bSMike Anderson 	list_add(elist, &md->uevent_list);
33907a8c3d3bSMike Anderson 	spin_unlock_irqrestore(&md->uevent_lock, flags);
33917a8c3d3bSMike Anderson }
33927a8c3d3bSMike Anderson 
33931da177e4SLinus Torvalds /*
33941da177e4SLinus Torvalds  * The gendisk is only valid as long as you have a reference
33951da177e4SLinus Torvalds  * count on 'md'.
33961da177e4SLinus Torvalds  */
33971da177e4SLinus Torvalds struct gendisk *dm_disk(struct mapped_device *md)
33981da177e4SLinus Torvalds {
33991da177e4SLinus Torvalds 	return md->disk;
34001da177e4SLinus Torvalds }
340165ff5b7dSSami Tolvanen EXPORT_SYMBOL_GPL(dm_disk);
34021da177e4SLinus Torvalds 
3403784aae73SMilan Broz struct kobject *dm_kobject(struct mapped_device *md)
3404784aae73SMilan Broz {
34052995fa78SMikulas Patocka 	return &md->kobj_holder.kobj;
3406784aae73SMilan Broz }
3407784aae73SMilan Broz 
3408784aae73SMilan Broz struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
3409784aae73SMilan Broz {
3410784aae73SMilan Broz 	struct mapped_device *md;
3411784aae73SMilan Broz 
34122995fa78SMikulas Patocka 	md = container_of(kobj, struct mapped_device, kobj_holder.kobj);
3413784aae73SMilan Broz 
34144d89b7b4SMilan Broz 	if (test_bit(DMF_FREEING, &md->flags) ||
3415432a212cSMike Anderson 	    dm_deleting_md(md))
34164d89b7b4SMilan Broz 		return NULL;
34174d89b7b4SMilan Broz 
3418784aae73SMilan Broz 	dm_get(md);
3419784aae73SMilan Broz 	return md;
3420784aae73SMilan Broz }
3421784aae73SMilan Broz 
34224f186f8bSKiyoshi Ueda int dm_suspended_md(struct mapped_device *md)
34231da177e4SLinus Torvalds {
34241da177e4SLinus Torvalds 	return test_bit(DMF_SUSPENDED, &md->flags);
34251da177e4SLinus Torvalds }
34261da177e4SLinus Torvalds 
3427ffcc3936SMike Snitzer int dm_suspended_internally_md(struct mapped_device *md)
3428ffcc3936SMike Snitzer {
3429ffcc3936SMike Snitzer 	return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
3430ffcc3936SMike Snitzer }
3431ffcc3936SMike Snitzer 
34322c140a24SMikulas Patocka int dm_test_deferred_remove_flag(struct mapped_device *md)
34332c140a24SMikulas Patocka {
34342c140a24SMikulas Patocka 	return test_bit(DMF_DEFERRED_REMOVE, &md->flags);
34352c140a24SMikulas Patocka }
34362c140a24SMikulas Patocka 
343764dbce58SKiyoshi Ueda int dm_suspended(struct dm_target *ti)
343864dbce58SKiyoshi Ueda {
3439ecdb2e25SKiyoshi Ueda 	return dm_suspended_md(dm_table_get_md(ti->table));
344064dbce58SKiyoshi Ueda }
344164dbce58SKiyoshi Ueda EXPORT_SYMBOL_GPL(dm_suspended);
344264dbce58SKiyoshi Ueda 
34432e93ccc1SKiyoshi Ueda int dm_noflush_suspending(struct dm_target *ti)
34442e93ccc1SKiyoshi Ueda {
3445ecdb2e25SKiyoshi Ueda 	return __noflush_suspending(dm_table_get_md(ti->table));
34462e93ccc1SKiyoshi Ueda }
34472e93ccc1SKiyoshi Ueda EXPORT_SYMBOL_GPL(dm_noflush_suspending);
34482e93ccc1SKiyoshi Ueda 
3449*5f1b670dSChristoph Hellwig struct dm_md_mempools *dm_alloc_bio_mempools(unsigned integrity,
3450*5f1b670dSChristoph Hellwig 					     unsigned per_bio_data_size)
3451e6ee8c0bSKiyoshi Ueda {
3452*5f1b670dSChristoph Hellwig 	struct dm_md_mempools *pools;
3453*5f1b670dSChristoph Hellwig 	unsigned int pool_size = dm_get_reserved_bio_based_ios();
34545f015204SJun'ichi Nomura 	unsigned int front_pad;
3455e6ee8c0bSKiyoshi Ueda 
3456*5f1b670dSChristoph Hellwig 	pools = kzalloc(sizeof(*pools), GFP_KERNEL);
3457e6ee8c0bSKiyoshi Ueda 	if (!pools)
3458e6ee8c0bSKiyoshi Ueda 		return NULL;
3459e6ee8c0bSKiyoshi Ueda 
3460*5f1b670dSChristoph Hellwig 	front_pad = roundup(per_bio_data_size, __alignof__(struct dm_target_io)) +
3461*5f1b670dSChristoph Hellwig 		offsetof(struct dm_target_io, clone);
346217e149b8SMike Snitzer 
3463*5f1b670dSChristoph Hellwig 	pools->io_pool = mempool_create_slab_pool(pool_size, _io_cache);
3464e6ee8c0bSKiyoshi Ueda 	if (!pools->io_pool)
34655f015204SJun'ichi Nomura 		goto out;
3466e6ee8c0bSKiyoshi Ueda 
34673d8aab2dSJunichi Nomura 	pools->bs = bioset_create_nobvec(pool_size, front_pad);
3468e6ee8c0bSKiyoshi Ueda 	if (!pools->bs)
34695f015204SJun'ichi Nomura 		goto out;
3470e6ee8c0bSKiyoshi Ueda 
3471a91a2785SMartin K. Petersen 	if (integrity && bioset_integrity_create(pools->bs, pool_size))
34725f015204SJun'ichi Nomura 		goto out;
3473a91a2785SMartin K. Petersen 
3474e6ee8c0bSKiyoshi Ueda 	return pools;
34755f015204SJun'ichi Nomura out:
34765f015204SJun'ichi Nomura 	dm_free_md_mempools(pools);
3477*5f1b670dSChristoph Hellwig 	return NULL;
3478*5f1b670dSChristoph Hellwig }
3479e6ee8c0bSKiyoshi Ueda 
3480*5f1b670dSChristoph Hellwig struct dm_md_mempools *dm_alloc_rq_mempools(struct mapped_device *md,
3481*5f1b670dSChristoph Hellwig 					    unsigned type)
3482*5f1b670dSChristoph Hellwig {
3483*5f1b670dSChristoph Hellwig 	unsigned int pool_size = dm_get_reserved_rq_based_ios();
3484*5f1b670dSChristoph Hellwig 	struct dm_md_mempools *pools;
3485*5f1b670dSChristoph Hellwig 
3486*5f1b670dSChristoph Hellwig 	pools = kzalloc(sizeof(*pools), GFP_KERNEL);
3487*5f1b670dSChristoph Hellwig 	if (!pools)
3488*5f1b670dSChristoph Hellwig 		return NULL;
3489*5f1b670dSChristoph Hellwig 
3490*5f1b670dSChristoph Hellwig 	if (filter_md_type(type, md) == DM_TYPE_REQUEST_BASED) {
3491*5f1b670dSChristoph Hellwig 		pools->rq_pool = mempool_create_slab_pool(pool_size, _rq_cache);
3492*5f1b670dSChristoph Hellwig 		if (!pools->rq_pool)
3493*5f1b670dSChristoph Hellwig 			goto out;
3494*5f1b670dSChristoph Hellwig 	}
3495*5f1b670dSChristoph Hellwig 
3496*5f1b670dSChristoph Hellwig 	pools->io_pool = mempool_create_slab_pool(pool_size, _rq_tio_cache);
3497*5f1b670dSChristoph Hellwig 	if (!pools->io_pool)
3498*5f1b670dSChristoph Hellwig 		goto out;
3499*5f1b670dSChristoph Hellwig 
3500*5f1b670dSChristoph Hellwig 	return pools;
3501*5f1b670dSChristoph Hellwig out:
3502*5f1b670dSChristoph Hellwig 	dm_free_md_mempools(pools);
3503e6ee8c0bSKiyoshi Ueda 	return NULL;
3504e6ee8c0bSKiyoshi Ueda }
3505e6ee8c0bSKiyoshi Ueda 
3506e6ee8c0bSKiyoshi Ueda void dm_free_md_mempools(struct dm_md_mempools *pools)
3507e6ee8c0bSKiyoshi Ueda {
3508e6ee8c0bSKiyoshi Ueda 	if (!pools)
3509e6ee8c0bSKiyoshi Ueda 		return;
3510e6ee8c0bSKiyoshi Ueda 
3511e6ee8c0bSKiyoshi Ueda 	if (pools->io_pool)
3512e6ee8c0bSKiyoshi Ueda 		mempool_destroy(pools->io_pool);
3513e6ee8c0bSKiyoshi Ueda 
35141ae49ea2SMike Snitzer 	if (pools->rq_pool)
35151ae49ea2SMike Snitzer 		mempool_destroy(pools->rq_pool);
35161ae49ea2SMike Snitzer 
3517e6ee8c0bSKiyoshi Ueda 	if (pools->bs)
3518e6ee8c0bSKiyoshi Ueda 		bioset_free(pools->bs);
3519e6ee8c0bSKiyoshi Ueda 
3520e6ee8c0bSKiyoshi Ueda 	kfree(pools);
3521e6ee8c0bSKiyoshi Ueda }
3522e6ee8c0bSKiyoshi Ueda 
352383d5cde4SAlexey Dobriyan static const struct block_device_operations dm_blk_dops = {
35241da177e4SLinus Torvalds 	.open = dm_blk_open,
35251da177e4SLinus Torvalds 	.release = dm_blk_close,
3526aa129a22SMilan Broz 	.ioctl = dm_blk_ioctl,
35273ac51e74SDarrick J. Wong 	.getgeo = dm_blk_getgeo,
35281da177e4SLinus Torvalds 	.owner = THIS_MODULE
35291da177e4SLinus Torvalds };
35301da177e4SLinus Torvalds 
35311da177e4SLinus Torvalds /*
35321da177e4SLinus Torvalds  * module hooks
35331da177e4SLinus Torvalds  */
35341da177e4SLinus Torvalds module_init(dm_init);
35351da177e4SLinus Torvalds module_exit(dm_exit);
35361da177e4SLinus Torvalds 
35371da177e4SLinus Torvalds module_param(major, uint, 0);
35381da177e4SLinus Torvalds MODULE_PARM_DESC(major, "The major number of the device mapper");
3539f4790826SMike Snitzer 
3540e8603136SMike Snitzer module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR);
3541e8603136SMike Snitzer MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools");
3542e8603136SMike Snitzer 
3543f4790826SMike Snitzer module_param(reserved_rq_based_ios, uint, S_IRUGO | S_IWUSR);
3544f4790826SMike Snitzer MODULE_PARM_DESC(reserved_rq_based_ios, "Reserved IOs in request-based mempools");
3545f4790826SMike Snitzer 
354617e149b8SMike Snitzer module_param(use_blk_mq, bool, S_IRUGO | S_IWUSR);
354717e149b8SMike Snitzer MODULE_PARM_DESC(use_blk_mq, "Use block multiqueue for request-based DM devices");
354817e149b8SMike Snitzer 
35491da177e4SLinus Torvalds MODULE_DESCRIPTION(DM_NAME " driver");
35501da177e4SLinus Torvalds MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
35511da177e4SLinus Torvalds MODULE_LICENSE("GPL");
3552