xref: /openbmc/linux/drivers/md/dm.c (revision 0cdb90f0f306384ecbc60dfd6dc48cdbc1f2d0d8)
11da177e4SLinus Torvalds /*
21da177e4SLinus Torvalds  * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3784aae73SMilan Broz  * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
41da177e4SLinus Torvalds  *
51da177e4SLinus Torvalds  * This file is released under the GPL.
61da177e4SLinus Torvalds  */
71da177e4SLinus Torvalds 
84cc96131SMike Snitzer #include "dm-core.h"
94cc96131SMike Snitzer #include "dm-rq.h"
1051e5b2bdSMike Anderson #include "dm-uevent.h"
1191ccbbacSTushar Sugandhi #include "dm-ima.h"
121da177e4SLinus Torvalds 
131da177e4SLinus Torvalds #include <linux/init.h>
141da177e4SLinus Torvalds #include <linux/module.h>
1548c9c27bSArjan van de Ven #include <linux/mutex.h>
166958c1c6SMikulas Patocka #include <linux/sched/mm.h>
17174cd4b1SIngo Molnar #include <linux/sched/signal.h>
181da177e4SLinus Torvalds #include <linux/blkpg.h>
191da177e4SLinus Torvalds #include <linux/bio.h>
201da177e4SLinus Torvalds #include <linux/mempool.h>
21f26c5719SDan Williams #include <linux/dax.h>
221da177e4SLinus Torvalds #include <linux/slab.h>
231da177e4SLinus Torvalds #include <linux/idr.h>
247e026c8cSDan Williams #include <linux/uio.h>
253ac51e74SDarrick J. Wong #include <linux/hdreg.h>
263f77316dSKiyoshi Ueda #include <linux/delay.h>
27ffcc3936SMike Snitzer #include <linux/wait.h>
2871cdb697SChristoph Hellwig #include <linux/pr.h>
29b0b4d7c6SElena Reshetova #include <linux/refcount.h>
30c6a564ffSChristoph Hellwig #include <linux/part_stat.h>
31a892c8d5SSatya Tangirala #include <linux/blk-crypto.h>
321e8d44bdSEric Biggers #include <linux/blk-crypto-profile.h>
3355782138SLi Zefan 
3472d94861SAlasdair G Kergon #define DM_MSG_PREFIX "core"
3572d94861SAlasdair G Kergon 
3660935eb2SMilan Broz /*
3760935eb2SMilan Broz  * Cookies are numeric values sent with CHANGE and REMOVE
3860935eb2SMilan Broz  * uevents while resuming, removing or renaming the device.
3960935eb2SMilan Broz  */
4060935eb2SMilan Broz #define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE"
4160935eb2SMilan Broz #define DM_COOKIE_LENGTH 24
4260935eb2SMilan Broz 
431da177e4SLinus Torvalds static const char *_name = DM_NAME;
441da177e4SLinus Torvalds 
451da177e4SLinus Torvalds static unsigned int major = 0;
461da177e4SLinus Torvalds static unsigned int _major = 0;
471da177e4SLinus Torvalds 
48d15b774cSAlasdair G Kergon static DEFINE_IDR(_minor_idr);
49d15b774cSAlasdair G Kergon 
50f32c10b0SJeff Mahoney static DEFINE_SPINLOCK(_minor_lock);
512c140a24SMikulas Patocka 
522c140a24SMikulas Patocka static void do_deferred_remove(struct work_struct *w);
532c140a24SMikulas Patocka 
542c140a24SMikulas Patocka static DECLARE_WORK(deferred_remove_work, do_deferred_remove);
552c140a24SMikulas Patocka 
56acfe0ad7SMikulas Patocka static struct workqueue_struct *deferred_remove_workqueue;
57acfe0ad7SMikulas Patocka 
5893e6442cSMikulas Patocka atomic_t dm_global_event_nr = ATOMIC_INIT(0);
5993e6442cSMikulas Patocka DECLARE_WAIT_QUEUE_HEAD(dm_global_eventq);
6093e6442cSMikulas Patocka 
6162e08243SMikulas Patocka void dm_issue_global_event(void)
6262e08243SMikulas Patocka {
6362e08243SMikulas Patocka 	atomic_inc(&dm_global_event_nr);
6462e08243SMikulas Patocka 	wake_up(&dm_global_eventq);
6562e08243SMikulas Patocka }
6662e08243SMikulas Patocka 
671da177e4SLinus Torvalds /*
6864f52b0eSMike Snitzer  * One of these is allocated (on-stack) per original bio.
691da177e4SLinus Torvalds  */
7064f52b0eSMike Snitzer struct clone_info {
7164f52b0eSMike Snitzer 	struct dm_table *map;
7264f52b0eSMike Snitzer 	struct bio *bio;
7364f52b0eSMike Snitzer 	struct dm_io *io;
7464f52b0eSMike Snitzer 	sector_t sector;
7564f52b0eSMike Snitzer 	unsigned sector_count;
7664f52b0eSMike Snitzer };
7764f52b0eSMike Snitzer 
7862f26317SJeffle Xu #define DM_TARGET_IO_BIO_OFFSET (offsetof(struct dm_target_io, clone))
7962f26317SJeffle Xu #define DM_IO_BIO_OFFSET \
8062f26317SJeffle Xu 	(offsetof(struct dm_target_io, clone) + offsetof(struct dm_io, tio))
8162f26317SJeffle Xu 
826c23f0bdSChristoph Hellwig static inline struct dm_target_io *clone_to_tio(struct bio *clone)
836c23f0bdSChristoph Hellwig {
846c23f0bdSChristoph Hellwig 	return container_of(clone, struct dm_target_io, clone);
856c23f0bdSChristoph Hellwig }
866c23f0bdSChristoph Hellwig 
8764f52b0eSMike Snitzer void *dm_per_bio_data(struct bio *bio, size_t data_size)
8864f52b0eSMike Snitzer {
896c23f0bdSChristoph Hellwig 	if (!clone_to_tio(bio)->inside_dm_io)
9062f26317SJeffle Xu 		return (char *)bio - DM_TARGET_IO_BIO_OFFSET - data_size;
9162f26317SJeffle Xu 	return (char *)bio - DM_IO_BIO_OFFSET - data_size;
9264f52b0eSMike Snitzer }
9364f52b0eSMike Snitzer EXPORT_SYMBOL_GPL(dm_per_bio_data);
9464f52b0eSMike Snitzer 
9564f52b0eSMike Snitzer struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size)
9664f52b0eSMike Snitzer {
9764f52b0eSMike Snitzer 	struct dm_io *io = (struct dm_io *)((char *)data + data_size);
9864f52b0eSMike Snitzer 	if (io->magic == DM_IO_MAGIC)
9962f26317SJeffle Xu 		return (struct bio *)((char *)io + DM_IO_BIO_OFFSET);
10064f52b0eSMike Snitzer 	BUG_ON(io->magic != DM_TIO_MAGIC);
10162f26317SJeffle Xu 	return (struct bio *)((char *)io + DM_TARGET_IO_BIO_OFFSET);
10264f52b0eSMike Snitzer }
10364f52b0eSMike Snitzer EXPORT_SYMBOL_GPL(dm_bio_from_per_bio_data);
10464f52b0eSMike Snitzer 
10564f52b0eSMike Snitzer unsigned dm_bio_get_target_bio_nr(const struct bio *bio)
10664f52b0eSMike Snitzer {
10764f52b0eSMike Snitzer 	return container_of(bio, struct dm_target_io, clone)->target_bio_nr;
10864f52b0eSMike Snitzer }
10964f52b0eSMike Snitzer EXPORT_SYMBOL_GPL(dm_bio_get_target_bio_nr);
11064f52b0eSMike Snitzer 
111ba61fdd1SJeff Mahoney #define MINOR_ALLOCED ((void *)-1)
112ba61fdd1SJeff Mahoney 
113115485e8SMike Snitzer #define DM_NUMA_NODE NUMA_NO_NODE
114115485e8SMike Snitzer static int dm_numa_node = DM_NUMA_NODE;
115faad87dfSMike Snitzer 
116a666e5c0SMikulas Patocka #define DEFAULT_SWAP_BIOS	(8 * 1048576 / PAGE_SIZE)
117a666e5c0SMikulas Patocka static int swap_bios = DEFAULT_SWAP_BIOS;
118a666e5c0SMikulas Patocka static int get_swap_bios(void)
119a666e5c0SMikulas Patocka {
120a666e5c0SMikulas Patocka 	int latch = READ_ONCE(swap_bios);
121a666e5c0SMikulas Patocka 	if (unlikely(latch <= 0))
122a666e5c0SMikulas Patocka 		latch = DEFAULT_SWAP_BIOS;
123a666e5c0SMikulas Patocka 	return latch;
124a666e5c0SMikulas Patocka }
125a666e5c0SMikulas Patocka 
126e6ee8c0bSKiyoshi Ueda /*
127e6ee8c0bSKiyoshi Ueda  * For mempools pre-allocation at the table loading time.
128e6ee8c0bSKiyoshi Ueda  */
129e6ee8c0bSKiyoshi Ueda struct dm_md_mempools {
1306f1c819cSKent Overstreet 	struct bio_set bs;
1316f1c819cSKent Overstreet 	struct bio_set io_bs;
132e6ee8c0bSKiyoshi Ueda };
133e6ee8c0bSKiyoshi Ueda 
13486f1152bSBenjamin Marzinski struct table_device {
13586f1152bSBenjamin Marzinski 	struct list_head list;
136b0b4d7c6SElena Reshetova 	refcount_t count;
13786f1152bSBenjamin Marzinski 	struct dm_dev dm_dev;
13886f1152bSBenjamin Marzinski };
13986f1152bSBenjamin Marzinski 
140f4790826SMike Snitzer /*
141e8603136SMike Snitzer  * Bio-based DM's mempools' reserved IOs set by the user.
142e8603136SMike Snitzer  */
1434cc96131SMike Snitzer #define RESERVED_BIO_BASED_IOS		16
144e8603136SMike Snitzer static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS;
145e8603136SMike Snitzer 
146115485e8SMike Snitzer static int __dm_get_module_param_int(int *module_param, int min, int max)
147115485e8SMike Snitzer {
1486aa7de05SMark Rutland 	int param = READ_ONCE(*module_param);
149115485e8SMike Snitzer 	int modified_param = 0;
150115485e8SMike Snitzer 	bool modified = true;
151115485e8SMike Snitzer 
152115485e8SMike Snitzer 	if (param < min)
153115485e8SMike Snitzer 		modified_param = min;
154115485e8SMike Snitzer 	else if (param > max)
155115485e8SMike Snitzer 		modified_param = max;
156115485e8SMike Snitzer 	else
157115485e8SMike Snitzer 		modified = false;
158115485e8SMike Snitzer 
159115485e8SMike Snitzer 	if (modified) {
160115485e8SMike Snitzer 		(void)cmpxchg(module_param, param, modified_param);
161115485e8SMike Snitzer 		param = modified_param;
162115485e8SMike Snitzer 	}
163115485e8SMike Snitzer 
164115485e8SMike Snitzer 	return param;
165115485e8SMike Snitzer }
166115485e8SMike Snitzer 
1674cc96131SMike Snitzer unsigned __dm_get_module_param(unsigned *module_param,
168f4790826SMike Snitzer 			       unsigned def, unsigned max)
169f4790826SMike Snitzer {
1706aa7de05SMark Rutland 	unsigned param = READ_ONCE(*module_param);
17109c2d531SMike Snitzer 	unsigned modified_param = 0;
172f4790826SMike Snitzer 
17309c2d531SMike Snitzer 	if (!param)
17409c2d531SMike Snitzer 		modified_param = def;
17509c2d531SMike Snitzer 	else if (param > max)
17609c2d531SMike Snitzer 		modified_param = max;
177f4790826SMike Snitzer 
17809c2d531SMike Snitzer 	if (modified_param) {
17909c2d531SMike Snitzer 		(void)cmpxchg(module_param, param, modified_param);
18009c2d531SMike Snitzer 		param = modified_param;
181f4790826SMike Snitzer 	}
182f4790826SMike Snitzer 
18309c2d531SMike Snitzer 	return param;
184f4790826SMike Snitzer }
185f4790826SMike Snitzer 
186e8603136SMike Snitzer unsigned dm_get_reserved_bio_based_ios(void)
187e8603136SMike Snitzer {
18809c2d531SMike Snitzer 	return __dm_get_module_param(&reserved_bio_based_ios,
1894cc96131SMike Snitzer 				     RESERVED_BIO_BASED_IOS, DM_RESERVED_MAX_IOS);
190e8603136SMike Snitzer }
191e8603136SMike Snitzer EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios);
192e8603136SMike Snitzer 
193115485e8SMike Snitzer static unsigned dm_get_numa_node(void)
194115485e8SMike Snitzer {
195115485e8SMike Snitzer 	return __dm_get_module_param_int(&dm_numa_node,
196115485e8SMike Snitzer 					 DM_NUMA_NODE, num_online_nodes() - 1);
197115485e8SMike Snitzer }
198115485e8SMike Snitzer 
1991da177e4SLinus Torvalds static int __init local_init(void)
2001da177e4SLinus Torvalds {
201e689fbabSMike Snitzer 	int r;
2021ae49ea2SMike Snitzer 
20351e5b2bdSMike Anderson 	r = dm_uevent_init();
20451157b4aSKiyoshi Ueda 	if (r)
205e689fbabSMike Snitzer 		return r;
20651e5b2bdSMike Anderson 
207acfe0ad7SMikulas Patocka 	deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1);
208acfe0ad7SMikulas Patocka 	if (!deferred_remove_workqueue) {
209acfe0ad7SMikulas Patocka 		r = -ENOMEM;
210acfe0ad7SMikulas Patocka 		goto out_uevent_exit;
211acfe0ad7SMikulas Patocka 	}
212acfe0ad7SMikulas Patocka 
2131da177e4SLinus Torvalds 	_major = major;
2141da177e4SLinus Torvalds 	r = register_blkdev(_major, _name);
21551157b4aSKiyoshi Ueda 	if (r < 0)
216acfe0ad7SMikulas Patocka 		goto out_free_workqueue;
2171da177e4SLinus Torvalds 
2181da177e4SLinus Torvalds 	if (!_major)
2191da177e4SLinus Torvalds 		_major = r;
2201da177e4SLinus Torvalds 
2211da177e4SLinus Torvalds 	return 0;
22251157b4aSKiyoshi Ueda 
223acfe0ad7SMikulas Patocka out_free_workqueue:
224acfe0ad7SMikulas Patocka 	destroy_workqueue(deferred_remove_workqueue);
22551157b4aSKiyoshi Ueda out_uevent_exit:
22651157b4aSKiyoshi Ueda 	dm_uevent_exit();
22751157b4aSKiyoshi Ueda 
22851157b4aSKiyoshi Ueda 	return r;
2291da177e4SLinus Torvalds }
2301da177e4SLinus Torvalds 
2311da177e4SLinus Torvalds static void local_exit(void)
2321da177e4SLinus Torvalds {
2332c140a24SMikulas Patocka 	flush_scheduled_work();
234acfe0ad7SMikulas Patocka 	destroy_workqueue(deferred_remove_workqueue);
2352c140a24SMikulas Patocka 
23600d59405SAkinobu Mita 	unregister_blkdev(_major, _name);
23751e5b2bdSMike Anderson 	dm_uevent_exit();
2381da177e4SLinus Torvalds 
2391da177e4SLinus Torvalds 	_major = 0;
2401da177e4SLinus Torvalds 
2411da177e4SLinus Torvalds 	DMINFO("cleaned up");
2421da177e4SLinus Torvalds }
2431da177e4SLinus Torvalds 
244b9249e55SAlasdair G Kergon static int (*_inits[])(void) __initdata = {
2451da177e4SLinus Torvalds 	local_init,
2461da177e4SLinus Torvalds 	dm_target_init,
2471da177e4SLinus Torvalds 	dm_linear_init,
2481da177e4SLinus Torvalds 	dm_stripe_init,
249952b3557SMikulas Patocka 	dm_io_init,
250945fa4d2SMikulas Patocka 	dm_kcopyd_init,
2511da177e4SLinus Torvalds 	dm_interface_init,
252fd2ed4d2SMikulas Patocka 	dm_statistics_init,
2531da177e4SLinus Torvalds };
2541da177e4SLinus Torvalds 
255b9249e55SAlasdair G Kergon static void (*_exits[])(void) = {
2561da177e4SLinus Torvalds 	local_exit,
2571da177e4SLinus Torvalds 	dm_target_exit,
2581da177e4SLinus Torvalds 	dm_linear_exit,
2591da177e4SLinus Torvalds 	dm_stripe_exit,
260952b3557SMikulas Patocka 	dm_io_exit,
261945fa4d2SMikulas Patocka 	dm_kcopyd_exit,
2621da177e4SLinus Torvalds 	dm_interface_exit,
263fd2ed4d2SMikulas Patocka 	dm_statistics_exit,
2641da177e4SLinus Torvalds };
2651da177e4SLinus Torvalds 
2661da177e4SLinus Torvalds static int __init dm_init(void)
2671da177e4SLinus Torvalds {
2681da177e4SLinus Torvalds 	const int count = ARRAY_SIZE(_inits);
2691da177e4SLinus Torvalds 	int r, i;
2701da177e4SLinus Torvalds 
271f1cd6cb2STushar Sugandhi #if (IS_ENABLED(CONFIG_IMA) && !IS_ENABLED(CONFIG_IMA_DISABLE_HTABLE))
272f1cd6cb2STushar Sugandhi 	DMWARN("CONFIG_IMA_DISABLE_HTABLE is disabled."
273f1cd6cb2STushar Sugandhi 	       " Duplicate IMA measurements will not be recorded in the IMA log.");
274f1cd6cb2STushar Sugandhi #endif
275f1cd6cb2STushar Sugandhi 
2761da177e4SLinus Torvalds 	for (i = 0; i < count; i++) {
2771da177e4SLinus Torvalds 		r = _inits[i]();
2781da177e4SLinus Torvalds 		if (r)
2791da177e4SLinus Torvalds 			goto bad;
2801da177e4SLinus Torvalds 	}
2811da177e4SLinus Torvalds 
2821da177e4SLinus Torvalds 	return 0;
2831da177e4SLinus Torvalds bad:
2841da177e4SLinus Torvalds 	while (i--)
2851da177e4SLinus Torvalds 		_exits[i]();
2861da177e4SLinus Torvalds 
2871da177e4SLinus Torvalds 	return r;
2881da177e4SLinus Torvalds }
2891da177e4SLinus Torvalds 
2901da177e4SLinus Torvalds static void __exit dm_exit(void)
2911da177e4SLinus Torvalds {
2921da177e4SLinus Torvalds 	int i = ARRAY_SIZE(_exits);
2931da177e4SLinus Torvalds 
2941da177e4SLinus Torvalds 	while (i--)
2951da177e4SLinus Torvalds 		_exits[i]();
296d15b774cSAlasdair G Kergon 
297d15b774cSAlasdair G Kergon 	/*
298d15b774cSAlasdair G Kergon 	 * Should be empty by this point.
299d15b774cSAlasdair G Kergon 	 */
300d15b774cSAlasdair G Kergon 	idr_destroy(&_minor_idr);
3011da177e4SLinus Torvalds }
3021da177e4SLinus Torvalds 
3031da177e4SLinus Torvalds /*
3041da177e4SLinus Torvalds  * Block device functions
3051da177e4SLinus Torvalds  */
306432a212cSMike Anderson int dm_deleting_md(struct mapped_device *md)
307432a212cSMike Anderson {
308432a212cSMike Anderson 	return test_bit(DMF_DELETING, &md->flags);
309432a212cSMike Anderson }
310432a212cSMike Anderson 
311fe5f9f2cSAl Viro static int dm_blk_open(struct block_device *bdev, fmode_t mode)
3121da177e4SLinus Torvalds {
3131da177e4SLinus Torvalds 	struct mapped_device *md;
3141da177e4SLinus Torvalds 
315fba9f90eSJeff Mahoney 	spin_lock(&_minor_lock);
316fba9f90eSJeff Mahoney 
317fe5f9f2cSAl Viro 	md = bdev->bd_disk->private_data;
318fba9f90eSJeff Mahoney 	if (!md)
319fba9f90eSJeff Mahoney 		goto out;
320fba9f90eSJeff Mahoney 
3215c6bd75dSAlasdair G Kergon 	if (test_bit(DMF_FREEING, &md->flags) ||
322432a212cSMike Anderson 	    dm_deleting_md(md)) {
323fba9f90eSJeff Mahoney 		md = NULL;
324fba9f90eSJeff Mahoney 		goto out;
325fba9f90eSJeff Mahoney 	}
326fba9f90eSJeff Mahoney 
3271da177e4SLinus Torvalds 	dm_get(md);
3285c6bd75dSAlasdair G Kergon 	atomic_inc(&md->open_count);
329fba9f90eSJeff Mahoney out:
330fba9f90eSJeff Mahoney 	spin_unlock(&_minor_lock);
331fba9f90eSJeff Mahoney 
332fba9f90eSJeff Mahoney 	return md ? 0 : -ENXIO;
3331da177e4SLinus Torvalds }
3341da177e4SLinus Torvalds 
335db2a144bSAl Viro static void dm_blk_close(struct gendisk *disk, fmode_t mode)
3361da177e4SLinus Torvalds {
33763a4f065SMike Snitzer 	struct mapped_device *md;
3386e9624b8SArnd Bergmann 
3394a1aeb98SMilan Broz 	spin_lock(&_minor_lock);
3404a1aeb98SMilan Broz 
34163a4f065SMike Snitzer 	md = disk->private_data;
34263a4f065SMike Snitzer 	if (WARN_ON(!md))
34363a4f065SMike Snitzer 		goto out;
34463a4f065SMike Snitzer 
3452c140a24SMikulas Patocka 	if (atomic_dec_and_test(&md->open_count) &&
3462c140a24SMikulas Patocka 	    (test_bit(DMF_DEFERRED_REMOVE, &md->flags)))
347acfe0ad7SMikulas Patocka 		queue_work(deferred_remove_workqueue, &deferred_remove_work);
3482c140a24SMikulas Patocka 
3491da177e4SLinus Torvalds 	dm_put(md);
35063a4f065SMike Snitzer out:
3514a1aeb98SMilan Broz 	spin_unlock(&_minor_lock);
3521da177e4SLinus Torvalds }
3531da177e4SLinus Torvalds 
3545c6bd75dSAlasdair G Kergon int dm_open_count(struct mapped_device *md)
3555c6bd75dSAlasdair G Kergon {
3565c6bd75dSAlasdair G Kergon 	return atomic_read(&md->open_count);
3575c6bd75dSAlasdair G Kergon }
3585c6bd75dSAlasdair G Kergon 
3595c6bd75dSAlasdair G Kergon /*
3605c6bd75dSAlasdair G Kergon  * Guarantees nothing is using the device before it's deleted.
3615c6bd75dSAlasdair G Kergon  */
3622c140a24SMikulas Patocka int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred)
3635c6bd75dSAlasdair G Kergon {
3645c6bd75dSAlasdair G Kergon 	int r = 0;
3655c6bd75dSAlasdair G Kergon 
3665c6bd75dSAlasdair G Kergon 	spin_lock(&_minor_lock);
3675c6bd75dSAlasdair G Kergon 
3682c140a24SMikulas Patocka 	if (dm_open_count(md)) {
3695c6bd75dSAlasdair G Kergon 		r = -EBUSY;
3702c140a24SMikulas Patocka 		if (mark_deferred)
3712c140a24SMikulas Patocka 			set_bit(DMF_DEFERRED_REMOVE, &md->flags);
3722c140a24SMikulas Patocka 	} else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags))
3732c140a24SMikulas Patocka 		r = -EEXIST;
3745c6bd75dSAlasdair G Kergon 	else
3755c6bd75dSAlasdair G Kergon 		set_bit(DMF_DELETING, &md->flags);
3765c6bd75dSAlasdair G Kergon 
3775c6bd75dSAlasdair G Kergon 	spin_unlock(&_minor_lock);
3785c6bd75dSAlasdair G Kergon 
3795c6bd75dSAlasdair G Kergon 	return r;
3805c6bd75dSAlasdair G Kergon }
3815c6bd75dSAlasdair G Kergon 
3822c140a24SMikulas Patocka int dm_cancel_deferred_remove(struct mapped_device *md)
3832c140a24SMikulas Patocka {
3842c140a24SMikulas Patocka 	int r = 0;
3852c140a24SMikulas Patocka 
3862c140a24SMikulas Patocka 	spin_lock(&_minor_lock);
3872c140a24SMikulas Patocka 
3882c140a24SMikulas Patocka 	if (test_bit(DMF_DELETING, &md->flags))
3892c140a24SMikulas Patocka 		r = -EBUSY;
3902c140a24SMikulas Patocka 	else
3912c140a24SMikulas Patocka 		clear_bit(DMF_DEFERRED_REMOVE, &md->flags);
3922c140a24SMikulas Patocka 
3932c140a24SMikulas Patocka 	spin_unlock(&_minor_lock);
3942c140a24SMikulas Patocka 
3952c140a24SMikulas Patocka 	return r;
3962c140a24SMikulas Patocka }
3972c140a24SMikulas Patocka 
3982c140a24SMikulas Patocka static void do_deferred_remove(struct work_struct *w)
3992c140a24SMikulas Patocka {
4002c140a24SMikulas Patocka 	dm_deferred_remove();
4012c140a24SMikulas Patocka }
4022c140a24SMikulas Patocka 
4033ac51e74SDarrick J. Wong static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
4043ac51e74SDarrick J. Wong {
4053ac51e74SDarrick J. Wong 	struct mapped_device *md = bdev->bd_disk->private_data;
4063ac51e74SDarrick J. Wong 
4073ac51e74SDarrick J. Wong 	return dm_get_geometry(md, geo);
4083ac51e74SDarrick J. Wong }
4093ac51e74SDarrick J. Wong 
410971888c4SMike Snitzer static int dm_prepare_ioctl(struct mapped_device *md, int *srcu_idx,
4115bd5e8d8SMike Snitzer 			    struct block_device **bdev)
412aa129a22SMilan Broz {
41366482026SMike Snitzer 	struct dm_target *tgt;
4146c182cd8SHannes Reinecke 	struct dm_table *map;
415971888c4SMike Snitzer 	int r;
416aa129a22SMilan Broz 
4176c182cd8SHannes Reinecke retry:
418e56f81e0SChristoph Hellwig 	r = -ENOTTY;
419971888c4SMike Snitzer 	map = dm_get_live_table(md, srcu_idx);
420aa129a22SMilan Broz 	if (!map || !dm_table_get_size(map))
421971888c4SMike Snitzer 		return r;
422aa129a22SMilan Broz 
423aa129a22SMilan Broz 	/* We only support devices that have a single target */
424aa129a22SMilan Broz 	if (dm_table_get_num_targets(map) != 1)
425971888c4SMike Snitzer 		return r;
426aa129a22SMilan Broz 
42766482026SMike Snitzer 	tgt = dm_table_get_target(map, 0);
42866482026SMike Snitzer 	if (!tgt->type->prepare_ioctl)
429e56f81e0SChristoph Hellwig 		return r;
430aa129a22SMilan Broz 
431971888c4SMike Snitzer 	if (dm_suspended_md(md))
432971888c4SMike Snitzer 		return -EAGAIN;
433971888c4SMike Snitzer 
4345bd5e8d8SMike Snitzer 	r = tgt->type->prepare_ioctl(tgt, bdev);
4355bbbfdf6SJunichi Nomura 	if (r == -ENOTCONN && !fatal_signal_pending(current)) {
436971888c4SMike Snitzer 		dm_put_live_table(md, *srcu_idx);
4376c182cd8SHannes Reinecke 		msleep(10);
4386c182cd8SHannes Reinecke 		goto retry;
4396c182cd8SHannes Reinecke 	}
440971888c4SMike Snitzer 
441e56f81e0SChristoph Hellwig 	return r;
442e56f81e0SChristoph Hellwig }
4436c182cd8SHannes Reinecke 
444971888c4SMike Snitzer static void dm_unprepare_ioctl(struct mapped_device *md, int srcu_idx)
445971888c4SMike Snitzer {
446971888c4SMike Snitzer 	dm_put_live_table(md, srcu_idx);
447971888c4SMike Snitzer }
448971888c4SMike Snitzer 
449e56f81e0SChristoph Hellwig static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
450e56f81e0SChristoph Hellwig 			unsigned int cmd, unsigned long arg)
451e56f81e0SChristoph Hellwig {
452e56f81e0SChristoph Hellwig 	struct mapped_device *md = bdev->bd_disk->private_data;
453971888c4SMike Snitzer 	int r, srcu_idx;
454e56f81e0SChristoph Hellwig 
4555bd5e8d8SMike Snitzer 	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
456e56f81e0SChristoph Hellwig 	if (r < 0)
457971888c4SMike Snitzer 		goto out;
458e56f81e0SChristoph Hellwig 
459e56f81e0SChristoph Hellwig 	if (r > 0) {
460e56f81e0SChristoph Hellwig 		/*
461e980f623SChristoph Hellwig 		 * Target determined this ioctl is being issued against a
462e980f623SChristoph Hellwig 		 * subset of the parent bdev; require extra privileges.
463e56f81e0SChristoph Hellwig 		 */
464e980f623SChristoph Hellwig 		if (!capable(CAP_SYS_RAWIO)) {
4650378c625SMike Snitzer 			DMDEBUG_LIMIT(
466e980f623SChristoph Hellwig 	"%s: sending ioctl %x to DM device without required privilege.",
467e980f623SChristoph Hellwig 				current->comm, cmd);
468e980f623SChristoph Hellwig 			r = -ENOIOCTLCMD;
469e56f81e0SChristoph Hellwig 			goto out;
470e56f81e0SChristoph Hellwig 		}
471e980f623SChristoph Hellwig 	}
472e56f81e0SChristoph Hellwig 
473a7cb3d2fSChristoph Hellwig 	if (!bdev->bd_disk->fops->ioctl)
474a7cb3d2fSChristoph Hellwig 		r = -ENOTTY;
475a7cb3d2fSChristoph Hellwig 	else
476a7cb3d2fSChristoph Hellwig 		r = bdev->bd_disk->fops->ioctl(bdev, mode, cmd, arg);
477e56f81e0SChristoph Hellwig out:
478971888c4SMike Snitzer 	dm_unprepare_ioctl(md, srcu_idx);
479aa129a22SMilan Broz 	return r;
480aa129a22SMilan Broz }
481aa129a22SMilan Broz 
4827465d7acSMike Snitzer u64 dm_start_time_ns_from_clone(struct bio *bio)
4837465d7acSMike Snitzer {
4846c23f0bdSChristoph Hellwig 	return jiffies_to_nsecs(clone_to_tio(bio)->io->start_time);
4857465d7acSMike Snitzer }
4867465d7acSMike Snitzer EXPORT_SYMBOL_GPL(dm_start_time_ns_from_clone);
4877465d7acSMike Snitzer 
4888d394bc4SMike Snitzer static bool bio_is_flush_with_data(struct bio *bio)
4897465d7acSMike Snitzer {
4908d394bc4SMike Snitzer 	return ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size);
4917465d7acSMike Snitzer }
4927465d7acSMike Snitzer 
4938d394bc4SMike Snitzer static void dm_io_acct(bool end, struct mapped_device *md, struct bio *bio,
494d208b894SJiazi Li 		       unsigned long start_time, struct dm_stats_aux *stats_aux)
4957465d7acSMike Snitzer {
4968d394bc4SMike Snitzer 	bool is_flush_with_data;
4978d394bc4SMike Snitzer 	unsigned int bi_size;
4987465d7acSMike Snitzer 
4998d394bc4SMike Snitzer 	/* If REQ_PREFLUSH set save any payload but do not account it */
5008d394bc4SMike Snitzer 	is_flush_with_data = bio_is_flush_with_data(bio);
5018d394bc4SMike Snitzer 	if (is_flush_with_data) {
5028d394bc4SMike Snitzer 		bi_size = bio->bi_iter.bi_size;
5038d394bc4SMike Snitzer 		bio->bi_iter.bi_size = 0;
5048d394bc4SMike Snitzer 	}
5058d394bc4SMike Snitzer 
5068d394bc4SMike Snitzer 	if (!end)
5078d394bc4SMike Snitzer 		bio_start_io_acct_time(bio, start_time);
5088d394bc4SMike Snitzer 	else
509d208b894SJiazi Li 		bio_end_io_acct(bio, start_time);
5107465d7acSMike Snitzer 
5117465d7acSMike Snitzer 	if (unlikely(dm_stats_used(&md->stats)))
5127465d7acSMike Snitzer 		dm_stats_account_io(&md->stats, bio_data_dir(bio),
5137465d7acSMike Snitzer 				    bio->bi_iter.bi_sector, bio_sectors(bio),
5148d394bc4SMike Snitzer 				    end, start_time, stats_aux);
5158d394bc4SMike Snitzer 
5168d394bc4SMike Snitzer 	/* Restore bio's payload so it does get accounted upon requeue */
5178d394bc4SMike Snitzer 	if (is_flush_with_data)
5188d394bc4SMike Snitzer 		bio->bi_iter.bi_size = bi_size;
5198d394bc4SMike Snitzer }
5208d394bc4SMike Snitzer 
5218d394bc4SMike Snitzer static void start_io_acct(struct dm_io *io)
5228d394bc4SMike Snitzer {
5238d394bc4SMike Snitzer 	dm_io_acct(false, io->md, io->orig_bio, io->start_time, &io->stats_aux);
5248d394bc4SMike Snitzer }
5258d394bc4SMike Snitzer 
5268d394bc4SMike Snitzer static void end_io_acct(struct mapped_device *md, struct bio *bio,
5278d394bc4SMike Snitzer 			unsigned long start_time, struct dm_stats_aux *stats_aux)
5288d394bc4SMike Snitzer {
5298d394bc4SMike Snitzer 	dm_io_acct(true, md, bio, start_time, stats_aux);
5307465d7acSMike Snitzer }
531978e51baSMike Snitzer 
532978e51baSMike Snitzer static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio)
5331da177e4SLinus Torvalds {
53464f52b0eSMike Snitzer 	struct dm_io *io;
53564f52b0eSMike Snitzer 	struct dm_target_io *tio;
53664f52b0eSMike Snitzer 	struct bio *clone;
53764f52b0eSMike Snitzer 
538abfc426dSChristoph Hellwig 	clone = bio_alloc_clone(bio->bi_bdev, bio, GFP_NOIO, &md->io_bs);
53964f52b0eSMike Snitzer 
5406c23f0bdSChristoph Hellwig 	tio = clone_to_tio(clone);
54164f52b0eSMike Snitzer 	tio->inside_dm_io = true;
54264f52b0eSMike Snitzer 	tio->io = NULL;
54364f52b0eSMike Snitzer 
54464f52b0eSMike Snitzer 	io = container_of(tio, struct dm_io, tio);
54564f52b0eSMike Snitzer 	io->magic = DM_IO_MAGIC;
546978e51baSMike Snitzer 	io->status = 0;
547978e51baSMike Snitzer 	atomic_set(&io->io_count, 1);
5489f6dc633SMike Snitzer 	this_cpu_inc(*md->pending_io);
549978e51baSMike Snitzer 	io->orig_bio = bio;
550978e51baSMike Snitzer 	io->md = md;
551978e51baSMike Snitzer 	spin_lock_init(&io->endio_lock);
552978e51baSMike Snitzer 
553b879f915SMike Snitzer 	io->start_time = jiffies;
55464f52b0eSMike Snitzer 
555*0cdb90f0SMike Snitzer 	dm_stats_record_start(&md->stats, &io->stats_aux);
556*0cdb90f0SMike Snitzer 
55764f52b0eSMike Snitzer 	return io;
5581da177e4SLinus Torvalds }
5591da177e4SLinus Torvalds 
560028867acSAlasdair G Kergon static void free_io(struct mapped_device *md, struct dm_io *io)
5611da177e4SLinus Torvalds {
56264f52b0eSMike Snitzer 	bio_put(&io->tio.clone);
56364f52b0eSMike Snitzer }
56464f52b0eSMike Snitzer 
5651d1068ceSChristoph Hellwig static struct bio *alloc_tio(struct clone_info *ci, struct dm_target *ti,
566dc8e2021SChristoph Hellwig 		unsigned target_bio_nr, unsigned *len, gfp_t gfp_mask)
56764f52b0eSMike Snitzer {
56864f52b0eSMike Snitzer 	struct dm_target_io *tio;
56964f52b0eSMike Snitzer 
57064f52b0eSMike Snitzer 	if (!ci->io->tio.io) {
57164f52b0eSMike Snitzer 		/* the dm_target_io embedded in ci->io is available */
57264f52b0eSMike Snitzer 		tio = &ci->io->tio;
57364f52b0eSMike Snitzer 	} else {
574abfc426dSChristoph Hellwig 		struct bio *clone = bio_alloc_clone(ci->bio->bi_bdev, ci->bio,
575abfc426dSChristoph Hellwig 						    gfp_mask, &ci->io->md->bs);
57664f52b0eSMike Snitzer 		if (!clone)
57764f52b0eSMike Snitzer 			return NULL;
57864f52b0eSMike Snitzer 
5796c23f0bdSChristoph Hellwig 		tio = clone_to_tio(clone);
58064f52b0eSMike Snitzer 		tio->inside_dm_io = false;
58164f52b0eSMike Snitzer 	}
58256b4b5abSChristoph Hellwig 
58364f52b0eSMike Snitzer 	tio->magic = DM_TIO_MAGIC;
58464f52b0eSMike Snitzer 	tio->io = ci->io;
58564f52b0eSMike Snitzer 	tio->ti = ti;
58664f52b0eSMike Snitzer 	tio->target_bio_nr = target_bio_nr;
587dc8e2021SChristoph Hellwig 	tio->len_ptr = len;
58864f52b0eSMike Snitzer 
5891d1068ceSChristoph Hellwig 	return &tio->clone;
5901da177e4SLinus Torvalds }
5911da177e4SLinus Torvalds 
5921d1068ceSChristoph Hellwig static void free_tio(struct bio *clone)
5931da177e4SLinus Torvalds {
5941d1068ceSChristoph Hellwig 	if (clone_to_tio(clone)->inside_dm_io)
59564f52b0eSMike Snitzer 		return;
5961d1068ceSChristoph Hellwig 	bio_put(clone);
5971da177e4SLinus Torvalds }
5981da177e4SLinus Torvalds 
5991da177e4SLinus Torvalds /*
6001da177e4SLinus Torvalds  * Add the bio to the list of deferred io.
6011da177e4SLinus Torvalds  */
60292c63902SMikulas Patocka static void queue_io(struct mapped_device *md, struct bio *bio)
6031da177e4SLinus Torvalds {
60405447420SKiyoshi Ueda 	unsigned long flags;
6051da177e4SLinus Torvalds 
60605447420SKiyoshi Ueda 	spin_lock_irqsave(&md->deferred_lock, flags);
6071da177e4SLinus Torvalds 	bio_list_add(&md->deferred, bio);
60805447420SKiyoshi Ueda 	spin_unlock_irqrestore(&md->deferred_lock, flags);
60992c63902SMikulas Patocka 	queue_work(md->wq, &md->work);
6101da177e4SLinus Torvalds }
6111da177e4SLinus Torvalds 
6121da177e4SLinus Torvalds /*
6131da177e4SLinus Torvalds  * Everyone (including functions in this file), should use this
6141da177e4SLinus Torvalds  * function to access the md->map field, and make sure they call
61583d5e5b0SMikulas Patocka  * dm_put_live_table() when finished.
6161da177e4SLinus Torvalds  */
61783d5e5b0SMikulas Patocka struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier)
6181da177e4SLinus Torvalds {
61983d5e5b0SMikulas Patocka 	*srcu_idx = srcu_read_lock(&md->io_barrier);
6201da177e4SLinus Torvalds 
62183d5e5b0SMikulas Patocka 	return srcu_dereference(md->map, &md->io_barrier);
62283d5e5b0SMikulas Patocka }
6231da177e4SLinus Torvalds 
62483d5e5b0SMikulas Patocka void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier)
62583d5e5b0SMikulas Patocka {
62683d5e5b0SMikulas Patocka 	srcu_read_unlock(&md->io_barrier, srcu_idx);
62783d5e5b0SMikulas Patocka }
62883d5e5b0SMikulas Patocka 
62983d5e5b0SMikulas Patocka void dm_sync_table(struct mapped_device *md)
63083d5e5b0SMikulas Patocka {
63183d5e5b0SMikulas Patocka 	synchronize_srcu(&md->io_barrier);
63283d5e5b0SMikulas Patocka 	synchronize_rcu_expedited();
63383d5e5b0SMikulas Patocka }
63483d5e5b0SMikulas Patocka 
63583d5e5b0SMikulas Patocka /*
63683d5e5b0SMikulas Patocka  * A fast alternative to dm_get_live_table/dm_put_live_table.
63783d5e5b0SMikulas Patocka  * The caller must not block between these two functions.
63883d5e5b0SMikulas Patocka  */
63983d5e5b0SMikulas Patocka static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU)
64083d5e5b0SMikulas Patocka {
64183d5e5b0SMikulas Patocka 	rcu_read_lock();
64283d5e5b0SMikulas Patocka 	return rcu_dereference(md->map);
64383d5e5b0SMikulas Patocka }
64483d5e5b0SMikulas Patocka 
64583d5e5b0SMikulas Patocka static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU)
64683d5e5b0SMikulas Patocka {
64783d5e5b0SMikulas Patocka 	rcu_read_unlock();
6481da177e4SLinus Torvalds }
6491da177e4SLinus Torvalds 
650971888c4SMike Snitzer static char *_dm_claim_ptr = "I belong to device-mapper";
651971888c4SMike Snitzer 
6523ac51e74SDarrick J. Wong /*
65386f1152bSBenjamin Marzinski  * Open a table device so we can use it as a map destination.
65486f1152bSBenjamin Marzinski  */
65586f1152bSBenjamin Marzinski static int open_table_device(struct table_device *td, dev_t dev,
65686f1152bSBenjamin Marzinski 			     struct mapped_device *md)
65786f1152bSBenjamin Marzinski {
65886f1152bSBenjamin Marzinski 	struct block_device *bdev;
659cd913c76SChristoph Hellwig 	u64 part_off;
66086f1152bSBenjamin Marzinski 	int r;
66186f1152bSBenjamin Marzinski 
66286f1152bSBenjamin Marzinski 	BUG_ON(td->dm_dev.bdev);
66386f1152bSBenjamin Marzinski 
664519049afSMike Snitzer 	bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _dm_claim_ptr);
66586f1152bSBenjamin Marzinski 	if (IS_ERR(bdev))
66686f1152bSBenjamin Marzinski 		return PTR_ERR(bdev);
66786f1152bSBenjamin Marzinski 
66886f1152bSBenjamin Marzinski 	r = bd_link_disk_holder(bdev, dm_disk(md));
66986f1152bSBenjamin Marzinski 	if (r) {
67086f1152bSBenjamin Marzinski 		blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL);
67186f1152bSBenjamin Marzinski 		return r;
67286f1152bSBenjamin Marzinski 	}
67386f1152bSBenjamin Marzinski 
67486f1152bSBenjamin Marzinski 	td->dm_dev.bdev = bdev;
675cd913c76SChristoph Hellwig 	td->dm_dev.dax_dev = fs_dax_get_by_bdev(bdev, &part_off);
67686f1152bSBenjamin Marzinski 	return 0;
67786f1152bSBenjamin Marzinski }
67886f1152bSBenjamin Marzinski 
67986f1152bSBenjamin Marzinski /*
68086f1152bSBenjamin Marzinski  * Close a table device that we've been using.
68186f1152bSBenjamin Marzinski  */
68286f1152bSBenjamin Marzinski static void close_table_device(struct table_device *td, struct mapped_device *md)
68386f1152bSBenjamin Marzinski {
68486f1152bSBenjamin Marzinski 	if (!td->dm_dev.bdev)
68586f1152bSBenjamin Marzinski 		return;
68686f1152bSBenjamin Marzinski 
68786f1152bSBenjamin Marzinski 	bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md));
68886f1152bSBenjamin Marzinski 	blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL);
689817bf402SDan Williams 	put_dax(td->dm_dev.dax_dev);
69086f1152bSBenjamin Marzinski 	td->dm_dev.bdev = NULL;
691817bf402SDan Williams 	td->dm_dev.dax_dev = NULL;
69286f1152bSBenjamin Marzinski }
69386f1152bSBenjamin Marzinski 
69486f1152bSBenjamin Marzinski static struct table_device *find_table_device(struct list_head *l, dev_t dev,
6958454fca4SSheetal Singala 					      fmode_t mode)
6968454fca4SSheetal Singala {
69786f1152bSBenjamin Marzinski 	struct table_device *td;
69886f1152bSBenjamin Marzinski 
69986f1152bSBenjamin Marzinski 	list_for_each_entry(td, l, list)
70086f1152bSBenjamin Marzinski 		if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode)
70186f1152bSBenjamin Marzinski 			return td;
70286f1152bSBenjamin Marzinski 
70386f1152bSBenjamin Marzinski 	return NULL;
70486f1152bSBenjamin Marzinski }
70586f1152bSBenjamin Marzinski 
70686f1152bSBenjamin Marzinski int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode,
7078454fca4SSheetal Singala 			struct dm_dev **result)
7088454fca4SSheetal Singala {
70986f1152bSBenjamin Marzinski 	int r;
71086f1152bSBenjamin Marzinski 	struct table_device *td;
71186f1152bSBenjamin Marzinski 
71286f1152bSBenjamin Marzinski 	mutex_lock(&md->table_devices_lock);
71386f1152bSBenjamin Marzinski 	td = find_table_device(&md->table_devices, dev, mode);
71486f1152bSBenjamin Marzinski 	if (!td) {
715115485e8SMike Snitzer 		td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id);
71686f1152bSBenjamin Marzinski 		if (!td) {
71786f1152bSBenjamin Marzinski 			mutex_unlock(&md->table_devices_lock);
71886f1152bSBenjamin Marzinski 			return -ENOMEM;
71986f1152bSBenjamin Marzinski 		}
72086f1152bSBenjamin Marzinski 
72186f1152bSBenjamin Marzinski 		td->dm_dev.mode = mode;
72286f1152bSBenjamin Marzinski 		td->dm_dev.bdev = NULL;
72386f1152bSBenjamin Marzinski 
72486f1152bSBenjamin Marzinski 		if ((r = open_table_device(td, dev, md))) {
72586f1152bSBenjamin Marzinski 			mutex_unlock(&md->table_devices_lock);
72686f1152bSBenjamin Marzinski 			kfree(td);
72786f1152bSBenjamin Marzinski 			return r;
72886f1152bSBenjamin Marzinski 		}
72986f1152bSBenjamin Marzinski 
73086f1152bSBenjamin Marzinski 		format_dev_t(td->dm_dev.name, dev);
73186f1152bSBenjamin Marzinski 
732b0b4d7c6SElena Reshetova 		refcount_set(&td->count, 1);
73386f1152bSBenjamin Marzinski 		list_add(&td->list, &md->table_devices);
734b0b4d7c6SElena Reshetova 	} else {
735b0b4d7c6SElena Reshetova 		refcount_inc(&td->count);
73686f1152bSBenjamin Marzinski 	}
73786f1152bSBenjamin Marzinski 	mutex_unlock(&md->table_devices_lock);
73886f1152bSBenjamin Marzinski 
73986f1152bSBenjamin Marzinski 	*result = &td->dm_dev;
74086f1152bSBenjamin Marzinski 	return 0;
74186f1152bSBenjamin Marzinski }
74286f1152bSBenjamin Marzinski 
74386f1152bSBenjamin Marzinski void dm_put_table_device(struct mapped_device *md, struct dm_dev *d)
74486f1152bSBenjamin Marzinski {
74586f1152bSBenjamin Marzinski 	struct table_device *td = container_of(d, struct table_device, dm_dev);
74686f1152bSBenjamin Marzinski 
74786f1152bSBenjamin Marzinski 	mutex_lock(&md->table_devices_lock);
748b0b4d7c6SElena Reshetova 	if (refcount_dec_and_test(&td->count)) {
74986f1152bSBenjamin Marzinski 		close_table_device(td, md);
75086f1152bSBenjamin Marzinski 		list_del(&td->list);
75186f1152bSBenjamin Marzinski 		kfree(td);
75286f1152bSBenjamin Marzinski 	}
75386f1152bSBenjamin Marzinski 	mutex_unlock(&md->table_devices_lock);
75486f1152bSBenjamin Marzinski }
75586f1152bSBenjamin Marzinski 
75686f1152bSBenjamin Marzinski static void free_table_devices(struct list_head *devices)
75786f1152bSBenjamin Marzinski {
75886f1152bSBenjamin Marzinski 	struct list_head *tmp, *next;
75986f1152bSBenjamin Marzinski 
76086f1152bSBenjamin Marzinski 	list_for_each_safe(tmp, next, devices) {
76186f1152bSBenjamin Marzinski 		struct table_device *td = list_entry(tmp, struct table_device, list);
76286f1152bSBenjamin Marzinski 
76386f1152bSBenjamin Marzinski 		DMWARN("dm_destroy: %s still exists with %d references",
764b0b4d7c6SElena Reshetova 		       td->dm_dev.name, refcount_read(&td->count));
76586f1152bSBenjamin Marzinski 		kfree(td);
76686f1152bSBenjamin Marzinski 	}
76786f1152bSBenjamin Marzinski }
76886f1152bSBenjamin Marzinski 
76986f1152bSBenjamin Marzinski /*
7703ac51e74SDarrick J. Wong  * Get the geometry associated with a dm device
7713ac51e74SDarrick J. Wong  */
7723ac51e74SDarrick J. Wong int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
7733ac51e74SDarrick J. Wong {
7743ac51e74SDarrick J. Wong 	*geo = md->geometry;
7753ac51e74SDarrick J. Wong 
7763ac51e74SDarrick J. Wong 	return 0;
7773ac51e74SDarrick J. Wong }
7783ac51e74SDarrick J. Wong 
7793ac51e74SDarrick J. Wong /*
7803ac51e74SDarrick J. Wong  * Set the geometry of a device.
7813ac51e74SDarrick J. Wong  */
7823ac51e74SDarrick J. Wong int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
7833ac51e74SDarrick J. Wong {
7843ac51e74SDarrick J. Wong 	sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
7853ac51e74SDarrick J. Wong 
7863ac51e74SDarrick J. Wong 	if (geo->start > sz) {
7873ac51e74SDarrick J. Wong 		DMWARN("Start sector is beyond the geometry limits.");
7883ac51e74SDarrick J. Wong 		return -EINVAL;
7893ac51e74SDarrick J. Wong 	}
7903ac51e74SDarrick J. Wong 
7913ac51e74SDarrick J. Wong 	md->geometry = *geo;
7923ac51e74SDarrick J. Wong 
7933ac51e74SDarrick J. Wong 	return 0;
7943ac51e74SDarrick J. Wong }
7953ac51e74SDarrick J. Wong 
7962e93ccc1SKiyoshi Ueda static int __noflush_suspending(struct mapped_device *md)
7972e93ccc1SKiyoshi Ueda {
7982e93ccc1SKiyoshi Ueda 	return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
7992e93ccc1SKiyoshi Ueda }
8002e93ccc1SKiyoshi Ueda 
8011da177e4SLinus Torvalds /*
8021da177e4SLinus Torvalds  * Decrements the number of outstanding ios that a bio has been
8031da177e4SLinus Torvalds  * cloned into, completing the original io if necc.
8041da177e4SLinus Torvalds  */
805e2118b3cSDamien Le Moal void dm_io_dec_pending(struct dm_io *io, blk_status_t error)
8061da177e4SLinus Torvalds {
8072e93ccc1SKiyoshi Ueda 	unsigned long flags;
8084e4cbee9SChristoph Hellwig 	blk_status_t io_error;
809b35f8caaSMilan Broz 	struct bio *bio;
810b35f8caaSMilan Broz 	struct mapped_device *md = io->md;
811d208b894SJiazi Li 	unsigned long start_time = 0;
812d208b894SJiazi Li 	struct dm_stats_aux stats_aux;
8132e93ccc1SKiyoshi Ueda 
8142e93ccc1SKiyoshi Ueda 	/* Push-back supersedes any I/O errors */
815f88fb981SKiyoshi Ueda 	if (unlikely(error)) {
816f88fb981SKiyoshi Ueda 		spin_lock_irqsave(&io->endio_lock, flags);
817745dc570SMike Snitzer 		if (!(io->status == BLK_STS_DM_REQUEUE && __noflush_suspending(md)))
8184e4cbee9SChristoph Hellwig 			io->status = error;
819f88fb981SKiyoshi Ueda 		spin_unlock_irqrestore(&io->endio_lock, flags);
820f88fb981SKiyoshi Ueda 	}
8211da177e4SLinus Torvalds 
8221da177e4SLinus Torvalds 	if (atomic_dec_and_test(&io->io_count)) {
823bf14e2b2SDamien Le Moal 		bio = io->orig_bio;
8244e4cbee9SChristoph Hellwig 		if (io->status == BLK_STS_DM_REQUEUE) {
8252e93ccc1SKiyoshi Ueda 			/*
8262e93ccc1SKiyoshi Ueda 			 * Target requested pushing back the I/O.
8272e93ccc1SKiyoshi Ueda 			 */
828022c2611SMikulas Patocka 			spin_lock_irqsave(&md->deferred_lock, flags);
829bf14e2b2SDamien Le Moal 			if (__noflush_suspending(md) &&
830bf14e2b2SDamien Le Moal 			    !WARN_ON_ONCE(dm_is_zone_write(md, bio))) {
831745dc570SMike Snitzer 				/* NOTE early return due to BLK_STS_DM_REQUEUE below */
832bf14e2b2SDamien Le Moal 				bio_list_add_head(&md->deferred, bio);
833bf14e2b2SDamien Le Moal 			} else {
834bf14e2b2SDamien Le Moal 				/*
835bf14e2b2SDamien Le Moal 				 * noflush suspend was interrupted or this is
836bf14e2b2SDamien Le Moal 				 * a write to a zoned target.
837bf14e2b2SDamien Le Moal 				 */
8384e4cbee9SChristoph Hellwig 				io->status = BLK_STS_IOERR;
839bf14e2b2SDamien Le Moal 			}
840022c2611SMikulas Patocka 			spin_unlock_irqrestore(&md->deferred_lock, flags);
8412e93ccc1SKiyoshi Ueda 		}
8422e93ccc1SKiyoshi Ueda 
8434e4cbee9SChristoph Hellwig 		io_error = io->status;
844d208b894SJiazi Li 		start_time = io->start_time;
845d208b894SJiazi Li 		stats_aux = io->stats_aux;
846a97f925aSMikulas Patocka 		free_io(md, io);
847d208b894SJiazi Li 		end_io_acct(md, bio, start_time, &stats_aux);
8489f6dc633SMike Snitzer 		smp_wmb();
8499f6dc633SMike Snitzer 		this_cpu_dec(*md->pending_io);
8509f6dc633SMike Snitzer 
8519f6dc633SMike Snitzer 		/* nudge anyone waiting on suspend queue */
8529f6dc633SMike Snitzer 		if (unlikely(wq_has_sleeper(&md->wait)))
8539f6dc633SMike Snitzer 			wake_up(&md->wait);
8541da177e4SLinus Torvalds 
8554e4cbee9SChristoph Hellwig 		if (io_error == BLK_STS_DM_REQUEUE)
8566a8736d1STejun Heo 			return;
8576a8736d1STejun Heo 
8588d394bc4SMike Snitzer 		if (bio_is_flush_with_data(bio)) {
8591da177e4SLinus Torvalds 			/*
8606a8736d1STejun Heo 			 * Preflush done for flush with data, reissue
86128a8f0d3SMike Christie 			 * without REQ_PREFLUSH.
862af7e466aSMikulas Patocka 			 */
8631eff9d32SJens Axboe 			bio->bi_opf &= ~REQ_PREFLUSH;
8646a8736d1STejun Heo 			queue_io(md, bio);
865af7e466aSMikulas Patocka 		} else {
866b372d360SMike Snitzer 			/* done with normal IO or empty flush */
8678dd601faSNeilBrown 			if (io_error)
8684e4cbee9SChristoph Hellwig 				bio->bi_status = io_error;
8694246a0b6SChristoph Hellwig 			bio_endio(bio);
8702e93ccc1SKiyoshi Ueda 		}
8711da177e4SLinus Torvalds 	}
872af7e466aSMikulas Patocka }
8731da177e4SLinus Torvalds 
874bcb44433SMike Snitzer void disable_discard(struct mapped_device *md)
875bcb44433SMike Snitzer {
876bcb44433SMike Snitzer 	struct queue_limits *limits = dm_get_queue_limits(md);
877bcb44433SMike Snitzer 
878bcb44433SMike Snitzer 	/* device doesn't really support DISCARD, disable it */
879bcb44433SMike Snitzer 	limits->max_discard_sectors = 0;
880bcb44433SMike Snitzer 	blk_queue_flag_clear(QUEUE_FLAG_DISCARD, md->queue);
881bcb44433SMike Snitzer }
882bcb44433SMike Snitzer 
8834cc96131SMike Snitzer void disable_write_same(struct mapped_device *md)
8847eee4ae2SMike Snitzer {
8857eee4ae2SMike Snitzer 	struct queue_limits *limits = dm_get_queue_limits(md);
8867eee4ae2SMike Snitzer 
8877eee4ae2SMike Snitzer 	/* device doesn't really support WRITE SAME, disable it */
8887eee4ae2SMike Snitzer 	limits->max_write_same_sectors = 0;
8897eee4ae2SMike Snitzer }
8907eee4ae2SMike Snitzer 
891ac62d620SChristoph Hellwig void disable_write_zeroes(struct mapped_device *md)
892ac62d620SChristoph Hellwig {
893ac62d620SChristoph Hellwig 	struct queue_limits *limits = dm_get_queue_limits(md);
894ac62d620SChristoph Hellwig 
895ac62d620SChristoph Hellwig 	/* device doesn't really support WRITE ZEROES, disable it */
896ac62d620SChristoph Hellwig 	limits->max_write_zeroes_sectors = 0;
897ac62d620SChristoph Hellwig }
898ac62d620SChristoph Hellwig 
899a666e5c0SMikulas Patocka static bool swap_bios_limit(struct dm_target *ti, struct bio *bio)
900a666e5c0SMikulas Patocka {
901a666e5c0SMikulas Patocka 	return unlikely((bio->bi_opf & REQ_SWAP) != 0) && unlikely(ti->limit_swap_bios);
902a666e5c0SMikulas Patocka }
903a666e5c0SMikulas Patocka 
9044246a0b6SChristoph Hellwig static void clone_endio(struct bio *bio)
9051da177e4SLinus Torvalds {
9064e4cbee9SChristoph Hellwig 	blk_status_t error = bio->bi_status;
9076c23f0bdSChristoph Hellwig 	struct dm_target_io *tio = clone_to_tio(bio);
908b35f8caaSMilan Broz 	struct dm_io *io = tio->io;
9099faf400fSStefan Bader 	struct mapped_device *md = tio->io->md;
9101da177e4SLinus Torvalds 	dm_endio_fn endio = tio->ti->type->end_io;
911309dca30SChristoph Hellwig 	struct request_queue *q = bio->bi_bdev->bd_disk->queue;
9121da177e4SLinus Torvalds 
9139c37de29SMike Snitzer 	if (unlikely(error == BLK_STS_TARGET)) {
914bcb44433SMike Snitzer 		if (bio_op(bio) == REQ_OP_DISCARD &&
915309dca30SChristoph Hellwig 		    !q->limits.max_discard_sectors)
916bcb44433SMike Snitzer 			disable_discard(md);
917bcb44433SMike Snitzer 		else if (bio_op(bio) == REQ_OP_WRITE_SAME &&
918309dca30SChristoph Hellwig 			 !q->limits.max_write_same_sectors)
9197eee4ae2SMike Snitzer 			disable_write_same(md);
920bcb44433SMike Snitzer 		else if (bio_op(bio) == REQ_OP_WRITE_ZEROES &&
921309dca30SChristoph Hellwig 			 !q->limits.max_write_zeroes_sectors)
922ac62d620SChristoph Hellwig 			disable_write_zeroes(md);
923ac62d620SChristoph Hellwig 	}
9247eee4ae2SMike Snitzer 
925bb37d772SDamien Le Moal 	if (blk_queue_is_zoned(q))
926bb37d772SDamien Le Moal 		dm_zone_endio(io, bio);
927415c79e1SJohannes Thumshirn 
9281be56909SChristoph Hellwig 	if (endio) {
9294e4cbee9SChristoph Hellwig 		int r = endio(tio->ti, bio, &error);
9301be56909SChristoph Hellwig 		switch (r) {
9311be56909SChristoph Hellwig 		case DM_ENDIO_REQUEUE:
932bf14e2b2SDamien Le Moal 			/*
933bf14e2b2SDamien Le Moal 			 * Requeuing writes to a sequential zone of a zoned
934bf14e2b2SDamien Le Moal 			 * target will break the sequential write pattern:
935bf14e2b2SDamien Le Moal 			 * fail such IO.
936bf14e2b2SDamien Le Moal 			 */
937bf14e2b2SDamien Le Moal 			if (WARN_ON_ONCE(dm_is_zone_write(md, bio)))
938bf14e2b2SDamien Le Moal 				error = BLK_STS_IOERR;
939bf14e2b2SDamien Le Moal 			else
9404e4cbee9SChristoph Hellwig 				error = BLK_STS_DM_REQUEUE;
941df561f66SGustavo A. R. Silva 			fallthrough;
9421be56909SChristoph Hellwig 		case DM_ENDIO_DONE:
9431be56909SChristoph Hellwig 			break;
9441be56909SChristoph Hellwig 		case DM_ENDIO_INCOMPLETE:
9451be56909SChristoph Hellwig 			/* The target will handle the io */
9461be56909SChristoph Hellwig 			return;
9471be56909SChristoph Hellwig 		default:
9481be56909SChristoph Hellwig 			DMWARN("unimplemented target endio return value: %d", r);
9491be56909SChristoph Hellwig 			BUG();
9501be56909SChristoph Hellwig 		}
9511be56909SChristoph Hellwig 	}
9521be56909SChristoph Hellwig 
953a666e5c0SMikulas Patocka 	if (unlikely(swap_bios_limit(tio->ti, bio))) {
954a666e5c0SMikulas Patocka 		struct mapped_device *md = io->md;
955a666e5c0SMikulas Patocka 		up(&md->swap_bios_semaphore);
956a666e5c0SMikulas Patocka 	}
957a666e5c0SMikulas Patocka 
9581d1068ceSChristoph Hellwig 	free_tio(bio);
959e2118b3cSDamien Le Moal 	dm_io_dec_pending(io, error);
9601da177e4SLinus Torvalds }
9611da177e4SLinus Torvalds 
96278d8e58aSMike Snitzer /*
96356a67df7SMike Snitzer  * Return maximum size of I/O possible at the supplied sector up to the current
96456a67df7SMike Snitzer  * target boundary.
96556a67df7SMike Snitzer  */
9663720281dSMike Snitzer static inline sector_t max_io_len_target_boundary(struct dm_target *ti,
9673720281dSMike Snitzer 						  sector_t target_offset)
9681da177e4SLinus Torvalds {
96956a67df7SMike Snitzer 	return ti->len - target_offset;
97056a67df7SMike Snitzer }
97156a67df7SMike Snitzer 
9723720281dSMike Snitzer static sector_t max_io_len(struct dm_target *ti, sector_t sector)
97356a67df7SMike Snitzer {
9743720281dSMike Snitzer 	sector_t target_offset = dm_target_offset(ti, sector);
9753720281dSMike Snitzer 	sector_t len = max_io_len_target_boundary(ti, target_offset);
9765091cdecSMike Snitzer 	sector_t max_len;
9771da177e4SLinus Torvalds 
9781da177e4SLinus Torvalds 	/*
9793ee16db3SMike Snitzer 	 * Does the target need to split IO even further?
9803ee16db3SMike Snitzer 	 * - varied (per target) IO splitting is a tenet of DM; this
9813ee16db3SMike Snitzer 	 *   explains why stacked chunk_sectors based splitting via
9823ee16db3SMike Snitzer 	 *   blk_max_size_offset() isn't possible here. So pass in
9833ee16db3SMike Snitzer 	 *   ti->max_io_len to override stacked chunk_sectors.
9841da177e4SLinus Torvalds 	 */
9853ee16db3SMike Snitzer 	if (ti->max_io_len) {
98633bd6f06SMike Snitzer 		max_len = blk_max_size_offset(ti->table->md->queue,
9873ee16db3SMike Snitzer 					      target_offset, ti->max_io_len);
988542f9038SMike Snitzer 		if (len > max_len)
989542f9038SMike Snitzer 			len = max_len;
9903ee16db3SMike Snitzer 	}
9911da177e4SLinus Torvalds 
9921da177e4SLinus Torvalds 	return len;
9931da177e4SLinus Torvalds }
9941da177e4SLinus Torvalds 
995542f9038SMike Snitzer int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
996542f9038SMike Snitzer {
997542f9038SMike Snitzer 	if (len > UINT_MAX) {
998542f9038SMike Snitzer 		DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)",
999542f9038SMike Snitzer 		      (unsigned long long)len, UINT_MAX);
1000542f9038SMike Snitzer 		ti->error = "Maximum size of target IO is too large";
1001542f9038SMike Snitzer 		return -EINVAL;
1002542f9038SMike Snitzer 	}
1003542f9038SMike Snitzer 
100475ae1936SMikulas Patocka 	ti->max_io_len = (uint32_t) len;
1005542f9038SMike Snitzer 
1006542f9038SMike Snitzer 	return 0;
1007542f9038SMike Snitzer }
1008542f9038SMike Snitzer EXPORT_SYMBOL_GPL(dm_set_target_max_io_len);
1009542f9038SMike Snitzer 
1010f26c5719SDan Williams static struct dm_target *dm_dax_get_live_target(struct mapped_device *md,
1011f26c5719SDan Williams 						sector_t sector, int *srcu_idx)
10123d97c829SMike Snitzer 	__acquires(md->io_barrier)
1013545ed20eSToshi Kani {
1014545ed20eSToshi Kani 	struct dm_table *map;
1015545ed20eSToshi Kani 	struct dm_target *ti;
1016545ed20eSToshi Kani 
1017f26c5719SDan Williams 	map = dm_get_live_table(md, srcu_idx);
1018545ed20eSToshi Kani 	if (!map)
1019f26c5719SDan Williams 		return NULL;
1020545ed20eSToshi Kani 
1021545ed20eSToshi Kani 	ti = dm_table_find_target(map, sector);
1022123d87d5SMikulas Patocka 	if (!ti)
1023f26c5719SDan Williams 		return NULL;
1024f26c5719SDan Williams 
1025f26c5719SDan Williams 	return ti;
1026f26c5719SDan Williams }
1027f26c5719SDan Williams 
1028f26c5719SDan Williams static long dm_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
1029f26c5719SDan Williams 				 long nr_pages, void **kaddr, pfn_t *pfn)
1030f26c5719SDan Williams {
1031f26c5719SDan Williams 	struct mapped_device *md = dax_get_private(dax_dev);
1032f26c5719SDan Williams 	sector_t sector = pgoff * PAGE_SECTORS;
1033f26c5719SDan Williams 	struct dm_target *ti;
1034f26c5719SDan Williams 	long len, ret = -EIO;
1035f26c5719SDan Williams 	int srcu_idx;
1036f26c5719SDan Williams 
1037f26c5719SDan Williams 	ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1038f26c5719SDan Williams 
1039f26c5719SDan Williams 	if (!ti)
1040545ed20eSToshi Kani 		goto out;
1041f26c5719SDan Williams 	if (!ti->type->direct_access)
1042f26c5719SDan Williams 		goto out;
10433720281dSMike Snitzer 	len = max_io_len(ti, sector) / PAGE_SECTORS;
1044f26c5719SDan Williams 	if (len < 1)
1045f26c5719SDan Williams 		goto out;
1046f26c5719SDan Williams 	nr_pages = min(len, nr_pages);
1047817bf402SDan Williams 	ret = ti->type->direct_access(ti, pgoff, nr_pages, kaddr, pfn);
1048817bf402SDan Williams 
1049545ed20eSToshi Kani  out:
1050545ed20eSToshi Kani 	dm_put_live_table(md, srcu_idx);
1051f26c5719SDan Williams 
1052f26c5719SDan Williams 	return ret;
1053545ed20eSToshi Kani }
1054545ed20eSToshi Kani 
1055cdf6cdcdSVivek Goyal static int dm_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,
1056cdf6cdcdSVivek Goyal 				  size_t nr_pages)
1057cdf6cdcdSVivek Goyal {
1058cdf6cdcdSVivek Goyal 	struct mapped_device *md = dax_get_private(dax_dev);
1059cdf6cdcdSVivek Goyal 	sector_t sector = pgoff * PAGE_SECTORS;
1060cdf6cdcdSVivek Goyal 	struct dm_target *ti;
1061cdf6cdcdSVivek Goyal 	int ret = -EIO;
1062cdf6cdcdSVivek Goyal 	int srcu_idx;
1063cdf6cdcdSVivek Goyal 
1064cdf6cdcdSVivek Goyal 	ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1065cdf6cdcdSVivek Goyal 
1066cdf6cdcdSVivek Goyal 	if (!ti)
1067cdf6cdcdSVivek Goyal 		goto out;
1068cdf6cdcdSVivek Goyal 	if (WARN_ON(!ti->type->dax_zero_page_range)) {
1069cdf6cdcdSVivek Goyal 		/*
1070cdf6cdcdSVivek Goyal 		 * ->zero_page_range() is mandatory dax operation. If we are
1071cdf6cdcdSVivek Goyal 		 *  here, something is wrong.
1072cdf6cdcdSVivek Goyal 		 */
1073cdf6cdcdSVivek Goyal 		goto out;
1074cdf6cdcdSVivek Goyal 	}
1075cdf6cdcdSVivek Goyal 	ret = ti->type->dax_zero_page_range(ti, pgoff, nr_pages);
1076cdf6cdcdSVivek Goyal  out:
1077cdf6cdcdSVivek Goyal 	dm_put_live_table(md, srcu_idx);
1078cdf6cdcdSVivek Goyal 
1079cdf6cdcdSVivek Goyal 	return ret;
1080cdf6cdcdSVivek Goyal }
1081cdf6cdcdSVivek Goyal 
10821dd40c3eSMikulas Patocka /*
10831dd40c3eSMikulas Patocka  * A target may call dm_accept_partial_bio only from the map routine.  It is
10846842d264SDamien Le Moal  * allowed for all bio types except REQ_PREFLUSH, REQ_OP_ZONE_* zone management
10856842d264SDamien Le Moal  * operations and REQ_OP_ZONE_APPEND (zone append writes).
10861dd40c3eSMikulas Patocka  *
10871dd40c3eSMikulas Patocka  * dm_accept_partial_bio informs the dm that the target only wants to process
10881dd40c3eSMikulas Patocka  * additional n_sectors sectors of the bio and the rest of the data should be
10891dd40c3eSMikulas Patocka  * sent in a next bio.
10901dd40c3eSMikulas Patocka  *
10911dd40c3eSMikulas Patocka  * A diagram that explains the arithmetics:
10921dd40c3eSMikulas Patocka  * +--------------------+---------------+-------+
10931dd40c3eSMikulas Patocka  * |         1          |       2       |   3   |
10941dd40c3eSMikulas Patocka  * +--------------------+---------------+-------+
10951dd40c3eSMikulas Patocka  *
10961dd40c3eSMikulas Patocka  * <-------------- *tio->len_ptr --------------->
10971dd40c3eSMikulas Patocka  *                      <------- bi_size ------->
10981dd40c3eSMikulas Patocka  *                      <-- n_sectors -->
10991dd40c3eSMikulas Patocka  *
11001dd40c3eSMikulas Patocka  * Region 1 was already iterated over with bio_advance or similar function.
11011dd40c3eSMikulas Patocka  *	(it may be empty if the target doesn't use bio_advance)
11021dd40c3eSMikulas Patocka  * Region 2 is the remaining bio size that the target wants to process.
11031dd40c3eSMikulas Patocka  *	(it may be empty if region 1 is non-empty, although there is no reason
11041dd40c3eSMikulas Patocka  *	 to make it empty)
11051dd40c3eSMikulas Patocka  * The target requires that region 3 is to be sent in the next bio.
11061dd40c3eSMikulas Patocka  *
11071dd40c3eSMikulas Patocka  * If the target wants to receive multiple copies of the bio (via num_*bios, etc),
11081dd40c3eSMikulas Patocka  * the partially processed part (the sum of regions 1+2) must be the same for all
11091dd40c3eSMikulas Patocka  * copies of the bio.
11101dd40c3eSMikulas Patocka  */
11111dd40c3eSMikulas Patocka void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors)
11121dd40c3eSMikulas Patocka {
11136c23f0bdSChristoph Hellwig 	struct dm_target_io *tio = clone_to_tio(bio);
11141dd40c3eSMikulas Patocka 	unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT;
11156842d264SDamien Le Moal 
11161eff9d32SJens Axboe 	BUG_ON(bio->bi_opf & REQ_PREFLUSH);
11176842d264SDamien Le Moal 	BUG_ON(op_is_zone_mgmt(bio_op(bio)));
11186842d264SDamien Le Moal 	BUG_ON(bio_op(bio) == REQ_OP_ZONE_APPEND);
11191dd40c3eSMikulas Patocka 	BUG_ON(bi_size > *tio->len_ptr);
11201dd40c3eSMikulas Patocka 	BUG_ON(n_sectors > bi_size);
11216842d264SDamien Le Moal 
11221dd40c3eSMikulas Patocka 	*tio->len_ptr -= bi_size - n_sectors;
11231dd40c3eSMikulas Patocka 	bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT;
11241dd40c3eSMikulas Patocka }
11251dd40c3eSMikulas Patocka EXPORT_SYMBOL_GPL(dm_accept_partial_bio);
11261dd40c3eSMikulas Patocka 
1127a666e5c0SMikulas Patocka static noinline void __set_swap_bios_limit(struct mapped_device *md, int latch)
1128a666e5c0SMikulas Patocka {
1129a666e5c0SMikulas Patocka 	mutex_lock(&md->swap_bios_lock);
1130a666e5c0SMikulas Patocka 	while (latch < md->swap_bios) {
1131a666e5c0SMikulas Patocka 		cond_resched();
1132a666e5c0SMikulas Patocka 		down(&md->swap_bios_semaphore);
1133a666e5c0SMikulas Patocka 		md->swap_bios--;
1134a666e5c0SMikulas Patocka 	}
1135a666e5c0SMikulas Patocka 	while (latch > md->swap_bios) {
1136a666e5c0SMikulas Patocka 		cond_resched();
1137a666e5c0SMikulas Patocka 		up(&md->swap_bios_semaphore);
1138a666e5c0SMikulas Patocka 		md->swap_bios++;
1139a666e5c0SMikulas Patocka 	}
1140a666e5c0SMikulas Patocka 	mutex_unlock(&md->swap_bios_lock);
1141a666e5c0SMikulas Patocka }
1142a666e5c0SMikulas Patocka 
11431561b396SChristoph Hellwig static void __map_bio(struct bio *clone)
11441da177e4SLinus Torvalds {
11451561b396SChristoph Hellwig 	struct dm_target_io *tio = clone_to_tio(clone);
11461da177e4SLinus Torvalds 	int r;
11472056a782SJens Axboe 	sector_t sector;
114864f52b0eSMike Snitzer 	struct dm_io *io = tio->io;
1149bd2a49b8SAlasdair G Kergon 	struct dm_target *ti = tio->ti;
11501da177e4SLinus Torvalds 
11511da177e4SLinus Torvalds 	clone->bi_end_io = clone_endio;
11521da177e4SLinus Torvalds 
11531da177e4SLinus Torvalds 	/*
11541da177e4SLinus Torvalds 	 * Map the clone.  If r == 0 we don't need to do
11551da177e4SLinus Torvalds 	 * anything, the target has assumed ownership of
11561da177e4SLinus Torvalds 	 * this io.
11571da177e4SLinus Torvalds 	 */
1158e2118b3cSDamien Le Moal 	dm_io_inc_pending(io);
11594f024f37SKent Overstreet 	sector = clone->bi_iter.bi_sector;
1160d67a5f4bSMikulas Patocka 
1161a666e5c0SMikulas Patocka 	if (unlikely(swap_bios_limit(ti, clone))) {
1162a666e5c0SMikulas Patocka 		struct mapped_device *md = io->md;
1163a666e5c0SMikulas Patocka 		int latch = get_swap_bios();
1164a666e5c0SMikulas Patocka 		if (unlikely(latch != md->swap_bios))
1165a666e5c0SMikulas Patocka 			__set_swap_bios_limit(md, latch);
1166a666e5c0SMikulas Patocka 		down(&md->swap_bios_semaphore);
1167a666e5c0SMikulas Patocka 	}
1168a666e5c0SMikulas Patocka 
1169bb37d772SDamien Le Moal 	/*
1170bb37d772SDamien Le Moal 	 * Check if the IO needs a special mapping due to zone append emulation
1171bb37d772SDamien Le Moal 	 * on zoned target. In this case, dm_zone_map_bio() calls the target
1172bb37d772SDamien Le Moal 	 * map operation.
1173bb37d772SDamien Le Moal 	 */
1174bb37d772SDamien Le Moal 	if (dm_emulate_zone_append(io->md))
1175bb37d772SDamien Le Moal 		r = dm_zone_map_bio(tio);
1176bb37d772SDamien Le Moal 	else
11777de3ee57SMikulas Patocka 		r = ti->type->map(ti, clone);
1178bb37d772SDamien Le Moal 
1179846785e6SChristoph Hellwig 	switch (r) {
1180846785e6SChristoph Hellwig 	case DM_MAPIO_SUBMITTED:
1181846785e6SChristoph Hellwig 		break;
1182846785e6SChristoph Hellwig 	case DM_MAPIO_REMAPPED:
11831da177e4SLinus Torvalds 		/* the bio has been remapped so dispatch it */
11841c02fca6SChristoph Hellwig 		trace_block_bio_remap(clone, bio_dev(io->orig_bio), sector);
11853e08773cSChristoph Hellwig 		submit_bio_noacct(clone);
1186846785e6SChristoph Hellwig 		break;
1187846785e6SChristoph Hellwig 	case DM_MAPIO_KILL:
1188a666e5c0SMikulas Patocka 		if (unlikely(swap_bios_limit(ti, clone))) {
1189a666e5c0SMikulas Patocka 			struct mapped_device *md = io->md;
1190a666e5c0SMikulas Patocka 			up(&md->swap_bios_semaphore);
1191a666e5c0SMikulas Patocka 		}
11921d1068ceSChristoph Hellwig 		free_tio(clone);
1193e2118b3cSDamien Le Moal 		dm_io_dec_pending(io, BLK_STS_IOERR);
11944e4cbee9SChristoph Hellwig 		break;
1195846785e6SChristoph Hellwig 	case DM_MAPIO_REQUEUE:
1196a666e5c0SMikulas Patocka 		if (unlikely(swap_bios_limit(ti, clone))) {
1197a666e5c0SMikulas Patocka 			struct mapped_device *md = io->md;
1198a666e5c0SMikulas Patocka 			up(&md->swap_bios_semaphore);
1199a666e5c0SMikulas Patocka 		}
12001d1068ceSChristoph Hellwig 		free_tio(clone);
1201e2118b3cSDamien Le Moal 		dm_io_dec_pending(io, BLK_STS_DM_REQUEUE);
1202846785e6SChristoph Hellwig 		break;
1203846785e6SChristoph Hellwig 	default:
120445cbcd79SKiyoshi Ueda 		DMWARN("unimplemented target map return value: %d", r);
120545cbcd79SKiyoshi Ueda 		BUG();
12061da177e4SLinus Torvalds 	}
12071da177e4SLinus Torvalds }
12081da177e4SLinus Torvalds 
1209e0d6609aSMikulas Patocka static void bio_setup_sector(struct bio *bio, sector_t sector, unsigned len)
1210bd2a49b8SAlasdair G Kergon {
12114f024f37SKent Overstreet 	bio->bi_iter.bi_sector = sector;
12124f024f37SKent Overstreet 	bio->bi_iter.bi_size = to_bytes(len);
12131da177e4SLinus Torvalds }
12141da177e4SLinus Torvalds 
12151da177e4SLinus Torvalds /*
12161da177e4SLinus Torvalds  * Creates a bio that consists of range of complete bvecs.
12171da177e4SLinus Torvalds  */
1218b1bee792SChristoph Hellwig static int __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti,
1219b1bee792SChristoph Hellwig 				    sector_t sector, unsigned *len)
12201da177e4SLinus Torvalds {
1221b1bee792SChristoph Hellwig 	struct bio *bio = ci->bio, *clone;
12221da177e4SLinus Torvalds 
12231d1068ceSChristoph Hellwig 	clone = alloc_tio(ci, ti, 0, len, GFP_NOIO);
1224fa8db494SMike Snitzer 	bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector));
1225b1bee792SChristoph Hellwig 	clone->bi_iter.bi_size = to_bytes(*len);
1226fa8db494SMike Snitzer 
1227fa8db494SMike Snitzer 	if (bio_integrity(bio))
1228fa8db494SMike Snitzer 		bio_integrity_trim(clone);
1229c80914e8SMike Snitzer 
12301561b396SChristoph Hellwig 	__map_bio(clone);
1231c80914e8SMike Snitzer 	return 0;
12321da177e4SLinus Torvalds }
12331da177e4SLinus Torvalds 
1234318716ddSMike Snitzer static void alloc_multiple_bios(struct bio_list *blist, struct clone_info *ci,
1235dc8e2021SChristoph Hellwig 				struct dm_target *ti, unsigned num_bios,
1236dc8e2021SChristoph Hellwig 				unsigned *len)
1237f9ab94ceSMikulas Patocka {
12381d1068ceSChristoph Hellwig 	struct bio *bio;
1239318716ddSMike Snitzer 	int try;
1240dba14160SMikulas Patocka 
1241318716ddSMike Snitzer 	for (try = 0; try < 2; try++) {
1242318716ddSMike Snitzer 		int bio_nr;
1243318716ddSMike Snitzer 
1244318716ddSMike Snitzer 		if (try)
1245bc02cdbeSMike Snitzer 			mutex_lock(&ci->io->md->table_devices_lock);
1246318716ddSMike Snitzer 		for (bio_nr = 0; bio_nr < num_bios; bio_nr++) {
12471d1068ceSChristoph Hellwig 			bio = alloc_tio(ci, ti, bio_nr, len,
1248dc8e2021SChristoph Hellwig 					try ? GFP_NOIO : GFP_NOWAIT);
12491d1068ceSChristoph Hellwig 			if (!bio)
1250318716ddSMike Snitzer 				break;
1251318716ddSMike Snitzer 
12521d1068ceSChristoph Hellwig 			bio_list_add(blist, bio);
1253318716ddSMike Snitzer 		}
1254318716ddSMike Snitzer 		if (try)
1255bc02cdbeSMike Snitzer 			mutex_unlock(&ci->io->md->table_devices_lock);
1256318716ddSMike Snitzer 		if (bio_nr == num_bios)
1257318716ddSMike Snitzer 			return;
1258318716ddSMike Snitzer 
12596c23f0bdSChristoph Hellwig 		while ((bio = bio_list_pop(blist)))
12601d1068ceSChristoph Hellwig 			free_tio(bio);
1261318716ddSMike Snitzer 	}
1262318716ddSMike Snitzer }
1263318716ddSMike Snitzer 
12648eabf5d0SChristoph Hellwig static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti,
12658eabf5d0SChristoph Hellwig 				  unsigned num_bios, unsigned *len)
12669015df24SAlasdair G Kergon {
12678eabf5d0SChristoph Hellwig 	struct bio_list blist = BIO_EMPTY_LIST;
12688eabf5d0SChristoph Hellwig 	struct bio *clone;
12698eabf5d0SChristoph Hellwig 
1270891fced6SChristoph Hellwig 	switch (num_bios) {
1271891fced6SChristoph Hellwig 	case 0:
1272891fced6SChristoph Hellwig 		break;
1273891fced6SChristoph Hellwig 	case 1:
1274891fced6SChristoph Hellwig 		clone = alloc_tio(ci, ti, 0, len, GFP_NOIO);
1275891fced6SChristoph Hellwig 		if (len)
1276891fced6SChristoph Hellwig 			bio_setup_sector(clone, ci->sector, *len);
1277891fced6SChristoph Hellwig 		__map_bio(clone);
1278891fced6SChristoph Hellwig 		break;
1279891fced6SChristoph Hellwig 	default:
1280dc8e2021SChristoph Hellwig 		alloc_multiple_bios(&blist, ci, ti, num_bios, len);
12818eabf5d0SChristoph Hellwig 		while ((clone = bio_list_pop(&blist))) {
1282bd2a49b8SAlasdair G Kergon 			if (len)
12831dd40c3eSMikulas Patocka 				bio_setup_sector(clone, ci->sector, *len);
12841561b396SChristoph Hellwig 			__map_bio(clone);
1285f9ab94ceSMikulas Patocka 		}
1286891fced6SChristoph Hellwig 		break;
1287891fced6SChristoph Hellwig 	}
128806a426ceSMike Snitzer }
128906a426ceSMike Snitzer 
129014fe594dSAlasdair G Kergon static int __send_empty_flush(struct clone_info *ci)
1291f9ab94ceSMikulas Patocka {
129206a426ceSMike Snitzer 	unsigned target_nr = 0;
1293f9ab94ceSMikulas Patocka 	struct dm_target *ti;
1294828678b8SMike Snitzer 	struct bio flush_bio;
1295828678b8SMike Snitzer 
1296828678b8SMike Snitzer 	/*
1297828678b8SMike Snitzer 	 * Use an on-stack bio for this, it's safe since we don't
1298828678b8SMike Snitzer 	 * need to reference it after submit. It's just used as
1299828678b8SMike Snitzer 	 * the basis for the clone(s).
1300828678b8SMike Snitzer 	 */
130149add496SChristoph Hellwig 	bio_init(&flush_bio, ci->io->md->disk->part0, NULL, 0,
130249add496SChristoph Hellwig 		 REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC);
130347d95102SChristoph Hellwig 
1304828678b8SMike Snitzer 	ci->bio = &flush_bio;
1305828678b8SMike Snitzer 	ci->sector_count = 0;
1306f9ab94ceSMikulas Patocka 
1307b372d360SMike Snitzer 	BUG_ON(bio_has_data(ci->bio));
1308f9ab94ceSMikulas Patocka 	while ((ti = dm_table_get_target(ci->map, target_nr++)))
13091dd40c3eSMikulas Patocka 		__send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL);
1310828678b8SMike Snitzer 
1311828678b8SMike Snitzer 	bio_uninit(ci->bio);
1312f9ab94ceSMikulas Patocka 	return 0;
1313f9ab94ceSMikulas Patocka }
1314f9ab94ceSMikulas Patocka 
13153d7f4562SMike Snitzer static int __send_changing_extent_only(struct clone_info *ci, struct dm_target *ti,
131661697a6aSMike Snitzer 				       unsigned num_bios)
13175ae89a87SMike Snitzer {
131851b86f9aSMichael Lass 	unsigned len;
13195ae89a87SMike Snitzer 
13205ae89a87SMike Snitzer 	/*
132123508a96SMike Snitzer 	 * Even though the device advertised support for this type of
132223508a96SMike Snitzer 	 * request, that does not mean every target supports it, and
1323936688d7SMike Snitzer 	 * reconfiguration might also have changed that since the
13245ae89a87SMike Snitzer 	 * check was performed.
13255ae89a87SMike Snitzer 	 */
132655a62eefSAlasdair G Kergon 	if (!num_bios)
13275ae89a87SMike Snitzer 		return -EOPNOTSUPP;
13285ae89a87SMike Snitzer 
13293720281dSMike Snitzer 	len = min_t(sector_t, ci->sector_count,
13303720281dSMike Snitzer 		    max_io_len_target_boundary(ti, dm_target_offset(ti, ci->sector)));
133151b86f9aSMichael Lass 
13321dd40c3eSMikulas Patocka 	__send_duplicate_bios(ci, ti, num_bios, &len);
13335ae89a87SMike Snitzer 
1334a79245b3SMike Snitzer 	ci->sector += len;
13353d7f4562SMike Snitzer 	ci->sector_count -= len;
13365ae89a87SMike Snitzer 
13375ae89a87SMike Snitzer 	return 0;
13385ae89a87SMike Snitzer }
13395ae89a87SMike Snitzer 
1340568c73a3SMike Snitzer static bool is_abnormal_io(struct bio *bio)
1341568c73a3SMike Snitzer {
1342568c73a3SMike Snitzer 	bool r = false;
1343568c73a3SMike Snitzer 
1344568c73a3SMike Snitzer 	switch (bio_op(bio)) {
1345568c73a3SMike Snitzer 	case REQ_OP_DISCARD:
1346568c73a3SMike Snitzer 	case REQ_OP_SECURE_ERASE:
1347568c73a3SMike Snitzer 	case REQ_OP_WRITE_SAME:
1348568c73a3SMike Snitzer 	case REQ_OP_WRITE_ZEROES:
1349568c73a3SMike Snitzer 		r = true;
1350568c73a3SMike Snitzer 		break;
1351568c73a3SMike Snitzer 	}
1352568c73a3SMike Snitzer 
1353568c73a3SMike Snitzer 	return r;
1354568c73a3SMike Snitzer }
1355568c73a3SMike Snitzer 
13560519c71eSMike Snitzer static bool __process_abnormal_io(struct clone_info *ci, struct dm_target *ti,
13570519c71eSMike Snitzer 				  int *result)
13580519c71eSMike Snitzer {
13590519c71eSMike Snitzer 	struct bio *bio = ci->bio;
13609679b5a7SMike Snitzer 	unsigned num_bios = 0;
13610519c71eSMike Snitzer 
13629679b5a7SMike Snitzer 	switch (bio_op(bio)) {
13639679b5a7SMike Snitzer 	case REQ_OP_DISCARD:
13649679b5a7SMike Snitzer 		num_bios = ti->num_discard_bios;
13659679b5a7SMike Snitzer 		break;
13669679b5a7SMike Snitzer 	case REQ_OP_SECURE_ERASE:
13679679b5a7SMike Snitzer 		num_bios = ti->num_secure_erase_bios;
13689679b5a7SMike Snitzer 		break;
13699679b5a7SMike Snitzer 	case REQ_OP_WRITE_SAME:
13709679b5a7SMike Snitzer 		num_bios = ti->num_write_same_bios;
13719679b5a7SMike Snitzer 		break;
13729679b5a7SMike Snitzer 	case REQ_OP_WRITE_ZEROES:
13739679b5a7SMike Snitzer 		num_bios = ti->num_write_zeroes_bios;
13749679b5a7SMike Snitzer 		break;
13759679b5a7SMike Snitzer 	default:
13760519c71eSMike Snitzer 		return false;
13779679b5a7SMike Snitzer 	}
13780519c71eSMike Snitzer 
13799679b5a7SMike Snitzer 	*result = __send_changing_extent_only(ci, ti, num_bios);
13800519c71eSMike Snitzer 	return true;
13810519c71eSMike Snitzer }
13820519c71eSMike Snitzer 
1383e4c93811SAlasdair G Kergon /*
1384e4c93811SAlasdair G Kergon  * Select the correct strategy for processing a non-flush bio.
1385e4c93811SAlasdair G Kergon  */
1386e4c93811SAlasdair G Kergon static int __split_and_process_non_flush(struct clone_info *ci)
1387e4c93811SAlasdair G Kergon {
1388e4c93811SAlasdair G Kergon 	struct dm_target *ti;
13891c3b13e6SKent Overstreet 	unsigned len;
1390c80914e8SMike Snitzer 	int r;
1391e4c93811SAlasdair G Kergon 
1392e4c93811SAlasdair G Kergon 	ti = dm_table_find_target(ci->map, ci->sector);
1393123d87d5SMikulas Patocka 	if (!ti)
1394e4c93811SAlasdair G Kergon 		return -EIO;
1395e4c93811SAlasdair G Kergon 
1396568c73a3SMike Snitzer 	if (__process_abnormal_io(ci, ti, &r))
13970519c71eSMike Snitzer 		return r;
13983d7f4562SMike Snitzer 
13993720281dSMike Snitzer 	len = min_t(sector_t, max_io_len(ti, ci->sector), ci->sector_count);
1400e4c93811SAlasdair G Kergon 
1401c80914e8SMike Snitzer 	r = __clone_and_map_data_bio(ci, ti, ci->sector, &len);
1402c80914e8SMike Snitzer 	if (r < 0)
1403c80914e8SMike Snitzer 		return r;
1404e4c93811SAlasdair G Kergon 
1405e4c93811SAlasdair G Kergon 	ci->sector += len;
1406e4c93811SAlasdair G Kergon 	ci->sector_count -= len;
1407e4c93811SAlasdair G Kergon 
1408e4c93811SAlasdair G Kergon 	return 0;
1409e4c93811SAlasdair G Kergon }
1410e4c93811SAlasdair G Kergon 
1411978e51baSMike Snitzer static void init_clone_info(struct clone_info *ci, struct mapped_device *md,
1412978e51baSMike Snitzer 			    struct dm_table *map, struct bio *bio)
1413978e51baSMike Snitzer {
1414978e51baSMike Snitzer 	ci->map = map;
1415978e51baSMike Snitzer 	ci->io = alloc_io(md, bio);
1416978e51baSMike Snitzer 	ci->sector = bio->bi_iter.bi_sector;
1417978e51baSMike Snitzer }
1418978e51baSMike Snitzer 
1419e4c93811SAlasdair G Kergon /*
142014fe594dSAlasdair G Kergon  * Entry point to split a bio into clones and submit them to the targets.
14211da177e4SLinus Torvalds  */
14223e08773cSChristoph Hellwig static void __split_and_process_bio(struct mapped_device *md,
142383d5e5b0SMikulas Patocka 					struct dm_table *map, struct bio *bio)
14241da177e4SLinus Torvalds {
14251da177e4SLinus Torvalds 	struct clone_info ci;
1426512875bdSJun'ichi Nomura 	int error = 0;
14271da177e4SLinus Torvalds 
1428978e51baSMike Snitzer 	init_clone_info(&ci, md, map, bio);
1429bd2a49b8SAlasdair G Kergon 
14301eff9d32SJens Axboe 	if (bio->bi_opf & REQ_PREFLUSH) {
143114fe594dSAlasdair G Kergon 		error = __send_empty_flush(&ci);
1432e2118b3cSDamien Le Moal 		/* dm_io_dec_pending submits any data associated with flush */
14332e2d6f7eSAjay Joshi 	} else if (op_is_zone_mgmt(bio_op(bio))) {
1434a4aa5e56SDamien Le Moal 		ci.bio = bio;
1435a4aa5e56SDamien Le Moal 		ci.sector_count = 0;
1436a4aa5e56SDamien Le Moal 		error = __split_and_process_non_flush(&ci);
1437b372d360SMike Snitzer 	} else {
14386a8736d1STejun Heo 		ci.bio = bio;
14391da177e4SLinus Torvalds 		ci.sector_count = bio_sectors(bio);
144014fe594dSAlasdair G Kergon 		error = __split_and_process_non_flush(&ci);
1441985eabdcSJeffle Xu 		if (ci.sector_count && !error) {
144218a25da8SNeilBrown 			/*
1443ed00aabdSChristoph Hellwig 			 * Remainder must be passed to submit_bio_noacct()
144418a25da8SNeilBrown 			 * so that it gets handled *after* bios already submitted
144518a25da8SNeilBrown 			 * have been completely processed.
144618a25da8SNeilBrown 			 * We take a clone of the original to store in
1447745dc570SMike Snitzer 			 * ci.io->orig_bio to be used by end_io_acct() and
144818a25da8SNeilBrown 			 * for dec_pending to use for completion handling.
144918a25da8SNeilBrown 			 */
1450f21c601aSMike Snitzer 			struct bio *b = bio_split(bio, bio_sectors(bio) - ci.sector_count,
1451f21c601aSMike Snitzer 						  GFP_NOIO, &md->queue->bio_split);
1452745dc570SMike Snitzer 			ci.io->orig_bio = b;
1453a1e1cb72SMike Snitzer 
145418a25da8SNeilBrown 			bio_chain(b, bio);
1455eb6f7f7cSChristoph Hellwig 			trace_block_split(b, bio->bi_iter.bi_sector);
14563e08773cSChristoph Hellwig 			submit_bio_noacct(bio);
145718a25da8SNeilBrown 		}
1458d87f4c14STejun Heo 	}
1459b879f915SMike Snitzer 	start_io_acct(ci.io);
14601da177e4SLinus Torvalds 
14611da177e4SLinus Torvalds 	/* drop the extra reference count */
1462e2118b3cSDamien Le Moal 	dm_io_dec_pending(ci.io, errno_to_blk_status(error));
14631da177e4SLinus Torvalds }
14641da177e4SLinus Torvalds 
14653e08773cSChristoph Hellwig static void dm_submit_bio(struct bio *bio)
14661da177e4SLinus Torvalds {
1467309dca30SChristoph Hellwig 	struct mapped_device *md = bio->bi_bdev->bd_disk->private_data;
146883d5e5b0SMikulas Patocka 	int srcu_idx;
146983d5e5b0SMikulas Patocka 	struct dm_table *map;
14701da177e4SLinus Torvalds 
147183d5e5b0SMikulas Patocka 	map = dm_get_live_table(md, &srcu_idx);
1472b2abdb1bSMike Snitzer 	if (unlikely(!map)) {
1473b2abdb1bSMike Snitzer 		DMERR_LIMIT("%s: mapping table unavailable, erroring io",
1474b2abdb1bSMike Snitzer 			    dm_device_name(md));
14756a8736d1STejun Heo 		bio_io_error(bio);
1476b2abdb1bSMike Snitzer 		goto out;
14771da177e4SLinus Torvalds 	}
147892c63902SMikulas Patocka 
1479b2abdb1bSMike Snitzer 	/* If suspended, queue this IO for later */
14801da177e4SLinus Torvalds 	if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) {
14816abc4946SKonstantin Khlebnikov 		if (bio->bi_opf & REQ_NOWAIT)
14826abc4946SKonstantin Khlebnikov 			bio_wouldblock_error(bio);
1483b2abdb1bSMike Snitzer 		else if (bio->bi_opf & REQ_RAHEAD)
14841da177e4SLinus Torvalds 			bio_io_error(bio);
1485b2abdb1bSMike Snitzer 		else
1486b2abdb1bSMike Snitzer 			queue_io(md, bio);
1487b2abdb1bSMike Snitzer 		goto out;
14881da177e4SLinus Torvalds 	}
14891da177e4SLinus Torvalds 
1490b2abdb1bSMike Snitzer 	/*
1491b2abdb1bSMike Snitzer 	 * Use blk_queue_split() for abnormal IO (e.g. discard, writesame, etc)
1492b2abdb1bSMike Snitzer 	 * otherwise associated queue_limits won't be imposed.
1493b2abdb1bSMike Snitzer 	 */
1494b2abdb1bSMike Snitzer 	if (is_abnormal_io(bio))
1495b2abdb1bSMike Snitzer 		blk_queue_split(&bio);
1496978e51baSMike Snitzer 
14973e08773cSChristoph Hellwig 	__split_and_process_bio(md, map, bio);
1498b2abdb1bSMike Snitzer out:
149983d5e5b0SMikulas Patocka 	dm_put_live_table(md, srcu_idx);
1500978e51baSMike Snitzer }
1501978e51baSMike Snitzer 
15021da177e4SLinus Torvalds /*-----------------------------------------------------------------
15031da177e4SLinus Torvalds  * An IDR is used to keep track of allocated minor numbers.
15041da177e4SLinus Torvalds  *---------------------------------------------------------------*/
15052b06cfffSAlasdair G Kergon static void free_minor(int minor)
15061da177e4SLinus Torvalds {
1507f32c10b0SJeff Mahoney 	spin_lock(&_minor_lock);
15081da177e4SLinus Torvalds 	idr_remove(&_minor_idr, minor);
1509f32c10b0SJeff Mahoney 	spin_unlock(&_minor_lock);
15101da177e4SLinus Torvalds }
15111da177e4SLinus Torvalds 
15121da177e4SLinus Torvalds /*
15131da177e4SLinus Torvalds  * See if the device with a specific minor # is free.
15141da177e4SLinus Torvalds  */
1515cf13ab8eSFrederik Deweerdt static int specific_minor(int minor)
15161da177e4SLinus Torvalds {
1517c9d76be6STejun Heo 	int r;
15181da177e4SLinus Torvalds 
15191da177e4SLinus Torvalds 	if (minor >= (1 << MINORBITS))
15201da177e4SLinus Torvalds 		return -EINVAL;
15211da177e4SLinus Torvalds 
1522c9d76be6STejun Heo 	idr_preload(GFP_KERNEL);
1523f32c10b0SJeff Mahoney 	spin_lock(&_minor_lock);
15241da177e4SLinus Torvalds 
1525c9d76be6STejun Heo 	r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT);
15261da177e4SLinus Torvalds 
1527f32c10b0SJeff Mahoney 	spin_unlock(&_minor_lock);
1528c9d76be6STejun Heo 	idr_preload_end();
1529c9d76be6STejun Heo 	if (r < 0)
1530c9d76be6STejun Heo 		return r == -ENOSPC ? -EBUSY : r;
1531c9d76be6STejun Heo 	return 0;
15321da177e4SLinus Torvalds }
15331da177e4SLinus Torvalds 
1534cf13ab8eSFrederik Deweerdt static int next_free_minor(int *minor)
15351da177e4SLinus Torvalds {
1536c9d76be6STejun Heo 	int r;
15371da177e4SLinus Torvalds 
1538c9d76be6STejun Heo 	idr_preload(GFP_KERNEL);
1539f32c10b0SJeff Mahoney 	spin_lock(&_minor_lock);
15401da177e4SLinus Torvalds 
1541c9d76be6STejun Heo 	r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT);
15421da177e4SLinus Torvalds 
1543f32c10b0SJeff Mahoney 	spin_unlock(&_minor_lock);
1544c9d76be6STejun Heo 	idr_preload_end();
1545c9d76be6STejun Heo 	if (r < 0)
15461da177e4SLinus Torvalds 		return r;
1547c9d76be6STejun Heo 	*minor = r;
1548c9d76be6STejun Heo 	return 0;
15491da177e4SLinus Torvalds }
15501da177e4SLinus Torvalds 
155183d5cde4SAlexey Dobriyan static const struct block_device_operations dm_blk_dops;
1552681cc5e8SMike Snitzer static const struct block_device_operations dm_rq_blk_dops;
1553f26c5719SDan Williams static const struct dax_operations dm_dax_ops;
15541da177e4SLinus Torvalds 
155553d5914fSMikulas Patocka static void dm_wq_work(struct work_struct *work);
155653d5914fSMikulas Patocka 
1557aa6ce87aSSatya Tangirala #ifdef CONFIG_BLK_INLINE_ENCRYPTION
1558cb77cb5aSEric Biggers static void dm_queue_destroy_crypto_profile(struct request_queue *q)
1559aa6ce87aSSatya Tangirala {
1560cb77cb5aSEric Biggers 	dm_destroy_crypto_profile(q->crypto_profile);
1561aa6ce87aSSatya Tangirala }
1562aa6ce87aSSatya Tangirala 
1563aa6ce87aSSatya Tangirala #else /* CONFIG_BLK_INLINE_ENCRYPTION */
1564aa6ce87aSSatya Tangirala 
1565cb77cb5aSEric Biggers static inline void dm_queue_destroy_crypto_profile(struct request_queue *q)
1566aa6ce87aSSatya Tangirala {
1567aa6ce87aSSatya Tangirala }
1568aa6ce87aSSatya Tangirala #endif /* !CONFIG_BLK_INLINE_ENCRYPTION */
1569aa6ce87aSSatya Tangirala 
15700f20972fSMike Snitzer static void cleanup_mapped_device(struct mapped_device *md)
15710f20972fSMike Snitzer {
15720f20972fSMike Snitzer 	if (md->wq)
15730f20972fSMike Snitzer 		destroy_workqueue(md->wq);
15746f1c819cSKent Overstreet 	bioset_exit(&md->bs);
15756f1c819cSKent Overstreet 	bioset_exit(&md->io_bs);
15760f20972fSMike Snitzer 
1577f26c5719SDan Williams 	if (md->dax_dev) {
1578fb08a190SChristoph Hellwig 		dax_remove_host(md->disk);
1579f26c5719SDan Williams 		kill_dax(md->dax_dev);
1580f26c5719SDan Williams 		put_dax(md->dax_dev);
1581f26c5719SDan Williams 		md->dax_dev = NULL;
1582f26c5719SDan Williams 	}
1583f26c5719SDan Williams 
15840f20972fSMike Snitzer 	if (md->disk) {
15850f20972fSMike Snitzer 		spin_lock(&_minor_lock);
15860f20972fSMike Snitzer 		md->disk->private_data = NULL;
15870f20972fSMike Snitzer 		spin_unlock(&_minor_lock);
158889f871afSChristoph Hellwig 		if (dm_get_md_type(md) != DM_TYPE_NONE) {
158989f871afSChristoph Hellwig 			dm_sysfs_exit(md);
15900f20972fSMike Snitzer 			del_gendisk(md->disk);
159189f871afSChristoph Hellwig 		}
1592cb77cb5aSEric Biggers 		dm_queue_destroy_crypto_profile(md->queue);
159374fe6ba9SChristoph Hellwig 		blk_cleanup_disk(md->disk);
159474a2b6ecSChristoph Hellwig 	}
15950f20972fSMike Snitzer 
15969f6dc633SMike Snitzer 	if (md->pending_io) {
15979f6dc633SMike Snitzer 		free_percpu(md->pending_io);
15989f6dc633SMike Snitzer 		md->pending_io = NULL;
15999f6dc633SMike Snitzer 	}
16009f6dc633SMike Snitzer 
1601d09960b0STahsin Erdogan 	cleanup_srcu_struct(&md->io_barrier);
1602d09960b0STahsin Erdogan 
1603d5ffebddSMike Snitzer 	mutex_destroy(&md->suspend_lock);
1604d5ffebddSMike Snitzer 	mutex_destroy(&md->type_lock);
1605d5ffebddSMike Snitzer 	mutex_destroy(&md->table_devices_lock);
1606a666e5c0SMikulas Patocka 	mutex_destroy(&md->swap_bios_lock);
1607d5ffebddSMike Snitzer 
16084cc96131SMike Snitzer 	dm_mq_cleanup_mapped_device(md);
1609bb37d772SDamien Le Moal 	dm_cleanup_zoned_dev(md);
16100f20972fSMike Snitzer }
16110f20972fSMike Snitzer 
16121da177e4SLinus Torvalds /*
16131da177e4SLinus Torvalds  * Allocate and initialise a blank device with a given minor.
16141da177e4SLinus Torvalds  */
16152b06cfffSAlasdair G Kergon static struct mapped_device *alloc_dev(int minor)
16161da177e4SLinus Torvalds {
1617115485e8SMike Snitzer 	int r, numa_node_id = dm_get_numa_node();
1618115485e8SMike Snitzer 	struct mapped_device *md;
1619ba61fdd1SJeff Mahoney 	void *old_md;
16201da177e4SLinus Torvalds 
1621856eb091SMikulas Patocka 	md = kvzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id);
16221da177e4SLinus Torvalds 	if (!md) {
16231da177e4SLinus Torvalds 		DMWARN("unable to allocate device, out of memory.");
16241da177e4SLinus Torvalds 		return NULL;
16251da177e4SLinus Torvalds 	}
16261da177e4SLinus Torvalds 
162710da4f79SJeff Mahoney 	if (!try_module_get(THIS_MODULE))
16286ed7ade8SMilan Broz 		goto bad_module_get;
162910da4f79SJeff Mahoney 
16301da177e4SLinus Torvalds 	/* get a minor number for the dev */
16312b06cfffSAlasdair G Kergon 	if (minor == DM_ANY_MINOR)
1632cf13ab8eSFrederik Deweerdt 		r = next_free_minor(&minor);
16332b06cfffSAlasdair G Kergon 	else
1634cf13ab8eSFrederik Deweerdt 		r = specific_minor(minor);
16351da177e4SLinus Torvalds 	if (r < 0)
16366ed7ade8SMilan Broz 		goto bad_minor;
16371da177e4SLinus Torvalds 
163883d5e5b0SMikulas Patocka 	r = init_srcu_struct(&md->io_barrier);
163983d5e5b0SMikulas Patocka 	if (r < 0)
164083d5e5b0SMikulas Patocka 		goto bad_io_barrier;
164183d5e5b0SMikulas Patocka 
1642115485e8SMike Snitzer 	md->numa_node_id = numa_node_id;
1643591ddcfcSMike Snitzer 	md->init_tio_pdu = false;
1644a5664dadSMike Snitzer 	md->type = DM_TYPE_NONE;
1645e61290a4SDaniel Walker 	mutex_init(&md->suspend_lock);
1646a5664dadSMike Snitzer 	mutex_init(&md->type_lock);
164786f1152bSBenjamin Marzinski 	mutex_init(&md->table_devices_lock);
1648022c2611SMikulas Patocka 	spin_lock_init(&md->deferred_lock);
16491da177e4SLinus Torvalds 	atomic_set(&md->holders, 1);
16505c6bd75dSAlasdair G Kergon 	atomic_set(&md->open_count, 0);
16511da177e4SLinus Torvalds 	atomic_set(&md->event_nr, 0);
16527a8c3d3bSMike Anderson 	atomic_set(&md->uevent_seq, 0);
16537a8c3d3bSMike Anderson 	INIT_LIST_HEAD(&md->uevent_list);
165486f1152bSBenjamin Marzinski 	INIT_LIST_HEAD(&md->table_devices);
16557a8c3d3bSMike Anderson 	spin_lock_init(&md->uevent_lock);
16561da177e4SLinus Torvalds 
165747ace7e0SMike Snitzer 	/*
1658c62b37d9SChristoph Hellwig 	 * default to bio-based until DM table is loaded and md->type
1659c62b37d9SChristoph Hellwig 	 * established. If request-based table is loaded: blk-mq will
1660c62b37d9SChristoph Hellwig 	 * override accordingly.
166147ace7e0SMike Snitzer 	 */
166274fe6ba9SChristoph Hellwig 	md->disk = blk_alloc_disk(md->numa_node_id);
16631da177e4SLinus Torvalds 	if (!md->disk)
16640f20972fSMike Snitzer 		goto bad;
166574fe6ba9SChristoph Hellwig 	md->queue = md->disk->queue;
16661da177e4SLinus Torvalds 
1667f0b04115SJeff Mahoney 	init_waitqueue_head(&md->wait);
166853d5914fSMikulas Patocka 	INIT_WORK(&md->work, dm_wq_work);
1669f0b04115SJeff Mahoney 	init_waitqueue_head(&md->eventq);
16702995fa78SMikulas Patocka 	init_completion(&md->kobj_holder.completion);
1671f0b04115SJeff Mahoney 
1672a666e5c0SMikulas Patocka 	md->swap_bios = get_swap_bios();
1673a666e5c0SMikulas Patocka 	sema_init(&md->swap_bios_semaphore, md->swap_bios);
1674a666e5c0SMikulas Patocka 	mutex_init(&md->swap_bios_lock);
1675a666e5c0SMikulas Patocka 
16761da177e4SLinus Torvalds 	md->disk->major = _major;
16771da177e4SLinus Torvalds 	md->disk->first_minor = minor;
167874fe6ba9SChristoph Hellwig 	md->disk->minors = 1;
16791ebe2e5fSChristoph Hellwig 	md->disk->flags |= GENHD_FL_NO_PART;
16801da177e4SLinus Torvalds 	md->disk->fops = &dm_blk_dops;
16811da177e4SLinus Torvalds 	md->disk->queue = md->queue;
16821da177e4SLinus Torvalds 	md->disk->private_data = md;
16831da177e4SLinus Torvalds 	sprintf(md->disk->disk_name, "dm-%d", minor);
1684f26c5719SDan Williams 
16855d2a228bSChristoph Hellwig 	if (IS_ENABLED(CONFIG_FS_DAX)) {
168630c6828aSChristoph Hellwig 		md->dax_dev = alloc_dax(md, &dm_dax_ops);
1687d7519392SChristoph Hellwig 		if (IS_ERR(md->dax_dev)) {
1688d7519392SChristoph Hellwig 			md->dax_dev = NULL;
1689f26c5719SDan Williams 			goto bad;
1690976431b0SDan Williams 		}
16917ac5360cSChristoph Hellwig 		set_dax_nocache(md->dax_dev);
16927ac5360cSChristoph Hellwig 		set_dax_nomc(md->dax_dev);
1693fb08a190SChristoph Hellwig 		if (dax_add_host(md->dax_dev, md->disk))
1694f26c5719SDan Williams 			goto bad;
1695f26c5719SDan Williams 	}
16961da177e4SLinus Torvalds 
16977e51f257SMike Anderson 	format_dev_t(md->name, MKDEV(_major, minor));
16981da177e4SLinus Torvalds 
1699c7c879eeSMichał Mirosław 	md->wq = alloc_workqueue("kdmflush/%s", WQ_MEM_RECLAIM, 0, md->name);
1700304f3f6aSMilan Broz 	if (!md->wq)
17010f20972fSMike Snitzer 		goto bad;
1702304f3f6aSMilan Broz 
17039f6dc633SMike Snitzer 	md->pending_io = alloc_percpu(unsigned long);
17049f6dc633SMike Snitzer 	if (!md->pending_io)
17059f6dc633SMike Snitzer 		goto bad;
17069f6dc633SMike Snitzer 
1707fd2ed4d2SMikulas Patocka 	dm_stats_init(&md->stats);
1708fd2ed4d2SMikulas Patocka 
1709ba61fdd1SJeff Mahoney 	/* Populate the mapping, nobody knows we exist yet */
1710f32c10b0SJeff Mahoney 	spin_lock(&_minor_lock);
1711ba61fdd1SJeff Mahoney 	old_md = idr_replace(&_minor_idr, md, minor);
1712f32c10b0SJeff Mahoney 	spin_unlock(&_minor_lock);
1713ba61fdd1SJeff Mahoney 
1714ba61fdd1SJeff Mahoney 	BUG_ON(old_md != MINOR_ALLOCED);
1715ba61fdd1SJeff Mahoney 
17161da177e4SLinus Torvalds 	return md;
17171da177e4SLinus Torvalds 
17180f20972fSMike Snitzer bad:
17190f20972fSMike Snitzer 	cleanup_mapped_device(md);
172083d5e5b0SMikulas Patocka bad_io_barrier:
17211da177e4SLinus Torvalds 	free_minor(minor);
17226ed7ade8SMilan Broz bad_minor:
172310da4f79SJeff Mahoney 	module_put(THIS_MODULE);
17246ed7ade8SMilan Broz bad_module_get:
1725856eb091SMikulas Patocka 	kvfree(md);
17261da177e4SLinus Torvalds 	return NULL;
17271da177e4SLinus Torvalds }
17281da177e4SLinus Torvalds 
1729ae9da83fSJun'ichi Nomura static void unlock_fs(struct mapped_device *md);
1730ae9da83fSJun'ichi Nomura 
17311da177e4SLinus Torvalds static void free_dev(struct mapped_device *md)
17321da177e4SLinus Torvalds {
1733f331c029STejun Heo 	int minor = MINOR(disk_devt(md->disk));
173463d94e48SJun'ichi Nomura 
1735ae9da83fSJun'ichi Nomura 	unlock_fs(md);
17362eb6e1e3SKeith Busch 
17370f20972fSMike Snitzer 	cleanup_mapped_device(md);
17380f20972fSMike Snitzer 
17390f20972fSMike Snitzer 	free_table_devices(&md->table_devices);
17400f20972fSMike Snitzer 	dm_stats_cleanup(&md->stats);
174163a4f065SMike Snitzer 	free_minor(minor);
174263a4f065SMike Snitzer 
174310da4f79SJeff Mahoney 	module_put(THIS_MODULE);
1744856eb091SMikulas Patocka 	kvfree(md);
17451da177e4SLinus Torvalds }
17461da177e4SLinus Torvalds 
17472a2a4c51SJens Axboe static int __bind_mempools(struct mapped_device *md, struct dm_table *t)
1748e6ee8c0bSKiyoshi Ueda {
1749c0820cf5SMikulas Patocka 	struct dm_md_mempools *p = dm_table_get_md_mempools(t);
17502a2a4c51SJens Axboe 	int ret = 0;
1751e6ee8c0bSKiyoshi Ueda 
1752545ed20eSToshi Kani 	if (dm_table_bio_based(t)) {
1753c0820cf5SMikulas Patocka 		/*
175464f52b0eSMike Snitzer 		 * The md may already have mempools that need changing.
175564f52b0eSMike Snitzer 		 * If so, reload bioset because front_pad may have changed
175616245bdcSJun'ichi Nomura 		 * because a different table was loaded.
1757c0820cf5SMikulas Patocka 		 */
17586f1c819cSKent Overstreet 		bioset_exit(&md->bs);
17596f1c819cSKent Overstreet 		bioset_exit(&md->io_bs);
17600776aa0eSMike Snitzer 
17616f1c819cSKent Overstreet 	} else if (bioset_initialized(&md->bs)) {
1762cbc4e3c1SMike Snitzer 		/*
17634e6e36c3SMike Snitzer 		 * There's no need to reload with request-based dm
17644e6e36c3SMike Snitzer 		 * because the size of front_pad doesn't change.
17654e6e36c3SMike Snitzer 		 * Note for future: If you are to reload bioset,
17664e6e36c3SMike Snitzer 		 * prep-ed requests in the queue may refer
17674e6e36c3SMike Snitzer 		 * to bio from the old bioset, so you must walk
17684e6e36c3SMike Snitzer 		 * through the queue to unprep.
1769cbc4e3c1SMike Snitzer 		 */
1770cbc4e3c1SMike Snitzer 		goto out;
1771cbc4e3c1SMike Snitzer 	}
1772cbc4e3c1SMike Snitzer 
17736f1c819cSKent Overstreet 	BUG_ON(!p ||
17746f1c819cSKent Overstreet 	       bioset_initialized(&md->bs) ||
17756f1c819cSKent Overstreet 	       bioset_initialized(&md->io_bs));
1776e6ee8c0bSKiyoshi Ueda 
17772a2a4c51SJens Axboe 	ret = bioset_init_from_src(&md->bs, &p->bs);
17782a2a4c51SJens Axboe 	if (ret)
17792a2a4c51SJens Axboe 		goto out;
17802a2a4c51SJens Axboe 	ret = bioset_init_from_src(&md->io_bs, &p->io_bs);
17812a2a4c51SJens Axboe 	if (ret)
17822a2a4c51SJens Axboe 		bioset_exit(&md->bs);
1783e6ee8c0bSKiyoshi Ueda out:
178402233342SMike Snitzer 	/* mempool bind completed, no longer need any mempools in the table */
1785e6ee8c0bSKiyoshi Ueda 	dm_table_free_md_mempools(t);
17862a2a4c51SJens Axboe 	return ret;
1787e6ee8c0bSKiyoshi Ueda }
1788e6ee8c0bSKiyoshi Ueda 
17891da177e4SLinus Torvalds /*
17901da177e4SLinus Torvalds  * Bind a table to the device.
17911da177e4SLinus Torvalds  */
17921da177e4SLinus Torvalds static void event_callback(void *context)
17931da177e4SLinus Torvalds {
17947a8c3d3bSMike Anderson 	unsigned long flags;
17957a8c3d3bSMike Anderson 	LIST_HEAD(uevents);
17961da177e4SLinus Torvalds 	struct mapped_device *md = (struct mapped_device *) context;
17971da177e4SLinus Torvalds 
17987a8c3d3bSMike Anderson 	spin_lock_irqsave(&md->uevent_lock, flags);
17997a8c3d3bSMike Anderson 	list_splice_init(&md->uevent_list, &uevents);
18007a8c3d3bSMike Anderson 	spin_unlock_irqrestore(&md->uevent_lock, flags);
18017a8c3d3bSMike Anderson 
1802ed9e1982STejun Heo 	dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
18037a8c3d3bSMike Anderson 
18041da177e4SLinus Torvalds 	atomic_inc(&md->event_nr);
18051da177e4SLinus Torvalds 	wake_up(&md->eventq);
180662e08243SMikulas Patocka 	dm_issue_global_event();
18071da177e4SLinus Torvalds }
18081da177e4SLinus Torvalds 
1809c217649bSMike Snitzer /*
1810042d2a9bSAlasdair G Kergon  * Returns old map, which caller must destroy.
1811042d2a9bSAlasdair G Kergon  */
1812042d2a9bSAlasdair G Kergon static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
1813754c5fc7SMike Snitzer 			       struct queue_limits *limits)
18141da177e4SLinus Torvalds {
1815042d2a9bSAlasdair G Kergon 	struct dm_table *old_map;
1816165125e1SJens Axboe 	struct request_queue *q = md->queue;
1817978e51baSMike Snitzer 	bool request_based = dm_table_request_based(t);
18181da177e4SLinus Torvalds 	sector_t size;
18192a2a4c51SJens Axboe 	int ret;
18201da177e4SLinus Torvalds 
18215a8f1f80SBart Van Assche 	lockdep_assert_held(&md->suspend_lock);
18225a8f1f80SBart Van Assche 
18231da177e4SLinus Torvalds 	size = dm_table_get_size(t);
18243ac51e74SDarrick J. Wong 
18253ac51e74SDarrick J. Wong 	/*
18263ac51e74SDarrick J. Wong 	 * Wipe any geometry if the size of the table changed.
18273ac51e74SDarrick J. Wong 	 */
1828fd2ed4d2SMikulas Patocka 	if (size != dm_get_size(md))
18293ac51e74SDarrick J. Wong 		memset(&md->geometry, 0, sizeof(md->geometry));
18303ac51e74SDarrick J. Wong 
18315424a0b8SMikulas Patocka 	if (!get_capacity(md->disk))
18325424a0b8SMikulas Patocka 		set_capacity(md->disk, size);
18335424a0b8SMikulas Patocka 	else
1834f64d9b2eSChristoph Hellwig 		set_capacity_and_notify(md->disk, size);
18351da177e4SLinus Torvalds 
1836cf222b37SAlasdair G Kergon 	dm_table_event_callback(t, event_callback, md);
18372ca3310eSAlasdair G Kergon 
18389c37de29SMike Snitzer 	if (request_based) {
183916f12266SMike Snitzer 		/*
18409c37de29SMike Snitzer 		 * Leverage the fact that request-based DM targets are
18419c37de29SMike Snitzer 		 * immutable singletons - used to optimize dm_mq_queue_rq.
184216f12266SMike Snitzer 		 */
184316f12266SMike Snitzer 		md->immutable_target = dm_table_get_immutable_target(t);
184416f12266SMike Snitzer 	}
1845e6ee8c0bSKiyoshi Ueda 
18462a2a4c51SJens Axboe 	ret = __bind_mempools(md, t);
18472a2a4c51SJens Axboe 	if (ret) {
18482a2a4c51SJens Axboe 		old_map = ERR_PTR(ret);
18492a2a4c51SJens Axboe 		goto out;
18502a2a4c51SJens Axboe 	}
1851e6ee8c0bSKiyoshi Ueda 
1852bb37d772SDamien Le Moal 	ret = dm_table_set_restrictions(t, q, limits);
1853bb37d772SDamien Le Moal 	if (ret) {
1854bb37d772SDamien Le Moal 		old_map = ERR_PTR(ret);
1855bb37d772SDamien Le Moal 		goto out;
1856bb37d772SDamien Le Moal 	}
1857bb37d772SDamien Le Moal 
1858a12f5d48SEric Dumazet 	old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
18591d3aa6f6SMike Snitzer 	rcu_assign_pointer(md->map, (void *)t);
186036a0456fSAlasdair G Kergon 	md->immutable_target_type = dm_table_get_immutable_target_type(t);
186136a0456fSAlasdair G Kergon 
186241abc4e1SHannes Reinecke 	if (old_map)
186383d5e5b0SMikulas Patocka 		dm_sync_table(md);
18642ca3310eSAlasdair G Kergon 
18652a2a4c51SJens Axboe out:
1866042d2a9bSAlasdair G Kergon 	return old_map;
18671da177e4SLinus Torvalds }
18681da177e4SLinus Torvalds 
1869a7940155SAlasdair G Kergon /*
1870a7940155SAlasdair G Kergon  * Returns unbound table for the caller to free.
1871a7940155SAlasdair G Kergon  */
1872a7940155SAlasdair G Kergon static struct dm_table *__unbind(struct mapped_device *md)
18731da177e4SLinus Torvalds {
1874a12f5d48SEric Dumazet 	struct dm_table *map = rcu_dereference_protected(md->map, 1);
18751da177e4SLinus Torvalds 
18761da177e4SLinus Torvalds 	if (!map)
1877a7940155SAlasdair G Kergon 		return NULL;
18781da177e4SLinus Torvalds 
18791da177e4SLinus Torvalds 	dm_table_event_callback(map, NULL, NULL);
18809cdb8520SMonam Agarwal 	RCU_INIT_POINTER(md->map, NULL);
188183d5e5b0SMikulas Patocka 	dm_sync_table(md);
1882a7940155SAlasdair G Kergon 
1883a7940155SAlasdair G Kergon 	return map;
18841da177e4SLinus Torvalds }
18851da177e4SLinus Torvalds 
18861da177e4SLinus Torvalds /*
18871da177e4SLinus Torvalds  * Constructor for a new device.
18881da177e4SLinus Torvalds  */
18892b06cfffSAlasdair G Kergon int dm_create(int minor, struct mapped_device **result)
18901da177e4SLinus Torvalds {
18911da177e4SLinus Torvalds 	struct mapped_device *md;
18921da177e4SLinus Torvalds 
18932b06cfffSAlasdair G Kergon 	md = alloc_dev(minor);
18941da177e4SLinus Torvalds 	if (!md)
18951da177e4SLinus Torvalds 		return -ENXIO;
18961da177e4SLinus Torvalds 
189791ccbbacSTushar Sugandhi 	dm_ima_reset_data(md);
189891ccbbacSTushar Sugandhi 
18991da177e4SLinus Torvalds 	*result = md;
19001da177e4SLinus Torvalds 	return 0;
19011da177e4SLinus Torvalds }
19021da177e4SLinus Torvalds 
1903a5664dadSMike Snitzer /*
1904a5664dadSMike Snitzer  * Functions to manage md->type.
1905a5664dadSMike Snitzer  * All are required to hold md->type_lock.
1906a5664dadSMike Snitzer  */
1907a5664dadSMike Snitzer void dm_lock_md_type(struct mapped_device *md)
1908a5664dadSMike Snitzer {
1909a5664dadSMike Snitzer 	mutex_lock(&md->type_lock);
1910a5664dadSMike Snitzer }
1911a5664dadSMike Snitzer 
1912a5664dadSMike Snitzer void dm_unlock_md_type(struct mapped_device *md)
1913a5664dadSMike Snitzer {
1914a5664dadSMike Snitzer 	mutex_unlock(&md->type_lock);
1915a5664dadSMike Snitzer }
1916a5664dadSMike Snitzer 
19177e0d574fSBart Van Assche void dm_set_md_type(struct mapped_device *md, enum dm_queue_mode type)
1918a5664dadSMike Snitzer {
191900c4fc3bSMike Snitzer 	BUG_ON(!mutex_is_locked(&md->type_lock));
1920a5664dadSMike Snitzer 	md->type = type;
1921a5664dadSMike Snitzer }
1922a5664dadSMike Snitzer 
19237e0d574fSBart Van Assche enum dm_queue_mode dm_get_md_type(struct mapped_device *md)
1924a5664dadSMike Snitzer {
1925a5664dadSMike Snitzer 	return md->type;
1926a5664dadSMike Snitzer }
1927a5664dadSMike Snitzer 
192836a0456fSAlasdair G Kergon struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
192936a0456fSAlasdair G Kergon {
193036a0456fSAlasdair G Kergon 	return md->immutable_target_type;
193136a0456fSAlasdair G Kergon }
193236a0456fSAlasdair G Kergon 
19334a0b4ddfSMike Snitzer /*
1934f84cb8a4SMike Snitzer  * The queue_limits are only valid as long as you have a reference
1935f84cb8a4SMike Snitzer  * count on 'md'.
1936f84cb8a4SMike Snitzer  */
1937f84cb8a4SMike Snitzer struct queue_limits *dm_get_queue_limits(struct mapped_device *md)
1938f84cb8a4SMike Snitzer {
1939f84cb8a4SMike Snitzer 	BUG_ON(!atomic_read(&md->holders));
1940f84cb8a4SMike Snitzer 	return &md->queue->limits;
1941f84cb8a4SMike Snitzer }
1942f84cb8a4SMike Snitzer EXPORT_SYMBOL_GPL(dm_get_queue_limits);
1943f84cb8a4SMike Snitzer 
19444a0b4ddfSMike Snitzer /*
19454a0b4ddfSMike Snitzer  * Setup the DM device's queue based on md's type
19464a0b4ddfSMike Snitzer  */
1947591ddcfcSMike Snitzer int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
19484a0b4ddfSMike Snitzer {
1949ba305859SChristoph Hellwig 	enum dm_queue_mode type = dm_table_get_type(t);
1950c100ec49SMike Snitzer 	struct queue_limits limits;
1951ba305859SChristoph Hellwig 	int r;
1952bfebd1cdSMike Snitzer 
1953545ed20eSToshi Kani 	switch (type) {
1954bfebd1cdSMike Snitzer 	case DM_TYPE_REQUEST_BASED:
1955681cc5e8SMike Snitzer 		md->disk->fops = &dm_rq_blk_dops;
1956e83068a5SMike Snitzer 		r = dm_mq_init_request_queue(md, t);
1957bfebd1cdSMike Snitzer 		if (r) {
1958681cc5e8SMike Snitzer 			DMERR("Cannot initialize queue for request-based dm mapped device");
1959bfebd1cdSMike Snitzer 			return r;
1960bfebd1cdSMike Snitzer 		}
1961bfebd1cdSMike Snitzer 		break;
1962bfebd1cdSMike Snitzer 	case DM_TYPE_BIO_BASED:
1963545ed20eSToshi Kani 	case DM_TYPE_DAX_BIO_BASED:
1964bfebd1cdSMike Snitzer 		break;
19657e0d574fSBart Van Assche 	case DM_TYPE_NONE:
19667e0d574fSBart Van Assche 		WARN_ON_ONCE(true);
19677e0d574fSBart Van Assche 		break;
1968ff36ab34SMike Snitzer 	}
19694a0b4ddfSMike Snitzer 
1970c100ec49SMike Snitzer 	r = dm_calculate_queue_limits(t, &limits);
1971c100ec49SMike Snitzer 	if (r) {
1972c100ec49SMike Snitzer 		DMERR("Cannot calculate initial queue limits");
1973c100ec49SMike Snitzer 		return r;
1974c100ec49SMike Snitzer 	}
1975bb37d772SDamien Le Moal 	r = dm_table_set_restrictions(t, md->queue, &limits);
1976bb37d772SDamien Le Moal 	if (r)
1977bb37d772SDamien Le Moal 		return r;
197889f871afSChristoph Hellwig 
1979e7089f65SLuis Chamberlain 	r = add_disk(md->disk);
1980e7089f65SLuis Chamberlain 	if (r)
1981e7089f65SLuis Chamberlain 		return r;
198289f871afSChristoph Hellwig 
198389f871afSChristoph Hellwig 	r = dm_sysfs_init(md);
198489f871afSChristoph Hellwig 	if (r) {
198589f871afSChristoph Hellwig 		del_gendisk(md->disk);
198689f871afSChristoph Hellwig 		return r;
198789f871afSChristoph Hellwig 	}
1988ba305859SChristoph Hellwig 	md->type = type;
19894a0b4ddfSMike Snitzer 	return 0;
19904a0b4ddfSMike Snitzer }
19914a0b4ddfSMike Snitzer 
19922bec1f4aSMikulas Patocka struct mapped_device *dm_get_md(dev_t dev)
19931da177e4SLinus Torvalds {
19941da177e4SLinus Torvalds 	struct mapped_device *md;
19951da177e4SLinus Torvalds 	unsigned minor = MINOR(dev);
19961da177e4SLinus Torvalds 
19971da177e4SLinus Torvalds 	if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
19981da177e4SLinus Torvalds 		return NULL;
19991da177e4SLinus Torvalds 
2000f32c10b0SJeff Mahoney 	spin_lock(&_minor_lock);
20011da177e4SLinus Torvalds 
20021da177e4SLinus Torvalds 	md = idr_find(&_minor_idr, minor);
200349de5769SMike Snitzer 	if (!md || md == MINOR_ALLOCED || (MINOR(disk_devt(dm_disk(md))) != minor) ||
200449de5769SMike Snitzer 	    test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
2005637842cfSDavid Teigland 		md = NULL;
2006fba9f90eSJeff Mahoney 		goto out;
2007fba9f90eSJeff Mahoney 	}
20082bec1f4aSMikulas Patocka 	dm_get(md);
2009fba9f90eSJeff Mahoney out:
2010f32c10b0SJeff Mahoney 	spin_unlock(&_minor_lock);
20111da177e4SLinus Torvalds 
2012637842cfSDavid Teigland 	return md;
2013637842cfSDavid Teigland }
20143cf2e4baSAlasdair G Kergon EXPORT_SYMBOL_GPL(dm_get_md);
2015d229a958SDavid Teigland 
20169ade92a9SAlasdair G Kergon void *dm_get_mdptr(struct mapped_device *md)
2017637842cfSDavid Teigland {
20189ade92a9SAlasdair G Kergon 	return md->interface_ptr;
20191da177e4SLinus Torvalds }
20201da177e4SLinus Torvalds 
20211da177e4SLinus Torvalds void dm_set_mdptr(struct mapped_device *md, void *ptr)
20221da177e4SLinus Torvalds {
20231da177e4SLinus Torvalds 	md->interface_ptr = ptr;
20241da177e4SLinus Torvalds }
20251da177e4SLinus Torvalds 
20261da177e4SLinus Torvalds void dm_get(struct mapped_device *md)
20271da177e4SLinus Torvalds {
20281da177e4SLinus Torvalds 	atomic_inc(&md->holders);
20293f77316dSKiyoshi Ueda 	BUG_ON(test_bit(DMF_FREEING, &md->flags));
20301da177e4SLinus Torvalds }
20311da177e4SLinus Torvalds 
203209ee96b2SMikulas Patocka int dm_hold(struct mapped_device *md)
203309ee96b2SMikulas Patocka {
203409ee96b2SMikulas Patocka 	spin_lock(&_minor_lock);
203509ee96b2SMikulas Patocka 	if (test_bit(DMF_FREEING, &md->flags)) {
203609ee96b2SMikulas Patocka 		spin_unlock(&_minor_lock);
203709ee96b2SMikulas Patocka 		return -EBUSY;
203809ee96b2SMikulas Patocka 	}
203909ee96b2SMikulas Patocka 	dm_get(md);
204009ee96b2SMikulas Patocka 	spin_unlock(&_minor_lock);
204109ee96b2SMikulas Patocka 	return 0;
204209ee96b2SMikulas Patocka }
204309ee96b2SMikulas Patocka EXPORT_SYMBOL_GPL(dm_hold);
204409ee96b2SMikulas Patocka 
204572d94861SAlasdair G Kergon const char *dm_device_name(struct mapped_device *md)
204672d94861SAlasdair G Kergon {
204772d94861SAlasdair G Kergon 	return md->name;
204872d94861SAlasdair G Kergon }
204972d94861SAlasdair G Kergon EXPORT_SYMBOL_GPL(dm_device_name);
205072d94861SAlasdair G Kergon 
20513f77316dSKiyoshi Ueda static void __dm_destroy(struct mapped_device *md, bool wait)
20521da177e4SLinus Torvalds {
20531134e5aeSMike Anderson 	struct dm_table *map;
205483d5e5b0SMikulas Patocka 	int srcu_idx;
20551da177e4SLinus Torvalds 
20563f77316dSKiyoshi Ueda 	might_sleep();
2057fba9f90eSJeff Mahoney 
205863a4f065SMike Snitzer 	spin_lock(&_minor_lock);
20593f77316dSKiyoshi Ueda 	idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));
2060fba9f90eSJeff Mahoney 	set_bit(DMF_FREEING, &md->flags);
2061f32c10b0SJeff Mahoney 	spin_unlock(&_minor_lock);
20623f77316dSKiyoshi Ueda 
2063c12c9a3cSMike Snitzer 	blk_set_queue_dying(md->queue);
20643b785fbcSBart Van Assche 
2065ab7c7bb6SMikulas Patocka 	/*
2066ab7c7bb6SMikulas Patocka 	 * Take suspend_lock so that presuspend and postsuspend methods
2067ab7c7bb6SMikulas Patocka 	 * do not race with internal suspend.
2068ab7c7bb6SMikulas Patocka 	 */
2069ab7c7bb6SMikulas Patocka 	mutex_lock(&md->suspend_lock);
20702a708cffSJunichi Nomura 	map = dm_get_live_table(md, &srcu_idx);
20714f186f8bSKiyoshi Ueda 	if (!dm_suspended_md(md)) {
20721da177e4SLinus Torvalds 		dm_table_presuspend_targets(map);
2073adc0daadSMikulas Patocka 		set_bit(DMF_SUSPENDED, &md->flags);
20745df96f2bSMikulas Patocka 		set_bit(DMF_POST_SUSPENDING, &md->flags);
20751da177e4SLinus Torvalds 		dm_table_postsuspend_targets(map);
20761da177e4SLinus Torvalds 	}
207783d5e5b0SMikulas Patocka 	/* dm_put_live_table must be before msleep, otherwise deadlock is possible */
207883d5e5b0SMikulas Patocka 	dm_put_live_table(md, srcu_idx);
20792a708cffSJunichi Nomura 	mutex_unlock(&md->suspend_lock);
208083d5e5b0SMikulas Patocka 
20813f77316dSKiyoshi Ueda 	/*
20823f77316dSKiyoshi Ueda 	 * Rare, but there may be I/O requests still going to complete,
20833f77316dSKiyoshi Ueda 	 * for example.  Wait for all references to disappear.
20843f77316dSKiyoshi Ueda 	 * No one should increment the reference count of the mapped_device,
20853f77316dSKiyoshi Ueda 	 * after the mapped_device state becomes DMF_FREEING.
20863f77316dSKiyoshi Ueda 	 */
20873f77316dSKiyoshi Ueda 	if (wait)
20883f77316dSKiyoshi Ueda 		while (atomic_read(&md->holders))
20893f77316dSKiyoshi Ueda 			msleep(1);
20903f77316dSKiyoshi Ueda 	else if (atomic_read(&md->holders))
20913f77316dSKiyoshi Ueda 		DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)",
20923f77316dSKiyoshi Ueda 		       dm_device_name(md), atomic_read(&md->holders));
20933f77316dSKiyoshi Ueda 
2094a7940155SAlasdair G Kergon 	dm_table_destroy(__unbind(md));
20951da177e4SLinus Torvalds 	free_dev(md);
20961da177e4SLinus Torvalds }
20973f77316dSKiyoshi Ueda 
20983f77316dSKiyoshi Ueda void dm_destroy(struct mapped_device *md)
20993f77316dSKiyoshi Ueda {
21003f77316dSKiyoshi Ueda 	__dm_destroy(md, true);
21013f77316dSKiyoshi Ueda }
21023f77316dSKiyoshi Ueda 
21033f77316dSKiyoshi Ueda void dm_destroy_immediate(struct mapped_device *md)
21043f77316dSKiyoshi Ueda {
21053f77316dSKiyoshi Ueda 	__dm_destroy(md, false);
21063f77316dSKiyoshi Ueda }
21073f77316dSKiyoshi Ueda 
21083f77316dSKiyoshi Ueda void dm_put(struct mapped_device *md)
21093f77316dSKiyoshi Ueda {
21103f77316dSKiyoshi Ueda 	atomic_dec(&md->holders);
21111da177e4SLinus Torvalds }
211279eb885cSEdward Goggin EXPORT_SYMBOL_GPL(dm_put);
21131da177e4SLinus Torvalds 
21149f6dc633SMike Snitzer static bool dm_in_flight_bios(struct mapped_device *md)
211585067747SMing Lei {
211685067747SMing Lei 	int cpu;
21179f6dc633SMike Snitzer 	unsigned long sum = 0;
211885067747SMing Lei 
21199f6dc633SMike Snitzer 	for_each_possible_cpu(cpu)
21209f6dc633SMike Snitzer 		sum += *per_cpu_ptr(md->pending_io, cpu);
212185067747SMing Lei 
212285067747SMing Lei 	return sum != 0;
212385067747SMing Lei }
212485067747SMing Lei 
21252f064a59SPeter Zijlstra static int dm_wait_for_bios_completion(struct mapped_device *md, unsigned int task_state)
212646125c1cSMilan Broz {
212746125c1cSMilan Broz 	int r = 0;
21289f4c3f87SBart Van Assche 	DEFINE_WAIT(wait);
212946125c1cSMilan Broz 
213085067747SMing Lei 	while (true) {
21319f4c3f87SBart Van Assche 		prepare_to_wait(&md->wait, &wait, task_state);
213246125c1cSMilan Broz 
21339f6dc633SMike Snitzer 		if (!dm_in_flight_bios(md))
213446125c1cSMilan Broz 			break;
213546125c1cSMilan Broz 
2136e3fabdfdSBart Van Assche 		if (signal_pending_state(task_state, current)) {
213746125c1cSMilan Broz 			r = -EINTR;
213846125c1cSMilan Broz 			break;
213946125c1cSMilan Broz 		}
214046125c1cSMilan Broz 
214146125c1cSMilan Broz 		io_schedule();
214246125c1cSMilan Broz 	}
21439f4c3f87SBart Van Assche 	finish_wait(&md->wait, &wait);
2144b44ebeb0SMikulas Patocka 
21459f6dc633SMike Snitzer 	smp_rmb();
21469f6dc633SMike Snitzer 
214746125c1cSMilan Broz 	return r;
214846125c1cSMilan Broz }
214946125c1cSMilan Broz 
21502f064a59SPeter Zijlstra static int dm_wait_for_completion(struct mapped_device *md, unsigned int task_state)
215185067747SMing Lei {
215285067747SMing Lei 	int r = 0;
215385067747SMing Lei 
215485067747SMing Lei 	if (!queue_is_mq(md->queue))
215585067747SMing Lei 		return dm_wait_for_bios_completion(md, task_state);
215685067747SMing Lei 
215785067747SMing Lei 	while (true) {
215885067747SMing Lei 		if (!blk_mq_queue_inflight(md->queue))
215985067747SMing Lei 			break;
216085067747SMing Lei 
216185067747SMing Lei 		if (signal_pending_state(task_state, current)) {
216285067747SMing Lei 			r = -EINTR;
216385067747SMing Lei 			break;
216485067747SMing Lei 		}
216585067747SMing Lei 
216685067747SMing Lei 		msleep(5);
216785067747SMing Lei 	}
216885067747SMing Lei 
216985067747SMing Lei 	return r;
217085067747SMing Lei }
217185067747SMing Lei 
21721da177e4SLinus Torvalds /*
21731da177e4SLinus Torvalds  * Process the deferred bios
21741da177e4SLinus Torvalds  */
2175ef208587SMikulas Patocka static void dm_wq_work(struct work_struct *work)
21761da177e4SLinus Torvalds {
21770c2915b8SMike Snitzer 	struct mapped_device *md = container_of(work, struct mapped_device, work);
21780c2915b8SMike Snitzer 	struct bio *bio;
2179ef208587SMikulas Patocka 
21803b00b203SMikulas Patocka 	while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
2181022c2611SMikulas Patocka 		spin_lock_irq(&md->deferred_lock);
21820c2915b8SMike Snitzer 		bio = bio_list_pop(&md->deferred);
2183022c2611SMikulas Patocka 		spin_unlock_irq(&md->deferred_lock);
2184022c2611SMikulas Patocka 
21850c2915b8SMike Snitzer 		if (!bio)
2186df12ee99SAlasdair G Kergon 			break;
218773d410c0SMilan Broz 
21880c2915b8SMike Snitzer 		submit_bio_noacct(bio);
2189e6ee8c0bSKiyoshi Ueda 	}
21901da177e4SLinus Torvalds }
21911da177e4SLinus Torvalds 
21929a1fb464SMikulas Patocka static void dm_queue_flush(struct mapped_device *md)
2193304f3f6aSMilan Broz {
21943b00b203SMikulas Patocka 	clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
21954e857c58SPeter Zijlstra 	smp_mb__after_atomic();
219653d5914fSMikulas Patocka 	queue_work(md->wq, &md->work);
2197304f3f6aSMilan Broz }
2198304f3f6aSMilan Broz 
21991da177e4SLinus Torvalds /*
2200042d2a9bSAlasdair G Kergon  * Swap in a new table, returning the old one for the caller to destroy.
22011da177e4SLinus Torvalds  */
2202042d2a9bSAlasdair G Kergon struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table)
22031da177e4SLinus Torvalds {
220487eb5b21SMike Christie 	struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL);
2205754c5fc7SMike Snitzer 	struct queue_limits limits;
2206042d2a9bSAlasdair G Kergon 	int r;
22071da177e4SLinus Torvalds 
2208e61290a4SDaniel Walker 	mutex_lock(&md->suspend_lock);
22091da177e4SLinus Torvalds 
22101da177e4SLinus Torvalds 	/* device must be suspended */
22114f186f8bSKiyoshi Ueda 	if (!dm_suspended_md(md))
221293c534aeSAlasdair G Kergon 		goto out;
22131da177e4SLinus Torvalds 
22143ae70656SMike Snitzer 	/*
22153ae70656SMike Snitzer 	 * If the new table has no data devices, retain the existing limits.
22163ae70656SMike Snitzer 	 * This helps multipath with queue_if_no_path if all paths disappear,
22173ae70656SMike Snitzer 	 * then new I/O is queued based on these limits, and then some paths
22183ae70656SMike Snitzer 	 * reappear.
22193ae70656SMike Snitzer 	 */
22203ae70656SMike Snitzer 	if (dm_table_has_no_data_devices(table)) {
222183d5e5b0SMikulas Patocka 		live_map = dm_get_live_table_fast(md);
22223ae70656SMike Snitzer 		if (live_map)
22233ae70656SMike Snitzer 			limits = md->queue->limits;
222483d5e5b0SMikulas Patocka 		dm_put_live_table_fast(md);
22253ae70656SMike Snitzer 	}
22263ae70656SMike Snitzer 
222787eb5b21SMike Christie 	if (!live_map) {
2228754c5fc7SMike Snitzer 		r = dm_calculate_queue_limits(table, &limits);
2229042d2a9bSAlasdair G Kergon 		if (r) {
2230042d2a9bSAlasdair G Kergon 			map = ERR_PTR(r);
2231754c5fc7SMike Snitzer 			goto out;
2232042d2a9bSAlasdair G Kergon 		}
223387eb5b21SMike Christie 	}
2234754c5fc7SMike Snitzer 
2235042d2a9bSAlasdair G Kergon 	map = __bind(md, table, &limits);
223662e08243SMikulas Patocka 	dm_issue_global_event();
22371da177e4SLinus Torvalds 
223893c534aeSAlasdair G Kergon out:
2239e61290a4SDaniel Walker 	mutex_unlock(&md->suspend_lock);
2240042d2a9bSAlasdair G Kergon 	return map;
22411da177e4SLinus Torvalds }
22421da177e4SLinus Torvalds 
22431da177e4SLinus Torvalds /*
22441da177e4SLinus Torvalds  * Functions to lock and unlock any filesystem running on the
22451da177e4SLinus Torvalds  * device.
22461da177e4SLinus Torvalds  */
22472ca3310eSAlasdair G Kergon static int lock_fs(struct mapped_device *md)
22481da177e4SLinus Torvalds {
2249e39e2e95SAlasdair G Kergon 	int r;
22501da177e4SLinus Torvalds 
2251040f04bdSChristoph Hellwig 	WARN_ON(test_bit(DMF_FROZEN, &md->flags));
2252dfbe03f6SAlasdair G Kergon 
2253977115c0SChristoph Hellwig 	r = freeze_bdev(md->disk->part0);
2254040f04bdSChristoph Hellwig 	if (!r)
2255aa8d7c2fSAlasdair G Kergon 		set_bit(DMF_FROZEN, &md->flags);
2256040f04bdSChristoph Hellwig 	return r;
22571da177e4SLinus Torvalds }
22581da177e4SLinus Torvalds 
22592ca3310eSAlasdair G Kergon static void unlock_fs(struct mapped_device *md)
22601da177e4SLinus Torvalds {
2261aa8d7c2fSAlasdair G Kergon 	if (!test_bit(DMF_FROZEN, &md->flags))
2262aa8d7c2fSAlasdair G Kergon 		return;
2263977115c0SChristoph Hellwig 	thaw_bdev(md->disk->part0);
2264aa8d7c2fSAlasdair G Kergon 	clear_bit(DMF_FROZEN, &md->flags);
22651da177e4SLinus Torvalds }
22661da177e4SLinus Torvalds 
22671da177e4SLinus Torvalds /*
2268b48633f8SBart Van Assche  * @suspend_flags: DM_SUSPEND_LOCKFS_FLAG and/or DM_SUSPEND_NOFLUSH_FLAG
2269b48633f8SBart Van Assche  * @task_state: e.g. TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE
2270b48633f8SBart Van Assche  * @dmf_suspended_flag: DMF_SUSPENDED or DMF_SUSPENDED_INTERNALLY
2271b48633f8SBart Van Assche  *
2272ffcc3936SMike Snitzer  * If __dm_suspend returns 0, the device is completely quiescent
2273ffcc3936SMike Snitzer  * now. There is no request-processing activity. All new requests
2274ffcc3936SMike Snitzer  * are being added to md->deferred list.
2275cec47e3dSKiyoshi Ueda  */
2276ffcc3936SMike Snitzer static int __dm_suspend(struct mapped_device *md, struct dm_table *map,
22772f064a59SPeter Zijlstra 			unsigned suspend_flags, unsigned int task_state,
2278eaf9a736SMike Snitzer 			int dmf_suspended_flag)
22791da177e4SLinus Torvalds {
2280ffcc3936SMike Snitzer 	bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG;
2281ffcc3936SMike Snitzer 	bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG;
2282ffcc3936SMike Snitzer 	int r;
2283cf222b37SAlasdair G Kergon 
22845a8f1f80SBart Van Assche 	lockdep_assert_held(&md->suspend_lock);
22855a8f1f80SBart Van Assche 
22862e93ccc1SKiyoshi Ueda 	/*
22872e93ccc1SKiyoshi Ueda 	 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
22882e93ccc1SKiyoshi Ueda 	 * This flag is cleared before dm_suspend returns.
22892e93ccc1SKiyoshi Ueda 	 */
22902e93ccc1SKiyoshi Ueda 	if (noflush)
22912e93ccc1SKiyoshi Ueda 		set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
229286331f39SBart Van Assche 	else
2293ac75b09fSMike Snitzer 		DMDEBUG("%s: suspending with flush", dm_device_name(md));
22942e93ccc1SKiyoshi Ueda 
2295d67ee213SMike Snitzer 	/*
2296d67ee213SMike Snitzer 	 * This gets reverted if there's an error later and the targets
2297d67ee213SMike Snitzer 	 * provide the .presuspend_undo hook.
2298d67ee213SMike Snitzer 	 */
22991da177e4SLinus Torvalds 	dm_table_presuspend_targets(map);
23001da177e4SLinus Torvalds 
23012e93ccc1SKiyoshi Ueda 	/*
23029f518b27SKiyoshi Ueda 	 * Flush I/O to the device.
23039f518b27SKiyoshi Ueda 	 * Any I/O submitted after lock_fs() may not be flushed.
23049f518b27SKiyoshi Ueda 	 * noflush takes precedence over do_lockfs.
23059f518b27SKiyoshi Ueda 	 * (lock_fs() flushes I/Os and waits for them to complete.)
23062e93ccc1SKiyoshi Ueda 	 */
230732a926daSMikulas Patocka 	if (!noflush && do_lockfs) {
23082ca3310eSAlasdair G Kergon 		r = lock_fs(md);
2309d67ee213SMike Snitzer 		if (r) {
2310d67ee213SMike Snitzer 			dm_table_presuspend_undo_targets(map);
2311ffcc3936SMike Snitzer 			return r;
2312aa8d7c2fSAlasdair G Kergon 		}
2313d67ee213SMike Snitzer 	}
23141da177e4SLinus Torvalds 
23151da177e4SLinus Torvalds 	/*
23163b00b203SMikulas Patocka 	 * Here we must make sure that no processes are submitting requests
23173b00b203SMikulas Patocka 	 * to target drivers i.e. no one may be executing
23180cede372SMike Snitzer 	 * __split_and_process_bio from dm_submit_bio.
23193b00b203SMikulas Patocka 	 *
23200cede372SMike Snitzer 	 * To get all processes out of __split_and_process_bio in dm_submit_bio,
23213b00b203SMikulas Patocka 	 * we take the write lock. To prevent any process from reentering
23220cede372SMike Snitzer 	 * __split_and_process_bio from dm_submit_bio and quiesce the thread
23230cede372SMike Snitzer 	 * (dm_wq_work), we set DMF_BLOCK_IO_FOR_SUSPEND and call
23246a8736d1STejun Heo 	 * flush_workqueue(md->wq).
23251da177e4SLinus Torvalds 	 */
23261eb787ecSAlasdair G Kergon 	set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
232741abc4e1SHannes Reinecke 	if (map)
232883d5e5b0SMikulas Patocka 		synchronize_srcu(&md->io_barrier);
23291da177e4SLinus Torvalds 
2330d0bcb878SKiyoshi Ueda 	/*
233129e4013dSTejun Heo 	 * Stop md->queue before flushing md->wq in case request-based
233229e4013dSTejun Heo 	 * dm defers requests to md->wq from md->queue.
2333d0bcb878SKiyoshi Ueda 	 */
23346a23e05cSJens Axboe 	if (dm_request_based(md))
2335eca7ee6dSMike Snitzer 		dm_stop_queue(md->queue);
2336cec47e3dSKiyoshi Ueda 
2337d0bcb878SKiyoshi Ueda 	flush_workqueue(md->wq);
2338d0bcb878SKiyoshi Ueda 
23391da177e4SLinus Torvalds 	/*
23403b00b203SMikulas Patocka 	 * At this point no more requests are entering target request routines.
23413b00b203SMikulas Patocka 	 * We call dm_wait_for_completion to wait for all existing requests
23423b00b203SMikulas Patocka 	 * to finish.
23431da177e4SLinus Torvalds 	 */
2344b48633f8SBart Van Assche 	r = dm_wait_for_completion(md, task_state);
2345eaf9a736SMike Snitzer 	if (!r)
2346eaf9a736SMike Snitzer 		set_bit(dmf_suspended_flag, &md->flags);
23471da177e4SLinus Torvalds 
23486d6f10dfSMilan Broz 	if (noflush)
2349022c2611SMikulas Patocka 		clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
235041abc4e1SHannes Reinecke 	if (map)
235183d5e5b0SMikulas Patocka 		synchronize_srcu(&md->io_barrier);
23522e93ccc1SKiyoshi Ueda 
23531da177e4SLinus Torvalds 	/* were we interrupted ? */
235446125c1cSMilan Broz 	if (r < 0) {
23559a1fb464SMikulas Patocka 		dm_queue_flush(md);
235673d410c0SMilan Broz 
2357cec47e3dSKiyoshi Ueda 		if (dm_request_based(md))
2358eca7ee6dSMike Snitzer 			dm_start_queue(md->queue);
2359cec47e3dSKiyoshi Ueda 
23602ca3310eSAlasdair G Kergon 		unlock_fs(md);
2361d67ee213SMike Snitzer 		dm_table_presuspend_undo_targets(map);
2362ffcc3936SMike Snitzer 		/* pushback list is already flushed, so skip flush */
2363ffcc3936SMike Snitzer 	}
2364ffcc3936SMike Snitzer 
2365ffcc3936SMike Snitzer 	return r;
23662ca3310eSAlasdair G Kergon }
23672ca3310eSAlasdair G Kergon 
23683b00b203SMikulas Patocka /*
2369ffcc3936SMike Snitzer  * We need to be able to change a mapping table under a mounted
2370ffcc3936SMike Snitzer  * filesystem.  For example we might want to move some data in
2371ffcc3936SMike Snitzer  * the background.  Before the table can be swapped with
2372ffcc3936SMike Snitzer  * dm_bind_table, dm_suspend must be called to flush any in
2373ffcc3936SMike Snitzer  * flight bios and ensure that any further io gets deferred.
23743b00b203SMikulas Patocka  */
2375ffcc3936SMike Snitzer /*
2376ffcc3936SMike Snitzer  * Suspend mechanism in request-based dm.
2377ffcc3936SMike Snitzer  *
2378ffcc3936SMike Snitzer  * 1. Flush all I/Os by lock_fs() if needed.
2379ffcc3936SMike Snitzer  * 2. Stop dispatching any I/O by stopping the request_queue.
2380ffcc3936SMike Snitzer  * 3. Wait for all in-flight I/Os to be completed or requeued.
2381ffcc3936SMike Snitzer  *
2382ffcc3936SMike Snitzer  * To abort suspend, start the request_queue.
2383ffcc3936SMike Snitzer  */
2384ffcc3936SMike Snitzer int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
2385ffcc3936SMike Snitzer {
2386ffcc3936SMike Snitzer 	struct dm_table *map = NULL;
2387ffcc3936SMike Snitzer 	int r = 0;
2388ffcc3936SMike Snitzer 
2389ffcc3936SMike Snitzer retry:
2390ffcc3936SMike Snitzer 	mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2391ffcc3936SMike Snitzer 
2392ffcc3936SMike Snitzer 	if (dm_suspended_md(md)) {
2393ffcc3936SMike Snitzer 		r = -EINVAL;
2394ffcc3936SMike Snitzer 		goto out_unlock;
2395ffcc3936SMike Snitzer 	}
2396ffcc3936SMike Snitzer 
2397ffcc3936SMike Snitzer 	if (dm_suspended_internally_md(md)) {
2398ffcc3936SMike Snitzer 		/* already internally suspended, wait for internal resume */
2399ffcc3936SMike Snitzer 		mutex_unlock(&md->suspend_lock);
2400ffcc3936SMike Snitzer 		r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2401ffcc3936SMike Snitzer 		if (r)
2402ffcc3936SMike Snitzer 			return r;
2403ffcc3936SMike Snitzer 		goto retry;
2404ffcc3936SMike Snitzer 	}
2405ffcc3936SMike Snitzer 
2406a12f5d48SEric Dumazet 	map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2407ffcc3936SMike Snitzer 
2408eaf9a736SMike Snitzer 	r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED);
2409ffcc3936SMike Snitzer 	if (r)
2410ffcc3936SMike Snitzer 		goto out_unlock;
24113b00b203SMikulas Patocka 
24125df96f2bSMikulas Patocka 	set_bit(DMF_POST_SUSPENDING, &md->flags);
24134d4471cbSKiyoshi Ueda 	dm_table_postsuspend_targets(map);
24145df96f2bSMikulas Patocka 	clear_bit(DMF_POST_SUSPENDING, &md->flags);
24154d4471cbSKiyoshi Ueda 
2416d287483dSAlasdair G Kergon out_unlock:
2417e61290a4SDaniel Walker 	mutex_unlock(&md->suspend_lock);
2418cf222b37SAlasdair G Kergon 	return r;
24191da177e4SLinus Torvalds }
24201da177e4SLinus Torvalds 
2421ffcc3936SMike Snitzer static int __dm_resume(struct mapped_device *md, struct dm_table *map)
24221da177e4SLinus Torvalds {
2423ffcc3936SMike Snitzer 	if (map) {
2424ffcc3936SMike Snitzer 		int r = dm_table_resume_targets(map);
24258757b776SMilan Broz 		if (r)
2426ffcc3936SMike Snitzer 			return r;
2427ffcc3936SMike Snitzer 	}
24282ca3310eSAlasdair G Kergon 
24299a1fb464SMikulas Patocka 	dm_queue_flush(md);
24302ca3310eSAlasdair G Kergon 
2431cec47e3dSKiyoshi Ueda 	/*
2432cec47e3dSKiyoshi Ueda 	 * Flushing deferred I/Os must be done after targets are resumed
2433cec47e3dSKiyoshi Ueda 	 * so that mapping of targets can work correctly.
2434cec47e3dSKiyoshi Ueda 	 * Request-based dm is queueing the deferred I/Os in its request_queue.
2435cec47e3dSKiyoshi Ueda 	 */
2436cec47e3dSKiyoshi Ueda 	if (dm_request_based(md))
2437eca7ee6dSMike Snitzer 		dm_start_queue(md->queue);
2438cec47e3dSKiyoshi Ueda 
24392ca3310eSAlasdair G Kergon 	unlock_fs(md);
24402ca3310eSAlasdair G Kergon 
2441ffcc3936SMike Snitzer 	return 0;
2442ffcc3936SMike Snitzer }
2443ffcc3936SMike Snitzer 
2444ffcc3936SMike Snitzer int dm_resume(struct mapped_device *md)
2445ffcc3936SMike Snitzer {
24468dc23658SMinfei Huang 	int r;
2447ffcc3936SMike Snitzer 	struct dm_table *map = NULL;
2448ffcc3936SMike Snitzer 
2449ffcc3936SMike Snitzer retry:
24508dc23658SMinfei Huang 	r = -EINVAL;
2451ffcc3936SMike Snitzer 	mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2452ffcc3936SMike Snitzer 
2453ffcc3936SMike Snitzer 	if (!dm_suspended_md(md))
2454ffcc3936SMike Snitzer 		goto out;
2455ffcc3936SMike Snitzer 
2456ffcc3936SMike Snitzer 	if (dm_suspended_internally_md(md)) {
2457ffcc3936SMike Snitzer 		/* already internally suspended, wait for internal resume */
2458ffcc3936SMike Snitzer 		mutex_unlock(&md->suspend_lock);
2459ffcc3936SMike Snitzer 		r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2460ffcc3936SMike Snitzer 		if (r)
2461ffcc3936SMike Snitzer 			return r;
2462ffcc3936SMike Snitzer 		goto retry;
2463ffcc3936SMike Snitzer 	}
2464ffcc3936SMike Snitzer 
2465a12f5d48SEric Dumazet 	map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2466ffcc3936SMike Snitzer 	if (!map || !dm_table_get_size(map))
2467ffcc3936SMike Snitzer 		goto out;
2468ffcc3936SMike Snitzer 
2469ffcc3936SMike Snitzer 	r = __dm_resume(md, map);
2470ffcc3936SMike Snitzer 	if (r)
2471ffcc3936SMike Snitzer 		goto out;
2472ffcc3936SMike Snitzer 
24732ca3310eSAlasdair G Kergon 	clear_bit(DMF_SUSPENDED, &md->flags);
2474cf222b37SAlasdair G Kergon out:
2475e61290a4SDaniel Walker 	mutex_unlock(&md->suspend_lock);
24762ca3310eSAlasdair G Kergon 
2477cf222b37SAlasdair G Kergon 	return r;
24781da177e4SLinus Torvalds }
24791da177e4SLinus Torvalds 
2480fd2ed4d2SMikulas Patocka /*
2481fd2ed4d2SMikulas Patocka  * Internal suspend/resume works like userspace-driven suspend. It waits
2482fd2ed4d2SMikulas Patocka  * until all bios finish and prevents issuing new bios to the target drivers.
2483fd2ed4d2SMikulas Patocka  * It may be used only from the kernel.
2484fd2ed4d2SMikulas Patocka  */
2485fd2ed4d2SMikulas Patocka 
2486ffcc3936SMike Snitzer static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags)
2487ffcc3936SMike Snitzer {
2488ffcc3936SMike Snitzer 	struct dm_table *map = NULL;
2489ffcc3936SMike Snitzer 
24901ea0654eSBart Van Assche 	lockdep_assert_held(&md->suspend_lock);
24911ea0654eSBart Van Assche 
249296b26c8cSMikulas Patocka 	if (md->internal_suspend_count++)
2493ffcc3936SMike Snitzer 		return; /* nested internal suspend */
2494ffcc3936SMike Snitzer 
2495ffcc3936SMike Snitzer 	if (dm_suspended_md(md)) {
2496ffcc3936SMike Snitzer 		set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2497ffcc3936SMike Snitzer 		return; /* nest suspend */
2498ffcc3936SMike Snitzer 	}
2499ffcc3936SMike Snitzer 
2500a12f5d48SEric Dumazet 	map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2501ffcc3936SMike Snitzer 
2502ffcc3936SMike Snitzer 	/*
2503ffcc3936SMike Snitzer 	 * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is
2504ffcc3936SMike Snitzer 	 * supported.  Properly supporting a TASK_INTERRUPTIBLE internal suspend
2505ffcc3936SMike Snitzer 	 * would require changing .presuspend to return an error -- avoid this
2506ffcc3936SMike Snitzer 	 * until there is a need for more elaborate variants of internal suspend.
2507ffcc3936SMike Snitzer 	 */
2508eaf9a736SMike Snitzer 	(void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE,
2509eaf9a736SMike Snitzer 			    DMF_SUSPENDED_INTERNALLY);
2510ffcc3936SMike Snitzer 
25115df96f2bSMikulas Patocka 	set_bit(DMF_POST_SUSPENDING, &md->flags);
2512ffcc3936SMike Snitzer 	dm_table_postsuspend_targets(map);
25135df96f2bSMikulas Patocka 	clear_bit(DMF_POST_SUSPENDING, &md->flags);
2514ffcc3936SMike Snitzer }
2515ffcc3936SMike Snitzer 
2516ffcc3936SMike Snitzer static void __dm_internal_resume(struct mapped_device *md)
2517ffcc3936SMike Snitzer {
251896b26c8cSMikulas Patocka 	BUG_ON(!md->internal_suspend_count);
251996b26c8cSMikulas Patocka 
252096b26c8cSMikulas Patocka 	if (--md->internal_suspend_count)
2521ffcc3936SMike Snitzer 		return; /* resume from nested internal suspend */
2522ffcc3936SMike Snitzer 
2523ffcc3936SMike Snitzer 	if (dm_suspended_md(md))
2524ffcc3936SMike Snitzer 		goto done; /* resume from nested suspend */
2525ffcc3936SMike Snitzer 
2526ffcc3936SMike Snitzer 	/*
2527ffcc3936SMike Snitzer 	 * NOTE: existing callers don't need to call dm_table_resume_targets
2528ffcc3936SMike Snitzer 	 * (which may fail -- so best to avoid it for now by passing NULL map)
2529ffcc3936SMike Snitzer 	 */
2530ffcc3936SMike Snitzer 	(void) __dm_resume(md, NULL);
2531ffcc3936SMike Snitzer 
2532ffcc3936SMike Snitzer done:
2533ffcc3936SMike Snitzer 	clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2534ffcc3936SMike Snitzer 	smp_mb__after_atomic();
2535ffcc3936SMike Snitzer 	wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY);
2536ffcc3936SMike Snitzer }
2537ffcc3936SMike Snitzer 
2538ffcc3936SMike Snitzer void dm_internal_suspend_noflush(struct mapped_device *md)
2539fd2ed4d2SMikulas Patocka {
2540fd2ed4d2SMikulas Patocka 	mutex_lock(&md->suspend_lock);
2541ffcc3936SMike Snitzer 	__dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG);
2542ffcc3936SMike Snitzer 	mutex_unlock(&md->suspend_lock);
2543ffcc3936SMike Snitzer }
2544ffcc3936SMike Snitzer EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush);
2545ffcc3936SMike Snitzer 
2546ffcc3936SMike Snitzer void dm_internal_resume(struct mapped_device *md)
2547ffcc3936SMike Snitzer {
2548ffcc3936SMike Snitzer 	mutex_lock(&md->suspend_lock);
2549ffcc3936SMike Snitzer 	__dm_internal_resume(md);
2550ffcc3936SMike Snitzer 	mutex_unlock(&md->suspend_lock);
2551ffcc3936SMike Snitzer }
2552ffcc3936SMike Snitzer EXPORT_SYMBOL_GPL(dm_internal_resume);
2553ffcc3936SMike Snitzer 
2554ffcc3936SMike Snitzer /*
2555ffcc3936SMike Snitzer  * Fast variants of internal suspend/resume hold md->suspend_lock,
2556ffcc3936SMike Snitzer  * which prevents interaction with userspace-driven suspend.
2557ffcc3936SMike Snitzer  */
2558ffcc3936SMike Snitzer 
2559ffcc3936SMike Snitzer void dm_internal_suspend_fast(struct mapped_device *md)
2560ffcc3936SMike Snitzer {
2561ffcc3936SMike Snitzer 	mutex_lock(&md->suspend_lock);
2562ffcc3936SMike Snitzer 	if (dm_suspended_md(md) || dm_suspended_internally_md(md))
2563fd2ed4d2SMikulas Patocka 		return;
2564fd2ed4d2SMikulas Patocka 
2565fd2ed4d2SMikulas Patocka 	set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2566fd2ed4d2SMikulas Patocka 	synchronize_srcu(&md->io_barrier);
2567fd2ed4d2SMikulas Patocka 	flush_workqueue(md->wq);
2568fd2ed4d2SMikulas Patocka 	dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
2569fd2ed4d2SMikulas Patocka }
2570b735fedeSMikulas Patocka EXPORT_SYMBOL_GPL(dm_internal_suspend_fast);
2571fd2ed4d2SMikulas Patocka 
2572ffcc3936SMike Snitzer void dm_internal_resume_fast(struct mapped_device *md)
2573fd2ed4d2SMikulas Patocka {
2574ffcc3936SMike Snitzer 	if (dm_suspended_md(md) || dm_suspended_internally_md(md))
2575fd2ed4d2SMikulas Patocka 		goto done;
2576fd2ed4d2SMikulas Patocka 
2577fd2ed4d2SMikulas Patocka 	dm_queue_flush(md);
2578fd2ed4d2SMikulas Patocka 
2579fd2ed4d2SMikulas Patocka done:
2580fd2ed4d2SMikulas Patocka 	mutex_unlock(&md->suspend_lock);
2581fd2ed4d2SMikulas Patocka }
2582b735fedeSMikulas Patocka EXPORT_SYMBOL_GPL(dm_internal_resume_fast);
2583fd2ed4d2SMikulas Patocka 
25841da177e4SLinus Torvalds /*-----------------------------------------------------------------
25851da177e4SLinus Torvalds  * Event notification.
25861da177e4SLinus Torvalds  *---------------------------------------------------------------*/
25873abf85b5SPeter Rajnoha int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action,
258860935eb2SMilan Broz 		       unsigned cookie)
258969267a30SAlasdair G Kergon {
25906958c1c6SMikulas Patocka 	int r;
25916958c1c6SMikulas Patocka 	unsigned noio_flag;
259260935eb2SMilan Broz 	char udev_cookie[DM_COOKIE_LENGTH];
259360935eb2SMilan Broz 	char *envp[] = { udev_cookie, NULL };
259460935eb2SMilan Broz 
25956958c1c6SMikulas Patocka 	noio_flag = memalloc_noio_save();
25966958c1c6SMikulas Patocka 
259760935eb2SMilan Broz 	if (!cookie)
25986958c1c6SMikulas Patocka 		r = kobject_uevent(&disk_to_dev(md->disk)->kobj, action);
259960935eb2SMilan Broz 	else {
260060935eb2SMilan Broz 		snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u",
260160935eb2SMilan Broz 			 DM_COOKIE_ENV_VAR_NAME, cookie);
26026958c1c6SMikulas Patocka 		r = kobject_uevent_env(&disk_to_dev(md->disk)->kobj,
26033abf85b5SPeter Rajnoha 				       action, envp);
260460935eb2SMilan Broz 	}
26056958c1c6SMikulas Patocka 
26066958c1c6SMikulas Patocka 	memalloc_noio_restore(noio_flag);
26076958c1c6SMikulas Patocka 
26086958c1c6SMikulas Patocka 	return r;
260969267a30SAlasdair G Kergon }
261069267a30SAlasdair G Kergon 
26117a8c3d3bSMike Anderson uint32_t dm_next_uevent_seq(struct mapped_device *md)
26127a8c3d3bSMike Anderson {
26137a8c3d3bSMike Anderson 	return atomic_add_return(1, &md->uevent_seq);
26147a8c3d3bSMike Anderson }
26157a8c3d3bSMike Anderson 
26161da177e4SLinus Torvalds uint32_t dm_get_event_nr(struct mapped_device *md)
26171da177e4SLinus Torvalds {
26181da177e4SLinus Torvalds 	return atomic_read(&md->event_nr);
26191da177e4SLinus Torvalds }
26201da177e4SLinus Torvalds 
26211da177e4SLinus Torvalds int dm_wait_event(struct mapped_device *md, int event_nr)
26221da177e4SLinus Torvalds {
26231da177e4SLinus Torvalds 	return wait_event_interruptible(md->eventq,
26241da177e4SLinus Torvalds 			(event_nr != atomic_read(&md->event_nr)));
26251da177e4SLinus Torvalds }
26261da177e4SLinus Torvalds 
26277a8c3d3bSMike Anderson void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
26287a8c3d3bSMike Anderson {
26297a8c3d3bSMike Anderson 	unsigned long flags;
26307a8c3d3bSMike Anderson 
26317a8c3d3bSMike Anderson 	spin_lock_irqsave(&md->uevent_lock, flags);
26327a8c3d3bSMike Anderson 	list_add(elist, &md->uevent_list);
26337a8c3d3bSMike Anderson 	spin_unlock_irqrestore(&md->uevent_lock, flags);
26347a8c3d3bSMike Anderson }
26357a8c3d3bSMike Anderson 
26361da177e4SLinus Torvalds /*
26371da177e4SLinus Torvalds  * The gendisk is only valid as long as you have a reference
26381da177e4SLinus Torvalds  * count on 'md'.
26391da177e4SLinus Torvalds  */
26401da177e4SLinus Torvalds struct gendisk *dm_disk(struct mapped_device *md)
26411da177e4SLinus Torvalds {
26421da177e4SLinus Torvalds 	return md->disk;
26431da177e4SLinus Torvalds }
264465ff5b7dSSami Tolvanen EXPORT_SYMBOL_GPL(dm_disk);
26451da177e4SLinus Torvalds 
2646784aae73SMilan Broz struct kobject *dm_kobject(struct mapped_device *md)
2647784aae73SMilan Broz {
26482995fa78SMikulas Patocka 	return &md->kobj_holder.kobj;
2649784aae73SMilan Broz }
2650784aae73SMilan Broz 
2651784aae73SMilan Broz struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
2652784aae73SMilan Broz {
2653784aae73SMilan Broz 	struct mapped_device *md;
2654784aae73SMilan Broz 
26552995fa78SMikulas Patocka 	md = container_of(kobj, struct mapped_device, kobj_holder.kobj);
2656784aae73SMilan Broz 
2657b9a41d21SHou Tao 	spin_lock(&_minor_lock);
2658b9a41d21SHou Tao 	if (test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
2659b9a41d21SHou Tao 		md = NULL;
2660b9a41d21SHou Tao 		goto out;
2661b9a41d21SHou Tao 	}
2662784aae73SMilan Broz 	dm_get(md);
2663b9a41d21SHou Tao out:
2664b9a41d21SHou Tao 	spin_unlock(&_minor_lock);
2665b9a41d21SHou Tao 
2666784aae73SMilan Broz 	return md;
2667784aae73SMilan Broz }
2668784aae73SMilan Broz 
26694f186f8bSKiyoshi Ueda int dm_suspended_md(struct mapped_device *md)
26701da177e4SLinus Torvalds {
26711da177e4SLinus Torvalds 	return test_bit(DMF_SUSPENDED, &md->flags);
26721da177e4SLinus Torvalds }
26731da177e4SLinus Torvalds 
26745df96f2bSMikulas Patocka static int dm_post_suspending_md(struct mapped_device *md)
26755df96f2bSMikulas Patocka {
26765df96f2bSMikulas Patocka 	return test_bit(DMF_POST_SUSPENDING, &md->flags);
26775df96f2bSMikulas Patocka }
26785df96f2bSMikulas Patocka 
2679ffcc3936SMike Snitzer int dm_suspended_internally_md(struct mapped_device *md)
2680ffcc3936SMike Snitzer {
2681ffcc3936SMike Snitzer 	return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2682ffcc3936SMike Snitzer }
2683ffcc3936SMike Snitzer 
26842c140a24SMikulas Patocka int dm_test_deferred_remove_flag(struct mapped_device *md)
26852c140a24SMikulas Patocka {
26862c140a24SMikulas Patocka 	return test_bit(DMF_DEFERRED_REMOVE, &md->flags);
26872c140a24SMikulas Patocka }
26882c140a24SMikulas Patocka 
268964dbce58SKiyoshi Ueda int dm_suspended(struct dm_target *ti)
269064dbce58SKiyoshi Ueda {
269133bd6f06SMike Snitzer 	return dm_suspended_md(ti->table->md);
269264dbce58SKiyoshi Ueda }
269364dbce58SKiyoshi Ueda EXPORT_SYMBOL_GPL(dm_suspended);
269464dbce58SKiyoshi Ueda 
26955df96f2bSMikulas Patocka int dm_post_suspending(struct dm_target *ti)
26965df96f2bSMikulas Patocka {
269733bd6f06SMike Snitzer 	return dm_post_suspending_md(ti->table->md);
26985df96f2bSMikulas Patocka }
26995df96f2bSMikulas Patocka EXPORT_SYMBOL_GPL(dm_post_suspending);
27005df96f2bSMikulas Patocka 
27012e93ccc1SKiyoshi Ueda int dm_noflush_suspending(struct dm_target *ti)
27022e93ccc1SKiyoshi Ueda {
270333bd6f06SMike Snitzer 	return __noflush_suspending(ti->table->md);
27042e93ccc1SKiyoshi Ueda }
27052e93ccc1SKiyoshi Ueda EXPORT_SYMBOL_GPL(dm_noflush_suspending);
27062e93ccc1SKiyoshi Ueda 
27077e0d574fSBart Van Assche struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_queue_mode type,
27080776aa0eSMike Snitzer 					    unsigned integrity, unsigned per_io_data_size,
27090776aa0eSMike Snitzer 					    unsigned min_pool_size)
2710e6ee8c0bSKiyoshi Ueda {
2711115485e8SMike Snitzer 	struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id);
271278d8e58aSMike Snitzer 	unsigned int pool_size = 0;
271364f52b0eSMike Snitzer 	unsigned int front_pad, io_front_pad;
27146f1c819cSKent Overstreet 	int ret;
2715e6ee8c0bSKiyoshi Ueda 
2716e6ee8c0bSKiyoshi Ueda 	if (!pools)
27174e6e36c3SMike Snitzer 		return NULL;
2718e6ee8c0bSKiyoshi Ueda 
271978d8e58aSMike Snitzer 	switch (type) {
272078d8e58aSMike Snitzer 	case DM_TYPE_BIO_BASED:
2721545ed20eSToshi Kani 	case DM_TYPE_DAX_BIO_BASED:
27220776aa0eSMike Snitzer 		pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size);
272362f26317SJeffle Xu 		front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + DM_TARGET_IO_BIO_OFFSET;
272462f26317SJeffle Xu 		io_front_pad = roundup(per_io_data_size,  __alignof__(struct dm_io)) + DM_IO_BIO_OFFSET;
27256f1c819cSKent Overstreet 		ret = bioset_init(&pools->io_bs, pool_size, io_front_pad, 0);
27266f1c819cSKent Overstreet 		if (ret)
272764f52b0eSMike Snitzer 			goto out;
27286f1c819cSKent Overstreet 		if (integrity && bioset_integrity_create(&pools->io_bs, pool_size))
2729eb8db831SChristoph Hellwig 			goto out;
273078d8e58aSMike Snitzer 		break;
273178d8e58aSMike Snitzer 	case DM_TYPE_REQUEST_BASED:
27320776aa0eSMike Snitzer 		pool_size = max(dm_get_reserved_rq_based_ios(), min_pool_size);
273378d8e58aSMike Snitzer 		front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
2734591ddcfcSMike Snitzer 		/* per_io_data_size is used for blk-mq pdu at queue allocation */
273578d8e58aSMike Snitzer 		break;
273678d8e58aSMike Snitzer 	default:
273778d8e58aSMike Snitzer 		BUG();
273878d8e58aSMike Snitzer 	}
273978d8e58aSMike Snitzer 
27406f1c819cSKent Overstreet 	ret = bioset_init(&pools->bs, pool_size, front_pad, 0);
27416f1c819cSKent Overstreet 	if (ret)
27425f015204SJun'ichi Nomura 		goto out;
2743e6ee8c0bSKiyoshi Ueda 
27446f1c819cSKent Overstreet 	if (integrity && bioset_integrity_create(&pools->bs, pool_size))
27455f015204SJun'ichi Nomura 		goto out;
2746a91a2785SMartin K. Petersen 
2747e6ee8c0bSKiyoshi Ueda 	return pools;
274878d8e58aSMike Snitzer 
27495f015204SJun'ichi Nomura out:
27505f015204SJun'ichi Nomura 	dm_free_md_mempools(pools);
2751e6ee8c0bSKiyoshi Ueda 
27524e6e36c3SMike Snitzer 	return NULL;
2753e6ee8c0bSKiyoshi Ueda }
2754e6ee8c0bSKiyoshi Ueda 
2755e6ee8c0bSKiyoshi Ueda void dm_free_md_mempools(struct dm_md_mempools *pools)
2756e6ee8c0bSKiyoshi Ueda {
2757e6ee8c0bSKiyoshi Ueda 	if (!pools)
2758e6ee8c0bSKiyoshi Ueda 		return;
2759e6ee8c0bSKiyoshi Ueda 
27606f1c819cSKent Overstreet 	bioset_exit(&pools->bs);
27616f1c819cSKent Overstreet 	bioset_exit(&pools->io_bs);
2762e6ee8c0bSKiyoshi Ueda 
2763e6ee8c0bSKiyoshi Ueda 	kfree(pools);
2764e6ee8c0bSKiyoshi Ueda }
2765e6ee8c0bSKiyoshi Ueda 
27669c72bad1SChristoph Hellwig struct dm_pr {
27679c72bad1SChristoph Hellwig 	u64	old_key;
27689c72bad1SChristoph Hellwig 	u64	new_key;
27699c72bad1SChristoph Hellwig 	u32	flags;
27709c72bad1SChristoph Hellwig 	bool	fail_early;
27719c72bad1SChristoph Hellwig };
27729c72bad1SChristoph Hellwig 
27739c72bad1SChristoph Hellwig static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn,
27749c72bad1SChristoph Hellwig 		      void *data)
27759c72bad1SChristoph Hellwig {
27769c72bad1SChristoph Hellwig 	struct mapped_device *md = bdev->bd_disk->private_data;
27779c72bad1SChristoph Hellwig 	struct dm_table *table;
27789c72bad1SChristoph Hellwig 	struct dm_target *ti;
27799c72bad1SChristoph Hellwig 	int ret = -ENOTTY, srcu_idx;
27809c72bad1SChristoph Hellwig 
27819c72bad1SChristoph Hellwig 	table = dm_get_live_table(md, &srcu_idx);
27829c72bad1SChristoph Hellwig 	if (!table || !dm_table_get_size(table))
27839c72bad1SChristoph Hellwig 		goto out;
27849c72bad1SChristoph Hellwig 
27859c72bad1SChristoph Hellwig 	/* We only support devices that have a single target */
27869c72bad1SChristoph Hellwig 	if (dm_table_get_num_targets(table) != 1)
27879c72bad1SChristoph Hellwig 		goto out;
27889c72bad1SChristoph Hellwig 	ti = dm_table_get_target(table, 0);
27899c72bad1SChristoph Hellwig 
27909c72bad1SChristoph Hellwig 	ret = -EINVAL;
27919c72bad1SChristoph Hellwig 	if (!ti->type->iterate_devices)
27929c72bad1SChristoph Hellwig 		goto out;
27939c72bad1SChristoph Hellwig 
27949c72bad1SChristoph Hellwig 	ret = ti->type->iterate_devices(ti, fn, data);
27959c72bad1SChristoph Hellwig out:
27969c72bad1SChristoph Hellwig 	dm_put_live_table(md, srcu_idx);
27979c72bad1SChristoph Hellwig 	return ret;
27989c72bad1SChristoph Hellwig }
27999c72bad1SChristoph Hellwig 
28009c72bad1SChristoph Hellwig /*
28019c72bad1SChristoph Hellwig  * For register / unregister we need to manually call out to every path.
28029c72bad1SChristoph Hellwig  */
28039c72bad1SChristoph Hellwig static int __dm_pr_register(struct dm_target *ti, struct dm_dev *dev,
28049c72bad1SChristoph Hellwig 			    sector_t start, sector_t len, void *data)
28059c72bad1SChristoph Hellwig {
28069c72bad1SChristoph Hellwig 	struct dm_pr *pr = data;
28079c72bad1SChristoph Hellwig 	const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
28089c72bad1SChristoph Hellwig 
28099c72bad1SChristoph Hellwig 	if (!ops || !ops->pr_register)
28109c72bad1SChristoph Hellwig 		return -EOPNOTSUPP;
28119c72bad1SChristoph Hellwig 	return ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags);
28129c72bad1SChristoph Hellwig }
28139c72bad1SChristoph Hellwig 
281471cdb697SChristoph Hellwig static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key,
281571cdb697SChristoph Hellwig 			  u32 flags)
281671cdb697SChristoph Hellwig {
28179c72bad1SChristoph Hellwig 	struct dm_pr pr = {
28189c72bad1SChristoph Hellwig 		.old_key	= old_key,
28199c72bad1SChristoph Hellwig 		.new_key	= new_key,
28209c72bad1SChristoph Hellwig 		.flags		= flags,
28219c72bad1SChristoph Hellwig 		.fail_early	= true,
28229c72bad1SChristoph Hellwig 	};
28239c72bad1SChristoph Hellwig 	int ret;
282471cdb697SChristoph Hellwig 
28259c72bad1SChristoph Hellwig 	ret = dm_call_pr(bdev, __dm_pr_register, &pr);
28269c72bad1SChristoph Hellwig 	if (ret && new_key) {
28279c72bad1SChristoph Hellwig 		/* unregister all paths if we failed to register any path */
28289c72bad1SChristoph Hellwig 		pr.old_key = new_key;
28299c72bad1SChristoph Hellwig 		pr.new_key = 0;
28309c72bad1SChristoph Hellwig 		pr.flags = 0;
28319c72bad1SChristoph Hellwig 		pr.fail_early = false;
28329c72bad1SChristoph Hellwig 		dm_call_pr(bdev, __dm_pr_register, &pr);
28339c72bad1SChristoph Hellwig 	}
283471cdb697SChristoph Hellwig 
28359c72bad1SChristoph Hellwig 	return ret;
283671cdb697SChristoph Hellwig }
283771cdb697SChristoph Hellwig 
283871cdb697SChristoph Hellwig static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type,
283971cdb697SChristoph Hellwig 			 u32 flags)
284071cdb697SChristoph Hellwig {
284171cdb697SChristoph Hellwig 	struct mapped_device *md = bdev->bd_disk->private_data;
284271cdb697SChristoph Hellwig 	const struct pr_ops *ops;
2843971888c4SMike Snitzer 	int r, srcu_idx;
284471cdb697SChristoph Hellwig 
28455bd5e8d8SMike Snitzer 	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
284671cdb697SChristoph Hellwig 	if (r < 0)
2847971888c4SMike Snitzer 		goto out;
284871cdb697SChristoph Hellwig 
284971cdb697SChristoph Hellwig 	ops = bdev->bd_disk->fops->pr_ops;
285071cdb697SChristoph Hellwig 	if (ops && ops->pr_reserve)
285171cdb697SChristoph Hellwig 		r = ops->pr_reserve(bdev, key, type, flags);
285271cdb697SChristoph Hellwig 	else
285371cdb697SChristoph Hellwig 		r = -EOPNOTSUPP;
2854971888c4SMike Snitzer out:
2855971888c4SMike Snitzer 	dm_unprepare_ioctl(md, srcu_idx);
285671cdb697SChristoph Hellwig 	return r;
285771cdb697SChristoph Hellwig }
285871cdb697SChristoph Hellwig 
285971cdb697SChristoph Hellwig static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
286071cdb697SChristoph Hellwig {
286171cdb697SChristoph Hellwig 	struct mapped_device *md = bdev->bd_disk->private_data;
286271cdb697SChristoph Hellwig 	const struct pr_ops *ops;
2863971888c4SMike Snitzer 	int r, srcu_idx;
286471cdb697SChristoph Hellwig 
28655bd5e8d8SMike Snitzer 	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
286671cdb697SChristoph Hellwig 	if (r < 0)
2867971888c4SMike Snitzer 		goto out;
286871cdb697SChristoph Hellwig 
286971cdb697SChristoph Hellwig 	ops = bdev->bd_disk->fops->pr_ops;
287071cdb697SChristoph Hellwig 	if (ops && ops->pr_release)
287171cdb697SChristoph Hellwig 		r = ops->pr_release(bdev, key, type);
287271cdb697SChristoph Hellwig 	else
287371cdb697SChristoph Hellwig 		r = -EOPNOTSUPP;
2874971888c4SMike Snitzer out:
2875971888c4SMike Snitzer 	dm_unprepare_ioctl(md, srcu_idx);
287671cdb697SChristoph Hellwig 	return r;
287771cdb697SChristoph Hellwig }
287871cdb697SChristoph Hellwig 
287971cdb697SChristoph Hellwig static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key,
288071cdb697SChristoph Hellwig 			 enum pr_type type, bool abort)
288171cdb697SChristoph Hellwig {
288271cdb697SChristoph Hellwig 	struct mapped_device *md = bdev->bd_disk->private_data;
288371cdb697SChristoph Hellwig 	const struct pr_ops *ops;
2884971888c4SMike Snitzer 	int r, srcu_idx;
288571cdb697SChristoph Hellwig 
28865bd5e8d8SMike Snitzer 	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
288771cdb697SChristoph Hellwig 	if (r < 0)
2888971888c4SMike Snitzer 		goto out;
288971cdb697SChristoph Hellwig 
289071cdb697SChristoph Hellwig 	ops = bdev->bd_disk->fops->pr_ops;
289171cdb697SChristoph Hellwig 	if (ops && ops->pr_preempt)
289271cdb697SChristoph Hellwig 		r = ops->pr_preempt(bdev, old_key, new_key, type, abort);
289371cdb697SChristoph Hellwig 	else
289471cdb697SChristoph Hellwig 		r = -EOPNOTSUPP;
2895971888c4SMike Snitzer out:
2896971888c4SMike Snitzer 	dm_unprepare_ioctl(md, srcu_idx);
289771cdb697SChristoph Hellwig 	return r;
289871cdb697SChristoph Hellwig }
289971cdb697SChristoph Hellwig 
290071cdb697SChristoph Hellwig static int dm_pr_clear(struct block_device *bdev, u64 key)
290171cdb697SChristoph Hellwig {
290271cdb697SChristoph Hellwig 	struct mapped_device *md = bdev->bd_disk->private_data;
290371cdb697SChristoph Hellwig 	const struct pr_ops *ops;
2904971888c4SMike Snitzer 	int r, srcu_idx;
290571cdb697SChristoph Hellwig 
29065bd5e8d8SMike Snitzer 	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
290771cdb697SChristoph Hellwig 	if (r < 0)
2908971888c4SMike Snitzer 		goto out;
290971cdb697SChristoph Hellwig 
291071cdb697SChristoph Hellwig 	ops = bdev->bd_disk->fops->pr_ops;
291171cdb697SChristoph Hellwig 	if (ops && ops->pr_clear)
291271cdb697SChristoph Hellwig 		r = ops->pr_clear(bdev, key);
291371cdb697SChristoph Hellwig 	else
291471cdb697SChristoph Hellwig 		r = -EOPNOTSUPP;
2915971888c4SMike Snitzer out:
2916971888c4SMike Snitzer 	dm_unprepare_ioctl(md, srcu_idx);
291771cdb697SChristoph Hellwig 	return r;
291871cdb697SChristoph Hellwig }
291971cdb697SChristoph Hellwig 
292071cdb697SChristoph Hellwig static const struct pr_ops dm_pr_ops = {
292171cdb697SChristoph Hellwig 	.pr_register	= dm_pr_register,
292271cdb697SChristoph Hellwig 	.pr_reserve	= dm_pr_reserve,
292371cdb697SChristoph Hellwig 	.pr_release	= dm_pr_release,
292471cdb697SChristoph Hellwig 	.pr_preempt	= dm_pr_preempt,
292571cdb697SChristoph Hellwig 	.pr_clear	= dm_pr_clear,
292671cdb697SChristoph Hellwig };
292771cdb697SChristoph Hellwig 
292883d5cde4SAlexey Dobriyan static const struct block_device_operations dm_blk_dops = {
2929c62b37d9SChristoph Hellwig 	.submit_bio = dm_submit_bio,
29301da177e4SLinus Torvalds 	.open = dm_blk_open,
29311da177e4SLinus Torvalds 	.release = dm_blk_close,
2932aa129a22SMilan Broz 	.ioctl = dm_blk_ioctl,
29333ac51e74SDarrick J. Wong 	.getgeo = dm_blk_getgeo,
2934e76239a3SChristoph Hellwig 	.report_zones = dm_blk_report_zones,
293571cdb697SChristoph Hellwig 	.pr_ops = &dm_pr_ops,
29361da177e4SLinus Torvalds 	.owner = THIS_MODULE
29371da177e4SLinus Torvalds };
29381da177e4SLinus Torvalds 
2939681cc5e8SMike Snitzer static const struct block_device_operations dm_rq_blk_dops = {
2940681cc5e8SMike Snitzer 	.open = dm_blk_open,
2941681cc5e8SMike Snitzer 	.release = dm_blk_close,
2942681cc5e8SMike Snitzer 	.ioctl = dm_blk_ioctl,
2943681cc5e8SMike Snitzer 	.getgeo = dm_blk_getgeo,
2944681cc5e8SMike Snitzer 	.pr_ops = &dm_pr_ops,
2945681cc5e8SMike Snitzer 	.owner = THIS_MODULE
2946681cc5e8SMike Snitzer };
2947681cc5e8SMike Snitzer 
2948f26c5719SDan Williams static const struct dax_operations dm_dax_ops = {
2949f26c5719SDan Williams 	.direct_access = dm_dax_direct_access,
2950cdf6cdcdSVivek Goyal 	.zero_page_range = dm_dax_zero_page_range,
2951f26c5719SDan Williams };
2952f26c5719SDan Williams 
29531da177e4SLinus Torvalds /*
29541da177e4SLinus Torvalds  * module hooks
29551da177e4SLinus Torvalds  */
29561da177e4SLinus Torvalds module_init(dm_init);
29571da177e4SLinus Torvalds module_exit(dm_exit);
29581da177e4SLinus Torvalds 
29591da177e4SLinus Torvalds module_param(major, uint, 0);
29601da177e4SLinus Torvalds MODULE_PARM_DESC(major, "The major number of the device mapper");
2961f4790826SMike Snitzer 
2962e8603136SMike Snitzer module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR);
2963e8603136SMike Snitzer MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools");
2964e8603136SMike Snitzer 
2965115485e8SMike Snitzer module_param(dm_numa_node, int, S_IRUGO | S_IWUSR);
2966115485e8SMike Snitzer MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations");
2967115485e8SMike Snitzer 
2968a666e5c0SMikulas Patocka module_param(swap_bios, int, S_IRUGO | S_IWUSR);
2969a666e5c0SMikulas Patocka MODULE_PARM_DESC(swap_bios, "Maximum allowed inflight swap IOs");
2970a666e5c0SMikulas Patocka 
29711da177e4SLinus Torvalds MODULE_DESCRIPTION(DM_NAME " driver");
29721da177e4SLinus Torvalds MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
29731da177e4SLinus Torvalds MODULE_LICENSE("GPL");
2974