xref: /openbmc/linux/drivers/md/dm.c (revision 0bfc24559d7945506184d86739fe365a181f06b7)
11da177e4SLinus Torvalds /*
21da177e4SLinus Torvalds  * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
32b06cfffSAlasdair G Kergon  * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved.
41da177e4SLinus Torvalds  *
51da177e4SLinus Torvalds  * This file is released under the GPL.
61da177e4SLinus Torvalds  */
71da177e4SLinus Torvalds 
81da177e4SLinus Torvalds #include "dm.h"
91da177e4SLinus Torvalds #include "dm-bio-list.h"
1051e5b2bdSMike Anderson #include "dm-uevent.h"
111da177e4SLinus Torvalds 
121da177e4SLinus Torvalds #include <linux/init.h>
131da177e4SLinus Torvalds #include <linux/module.h>
1448c9c27bSArjan van de Ven #include <linux/mutex.h>
151da177e4SLinus Torvalds #include <linux/moduleparam.h>
161da177e4SLinus Torvalds #include <linux/blkpg.h>
171da177e4SLinus Torvalds #include <linux/bio.h>
181da177e4SLinus Torvalds #include <linux/buffer_head.h>
191da177e4SLinus Torvalds #include <linux/mempool.h>
201da177e4SLinus Torvalds #include <linux/slab.h>
211da177e4SLinus Torvalds #include <linux/idr.h>
223ac51e74SDarrick J. Wong #include <linux/hdreg.h>
232056a782SJens Axboe #include <linux/blktrace_api.h>
245f3ea37cSArnaldo Carvalho de Melo #include <trace/block.h>
251da177e4SLinus Torvalds 
2672d94861SAlasdair G Kergon #define DM_MSG_PREFIX "core"
2772d94861SAlasdair G Kergon 
281da177e4SLinus Torvalds static const char *_name = DM_NAME;
291da177e4SLinus Torvalds 
301da177e4SLinus Torvalds static unsigned int major = 0;
311da177e4SLinus Torvalds static unsigned int _major = 0;
321da177e4SLinus Torvalds 
33f32c10b0SJeff Mahoney static DEFINE_SPINLOCK(_minor_lock);
341da177e4SLinus Torvalds /*
351da177e4SLinus Torvalds  * One of these is allocated per bio.
361da177e4SLinus Torvalds  */
371da177e4SLinus Torvalds struct dm_io {
381da177e4SLinus Torvalds 	struct mapped_device *md;
391da177e4SLinus Torvalds 	int error;
401da177e4SLinus Torvalds 	atomic_t io_count;
416ae2fa67SRichard Kennedy 	struct bio *bio;
423eaf840eSJun'ichi "Nick" Nomura 	unsigned long start_time;
431da177e4SLinus Torvalds };
441da177e4SLinus Torvalds 
451da177e4SLinus Torvalds /*
461da177e4SLinus Torvalds  * One of these is allocated per target within a bio.  Hopefully
471da177e4SLinus Torvalds  * this will be simplified out one day.
481da177e4SLinus Torvalds  */
49028867acSAlasdair G Kergon struct dm_target_io {
501da177e4SLinus Torvalds 	struct dm_io *io;
511da177e4SLinus Torvalds 	struct dm_target *ti;
521da177e4SLinus Torvalds 	union map_info info;
531da177e4SLinus Torvalds };
541da177e4SLinus Torvalds 
55*0bfc2455SIngo Molnar DEFINE_TRACE(block_bio_complete);
56*0bfc2455SIngo Molnar 
571da177e4SLinus Torvalds union map_info *dm_get_mapinfo(struct bio *bio)
581da177e4SLinus Torvalds {
591da177e4SLinus Torvalds 	if (bio && bio->bi_private)
60028867acSAlasdair G Kergon 		return &((struct dm_target_io *)bio->bi_private)->info;
611da177e4SLinus Torvalds 	return NULL;
621da177e4SLinus Torvalds }
631da177e4SLinus Torvalds 
64ba61fdd1SJeff Mahoney #define MINOR_ALLOCED ((void *)-1)
65ba61fdd1SJeff Mahoney 
661da177e4SLinus Torvalds /*
671da177e4SLinus Torvalds  * Bits for the md->flags field.
681da177e4SLinus Torvalds  */
691da177e4SLinus Torvalds #define DMF_BLOCK_IO 0
701da177e4SLinus Torvalds #define DMF_SUSPENDED 1
71aa8d7c2fSAlasdair G Kergon #define DMF_FROZEN 2
72fba9f90eSJeff Mahoney #define DMF_FREEING 3
735c6bd75dSAlasdair G Kergon #define DMF_DELETING 4
742e93ccc1SKiyoshi Ueda #define DMF_NOFLUSH_SUSPENDING 5
751da177e4SLinus Torvalds 
76304f3f6aSMilan Broz /*
77304f3f6aSMilan Broz  * Work processed by per-device workqueue.
78304f3f6aSMilan Broz  */
79304f3f6aSMilan Broz struct dm_wq_req {
80304f3f6aSMilan Broz 	enum {
81304f3f6aSMilan Broz 		DM_WQ_FLUSH_DEFERRED,
82304f3f6aSMilan Broz 	} type;
83304f3f6aSMilan Broz 	struct work_struct work;
84304f3f6aSMilan Broz 	struct mapped_device *md;
85304f3f6aSMilan Broz 	void *context;
86304f3f6aSMilan Broz };
87304f3f6aSMilan Broz 
881da177e4SLinus Torvalds struct mapped_device {
892ca3310eSAlasdair G Kergon 	struct rw_semaphore io_lock;
90e61290a4SDaniel Walker 	struct mutex suspend_lock;
912e93ccc1SKiyoshi Ueda 	spinlock_t pushback_lock;
921da177e4SLinus Torvalds 	rwlock_t map_lock;
931da177e4SLinus Torvalds 	atomic_t holders;
945c6bd75dSAlasdair G Kergon 	atomic_t open_count;
951da177e4SLinus Torvalds 
961da177e4SLinus Torvalds 	unsigned long flags;
971da177e4SLinus Torvalds 
98165125e1SJens Axboe 	struct request_queue *queue;
991da177e4SLinus Torvalds 	struct gendisk *disk;
1007e51f257SMike Anderson 	char name[16];
1011da177e4SLinus Torvalds 
1021da177e4SLinus Torvalds 	void *interface_ptr;
1031da177e4SLinus Torvalds 
1041da177e4SLinus Torvalds 	/*
1051da177e4SLinus Torvalds 	 * A list of ios that arrived while we were suspended.
1061da177e4SLinus Torvalds 	 */
1071da177e4SLinus Torvalds 	atomic_t pending;
1081da177e4SLinus Torvalds 	wait_queue_head_t wait;
1091da177e4SLinus Torvalds 	struct bio_list deferred;
1102e93ccc1SKiyoshi Ueda 	struct bio_list pushback;
1111da177e4SLinus Torvalds 
1121da177e4SLinus Torvalds 	/*
113304f3f6aSMilan Broz 	 * Processing queue (flush/barriers)
114304f3f6aSMilan Broz 	 */
115304f3f6aSMilan Broz 	struct workqueue_struct *wq;
116304f3f6aSMilan Broz 
117304f3f6aSMilan Broz 	/*
1181da177e4SLinus Torvalds 	 * The current mapping.
1191da177e4SLinus Torvalds 	 */
1201da177e4SLinus Torvalds 	struct dm_table *map;
1211da177e4SLinus Torvalds 
1221da177e4SLinus Torvalds 	/*
1231da177e4SLinus Torvalds 	 * io objects are allocated from here.
1241da177e4SLinus Torvalds 	 */
1251da177e4SLinus Torvalds 	mempool_t *io_pool;
1261da177e4SLinus Torvalds 	mempool_t *tio_pool;
1271da177e4SLinus Torvalds 
1289faf400fSStefan Bader 	struct bio_set *bs;
1299faf400fSStefan Bader 
1301da177e4SLinus Torvalds 	/*
1311da177e4SLinus Torvalds 	 * Event handling.
1321da177e4SLinus Torvalds 	 */
1331da177e4SLinus Torvalds 	atomic_t event_nr;
1341da177e4SLinus Torvalds 	wait_queue_head_t eventq;
1357a8c3d3bSMike Anderson 	atomic_t uevent_seq;
1367a8c3d3bSMike Anderson 	struct list_head uevent_list;
1377a8c3d3bSMike Anderson 	spinlock_t uevent_lock; /* Protect access to uevent_list */
1381da177e4SLinus Torvalds 
1391da177e4SLinus Torvalds 	/*
1401da177e4SLinus Torvalds 	 * freeze/thaw support require holding onto a super block
1411da177e4SLinus Torvalds 	 */
1421da177e4SLinus Torvalds 	struct super_block *frozen_sb;
143e39e2e95SAlasdair G Kergon 	struct block_device *suspended_bdev;
1443ac51e74SDarrick J. Wong 
1453ac51e74SDarrick J. Wong 	/* forced geometry settings */
1463ac51e74SDarrick J. Wong 	struct hd_geometry geometry;
1471da177e4SLinus Torvalds };
1481da177e4SLinus Torvalds 
1491da177e4SLinus Torvalds #define MIN_IOS 256
150e18b890bSChristoph Lameter static struct kmem_cache *_io_cache;
151e18b890bSChristoph Lameter static struct kmem_cache *_tio_cache;
1521da177e4SLinus Torvalds 
1531da177e4SLinus Torvalds static int __init local_init(void)
1541da177e4SLinus Torvalds {
15551157b4aSKiyoshi Ueda 	int r = -ENOMEM;
1561da177e4SLinus Torvalds 
1571da177e4SLinus Torvalds 	/* allocate a slab for the dm_ios */
158028867acSAlasdair G Kergon 	_io_cache = KMEM_CACHE(dm_io, 0);
1591da177e4SLinus Torvalds 	if (!_io_cache)
16051157b4aSKiyoshi Ueda 		return r;
1611da177e4SLinus Torvalds 
1621da177e4SLinus Torvalds 	/* allocate a slab for the target ios */
163028867acSAlasdair G Kergon 	_tio_cache = KMEM_CACHE(dm_target_io, 0);
16451157b4aSKiyoshi Ueda 	if (!_tio_cache)
16551157b4aSKiyoshi Ueda 		goto out_free_io_cache;
1661da177e4SLinus Torvalds 
16751e5b2bdSMike Anderson 	r = dm_uevent_init();
16851157b4aSKiyoshi Ueda 	if (r)
16951157b4aSKiyoshi Ueda 		goto out_free_tio_cache;
17051e5b2bdSMike Anderson 
1711da177e4SLinus Torvalds 	_major = major;
1721da177e4SLinus Torvalds 	r = register_blkdev(_major, _name);
17351157b4aSKiyoshi Ueda 	if (r < 0)
17451157b4aSKiyoshi Ueda 		goto out_uevent_exit;
1751da177e4SLinus Torvalds 
1761da177e4SLinus Torvalds 	if (!_major)
1771da177e4SLinus Torvalds 		_major = r;
1781da177e4SLinus Torvalds 
1791da177e4SLinus Torvalds 	return 0;
18051157b4aSKiyoshi Ueda 
18151157b4aSKiyoshi Ueda out_uevent_exit:
18251157b4aSKiyoshi Ueda 	dm_uevent_exit();
18351157b4aSKiyoshi Ueda out_free_tio_cache:
18451157b4aSKiyoshi Ueda 	kmem_cache_destroy(_tio_cache);
18551157b4aSKiyoshi Ueda out_free_io_cache:
18651157b4aSKiyoshi Ueda 	kmem_cache_destroy(_io_cache);
18751157b4aSKiyoshi Ueda 
18851157b4aSKiyoshi Ueda 	return r;
1891da177e4SLinus Torvalds }
1901da177e4SLinus Torvalds 
1911da177e4SLinus Torvalds static void local_exit(void)
1921da177e4SLinus Torvalds {
1931da177e4SLinus Torvalds 	kmem_cache_destroy(_tio_cache);
1941da177e4SLinus Torvalds 	kmem_cache_destroy(_io_cache);
19500d59405SAkinobu Mita 	unregister_blkdev(_major, _name);
19651e5b2bdSMike Anderson 	dm_uevent_exit();
1971da177e4SLinus Torvalds 
1981da177e4SLinus Torvalds 	_major = 0;
1991da177e4SLinus Torvalds 
2001da177e4SLinus Torvalds 	DMINFO("cleaned up");
2011da177e4SLinus Torvalds }
2021da177e4SLinus Torvalds 
203b9249e55SAlasdair G Kergon static int (*_inits[])(void) __initdata = {
2041da177e4SLinus Torvalds 	local_init,
2051da177e4SLinus Torvalds 	dm_target_init,
2061da177e4SLinus Torvalds 	dm_linear_init,
2071da177e4SLinus Torvalds 	dm_stripe_init,
208945fa4d2SMikulas Patocka 	dm_kcopyd_init,
2091da177e4SLinus Torvalds 	dm_interface_init,
2101da177e4SLinus Torvalds };
2111da177e4SLinus Torvalds 
212b9249e55SAlasdair G Kergon static void (*_exits[])(void) = {
2131da177e4SLinus Torvalds 	local_exit,
2141da177e4SLinus Torvalds 	dm_target_exit,
2151da177e4SLinus Torvalds 	dm_linear_exit,
2161da177e4SLinus Torvalds 	dm_stripe_exit,
217945fa4d2SMikulas Patocka 	dm_kcopyd_exit,
2181da177e4SLinus Torvalds 	dm_interface_exit,
2191da177e4SLinus Torvalds };
2201da177e4SLinus Torvalds 
2211da177e4SLinus Torvalds static int __init dm_init(void)
2221da177e4SLinus Torvalds {
2231da177e4SLinus Torvalds 	const int count = ARRAY_SIZE(_inits);
2241da177e4SLinus Torvalds 
2251da177e4SLinus Torvalds 	int r, i;
2261da177e4SLinus Torvalds 
2271da177e4SLinus Torvalds 	for (i = 0; i < count; i++) {
2281da177e4SLinus Torvalds 		r = _inits[i]();
2291da177e4SLinus Torvalds 		if (r)
2301da177e4SLinus Torvalds 			goto bad;
2311da177e4SLinus Torvalds 	}
2321da177e4SLinus Torvalds 
2331da177e4SLinus Torvalds 	return 0;
2341da177e4SLinus Torvalds 
2351da177e4SLinus Torvalds       bad:
2361da177e4SLinus Torvalds 	while (i--)
2371da177e4SLinus Torvalds 		_exits[i]();
2381da177e4SLinus Torvalds 
2391da177e4SLinus Torvalds 	return r;
2401da177e4SLinus Torvalds }
2411da177e4SLinus Torvalds 
2421da177e4SLinus Torvalds static void __exit dm_exit(void)
2431da177e4SLinus Torvalds {
2441da177e4SLinus Torvalds 	int i = ARRAY_SIZE(_exits);
2451da177e4SLinus Torvalds 
2461da177e4SLinus Torvalds 	while (i--)
2471da177e4SLinus Torvalds 		_exits[i]();
2481da177e4SLinus Torvalds }
2491da177e4SLinus Torvalds 
2501da177e4SLinus Torvalds /*
2511da177e4SLinus Torvalds  * Block device functions
2521da177e4SLinus Torvalds  */
253fe5f9f2cSAl Viro static int dm_blk_open(struct block_device *bdev, fmode_t mode)
2541da177e4SLinus Torvalds {
2551da177e4SLinus Torvalds 	struct mapped_device *md;
2561da177e4SLinus Torvalds 
257fba9f90eSJeff Mahoney 	spin_lock(&_minor_lock);
258fba9f90eSJeff Mahoney 
259fe5f9f2cSAl Viro 	md = bdev->bd_disk->private_data;
260fba9f90eSJeff Mahoney 	if (!md)
261fba9f90eSJeff Mahoney 		goto out;
262fba9f90eSJeff Mahoney 
2635c6bd75dSAlasdair G Kergon 	if (test_bit(DMF_FREEING, &md->flags) ||
2645c6bd75dSAlasdair G Kergon 	    test_bit(DMF_DELETING, &md->flags)) {
265fba9f90eSJeff Mahoney 		md = NULL;
266fba9f90eSJeff Mahoney 		goto out;
267fba9f90eSJeff Mahoney 	}
268fba9f90eSJeff Mahoney 
2691da177e4SLinus Torvalds 	dm_get(md);
2705c6bd75dSAlasdair G Kergon 	atomic_inc(&md->open_count);
271fba9f90eSJeff Mahoney 
272fba9f90eSJeff Mahoney out:
273fba9f90eSJeff Mahoney 	spin_unlock(&_minor_lock);
274fba9f90eSJeff Mahoney 
275fba9f90eSJeff Mahoney 	return md ? 0 : -ENXIO;
2761da177e4SLinus Torvalds }
2771da177e4SLinus Torvalds 
278fe5f9f2cSAl Viro static int dm_blk_close(struct gendisk *disk, fmode_t mode)
2791da177e4SLinus Torvalds {
280fe5f9f2cSAl Viro 	struct mapped_device *md = disk->private_data;
2815c6bd75dSAlasdair G Kergon 	atomic_dec(&md->open_count);
2821da177e4SLinus Torvalds 	dm_put(md);
2831da177e4SLinus Torvalds 	return 0;
2841da177e4SLinus Torvalds }
2851da177e4SLinus Torvalds 
2865c6bd75dSAlasdair G Kergon int dm_open_count(struct mapped_device *md)
2875c6bd75dSAlasdair G Kergon {
2885c6bd75dSAlasdair G Kergon 	return atomic_read(&md->open_count);
2895c6bd75dSAlasdair G Kergon }
2905c6bd75dSAlasdair G Kergon 
2915c6bd75dSAlasdair G Kergon /*
2925c6bd75dSAlasdair G Kergon  * Guarantees nothing is using the device before it's deleted.
2935c6bd75dSAlasdair G Kergon  */
2945c6bd75dSAlasdair G Kergon int dm_lock_for_deletion(struct mapped_device *md)
2955c6bd75dSAlasdair G Kergon {
2965c6bd75dSAlasdair G Kergon 	int r = 0;
2975c6bd75dSAlasdair G Kergon 
2985c6bd75dSAlasdair G Kergon 	spin_lock(&_minor_lock);
2995c6bd75dSAlasdair G Kergon 
3005c6bd75dSAlasdair G Kergon 	if (dm_open_count(md))
3015c6bd75dSAlasdair G Kergon 		r = -EBUSY;
3025c6bd75dSAlasdair G Kergon 	else
3035c6bd75dSAlasdair G Kergon 		set_bit(DMF_DELETING, &md->flags);
3045c6bd75dSAlasdair G Kergon 
3055c6bd75dSAlasdair G Kergon 	spin_unlock(&_minor_lock);
3065c6bd75dSAlasdair G Kergon 
3075c6bd75dSAlasdair G Kergon 	return r;
3085c6bd75dSAlasdair G Kergon }
3095c6bd75dSAlasdair G Kergon 
3103ac51e74SDarrick J. Wong static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
3113ac51e74SDarrick J. Wong {
3123ac51e74SDarrick J. Wong 	struct mapped_device *md = bdev->bd_disk->private_data;
3133ac51e74SDarrick J. Wong 
3143ac51e74SDarrick J. Wong 	return dm_get_geometry(md, geo);
3153ac51e74SDarrick J. Wong }
3163ac51e74SDarrick J. Wong 
317fe5f9f2cSAl Viro static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
318aa129a22SMilan Broz 			unsigned int cmd, unsigned long arg)
319aa129a22SMilan Broz {
320fe5f9f2cSAl Viro 	struct mapped_device *md = bdev->bd_disk->private_data;
321fe5f9f2cSAl Viro 	struct dm_table *map = dm_get_table(md);
322aa129a22SMilan Broz 	struct dm_target *tgt;
323aa129a22SMilan Broz 	int r = -ENOTTY;
324aa129a22SMilan Broz 
325aa129a22SMilan Broz 	if (!map || !dm_table_get_size(map))
326aa129a22SMilan Broz 		goto out;
327aa129a22SMilan Broz 
328aa129a22SMilan Broz 	/* We only support devices that have a single target */
329aa129a22SMilan Broz 	if (dm_table_get_num_targets(map) != 1)
330aa129a22SMilan Broz 		goto out;
331aa129a22SMilan Broz 
332aa129a22SMilan Broz 	tgt = dm_table_get_target(map, 0);
333aa129a22SMilan Broz 
334aa129a22SMilan Broz 	if (dm_suspended(md)) {
335aa129a22SMilan Broz 		r = -EAGAIN;
336aa129a22SMilan Broz 		goto out;
337aa129a22SMilan Broz 	}
338aa129a22SMilan Broz 
339aa129a22SMilan Broz 	if (tgt->type->ioctl)
340647b3d00SAl Viro 		r = tgt->type->ioctl(tgt, cmd, arg);
341aa129a22SMilan Broz 
342aa129a22SMilan Broz out:
343aa129a22SMilan Broz 	dm_table_put(map);
344aa129a22SMilan Broz 
345aa129a22SMilan Broz 	return r;
346aa129a22SMilan Broz }
347aa129a22SMilan Broz 
348028867acSAlasdair G Kergon static struct dm_io *alloc_io(struct mapped_device *md)
3491da177e4SLinus Torvalds {
3501da177e4SLinus Torvalds 	return mempool_alloc(md->io_pool, GFP_NOIO);
3511da177e4SLinus Torvalds }
3521da177e4SLinus Torvalds 
353028867acSAlasdair G Kergon static void free_io(struct mapped_device *md, struct dm_io *io)
3541da177e4SLinus Torvalds {
3551da177e4SLinus Torvalds 	mempool_free(io, md->io_pool);
3561da177e4SLinus Torvalds }
3571da177e4SLinus Torvalds 
358028867acSAlasdair G Kergon static struct dm_target_io *alloc_tio(struct mapped_device *md)
3591da177e4SLinus Torvalds {
3601da177e4SLinus Torvalds 	return mempool_alloc(md->tio_pool, GFP_NOIO);
3611da177e4SLinus Torvalds }
3621da177e4SLinus Torvalds 
363028867acSAlasdair G Kergon static void free_tio(struct mapped_device *md, struct dm_target_io *tio)
3641da177e4SLinus Torvalds {
3651da177e4SLinus Torvalds 	mempool_free(tio, md->tio_pool);
3661da177e4SLinus Torvalds }
3671da177e4SLinus Torvalds 
3683eaf840eSJun'ichi "Nick" Nomura static void start_io_acct(struct dm_io *io)
3693eaf840eSJun'ichi "Nick" Nomura {
3703eaf840eSJun'ichi "Nick" Nomura 	struct mapped_device *md = io->md;
371c9959059STejun Heo 	int cpu;
3723eaf840eSJun'ichi "Nick" Nomura 
3733eaf840eSJun'ichi "Nick" Nomura 	io->start_time = jiffies;
3743eaf840eSJun'ichi "Nick" Nomura 
375074a7acaSTejun Heo 	cpu = part_stat_lock();
376074a7acaSTejun Heo 	part_round_stats(cpu, &dm_disk(md)->part0);
377074a7acaSTejun Heo 	part_stat_unlock();
378074a7acaSTejun Heo 	dm_disk(md)->part0.in_flight = atomic_inc_return(&md->pending);
3793eaf840eSJun'ichi "Nick" Nomura }
3803eaf840eSJun'ichi "Nick" Nomura 
381d221d2e7SMikulas Patocka static void end_io_acct(struct dm_io *io)
3823eaf840eSJun'ichi "Nick" Nomura {
3833eaf840eSJun'ichi "Nick" Nomura 	struct mapped_device *md = io->md;
3843eaf840eSJun'ichi "Nick" Nomura 	struct bio *bio = io->bio;
3853eaf840eSJun'ichi "Nick" Nomura 	unsigned long duration = jiffies - io->start_time;
386c9959059STejun Heo 	int pending, cpu;
3873eaf840eSJun'ichi "Nick" Nomura 	int rw = bio_data_dir(bio);
3883eaf840eSJun'ichi "Nick" Nomura 
389074a7acaSTejun Heo 	cpu = part_stat_lock();
390074a7acaSTejun Heo 	part_round_stats(cpu, &dm_disk(md)->part0);
391074a7acaSTejun Heo 	part_stat_add(cpu, &dm_disk(md)->part0, ticks[rw], duration);
392074a7acaSTejun Heo 	part_stat_unlock();
3933eaf840eSJun'ichi "Nick" Nomura 
394074a7acaSTejun Heo 	dm_disk(md)->part0.in_flight = pending =
395074a7acaSTejun Heo 		atomic_dec_return(&md->pending);
3963eaf840eSJun'ichi "Nick" Nomura 
397d221d2e7SMikulas Patocka 	/* nudge anyone waiting on suspend queue */
398d221d2e7SMikulas Patocka 	if (!pending)
399d221d2e7SMikulas Patocka 		wake_up(&md->wait);
4003eaf840eSJun'ichi "Nick" Nomura }
4013eaf840eSJun'ichi "Nick" Nomura 
4021da177e4SLinus Torvalds /*
4031da177e4SLinus Torvalds  * Add the bio to the list of deferred io.
4041da177e4SLinus Torvalds  */
4051da177e4SLinus Torvalds static int queue_io(struct mapped_device *md, struct bio *bio)
4061da177e4SLinus Torvalds {
4072ca3310eSAlasdair G Kergon 	down_write(&md->io_lock);
4081da177e4SLinus Torvalds 
4091da177e4SLinus Torvalds 	if (!test_bit(DMF_BLOCK_IO, &md->flags)) {
4102ca3310eSAlasdair G Kergon 		up_write(&md->io_lock);
4111da177e4SLinus Torvalds 		return 1;
4121da177e4SLinus Torvalds 	}
4131da177e4SLinus Torvalds 
4141da177e4SLinus Torvalds 	bio_list_add(&md->deferred, bio);
4151da177e4SLinus Torvalds 
4162ca3310eSAlasdair G Kergon 	up_write(&md->io_lock);
4171da177e4SLinus Torvalds 	return 0;		/* deferred successfully */
4181da177e4SLinus Torvalds }
4191da177e4SLinus Torvalds 
4201da177e4SLinus Torvalds /*
4211da177e4SLinus Torvalds  * Everyone (including functions in this file), should use this
4221da177e4SLinus Torvalds  * function to access the md->map field, and make sure they call
4231da177e4SLinus Torvalds  * dm_table_put() when finished.
4241da177e4SLinus Torvalds  */
4251da177e4SLinus Torvalds struct dm_table *dm_get_table(struct mapped_device *md)
4261da177e4SLinus Torvalds {
4271da177e4SLinus Torvalds 	struct dm_table *t;
4281da177e4SLinus Torvalds 
4291da177e4SLinus Torvalds 	read_lock(&md->map_lock);
4301da177e4SLinus Torvalds 	t = md->map;
4311da177e4SLinus Torvalds 	if (t)
4321da177e4SLinus Torvalds 		dm_table_get(t);
4331da177e4SLinus Torvalds 	read_unlock(&md->map_lock);
4341da177e4SLinus Torvalds 
4351da177e4SLinus Torvalds 	return t;
4361da177e4SLinus Torvalds }
4371da177e4SLinus Torvalds 
4383ac51e74SDarrick J. Wong /*
4393ac51e74SDarrick J. Wong  * Get the geometry associated with a dm device
4403ac51e74SDarrick J. Wong  */
4413ac51e74SDarrick J. Wong int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
4423ac51e74SDarrick J. Wong {
4433ac51e74SDarrick J. Wong 	*geo = md->geometry;
4443ac51e74SDarrick J. Wong 
4453ac51e74SDarrick J. Wong 	return 0;
4463ac51e74SDarrick J. Wong }
4473ac51e74SDarrick J. Wong 
4483ac51e74SDarrick J. Wong /*
4493ac51e74SDarrick J. Wong  * Set the geometry of a device.
4503ac51e74SDarrick J. Wong  */
4513ac51e74SDarrick J. Wong int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
4523ac51e74SDarrick J. Wong {
4533ac51e74SDarrick J. Wong 	sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
4543ac51e74SDarrick J. Wong 
4553ac51e74SDarrick J. Wong 	if (geo->start > sz) {
4563ac51e74SDarrick J. Wong 		DMWARN("Start sector is beyond the geometry limits.");
4573ac51e74SDarrick J. Wong 		return -EINVAL;
4583ac51e74SDarrick J. Wong 	}
4593ac51e74SDarrick J. Wong 
4603ac51e74SDarrick J. Wong 	md->geometry = *geo;
4613ac51e74SDarrick J. Wong 
4623ac51e74SDarrick J. Wong 	return 0;
4633ac51e74SDarrick J. Wong }
4643ac51e74SDarrick J. Wong 
4651da177e4SLinus Torvalds /*-----------------------------------------------------------------
4661da177e4SLinus Torvalds  * CRUD START:
4671da177e4SLinus Torvalds  *   A more elegant soln is in the works that uses the queue
4681da177e4SLinus Torvalds  *   merge fn, unfortunately there are a couple of changes to
4691da177e4SLinus Torvalds  *   the block layer that I want to make for this.  So in the
4701da177e4SLinus Torvalds  *   interests of getting something for people to use I give
4711da177e4SLinus Torvalds  *   you this clearly demarcated crap.
4721da177e4SLinus Torvalds  *---------------------------------------------------------------*/
4731da177e4SLinus Torvalds 
4742e93ccc1SKiyoshi Ueda static int __noflush_suspending(struct mapped_device *md)
4752e93ccc1SKiyoshi Ueda {
4762e93ccc1SKiyoshi Ueda 	return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
4772e93ccc1SKiyoshi Ueda }
4782e93ccc1SKiyoshi Ueda 
4791da177e4SLinus Torvalds /*
4801da177e4SLinus Torvalds  * Decrements the number of outstanding ios that a bio has been
4811da177e4SLinus Torvalds  * cloned into, completing the original io if necc.
4821da177e4SLinus Torvalds  */
483858119e1SArjan van de Ven static void dec_pending(struct dm_io *io, int error)
4841da177e4SLinus Torvalds {
4852e93ccc1SKiyoshi Ueda 	unsigned long flags;
4862e93ccc1SKiyoshi Ueda 
4872e93ccc1SKiyoshi Ueda 	/* Push-back supersedes any I/O errors */
4882e93ccc1SKiyoshi Ueda 	if (error && !(io->error > 0 && __noflush_suspending(io->md)))
4891da177e4SLinus Torvalds 		io->error = error;
4901da177e4SLinus Torvalds 
4911da177e4SLinus Torvalds 	if (atomic_dec_and_test(&io->io_count)) {
4922e93ccc1SKiyoshi Ueda 		if (io->error == DM_ENDIO_REQUEUE) {
4932e93ccc1SKiyoshi Ueda 			/*
4942e93ccc1SKiyoshi Ueda 			 * Target requested pushing back the I/O.
4952e93ccc1SKiyoshi Ueda 			 * This must be handled before the sleeper on
4962e93ccc1SKiyoshi Ueda 			 * suspend queue merges the pushback list.
4972e93ccc1SKiyoshi Ueda 			 */
4982e93ccc1SKiyoshi Ueda 			spin_lock_irqsave(&io->md->pushback_lock, flags);
4992e93ccc1SKiyoshi Ueda 			if (__noflush_suspending(io->md))
5002e93ccc1SKiyoshi Ueda 				bio_list_add(&io->md->pushback, io->bio);
5012e93ccc1SKiyoshi Ueda 			else
5022e93ccc1SKiyoshi Ueda 				/* noflush suspend was interrupted. */
5032e93ccc1SKiyoshi Ueda 				io->error = -EIO;
5042e93ccc1SKiyoshi Ueda 			spin_unlock_irqrestore(&io->md->pushback_lock, flags);
5052e93ccc1SKiyoshi Ueda 		}
5062e93ccc1SKiyoshi Ueda 
507d221d2e7SMikulas Patocka 		end_io_acct(io);
5081da177e4SLinus Torvalds 
5092e93ccc1SKiyoshi Ueda 		if (io->error != DM_ENDIO_REQUEUE) {
5105f3ea37cSArnaldo Carvalho de Melo 			trace_block_bio_complete(io->md->queue, io->bio);
5112056a782SJens Axboe 
5126712ecf8SNeilBrown 			bio_endio(io->bio, io->error);
5132e93ccc1SKiyoshi Ueda 		}
5142e93ccc1SKiyoshi Ueda 
5151da177e4SLinus Torvalds 		free_io(io->md, io);
5161da177e4SLinus Torvalds 	}
5171da177e4SLinus Torvalds }
5181da177e4SLinus Torvalds 
5196712ecf8SNeilBrown static void clone_endio(struct bio *bio, int error)
5201da177e4SLinus Torvalds {
5211da177e4SLinus Torvalds 	int r = 0;
522028867acSAlasdair G Kergon 	struct dm_target_io *tio = bio->bi_private;
5239faf400fSStefan Bader 	struct mapped_device *md = tio->io->md;
5241da177e4SLinus Torvalds 	dm_endio_fn endio = tio->ti->type->end_io;
5251da177e4SLinus Torvalds 
5261da177e4SLinus Torvalds 	if (!bio_flagged(bio, BIO_UPTODATE) && !error)
5271da177e4SLinus Torvalds 		error = -EIO;
5281da177e4SLinus Torvalds 
5291da177e4SLinus Torvalds 	if (endio) {
5301da177e4SLinus Torvalds 		r = endio(tio->ti, bio, error, &tio->info);
5312e93ccc1SKiyoshi Ueda 		if (r < 0 || r == DM_ENDIO_REQUEUE)
5322e93ccc1SKiyoshi Ueda 			/*
5332e93ccc1SKiyoshi Ueda 			 * error and requeue request are handled
5342e93ccc1SKiyoshi Ueda 			 * in dec_pending().
5352e93ccc1SKiyoshi Ueda 			 */
5361da177e4SLinus Torvalds 			error = r;
53745cbcd79SKiyoshi Ueda 		else if (r == DM_ENDIO_INCOMPLETE)
53845cbcd79SKiyoshi Ueda 			/* The target will handle the io */
5396712ecf8SNeilBrown 			return;
54045cbcd79SKiyoshi Ueda 		else if (r) {
54145cbcd79SKiyoshi Ueda 			DMWARN("unimplemented target endio return value: %d", r);
54245cbcd79SKiyoshi Ueda 			BUG();
54345cbcd79SKiyoshi Ueda 		}
5441da177e4SLinus Torvalds 	}
5451da177e4SLinus Torvalds 
5469faf400fSStefan Bader 	dec_pending(tio->io, error);
5479faf400fSStefan Bader 
5489faf400fSStefan Bader 	/*
5499faf400fSStefan Bader 	 * Store md for cleanup instead of tio which is about to get freed.
5509faf400fSStefan Bader 	 */
5519faf400fSStefan Bader 	bio->bi_private = md->bs;
5529faf400fSStefan Bader 
5531da177e4SLinus Torvalds 	bio_put(bio);
5549faf400fSStefan Bader 	free_tio(md, tio);
5551da177e4SLinus Torvalds }
5561da177e4SLinus Torvalds 
5571da177e4SLinus Torvalds static sector_t max_io_len(struct mapped_device *md,
5581da177e4SLinus Torvalds 			   sector_t sector, struct dm_target *ti)
5591da177e4SLinus Torvalds {
5601da177e4SLinus Torvalds 	sector_t offset = sector - ti->begin;
5611da177e4SLinus Torvalds 	sector_t len = ti->len - offset;
5621da177e4SLinus Torvalds 
5631da177e4SLinus Torvalds 	/*
5641da177e4SLinus Torvalds 	 * Does the target need to split even further ?
5651da177e4SLinus Torvalds 	 */
5661da177e4SLinus Torvalds 	if (ti->split_io) {
5671da177e4SLinus Torvalds 		sector_t boundary;
5681da177e4SLinus Torvalds 		boundary = ((offset + ti->split_io) & ~(ti->split_io - 1))
5691da177e4SLinus Torvalds 			   - offset;
5701da177e4SLinus Torvalds 		if (len > boundary)
5711da177e4SLinus Torvalds 			len = boundary;
5721da177e4SLinus Torvalds 	}
5731da177e4SLinus Torvalds 
5741da177e4SLinus Torvalds 	return len;
5751da177e4SLinus Torvalds }
5761da177e4SLinus Torvalds 
5771da177e4SLinus Torvalds static void __map_bio(struct dm_target *ti, struct bio *clone,
578028867acSAlasdair G Kergon 		      struct dm_target_io *tio)
5791da177e4SLinus Torvalds {
5801da177e4SLinus Torvalds 	int r;
5812056a782SJens Axboe 	sector_t sector;
5829faf400fSStefan Bader 	struct mapped_device *md;
5831da177e4SLinus Torvalds 
5841da177e4SLinus Torvalds 	/*
5851da177e4SLinus Torvalds 	 * Sanity checks.
5861da177e4SLinus Torvalds 	 */
5871da177e4SLinus Torvalds 	BUG_ON(!clone->bi_size);
5881da177e4SLinus Torvalds 
5891da177e4SLinus Torvalds 	clone->bi_end_io = clone_endio;
5901da177e4SLinus Torvalds 	clone->bi_private = tio;
5911da177e4SLinus Torvalds 
5921da177e4SLinus Torvalds 	/*
5931da177e4SLinus Torvalds 	 * Map the clone.  If r == 0 we don't need to do
5941da177e4SLinus Torvalds 	 * anything, the target has assumed ownership of
5951da177e4SLinus Torvalds 	 * this io.
5961da177e4SLinus Torvalds 	 */
5971da177e4SLinus Torvalds 	atomic_inc(&tio->io->io_count);
5982056a782SJens Axboe 	sector = clone->bi_sector;
5991da177e4SLinus Torvalds 	r = ti->type->map(ti, clone, &tio->info);
60045cbcd79SKiyoshi Ueda 	if (r == DM_MAPIO_REMAPPED) {
6011da177e4SLinus Torvalds 		/* the bio has been remapped so dispatch it */
6022056a782SJens Axboe 
6035f3ea37cSArnaldo Carvalho de Melo 		trace_block_remap(bdev_get_queue(clone->bi_bdev), clone,
604c7149d6bSAlan D. Brunelle 				    tio->io->bio->bi_bdev->bd_dev,
605c7149d6bSAlan D. Brunelle 				    clone->bi_sector, sector);
6062056a782SJens Axboe 
6071da177e4SLinus Torvalds 		generic_make_request(clone);
6082e93ccc1SKiyoshi Ueda 	} else if (r < 0 || r == DM_MAPIO_REQUEUE) {
6092e93ccc1SKiyoshi Ueda 		/* error the io and bail out, or requeue it if needed */
6109faf400fSStefan Bader 		md = tio->io->md;
6119faf400fSStefan Bader 		dec_pending(tio->io, r);
6129faf400fSStefan Bader 		/*
6139faf400fSStefan Bader 		 * Store bio_set for cleanup.
6149faf400fSStefan Bader 		 */
6159faf400fSStefan Bader 		clone->bi_private = md->bs;
6161da177e4SLinus Torvalds 		bio_put(clone);
6179faf400fSStefan Bader 		free_tio(md, tio);
61845cbcd79SKiyoshi Ueda 	} else if (r) {
61945cbcd79SKiyoshi Ueda 		DMWARN("unimplemented target map return value: %d", r);
62045cbcd79SKiyoshi Ueda 		BUG();
6211da177e4SLinus Torvalds 	}
6221da177e4SLinus Torvalds }
6231da177e4SLinus Torvalds 
6241da177e4SLinus Torvalds struct clone_info {
6251da177e4SLinus Torvalds 	struct mapped_device *md;
6261da177e4SLinus Torvalds 	struct dm_table *map;
6271da177e4SLinus Torvalds 	struct bio *bio;
6281da177e4SLinus Torvalds 	struct dm_io *io;
6291da177e4SLinus Torvalds 	sector_t sector;
6301da177e4SLinus Torvalds 	sector_t sector_count;
6311da177e4SLinus Torvalds 	unsigned short idx;
6321da177e4SLinus Torvalds };
6331da177e4SLinus Torvalds 
6343676347aSPeter Osterlund static void dm_bio_destructor(struct bio *bio)
6353676347aSPeter Osterlund {
6369faf400fSStefan Bader 	struct bio_set *bs = bio->bi_private;
6379faf400fSStefan Bader 
6389faf400fSStefan Bader 	bio_free(bio, bs);
6393676347aSPeter Osterlund }
6403676347aSPeter Osterlund 
6411da177e4SLinus Torvalds /*
6421da177e4SLinus Torvalds  * Creates a little bio that is just does part of a bvec.
6431da177e4SLinus Torvalds  */
6441da177e4SLinus Torvalds static struct bio *split_bvec(struct bio *bio, sector_t sector,
6451da177e4SLinus Torvalds 			      unsigned short idx, unsigned int offset,
6469faf400fSStefan Bader 			      unsigned int len, struct bio_set *bs)
6471da177e4SLinus Torvalds {
6481da177e4SLinus Torvalds 	struct bio *clone;
6491da177e4SLinus Torvalds 	struct bio_vec *bv = bio->bi_io_vec + idx;
6501da177e4SLinus Torvalds 
6519faf400fSStefan Bader 	clone = bio_alloc_bioset(GFP_NOIO, 1, bs);
6523676347aSPeter Osterlund 	clone->bi_destructor = dm_bio_destructor;
6531da177e4SLinus Torvalds 	*clone->bi_io_vec = *bv;
6541da177e4SLinus Torvalds 
6551da177e4SLinus Torvalds 	clone->bi_sector = sector;
6561da177e4SLinus Torvalds 	clone->bi_bdev = bio->bi_bdev;
6571da177e4SLinus Torvalds 	clone->bi_rw = bio->bi_rw;
6581da177e4SLinus Torvalds 	clone->bi_vcnt = 1;
6591da177e4SLinus Torvalds 	clone->bi_size = to_bytes(len);
6601da177e4SLinus Torvalds 	clone->bi_io_vec->bv_offset = offset;
6611da177e4SLinus Torvalds 	clone->bi_io_vec->bv_len = clone->bi_size;
662f3e1d26eSMartin K. Petersen 	clone->bi_flags |= 1 << BIO_CLONED;
6631da177e4SLinus Torvalds 
6641da177e4SLinus Torvalds 	return clone;
6651da177e4SLinus Torvalds }
6661da177e4SLinus Torvalds 
6671da177e4SLinus Torvalds /*
6681da177e4SLinus Torvalds  * Creates a bio that consists of range of complete bvecs.
6691da177e4SLinus Torvalds  */
6701da177e4SLinus Torvalds static struct bio *clone_bio(struct bio *bio, sector_t sector,
6711da177e4SLinus Torvalds 			     unsigned short idx, unsigned short bv_count,
6729faf400fSStefan Bader 			     unsigned int len, struct bio_set *bs)
6731da177e4SLinus Torvalds {
6741da177e4SLinus Torvalds 	struct bio *clone;
6751da177e4SLinus Torvalds 
6769faf400fSStefan Bader 	clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs);
6779faf400fSStefan Bader 	__bio_clone(clone, bio);
6789faf400fSStefan Bader 	clone->bi_destructor = dm_bio_destructor;
6791da177e4SLinus Torvalds 	clone->bi_sector = sector;
6801da177e4SLinus Torvalds 	clone->bi_idx = idx;
6811da177e4SLinus Torvalds 	clone->bi_vcnt = idx + bv_count;
6821da177e4SLinus Torvalds 	clone->bi_size = to_bytes(len);
6831da177e4SLinus Torvalds 	clone->bi_flags &= ~(1 << BIO_SEG_VALID);
6841da177e4SLinus Torvalds 
6851da177e4SLinus Torvalds 	return clone;
6861da177e4SLinus Torvalds }
6871da177e4SLinus Torvalds 
688512875bdSJun'ichi Nomura static int __clone_and_map(struct clone_info *ci)
6891da177e4SLinus Torvalds {
6901da177e4SLinus Torvalds 	struct bio *clone, *bio = ci->bio;
691512875bdSJun'ichi Nomura 	struct dm_target *ti;
692512875bdSJun'ichi Nomura 	sector_t len = 0, max;
693028867acSAlasdair G Kergon 	struct dm_target_io *tio;
6941da177e4SLinus Torvalds 
695512875bdSJun'ichi Nomura 	ti = dm_table_find_target(ci->map, ci->sector);
696512875bdSJun'ichi Nomura 	if (!dm_target_is_valid(ti))
697512875bdSJun'ichi Nomura 		return -EIO;
698512875bdSJun'ichi Nomura 
699512875bdSJun'ichi Nomura 	max = max_io_len(ci->md, ci->sector, ti);
700512875bdSJun'ichi Nomura 
7011da177e4SLinus Torvalds 	/*
7021da177e4SLinus Torvalds 	 * Allocate a target io object.
7031da177e4SLinus Torvalds 	 */
7041da177e4SLinus Torvalds 	tio = alloc_tio(ci->md);
7051da177e4SLinus Torvalds 	tio->io = ci->io;
7061da177e4SLinus Torvalds 	tio->ti = ti;
7071da177e4SLinus Torvalds 	memset(&tio->info, 0, sizeof(tio->info));
7081da177e4SLinus Torvalds 
7091da177e4SLinus Torvalds 	if (ci->sector_count <= max) {
7101da177e4SLinus Torvalds 		/*
7111da177e4SLinus Torvalds 		 * Optimise for the simple case where we can do all of
7121da177e4SLinus Torvalds 		 * the remaining io with a single clone.
7131da177e4SLinus Torvalds 		 */
7141da177e4SLinus Torvalds 		clone = clone_bio(bio, ci->sector, ci->idx,
7159faf400fSStefan Bader 				  bio->bi_vcnt - ci->idx, ci->sector_count,
7169faf400fSStefan Bader 				  ci->md->bs);
7171da177e4SLinus Torvalds 		__map_bio(ti, clone, tio);
7181da177e4SLinus Torvalds 		ci->sector_count = 0;
7191da177e4SLinus Torvalds 
7201da177e4SLinus Torvalds 	} else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) {
7211da177e4SLinus Torvalds 		/*
7221da177e4SLinus Torvalds 		 * There are some bvecs that don't span targets.
7231da177e4SLinus Torvalds 		 * Do as many of these as possible.
7241da177e4SLinus Torvalds 		 */
7251da177e4SLinus Torvalds 		int i;
7261da177e4SLinus Torvalds 		sector_t remaining = max;
7271da177e4SLinus Torvalds 		sector_t bv_len;
7281da177e4SLinus Torvalds 
7291da177e4SLinus Torvalds 		for (i = ci->idx; remaining && (i < bio->bi_vcnt); i++) {
7301da177e4SLinus Torvalds 			bv_len = to_sector(bio->bi_io_vec[i].bv_len);
7311da177e4SLinus Torvalds 
7321da177e4SLinus Torvalds 			if (bv_len > remaining)
7331da177e4SLinus Torvalds 				break;
7341da177e4SLinus Torvalds 
7351da177e4SLinus Torvalds 			remaining -= bv_len;
7361da177e4SLinus Torvalds 			len += bv_len;
7371da177e4SLinus Torvalds 		}
7381da177e4SLinus Torvalds 
7399faf400fSStefan Bader 		clone = clone_bio(bio, ci->sector, ci->idx, i - ci->idx, len,
7409faf400fSStefan Bader 				  ci->md->bs);
7411da177e4SLinus Torvalds 		__map_bio(ti, clone, tio);
7421da177e4SLinus Torvalds 
7431da177e4SLinus Torvalds 		ci->sector += len;
7441da177e4SLinus Torvalds 		ci->sector_count -= len;
7451da177e4SLinus Torvalds 		ci->idx = i;
7461da177e4SLinus Torvalds 
7471da177e4SLinus Torvalds 	} else {
7481da177e4SLinus Torvalds 		/*
749d2044a94SAlasdair G Kergon 		 * Handle a bvec that must be split between two or more targets.
7501da177e4SLinus Torvalds 		 */
7511da177e4SLinus Torvalds 		struct bio_vec *bv = bio->bi_io_vec + ci->idx;
752d2044a94SAlasdair G Kergon 		sector_t remaining = to_sector(bv->bv_len);
753d2044a94SAlasdair G Kergon 		unsigned int offset = 0;
7541da177e4SLinus Torvalds 
755d2044a94SAlasdair G Kergon 		do {
756d2044a94SAlasdair G Kergon 			if (offset) {
7571da177e4SLinus Torvalds 				ti = dm_table_find_target(ci->map, ci->sector);
758512875bdSJun'ichi Nomura 				if (!dm_target_is_valid(ti))
759512875bdSJun'ichi Nomura 					return -EIO;
760512875bdSJun'ichi Nomura 
761d2044a94SAlasdair G Kergon 				max = max_io_len(ci->md, ci->sector, ti);
7621da177e4SLinus Torvalds 
7631da177e4SLinus Torvalds 				tio = alloc_tio(ci->md);
7641da177e4SLinus Torvalds 				tio->io = ci->io;
7651da177e4SLinus Torvalds 				tio->ti = ti;
7661da177e4SLinus Torvalds 				memset(&tio->info, 0, sizeof(tio->info));
767d2044a94SAlasdair G Kergon 			}
768d2044a94SAlasdair G Kergon 
769d2044a94SAlasdair G Kergon 			len = min(remaining, max);
770d2044a94SAlasdair G Kergon 
771d2044a94SAlasdair G Kergon 			clone = split_bvec(bio, ci->sector, ci->idx,
7729faf400fSStefan Bader 					   bv->bv_offset + offset, len,
7739faf400fSStefan Bader 					   ci->md->bs);
774d2044a94SAlasdair G Kergon 
7751da177e4SLinus Torvalds 			__map_bio(ti, clone, tio);
7761da177e4SLinus Torvalds 
7771da177e4SLinus Torvalds 			ci->sector += len;
7781da177e4SLinus Torvalds 			ci->sector_count -= len;
779d2044a94SAlasdair G Kergon 			offset += to_bytes(len);
780d2044a94SAlasdair G Kergon 		} while (remaining -= len);
781d2044a94SAlasdair G Kergon 
7821da177e4SLinus Torvalds 		ci->idx++;
7831da177e4SLinus Torvalds 	}
784512875bdSJun'ichi Nomura 
785512875bdSJun'ichi Nomura 	return 0;
7861da177e4SLinus Torvalds }
7871da177e4SLinus Torvalds 
7881da177e4SLinus Torvalds /*
7891da177e4SLinus Torvalds  * Split the bio into several clones.
7901da177e4SLinus Torvalds  */
7919e4e5f87SMilan Broz static int __split_bio(struct mapped_device *md, struct bio *bio)
7921da177e4SLinus Torvalds {
7931da177e4SLinus Torvalds 	struct clone_info ci;
794512875bdSJun'ichi Nomura 	int error = 0;
7951da177e4SLinus Torvalds 
7961da177e4SLinus Torvalds 	ci.map = dm_get_table(md);
7979e4e5f87SMilan Broz 	if (unlikely(!ci.map))
7989e4e5f87SMilan Broz 		return -EIO;
7991da177e4SLinus Torvalds 
8001da177e4SLinus Torvalds 	ci.md = md;
8011da177e4SLinus Torvalds 	ci.bio = bio;
8021da177e4SLinus Torvalds 	ci.io = alloc_io(md);
8031da177e4SLinus Torvalds 	ci.io->error = 0;
8041da177e4SLinus Torvalds 	atomic_set(&ci.io->io_count, 1);
8051da177e4SLinus Torvalds 	ci.io->bio = bio;
8061da177e4SLinus Torvalds 	ci.io->md = md;
8071da177e4SLinus Torvalds 	ci.sector = bio->bi_sector;
8081da177e4SLinus Torvalds 	ci.sector_count = bio_sectors(bio);
8091da177e4SLinus Torvalds 	ci.idx = bio->bi_idx;
8101da177e4SLinus Torvalds 
8113eaf840eSJun'ichi "Nick" Nomura 	start_io_acct(ci.io);
812512875bdSJun'ichi Nomura 	while (ci.sector_count && !error)
813512875bdSJun'ichi Nomura 		error = __clone_and_map(&ci);
8141da177e4SLinus Torvalds 
8151da177e4SLinus Torvalds 	/* drop the extra reference count */
816512875bdSJun'ichi Nomura 	dec_pending(ci.io, error);
8171da177e4SLinus Torvalds 	dm_table_put(ci.map);
8189e4e5f87SMilan Broz 
8199e4e5f87SMilan Broz 	return 0;
8201da177e4SLinus Torvalds }
8211da177e4SLinus Torvalds /*-----------------------------------------------------------------
8221da177e4SLinus Torvalds  * CRUD END
8231da177e4SLinus Torvalds  *---------------------------------------------------------------*/
8241da177e4SLinus Torvalds 
825f6fccb12SMilan Broz static int dm_merge_bvec(struct request_queue *q,
826f6fccb12SMilan Broz 			 struct bvec_merge_data *bvm,
827f6fccb12SMilan Broz 			 struct bio_vec *biovec)
828f6fccb12SMilan Broz {
829f6fccb12SMilan Broz 	struct mapped_device *md = q->queuedata;
830f6fccb12SMilan Broz 	struct dm_table *map = dm_get_table(md);
831f6fccb12SMilan Broz 	struct dm_target *ti;
832f6fccb12SMilan Broz 	sector_t max_sectors;
8335037108aSMikulas Patocka 	int max_size = 0;
834f6fccb12SMilan Broz 
835f6fccb12SMilan Broz 	if (unlikely(!map))
8365037108aSMikulas Patocka 		goto out;
837f6fccb12SMilan Broz 
838f6fccb12SMilan Broz 	ti = dm_table_find_target(map, bvm->bi_sector);
839b01cd5acSMikulas Patocka 	if (!dm_target_is_valid(ti))
840b01cd5acSMikulas Patocka 		goto out_table;
841f6fccb12SMilan Broz 
842f6fccb12SMilan Broz 	/*
843f6fccb12SMilan Broz 	 * Find maximum amount of I/O that won't need splitting
844f6fccb12SMilan Broz 	 */
845f6fccb12SMilan Broz 	max_sectors = min(max_io_len(md, bvm->bi_sector, ti),
846f6fccb12SMilan Broz 			  (sector_t) BIO_MAX_SECTORS);
847f6fccb12SMilan Broz 	max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size;
848f6fccb12SMilan Broz 	if (max_size < 0)
849f6fccb12SMilan Broz 		max_size = 0;
850f6fccb12SMilan Broz 
851f6fccb12SMilan Broz 	/*
852f6fccb12SMilan Broz 	 * merge_bvec_fn() returns number of bytes
853f6fccb12SMilan Broz 	 * it can accept at this offset
854f6fccb12SMilan Broz 	 * max is precomputed maximal io size
855f6fccb12SMilan Broz 	 */
856f6fccb12SMilan Broz 	if (max_size && ti->type->merge)
857f6fccb12SMilan Broz 		max_size = ti->type->merge(ti, bvm, biovec, max_size);
858f6fccb12SMilan Broz 
859b01cd5acSMikulas Patocka out_table:
8605037108aSMikulas Patocka 	dm_table_put(map);
8615037108aSMikulas Patocka 
8625037108aSMikulas Patocka out:
863f6fccb12SMilan Broz 	/*
864f6fccb12SMilan Broz 	 * Always allow an entire first page
865f6fccb12SMilan Broz 	 */
866f6fccb12SMilan Broz 	if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT))
867f6fccb12SMilan Broz 		max_size = biovec->bv_len;
868f6fccb12SMilan Broz 
869f6fccb12SMilan Broz 	return max_size;
870f6fccb12SMilan Broz }
871f6fccb12SMilan Broz 
8721da177e4SLinus Torvalds /*
8731da177e4SLinus Torvalds  * The request function that just remaps the bio built up by
8741da177e4SLinus Torvalds  * dm_merge_bvec.
8751da177e4SLinus Torvalds  */
876165125e1SJens Axboe static int dm_request(struct request_queue *q, struct bio *bio)
8771da177e4SLinus Torvalds {
8789e4e5f87SMilan Broz 	int r = -EIO;
87912f03a49SKevin Corry 	int rw = bio_data_dir(bio);
8801da177e4SLinus Torvalds 	struct mapped_device *md = q->queuedata;
881c9959059STejun Heo 	int cpu;
8821da177e4SLinus Torvalds 
88307a83c47SStefan Bader 	/*
88407a83c47SStefan Bader 	 * There is no use in forwarding any barrier request since we can't
88507a83c47SStefan Bader 	 * guarantee it is (or can be) handled by the targets correctly.
88607a83c47SStefan Bader 	 */
88707a83c47SStefan Bader 	if (unlikely(bio_barrier(bio))) {
8886712ecf8SNeilBrown 		bio_endio(bio, -EOPNOTSUPP);
88907a83c47SStefan Bader 		return 0;
89007a83c47SStefan Bader 	}
89107a83c47SStefan Bader 
8922ca3310eSAlasdair G Kergon 	down_read(&md->io_lock);
8931da177e4SLinus Torvalds 
894074a7acaSTejun Heo 	cpu = part_stat_lock();
895074a7acaSTejun Heo 	part_stat_inc(cpu, &dm_disk(md)->part0, ios[rw]);
896074a7acaSTejun Heo 	part_stat_add(cpu, &dm_disk(md)->part0, sectors[rw], bio_sectors(bio));
897074a7acaSTejun Heo 	part_stat_unlock();
89812f03a49SKevin Corry 
8991da177e4SLinus Torvalds 	/*
9001da177e4SLinus Torvalds 	 * If we're suspended we have to queue
9011da177e4SLinus Torvalds 	 * this io for later.
9021da177e4SLinus Torvalds 	 */
9031da177e4SLinus Torvalds 	while (test_bit(DMF_BLOCK_IO, &md->flags)) {
9042ca3310eSAlasdair G Kergon 		up_read(&md->io_lock);
9051da177e4SLinus Torvalds 
9069e4e5f87SMilan Broz 		if (bio_rw(bio) != READA)
9071da177e4SLinus Torvalds 			r = queue_io(md, bio);
9081da177e4SLinus Torvalds 
9099e4e5f87SMilan Broz 		if (r <= 0)
9109e4e5f87SMilan Broz 			goto out_req;
9111da177e4SLinus Torvalds 
9121da177e4SLinus Torvalds 		/*
9131da177e4SLinus Torvalds 		 * We're in a while loop, because someone could suspend
9141da177e4SLinus Torvalds 		 * before we get to the following read lock.
9151da177e4SLinus Torvalds 		 */
9162ca3310eSAlasdair G Kergon 		down_read(&md->io_lock);
9171da177e4SLinus Torvalds 	}
9181da177e4SLinus Torvalds 
9199e4e5f87SMilan Broz 	r = __split_bio(md, bio);
9202ca3310eSAlasdair G Kergon 	up_read(&md->io_lock);
9219e4e5f87SMilan Broz 
9229e4e5f87SMilan Broz out_req:
9239e4e5f87SMilan Broz 	if (r < 0)
9249e4e5f87SMilan Broz 		bio_io_error(bio);
9259e4e5f87SMilan Broz 
9261da177e4SLinus Torvalds 	return 0;
9271da177e4SLinus Torvalds }
9281da177e4SLinus Torvalds 
929165125e1SJens Axboe static void dm_unplug_all(struct request_queue *q)
9301da177e4SLinus Torvalds {
9311da177e4SLinus Torvalds 	struct mapped_device *md = q->queuedata;
9321da177e4SLinus Torvalds 	struct dm_table *map = dm_get_table(md);
9331da177e4SLinus Torvalds 
9341da177e4SLinus Torvalds 	if (map) {
9351da177e4SLinus Torvalds 		dm_table_unplug_all(map);
9361da177e4SLinus Torvalds 		dm_table_put(map);
9371da177e4SLinus Torvalds 	}
9381da177e4SLinus Torvalds }
9391da177e4SLinus Torvalds 
9401da177e4SLinus Torvalds static int dm_any_congested(void *congested_data, int bdi_bits)
9411da177e4SLinus Torvalds {
9428a57dfc6SChandra Seetharaman 	int r = bdi_bits;
9438a57dfc6SChandra Seetharaman 	struct mapped_device *md = congested_data;
9448a57dfc6SChandra Seetharaman 	struct dm_table *map;
9451da177e4SLinus Torvalds 
9468a57dfc6SChandra Seetharaman 	atomic_inc(&md->pending);
9478a57dfc6SChandra Seetharaman 
9488a57dfc6SChandra Seetharaman 	if (!test_bit(DMF_BLOCK_IO, &md->flags)) {
9498a57dfc6SChandra Seetharaman 		map = dm_get_table(md);
9508a57dfc6SChandra Seetharaman 		if (map) {
9511da177e4SLinus Torvalds 			r = dm_table_any_congested(map, bdi_bits);
9521da177e4SLinus Torvalds 			dm_table_put(map);
9538a57dfc6SChandra Seetharaman 		}
9548a57dfc6SChandra Seetharaman 	}
9558a57dfc6SChandra Seetharaman 
9568a57dfc6SChandra Seetharaman 	if (!atomic_dec_return(&md->pending))
9578a57dfc6SChandra Seetharaman 		/* nudge anyone waiting on suspend queue */
9588a57dfc6SChandra Seetharaman 		wake_up(&md->wait);
9598a57dfc6SChandra Seetharaman 
9601da177e4SLinus Torvalds 	return r;
9611da177e4SLinus Torvalds }
9621da177e4SLinus Torvalds 
9631da177e4SLinus Torvalds /*-----------------------------------------------------------------
9641da177e4SLinus Torvalds  * An IDR is used to keep track of allocated minor numbers.
9651da177e4SLinus Torvalds  *---------------------------------------------------------------*/
9661da177e4SLinus Torvalds static DEFINE_IDR(_minor_idr);
9671da177e4SLinus Torvalds 
9682b06cfffSAlasdair G Kergon static void free_minor(int minor)
9691da177e4SLinus Torvalds {
970f32c10b0SJeff Mahoney 	spin_lock(&_minor_lock);
9711da177e4SLinus Torvalds 	idr_remove(&_minor_idr, minor);
972f32c10b0SJeff Mahoney 	spin_unlock(&_minor_lock);
9731da177e4SLinus Torvalds }
9741da177e4SLinus Torvalds 
9751da177e4SLinus Torvalds /*
9761da177e4SLinus Torvalds  * See if the device with a specific minor # is free.
9771da177e4SLinus Torvalds  */
978cf13ab8eSFrederik Deweerdt static int specific_minor(int minor)
9791da177e4SLinus Torvalds {
9801da177e4SLinus Torvalds 	int r, m;
9811da177e4SLinus Torvalds 
9821da177e4SLinus Torvalds 	if (minor >= (1 << MINORBITS))
9831da177e4SLinus Torvalds 		return -EINVAL;
9841da177e4SLinus Torvalds 
98562f75c2fSJeff Mahoney 	r = idr_pre_get(&_minor_idr, GFP_KERNEL);
98662f75c2fSJeff Mahoney 	if (!r)
98762f75c2fSJeff Mahoney 		return -ENOMEM;
98862f75c2fSJeff Mahoney 
989f32c10b0SJeff Mahoney 	spin_lock(&_minor_lock);
9901da177e4SLinus Torvalds 
9911da177e4SLinus Torvalds 	if (idr_find(&_minor_idr, minor)) {
9921da177e4SLinus Torvalds 		r = -EBUSY;
9931da177e4SLinus Torvalds 		goto out;
9941da177e4SLinus Torvalds 	}
9951da177e4SLinus Torvalds 
996ba61fdd1SJeff Mahoney 	r = idr_get_new_above(&_minor_idr, MINOR_ALLOCED, minor, &m);
99762f75c2fSJeff Mahoney 	if (r)
9981da177e4SLinus Torvalds 		goto out;
9991da177e4SLinus Torvalds 
10001da177e4SLinus Torvalds 	if (m != minor) {
10011da177e4SLinus Torvalds 		idr_remove(&_minor_idr, m);
10021da177e4SLinus Torvalds 		r = -EBUSY;
10031da177e4SLinus Torvalds 		goto out;
10041da177e4SLinus Torvalds 	}
10051da177e4SLinus Torvalds 
10061da177e4SLinus Torvalds out:
1007f32c10b0SJeff Mahoney 	spin_unlock(&_minor_lock);
10081da177e4SLinus Torvalds 	return r;
10091da177e4SLinus Torvalds }
10101da177e4SLinus Torvalds 
1011cf13ab8eSFrederik Deweerdt static int next_free_minor(int *minor)
10121da177e4SLinus Torvalds {
10132b06cfffSAlasdair G Kergon 	int r, m;
10141da177e4SLinus Torvalds 
10151da177e4SLinus Torvalds 	r = idr_pre_get(&_minor_idr, GFP_KERNEL);
101662f75c2fSJeff Mahoney 	if (!r)
101762f75c2fSJeff Mahoney 		return -ENOMEM;
101862f75c2fSJeff Mahoney 
1019f32c10b0SJeff Mahoney 	spin_lock(&_minor_lock);
10201da177e4SLinus Torvalds 
1021ba61fdd1SJeff Mahoney 	r = idr_get_new(&_minor_idr, MINOR_ALLOCED, &m);
1022cf13ab8eSFrederik Deweerdt 	if (r)
10231da177e4SLinus Torvalds 		goto out;
10241da177e4SLinus Torvalds 
10251da177e4SLinus Torvalds 	if (m >= (1 << MINORBITS)) {
10261da177e4SLinus Torvalds 		idr_remove(&_minor_idr, m);
10271da177e4SLinus Torvalds 		r = -ENOSPC;
10281da177e4SLinus Torvalds 		goto out;
10291da177e4SLinus Torvalds 	}
10301da177e4SLinus Torvalds 
10311da177e4SLinus Torvalds 	*minor = m;
10321da177e4SLinus Torvalds 
10331da177e4SLinus Torvalds out:
1034f32c10b0SJeff Mahoney 	spin_unlock(&_minor_lock);
10351da177e4SLinus Torvalds 	return r;
10361da177e4SLinus Torvalds }
10371da177e4SLinus Torvalds 
10381da177e4SLinus Torvalds static struct block_device_operations dm_blk_dops;
10391da177e4SLinus Torvalds 
10401da177e4SLinus Torvalds /*
10411da177e4SLinus Torvalds  * Allocate and initialise a blank device with a given minor.
10421da177e4SLinus Torvalds  */
10432b06cfffSAlasdair G Kergon static struct mapped_device *alloc_dev(int minor)
10441da177e4SLinus Torvalds {
10451da177e4SLinus Torvalds 	int r;
1046cf13ab8eSFrederik Deweerdt 	struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL);
1047ba61fdd1SJeff Mahoney 	void *old_md;
10481da177e4SLinus Torvalds 
10491da177e4SLinus Torvalds 	if (!md) {
10501da177e4SLinus Torvalds 		DMWARN("unable to allocate device, out of memory.");
10511da177e4SLinus Torvalds 		return NULL;
10521da177e4SLinus Torvalds 	}
10531da177e4SLinus Torvalds 
105410da4f79SJeff Mahoney 	if (!try_module_get(THIS_MODULE))
10556ed7ade8SMilan Broz 		goto bad_module_get;
105610da4f79SJeff Mahoney 
10571da177e4SLinus Torvalds 	/* get a minor number for the dev */
10582b06cfffSAlasdair G Kergon 	if (minor == DM_ANY_MINOR)
1059cf13ab8eSFrederik Deweerdt 		r = next_free_minor(&minor);
10602b06cfffSAlasdair G Kergon 	else
1061cf13ab8eSFrederik Deweerdt 		r = specific_minor(minor);
10621da177e4SLinus Torvalds 	if (r < 0)
10636ed7ade8SMilan Broz 		goto bad_minor;
10641da177e4SLinus Torvalds 
10652ca3310eSAlasdair G Kergon 	init_rwsem(&md->io_lock);
1066e61290a4SDaniel Walker 	mutex_init(&md->suspend_lock);
10672e93ccc1SKiyoshi Ueda 	spin_lock_init(&md->pushback_lock);
10681da177e4SLinus Torvalds 	rwlock_init(&md->map_lock);
10691da177e4SLinus Torvalds 	atomic_set(&md->holders, 1);
10705c6bd75dSAlasdair G Kergon 	atomic_set(&md->open_count, 0);
10711da177e4SLinus Torvalds 	atomic_set(&md->event_nr, 0);
10727a8c3d3bSMike Anderson 	atomic_set(&md->uevent_seq, 0);
10737a8c3d3bSMike Anderson 	INIT_LIST_HEAD(&md->uevent_list);
10747a8c3d3bSMike Anderson 	spin_lock_init(&md->uevent_lock);
10751da177e4SLinus Torvalds 
10761da177e4SLinus Torvalds 	md->queue = blk_alloc_queue(GFP_KERNEL);
10771da177e4SLinus Torvalds 	if (!md->queue)
10786ed7ade8SMilan Broz 		goto bad_queue;
10791da177e4SLinus Torvalds 
10801da177e4SLinus Torvalds 	md->queue->queuedata = md;
10811da177e4SLinus Torvalds 	md->queue->backing_dev_info.congested_fn = dm_any_congested;
10821da177e4SLinus Torvalds 	md->queue->backing_dev_info.congested_data = md;
10831da177e4SLinus Torvalds 	blk_queue_make_request(md->queue, dm_request);
1084daef265fSJens Axboe 	blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
10851da177e4SLinus Torvalds 	md->queue->unplug_fn = dm_unplug_all;
1086f6fccb12SMilan Broz 	blk_queue_merge_bvec(md->queue, dm_merge_bvec);
10871da177e4SLinus Torvalds 
108893d2341cSMatthew Dobson 	md->io_pool = mempool_create_slab_pool(MIN_IOS, _io_cache);
10891da177e4SLinus Torvalds 	if (!md->io_pool)
10906ed7ade8SMilan Broz 		goto bad_io_pool;
10911da177e4SLinus Torvalds 
109293d2341cSMatthew Dobson 	md->tio_pool = mempool_create_slab_pool(MIN_IOS, _tio_cache);
10931da177e4SLinus Torvalds 	if (!md->tio_pool)
10946ed7ade8SMilan Broz 		goto bad_tio_pool;
10951da177e4SLinus Torvalds 
10965972511bSJens Axboe 	md->bs = bioset_create(16, 16);
10979faf400fSStefan Bader 	if (!md->bs)
10989faf400fSStefan Bader 		goto bad_no_bioset;
10999faf400fSStefan Bader 
11001da177e4SLinus Torvalds 	md->disk = alloc_disk(1);
11011da177e4SLinus Torvalds 	if (!md->disk)
11026ed7ade8SMilan Broz 		goto bad_disk;
11031da177e4SLinus Torvalds 
1104f0b04115SJeff Mahoney 	atomic_set(&md->pending, 0);
1105f0b04115SJeff Mahoney 	init_waitqueue_head(&md->wait);
1106f0b04115SJeff Mahoney 	init_waitqueue_head(&md->eventq);
1107f0b04115SJeff Mahoney 
11081da177e4SLinus Torvalds 	md->disk->major = _major;
11091da177e4SLinus Torvalds 	md->disk->first_minor = minor;
11101da177e4SLinus Torvalds 	md->disk->fops = &dm_blk_dops;
11111da177e4SLinus Torvalds 	md->disk->queue = md->queue;
11121da177e4SLinus Torvalds 	md->disk->private_data = md;
11131da177e4SLinus Torvalds 	sprintf(md->disk->disk_name, "dm-%d", minor);
11141da177e4SLinus Torvalds 	add_disk(md->disk);
11157e51f257SMike Anderson 	format_dev_t(md->name, MKDEV(_major, minor));
11161da177e4SLinus Torvalds 
1117304f3f6aSMilan Broz 	md->wq = create_singlethread_workqueue("kdmflush");
1118304f3f6aSMilan Broz 	if (!md->wq)
1119304f3f6aSMilan Broz 		goto bad_thread;
1120304f3f6aSMilan Broz 
1121ba61fdd1SJeff Mahoney 	/* Populate the mapping, nobody knows we exist yet */
1122f32c10b0SJeff Mahoney 	spin_lock(&_minor_lock);
1123ba61fdd1SJeff Mahoney 	old_md = idr_replace(&_minor_idr, md, minor);
1124f32c10b0SJeff Mahoney 	spin_unlock(&_minor_lock);
1125ba61fdd1SJeff Mahoney 
1126ba61fdd1SJeff Mahoney 	BUG_ON(old_md != MINOR_ALLOCED);
1127ba61fdd1SJeff Mahoney 
11281da177e4SLinus Torvalds 	return md;
11291da177e4SLinus Torvalds 
1130304f3f6aSMilan Broz bad_thread:
1131304f3f6aSMilan Broz 	put_disk(md->disk);
11326ed7ade8SMilan Broz bad_disk:
11339faf400fSStefan Bader 	bioset_free(md->bs);
11349faf400fSStefan Bader bad_no_bioset:
11351da177e4SLinus Torvalds 	mempool_destroy(md->tio_pool);
11366ed7ade8SMilan Broz bad_tio_pool:
11371da177e4SLinus Torvalds 	mempool_destroy(md->io_pool);
11386ed7ade8SMilan Broz bad_io_pool:
11391312f40eSAl Viro 	blk_cleanup_queue(md->queue);
11406ed7ade8SMilan Broz bad_queue:
11411da177e4SLinus Torvalds 	free_minor(minor);
11426ed7ade8SMilan Broz bad_minor:
114310da4f79SJeff Mahoney 	module_put(THIS_MODULE);
11446ed7ade8SMilan Broz bad_module_get:
11451da177e4SLinus Torvalds 	kfree(md);
11461da177e4SLinus Torvalds 	return NULL;
11471da177e4SLinus Torvalds }
11481da177e4SLinus Torvalds 
1149ae9da83fSJun'ichi Nomura static void unlock_fs(struct mapped_device *md);
1150ae9da83fSJun'ichi Nomura 
11511da177e4SLinus Torvalds static void free_dev(struct mapped_device *md)
11521da177e4SLinus Torvalds {
1153f331c029STejun Heo 	int minor = MINOR(disk_devt(md->disk));
115463d94e48SJun'ichi Nomura 
1155d9dde59bSJun'ichi Nomura 	if (md->suspended_bdev) {
1156ae9da83fSJun'ichi Nomura 		unlock_fs(md);
1157d9dde59bSJun'ichi Nomura 		bdput(md->suspended_bdev);
1158d9dde59bSJun'ichi Nomura 	}
1159304f3f6aSMilan Broz 	destroy_workqueue(md->wq);
11601da177e4SLinus Torvalds 	mempool_destroy(md->tio_pool);
11611da177e4SLinus Torvalds 	mempool_destroy(md->io_pool);
11629faf400fSStefan Bader 	bioset_free(md->bs);
11631da177e4SLinus Torvalds 	del_gendisk(md->disk);
116463d94e48SJun'ichi Nomura 	free_minor(minor);
1165fba9f90eSJeff Mahoney 
1166fba9f90eSJeff Mahoney 	spin_lock(&_minor_lock);
1167fba9f90eSJeff Mahoney 	md->disk->private_data = NULL;
1168fba9f90eSJeff Mahoney 	spin_unlock(&_minor_lock);
1169fba9f90eSJeff Mahoney 
11701da177e4SLinus Torvalds 	put_disk(md->disk);
11711312f40eSAl Viro 	blk_cleanup_queue(md->queue);
117210da4f79SJeff Mahoney 	module_put(THIS_MODULE);
11731da177e4SLinus Torvalds 	kfree(md);
11741da177e4SLinus Torvalds }
11751da177e4SLinus Torvalds 
11761da177e4SLinus Torvalds /*
11771da177e4SLinus Torvalds  * Bind a table to the device.
11781da177e4SLinus Torvalds  */
11791da177e4SLinus Torvalds static void event_callback(void *context)
11801da177e4SLinus Torvalds {
11817a8c3d3bSMike Anderson 	unsigned long flags;
11827a8c3d3bSMike Anderson 	LIST_HEAD(uevents);
11831da177e4SLinus Torvalds 	struct mapped_device *md = (struct mapped_device *) context;
11841da177e4SLinus Torvalds 
11857a8c3d3bSMike Anderson 	spin_lock_irqsave(&md->uevent_lock, flags);
11867a8c3d3bSMike Anderson 	list_splice_init(&md->uevent_list, &uevents);
11877a8c3d3bSMike Anderson 	spin_unlock_irqrestore(&md->uevent_lock, flags);
11887a8c3d3bSMike Anderson 
1189ed9e1982STejun Heo 	dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
11907a8c3d3bSMike Anderson 
11911da177e4SLinus Torvalds 	atomic_inc(&md->event_nr);
11921da177e4SLinus Torvalds 	wake_up(&md->eventq);
11931da177e4SLinus Torvalds }
11941da177e4SLinus Torvalds 
11954e90188bSAlasdair G Kergon static void __set_size(struct mapped_device *md, sector_t size)
11961da177e4SLinus Torvalds {
11974e90188bSAlasdair G Kergon 	set_capacity(md->disk, size);
11981da177e4SLinus Torvalds 
11991b1dcc1bSJes Sorensen 	mutex_lock(&md->suspended_bdev->bd_inode->i_mutex);
1200e39e2e95SAlasdair G Kergon 	i_size_write(md->suspended_bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
12011b1dcc1bSJes Sorensen 	mutex_unlock(&md->suspended_bdev->bd_inode->i_mutex);
12021da177e4SLinus Torvalds }
12031da177e4SLinus Torvalds 
12041da177e4SLinus Torvalds static int __bind(struct mapped_device *md, struct dm_table *t)
12051da177e4SLinus Torvalds {
1206165125e1SJens Axboe 	struct request_queue *q = md->queue;
12071da177e4SLinus Torvalds 	sector_t size;
12081da177e4SLinus Torvalds 
12091da177e4SLinus Torvalds 	size = dm_table_get_size(t);
12103ac51e74SDarrick J. Wong 
12113ac51e74SDarrick J. Wong 	/*
12123ac51e74SDarrick J. Wong 	 * Wipe any geometry if the size of the table changed.
12133ac51e74SDarrick J. Wong 	 */
12143ac51e74SDarrick J. Wong 	if (size != get_capacity(md->disk))
12153ac51e74SDarrick J. Wong 		memset(&md->geometry, 0, sizeof(md->geometry));
12163ac51e74SDarrick J. Wong 
1217bfa152faSJun'ichi Nomura 	if (md->suspended_bdev)
12184e90188bSAlasdair G Kergon 		__set_size(md, size);
12191da177e4SLinus Torvalds 	if (size == 0)
12201da177e4SLinus Torvalds 		return 0;
12211da177e4SLinus Torvalds 
12221da177e4SLinus Torvalds 	dm_table_get(t);
1223cf222b37SAlasdair G Kergon 	dm_table_event_callback(t, event_callback, md);
12242ca3310eSAlasdair G Kergon 
12252ca3310eSAlasdair G Kergon 	write_lock(&md->map_lock);
12262ca3310eSAlasdair G Kergon 	md->map = t;
12271da177e4SLinus Torvalds 	dm_table_set_restrictions(t, q);
12282ca3310eSAlasdair G Kergon 	write_unlock(&md->map_lock);
12292ca3310eSAlasdair G Kergon 
12301da177e4SLinus Torvalds 	return 0;
12311da177e4SLinus Torvalds }
12321da177e4SLinus Torvalds 
12331da177e4SLinus Torvalds static void __unbind(struct mapped_device *md)
12341da177e4SLinus Torvalds {
12351da177e4SLinus Torvalds 	struct dm_table *map = md->map;
12361da177e4SLinus Torvalds 
12371da177e4SLinus Torvalds 	if (!map)
12381da177e4SLinus Torvalds 		return;
12391da177e4SLinus Torvalds 
12401da177e4SLinus Torvalds 	dm_table_event_callback(map, NULL, NULL);
12411da177e4SLinus Torvalds 	write_lock(&md->map_lock);
12421da177e4SLinus Torvalds 	md->map = NULL;
12431da177e4SLinus Torvalds 	write_unlock(&md->map_lock);
12441da177e4SLinus Torvalds 	dm_table_put(map);
12451da177e4SLinus Torvalds }
12461da177e4SLinus Torvalds 
12471da177e4SLinus Torvalds /*
12481da177e4SLinus Torvalds  * Constructor for a new device.
12491da177e4SLinus Torvalds  */
12502b06cfffSAlasdair G Kergon int dm_create(int minor, struct mapped_device **result)
12511da177e4SLinus Torvalds {
12521da177e4SLinus Torvalds 	struct mapped_device *md;
12531da177e4SLinus Torvalds 
12542b06cfffSAlasdair G Kergon 	md = alloc_dev(minor);
12551da177e4SLinus Torvalds 	if (!md)
12561da177e4SLinus Torvalds 		return -ENXIO;
12571da177e4SLinus Torvalds 
12581da177e4SLinus Torvalds 	*result = md;
12591da177e4SLinus Torvalds 	return 0;
12601da177e4SLinus Torvalds }
12611da177e4SLinus Torvalds 
1262637842cfSDavid Teigland static struct mapped_device *dm_find_md(dev_t dev)
12631da177e4SLinus Torvalds {
12641da177e4SLinus Torvalds 	struct mapped_device *md;
12651da177e4SLinus Torvalds 	unsigned minor = MINOR(dev);
12661da177e4SLinus Torvalds 
12671da177e4SLinus Torvalds 	if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
12681da177e4SLinus Torvalds 		return NULL;
12691da177e4SLinus Torvalds 
1270f32c10b0SJeff Mahoney 	spin_lock(&_minor_lock);
12711da177e4SLinus Torvalds 
12721da177e4SLinus Torvalds 	md = idr_find(&_minor_idr, minor);
1273fba9f90eSJeff Mahoney 	if (md && (md == MINOR_ALLOCED ||
1274f331c029STejun Heo 		   (MINOR(disk_devt(dm_disk(md))) != minor) ||
1275fba9f90eSJeff Mahoney 		   test_bit(DMF_FREEING, &md->flags))) {
1276637842cfSDavid Teigland 		md = NULL;
1277fba9f90eSJeff Mahoney 		goto out;
1278fba9f90eSJeff Mahoney 	}
12791da177e4SLinus Torvalds 
1280fba9f90eSJeff Mahoney out:
1281f32c10b0SJeff Mahoney 	spin_unlock(&_minor_lock);
12821da177e4SLinus Torvalds 
1283637842cfSDavid Teigland 	return md;
1284637842cfSDavid Teigland }
1285637842cfSDavid Teigland 
1286d229a958SDavid Teigland struct mapped_device *dm_get_md(dev_t dev)
1287d229a958SDavid Teigland {
1288d229a958SDavid Teigland 	struct mapped_device *md = dm_find_md(dev);
1289d229a958SDavid Teigland 
1290d229a958SDavid Teigland 	if (md)
1291d229a958SDavid Teigland 		dm_get(md);
1292d229a958SDavid Teigland 
1293d229a958SDavid Teigland 	return md;
1294d229a958SDavid Teigland }
1295d229a958SDavid Teigland 
12969ade92a9SAlasdair G Kergon void *dm_get_mdptr(struct mapped_device *md)
1297637842cfSDavid Teigland {
12989ade92a9SAlasdair G Kergon 	return md->interface_ptr;
12991da177e4SLinus Torvalds }
13001da177e4SLinus Torvalds 
13011da177e4SLinus Torvalds void dm_set_mdptr(struct mapped_device *md, void *ptr)
13021da177e4SLinus Torvalds {
13031da177e4SLinus Torvalds 	md->interface_ptr = ptr;
13041da177e4SLinus Torvalds }
13051da177e4SLinus Torvalds 
13061da177e4SLinus Torvalds void dm_get(struct mapped_device *md)
13071da177e4SLinus Torvalds {
13081da177e4SLinus Torvalds 	atomic_inc(&md->holders);
13091da177e4SLinus Torvalds }
13101da177e4SLinus Torvalds 
131172d94861SAlasdair G Kergon const char *dm_device_name(struct mapped_device *md)
131272d94861SAlasdair G Kergon {
131372d94861SAlasdair G Kergon 	return md->name;
131472d94861SAlasdair G Kergon }
131572d94861SAlasdair G Kergon EXPORT_SYMBOL_GPL(dm_device_name);
131672d94861SAlasdair G Kergon 
13171da177e4SLinus Torvalds void dm_put(struct mapped_device *md)
13181da177e4SLinus Torvalds {
13191134e5aeSMike Anderson 	struct dm_table *map;
13201da177e4SLinus Torvalds 
1321fba9f90eSJeff Mahoney 	BUG_ON(test_bit(DMF_FREEING, &md->flags));
1322fba9f90eSJeff Mahoney 
1323f32c10b0SJeff Mahoney 	if (atomic_dec_and_lock(&md->holders, &_minor_lock)) {
13241134e5aeSMike Anderson 		map = dm_get_table(md);
1325f331c029STejun Heo 		idr_replace(&_minor_idr, MINOR_ALLOCED,
1326f331c029STejun Heo 			    MINOR(disk_devt(dm_disk(md))));
1327fba9f90eSJeff Mahoney 		set_bit(DMF_FREEING, &md->flags);
1328f32c10b0SJeff Mahoney 		spin_unlock(&_minor_lock);
1329cf222b37SAlasdair G Kergon 		if (!dm_suspended(md)) {
13301da177e4SLinus Torvalds 			dm_table_presuspend_targets(map);
13311da177e4SLinus Torvalds 			dm_table_postsuspend_targets(map);
13321da177e4SLinus Torvalds 		}
13331da177e4SLinus Torvalds 		__unbind(md);
13341134e5aeSMike Anderson 		dm_table_put(map);
13351da177e4SLinus Torvalds 		free_dev(md);
13361da177e4SLinus Torvalds 	}
13371da177e4SLinus Torvalds }
133879eb885cSEdward Goggin EXPORT_SYMBOL_GPL(dm_put);
13391da177e4SLinus Torvalds 
134046125c1cSMilan Broz static int dm_wait_for_completion(struct mapped_device *md)
134146125c1cSMilan Broz {
134246125c1cSMilan Broz 	int r = 0;
134346125c1cSMilan Broz 
134446125c1cSMilan Broz 	while (1) {
134546125c1cSMilan Broz 		set_current_state(TASK_INTERRUPTIBLE);
134646125c1cSMilan Broz 
134746125c1cSMilan Broz 		smp_mb();
134846125c1cSMilan Broz 		if (!atomic_read(&md->pending))
134946125c1cSMilan Broz 			break;
135046125c1cSMilan Broz 
135146125c1cSMilan Broz 		if (signal_pending(current)) {
135246125c1cSMilan Broz 			r = -EINTR;
135346125c1cSMilan Broz 			break;
135446125c1cSMilan Broz 		}
135546125c1cSMilan Broz 
135646125c1cSMilan Broz 		io_schedule();
135746125c1cSMilan Broz 	}
135846125c1cSMilan Broz 	set_current_state(TASK_RUNNING);
135946125c1cSMilan Broz 
136046125c1cSMilan Broz 	return r;
136146125c1cSMilan Broz }
136246125c1cSMilan Broz 
13631da177e4SLinus Torvalds /*
13641da177e4SLinus Torvalds  * Process the deferred bios
13651da177e4SLinus Torvalds  */
13666d6f10dfSMilan Broz static void __flush_deferred_io(struct mapped_device *md)
13671da177e4SLinus Torvalds {
13686d6f10dfSMilan Broz 	struct bio *c;
13691da177e4SLinus Torvalds 
13706d6f10dfSMilan Broz 	while ((c = bio_list_pop(&md->deferred))) {
13719e4e5f87SMilan Broz 		if (__split_bio(md, c))
13729e4e5f87SMilan Broz 			bio_io_error(c);
13731da177e4SLinus Torvalds 	}
137473d410c0SMilan Broz 
137573d410c0SMilan Broz 	clear_bit(DMF_BLOCK_IO, &md->flags);
13761da177e4SLinus Torvalds }
13771da177e4SLinus Torvalds 
13786d6f10dfSMilan Broz static void __merge_pushback_list(struct mapped_device *md)
13796d6f10dfSMilan Broz {
13806d6f10dfSMilan Broz 	unsigned long flags;
13816d6f10dfSMilan Broz 
13826d6f10dfSMilan Broz 	spin_lock_irqsave(&md->pushback_lock, flags);
13836d6f10dfSMilan Broz 	clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
13846d6f10dfSMilan Broz 	bio_list_merge_head(&md->deferred, &md->pushback);
13856d6f10dfSMilan Broz 	bio_list_init(&md->pushback);
13866d6f10dfSMilan Broz 	spin_unlock_irqrestore(&md->pushback_lock, flags);
13876d6f10dfSMilan Broz }
13886d6f10dfSMilan Broz 
1389304f3f6aSMilan Broz static void dm_wq_work(struct work_struct *work)
1390304f3f6aSMilan Broz {
1391304f3f6aSMilan Broz 	struct dm_wq_req *req = container_of(work, struct dm_wq_req, work);
1392304f3f6aSMilan Broz 	struct mapped_device *md = req->md;
1393304f3f6aSMilan Broz 
1394304f3f6aSMilan Broz 	down_write(&md->io_lock);
1395304f3f6aSMilan Broz 	switch (req->type) {
1396304f3f6aSMilan Broz 	case DM_WQ_FLUSH_DEFERRED:
1397304f3f6aSMilan Broz 		__flush_deferred_io(md);
1398304f3f6aSMilan Broz 		break;
1399304f3f6aSMilan Broz 	default:
1400304f3f6aSMilan Broz 		DMERR("dm_wq_work: unrecognised work type %d", req->type);
1401304f3f6aSMilan Broz 		BUG();
1402304f3f6aSMilan Broz 	}
1403304f3f6aSMilan Broz 	up_write(&md->io_lock);
1404304f3f6aSMilan Broz }
1405304f3f6aSMilan Broz 
1406304f3f6aSMilan Broz static void dm_wq_queue(struct mapped_device *md, int type, void *context,
1407304f3f6aSMilan Broz 			struct dm_wq_req *req)
1408304f3f6aSMilan Broz {
1409304f3f6aSMilan Broz 	req->type = type;
1410304f3f6aSMilan Broz 	req->md = md;
1411304f3f6aSMilan Broz 	req->context = context;
1412304f3f6aSMilan Broz 	INIT_WORK(&req->work, dm_wq_work);
1413304f3f6aSMilan Broz 	queue_work(md->wq, &req->work);
1414304f3f6aSMilan Broz }
1415304f3f6aSMilan Broz 
1416304f3f6aSMilan Broz static void dm_queue_flush(struct mapped_device *md, int type, void *context)
1417304f3f6aSMilan Broz {
1418304f3f6aSMilan Broz 	struct dm_wq_req req;
1419304f3f6aSMilan Broz 
1420304f3f6aSMilan Broz 	dm_wq_queue(md, type, context, &req);
1421304f3f6aSMilan Broz 	flush_workqueue(md->wq);
1422304f3f6aSMilan Broz }
1423304f3f6aSMilan Broz 
14241da177e4SLinus Torvalds /*
14251da177e4SLinus Torvalds  * Swap in a new table (destroying old one).
14261da177e4SLinus Torvalds  */
14271da177e4SLinus Torvalds int dm_swap_table(struct mapped_device *md, struct dm_table *table)
14281da177e4SLinus Torvalds {
142993c534aeSAlasdair G Kergon 	int r = -EINVAL;
14301da177e4SLinus Torvalds 
1431e61290a4SDaniel Walker 	mutex_lock(&md->suspend_lock);
14321da177e4SLinus Torvalds 
14331da177e4SLinus Torvalds 	/* device must be suspended */
1434cf222b37SAlasdair G Kergon 	if (!dm_suspended(md))
143593c534aeSAlasdair G Kergon 		goto out;
14361da177e4SLinus Torvalds 
1437bfa152faSJun'ichi Nomura 	/* without bdev, the device size cannot be changed */
1438bfa152faSJun'ichi Nomura 	if (!md->suspended_bdev)
1439bfa152faSJun'ichi Nomura 		if (get_capacity(md->disk) != dm_table_get_size(table))
1440bfa152faSJun'ichi Nomura 			goto out;
1441bfa152faSJun'ichi Nomura 
14421da177e4SLinus Torvalds 	__unbind(md);
14431da177e4SLinus Torvalds 	r = __bind(md, table);
14441da177e4SLinus Torvalds 
144593c534aeSAlasdair G Kergon out:
1446e61290a4SDaniel Walker 	mutex_unlock(&md->suspend_lock);
144793c534aeSAlasdair G Kergon 	return r;
14481da177e4SLinus Torvalds }
14491da177e4SLinus Torvalds 
14501da177e4SLinus Torvalds /*
14511da177e4SLinus Torvalds  * Functions to lock and unlock any filesystem running on the
14521da177e4SLinus Torvalds  * device.
14531da177e4SLinus Torvalds  */
14542ca3310eSAlasdair G Kergon static int lock_fs(struct mapped_device *md)
14551da177e4SLinus Torvalds {
1456e39e2e95SAlasdair G Kergon 	int r;
14571da177e4SLinus Torvalds 
14581da177e4SLinus Torvalds 	WARN_ON(md->frozen_sb);
1459dfbe03f6SAlasdair G Kergon 
1460e39e2e95SAlasdair G Kergon 	md->frozen_sb = freeze_bdev(md->suspended_bdev);
1461dfbe03f6SAlasdair G Kergon 	if (IS_ERR(md->frozen_sb)) {
1462cf222b37SAlasdair G Kergon 		r = PTR_ERR(md->frozen_sb);
1463e39e2e95SAlasdair G Kergon 		md->frozen_sb = NULL;
1464e39e2e95SAlasdair G Kergon 		return r;
1465dfbe03f6SAlasdair G Kergon 	}
1466dfbe03f6SAlasdair G Kergon 
1467aa8d7c2fSAlasdair G Kergon 	set_bit(DMF_FROZEN, &md->flags);
1468aa8d7c2fSAlasdair G Kergon 
14691da177e4SLinus Torvalds 	/* don't bdput right now, we don't want the bdev
1470e39e2e95SAlasdair G Kergon 	 * to go away while it is locked.
14711da177e4SLinus Torvalds 	 */
14721da177e4SLinus Torvalds 	return 0;
14731da177e4SLinus Torvalds }
14741da177e4SLinus Torvalds 
14752ca3310eSAlasdair G Kergon static void unlock_fs(struct mapped_device *md)
14761da177e4SLinus Torvalds {
1477aa8d7c2fSAlasdair G Kergon 	if (!test_bit(DMF_FROZEN, &md->flags))
1478aa8d7c2fSAlasdair G Kergon 		return;
1479aa8d7c2fSAlasdair G Kergon 
1480e39e2e95SAlasdair G Kergon 	thaw_bdev(md->suspended_bdev, md->frozen_sb);
14811da177e4SLinus Torvalds 	md->frozen_sb = NULL;
1482aa8d7c2fSAlasdair G Kergon 	clear_bit(DMF_FROZEN, &md->flags);
14831da177e4SLinus Torvalds }
14841da177e4SLinus Torvalds 
14851da177e4SLinus Torvalds /*
14861da177e4SLinus Torvalds  * We need to be able to change a mapping table under a mounted
14871da177e4SLinus Torvalds  * filesystem.  For example we might want to move some data in
14881da177e4SLinus Torvalds  * the background.  Before the table can be swapped with
14891da177e4SLinus Torvalds  * dm_bind_table, dm_suspend must be called to flush any in
14901da177e4SLinus Torvalds  * flight bios and ensure that any further io gets deferred.
14911da177e4SLinus Torvalds  */
1492a3d77d35SKiyoshi Ueda int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
14931da177e4SLinus Torvalds {
14942ca3310eSAlasdair G Kergon 	struct dm_table *map = NULL;
14951da177e4SLinus Torvalds 	DECLARE_WAITQUEUE(wait, current);
149646125c1cSMilan Broz 	int r = 0;
1497a3d77d35SKiyoshi Ueda 	int do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG ? 1 : 0;
14982e93ccc1SKiyoshi Ueda 	int noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG ? 1 : 0;
14991da177e4SLinus Torvalds 
1500e61290a4SDaniel Walker 	mutex_lock(&md->suspend_lock);
15012ca3310eSAlasdair G Kergon 
150273d410c0SMilan Broz 	if (dm_suspended(md)) {
150373d410c0SMilan Broz 		r = -EINVAL;
1504d287483dSAlasdair G Kergon 		goto out_unlock;
150573d410c0SMilan Broz 	}
15061da177e4SLinus Torvalds 
15071da177e4SLinus Torvalds 	map = dm_get_table(md);
1508cf222b37SAlasdair G Kergon 
15092e93ccc1SKiyoshi Ueda 	/*
15102e93ccc1SKiyoshi Ueda 	 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
15112e93ccc1SKiyoshi Ueda 	 * This flag is cleared before dm_suspend returns.
15122e93ccc1SKiyoshi Ueda 	 */
15132e93ccc1SKiyoshi Ueda 	if (noflush)
15142e93ccc1SKiyoshi Ueda 		set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
15152e93ccc1SKiyoshi Ueda 
1516436d4108SAlasdair G Kergon 	/* This does not get reverted if there's an error later. */
15171da177e4SLinus Torvalds 	dm_table_presuspend_targets(map);
15181da177e4SLinus Torvalds 
1519bfa152faSJun'ichi Nomura 	/* bdget() can stall if the pending I/Os are not flushed */
1520bfa152faSJun'ichi Nomura 	if (!noflush) {
1521e39e2e95SAlasdair G Kergon 		md->suspended_bdev = bdget_disk(md->disk, 0);
1522e39e2e95SAlasdair G Kergon 		if (!md->suspended_bdev) {
1523e39e2e95SAlasdair G Kergon 			DMWARN("bdget failed in dm_suspend");
1524e39e2e95SAlasdair G Kergon 			r = -ENOMEM;
1525f431d966SKiyoshi Ueda 			goto out;
1526e39e2e95SAlasdair G Kergon 		}
1527e39e2e95SAlasdair G Kergon 
15282e93ccc1SKiyoshi Ueda 		/*
15296d6f10dfSMilan Broz 		 * Flush I/O to the device. noflush supersedes do_lockfs,
15306d6f10dfSMilan Broz 		 * because lock_fs() needs to flush I/Os.
15312e93ccc1SKiyoshi Ueda 		 */
15326d6f10dfSMilan Broz 		if (do_lockfs) {
15332ca3310eSAlasdair G Kergon 			r = lock_fs(md);
15342ca3310eSAlasdair G Kergon 			if (r)
15352ca3310eSAlasdair G Kergon 				goto out;
1536aa8d7c2fSAlasdair G Kergon 		}
15376d6f10dfSMilan Broz 	}
15381da177e4SLinus Torvalds 
15391da177e4SLinus Torvalds 	/*
1540354e0071SAlasdair G Kergon 	 * First we set the BLOCK_IO flag so no more ios will be mapped.
15411da177e4SLinus Torvalds 	 */
15422ca3310eSAlasdair G Kergon 	down_write(&md->io_lock);
15432ca3310eSAlasdair G Kergon 	set_bit(DMF_BLOCK_IO, &md->flags);
15441da177e4SLinus Torvalds 
15451da177e4SLinus Torvalds 	add_wait_queue(&md->wait, &wait);
15462ca3310eSAlasdair G Kergon 	up_write(&md->io_lock);
15471da177e4SLinus Torvalds 
15481da177e4SLinus Torvalds 	/* unplug */
15492ca3310eSAlasdair G Kergon 	if (map)
15501da177e4SLinus Torvalds 		dm_table_unplug_all(map);
15511da177e4SLinus Torvalds 
15521da177e4SLinus Torvalds 	/*
155346125c1cSMilan Broz 	 * Wait for the already-mapped ios to complete.
15541da177e4SLinus Torvalds 	 */
155546125c1cSMilan Broz 	r = dm_wait_for_completion(md);
15561da177e4SLinus Torvalds 
15572ca3310eSAlasdair G Kergon 	down_write(&md->io_lock);
15581da177e4SLinus Torvalds 	remove_wait_queue(&md->wait, &wait);
15591da177e4SLinus Torvalds 
15606d6f10dfSMilan Broz 	if (noflush)
15616d6f10dfSMilan Broz 		__merge_pushback_list(md);
156294d6351eSMilan Broz 	up_write(&md->io_lock);
15632e93ccc1SKiyoshi Ueda 
15641da177e4SLinus Torvalds 	/* were we interrupted ? */
156546125c1cSMilan Broz 	if (r < 0) {
1566304f3f6aSMilan Broz 		dm_queue_flush(md, DM_WQ_FLUSH_DEFERRED, NULL);
156773d410c0SMilan Broz 
15682ca3310eSAlasdair G Kergon 		unlock_fs(md);
15692e93ccc1SKiyoshi Ueda 		goto out; /* pushback list is already flushed, so skip flush */
15702ca3310eSAlasdair G Kergon 	}
15712ca3310eSAlasdair G Kergon 
15722ca3310eSAlasdair G Kergon 	dm_table_postsuspend_targets(map);
15731da177e4SLinus Torvalds 
15741da177e4SLinus Torvalds 	set_bit(DMF_SUSPENDED, &md->flags);
15751da177e4SLinus Torvalds 
15762ca3310eSAlasdair G Kergon out:
1577e39e2e95SAlasdair G Kergon 	if (r && md->suspended_bdev) {
1578e39e2e95SAlasdair G Kergon 		bdput(md->suspended_bdev);
1579e39e2e95SAlasdair G Kergon 		md->suspended_bdev = NULL;
1580e39e2e95SAlasdair G Kergon 	}
1581e39e2e95SAlasdair G Kergon 
15821da177e4SLinus Torvalds 	dm_table_put(map);
1583d287483dSAlasdair G Kergon 
1584d287483dSAlasdair G Kergon out_unlock:
1585e61290a4SDaniel Walker 	mutex_unlock(&md->suspend_lock);
1586cf222b37SAlasdair G Kergon 	return r;
15871da177e4SLinus Torvalds }
15881da177e4SLinus Torvalds 
15891da177e4SLinus Torvalds int dm_resume(struct mapped_device *md)
15901da177e4SLinus Torvalds {
1591cf222b37SAlasdair G Kergon 	int r = -EINVAL;
1592cf222b37SAlasdair G Kergon 	struct dm_table *map = NULL;
15931da177e4SLinus Torvalds 
1594e61290a4SDaniel Walker 	mutex_lock(&md->suspend_lock);
15952ca3310eSAlasdair G Kergon 	if (!dm_suspended(md))
1596cf222b37SAlasdair G Kergon 		goto out;
1597cf222b37SAlasdair G Kergon 
1598cf222b37SAlasdair G Kergon 	map = dm_get_table(md);
15992ca3310eSAlasdair G Kergon 	if (!map || !dm_table_get_size(map))
1600cf222b37SAlasdair G Kergon 		goto out;
16011da177e4SLinus Torvalds 
16028757b776SMilan Broz 	r = dm_table_resume_targets(map);
16038757b776SMilan Broz 	if (r)
16048757b776SMilan Broz 		goto out;
16052ca3310eSAlasdair G Kergon 
1606304f3f6aSMilan Broz 	dm_queue_flush(md, DM_WQ_FLUSH_DEFERRED, NULL);
16072ca3310eSAlasdair G Kergon 
16082ca3310eSAlasdair G Kergon 	unlock_fs(md);
16092ca3310eSAlasdair G Kergon 
1610bfa152faSJun'ichi Nomura 	if (md->suspended_bdev) {
1611e39e2e95SAlasdair G Kergon 		bdput(md->suspended_bdev);
1612e39e2e95SAlasdair G Kergon 		md->suspended_bdev = NULL;
1613bfa152faSJun'ichi Nomura 	}
1614e39e2e95SAlasdair G Kergon 
16152ca3310eSAlasdair G Kergon 	clear_bit(DMF_SUSPENDED, &md->flags);
16162ca3310eSAlasdair G Kergon 
16171da177e4SLinus Torvalds 	dm_table_unplug_all(map);
16181da177e4SLinus Torvalds 
161969267a30SAlasdair G Kergon 	dm_kobject_uevent(md);
16208560ed6fSHannes Reinecke 
1621cf222b37SAlasdair G Kergon 	r = 0;
16222ca3310eSAlasdair G Kergon 
1623cf222b37SAlasdair G Kergon out:
1624cf222b37SAlasdair G Kergon 	dm_table_put(map);
1625e61290a4SDaniel Walker 	mutex_unlock(&md->suspend_lock);
16262ca3310eSAlasdair G Kergon 
1627cf222b37SAlasdair G Kergon 	return r;
16281da177e4SLinus Torvalds }
16291da177e4SLinus Torvalds 
16301da177e4SLinus Torvalds /*-----------------------------------------------------------------
16311da177e4SLinus Torvalds  * Event notification.
16321da177e4SLinus Torvalds  *---------------------------------------------------------------*/
163369267a30SAlasdair G Kergon void dm_kobject_uevent(struct mapped_device *md)
163469267a30SAlasdair G Kergon {
1635ed9e1982STejun Heo 	kobject_uevent(&disk_to_dev(md->disk)->kobj, KOBJ_CHANGE);
163669267a30SAlasdair G Kergon }
163769267a30SAlasdair G Kergon 
16387a8c3d3bSMike Anderson uint32_t dm_next_uevent_seq(struct mapped_device *md)
16397a8c3d3bSMike Anderson {
16407a8c3d3bSMike Anderson 	return atomic_add_return(1, &md->uevent_seq);
16417a8c3d3bSMike Anderson }
16427a8c3d3bSMike Anderson 
16431da177e4SLinus Torvalds uint32_t dm_get_event_nr(struct mapped_device *md)
16441da177e4SLinus Torvalds {
16451da177e4SLinus Torvalds 	return atomic_read(&md->event_nr);
16461da177e4SLinus Torvalds }
16471da177e4SLinus Torvalds 
16481da177e4SLinus Torvalds int dm_wait_event(struct mapped_device *md, int event_nr)
16491da177e4SLinus Torvalds {
16501da177e4SLinus Torvalds 	return wait_event_interruptible(md->eventq,
16511da177e4SLinus Torvalds 			(event_nr != atomic_read(&md->event_nr)));
16521da177e4SLinus Torvalds }
16531da177e4SLinus Torvalds 
16547a8c3d3bSMike Anderson void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
16557a8c3d3bSMike Anderson {
16567a8c3d3bSMike Anderson 	unsigned long flags;
16577a8c3d3bSMike Anderson 
16587a8c3d3bSMike Anderson 	spin_lock_irqsave(&md->uevent_lock, flags);
16597a8c3d3bSMike Anderson 	list_add(elist, &md->uevent_list);
16607a8c3d3bSMike Anderson 	spin_unlock_irqrestore(&md->uevent_lock, flags);
16617a8c3d3bSMike Anderson }
16627a8c3d3bSMike Anderson 
16631da177e4SLinus Torvalds /*
16641da177e4SLinus Torvalds  * The gendisk is only valid as long as you have a reference
16651da177e4SLinus Torvalds  * count on 'md'.
16661da177e4SLinus Torvalds  */
16671da177e4SLinus Torvalds struct gendisk *dm_disk(struct mapped_device *md)
16681da177e4SLinus Torvalds {
16691da177e4SLinus Torvalds 	return md->disk;
16701da177e4SLinus Torvalds }
16711da177e4SLinus Torvalds 
16721da177e4SLinus Torvalds int dm_suspended(struct mapped_device *md)
16731da177e4SLinus Torvalds {
16741da177e4SLinus Torvalds 	return test_bit(DMF_SUSPENDED, &md->flags);
16751da177e4SLinus Torvalds }
16761da177e4SLinus Torvalds 
16772e93ccc1SKiyoshi Ueda int dm_noflush_suspending(struct dm_target *ti)
16782e93ccc1SKiyoshi Ueda {
16792e93ccc1SKiyoshi Ueda 	struct mapped_device *md = dm_table_get_md(ti->table);
16802e93ccc1SKiyoshi Ueda 	int r = __noflush_suspending(md);
16812e93ccc1SKiyoshi Ueda 
16822e93ccc1SKiyoshi Ueda 	dm_put(md);
16832e93ccc1SKiyoshi Ueda 
16842e93ccc1SKiyoshi Ueda 	return r;
16852e93ccc1SKiyoshi Ueda }
16862e93ccc1SKiyoshi Ueda EXPORT_SYMBOL_GPL(dm_noflush_suspending);
16872e93ccc1SKiyoshi Ueda 
16881da177e4SLinus Torvalds static struct block_device_operations dm_blk_dops = {
16891da177e4SLinus Torvalds 	.open = dm_blk_open,
16901da177e4SLinus Torvalds 	.release = dm_blk_close,
1691aa129a22SMilan Broz 	.ioctl = dm_blk_ioctl,
16923ac51e74SDarrick J. Wong 	.getgeo = dm_blk_getgeo,
16931da177e4SLinus Torvalds 	.owner = THIS_MODULE
16941da177e4SLinus Torvalds };
16951da177e4SLinus Torvalds 
16961da177e4SLinus Torvalds EXPORT_SYMBOL(dm_get_mapinfo);
16971da177e4SLinus Torvalds 
16981da177e4SLinus Torvalds /*
16991da177e4SLinus Torvalds  * module hooks
17001da177e4SLinus Torvalds  */
17011da177e4SLinus Torvalds module_init(dm_init);
17021da177e4SLinus Torvalds module_exit(dm_exit);
17031da177e4SLinus Torvalds 
17041da177e4SLinus Torvalds module_param(major, uint, 0);
17051da177e4SLinus Torvalds MODULE_PARM_DESC(major, "The major number of the device mapper");
17061da177e4SLinus Torvalds MODULE_DESCRIPTION(DM_NAME " driver");
17071da177e4SLinus Torvalds MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
17081da177e4SLinus Torvalds MODULE_LICENSE("GPL");
1709