11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * Copyright (C) 2001, 2002 Sistina Software (UK) Limited. 32b06cfffSAlasdair G Kergon * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. 41da177e4SLinus Torvalds * 51da177e4SLinus Torvalds * This file is released under the GPL. 61da177e4SLinus Torvalds */ 71da177e4SLinus Torvalds 81da177e4SLinus Torvalds #include "dm.h" 91da177e4SLinus Torvalds #include "dm-bio-list.h" 1051e5b2bdSMike Anderson #include "dm-uevent.h" 111da177e4SLinus Torvalds 121da177e4SLinus Torvalds #include <linux/init.h> 131da177e4SLinus Torvalds #include <linux/module.h> 1448c9c27bSArjan van de Ven #include <linux/mutex.h> 151da177e4SLinus Torvalds #include <linux/moduleparam.h> 161da177e4SLinus Torvalds #include <linux/blkpg.h> 171da177e4SLinus Torvalds #include <linux/bio.h> 181da177e4SLinus Torvalds #include <linux/buffer_head.h> 191da177e4SLinus Torvalds #include <linux/mempool.h> 201da177e4SLinus Torvalds #include <linux/slab.h> 211da177e4SLinus Torvalds #include <linux/idr.h> 223ac51e74SDarrick J. Wong #include <linux/hdreg.h> 232056a782SJens Axboe #include <linux/blktrace_api.h> 245f3ea37cSArnaldo Carvalho de Melo #include <trace/block.h> 251da177e4SLinus Torvalds 2672d94861SAlasdair G Kergon #define DM_MSG_PREFIX "core" 2772d94861SAlasdair G Kergon 281da177e4SLinus Torvalds static const char *_name = DM_NAME; 291da177e4SLinus Torvalds 301da177e4SLinus Torvalds static unsigned int major = 0; 311da177e4SLinus Torvalds static unsigned int _major = 0; 321da177e4SLinus Torvalds 33f32c10b0SJeff Mahoney static DEFINE_SPINLOCK(_minor_lock); 341da177e4SLinus Torvalds /* 351da177e4SLinus Torvalds * One of these is allocated per bio. 361da177e4SLinus Torvalds */ 371da177e4SLinus Torvalds struct dm_io { 381da177e4SLinus Torvalds struct mapped_device *md; 391da177e4SLinus Torvalds int error; 401da177e4SLinus Torvalds atomic_t io_count; 416ae2fa67SRichard Kennedy struct bio *bio; 423eaf840eSJun'ichi "Nick" Nomura unsigned long start_time; 431da177e4SLinus Torvalds }; 441da177e4SLinus Torvalds 451da177e4SLinus Torvalds /* 461da177e4SLinus Torvalds * One of these is allocated per target within a bio. Hopefully 471da177e4SLinus Torvalds * this will be simplified out one day. 481da177e4SLinus Torvalds */ 49028867acSAlasdair G Kergon struct dm_target_io { 501da177e4SLinus Torvalds struct dm_io *io; 511da177e4SLinus Torvalds struct dm_target *ti; 521da177e4SLinus Torvalds union map_info info; 531da177e4SLinus Torvalds }; 541da177e4SLinus Torvalds 55*0bfc2455SIngo Molnar DEFINE_TRACE(block_bio_complete); 56*0bfc2455SIngo Molnar 571da177e4SLinus Torvalds union map_info *dm_get_mapinfo(struct bio *bio) 581da177e4SLinus Torvalds { 591da177e4SLinus Torvalds if (bio && bio->bi_private) 60028867acSAlasdair G Kergon return &((struct dm_target_io *)bio->bi_private)->info; 611da177e4SLinus Torvalds return NULL; 621da177e4SLinus Torvalds } 631da177e4SLinus Torvalds 64ba61fdd1SJeff Mahoney #define MINOR_ALLOCED ((void *)-1) 65ba61fdd1SJeff Mahoney 661da177e4SLinus Torvalds /* 671da177e4SLinus Torvalds * Bits for the md->flags field. 681da177e4SLinus Torvalds */ 691da177e4SLinus Torvalds #define DMF_BLOCK_IO 0 701da177e4SLinus Torvalds #define DMF_SUSPENDED 1 71aa8d7c2fSAlasdair G Kergon #define DMF_FROZEN 2 72fba9f90eSJeff Mahoney #define DMF_FREEING 3 735c6bd75dSAlasdair G Kergon #define DMF_DELETING 4 742e93ccc1SKiyoshi Ueda #define DMF_NOFLUSH_SUSPENDING 5 751da177e4SLinus Torvalds 76304f3f6aSMilan Broz /* 77304f3f6aSMilan Broz * Work processed by per-device workqueue. 78304f3f6aSMilan Broz */ 79304f3f6aSMilan Broz struct dm_wq_req { 80304f3f6aSMilan Broz enum { 81304f3f6aSMilan Broz DM_WQ_FLUSH_DEFERRED, 82304f3f6aSMilan Broz } type; 83304f3f6aSMilan Broz struct work_struct work; 84304f3f6aSMilan Broz struct mapped_device *md; 85304f3f6aSMilan Broz void *context; 86304f3f6aSMilan Broz }; 87304f3f6aSMilan Broz 881da177e4SLinus Torvalds struct mapped_device { 892ca3310eSAlasdair G Kergon struct rw_semaphore io_lock; 90e61290a4SDaniel Walker struct mutex suspend_lock; 912e93ccc1SKiyoshi Ueda spinlock_t pushback_lock; 921da177e4SLinus Torvalds rwlock_t map_lock; 931da177e4SLinus Torvalds atomic_t holders; 945c6bd75dSAlasdair G Kergon atomic_t open_count; 951da177e4SLinus Torvalds 961da177e4SLinus Torvalds unsigned long flags; 971da177e4SLinus Torvalds 98165125e1SJens Axboe struct request_queue *queue; 991da177e4SLinus Torvalds struct gendisk *disk; 1007e51f257SMike Anderson char name[16]; 1011da177e4SLinus Torvalds 1021da177e4SLinus Torvalds void *interface_ptr; 1031da177e4SLinus Torvalds 1041da177e4SLinus Torvalds /* 1051da177e4SLinus Torvalds * A list of ios that arrived while we were suspended. 1061da177e4SLinus Torvalds */ 1071da177e4SLinus Torvalds atomic_t pending; 1081da177e4SLinus Torvalds wait_queue_head_t wait; 1091da177e4SLinus Torvalds struct bio_list deferred; 1102e93ccc1SKiyoshi Ueda struct bio_list pushback; 1111da177e4SLinus Torvalds 1121da177e4SLinus Torvalds /* 113304f3f6aSMilan Broz * Processing queue (flush/barriers) 114304f3f6aSMilan Broz */ 115304f3f6aSMilan Broz struct workqueue_struct *wq; 116304f3f6aSMilan Broz 117304f3f6aSMilan Broz /* 1181da177e4SLinus Torvalds * The current mapping. 1191da177e4SLinus Torvalds */ 1201da177e4SLinus Torvalds struct dm_table *map; 1211da177e4SLinus Torvalds 1221da177e4SLinus Torvalds /* 1231da177e4SLinus Torvalds * io objects are allocated from here. 1241da177e4SLinus Torvalds */ 1251da177e4SLinus Torvalds mempool_t *io_pool; 1261da177e4SLinus Torvalds mempool_t *tio_pool; 1271da177e4SLinus Torvalds 1289faf400fSStefan Bader struct bio_set *bs; 1299faf400fSStefan Bader 1301da177e4SLinus Torvalds /* 1311da177e4SLinus Torvalds * Event handling. 1321da177e4SLinus Torvalds */ 1331da177e4SLinus Torvalds atomic_t event_nr; 1341da177e4SLinus Torvalds wait_queue_head_t eventq; 1357a8c3d3bSMike Anderson atomic_t uevent_seq; 1367a8c3d3bSMike Anderson struct list_head uevent_list; 1377a8c3d3bSMike Anderson spinlock_t uevent_lock; /* Protect access to uevent_list */ 1381da177e4SLinus Torvalds 1391da177e4SLinus Torvalds /* 1401da177e4SLinus Torvalds * freeze/thaw support require holding onto a super block 1411da177e4SLinus Torvalds */ 1421da177e4SLinus Torvalds struct super_block *frozen_sb; 143e39e2e95SAlasdair G Kergon struct block_device *suspended_bdev; 1443ac51e74SDarrick J. Wong 1453ac51e74SDarrick J. Wong /* forced geometry settings */ 1463ac51e74SDarrick J. Wong struct hd_geometry geometry; 1471da177e4SLinus Torvalds }; 1481da177e4SLinus Torvalds 1491da177e4SLinus Torvalds #define MIN_IOS 256 150e18b890bSChristoph Lameter static struct kmem_cache *_io_cache; 151e18b890bSChristoph Lameter static struct kmem_cache *_tio_cache; 1521da177e4SLinus Torvalds 1531da177e4SLinus Torvalds static int __init local_init(void) 1541da177e4SLinus Torvalds { 15551157b4aSKiyoshi Ueda int r = -ENOMEM; 1561da177e4SLinus Torvalds 1571da177e4SLinus Torvalds /* allocate a slab for the dm_ios */ 158028867acSAlasdair G Kergon _io_cache = KMEM_CACHE(dm_io, 0); 1591da177e4SLinus Torvalds if (!_io_cache) 16051157b4aSKiyoshi Ueda return r; 1611da177e4SLinus Torvalds 1621da177e4SLinus Torvalds /* allocate a slab for the target ios */ 163028867acSAlasdair G Kergon _tio_cache = KMEM_CACHE(dm_target_io, 0); 16451157b4aSKiyoshi Ueda if (!_tio_cache) 16551157b4aSKiyoshi Ueda goto out_free_io_cache; 1661da177e4SLinus Torvalds 16751e5b2bdSMike Anderson r = dm_uevent_init(); 16851157b4aSKiyoshi Ueda if (r) 16951157b4aSKiyoshi Ueda goto out_free_tio_cache; 17051e5b2bdSMike Anderson 1711da177e4SLinus Torvalds _major = major; 1721da177e4SLinus Torvalds r = register_blkdev(_major, _name); 17351157b4aSKiyoshi Ueda if (r < 0) 17451157b4aSKiyoshi Ueda goto out_uevent_exit; 1751da177e4SLinus Torvalds 1761da177e4SLinus Torvalds if (!_major) 1771da177e4SLinus Torvalds _major = r; 1781da177e4SLinus Torvalds 1791da177e4SLinus Torvalds return 0; 18051157b4aSKiyoshi Ueda 18151157b4aSKiyoshi Ueda out_uevent_exit: 18251157b4aSKiyoshi Ueda dm_uevent_exit(); 18351157b4aSKiyoshi Ueda out_free_tio_cache: 18451157b4aSKiyoshi Ueda kmem_cache_destroy(_tio_cache); 18551157b4aSKiyoshi Ueda out_free_io_cache: 18651157b4aSKiyoshi Ueda kmem_cache_destroy(_io_cache); 18751157b4aSKiyoshi Ueda 18851157b4aSKiyoshi Ueda return r; 1891da177e4SLinus Torvalds } 1901da177e4SLinus Torvalds 1911da177e4SLinus Torvalds static void local_exit(void) 1921da177e4SLinus Torvalds { 1931da177e4SLinus Torvalds kmem_cache_destroy(_tio_cache); 1941da177e4SLinus Torvalds kmem_cache_destroy(_io_cache); 19500d59405SAkinobu Mita unregister_blkdev(_major, _name); 19651e5b2bdSMike Anderson dm_uevent_exit(); 1971da177e4SLinus Torvalds 1981da177e4SLinus Torvalds _major = 0; 1991da177e4SLinus Torvalds 2001da177e4SLinus Torvalds DMINFO("cleaned up"); 2011da177e4SLinus Torvalds } 2021da177e4SLinus Torvalds 203b9249e55SAlasdair G Kergon static int (*_inits[])(void) __initdata = { 2041da177e4SLinus Torvalds local_init, 2051da177e4SLinus Torvalds dm_target_init, 2061da177e4SLinus Torvalds dm_linear_init, 2071da177e4SLinus Torvalds dm_stripe_init, 208945fa4d2SMikulas Patocka dm_kcopyd_init, 2091da177e4SLinus Torvalds dm_interface_init, 2101da177e4SLinus Torvalds }; 2111da177e4SLinus Torvalds 212b9249e55SAlasdair G Kergon static void (*_exits[])(void) = { 2131da177e4SLinus Torvalds local_exit, 2141da177e4SLinus Torvalds dm_target_exit, 2151da177e4SLinus Torvalds dm_linear_exit, 2161da177e4SLinus Torvalds dm_stripe_exit, 217945fa4d2SMikulas Patocka dm_kcopyd_exit, 2181da177e4SLinus Torvalds dm_interface_exit, 2191da177e4SLinus Torvalds }; 2201da177e4SLinus Torvalds 2211da177e4SLinus Torvalds static int __init dm_init(void) 2221da177e4SLinus Torvalds { 2231da177e4SLinus Torvalds const int count = ARRAY_SIZE(_inits); 2241da177e4SLinus Torvalds 2251da177e4SLinus Torvalds int r, i; 2261da177e4SLinus Torvalds 2271da177e4SLinus Torvalds for (i = 0; i < count; i++) { 2281da177e4SLinus Torvalds r = _inits[i](); 2291da177e4SLinus Torvalds if (r) 2301da177e4SLinus Torvalds goto bad; 2311da177e4SLinus Torvalds } 2321da177e4SLinus Torvalds 2331da177e4SLinus Torvalds return 0; 2341da177e4SLinus Torvalds 2351da177e4SLinus Torvalds bad: 2361da177e4SLinus Torvalds while (i--) 2371da177e4SLinus Torvalds _exits[i](); 2381da177e4SLinus Torvalds 2391da177e4SLinus Torvalds return r; 2401da177e4SLinus Torvalds } 2411da177e4SLinus Torvalds 2421da177e4SLinus Torvalds static void __exit dm_exit(void) 2431da177e4SLinus Torvalds { 2441da177e4SLinus Torvalds int i = ARRAY_SIZE(_exits); 2451da177e4SLinus Torvalds 2461da177e4SLinus Torvalds while (i--) 2471da177e4SLinus Torvalds _exits[i](); 2481da177e4SLinus Torvalds } 2491da177e4SLinus Torvalds 2501da177e4SLinus Torvalds /* 2511da177e4SLinus Torvalds * Block device functions 2521da177e4SLinus Torvalds */ 253fe5f9f2cSAl Viro static int dm_blk_open(struct block_device *bdev, fmode_t mode) 2541da177e4SLinus Torvalds { 2551da177e4SLinus Torvalds struct mapped_device *md; 2561da177e4SLinus Torvalds 257fba9f90eSJeff Mahoney spin_lock(&_minor_lock); 258fba9f90eSJeff Mahoney 259fe5f9f2cSAl Viro md = bdev->bd_disk->private_data; 260fba9f90eSJeff Mahoney if (!md) 261fba9f90eSJeff Mahoney goto out; 262fba9f90eSJeff Mahoney 2635c6bd75dSAlasdair G Kergon if (test_bit(DMF_FREEING, &md->flags) || 2645c6bd75dSAlasdair G Kergon test_bit(DMF_DELETING, &md->flags)) { 265fba9f90eSJeff Mahoney md = NULL; 266fba9f90eSJeff Mahoney goto out; 267fba9f90eSJeff Mahoney } 268fba9f90eSJeff Mahoney 2691da177e4SLinus Torvalds dm_get(md); 2705c6bd75dSAlasdair G Kergon atomic_inc(&md->open_count); 271fba9f90eSJeff Mahoney 272fba9f90eSJeff Mahoney out: 273fba9f90eSJeff Mahoney spin_unlock(&_minor_lock); 274fba9f90eSJeff Mahoney 275fba9f90eSJeff Mahoney return md ? 0 : -ENXIO; 2761da177e4SLinus Torvalds } 2771da177e4SLinus Torvalds 278fe5f9f2cSAl Viro static int dm_blk_close(struct gendisk *disk, fmode_t mode) 2791da177e4SLinus Torvalds { 280fe5f9f2cSAl Viro struct mapped_device *md = disk->private_data; 2815c6bd75dSAlasdair G Kergon atomic_dec(&md->open_count); 2821da177e4SLinus Torvalds dm_put(md); 2831da177e4SLinus Torvalds return 0; 2841da177e4SLinus Torvalds } 2851da177e4SLinus Torvalds 2865c6bd75dSAlasdair G Kergon int dm_open_count(struct mapped_device *md) 2875c6bd75dSAlasdair G Kergon { 2885c6bd75dSAlasdair G Kergon return atomic_read(&md->open_count); 2895c6bd75dSAlasdair G Kergon } 2905c6bd75dSAlasdair G Kergon 2915c6bd75dSAlasdair G Kergon /* 2925c6bd75dSAlasdair G Kergon * Guarantees nothing is using the device before it's deleted. 2935c6bd75dSAlasdair G Kergon */ 2945c6bd75dSAlasdair G Kergon int dm_lock_for_deletion(struct mapped_device *md) 2955c6bd75dSAlasdair G Kergon { 2965c6bd75dSAlasdair G Kergon int r = 0; 2975c6bd75dSAlasdair G Kergon 2985c6bd75dSAlasdair G Kergon spin_lock(&_minor_lock); 2995c6bd75dSAlasdair G Kergon 3005c6bd75dSAlasdair G Kergon if (dm_open_count(md)) 3015c6bd75dSAlasdair G Kergon r = -EBUSY; 3025c6bd75dSAlasdair G Kergon else 3035c6bd75dSAlasdair G Kergon set_bit(DMF_DELETING, &md->flags); 3045c6bd75dSAlasdair G Kergon 3055c6bd75dSAlasdair G Kergon spin_unlock(&_minor_lock); 3065c6bd75dSAlasdair G Kergon 3075c6bd75dSAlasdair G Kergon return r; 3085c6bd75dSAlasdair G Kergon } 3095c6bd75dSAlasdair G Kergon 3103ac51e74SDarrick J. Wong static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo) 3113ac51e74SDarrick J. Wong { 3123ac51e74SDarrick J. Wong struct mapped_device *md = bdev->bd_disk->private_data; 3133ac51e74SDarrick J. Wong 3143ac51e74SDarrick J. Wong return dm_get_geometry(md, geo); 3153ac51e74SDarrick J. Wong } 3163ac51e74SDarrick J. Wong 317fe5f9f2cSAl Viro static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode, 318aa129a22SMilan Broz unsigned int cmd, unsigned long arg) 319aa129a22SMilan Broz { 320fe5f9f2cSAl Viro struct mapped_device *md = bdev->bd_disk->private_data; 321fe5f9f2cSAl Viro struct dm_table *map = dm_get_table(md); 322aa129a22SMilan Broz struct dm_target *tgt; 323aa129a22SMilan Broz int r = -ENOTTY; 324aa129a22SMilan Broz 325aa129a22SMilan Broz if (!map || !dm_table_get_size(map)) 326aa129a22SMilan Broz goto out; 327aa129a22SMilan Broz 328aa129a22SMilan Broz /* We only support devices that have a single target */ 329aa129a22SMilan Broz if (dm_table_get_num_targets(map) != 1) 330aa129a22SMilan Broz goto out; 331aa129a22SMilan Broz 332aa129a22SMilan Broz tgt = dm_table_get_target(map, 0); 333aa129a22SMilan Broz 334aa129a22SMilan Broz if (dm_suspended(md)) { 335aa129a22SMilan Broz r = -EAGAIN; 336aa129a22SMilan Broz goto out; 337aa129a22SMilan Broz } 338aa129a22SMilan Broz 339aa129a22SMilan Broz if (tgt->type->ioctl) 340647b3d00SAl Viro r = tgt->type->ioctl(tgt, cmd, arg); 341aa129a22SMilan Broz 342aa129a22SMilan Broz out: 343aa129a22SMilan Broz dm_table_put(map); 344aa129a22SMilan Broz 345aa129a22SMilan Broz return r; 346aa129a22SMilan Broz } 347aa129a22SMilan Broz 348028867acSAlasdair G Kergon static struct dm_io *alloc_io(struct mapped_device *md) 3491da177e4SLinus Torvalds { 3501da177e4SLinus Torvalds return mempool_alloc(md->io_pool, GFP_NOIO); 3511da177e4SLinus Torvalds } 3521da177e4SLinus Torvalds 353028867acSAlasdair G Kergon static void free_io(struct mapped_device *md, struct dm_io *io) 3541da177e4SLinus Torvalds { 3551da177e4SLinus Torvalds mempool_free(io, md->io_pool); 3561da177e4SLinus Torvalds } 3571da177e4SLinus Torvalds 358028867acSAlasdair G Kergon static struct dm_target_io *alloc_tio(struct mapped_device *md) 3591da177e4SLinus Torvalds { 3601da177e4SLinus Torvalds return mempool_alloc(md->tio_pool, GFP_NOIO); 3611da177e4SLinus Torvalds } 3621da177e4SLinus Torvalds 363028867acSAlasdair G Kergon static void free_tio(struct mapped_device *md, struct dm_target_io *tio) 3641da177e4SLinus Torvalds { 3651da177e4SLinus Torvalds mempool_free(tio, md->tio_pool); 3661da177e4SLinus Torvalds } 3671da177e4SLinus Torvalds 3683eaf840eSJun'ichi "Nick" Nomura static void start_io_acct(struct dm_io *io) 3693eaf840eSJun'ichi "Nick" Nomura { 3703eaf840eSJun'ichi "Nick" Nomura struct mapped_device *md = io->md; 371c9959059STejun Heo int cpu; 3723eaf840eSJun'ichi "Nick" Nomura 3733eaf840eSJun'ichi "Nick" Nomura io->start_time = jiffies; 3743eaf840eSJun'ichi "Nick" Nomura 375074a7acaSTejun Heo cpu = part_stat_lock(); 376074a7acaSTejun Heo part_round_stats(cpu, &dm_disk(md)->part0); 377074a7acaSTejun Heo part_stat_unlock(); 378074a7acaSTejun Heo dm_disk(md)->part0.in_flight = atomic_inc_return(&md->pending); 3793eaf840eSJun'ichi "Nick" Nomura } 3803eaf840eSJun'ichi "Nick" Nomura 381d221d2e7SMikulas Patocka static void end_io_acct(struct dm_io *io) 3823eaf840eSJun'ichi "Nick" Nomura { 3833eaf840eSJun'ichi "Nick" Nomura struct mapped_device *md = io->md; 3843eaf840eSJun'ichi "Nick" Nomura struct bio *bio = io->bio; 3853eaf840eSJun'ichi "Nick" Nomura unsigned long duration = jiffies - io->start_time; 386c9959059STejun Heo int pending, cpu; 3873eaf840eSJun'ichi "Nick" Nomura int rw = bio_data_dir(bio); 3883eaf840eSJun'ichi "Nick" Nomura 389074a7acaSTejun Heo cpu = part_stat_lock(); 390074a7acaSTejun Heo part_round_stats(cpu, &dm_disk(md)->part0); 391074a7acaSTejun Heo part_stat_add(cpu, &dm_disk(md)->part0, ticks[rw], duration); 392074a7acaSTejun Heo part_stat_unlock(); 3933eaf840eSJun'ichi "Nick" Nomura 394074a7acaSTejun Heo dm_disk(md)->part0.in_flight = pending = 395074a7acaSTejun Heo atomic_dec_return(&md->pending); 3963eaf840eSJun'ichi "Nick" Nomura 397d221d2e7SMikulas Patocka /* nudge anyone waiting on suspend queue */ 398d221d2e7SMikulas Patocka if (!pending) 399d221d2e7SMikulas Patocka wake_up(&md->wait); 4003eaf840eSJun'ichi "Nick" Nomura } 4013eaf840eSJun'ichi "Nick" Nomura 4021da177e4SLinus Torvalds /* 4031da177e4SLinus Torvalds * Add the bio to the list of deferred io. 4041da177e4SLinus Torvalds */ 4051da177e4SLinus Torvalds static int queue_io(struct mapped_device *md, struct bio *bio) 4061da177e4SLinus Torvalds { 4072ca3310eSAlasdair G Kergon down_write(&md->io_lock); 4081da177e4SLinus Torvalds 4091da177e4SLinus Torvalds if (!test_bit(DMF_BLOCK_IO, &md->flags)) { 4102ca3310eSAlasdair G Kergon up_write(&md->io_lock); 4111da177e4SLinus Torvalds return 1; 4121da177e4SLinus Torvalds } 4131da177e4SLinus Torvalds 4141da177e4SLinus Torvalds bio_list_add(&md->deferred, bio); 4151da177e4SLinus Torvalds 4162ca3310eSAlasdair G Kergon up_write(&md->io_lock); 4171da177e4SLinus Torvalds return 0; /* deferred successfully */ 4181da177e4SLinus Torvalds } 4191da177e4SLinus Torvalds 4201da177e4SLinus Torvalds /* 4211da177e4SLinus Torvalds * Everyone (including functions in this file), should use this 4221da177e4SLinus Torvalds * function to access the md->map field, and make sure they call 4231da177e4SLinus Torvalds * dm_table_put() when finished. 4241da177e4SLinus Torvalds */ 4251da177e4SLinus Torvalds struct dm_table *dm_get_table(struct mapped_device *md) 4261da177e4SLinus Torvalds { 4271da177e4SLinus Torvalds struct dm_table *t; 4281da177e4SLinus Torvalds 4291da177e4SLinus Torvalds read_lock(&md->map_lock); 4301da177e4SLinus Torvalds t = md->map; 4311da177e4SLinus Torvalds if (t) 4321da177e4SLinus Torvalds dm_table_get(t); 4331da177e4SLinus Torvalds read_unlock(&md->map_lock); 4341da177e4SLinus Torvalds 4351da177e4SLinus Torvalds return t; 4361da177e4SLinus Torvalds } 4371da177e4SLinus Torvalds 4383ac51e74SDarrick J. Wong /* 4393ac51e74SDarrick J. Wong * Get the geometry associated with a dm device 4403ac51e74SDarrick J. Wong */ 4413ac51e74SDarrick J. Wong int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo) 4423ac51e74SDarrick J. Wong { 4433ac51e74SDarrick J. Wong *geo = md->geometry; 4443ac51e74SDarrick J. Wong 4453ac51e74SDarrick J. Wong return 0; 4463ac51e74SDarrick J. Wong } 4473ac51e74SDarrick J. Wong 4483ac51e74SDarrick J. Wong /* 4493ac51e74SDarrick J. Wong * Set the geometry of a device. 4503ac51e74SDarrick J. Wong */ 4513ac51e74SDarrick J. Wong int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo) 4523ac51e74SDarrick J. Wong { 4533ac51e74SDarrick J. Wong sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors; 4543ac51e74SDarrick J. Wong 4553ac51e74SDarrick J. Wong if (geo->start > sz) { 4563ac51e74SDarrick J. Wong DMWARN("Start sector is beyond the geometry limits."); 4573ac51e74SDarrick J. Wong return -EINVAL; 4583ac51e74SDarrick J. Wong } 4593ac51e74SDarrick J. Wong 4603ac51e74SDarrick J. Wong md->geometry = *geo; 4613ac51e74SDarrick J. Wong 4623ac51e74SDarrick J. Wong return 0; 4633ac51e74SDarrick J. Wong } 4643ac51e74SDarrick J. Wong 4651da177e4SLinus Torvalds /*----------------------------------------------------------------- 4661da177e4SLinus Torvalds * CRUD START: 4671da177e4SLinus Torvalds * A more elegant soln is in the works that uses the queue 4681da177e4SLinus Torvalds * merge fn, unfortunately there are a couple of changes to 4691da177e4SLinus Torvalds * the block layer that I want to make for this. So in the 4701da177e4SLinus Torvalds * interests of getting something for people to use I give 4711da177e4SLinus Torvalds * you this clearly demarcated crap. 4721da177e4SLinus Torvalds *---------------------------------------------------------------*/ 4731da177e4SLinus Torvalds 4742e93ccc1SKiyoshi Ueda static int __noflush_suspending(struct mapped_device *md) 4752e93ccc1SKiyoshi Ueda { 4762e93ccc1SKiyoshi Ueda return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 4772e93ccc1SKiyoshi Ueda } 4782e93ccc1SKiyoshi Ueda 4791da177e4SLinus Torvalds /* 4801da177e4SLinus Torvalds * Decrements the number of outstanding ios that a bio has been 4811da177e4SLinus Torvalds * cloned into, completing the original io if necc. 4821da177e4SLinus Torvalds */ 483858119e1SArjan van de Ven static void dec_pending(struct dm_io *io, int error) 4841da177e4SLinus Torvalds { 4852e93ccc1SKiyoshi Ueda unsigned long flags; 4862e93ccc1SKiyoshi Ueda 4872e93ccc1SKiyoshi Ueda /* Push-back supersedes any I/O errors */ 4882e93ccc1SKiyoshi Ueda if (error && !(io->error > 0 && __noflush_suspending(io->md))) 4891da177e4SLinus Torvalds io->error = error; 4901da177e4SLinus Torvalds 4911da177e4SLinus Torvalds if (atomic_dec_and_test(&io->io_count)) { 4922e93ccc1SKiyoshi Ueda if (io->error == DM_ENDIO_REQUEUE) { 4932e93ccc1SKiyoshi Ueda /* 4942e93ccc1SKiyoshi Ueda * Target requested pushing back the I/O. 4952e93ccc1SKiyoshi Ueda * This must be handled before the sleeper on 4962e93ccc1SKiyoshi Ueda * suspend queue merges the pushback list. 4972e93ccc1SKiyoshi Ueda */ 4982e93ccc1SKiyoshi Ueda spin_lock_irqsave(&io->md->pushback_lock, flags); 4992e93ccc1SKiyoshi Ueda if (__noflush_suspending(io->md)) 5002e93ccc1SKiyoshi Ueda bio_list_add(&io->md->pushback, io->bio); 5012e93ccc1SKiyoshi Ueda else 5022e93ccc1SKiyoshi Ueda /* noflush suspend was interrupted. */ 5032e93ccc1SKiyoshi Ueda io->error = -EIO; 5042e93ccc1SKiyoshi Ueda spin_unlock_irqrestore(&io->md->pushback_lock, flags); 5052e93ccc1SKiyoshi Ueda } 5062e93ccc1SKiyoshi Ueda 507d221d2e7SMikulas Patocka end_io_acct(io); 5081da177e4SLinus Torvalds 5092e93ccc1SKiyoshi Ueda if (io->error != DM_ENDIO_REQUEUE) { 5105f3ea37cSArnaldo Carvalho de Melo trace_block_bio_complete(io->md->queue, io->bio); 5112056a782SJens Axboe 5126712ecf8SNeilBrown bio_endio(io->bio, io->error); 5132e93ccc1SKiyoshi Ueda } 5142e93ccc1SKiyoshi Ueda 5151da177e4SLinus Torvalds free_io(io->md, io); 5161da177e4SLinus Torvalds } 5171da177e4SLinus Torvalds } 5181da177e4SLinus Torvalds 5196712ecf8SNeilBrown static void clone_endio(struct bio *bio, int error) 5201da177e4SLinus Torvalds { 5211da177e4SLinus Torvalds int r = 0; 522028867acSAlasdair G Kergon struct dm_target_io *tio = bio->bi_private; 5239faf400fSStefan Bader struct mapped_device *md = tio->io->md; 5241da177e4SLinus Torvalds dm_endio_fn endio = tio->ti->type->end_io; 5251da177e4SLinus Torvalds 5261da177e4SLinus Torvalds if (!bio_flagged(bio, BIO_UPTODATE) && !error) 5271da177e4SLinus Torvalds error = -EIO; 5281da177e4SLinus Torvalds 5291da177e4SLinus Torvalds if (endio) { 5301da177e4SLinus Torvalds r = endio(tio->ti, bio, error, &tio->info); 5312e93ccc1SKiyoshi Ueda if (r < 0 || r == DM_ENDIO_REQUEUE) 5322e93ccc1SKiyoshi Ueda /* 5332e93ccc1SKiyoshi Ueda * error and requeue request are handled 5342e93ccc1SKiyoshi Ueda * in dec_pending(). 5352e93ccc1SKiyoshi Ueda */ 5361da177e4SLinus Torvalds error = r; 53745cbcd79SKiyoshi Ueda else if (r == DM_ENDIO_INCOMPLETE) 53845cbcd79SKiyoshi Ueda /* The target will handle the io */ 5396712ecf8SNeilBrown return; 54045cbcd79SKiyoshi Ueda else if (r) { 54145cbcd79SKiyoshi Ueda DMWARN("unimplemented target endio return value: %d", r); 54245cbcd79SKiyoshi Ueda BUG(); 54345cbcd79SKiyoshi Ueda } 5441da177e4SLinus Torvalds } 5451da177e4SLinus Torvalds 5469faf400fSStefan Bader dec_pending(tio->io, error); 5479faf400fSStefan Bader 5489faf400fSStefan Bader /* 5499faf400fSStefan Bader * Store md for cleanup instead of tio which is about to get freed. 5509faf400fSStefan Bader */ 5519faf400fSStefan Bader bio->bi_private = md->bs; 5529faf400fSStefan Bader 5531da177e4SLinus Torvalds bio_put(bio); 5549faf400fSStefan Bader free_tio(md, tio); 5551da177e4SLinus Torvalds } 5561da177e4SLinus Torvalds 5571da177e4SLinus Torvalds static sector_t max_io_len(struct mapped_device *md, 5581da177e4SLinus Torvalds sector_t sector, struct dm_target *ti) 5591da177e4SLinus Torvalds { 5601da177e4SLinus Torvalds sector_t offset = sector - ti->begin; 5611da177e4SLinus Torvalds sector_t len = ti->len - offset; 5621da177e4SLinus Torvalds 5631da177e4SLinus Torvalds /* 5641da177e4SLinus Torvalds * Does the target need to split even further ? 5651da177e4SLinus Torvalds */ 5661da177e4SLinus Torvalds if (ti->split_io) { 5671da177e4SLinus Torvalds sector_t boundary; 5681da177e4SLinus Torvalds boundary = ((offset + ti->split_io) & ~(ti->split_io - 1)) 5691da177e4SLinus Torvalds - offset; 5701da177e4SLinus Torvalds if (len > boundary) 5711da177e4SLinus Torvalds len = boundary; 5721da177e4SLinus Torvalds } 5731da177e4SLinus Torvalds 5741da177e4SLinus Torvalds return len; 5751da177e4SLinus Torvalds } 5761da177e4SLinus Torvalds 5771da177e4SLinus Torvalds static void __map_bio(struct dm_target *ti, struct bio *clone, 578028867acSAlasdair G Kergon struct dm_target_io *tio) 5791da177e4SLinus Torvalds { 5801da177e4SLinus Torvalds int r; 5812056a782SJens Axboe sector_t sector; 5829faf400fSStefan Bader struct mapped_device *md; 5831da177e4SLinus Torvalds 5841da177e4SLinus Torvalds /* 5851da177e4SLinus Torvalds * Sanity checks. 5861da177e4SLinus Torvalds */ 5871da177e4SLinus Torvalds BUG_ON(!clone->bi_size); 5881da177e4SLinus Torvalds 5891da177e4SLinus Torvalds clone->bi_end_io = clone_endio; 5901da177e4SLinus Torvalds clone->bi_private = tio; 5911da177e4SLinus Torvalds 5921da177e4SLinus Torvalds /* 5931da177e4SLinus Torvalds * Map the clone. If r == 0 we don't need to do 5941da177e4SLinus Torvalds * anything, the target has assumed ownership of 5951da177e4SLinus Torvalds * this io. 5961da177e4SLinus Torvalds */ 5971da177e4SLinus Torvalds atomic_inc(&tio->io->io_count); 5982056a782SJens Axboe sector = clone->bi_sector; 5991da177e4SLinus Torvalds r = ti->type->map(ti, clone, &tio->info); 60045cbcd79SKiyoshi Ueda if (r == DM_MAPIO_REMAPPED) { 6011da177e4SLinus Torvalds /* the bio has been remapped so dispatch it */ 6022056a782SJens Axboe 6035f3ea37cSArnaldo Carvalho de Melo trace_block_remap(bdev_get_queue(clone->bi_bdev), clone, 604c7149d6bSAlan D. Brunelle tio->io->bio->bi_bdev->bd_dev, 605c7149d6bSAlan D. Brunelle clone->bi_sector, sector); 6062056a782SJens Axboe 6071da177e4SLinus Torvalds generic_make_request(clone); 6082e93ccc1SKiyoshi Ueda } else if (r < 0 || r == DM_MAPIO_REQUEUE) { 6092e93ccc1SKiyoshi Ueda /* error the io and bail out, or requeue it if needed */ 6109faf400fSStefan Bader md = tio->io->md; 6119faf400fSStefan Bader dec_pending(tio->io, r); 6129faf400fSStefan Bader /* 6139faf400fSStefan Bader * Store bio_set for cleanup. 6149faf400fSStefan Bader */ 6159faf400fSStefan Bader clone->bi_private = md->bs; 6161da177e4SLinus Torvalds bio_put(clone); 6179faf400fSStefan Bader free_tio(md, tio); 61845cbcd79SKiyoshi Ueda } else if (r) { 61945cbcd79SKiyoshi Ueda DMWARN("unimplemented target map return value: %d", r); 62045cbcd79SKiyoshi Ueda BUG(); 6211da177e4SLinus Torvalds } 6221da177e4SLinus Torvalds } 6231da177e4SLinus Torvalds 6241da177e4SLinus Torvalds struct clone_info { 6251da177e4SLinus Torvalds struct mapped_device *md; 6261da177e4SLinus Torvalds struct dm_table *map; 6271da177e4SLinus Torvalds struct bio *bio; 6281da177e4SLinus Torvalds struct dm_io *io; 6291da177e4SLinus Torvalds sector_t sector; 6301da177e4SLinus Torvalds sector_t sector_count; 6311da177e4SLinus Torvalds unsigned short idx; 6321da177e4SLinus Torvalds }; 6331da177e4SLinus Torvalds 6343676347aSPeter Osterlund static void dm_bio_destructor(struct bio *bio) 6353676347aSPeter Osterlund { 6369faf400fSStefan Bader struct bio_set *bs = bio->bi_private; 6379faf400fSStefan Bader 6389faf400fSStefan Bader bio_free(bio, bs); 6393676347aSPeter Osterlund } 6403676347aSPeter Osterlund 6411da177e4SLinus Torvalds /* 6421da177e4SLinus Torvalds * Creates a little bio that is just does part of a bvec. 6431da177e4SLinus Torvalds */ 6441da177e4SLinus Torvalds static struct bio *split_bvec(struct bio *bio, sector_t sector, 6451da177e4SLinus Torvalds unsigned short idx, unsigned int offset, 6469faf400fSStefan Bader unsigned int len, struct bio_set *bs) 6471da177e4SLinus Torvalds { 6481da177e4SLinus Torvalds struct bio *clone; 6491da177e4SLinus Torvalds struct bio_vec *bv = bio->bi_io_vec + idx; 6501da177e4SLinus Torvalds 6519faf400fSStefan Bader clone = bio_alloc_bioset(GFP_NOIO, 1, bs); 6523676347aSPeter Osterlund clone->bi_destructor = dm_bio_destructor; 6531da177e4SLinus Torvalds *clone->bi_io_vec = *bv; 6541da177e4SLinus Torvalds 6551da177e4SLinus Torvalds clone->bi_sector = sector; 6561da177e4SLinus Torvalds clone->bi_bdev = bio->bi_bdev; 6571da177e4SLinus Torvalds clone->bi_rw = bio->bi_rw; 6581da177e4SLinus Torvalds clone->bi_vcnt = 1; 6591da177e4SLinus Torvalds clone->bi_size = to_bytes(len); 6601da177e4SLinus Torvalds clone->bi_io_vec->bv_offset = offset; 6611da177e4SLinus Torvalds clone->bi_io_vec->bv_len = clone->bi_size; 662f3e1d26eSMartin K. Petersen clone->bi_flags |= 1 << BIO_CLONED; 6631da177e4SLinus Torvalds 6641da177e4SLinus Torvalds return clone; 6651da177e4SLinus Torvalds } 6661da177e4SLinus Torvalds 6671da177e4SLinus Torvalds /* 6681da177e4SLinus Torvalds * Creates a bio that consists of range of complete bvecs. 6691da177e4SLinus Torvalds */ 6701da177e4SLinus Torvalds static struct bio *clone_bio(struct bio *bio, sector_t sector, 6711da177e4SLinus Torvalds unsigned short idx, unsigned short bv_count, 6729faf400fSStefan Bader unsigned int len, struct bio_set *bs) 6731da177e4SLinus Torvalds { 6741da177e4SLinus Torvalds struct bio *clone; 6751da177e4SLinus Torvalds 6769faf400fSStefan Bader clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs); 6779faf400fSStefan Bader __bio_clone(clone, bio); 6789faf400fSStefan Bader clone->bi_destructor = dm_bio_destructor; 6791da177e4SLinus Torvalds clone->bi_sector = sector; 6801da177e4SLinus Torvalds clone->bi_idx = idx; 6811da177e4SLinus Torvalds clone->bi_vcnt = idx + bv_count; 6821da177e4SLinus Torvalds clone->bi_size = to_bytes(len); 6831da177e4SLinus Torvalds clone->bi_flags &= ~(1 << BIO_SEG_VALID); 6841da177e4SLinus Torvalds 6851da177e4SLinus Torvalds return clone; 6861da177e4SLinus Torvalds } 6871da177e4SLinus Torvalds 688512875bdSJun'ichi Nomura static int __clone_and_map(struct clone_info *ci) 6891da177e4SLinus Torvalds { 6901da177e4SLinus Torvalds struct bio *clone, *bio = ci->bio; 691512875bdSJun'ichi Nomura struct dm_target *ti; 692512875bdSJun'ichi Nomura sector_t len = 0, max; 693028867acSAlasdair G Kergon struct dm_target_io *tio; 6941da177e4SLinus Torvalds 695512875bdSJun'ichi Nomura ti = dm_table_find_target(ci->map, ci->sector); 696512875bdSJun'ichi Nomura if (!dm_target_is_valid(ti)) 697512875bdSJun'ichi Nomura return -EIO; 698512875bdSJun'ichi Nomura 699512875bdSJun'ichi Nomura max = max_io_len(ci->md, ci->sector, ti); 700512875bdSJun'ichi Nomura 7011da177e4SLinus Torvalds /* 7021da177e4SLinus Torvalds * Allocate a target io object. 7031da177e4SLinus Torvalds */ 7041da177e4SLinus Torvalds tio = alloc_tio(ci->md); 7051da177e4SLinus Torvalds tio->io = ci->io; 7061da177e4SLinus Torvalds tio->ti = ti; 7071da177e4SLinus Torvalds memset(&tio->info, 0, sizeof(tio->info)); 7081da177e4SLinus Torvalds 7091da177e4SLinus Torvalds if (ci->sector_count <= max) { 7101da177e4SLinus Torvalds /* 7111da177e4SLinus Torvalds * Optimise for the simple case where we can do all of 7121da177e4SLinus Torvalds * the remaining io with a single clone. 7131da177e4SLinus Torvalds */ 7141da177e4SLinus Torvalds clone = clone_bio(bio, ci->sector, ci->idx, 7159faf400fSStefan Bader bio->bi_vcnt - ci->idx, ci->sector_count, 7169faf400fSStefan Bader ci->md->bs); 7171da177e4SLinus Torvalds __map_bio(ti, clone, tio); 7181da177e4SLinus Torvalds ci->sector_count = 0; 7191da177e4SLinus Torvalds 7201da177e4SLinus Torvalds } else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) { 7211da177e4SLinus Torvalds /* 7221da177e4SLinus Torvalds * There are some bvecs that don't span targets. 7231da177e4SLinus Torvalds * Do as many of these as possible. 7241da177e4SLinus Torvalds */ 7251da177e4SLinus Torvalds int i; 7261da177e4SLinus Torvalds sector_t remaining = max; 7271da177e4SLinus Torvalds sector_t bv_len; 7281da177e4SLinus Torvalds 7291da177e4SLinus Torvalds for (i = ci->idx; remaining && (i < bio->bi_vcnt); i++) { 7301da177e4SLinus Torvalds bv_len = to_sector(bio->bi_io_vec[i].bv_len); 7311da177e4SLinus Torvalds 7321da177e4SLinus Torvalds if (bv_len > remaining) 7331da177e4SLinus Torvalds break; 7341da177e4SLinus Torvalds 7351da177e4SLinus Torvalds remaining -= bv_len; 7361da177e4SLinus Torvalds len += bv_len; 7371da177e4SLinus Torvalds } 7381da177e4SLinus Torvalds 7399faf400fSStefan Bader clone = clone_bio(bio, ci->sector, ci->idx, i - ci->idx, len, 7409faf400fSStefan Bader ci->md->bs); 7411da177e4SLinus Torvalds __map_bio(ti, clone, tio); 7421da177e4SLinus Torvalds 7431da177e4SLinus Torvalds ci->sector += len; 7441da177e4SLinus Torvalds ci->sector_count -= len; 7451da177e4SLinus Torvalds ci->idx = i; 7461da177e4SLinus Torvalds 7471da177e4SLinus Torvalds } else { 7481da177e4SLinus Torvalds /* 749d2044a94SAlasdair G Kergon * Handle a bvec that must be split between two or more targets. 7501da177e4SLinus Torvalds */ 7511da177e4SLinus Torvalds struct bio_vec *bv = bio->bi_io_vec + ci->idx; 752d2044a94SAlasdair G Kergon sector_t remaining = to_sector(bv->bv_len); 753d2044a94SAlasdair G Kergon unsigned int offset = 0; 7541da177e4SLinus Torvalds 755d2044a94SAlasdair G Kergon do { 756d2044a94SAlasdair G Kergon if (offset) { 7571da177e4SLinus Torvalds ti = dm_table_find_target(ci->map, ci->sector); 758512875bdSJun'ichi Nomura if (!dm_target_is_valid(ti)) 759512875bdSJun'ichi Nomura return -EIO; 760512875bdSJun'ichi Nomura 761d2044a94SAlasdair G Kergon max = max_io_len(ci->md, ci->sector, ti); 7621da177e4SLinus Torvalds 7631da177e4SLinus Torvalds tio = alloc_tio(ci->md); 7641da177e4SLinus Torvalds tio->io = ci->io; 7651da177e4SLinus Torvalds tio->ti = ti; 7661da177e4SLinus Torvalds memset(&tio->info, 0, sizeof(tio->info)); 767d2044a94SAlasdair G Kergon } 768d2044a94SAlasdair G Kergon 769d2044a94SAlasdair G Kergon len = min(remaining, max); 770d2044a94SAlasdair G Kergon 771d2044a94SAlasdair G Kergon clone = split_bvec(bio, ci->sector, ci->idx, 7729faf400fSStefan Bader bv->bv_offset + offset, len, 7739faf400fSStefan Bader ci->md->bs); 774d2044a94SAlasdair G Kergon 7751da177e4SLinus Torvalds __map_bio(ti, clone, tio); 7761da177e4SLinus Torvalds 7771da177e4SLinus Torvalds ci->sector += len; 7781da177e4SLinus Torvalds ci->sector_count -= len; 779d2044a94SAlasdair G Kergon offset += to_bytes(len); 780d2044a94SAlasdair G Kergon } while (remaining -= len); 781d2044a94SAlasdair G Kergon 7821da177e4SLinus Torvalds ci->idx++; 7831da177e4SLinus Torvalds } 784512875bdSJun'ichi Nomura 785512875bdSJun'ichi Nomura return 0; 7861da177e4SLinus Torvalds } 7871da177e4SLinus Torvalds 7881da177e4SLinus Torvalds /* 7891da177e4SLinus Torvalds * Split the bio into several clones. 7901da177e4SLinus Torvalds */ 7919e4e5f87SMilan Broz static int __split_bio(struct mapped_device *md, struct bio *bio) 7921da177e4SLinus Torvalds { 7931da177e4SLinus Torvalds struct clone_info ci; 794512875bdSJun'ichi Nomura int error = 0; 7951da177e4SLinus Torvalds 7961da177e4SLinus Torvalds ci.map = dm_get_table(md); 7979e4e5f87SMilan Broz if (unlikely(!ci.map)) 7989e4e5f87SMilan Broz return -EIO; 7991da177e4SLinus Torvalds 8001da177e4SLinus Torvalds ci.md = md; 8011da177e4SLinus Torvalds ci.bio = bio; 8021da177e4SLinus Torvalds ci.io = alloc_io(md); 8031da177e4SLinus Torvalds ci.io->error = 0; 8041da177e4SLinus Torvalds atomic_set(&ci.io->io_count, 1); 8051da177e4SLinus Torvalds ci.io->bio = bio; 8061da177e4SLinus Torvalds ci.io->md = md; 8071da177e4SLinus Torvalds ci.sector = bio->bi_sector; 8081da177e4SLinus Torvalds ci.sector_count = bio_sectors(bio); 8091da177e4SLinus Torvalds ci.idx = bio->bi_idx; 8101da177e4SLinus Torvalds 8113eaf840eSJun'ichi "Nick" Nomura start_io_acct(ci.io); 812512875bdSJun'ichi Nomura while (ci.sector_count && !error) 813512875bdSJun'ichi Nomura error = __clone_and_map(&ci); 8141da177e4SLinus Torvalds 8151da177e4SLinus Torvalds /* drop the extra reference count */ 816512875bdSJun'ichi Nomura dec_pending(ci.io, error); 8171da177e4SLinus Torvalds dm_table_put(ci.map); 8189e4e5f87SMilan Broz 8199e4e5f87SMilan Broz return 0; 8201da177e4SLinus Torvalds } 8211da177e4SLinus Torvalds /*----------------------------------------------------------------- 8221da177e4SLinus Torvalds * CRUD END 8231da177e4SLinus Torvalds *---------------------------------------------------------------*/ 8241da177e4SLinus Torvalds 825f6fccb12SMilan Broz static int dm_merge_bvec(struct request_queue *q, 826f6fccb12SMilan Broz struct bvec_merge_data *bvm, 827f6fccb12SMilan Broz struct bio_vec *biovec) 828f6fccb12SMilan Broz { 829f6fccb12SMilan Broz struct mapped_device *md = q->queuedata; 830f6fccb12SMilan Broz struct dm_table *map = dm_get_table(md); 831f6fccb12SMilan Broz struct dm_target *ti; 832f6fccb12SMilan Broz sector_t max_sectors; 8335037108aSMikulas Patocka int max_size = 0; 834f6fccb12SMilan Broz 835f6fccb12SMilan Broz if (unlikely(!map)) 8365037108aSMikulas Patocka goto out; 837f6fccb12SMilan Broz 838f6fccb12SMilan Broz ti = dm_table_find_target(map, bvm->bi_sector); 839b01cd5acSMikulas Patocka if (!dm_target_is_valid(ti)) 840b01cd5acSMikulas Patocka goto out_table; 841f6fccb12SMilan Broz 842f6fccb12SMilan Broz /* 843f6fccb12SMilan Broz * Find maximum amount of I/O that won't need splitting 844f6fccb12SMilan Broz */ 845f6fccb12SMilan Broz max_sectors = min(max_io_len(md, bvm->bi_sector, ti), 846f6fccb12SMilan Broz (sector_t) BIO_MAX_SECTORS); 847f6fccb12SMilan Broz max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size; 848f6fccb12SMilan Broz if (max_size < 0) 849f6fccb12SMilan Broz max_size = 0; 850f6fccb12SMilan Broz 851f6fccb12SMilan Broz /* 852f6fccb12SMilan Broz * merge_bvec_fn() returns number of bytes 853f6fccb12SMilan Broz * it can accept at this offset 854f6fccb12SMilan Broz * max is precomputed maximal io size 855f6fccb12SMilan Broz */ 856f6fccb12SMilan Broz if (max_size && ti->type->merge) 857f6fccb12SMilan Broz max_size = ti->type->merge(ti, bvm, biovec, max_size); 858f6fccb12SMilan Broz 859b01cd5acSMikulas Patocka out_table: 8605037108aSMikulas Patocka dm_table_put(map); 8615037108aSMikulas Patocka 8625037108aSMikulas Patocka out: 863f6fccb12SMilan Broz /* 864f6fccb12SMilan Broz * Always allow an entire first page 865f6fccb12SMilan Broz */ 866f6fccb12SMilan Broz if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT)) 867f6fccb12SMilan Broz max_size = biovec->bv_len; 868f6fccb12SMilan Broz 869f6fccb12SMilan Broz return max_size; 870f6fccb12SMilan Broz } 871f6fccb12SMilan Broz 8721da177e4SLinus Torvalds /* 8731da177e4SLinus Torvalds * The request function that just remaps the bio built up by 8741da177e4SLinus Torvalds * dm_merge_bvec. 8751da177e4SLinus Torvalds */ 876165125e1SJens Axboe static int dm_request(struct request_queue *q, struct bio *bio) 8771da177e4SLinus Torvalds { 8789e4e5f87SMilan Broz int r = -EIO; 87912f03a49SKevin Corry int rw = bio_data_dir(bio); 8801da177e4SLinus Torvalds struct mapped_device *md = q->queuedata; 881c9959059STejun Heo int cpu; 8821da177e4SLinus Torvalds 88307a83c47SStefan Bader /* 88407a83c47SStefan Bader * There is no use in forwarding any barrier request since we can't 88507a83c47SStefan Bader * guarantee it is (or can be) handled by the targets correctly. 88607a83c47SStefan Bader */ 88707a83c47SStefan Bader if (unlikely(bio_barrier(bio))) { 8886712ecf8SNeilBrown bio_endio(bio, -EOPNOTSUPP); 88907a83c47SStefan Bader return 0; 89007a83c47SStefan Bader } 89107a83c47SStefan Bader 8922ca3310eSAlasdair G Kergon down_read(&md->io_lock); 8931da177e4SLinus Torvalds 894074a7acaSTejun Heo cpu = part_stat_lock(); 895074a7acaSTejun Heo part_stat_inc(cpu, &dm_disk(md)->part0, ios[rw]); 896074a7acaSTejun Heo part_stat_add(cpu, &dm_disk(md)->part0, sectors[rw], bio_sectors(bio)); 897074a7acaSTejun Heo part_stat_unlock(); 89812f03a49SKevin Corry 8991da177e4SLinus Torvalds /* 9001da177e4SLinus Torvalds * If we're suspended we have to queue 9011da177e4SLinus Torvalds * this io for later. 9021da177e4SLinus Torvalds */ 9031da177e4SLinus Torvalds while (test_bit(DMF_BLOCK_IO, &md->flags)) { 9042ca3310eSAlasdair G Kergon up_read(&md->io_lock); 9051da177e4SLinus Torvalds 9069e4e5f87SMilan Broz if (bio_rw(bio) != READA) 9071da177e4SLinus Torvalds r = queue_io(md, bio); 9081da177e4SLinus Torvalds 9099e4e5f87SMilan Broz if (r <= 0) 9109e4e5f87SMilan Broz goto out_req; 9111da177e4SLinus Torvalds 9121da177e4SLinus Torvalds /* 9131da177e4SLinus Torvalds * We're in a while loop, because someone could suspend 9141da177e4SLinus Torvalds * before we get to the following read lock. 9151da177e4SLinus Torvalds */ 9162ca3310eSAlasdair G Kergon down_read(&md->io_lock); 9171da177e4SLinus Torvalds } 9181da177e4SLinus Torvalds 9199e4e5f87SMilan Broz r = __split_bio(md, bio); 9202ca3310eSAlasdair G Kergon up_read(&md->io_lock); 9219e4e5f87SMilan Broz 9229e4e5f87SMilan Broz out_req: 9239e4e5f87SMilan Broz if (r < 0) 9249e4e5f87SMilan Broz bio_io_error(bio); 9259e4e5f87SMilan Broz 9261da177e4SLinus Torvalds return 0; 9271da177e4SLinus Torvalds } 9281da177e4SLinus Torvalds 929165125e1SJens Axboe static void dm_unplug_all(struct request_queue *q) 9301da177e4SLinus Torvalds { 9311da177e4SLinus Torvalds struct mapped_device *md = q->queuedata; 9321da177e4SLinus Torvalds struct dm_table *map = dm_get_table(md); 9331da177e4SLinus Torvalds 9341da177e4SLinus Torvalds if (map) { 9351da177e4SLinus Torvalds dm_table_unplug_all(map); 9361da177e4SLinus Torvalds dm_table_put(map); 9371da177e4SLinus Torvalds } 9381da177e4SLinus Torvalds } 9391da177e4SLinus Torvalds 9401da177e4SLinus Torvalds static int dm_any_congested(void *congested_data, int bdi_bits) 9411da177e4SLinus Torvalds { 9428a57dfc6SChandra Seetharaman int r = bdi_bits; 9438a57dfc6SChandra Seetharaman struct mapped_device *md = congested_data; 9448a57dfc6SChandra Seetharaman struct dm_table *map; 9451da177e4SLinus Torvalds 9468a57dfc6SChandra Seetharaman atomic_inc(&md->pending); 9478a57dfc6SChandra Seetharaman 9488a57dfc6SChandra Seetharaman if (!test_bit(DMF_BLOCK_IO, &md->flags)) { 9498a57dfc6SChandra Seetharaman map = dm_get_table(md); 9508a57dfc6SChandra Seetharaman if (map) { 9511da177e4SLinus Torvalds r = dm_table_any_congested(map, bdi_bits); 9521da177e4SLinus Torvalds dm_table_put(map); 9538a57dfc6SChandra Seetharaman } 9548a57dfc6SChandra Seetharaman } 9558a57dfc6SChandra Seetharaman 9568a57dfc6SChandra Seetharaman if (!atomic_dec_return(&md->pending)) 9578a57dfc6SChandra Seetharaman /* nudge anyone waiting on suspend queue */ 9588a57dfc6SChandra Seetharaman wake_up(&md->wait); 9598a57dfc6SChandra Seetharaman 9601da177e4SLinus Torvalds return r; 9611da177e4SLinus Torvalds } 9621da177e4SLinus Torvalds 9631da177e4SLinus Torvalds /*----------------------------------------------------------------- 9641da177e4SLinus Torvalds * An IDR is used to keep track of allocated minor numbers. 9651da177e4SLinus Torvalds *---------------------------------------------------------------*/ 9661da177e4SLinus Torvalds static DEFINE_IDR(_minor_idr); 9671da177e4SLinus Torvalds 9682b06cfffSAlasdair G Kergon static void free_minor(int minor) 9691da177e4SLinus Torvalds { 970f32c10b0SJeff Mahoney spin_lock(&_minor_lock); 9711da177e4SLinus Torvalds idr_remove(&_minor_idr, minor); 972f32c10b0SJeff Mahoney spin_unlock(&_minor_lock); 9731da177e4SLinus Torvalds } 9741da177e4SLinus Torvalds 9751da177e4SLinus Torvalds /* 9761da177e4SLinus Torvalds * See if the device with a specific minor # is free. 9771da177e4SLinus Torvalds */ 978cf13ab8eSFrederik Deweerdt static int specific_minor(int minor) 9791da177e4SLinus Torvalds { 9801da177e4SLinus Torvalds int r, m; 9811da177e4SLinus Torvalds 9821da177e4SLinus Torvalds if (minor >= (1 << MINORBITS)) 9831da177e4SLinus Torvalds return -EINVAL; 9841da177e4SLinus Torvalds 98562f75c2fSJeff Mahoney r = idr_pre_get(&_minor_idr, GFP_KERNEL); 98662f75c2fSJeff Mahoney if (!r) 98762f75c2fSJeff Mahoney return -ENOMEM; 98862f75c2fSJeff Mahoney 989f32c10b0SJeff Mahoney spin_lock(&_minor_lock); 9901da177e4SLinus Torvalds 9911da177e4SLinus Torvalds if (idr_find(&_minor_idr, minor)) { 9921da177e4SLinus Torvalds r = -EBUSY; 9931da177e4SLinus Torvalds goto out; 9941da177e4SLinus Torvalds } 9951da177e4SLinus Torvalds 996ba61fdd1SJeff Mahoney r = idr_get_new_above(&_minor_idr, MINOR_ALLOCED, minor, &m); 99762f75c2fSJeff Mahoney if (r) 9981da177e4SLinus Torvalds goto out; 9991da177e4SLinus Torvalds 10001da177e4SLinus Torvalds if (m != minor) { 10011da177e4SLinus Torvalds idr_remove(&_minor_idr, m); 10021da177e4SLinus Torvalds r = -EBUSY; 10031da177e4SLinus Torvalds goto out; 10041da177e4SLinus Torvalds } 10051da177e4SLinus Torvalds 10061da177e4SLinus Torvalds out: 1007f32c10b0SJeff Mahoney spin_unlock(&_minor_lock); 10081da177e4SLinus Torvalds return r; 10091da177e4SLinus Torvalds } 10101da177e4SLinus Torvalds 1011cf13ab8eSFrederik Deweerdt static int next_free_minor(int *minor) 10121da177e4SLinus Torvalds { 10132b06cfffSAlasdair G Kergon int r, m; 10141da177e4SLinus Torvalds 10151da177e4SLinus Torvalds r = idr_pre_get(&_minor_idr, GFP_KERNEL); 101662f75c2fSJeff Mahoney if (!r) 101762f75c2fSJeff Mahoney return -ENOMEM; 101862f75c2fSJeff Mahoney 1019f32c10b0SJeff Mahoney spin_lock(&_minor_lock); 10201da177e4SLinus Torvalds 1021ba61fdd1SJeff Mahoney r = idr_get_new(&_minor_idr, MINOR_ALLOCED, &m); 1022cf13ab8eSFrederik Deweerdt if (r) 10231da177e4SLinus Torvalds goto out; 10241da177e4SLinus Torvalds 10251da177e4SLinus Torvalds if (m >= (1 << MINORBITS)) { 10261da177e4SLinus Torvalds idr_remove(&_minor_idr, m); 10271da177e4SLinus Torvalds r = -ENOSPC; 10281da177e4SLinus Torvalds goto out; 10291da177e4SLinus Torvalds } 10301da177e4SLinus Torvalds 10311da177e4SLinus Torvalds *minor = m; 10321da177e4SLinus Torvalds 10331da177e4SLinus Torvalds out: 1034f32c10b0SJeff Mahoney spin_unlock(&_minor_lock); 10351da177e4SLinus Torvalds return r; 10361da177e4SLinus Torvalds } 10371da177e4SLinus Torvalds 10381da177e4SLinus Torvalds static struct block_device_operations dm_blk_dops; 10391da177e4SLinus Torvalds 10401da177e4SLinus Torvalds /* 10411da177e4SLinus Torvalds * Allocate and initialise a blank device with a given minor. 10421da177e4SLinus Torvalds */ 10432b06cfffSAlasdair G Kergon static struct mapped_device *alloc_dev(int minor) 10441da177e4SLinus Torvalds { 10451da177e4SLinus Torvalds int r; 1046cf13ab8eSFrederik Deweerdt struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL); 1047ba61fdd1SJeff Mahoney void *old_md; 10481da177e4SLinus Torvalds 10491da177e4SLinus Torvalds if (!md) { 10501da177e4SLinus Torvalds DMWARN("unable to allocate device, out of memory."); 10511da177e4SLinus Torvalds return NULL; 10521da177e4SLinus Torvalds } 10531da177e4SLinus Torvalds 105410da4f79SJeff Mahoney if (!try_module_get(THIS_MODULE)) 10556ed7ade8SMilan Broz goto bad_module_get; 105610da4f79SJeff Mahoney 10571da177e4SLinus Torvalds /* get a minor number for the dev */ 10582b06cfffSAlasdair G Kergon if (minor == DM_ANY_MINOR) 1059cf13ab8eSFrederik Deweerdt r = next_free_minor(&minor); 10602b06cfffSAlasdair G Kergon else 1061cf13ab8eSFrederik Deweerdt r = specific_minor(minor); 10621da177e4SLinus Torvalds if (r < 0) 10636ed7ade8SMilan Broz goto bad_minor; 10641da177e4SLinus Torvalds 10652ca3310eSAlasdair G Kergon init_rwsem(&md->io_lock); 1066e61290a4SDaniel Walker mutex_init(&md->suspend_lock); 10672e93ccc1SKiyoshi Ueda spin_lock_init(&md->pushback_lock); 10681da177e4SLinus Torvalds rwlock_init(&md->map_lock); 10691da177e4SLinus Torvalds atomic_set(&md->holders, 1); 10705c6bd75dSAlasdair G Kergon atomic_set(&md->open_count, 0); 10711da177e4SLinus Torvalds atomic_set(&md->event_nr, 0); 10727a8c3d3bSMike Anderson atomic_set(&md->uevent_seq, 0); 10737a8c3d3bSMike Anderson INIT_LIST_HEAD(&md->uevent_list); 10747a8c3d3bSMike Anderson spin_lock_init(&md->uevent_lock); 10751da177e4SLinus Torvalds 10761da177e4SLinus Torvalds md->queue = blk_alloc_queue(GFP_KERNEL); 10771da177e4SLinus Torvalds if (!md->queue) 10786ed7ade8SMilan Broz goto bad_queue; 10791da177e4SLinus Torvalds 10801da177e4SLinus Torvalds md->queue->queuedata = md; 10811da177e4SLinus Torvalds md->queue->backing_dev_info.congested_fn = dm_any_congested; 10821da177e4SLinus Torvalds md->queue->backing_dev_info.congested_data = md; 10831da177e4SLinus Torvalds blk_queue_make_request(md->queue, dm_request); 1084daef265fSJens Axboe blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY); 10851da177e4SLinus Torvalds md->queue->unplug_fn = dm_unplug_all; 1086f6fccb12SMilan Broz blk_queue_merge_bvec(md->queue, dm_merge_bvec); 10871da177e4SLinus Torvalds 108893d2341cSMatthew Dobson md->io_pool = mempool_create_slab_pool(MIN_IOS, _io_cache); 10891da177e4SLinus Torvalds if (!md->io_pool) 10906ed7ade8SMilan Broz goto bad_io_pool; 10911da177e4SLinus Torvalds 109293d2341cSMatthew Dobson md->tio_pool = mempool_create_slab_pool(MIN_IOS, _tio_cache); 10931da177e4SLinus Torvalds if (!md->tio_pool) 10946ed7ade8SMilan Broz goto bad_tio_pool; 10951da177e4SLinus Torvalds 10965972511bSJens Axboe md->bs = bioset_create(16, 16); 10979faf400fSStefan Bader if (!md->bs) 10989faf400fSStefan Bader goto bad_no_bioset; 10999faf400fSStefan Bader 11001da177e4SLinus Torvalds md->disk = alloc_disk(1); 11011da177e4SLinus Torvalds if (!md->disk) 11026ed7ade8SMilan Broz goto bad_disk; 11031da177e4SLinus Torvalds 1104f0b04115SJeff Mahoney atomic_set(&md->pending, 0); 1105f0b04115SJeff Mahoney init_waitqueue_head(&md->wait); 1106f0b04115SJeff Mahoney init_waitqueue_head(&md->eventq); 1107f0b04115SJeff Mahoney 11081da177e4SLinus Torvalds md->disk->major = _major; 11091da177e4SLinus Torvalds md->disk->first_minor = minor; 11101da177e4SLinus Torvalds md->disk->fops = &dm_blk_dops; 11111da177e4SLinus Torvalds md->disk->queue = md->queue; 11121da177e4SLinus Torvalds md->disk->private_data = md; 11131da177e4SLinus Torvalds sprintf(md->disk->disk_name, "dm-%d", minor); 11141da177e4SLinus Torvalds add_disk(md->disk); 11157e51f257SMike Anderson format_dev_t(md->name, MKDEV(_major, minor)); 11161da177e4SLinus Torvalds 1117304f3f6aSMilan Broz md->wq = create_singlethread_workqueue("kdmflush"); 1118304f3f6aSMilan Broz if (!md->wq) 1119304f3f6aSMilan Broz goto bad_thread; 1120304f3f6aSMilan Broz 1121ba61fdd1SJeff Mahoney /* Populate the mapping, nobody knows we exist yet */ 1122f32c10b0SJeff Mahoney spin_lock(&_minor_lock); 1123ba61fdd1SJeff Mahoney old_md = idr_replace(&_minor_idr, md, minor); 1124f32c10b0SJeff Mahoney spin_unlock(&_minor_lock); 1125ba61fdd1SJeff Mahoney 1126ba61fdd1SJeff Mahoney BUG_ON(old_md != MINOR_ALLOCED); 1127ba61fdd1SJeff Mahoney 11281da177e4SLinus Torvalds return md; 11291da177e4SLinus Torvalds 1130304f3f6aSMilan Broz bad_thread: 1131304f3f6aSMilan Broz put_disk(md->disk); 11326ed7ade8SMilan Broz bad_disk: 11339faf400fSStefan Bader bioset_free(md->bs); 11349faf400fSStefan Bader bad_no_bioset: 11351da177e4SLinus Torvalds mempool_destroy(md->tio_pool); 11366ed7ade8SMilan Broz bad_tio_pool: 11371da177e4SLinus Torvalds mempool_destroy(md->io_pool); 11386ed7ade8SMilan Broz bad_io_pool: 11391312f40eSAl Viro blk_cleanup_queue(md->queue); 11406ed7ade8SMilan Broz bad_queue: 11411da177e4SLinus Torvalds free_minor(minor); 11426ed7ade8SMilan Broz bad_minor: 114310da4f79SJeff Mahoney module_put(THIS_MODULE); 11446ed7ade8SMilan Broz bad_module_get: 11451da177e4SLinus Torvalds kfree(md); 11461da177e4SLinus Torvalds return NULL; 11471da177e4SLinus Torvalds } 11481da177e4SLinus Torvalds 1149ae9da83fSJun'ichi Nomura static void unlock_fs(struct mapped_device *md); 1150ae9da83fSJun'ichi Nomura 11511da177e4SLinus Torvalds static void free_dev(struct mapped_device *md) 11521da177e4SLinus Torvalds { 1153f331c029STejun Heo int minor = MINOR(disk_devt(md->disk)); 115463d94e48SJun'ichi Nomura 1155d9dde59bSJun'ichi Nomura if (md->suspended_bdev) { 1156ae9da83fSJun'ichi Nomura unlock_fs(md); 1157d9dde59bSJun'ichi Nomura bdput(md->suspended_bdev); 1158d9dde59bSJun'ichi Nomura } 1159304f3f6aSMilan Broz destroy_workqueue(md->wq); 11601da177e4SLinus Torvalds mempool_destroy(md->tio_pool); 11611da177e4SLinus Torvalds mempool_destroy(md->io_pool); 11629faf400fSStefan Bader bioset_free(md->bs); 11631da177e4SLinus Torvalds del_gendisk(md->disk); 116463d94e48SJun'ichi Nomura free_minor(minor); 1165fba9f90eSJeff Mahoney 1166fba9f90eSJeff Mahoney spin_lock(&_minor_lock); 1167fba9f90eSJeff Mahoney md->disk->private_data = NULL; 1168fba9f90eSJeff Mahoney spin_unlock(&_minor_lock); 1169fba9f90eSJeff Mahoney 11701da177e4SLinus Torvalds put_disk(md->disk); 11711312f40eSAl Viro blk_cleanup_queue(md->queue); 117210da4f79SJeff Mahoney module_put(THIS_MODULE); 11731da177e4SLinus Torvalds kfree(md); 11741da177e4SLinus Torvalds } 11751da177e4SLinus Torvalds 11761da177e4SLinus Torvalds /* 11771da177e4SLinus Torvalds * Bind a table to the device. 11781da177e4SLinus Torvalds */ 11791da177e4SLinus Torvalds static void event_callback(void *context) 11801da177e4SLinus Torvalds { 11817a8c3d3bSMike Anderson unsigned long flags; 11827a8c3d3bSMike Anderson LIST_HEAD(uevents); 11831da177e4SLinus Torvalds struct mapped_device *md = (struct mapped_device *) context; 11841da177e4SLinus Torvalds 11857a8c3d3bSMike Anderson spin_lock_irqsave(&md->uevent_lock, flags); 11867a8c3d3bSMike Anderson list_splice_init(&md->uevent_list, &uevents); 11877a8c3d3bSMike Anderson spin_unlock_irqrestore(&md->uevent_lock, flags); 11887a8c3d3bSMike Anderson 1189ed9e1982STejun Heo dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj); 11907a8c3d3bSMike Anderson 11911da177e4SLinus Torvalds atomic_inc(&md->event_nr); 11921da177e4SLinus Torvalds wake_up(&md->eventq); 11931da177e4SLinus Torvalds } 11941da177e4SLinus Torvalds 11954e90188bSAlasdair G Kergon static void __set_size(struct mapped_device *md, sector_t size) 11961da177e4SLinus Torvalds { 11974e90188bSAlasdair G Kergon set_capacity(md->disk, size); 11981da177e4SLinus Torvalds 11991b1dcc1bSJes Sorensen mutex_lock(&md->suspended_bdev->bd_inode->i_mutex); 1200e39e2e95SAlasdair G Kergon i_size_write(md->suspended_bdev->bd_inode, (loff_t)size << SECTOR_SHIFT); 12011b1dcc1bSJes Sorensen mutex_unlock(&md->suspended_bdev->bd_inode->i_mutex); 12021da177e4SLinus Torvalds } 12031da177e4SLinus Torvalds 12041da177e4SLinus Torvalds static int __bind(struct mapped_device *md, struct dm_table *t) 12051da177e4SLinus Torvalds { 1206165125e1SJens Axboe struct request_queue *q = md->queue; 12071da177e4SLinus Torvalds sector_t size; 12081da177e4SLinus Torvalds 12091da177e4SLinus Torvalds size = dm_table_get_size(t); 12103ac51e74SDarrick J. Wong 12113ac51e74SDarrick J. Wong /* 12123ac51e74SDarrick J. Wong * Wipe any geometry if the size of the table changed. 12133ac51e74SDarrick J. Wong */ 12143ac51e74SDarrick J. Wong if (size != get_capacity(md->disk)) 12153ac51e74SDarrick J. Wong memset(&md->geometry, 0, sizeof(md->geometry)); 12163ac51e74SDarrick J. Wong 1217bfa152faSJun'ichi Nomura if (md->suspended_bdev) 12184e90188bSAlasdair G Kergon __set_size(md, size); 12191da177e4SLinus Torvalds if (size == 0) 12201da177e4SLinus Torvalds return 0; 12211da177e4SLinus Torvalds 12221da177e4SLinus Torvalds dm_table_get(t); 1223cf222b37SAlasdair G Kergon dm_table_event_callback(t, event_callback, md); 12242ca3310eSAlasdair G Kergon 12252ca3310eSAlasdair G Kergon write_lock(&md->map_lock); 12262ca3310eSAlasdair G Kergon md->map = t; 12271da177e4SLinus Torvalds dm_table_set_restrictions(t, q); 12282ca3310eSAlasdair G Kergon write_unlock(&md->map_lock); 12292ca3310eSAlasdair G Kergon 12301da177e4SLinus Torvalds return 0; 12311da177e4SLinus Torvalds } 12321da177e4SLinus Torvalds 12331da177e4SLinus Torvalds static void __unbind(struct mapped_device *md) 12341da177e4SLinus Torvalds { 12351da177e4SLinus Torvalds struct dm_table *map = md->map; 12361da177e4SLinus Torvalds 12371da177e4SLinus Torvalds if (!map) 12381da177e4SLinus Torvalds return; 12391da177e4SLinus Torvalds 12401da177e4SLinus Torvalds dm_table_event_callback(map, NULL, NULL); 12411da177e4SLinus Torvalds write_lock(&md->map_lock); 12421da177e4SLinus Torvalds md->map = NULL; 12431da177e4SLinus Torvalds write_unlock(&md->map_lock); 12441da177e4SLinus Torvalds dm_table_put(map); 12451da177e4SLinus Torvalds } 12461da177e4SLinus Torvalds 12471da177e4SLinus Torvalds /* 12481da177e4SLinus Torvalds * Constructor for a new device. 12491da177e4SLinus Torvalds */ 12502b06cfffSAlasdair G Kergon int dm_create(int minor, struct mapped_device **result) 12511da177e4SLinus Torvalds { 12521da177e4SLinus Torvalds struct mapped_device *md; 12531da177e4SLinus Torvalds 12542b06cfffSAlasdair G Kergon md = alloc_dev(minor); 12551da177e4SLinus Torvalds if (!md) 12561da177e4SLinus Torvalds return -ENXIO; 12571da177e4SLinus Torvalds 12581da177e4SLinus Torvalds *result = md; 12591da177e4SLinus Torvalds return 0; 12601da177e4SLinus Torvalds } 12611da177e4SLinus Torvalds 1262637842cfSDavid Teigland static struct mapped_device *dm_find_md(dev_t dev) 12631da177e4SLinus Torvalds { 12641da177e4SLinus Torvalds struct mapped_device *md; 12651da177e4SLinus Torvalds unsigned minor = MINOR(dev); 12661da177e4SLinus Torvalds 12671da177e4SLinus Torvalds if (MAJOR(dev) != _major || minor >= (1 << MINORBITS)) 12681da177e4SLinus Torvalds return NULL; 12691da177e4SLinus Torvalds 1270f32c10b0SJeff Mahoney spin_lock(&_minor_lock); 12711da177e4SLinus Torvalds 12721da177e4SLinus Torvalds md = idr_find(&_minor_idr, minor); 1273fba9f90eSJeff Mahoney if (md && (md == MINOR_ALLOCED || 1274f331c029STejun Heo (MINOR(disk_devt(dm_disk(md))) != minor) || 1275fba9f90eSJeff Mahoney test_bit(DMF_FREEING, &md->flags))) { 1276637842cfSDavid Teigland md = NULL; 1277fba9f90eSJeff Mahoney goto out; 1278fba9f90eSJeff Mahoney } 12791da177e4SLinus Torvalds 1280fba9f90eSJeff Mahoney out: 1281f32c10b0SJeff Mahoney spin_unlock(&_minor_lock); 12821da177e4SLinus Torvalds 1283637842cfSDavid Teigland return md; 1284637842cfSDavid Teigland } 1285637842cfSDavid Teigland 1286d229a958SDavid Teigland struct mapped_device *dm_get_md(dev_t dev) 1287d229a958SDavid Teigland { 1288d229a958SDavid Teigland struct mapped_device *md = dm_find_md(dev); 1289d229a958SDavid Teigland 1290d229a958SDavid Teigland if (md) 1291d229a958SDavid Teigland dm_get(md); 1292d229a958SDavid Teigland 1293d229a958SDavid Teigland return md; 1294d229a958SDavid Teigland } 1295d229a958SDavid Teigland 12969ade92a9SAlasdair G Kergon void *dm_get_mdptr(struct mapped_device *md) 1297637842cfSDavid Teigland { 12989ade92a9SAlasdair G Kergon return md->interface_ptr; 12991da177e4SLinus Torvalds } 13001da177e4SLinus Torvalds 13011da177e4SLinus Torvalds void dm_set_mdptr(struct mapped_device *md, void *ptr) 13021da177e4SLinus Torvalds { 13031da177e4SLinus Torvalds md->interface_ptr = ptr; 13041da177e4SLinus Torvalds } 13051da177e4SLinus Torvalds 13061da177e4SLinus Torvalds void dm_get(struct mapped_device *md) 13071da177e4SLinus Torvalds { 13081da177e4SLinus Torvalds atomic_inc(&md->holders); 13091da177e4SLinus Torvalds } 13101da177e4SLinus Torvalds 131172d94861SAlasdair G Kergon const char *dm_device_name(struct mapped_device *md) 131272d94861SAlasdair G Kergon { 131372d94861SAlasdair G Kergon return md->name; 131472d94861SAlasdair G Kergon } 131572d94861SAlasdair G Kergon EXPORT_SYMBOL_GPL(dm_device_name); 131672d94861SAlasdair G Kergon 13171da177e4SLinus Torvalds void dm_put(struct mapped_device *md) 13181da177e4SLinus Torvalds { 13191134e5aeSMike Anderson struct dm_table *map; 13201da177e4SLinus Torvalds 1321fba9f90eSJeff Mahoney BUG_ON(test_bit(DMF_FREEING, &md->flags)); 1322fba9f90eSJeff Mahoney 1323f32c10b0SJeff Mahoney if (atomic_dec_and_lock(&md->holders, &_minor_lock)) { 13241134e5aeSMike Anderson map = dm_get_table(md); 1325f331c029STejun Heo idr_replace(&_minor_idr, MINOR_ALLOCED, 1326f331c029STejun Heo MINOR(disk_devt(dm_disk(md)))); 1327fba9f90eSJeff Mahoney set_bit(DMF_FREEING, &md->flags); 1328f32c10b0SJeff Mahoney spin_unlock(&_minor_lock); 1329cf222b37SAlasdair G Kergon if (!dm_suspended(md)) { 13301da177e4SLinus Torvalds dm_table_presuspend_targets(map); 13311da177e4SLinus Torvalds dm_table_postsuspend_targets(map); 13321da177e4SLinus Torvalds } 13331da177e4SLinus Torvalds __unbind(md); 13341134e5aeSMike Anderson dm_table_put(map); 13351da177e4SLinus Torvalds free_dev(md); 13361da177e4SLinus Torvalds } 13371da177e4SLinus Torvalds } 133879eb885cSEdward Goggin EXPORT_SYMBOL_GPL(dm_put); 13391da177e4SLinus Torvalds 134046125c1cSMilan Broz static int dm_wait_for_completion(struct mapped_device *md) 134146125c1cSMilan Broz { 134246125c1cSMilan Broz int r = 0; 134346125c1cSMilan Broz 134446125c1cSMilan Broz while (1) { 134546125c1cSMilan Broz set_current_state(TASK_INTERRUPTIBLE); 134646125c1cSMilan Broz 134746125c1cSMilan Broz smp_mb(); 134846125c1cSMilan Broz if (!atomic_read(&md->pending)) 134946125c1cSMilan Broz break; 135046125c1cSMilan Broz 135146125c1cSMilan Broz if (signal_pending(current)) { 135246125c1cSMilan Broz r = -EINTR; 135346125c1cSMilan Broz break; 135446125c1cSMilan Broz } 135546125c1cSMilan Broz 135646125c1cSMilan Broz io_schedule(); 135746125c1cSMilan Broz } 135846125c1cSMilan Broz set_current_state(TASK_RUNNING); 135946125c1cSMilan Broz 136046125c1cSMilan Broz return r; 136146125c1cSMilan Broz } 136246125c1cSMilan Broz 13631da177e4SLinus Torvalds /* 13641da177e4SLinus Torvalds * Process the deferred bios 13651da177e4SLinus Torvalds */ 13666d6f10dfSMilan Broz static void __flush_deferred_io(struct mapped_device *md) 13671da177e4SLinus Torvalds { 13686d6f10dfSMilan Broz struct bio *c; 13691da177e4SLinus Torvalds 13706d6f10dfSMilan Broz while ((c = bio_list_pop(&md->deferred))) { 13719e4e5f87SMilan Broz if (__split_bio(md, c)) 13729e4e5f87SMilan Broz bio_io_error(c); 13731da177e4SLinus Torvalds } 137473d410c0SMilan Broz 137573d410c0SMilan Broz clear_bit(DMF_BLOCK_IO, &md->flags); 13761da177e4SLinus Torvalds } 13771da177e4SLinus Torvalds 13786d6f10dfSMilan Broz static void __merge_pushback_list(struct mapped_device *md) 13796d6f10dfSMilan Broz { 13806d6f10dfSMilan Broz unsigned long flags; 13816d6f10dfSMilan Broz 13826d6f10dfSMilan Broz spin_lock_irqsave(&md->pushback_lock, flags); 13836d6f10dfSMilan Broz clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 13846d6f10dfSMilan Broz bio_list_merge_head(&md->deferred, &md->pushback); 13856d6f10dfSMilan Broz bio_list_init(&md->pushback); 13866d6f10dfSMilan Broz spin_unlock_irqrestore(&md->pushback_lock, flags); 13876d6f10dfSMilan Broz } 13886d6f10dfSMilan Broz 1389304f3f6aSMilan Broz static void dm_wq_work(struct work_struct *work) 1390304f3f6aSMilan Broz { 1391304f3f6aSMilan Broz struct dm_wq_req *req = container_of(work, struct dm_wq_req, work); 1392304f3f6aSMilan Broz struct mapped_device *md = req->md; 1393304f3f6aSMilan Broz 1394304f3f6aSMilan Broz down_write(&md->io_lock); 1395304f3f6aSMilan Broz switch (req->type) { 1396304f3f6aSMilan Broz case DM_WQ_FLUSH_DEFERRED: 1397304f3f6aSMilan Broz __flush_deferred_io(md); 1398304f3f6aSMilan Broz break; 1399304f3f6aSMilan Broz default: 1400304f3f6aSMilan Broz DMERR("dm_wq_work: unrecognised work type %d", req->type); 1401304f3f6aSMilan Broz BUG(); 1402304f3f6aSMilan Broz } 1403304f3f6aSMilan Broz up_write(&md->io_lock); 1404304f3f6aSMilan Broz } 1405304f3f6aSMilan Broz 1406304f3f6aSMilan Broz static void dm_wq_queue(struct mapped_device *md, int type, void *context, 1407304f3f6aSMilan Broz struct dm_wq_req *req) 1408304f3f6aSMilan Broz { 1409304f3f6aSMilan Broz req->type = type; 1410304f3f6aSMilan Broz req->md = md; 1411304f3f6aSMilan Broz req->context = context; 1412304f3f6aSMilan Broz INIT_WORK(&req->work, dm_wq_work); 1413304f3f6aSMilan Broz queue_work(md->wq, &req->work); 1414304f3f6aSMilan Broz } 1415304f3f6aSMilan Broz 1416304f3f6aSMilan Broz static void dm_queue_flush(struct mapped_device *md, int type, void *context) 1417304f3f6aSMilan Broz { 1418304f3f6aSMilan Broz struct dm_wq_req req; 1419304f3f6aSMilan Broz 1420304f3f6aSMilan Broz dm_wq_queue(md, type, context, &req); 1421304f3f6aSMilan Broz flush_workqueue(md->wq); 1422304f3f6aSMilan Broz } 1423304f3f6aSMilan Broz 14241da177e4SLinus Torvalds /* 14251da177e4SLinus Torvalds * Swap in a new table (destroying old one). 14261da177e4SLinus Torvalds */ 14271da177e4SLinus Torvalds int dm_swap_table(struct mapped_device *md, struct dm_table *table) 14281da177e4SLinus Torvalds { 142993c534aeSAlasdair G Kergon int r = -EINVAL; 14301da177e4SLinus Torvalds 1431e61290a4SDaniel Walker mutex_lock(&md->suspend_lock); 14321da177e4SLinus Torvalds 14331da177e4SLinus Torvalds /* device must be suspended */ 1434cf222b37SAlasdair G Kergon if (!dm_suspended(md)) 143593c534aeSAlasdair G Kergon goto out; 14361da177e4SLinus Torvalds 1437bfa152faSJun'ichi Nomura /* without bdev, the device size cannot be changed */ 1438bfa152faSJun'ichi Nomura if (!md->suspended_bdev) 1439bfa152faSJun'ichi Nomura if (get_capacity(md->disk) != dm_table_get_size(table)) 1440bfa152faSJun'ichi Nomura goto out; 1441bfa152faSJun'ichi Nomura 14421da177e4SLinus Torvalds __unbind(md); 14431da177e4SLinus Torvalds r = __bind(md, table); 14441da177e4SLinus Torvalds 144593c534aeSAlasdair G Kergon out: 1446e61290a4SDaniel Walker mutex_unlock(&md->suspend_lock); 144793c534aeSAlasdair G Kergon return r; 14481da177e4SLinus Torvalds } 14491da177e4SLinus Torvalds 14501da177e4SLinus Torvalds /* 14511da177e4SLinus Torvalds * Functions to lock and unlock any filesystem running on the 14521da177e4SLinus Torvalds * device. 14531da177e4SLinus Torvalds */ 14542ca3310eSAlasdair G Kergon static int lock_fs(struct mapped_device *md) 14551da177e4SLinus Torvalds { 1456e39e2e95SAlasdair G Kergon int r; 14571da177e4SLinus Torvalds 14581da177e4SLinus Torvalds WARN_ON(md->frozen_sb); 1459dfbe03f6SAlasdair G Kergon 1460e39e2e95SAlasdair G Kergon md->frozen_sb = freeze_bdev(md->suspended_bdev); 1461dfbe03f6SAlasdair G Kergon if (IS_ERR(md->frozen_sb)) { 1462cf222b37SAlasdair G Kergon r = PTR_ERR(md->frozen_sb); 1463e39e2e95SAlasdair G Kergon md->frozen_sb = NULL; 1464e39e2e95SAlasdair G Kergon return r; 1465dfbe03f6SAlasdair G Kergon } 1466dfbe03f6SAlasdair G Kergon 1467aa8d7c2fSAlasdair G Kergon set_bit(DMF_FROZEN, &md->flags); 1468aa8d7c2fSAlasdair G Kergon 14691da177e4SLinus Torvalds /* don't bdput right now, we don't want the bdev 1470e39e2e95SAlasdair G Kergon * to go away while it is locked. 14711da177e4SLinus Torvalds */ 14721da177e4SLinus Torvalds return 0; 14731da177e4SLinus Torvalds } 14741da177e4SLinus Torvalds 14752ca3310eSAlasdair G Kergon static void unlock_fs(struct mapped_device *md) 14761da177e4SLinus Torvalds { 1477aa8d7c2fSAlasdair G Kergon if (!test_bit(DMF_FROZEN, &md->flags)) 1478aa8d7c2fSAlasdair G Kergon return; 1479aa8d7c2fSAlasdair G Kergon 1480e39e2e95SAlasdair G Kergon thaw_bdev(md->suspended_bdev, md->frozen_sb); 14811da177e4SLinus Torvalds md->frozen_sb = NULL; 1482aa8d7c2fSAlasdair G Kergon clear_bit(DMF_FROZEN, &md->flags); 14831da177e4SLinus Torvalds } 14841da177e4SLinus Torvalds 14851da177e4SLinus Torvalds /* 14861da177e4SLinus Torvalds * We need to be able to change a mapping table under a mounted 14871da177e4SLinus Torvalds * filesystem. For example we might want to move some data in 14881da177e4SLinus Torvalds * the background. Before the table can be swapped with 14891da177e4SLinus Torvalds * dm_bind_table, dm_suspend must be called to flush any in 14901da177e4SLinus Torvalds * flight bios and ensure that any further io gets deferred. 14911da177e4SLinus Torvalds */ 1492a3d77d35SKiyoshi Ueda int dm_suspend(struct mapped_device *md, unsigned suspend_flags) 14931da177e4SLinus Torvalds { 14942ca3310eSAlasdair G Kergon struct dm_table *map = NULL; 14951da177e4SLinus Torvalds DECLARE_WAITQUEUE(wait, current); 149646125c1cSMilan Broz int r = 0; 1497a3d77d35SKiyoshi Ueda int do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG ? 1 : 0; 14982e93ccc1SKiyoshi Ueda int noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG ? 1 : 0; 14991da177e4SLinus Torvalds 1500e61290a4SDaniel Walker mutex_lock(&md->suspend_lock); 15012ca3310eSAlasdair G Kergon 150273d410c0SMilan Broz if (dm_suspended(md)) { 150373d410c0SMilan Broz r = -EINVAL; 1504d287483dSAlasdair G Kergon goto out_unlock; 150573d410c0SMilan Broz } 15061da177e4SLinus Torvalds 15071da177e4SLinus Torvalds map = dm_get_table(md); 1508cf222b37SAlasdair G Kergon 15092e93ccc1SKiyoshi Ueda /* 15102e93ccc1SKiyoshi Ueda * DMF_NOFLUSH_SUSPENDING must be set before presuspend. 15112e93ccc1SKiyoshi Ueda * This flag is cleared before dm_suspend returns. 15122e93ccc1SKiyoshi Ueda */ 15132e93ccc1SKiyoshi Ueda if (noflush) 15142e93ccc1SKiyoshi Ueda set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 15152e93ccc1SKiyoshi Ueda 1516436d4108SAlasdair G Kergon /* This does not get reverted if there's an error later. */ 15171da177e4SLinus Torvalds dm_table_presuspend_targets(map); 15181da177e4SLinus Torvalds 1519bfa152faSJun'ichi Nomura /* bdget() can stall if the pending I/Os are not flushed */ 1520bfa152faSJun'ichi Nomura if (!noflush) { 1521e39e2e95SAlasdair G Kergon md->suspended_bdev = bdget_disk(md->disk, 0); 1522e39e2e95SAlasdair G Kergon if (!md->suspended_bdev) { 1523e39e2e95SAlasdair G Kergon DMWARN("bdget failed in dm_suspend"); 1524e39e2e95SAlasdair G Kergon r = -ENOMEM; 1525f431d966SKiyoshi Ueda goto out; 1526e39e2e95SAlasdair G Kergon } 1527e39e2e95SAlasdair G Kergon 15282e93ccc1SKiyoshi Ueda /* 15296d6f10dfSMilan Broz * Flush I/O to the device. noflush supersedes do_lockfs, 15306d6f10dfSMilan Broz * because lock_fs() needs to flush I/Os. 15312e93ccc1SKiyoshi Ueda */ 15326d6f10dfSMilan Broz if (do_lockfs) { 15332ca3310eSAlasdair G Kergon r = lock_fs(md); 15342ca3310eSAlasdair G Kergon if (r) 15352ca3310eSAlasdair G Kergon goto out; 1536aa8d7c2fSAlasdair G Kergon } 15376d6f10dfSMilan Broz } 15381da177e4SLinus Torvalds 15391da177e4SLinus Torvalds /* 1540354e0071SAlasdair G Kergon * First we set the BLOCK_IO flag so no more ios will be mapped. 15411da177e4SLinus Torvalds */ 15422ca3310eSAlasdair G Kergon down_write(&md->io_lock); 15432ca3310eSAlasdair G Kergon set_bit(DMF_BLOCK_IO, &md->flags); 15441da177e4SLinus Torvalds 15451da177e4SLinus Torvalds add_wait_queue(&md->wait, &wait); 15462ca3310eSAlasdair G Kergon up_write(&md->io_lock); 15471da177e4SLinus Torvalds 15481da177e4SLinus Torvalds /* unplug */ 15492ca3310eSAlasdair G Kergon if (map) 15501da177e4SLinus Torvalds dm_table_unplug_all(map); 15511da177e4SLinus Torvalds 15521da177e4SLinus Torvalds /* 155346125c1cSMilan Broz * Wait for the already-mapped ios to complete. 15541da177e4SLinus Torvalds */ 155546125c1cSMilan Broz r = dm_wait_for_completion(md); 15561da177e4SLinus Torvalds 15572ca3310eSAlasdair G Kergon down_write(&md->io_lock); 15581da177e4SLinus Torvalds remove_wait_queue(&md->wait, &wait); 15591da177e4SLinus Torvalds 15606d6f10dfSMilan Broz if (noflush) 15616d6f10dfSMilan Broz __merge_pushback_list(md); 156294d6351eSMilan Broz up_write(&md->io_lock); 15632e93ccc1SKiyoshi Ueda 15641da177e4SLinus Torvalds /* were we interrupted ? */ 156546125c1cSMilan Broz if (r < 0) { 1566304f3f6aSMilan Broz dm_queue_flush(md, DM_WQ_FLUSH_DEFERRED, NULL); 156773d410c0SMilan Broz 15682ca3310eSAlasdair G Kergon unlock_fs(md); 15692e93ccc1SKiyoshi Ueda goto out; /* pushback list is already flushed, so skip flush */ 15702ca3310eSAlasdair G Kergon } 15712ca3310eSAlasdair G Kergon 15722ca3310eSAlasdair G Kergon dm_table_postsuspend_targets(map); 15731da177e4SLinus Torvalds 15741da177e4SLinus Torvalds set_bit(DMF_SUSPENDED, &md->flags); 15751da177e4SLinus Torvalds 15762ca3310eSAlasdair G Kergon out: 1577e39e2e95SAlasdair G Kergon if (r && md->suspended_bdev) { 1578e39e2e95SAlasdair G Kergon bdput(md->suspended_bdev); 1579e39e2e95SAlasdair G Kergon md->suspended_bdev = NULL; 1580e39e2e95SAlasdair G Kergon } 1581e39e2e95SAlasdair G Kergon 15821da177e4SLinus Torvalds dm_table_put(map); 1583d287483dSAlasdair G Kergon 1584d287483dSAlasdair G Kergon out_unlock: 1585e61290a4SDaniel Walker mutex_unlock(&md->suspend_lock); 1586cf222b37SAlasdair G Kergon return r; 15871da177e4SLinus Torvalds } 15881da177e4SLinus Torvalds 15891da177e4SLinus Torvalds int dm_resume(struct mapped_device *md) 15901da177e4SLinus Torvalds { 1591cf222b37SAlasdair G Kergon int r = -EINVAL; 1592cf222b37SAlasdair G Kergon struct dm_table *map = NULL; 15931da177e4SLinus Torvalds 1594e61290a4SDaniel Walker mutex_lock(&md->suspend_lock); 15952ca3310eSAlasdair G Kergon if (!dm_suspended(md)) 1596cf222b37SAlasdair G Kergon goto out; 1597cf222b37SAlasdair G Kergon 1598cf222b37SAlasdair G Kergon map = dm_get_table(md); 15992ca3310eSAlasdair G Kergon if (!map || !dm_table_get_size(map)) 1600cf222b37SAlasdair G Kergon goto out; 16011da177e4SLinus Torvalds 16028757b776SMilan Broz r = dm_table_resume_targets(map); 16038757b776SMilan Broz if (r) 16048757b776SMilan Broz goto out; 16052ca3310eSAlasdair G Kergon 1606304f3f6aSMilan Broz dm_queue_flush(md, DM_WQ_FLUSH_DEFERRED, NULL); 16072ca3310eSAlasdair G Kergon 16082ca3310eSAlasdair G Kergon unlock_fs(md); 16092ca3310eSAlasdair G Kergon 1610bfa152faSJun'ichi Nomura if (md->suspended_bdev) { 1611e39e2e95SAlasdair G Kergon bdput(md->suspended_bdev); 1612e39e2e95SAlasdair G Kergon md->suspended_bdev = NULL; 1613bfa152faSJun'ichi Nomura } 1614e39e2e95SAlasdair G Kergon 16152ca3310eSAlasdair G Kergon clear_bit(DMF_SUSPENDED, &md->flags); 16162ca3310eSAlasdair G Kergon 16171da177e4SLinus Torvalds dm_table_unplug_all(map); 16181da177e4SLinus Torvalds 161969267a30SAlasdair G Kergon dm_kobject_uevent(md); 16208560ed6fSHannes Reinecke 1621cf222b37SAlasdair G Kergon r = 0; 16222ca3310eSAlasdair G Kergon 1623cf222b37SAlasdair G Kergon out: 1624cf222b37SAlasdair G Kergon dm_table_put(map); 1625e61290a4SDaniel Walker mutex_unlock(&md->suspend_lock); 16262ca3310eSAlasdair G Kergon 1627cf222b37SAlasdair G Kergon return r; 16281da177e4SLinus Torvalds } 16291da177e4SLinus Torvalds 16301da177e4SLinus Torvalds /*----------------------------------------------------------------- 16311da177e4SLinus Torvalds * Event notification. 16321da177e4SLinus Torvalds *---------------------------------------------------------------*/ 163369267a30SAlasdair G Kergon void dm_kobject_uevent(struct mapped_device *md) 163469267a30SAlasdair G Kergon { 1635ed9e1982STejun Heo kobject_uevent(&disk_to_dev(md->disk)->kobj, KOBJ_CHANGE); 163669267a30SAlasdair G Kergon } 163769267a30SAlasdair G Kergon 16387a8c3d3bSMike Anderson uint32_t dm_next_uevent_seq(struct mapped_device *md) 16397a8c3d3bSMike Anderson { 16407a8c3d3bSMike Anderson return atomic_add_return(1, &md->uevent_seq); 16417a8c3d3bSMike Anderson } 16427a8c3d3bSMike Anderson 16431da177e4SLinus Torvalds uint32_t dm_get_event_nr(struct mapped_device *md) 16441da177e4SLinus Torvalds { 16451da177e4SLinus Torvalds return atomic_read(&md->event_nr); 16461da177e4SLinus Torvalds } 16471da177e4SLinus Torvalds 16481da177e4SLinus Torvalds int dm_wait_event(struct mapped_device *md, int event_nr) 16491da177e4SLinus Torvalds { 16501da177e4SLinus Torvalds return wait_event_interruptible(md->eventq, 16511da177e4SLinus Torvalds (event_nr != atomic_read(&md->event_nr))); 16521da177e4SLinus Torvalds } 16531da177e4SLinus Torvalds 16547a8c3d3bSMike Anderson void dm_uevent_add(struct mapped_device *md, struct list_head *elist) 16557a8c3d3bSMike Anderson { 16567a8c3d3bSMike Anderson unsigned long flags; 16577a8c3d3bSMike Anderson 16587a8c3d3bSMike Anderson spin_lock_irqsave(&md->uevent_lock, flags); 16597a8c3d3bSMike Anderson list_add(elist, &md->uevent_list); 16607a8c3d3bSMike Anderson spin_unlock_irqrestore(&md->uevent_lock, flags); 16617a8c3d3bSMike Anderson } 16627a8c3d3bSMike Anderson 16631da177e4SLinus Torvalds /* 16641da177e4SLinus Torvalds * The gendisk is only valid as long as you have a reference 16651da177e4SLinus Torvalds * count on 'md'. 16661da177e4SLinus Torvalds */ 16671da177e4SLinus Torvalds struct gendisk *dm_disk(struct mapped_device *md) 16681da177e4SLinus Torvalds { 16691da177e4SLinus Torvalds return md->disk; 16701da177e4SLinus Torvalds } 16711da177e4SLinus Torvalds 16721da177e4SLinus Torvalds int dm_suspended(struct mapped_device *md) 16731da177e4SLinus Torvalds { 16741da177e4SLinus Torvalds return test_bit(DMF_SUSPENDED, &md->flags); 16751da177e4SLinus Torvalds } 16761da177e4SLinus Torvalds 16772e93ccc1SKiyoshi Ueda int dm_noflush_suspending(struct dm_target *ti) 16782e93ccc1SKiyoshi Ueda { 16792e93ccc1SKiyoshi Ueda struct mapped_device *md = dm_table_get_md(ti->table); 16802e93ccc1SKiyoshi Ueda int r = __noflush_suspending(md); 16812e93ccc1SKiyoshi Ueda 16822e93ccc1SKiyoshi Ueda dm_put(md); 16832e93ccc1SKiyoshi Ueda 16842e93ccc1SKiyoshi Ueda return r; 16852e93ccc1SKiyoshi Ueda } 16862e93ccc1SKiyoshi Ueda EXPORT_SYMBOL_GPL(dm_noflush_suspending); 16872e93ccc1SKiyoshi Ueda 16881da177e4SLinus Torvalds static struct block_device_operations dm_blk_dops = { 16891da177e4SLinus Torvalds .open = dm_blk_open, 16901da177e4SLinus Torvalds .release = dm_blk_close, 1691aa129a22SMilan Broz .ioctl = dm_blk_ioctl, 16923ac51e74SDarrick J. Wong .getgeo = dm_blk_getgeo, 16931da177e4SLinus Torvalds .owner = THIS_MODULE 16941da177e4SLinus Torvalds }; 16951da177e4SLinus Torvalds 16961da177e4SLinus Torvalds EXPORT_SYMBOL(dm_get_mapinfo); 16971da177e4SLinus Torvalds 16981da177e4SLinus Torvalds /* 16991da177e4SLinus Torvalds * module hooks 17001da177e4SLinus Torvalds */ 17011da177e4SLinus Torvalds module_init(dm_init); 17021da177e4SLinus Torvalds module_exit(dm_exit); 17031da177e4SLinus Torvalds 17041da177e4SLinus Torvalds module_param(major, uint, 0); 17051da177e4SLinus Torvalds MODULE_PARM_DESC(major, "The major number of the device mapper"); 17061da177e4SLinus Torvalds MODULE_DESCRIPTION(DM_NAME " driver"); 17071da177e4SLinus Torvalds MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>"); 17081da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 1709