11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * dm-snapshot.c 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Copyright (C) 2001-2002 Sistina Software (UK) Limited. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * This file is released under the GPL. 71da177e4SLinus Torvalds */ 81da177e4SLinus Torvalds 91da177e4SLinus Torvalds #include <linux/blkdev.h> 101da177e4SLinus Torvalds #include <linux/device-mapper.h> 1190fa1527SMikulas Patocka #include <linux/delay.h> 121da177e4SLinus Torvalds #include <linux/fs.h> 131da177e4SLinus Torvalds #include <linux/init.h> 141da177e4SLinus Torvalds #include <linux/kdev_t.h> 151da177e4SLinus Torvalds #include <linux/list.h> 161da177e4SLinus Torvalds #include <linux/mempool.h> 171da177e4SLinus Torvalds #include <linux/module.h> 181da177e4SLinus Torvalds #include <linux/slab.h> 191da177e4SLinus Torvalds #include <linux/vmalloc.h> 206f3c3f0aSvignesh babu #include <linux/log2.h> 21a765e20eSAlasdair G Kergon #include <linux/dm-kcopyd.h> 221da177e4SLinus Torvalds 23b735fedeSMikulas Patocka #include "dm.h" 24b735fedeSMikulas Patocka 25aea53d92SJonathan Brassow #include "dm-exception-store.h" 261da177e4SLinus Torvalds 2772d94861SAlasdair G Kergon #define DM_MSG_PREFIX "snapshots" 2872d94861SAlasdair G Kergon 29d698aa45SMikulas Patocka static const char dm_snapshot_merge_target_name[] = "snapshot-merge"; 30d698aa45SMikulas Patocka 31d698aa45SMikulas Patocka #define dm_target_is_snapshot_merge(ti) \ 32d698aa45SMikulas Patocka ((ti)->type->name == dm_snapshot_merge_target_name) 33d698aa45SMikulas Patocka 341da177e4SLinus Torvalds /* 35cd45daffSMikulas Patocka * The size of the mempool used to track chunks in use. 36cd45daffSMikulas Patocka */ 37cd45daffSMikulas Patocka #define MIN_IOS 256 38cd45daffSMikulas Patocka 39ccc45ea8SJonathan Brassow #define DM_TRACKED_CHUNK_HASH_SIZE 16 40ccc45ea8SJonathan Brassow #define DM_TRACKED_CHUNK_HASH(x) ((unsigned long)(x) & \ 41ccc45ea8SJonathan Brassow (DM_TRACKED_CHUNK_HASH_SIZE - 1)) 42ccc45ea8SJonathan Brassow 43191437a5SJon Brassow struct dm_exception_table { 44ccc45ea8SJonathan Brassow uint32_t hash_mask; 45ccc45ea8SJonathan Brassow unsigned hash_shift; 46ccc45ea8SJonathan Brassow struct list_head *table; 47ccc45ea8SJonathan Brassow }; 48ccc45ea8SJonathan Brassow 49ccc45ea8SJonathan Brassow struct dm_snapshot { 50ccc45ea8SJonathan Brassow struct rw_semaphore lock; 51ccc45ea8SJonathan Brassow 52ccc45ea8SJonathan Brassow struct dm_dev *origin; 53fc56f6fbSMike Snitzer struct dm_dev *cow; 54fc56f6fbSMike Snitzer 55fc56f6fbSMike Snitzer struct dm_target *ti; 56ccc45ea8SJonathan Brassow 57ccc45ea8SJonathan Brassow /* List of snapshots per Origin */ 58ccc45ea8SJonathan Brassow struct list_head list; 59ccc45ea8SJonathan Brassow 60d8ddb1cfSMike Snitzer /* 61d8ddb1cfSMike Snitzer * You can't use a snapshot if this is 0 (e.g. if full). 62d8ddb1cfSMike Snitzer * A snapshot-merge target never clears this. 63d8ddb1cfSMike Snitzer */ 64ccc45ea8SJonathan Brassow int valid; 65ccc45ea8SJonathan Brassow 66ccc45ea8SJonathan Brassow /* Origin writes don't trigger exceptions until this is set */ 67ccc45ea8SJonathan Brassow int active; 68ccc45ea8SJonathan Brassow 69ccc45ea8SJonathan Brassow atomic_t pending_exceptions_count; 70ccc45ea8SJonathan Brassow 71230c83afSMikulas Patocka /* Protected by "lock" */ 72230c83afSMikulas Patocka sector_t exception_start_sequence; 73230c83afSMikulas Patocka 74230c83afSMikulas Patocka /* Protected by kcopyd single-threaded callback */ 75230c83afSMikulas Patocka sector_t exception_complete_sequence; 76230c83afSMikulas Patocka 77230c83afSMikulas Patocka /* 78230c83afSMikulas Patocka * A list of pending exceptions that completed out of order. 79230c83afSMikulas Patocka * Protected by kcopyd single-threaded callback. 80230c83afSMikulas Patocka */ 81230c83afSMikulas Patocka struct list_head out_of_order_list; 82230c83afSMikulas Patocka 83924e600dSMike Snitzer mempool_t *pending_pool; 84924e600dSMike Snitzer 85191437a5SJon Brassow struct dm_exception_table pending; 86191437a5SJon Brassow struct dm_exception_table complete; 87ccc45ea8SJonathan Brassow 88ccc45ea8SJonathan Brassow /* 89ccc45ea8SJonathan Brassow * pe_lock protects all pending_exception operations and access 90ccc45ea8SJonathan Brassow * as well as the snapshot_bios list. 91ccc45ea8SJonathan Brassow */ 92ccc45ea8SJonathan Brassow spinlock_t pe_lock; 93ccc45ea8SJonathan Brassow 94924e600dSMike Snitzer /* Chunks with outstanding reads */ 95924e600dSMike Snitzer spinlock_t tracked_chunk_lock; 96924e600dSMike Snitzer struct hlist_head tracked_chunk_hash[DM_TRACKED_CHUNK_HASH_SIZE]; 97924e600dSMike Snitzer 98ccc45ea8SJonathan Brassow /* The on disk metadata handler */ 99ccc45ea8SJonathan Brassow struct dm_exception_store *store; 100ccc45ea8SJonathan Brassow 101ccc45ea8SJonathan Brassow struct dm_kcopyd_client *kcopyd_client; 102ccc45ea8SJonathan Brassow 103924e600dSMike Snitzer /* Wait for events based on state_bits */ 104924e600dSMike Snitzer unsigned long state_bits; 105924e600dSMike Snitzer 106924e600dSMike Snitzer /* Range of chunks currently being merged. */ 107924e600dSMike Snitzer chunk_t first_merging_chunk; 108924e600dSMike Snitzer int num_merging_chunks; 1091e03f97eSMikulas Patocka 110d8ddb1cfSMike Snitzer /* 111d8ddb1cfSMike Snitzer * The merge operation failed if this flag is set. 112d8ddb1cfSMike Snitzer * Failure modes are handled as follows: 113d8ddb1cfSMike Snitzer * - I/O error reading the header 114d8ddb1cfSMike Snitzer * => don't load the target; abort. 115d8ddb1cfSMike Snitzer * - Header does not have "valid" flag set 116d8ddb1cfSMike Snitzer * => use the origin; forget about the snapshot. 117d8ddb1cfSMike Snitzer * - I/O error when reading exceptions 118d8ddb1cfSMike Snitzer * => don't load the target; abort. 119d8ddb1cfSMike Snitzer * (We can't use the intermediate origin state.) 120d8ddb1cfSMike Snitzer * - I/O error while merging 121d8ddb1cfSMike Snitzer * => stop merging; set merge_failed; process I/O normally. 122d8ddb1cfSMike Snitzer */ 123d8ddb1cfSMike Snitzer int merge_failed; 124d8ddb1cfSMike Snitzer 1259fe86254SMikulas Patocka /* 1269fe86254SMikulas Patocka * Incoming bios that overlap with chunks being merged must wait 1279fe86254SMikulas Patocka * for them to be committed. 1289fe86254SMikulas Patocka */ 1299fe86254SMikulas Patocka struct bio_list bios_queued_during_merge; 130ccc45ea8SJonathan Brassow }; 131ccc45ea8SJonathan Brassow 1321e03f97eSMikulas Patocka /* 1331e03f97eSMikulas Patocka * state_bits: 1341e03f97eSMikulas Patocka * RUNNING_MERGE - Merge operation is in progress. 1351e03f97eSMikulas Patocka * SHUTDOWN_MERGE - Set to signal that merge needs to be stopped; 1361e03f97eSMikulas Patocka * cleared afterwards. 1371e03f97eSMikulas Patocka */ 1381e03f97eSMikulas Patocka #define RUNNING_MERGE 0 1391e03f97eSMikulas Patocka #define SHUTDOWN_MERGE 1 1401e03f97eSMikulas Patocka 141df5d2e90SMikulas Patocka DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(snapshot_copy_throttle, 142df5d2e90SMikulas Patocka "A percentage of time allocated for copy on write"); 143df5d2e90SMikulas Patocka 144c2411045SMikulas Patocka struct dm_dev *dm_snap_origin(struct dm_snapshot *s) 145c2411045SMikulas Patocka { 146c2411045SMikulas Patocka return s->origin; 147c2411045SMikulas Patocka } 148c2411045SMikulas Patocka EXPORT_SYMBOL(dm_snap_origin); 149c2411045SMikulas Patocka 150fc56f6fbSMike Snitzer struct dm_dev *dm_snap_cow(struct dm_snapshot *s) 151fc56f6fbSMike Snitzer { 152fc56f6fbSMike Snitzer return s->cow; 153fc56f6fbSMike Snitzer } 154fc56f6fbSMike Snitzer EXPORT_SYMBOL(dm_snap_cow); 155fc56f6fbSMike Snitzer 156ccc45ea8SJonathan Brassow static sector_t chunk_to_sector(struct dm_exception_store *store, 157ccc45ea8SJonathan Brassow chunk_t chunk) 158ccc45ea8SJonathan Brassow { 159ccc45ea8SJonathan Brassow return chunk << store->chunk_shift; 160ccc45ea8SJonathan Brassow } 161ccc45ea8SJonathan Brassow 162ccc45ea8SJonathan Brassow static int bdev_equal(struct block_device *lhs, struct block_device *rhs) 163ccc45ea8SJonathan Brassow { 164ccc45ea8SJonathan Brassow /* 165ccc45ea8SJonathan Brassow * There is only ever one instance of a particular block 166ccc45ea8SJonathan Brassow * device so we can compare pointers safely. 167ccc45ea8SJonathan Brassow */ 168ccc45ea8SJonathan Brassow return lhs == rhs; 169ccc45ea8SJonathan Brassow } 170ccc45ea8SJonathan Brassow 171028867acSAlasdair G Kergon struct dm_snap_pending_exception { 1721d4989c8SJon Brassow struct dm_exception e; 1731da177e4SLinus Torvalds 1741da177e4SLinus Torvalds /* 1751da177e4SLinus Torvalds * Origin buffers waiting for this to complete are held 1761da177e4SLinus Torvalds * in a bio list 1771da177e4SLinus Torvalds */ 1781da177e4SLinus Torvalds struct bio_list origin_bios; 1791da177e4SLinus Torvalds struct bio_list snapshot_bios; 1801da177e4SLinus Torvalds 1811da177e4SLinus Torvalds /* Pointer back to snapshot context */ 1821da177e4SLinus Torvalds struct dm_snapshot *snap; 1831da177e4SLinus Torvalds 1841da177e4SLinus Torvalds /* 1851da177e4SLinus Torvalds * 1 indicates the exception has already been sent to 1861da177e4SLinus Torvalds * kcopyd. 1871da177e4SLinus Torvalds */ 1881da177e4SLinus Torvalds int started; 189a6e50b40SMikulas Patocka 190230c83afSMikulas Patocka /* There was copying error. */ 191230c83afSMikulas Patocka int copy_error; 192230c83afSMikulas Patocka 193230c83afSMikulas Patocka /* A sequence number, it is used for in-order completion. */ 194230c83afSMikulas Patocka sector_t exception_sequence; 195230c83afSMikulas Patocka 196230c83afSMikulas Patocka struct list_head out_of_order_entry; 197230c83afSMikulas Patocka 198a6e50b40SMikulas Patocka /* 199a6e50b40SMikulas Patocka * For writing a complete chunk, bypassing the copy. 200a6e50b40SMikulas Patocka */ 201a6e50b40SMikulas Patocka struct bio *full_bio; 202a6e50b40SMikulas Patocka bio_end_io_t *full_bio_end_io; 203a6e50b40SMikulas Patocka void *full_bio_private; 2041da177e4SLinus Torvalds }; 2051da177e4SLinus Torvalds 2061da177e4SLinus Torvalds /* 2071da177e4SLinus Torvalds * Hash table mapping origin volumes to lists of snapshots and 2081da177e4SLinus Torvalds * a lock to protect it 2091da177e4SLinus Torvalds */ 210e18b890bSChristoph Lameter static struct kmem_cache *exception_cache; 211e18b890bSChristoph Lameter static struct kmem_cache *pending_cache; 2121da177e4SLinus Torvalds 213cd45daffSMikulas Patocka struct dm_snap_tracked_chunk { 214cd45daffSMikulas Patocka struct hlist_node node; 215cd45daffSMikulas Patocka chunk_t chunk; 216cd45daffSMikulas Patocka }; 217cd45daffSMikulas Patocka 218ee18026aSMikulas Patocka static void init_tracked_chunk(struct bio *bio) 219ee18026aSMikulas Patocka { 220ee18026aSMikulas Patocka struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk)); 221ee18026aSMikulas Patocka INIT_HLIST_NODE(&c->node); 222ee18026aSMikulas Patocka } 223ee18026aSMikulas Patocka 224ee18026aSMikulas Patocka static bool is_bio_tracked(struct bio *bio) 225ee18026aSMikulas Patocka { 226ee18026aSMikulas Patocka struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk)); 227ee18026aSMikulas Patocka return !hlist_unhashed(&c->node); 228ee18026aSMikulas Patocka } 229ee18026aSMikulas Patocka 230ee18026aSMikulas Patocka static void track_chunk(struct dm_snapshot *s, struct bio *bio, chunk_t chunk) 231cd45daffSMikulas Patocka { 23242bc954fSMikulas Patocka struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk)); 233cd45daffSMikulas Patocka 234cd45daffSMikulas Patocka c->chunk = chunk; 235cd45daffSMikulas Patocka 2369aa0c0e6SMikulas Patocka spin_lock_irq(&s->tracked_chunk_lock); 237cd45daffSMikulas Patocka hlist_add_head(&c->node, 238cd45daffSMikulas Patocka &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)]); 2399aa0c0e6SMikulas Patocka spin_unlock_irq(&s->tracked_chunk_lock); 240cd45daffSMikulas Patocka } 241cd45daffSMikulas Patocka 242ee18026aSMikulas Patocka static void stop_tracking_chunk(struct dm_snapshot *s, struct bio *bio) 243cd45daffSMikulas Patocka { 244ee18026aSMikulas Patocka struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk)); 245cd45daffSMikulas Patocka unsigned long flags; 246cd45daffSMikulas Patocka 247cd45daffSMikulas Patocka spin_lock_irqsave(&s->tracked_chunk_lock, flags); 248cd45daffSMikulas Patocka hlist_del(&c->node); 249cd45daffSMikulas Patocka spin_unlock_irqrestore(&s->tracked_chunk_lock, flags); 250cd45daffSMikulas Patocka } 251cd45daffSMikulas Patocka 252a8d41b59SMikulas Patocka static int __chunk_is_tracked(struct dm_snapshot *s, chunk_t chunk) 253a8d41b59SMikulas Patocka { 254a8d41b59SMikulas Patocka struct dm_snap_tracked_chunk *c; 255a8d41b59SMikulas Patocka int found = 0; 256a8d41b59SMikulas Patocka 257a8d41b59SMikulas Patocka spin_lock_irq(&s->tracked_chunk_lock); 258a8d41b59SMikulas Patocka 259b67bfe0dSSasha Levin hlist_for_each_entry(c, 260a8d41b59SMikulas Patocka &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)], node) { 261a8d41b59SMikulas Patocka if (c->chunk == chunk) { 262a8d41b59SMikulas Patocka found = 1; 263a8d41b59SMikulas Patocka break; 264a8d41b59SMikulas Patocka } 265a8d41b59SMikulas Patocka } 266a8d41b59SMikulas Patocka 267a8d41b59SMikulas Patocka spin_unlock_irq(&s->tracked_chunk_lock); 268a8d41b59SMikulas Patocka 269a8d41b59SMikulas Patocka return found; 270a8d41b59SMikulas Patocka } 271a8d41b59SMikulas Patocka 2721da177e4SLinus Torvalds /* 273615d1eb9SMike Snitzer * This conflicting I/O is extremely improbable in the caller, 274615d1eb9SMike Snitzer * so msleep(1) is sufficient and there is no need for a wait queue. 275615d1eb9SMike Snitzer */ 276615d1eb9SMike Snitzer static void __check_for_conflicting_io(struct dm_snapshot *s, chunk_t chunk) 277615d1eb9SMike Snitzer { 278615d1eb9SMike Snitzer while (__chunk_is_tracked(s, chunk)) 279615d1eb9SMike Snitzer msleep(1); 280615d1eb9SMike Snitzer } 281615d1eb9SMike Snitzer 282615d1eb9SMike Snitzer /* 2831da177e4SLinus Torvalds * One of these per registered origin, held in the snapshot_origins hash 2841da177e4SLinus Torvalds */ 2851da177e4SLinus Torvalds struct origin { 2861da177e4SLinus Torvalds /* The origin device */ 2871da177e4SLinus Torvalds struct block_device *bdev; 2881da177e4SLinus Torvalds 2891da177e4SLinus Torvalds struct list_head hash_list; 2901da177e4SLinus Torvalds 2911da177e4SLinus Torvalds /* List of snapshots for this origin */ 2921da177e4SLinus Torvalds struct list_head snapshots; 2931da177e4SLinus Torvalds }; 2941da177e4SLinus Torvalds 2951da177e4SLinus Torvalds /* 296b735fedeSMikulas Patocka * This structure is allocated for each origin target 297b735fedeSMikulas Patocka */ 298b735fedeSMikulas Patocka struct dm_origin { 299b735fedeSMikulas Patocka struct dm_dev *dev; 300b735fedeSMikulas Patocka struct dm_target *ti; 301b735fedeSMikulas Patocka unsigned split_boundary; 302b735fedeSMikulas Patocka struct list_head hash_list; 303b735fedeSMikulas Patocka }; 304b735fedeSMikulas Patocka 305b735fedeSMikulas Patocka /* 3061da177e4SLinus Torvalds * Size of the hash table for origin volumes. If we make this 3071da177e4SLinus Torvalds * the size of the minors list then it should be nearly perfect 3081da177e4SLinus Torvalds */ 3091da177e4SLinus Torvalds #define ORIGIN_HASH_SIZE 256 3101da177e4SLinus Torvalds #define ORIGIN_MASK 0xFF 3111da177e4SLinus Torvalds static struct list_head *_origins; 312b735fedeSMikulas Patocka static struct list_head *_dm_origins; 3131da177e4SLinus Torvalds static struct rw_semaphore _origins_lock; 3141da177e4SLinus Torvalds 31573dfd078SMikulas Patocka static DECLARE_WAIT_QUEUE_HEAD(_pending_exceptions_done); 31673dfd078SMikulas Patocka static DEFINE_SPINLOCK(_pending_exceptions_done_spinlock); 31773dfd078SMikulas Patocka static uint64_t _pending_exceptions_done_count; 31873dfd078SMikulas Patocka 3191da177e4SLinus Torvalds static int init_origin_hash(void) 3201da177e4SLinus Torvalds { 3211da177e4SLinus Torvalds int i; 3221da177e4SLinus Torvalds 3231da177e4SLinus Torvalds _origins = kmalloc(ORIGIN_HASH_SIZE * sizeof(struct list_head), 3241da177e4SLinus Torvalds GFP_KERNEL); 3251da177e4SLinus Torvalds if (!_origins) { 326b735fedeSMikulas Patocka DMERR("unable to allocate memory for _origins"); 3271da177e4SLinus Torvalds return -ENOMEM; 3281da177e4SLinus Torvalds } 3291da177e4SLinus Torvalds for (i = 0; i < ORIGIN_HASH_SIZE; i++) 3301da177e4SLinus Torvalds INIT_LIST_HEAD(_origins + i); 331b735fedeSMikulas Patocka 332b735fedeSMikulas Patocka _dm_origins = kmalloc(ORIGIN_HASH_SIZE * sizeof(struct list_head), 333b735fedeSMikulas Patocka GFP_KERNEL); 334b735fedeSMikulas Patocka if (!_dm_origins) { 335b735fedeSMikulas Patocka DMERR("unable to allocate memory for _dm_origins"); 336b735fedeSMikulas Patocka kfree(_origins); 337b735fedeSMikulas Patocka return -ENOMEM; 338b735fedeSMikulas Patocka } 339b735fedeSMikulas Patocka for (i = 0; i < ORIGIN_HASH_SIZE; i++) 340b735fedeSMikulas Patocka INIT_LIST_HEAD(_dm_origins + i); 341b735fedeSMikulas Patocka 3421da177e4SLinus Torvalds init_rwsem(&_origins_lock); 3431da177e4SLinus Torvalds 3441da177e4SLinus Torvalds return 0; 3451da177e4SLinus Torvalds } 3461da177e4SLinus Torvalds 3471da177e4SLinus Torvalds static void exit_origin_hash(void) 3481da177e4SLinus Torvalds { 3491da177e4SLinus Torvalds kfree(_origins); 350b735fedeSMikulas Patocka kfree(_dm_origins); 3511da177e4SLinus Torvalds } 3521da177e4SLinus Torvalds 353028867acSAlasdair G Kergon static unsigned origin_hash(struct block_device *bdev) 3541da177e4SLinus Torvalds { 3551da177e4SLinus Torvalds return bdev->bd_dev & ORIGIN_MASK; 3561da177e4SLinus Torvalds } 3571da177e4SLinus Torvalds 3581da177e4SLinus Torvalds static struct origin *__lookup_origin(struct block_device *origin) 3591da177e4SLinus Torvalds { 3601da177e4SLinus Torvalds struct list_head *ol; 3611da177e4SLinus Torvalds struct origin *o; 3621da177e4SLinus Torvalds 3631da177e4SLinus Torvalds ol = &_origins[origin_hash(origin)]; 3641da177e4SLinus Torvalds list_for_each_entry (o, ol, hash_list) 3651da177e4SLinus Torvalds if (bdev_equal(o->bdev, origin)) 3661da177e4SLinus Torvalds return o; 3671da177e4SLinus Torvalds 3681da177e4SLinus Torvalds return NULL; 3691da177e4SLinus Torvalds } 3701da177e4SLinus Torvalds 3711da177e4SLinus Torvalds static void __insert_origin(struct origin *o) 3721da177e4SLinus Torvalds { 3731da177e4SLinus Torvalds struct list_head *sl = &_origins[origin_hash(o->bdev)]; 3741da177e4SLinus Torvalds list_add_tail(&o->hash_list, sl); 3751da177e4SLinus Torvalds } 3761da177e4SLinus Torvalds 377b735fedeSMikulas Patocka static struct dm_origin *__lookup_dm_origin(struct block_device *origin) 378b735fedeSMikulas Patocka { 379b735fedeSMikulas Patocka struct list_head *ol; 380b735fedeSMikulas Patocka struct dm_origin *o; 381b735fedeSMikulas Patocka 382b735fedeSMikulas Patocka ol = &_dm_origins[origin_hash(origin)]; 383b735fedeSMikulas Patocka list_for_each_entry (o, ol, hash_list) 384b735fedeSMikulas Patocka if (bdev_equal(o->dev->bdev, origin)) 385b735fedeSMikulas Patocka return o; 386b735fedeSMikulas Patocka 387b735fedeSMikulas Patocka return NULL; 388b735fedeSMikulas Patocka } 389b735fedeSMikulas Patocka 390b735fedeSMikulas Patocka static void __insert_dm_origin(struct dm_origin *o) 391b735fedeSMikulas Patocka { 392b735fedeSMikulas Patocka struct list_head *sl = &_dm_origins[origin_hash(o->dev->bdev)]; 393b735fedeSMikulas Patocka list_add_tail(&o->hash_list, sl); 394b735fedeSMikulas Patocka } 395b735fedeSMikulas Patocka 396b735fedeSMikulas Patocka static void __remove_dm_origin(struct dm_origin *o) 397b735fedeSMikulas Patocka { 398b735fedeSMikulas Patocka list_del(&o->hash_list); 399b735fedeSMikulas Patocka } 400b735fedeSMikulas Patocka 4011da177e4SLinus Torvalds /* 402c1f0c183SMike Snitzer * _origins_lock must be held when calling this function. 403c1f0c183SMike Snitzer * Returns number of snapshots registered using the supplied cow device, plus: 404c1f0c183SMike Snitzer * snap_src - a snapshot suitable for use as a source of exception handover 405c1f0c183SMike Snitzer * snap_dest - a snapshot capable of receiving exception handover. 4069d3b15c4SMikulas Patocka * snap_merge - an existing snapshot-merge target linked to the same origin. 4079d3b15c4SMikulas Patocka * There can be at most one snapshot-merge target. The parameter is optional. 408c1f0c183SMike Snitzer * 4099d3b15c4SMikulas Patocka * Possible return values and states of snap_src and snap_dest. 410c1f0c183SMike Snitzer * 0: NULL, NULL - first new snapshot 411c1f0c183SMike Snitzer * 1: snap_src, NULL - normal snapshot 412c1f0c183SMike Snitzer * 2: snap_src, snap_dest - waiting for handover 413c1f0c183SMike Snitzer * 2: snap_src, NULL - handed over, waiting for old to be deleted 414c1f0c183SMike Snitzer * 1: NULL, snap_dest - source got destroyed without handover 415c1f0c183SMike Snitzer */ 416c1f0c183SMike Snitzer static int __find_snapshots_sharing_cow(struct dm_snapshot *snap, 417c1f0c183SMike Snitzer struct dm_snapshot **snap_src, 4189d3b15c4SMikulas Patocka struct dm_snapshot **snap_dest, 4199d3b15c4SMikulas Patocka struct dm_snapshot **snap_merge) 420c1f0c183SMike Snitzer { 421c1f0c183SMike Snitzer struct dm_snapshot *s; 422c1f0c183SMike Snitzer struct origin *o; 423c1f0c183SMike Snitzer int count = 0; 424c1f0c183SMike Snitzer int active; 425c1f0c183SMike Snitzer 426c1f0c183SMike Snitzer o = __lookup_origin(snap->origin->bdev); 427c1f0c183SMike Snitzer if (!o) 428c1f0c183SMike Snitzer goto out; 429c1f0c183SMike Snitzer 430c1f0c183SMike Snitzer list_for_each_entry(s, &o->snapshots, list) { 4319d3b15c4SMikulas Patocka if (dm_target_is_snapshot_merge(s->ti) && snap_merge) 4329d3b15c4SMikulas Patocka *snap_merge = s; 433c1f0c183SMike Snitzer if (!bdev_equal(s->cow->bdev, snap->cow->bdev)) 434c1f0c183SMike Snitzer continue; 435c1f0c183SMike Snitzer 436c1f0c183SMike Snitzer down_read(&s->lock); 437c1f0c183SMike Snitzer active = s->active; 438c1f0c183SMike Snitzer up_read(&s->lock); 439c1f0c183SMike Snitzer 440c1f0c183SMike Snitzer if (active) { 441c1f0c183SMike Snitzer if (snap_src) 442c1f0c183SMike Snitzer *snap_src = s; 443c1f0c183SMike Snitzer } else if (snap_dest) 444c1f0c183SMike Snitzer *snap_dest = s; 445c1f0c183SMike Snitzer 446c1f0c183SMike Snitzer count++; 447c1f0c183SMike Snitzer } 448c1f0c183SMike Snitzer 449c1f0c183SMike Snitzer out: 450c1f0c183SMike Snitzer return count; 451c1f0c183SMike Snitzer } 452c1f0c183SMike Snitzer 453c1f0c183SMike Snitzer /* 454c1f0c183SMike Snitzer * On success, returns 1 if this snapshot is a handover destination, 455c1f0c183SMike Snitzer * otherwise returns 0. 456c1f0c183SMike Snitzer */ 457c1f0c183SMike Snitzer static int __validate_exception_handover(struct dm_snapshot *snap) 458c1f0c183SMike Snitzer { 459c1f0c183SMike Snitzer struct dm_snapshot *snap_src = NULL, *snap_dest = NULL; 4609d3b15c4SMikulas Patocka struct dm_snapshot *snap_merge = NULL; 461c1f0c183SMike Snitzer 462c1f0c183SMike Snitzer /* Does snapshot need exceptions handed over to it? */ 4639d3b15c4SMikulas Patocka if ((__find_snapshots_sharing_cow(snap, &snap_src, &snap_dest, 4649d3b15c4SMikulas Patocka &snap_merge) == 2) || 465c1f0c183SMike Snitzer snap_dest) { 466c1f0c183SMike Snitzer snap->ti->error = "Snapshot cow pairing for exception " 467c1f0c183SMike Snitzer "table handover failed"; 468c1f0c183SMike Snitzer return -EINVAL; 469c1f0c183SMike Snitzer } 470c1f0c183SMike Snitzer 471c1f0c183SMike Snitzer /* 472c1f0c183SMike Snitzer * If no snap_src was found, snap cannot become a handover 473c1f0c183SMike Snitzer * destination. 474c1f0c183SMike Snitzer */ 475c1f0c183SMike Snitzer if (!snap_src) 476c1f0c183SMike Snitzer return 0; 477c1f0c183SMike Snitzer 4789d3b15c4SMikulas Patocka /* 4799d3b15c4SMikulas Patocka * Non-snapshot-merge handover? 4809d3b15c4SMikulas Patocka */ 4819d3b15c4SMikulas Patocka if (!dm_target_is_snapshot_merge(snap->ti)) 4829d3b15c4SMikulas Patocka return 1; 4839d3b15c4SMikulas Patocka 4849d3b15c4SMikulas Patocka /* 4859d3b15c4SMikulas Patocka * Do not allow more than one merging snapshot. 4869d3b15c4SMikulas Patocka */ 4879d3b15c4SMikulas Patocka if (snap_merge) { 4889d3b15c4SMikulas Patocka snap->ti->error = "A snapshot is already merging."; 4899d3b15c4SMikulas Patocka return -EINVAL; 4909d3b15c4SMikulas Patocka } 4919d3b15c4SMikulas Patocka 4921e03f97eSMikulas Patocka if (!snap_src->store->type->prepare_merge || 4931e03f97eSMikulas Patocka !snap_src->store->type->commit_merge) { 4941e03f97eSMikulas Patocka snap->ti->error = "Snapshot exception store does not " 4951e03f97eSMikulas Patocka "support snapshot-merge."; 4961e03f97eSMikulas Patocka return -EINVAL; 4971e03f97eSMikulas Patocka } 4981e03f97eSMikulas Patocka 499c1f0c183SMike Snitzer return 1; 500c1f0c183SMike Snitzer } 501c1f0c183SMike Snitzer 502c1f0c183SMike Snitzer static void __insert_snapshot(struct origin *o, struct dm_snapshot *s) 503c1f0c183SMike Snitzer { 504c1f0c183SMike Snitzer struct dm_snapshot *l; 505c1f0c183SMike Snitzer 506c1f0c183SMike Snitzer /* Sort the list according to chunk size, largest-first smallest-last */ 507c1f0c183SMike Snitzer list_for_each_entry(l, &o->snapshots, list) 508c1f0c183SMike Snitzer if (l->store->chunk_size < s->store->chunk_size) 509c1f0c183SMike Snitzer break; 510c1f0c183SMike Snitzer list_add_tail(&s->list, &l->list); 511c1f0c183SMike Snitzer } 512c1f0c183SMike Snitzer 513c1f0c183SMike Snitzer /* 5141da177e4SLinus Torvalds * Make a note of the snapshot and its origin so we can look it 5151da177e4SLinus Torvalds * up when the origin has a write on it. 516c1f0c183SMike Snitzer * 517c1f0c183SMike Snitzer * Also validate snapshot exception store handovers. 518c1f0c183SMike Snitzer * On success, returns 1 if this registration is a handover destination, 519c1f0c183SMike Snitzer * otherwise returns 0. 5201da177e4SLinus Torvalds */ 5211da177e4SLinus Torvalds static int register_snapshot(struct dm_snapshot *snap) 5221da177e4SLinus Torvalds { 523c1f0c183SMike Snitzer struct origin *o, *new_o = NULL; 5241da177e4SLinus Torvalds struct block_device *bdev = snap->origin->bdev; 525c1f0c183SMike Snitzer int r = 0; 5261da177e4SLinus Torvalds 52760c856c8SMikulas Patocka new_o = kmalloc(sizeof(*new_o), GFP_KERNEL); 52860c856c8SMikulas Patocka if (!new_o) 52960c856c8SMikulas Patocka return -ENOMEM; 53060c856c8SMikulas Patocka 5311da177e4SLinus Torvalds down_write(&_origins_lock); 5321da177e4SLinus Torvalds 533c1f0c183SMike Snitzer r = __validate_exception_handover(snap); 534c1f0c183SMike Snitzer if (r < 0) { 535c1f0c183SMike Snitzer kfree(new_o); 536c1f0c183SMike Snitzer goto out; 537c1f0c183SMike Snitzer } 538c1f0c183SMike Snitzer 539c1f0c183SMike Snitzer o = __lookup_origin(bdev); 54060c856c8SMikulas Patocka if (o) 54160c856c8SMikulas Patocka kfree(new_o); 54260c856c8SMikulas Patocka else { 5431da177e4SLinus Torvalds /* New origin */ 54460c856c8SMikulas Patocka o = new_o; 5451da177e4SLinus Torvalds 5461da177e4SLinus Torvalds /* Initialise the struct */ 5471da177e4SLinus Torvalds INIT_LIST_HEAD(&o->snapshots); 5481da177e4SLinus Torvalds o->bdev = bdev; 5491da177e4SLinus Torvalds 5501da177e4SLinus Torvalds __insert_origin(o); 5511da177e4SLinus Torvalds } 5521da177e4SLinus Torvalds 553c1f0c183SMike Snitzer __insert_snapshot(o, snap); 554c1f0c183SMike Snitzer 555c1f0c183SMike Snitzer out: 556c1f0c183SMike Snitzer up_write(&_origins_lock); 557c1f0c183SMike Snitzer 558c1f0c183SMike Snitzer return r; 559c1f0c183SMike Snitzer } 560c1f0c183SMike Snitzer 561c1f0c183SMike Snitzer /* 562c1f0c183SMike Snitzer * Move snapshot to correct place in list according to chunk size. 563c1f0c183SMike Snitzer */ 564c1f0c183SMike Snitzer static void reregister_snapshot(struct dm_snapshot *s) 565c1f0c183SMike Snitzer { 566c1f0c183SMike Snitzer struct block_device *bdev = s->origin->bdev; 567c1f0c183SMike Snitzer 568c1f0c183SMike Snitzer down_write(&_origins_lock); 569c1f0c183SMike Snitzer 570c1f0c183SMike Snitzer list_del(&s->list); 571c1f0c183SMike Snitzer __insert_snapshot(__lookup_origin(bdev), s); 5721da177e4SLinus Torvalds 5731da177e4SLinus Torvalds up_write(&_origins_lock); 5741da177e4SLinus Torvalds } 5751da177e4SLinus Torvalds 5761da177e4SLinus Torvalds static void unregister_snapshot(struct dm_snapshot *s) 5771da177e4SLinus Torvalds { 5781da177e4SLinus Torvalds struct origin *o; 5791da177e4SLinus Torvalds 5801da177e4SLinus Torvalds down_write(&_origins_lock); 5811da177e4SLinus Torvalds o = __lookup_origin(s->origin->bdev); 5821da177e4SLinus Torvalds 5831da177e4SLinus Torvalds list_del(&s->list); 584c1f0c183SMike Snitzer if (o && list_empty(&o->snapshots)) { 5851da177e4SLinus Torvalds list_del(&o->hash_list); 5861da177e4SLinus Torvalds kfree(o); 5871da177e4SLinus Torvalds } 5881da177e4SLinus Torvalds 5891da177e4SLinus Torvalds up_write(&_origins_lock); 5901da177e4SLinus Torvalds } 5911da177e4SLinus Torvalds 5921da177e4SLinus Torvalds /* 5931da177e4SLinus Torvalds * Implementation of the exception hash tables. 594d74f81f8SMilan Broz * The lowest hash_shift bits of the chunk number are ignored, allowing 595d74f81f8SMilan Broz * some consecutive chunks to be grouped together. 5961da177e4SLinus Torvalds */ 5973510cb94SJon Brassow static int dm_exception_table_init(struct dm_exception_table *et, 5983510cb94SJon Brassow uint32_t size, unsigned hash_shift) 5991da177e4SLinus Torvalds { 6001da177e4SLinus Torvalds unsigned int i; 6011da177e4SLinus Torvalds 602d74f81f8SMilan Broz et->hash_shift = hash_shift; 6031da177e4SLinus Torvalds et->hash_mask = size - 1; 6041da177e4SLinus Torvalds et->table = dm_vcalloc(size, sizeof(struct list_head)); 6051da177e4SLinus Torvalds if (!et->table) 6061da177e4SLinus Torvalds return -ENOMEM; 6071da177e4SLinus Torvalds 6081da177e4SLinus Torvalds for (i = 0; i < size; i++) 6091da177e4SLinus Torvalds INIT_LIST_HEAD(et->table + i); 6101da177e4SLinus Torvalds 6111da177e4SLinus Torvalds return 0; 6121da177e4SLinus Torvalds } 6131da177e4SLinus Torvalds 6143510cb94SJon Brassow static void dm_exception_table_exit(struct dm_exception_table *et, 615191437a5SJon Brassow struct kmem_cache *mem) 6161da177e4SLinus Torvalds { 6171da177e4SLinus Torvalds struct list_head *slot; 6181d4989c8SJon Brassow struct dm_exception *ex, *next; 6191da177e4SLinus Torvalds int i, size; 6201da177e4SLinus Torvalds 6211da177e4SLinus Torvalds size = et->hash_mask + 1; 6221da177e4SLinus Torvalds for (i = 0; i < size; i++) { 6231da177e4SLinus Torvalds slot = et->table + i; 6241da177e4SLinus Torvalds 6251da177e4SLinus Torvalds list_for_each_entry_safe (ex, next, slot, hash_list) 6261da177e4SLinus Torvalds kmem_cache_free(mem, ex); 6271da177e4SLinus Torvalds } 6281da177e4SLinus Torvalds 6291da177e4SLinus Torvalds vfree(et->table); 6301da177e4SLinus Torvalds } 6311da177e4SLinus Torvalds 632191437a5SJon Brassow static uint32_t exception_hash(struct dm_exception_table *et, chunk_t chunk) 6331da177e4SLinus Torvalds { 634d74f81f8SMilan Broz return (chunk >> et->hash_shift) & et->hash_mask; 6351da177e4SLinus Torvalds } 6361da177e4SLinus Torvalds 6373510cb94SJon Brassow static void dm_remove_exception(struct dm_exception *e) 6381da177e4SLinus Torvalds { 6391da177e4SLinus Torvalds list_del(&e->hash_list); 6401da177e4SLinus Torvalds } 6411da177e4SLinus Torvalds 6421da177e4SLinus Torvalds /* 6431da177e4SLinus Torvalds * Return the exception data for a sector, or NULL if not 6441da177e4SLinus Torvalds * remapped. 6451da177e4SLinus Torvalds */ 6463510cb94SJon Brassow static struct dm_exception *dm_lookup_exception(struct dm_exception_table *et, 6471da177e4SLinus Torvalds chunk_t chunk) 6481da177e4SLinus Torvalds { 6491da177e4SLinus Torvalds struct list_head *slot; 6501d4989c8SJon Brassow struct dm_exception *e; 6511da177e4SLinus Torvalds 6521da177e4SLinus Torvalds slot = &et->table[exception_hash(et, chunk)]; 6531da177e4SLinus Torvalds list_for_each_entry (e, slot, hash_list) 654d74f81f8SMilan Broz if (chunk >= e->old_chunk && 655d74f81f8SMilan Broz chunk <= e->old_chunk + dm_consecutive_chunk_count(e)) 6561da177e4SLinus Torvalds return e; 6571da177e4SLinus Torvalds 6581da177e4SLinus Torvalds return NULL; 6591da177e4SLinus Torvalds } 6601da177e4SLinus Torvalds 661119bc547SMikulas Patocka static struct dm_exception *alloc_completed_exception(gfp_t gfp) 6621da177e4SLinus Torvalds { 6631d4989c8SJon Brassow struct dm_exception *e; 6641da177e4SLinus Torvalds 665119bc547SMikulas Patocka e = kmem_cache_alloc(exception_cache, gfp); 666119bc547SMikulas Patocka if (!e && gfp == GFP_NOIO) 6671da177e4SLinus Torvalds e = kmem_cache_alloc(exception_cache, GFP_ATOMIC); 6681da177e4SLinus Torvalds 6691da177e4SLinus Torvalds return e; 6701da177e4SLinus Torvalds } 6711da177e4SLinus Torvalds 6723510cb94SJon Brassow static void free_completed_exception(struct dm_exception *e) 6731da177e4SLinus Torvalds { 6741da177e4SLinus Torvalds kmem_cache_free(exception_cache, e); 6751da177e4SLinus Torvalds } 6761da177e4SLinus Torvalds 67792e86812SMikulas Patocka static struct dm_snap_pending_exception *alloc_pending_exception(struct dm_snapshot *s) 6781da177e4SLinus Torvalds { 67992e86812SMikulas Patocka struct dm_snap_pending_exception *pe = mempool_alloc(s->pending_pool, 68092e86812SMikulas Patocka GFP_NOIO); 68192e86812SMikulas Patocka 682879129d2SMikulas Patocka atomic_inc(&s->pending_exceptions_count); 68392e86812SMikulas Patocka pe->snap = s; 68492e86812SMikulas Patocka 68592e86812SMikulas Patocka return pe; 6861da177e4SLinus Torvalds } 6871da177e4SLinus Torvalds 688028867acSAlasdair G Kergon static void free_pending_exception(struct dm_snap_pending_exception *pe) 6891da177e4SLinus Torvalds { 690879129d2SMikulas Patocka struct dm_snapshot *s = pe->snap; 691879129d2SMikulas Patocka 692879129d2SMikulas Patocka mempool_free(pe, s->pending_pool); 6934e857c58SPeter Zijlstra smp_mb__before_atomic(); 694879129d2SMikulas Patocka atomic_dec(&s->pending_exceptions_count); 6951da177e4SLinus Torvalds } 6961da177e4SLinus Torvalds 6973510cb94SJon Brassow static void dm_insert_exception(struct dm_exception_table *eh, 6981d4989c8SJon Brassow struct dm_exception *new_e) 699d74f81f8SMilan Broz { 700d74f81f8SMilan Broz struct list_head *l; 7011d4989c8SJon Brassow struct dm_exception *e = NULL; 702d74f81f8SMilan Broz 703d74f81f8SMilan Broz l = &eh->table[exception_hash(eh, new_e->old_chunk)]; 704d74f81f8SMilan Broz 705d74f81f8SMilan Broz /* Add immediately if this table doesn't support consecutive chunks */ 706d74f81f8SMilan Broz if (!eh->hash_shift) 707d74f81f8SMilan Broz goto out; 708d74f81f8SMilan Broz 709d74f81f8SMilan Broz /* List is ordered by old_chunk */ 710d74f81f8SMilan Broz list_for_each_entry_reverse(e, l, hash_list) { 711d74f81f8SMilan Broz /* Insert after an existing chunk? */ 712d74f81f8SMilan Broz if (new_e->old_chunk == (e->old_chunk + 713d74f81f8SMilan Broz dm_consecutive_chunk_count(e) + 1) && 714d74f81f8SMilan Broz new_e->new_chunk == (dm_chunk_number(e->new_chunk) + 715d74f81f8SMilan Broz dm_consecutive_chunk_count(e) + 1)) { 716d74f81f8SMilan Broz dm_consecutive_chunk_count_inc(e); 7173510cb94SJon Brassow free_completed_exception(new_e); 718d74f81f8SMilan Broz return; 719d74f81f8SMilan Broz } 720d74f81f8SMilan Broz 721d74f81f8SMilan Broz /* Insert before an existing chunk? */ 722d74f81f8SMilan Broz if (new_e->old_chunk == (e->old_chunk - 1) && 723d74f81f8SMilan Broz new_e->new_chunk == (dm_chunk_number(e->new_chunk) - 1)) { 724d74f81f8SMilan Broz dm_consecutive_chunk_count_inc(e); 725d74f81f8SMilan Broz e->old_chunk--; 726d74f81f8SMilan Broz e->new_chunk--; 7273510cb94SJon Brassow free_completed_exception(new_e); 728d74f81f8SMilan Broz return; 729d74f81f8SMilan Broz } 730d74f81f8SMilan Broz 731d74f81f8SMilan Broz if (new_e->old_chunk > e->old_chunk) 732d74f81f8SMilan Broz break; 733d74f81f8SMilan Broz } 734d74f81f8SMilan Broz 735d74f81f8SMilan Broz out: 736d74f81f8SMilan Broz list_add(&new_e->hash_list, e ? &e->hash_list : l); 737d74f81f8SMilan Broz } 738d74f81f8SMilan Broz 739a159c1acSJonathan Brassow /* 740a159c1acSJonathan Brassow * Callback used by the exception stores to load exceptions when 741a159c1acSJonathan Brassow * initialising. 742a159c1acSJonathan Brassow */ 743a159c1acSJonathan Brassow static int dm_add_exception(void *context, chunk_t old, chunk_t new) 7441da177e4SLinus Torvalds { 745a159c1acSJonathan Brassow struct dm_snapshot *s = context; 7461d4989c8SJon Brassow struct dm_exception *e; 7471da177e4SLinus Torvalds 748119bc547SMikulas Patocka e = alloc_completed_exception(GFP_KERNEL); 7491da177e4SLinus Torvalds if (!e) 7501da177e4SLinus Torvalds return -ENOMEM; 7511da177e4SLinus Torvalds 7521da177e4SLinus Torvalds e->old_chunk = old; 753d74f81f8SMilan Broz 754d74f81f8SMilan Broz /* Consecutive_count is implicitly initialised to zero */ 7551da177e4SLinus Torvalds e->new_chunk = new; 756d74f81f8SMilan Broz 7573510cb94SJon Brassow dm_insert_exception(&s->complete, e); 758d74f81f8SMilan Broz 7591da177e4SLinus Torvalds return 0; 7601da177e4SLinus Torvalds } 7611da177e4SLinus Torvalds 7627e201b35SMikulas Patocka /* 7637e201b35SMikulas Patocka * Return a minimum chunk size of all snapshots that have the specified origin. 7647e201b35SMikulas Patocka * Return zero if the origin has no snapshots. 7657e201b35SMikulas Patocka */ 766542f9038SMike Snitzer static uint32_t __minimum_chunk_size(struct origin *o) 7677e201b35SMikulas Patocka { 7687e201b35SMikulas Patocka struct dm_snapshot *snap; 7697e201b35SMikulas Patocka unsigned chunk_size = 0; 7707e201b35SMikulas Patocka 7717e201b35SMikulas Patocka if (o) 7727e201b35SMikulas Patocka list_for_each_entry(snap, &o->snapshots, list) 7737e201b35SMikulas Patocka chunk_size = min_not_zero(chunk_size, 7747e201b35SMikulas Patocka snap->store->chunk_size); 7757e201b35SMikulas Patocka 776542f9038SMike Snitzer return (uint32_t) chunk_size; 7777e201b35SMikulas Patocka } 7787e201b35SMikulas Patocka 7791da177e4SLinus Torvalds /* 7801da177e4SLinus Torvalds * Hard coded magic. 7811da177e4SLinus Torvalds */ 7821da177e4SLinus Torvalds static int calc_max_buckets(void) 7831da177e4SLinus Torvalds { 7841da177e4SLinus Torvalds /* use a fixed size of 2MB */ 7851da177e4SLinus Torvalds unsigned long mem = 2 * 1024 * 1024; 7861da177e4SLinus Torvalds mem /= sizeof(struct list_head); 7871da177e4SLinus Torvalds 7881da177e4SLinus Torvalds return mem; 7891da177e4SLinus Torvalds } 7901da177e4SLinus Torvalds 7911da177e4SLinus Torvalds /* 7921da177e4SLinus Torvalds * Allocate room for a suitable hash table. 7931da177e4SLinus Torvalds */ 794fee1998eSJonathan Brassow static int init_hash_tables(struct dm_snapshot *s) 7951da177e4SLinus Torvalds { 79660e356f3SMikulas Patocka sector_t hash_size, cow_dev_size, max_buckets; 7971da177e4SLinus Torvalds 7981da177e4SLinus Torvalds /* 7991da177e4SLinus Torvalds * Calculate based on the size of the original volume or 8001da177e4SLinus Torvalds * the COW volume... 8011da177e4SLinus Torvalds */ 802fc56f6fbSMike Snitzer cow_dev_size = get_dev_size(s->cow->bdev); 8031da177e4SLinus Torvalds max_buckets = calc_max_buckets(); 8041da177e4SLinus Torvalds 80560e356f3SMikulas Patocka hash_size = cow_dev_size >> s->store->chunk_shift; 8061da177e4SLinus Torvalds hash_size = min(hash_size, max_buckets); 8071da177e4SLinus Torvalds 8088e87b9b8SMikulas Patocka if (hash_size < 64) 8098e87b9b8SMikulas Patocka hash_size = 64; 8108defd830SRobert P. J. Day hash_size = rounddown_pow_of_two(hash_size); 8113510cb94SJon Brassow if (dm_exception_table_init(&s->complete, hash_size, 812d74f81f8SMilan Broz DM_CHUNK_CONSECUTIVE_BITS)) 8131da177e4SLinus Torvalds return -ENOMEM; 8141da177e4SLinus Torvalds 8151da177e4SLinus Torvalds /* 8161da177e4SLinus Torvalds * Allocate hash table for in-flight exceptions 8171da177e4SLinus Torvalds * Make this smaller than the real hash table 8181da177e4SLinus Torvalds */ 8191da177e4SLinus Torvalds hash_size >>= 3; 8201da177e4SLinus Torvalds if (hash_size < 64) 8211da177e4SLinus Torvalds hash_size = 64; 8221da177e4SLinus Torvalds 8233510cb94SJon Brassow if (dm_exception_table_init(&s->pending, hash_size, 0)) { 8243510cb94SJon Brassow dm_exception_table_exit(&s->complete, exception_cache); 8251da177e4SLinus Torvalds return -ENOMEM; 8261da177e4SLinus Torvalds } 8271da177e4SLinus Torvalds 8281da177e4SLinus Torvalds return 0; 8291da177e4SLinus Torvalds } 8301da177e4SLinus Torvalds 8311e03f97eSMikulas Patocka static void merge_shutdown(struct dm_snapshot *s) 8321e03f97eSMikulas Patocka { 8331e03f97eSMikulas Patocka clear_bit_unlock(RUNNING_MERGE, &s->state_bits); 8344e857c58SPeter Zijlstra smp_mb__after_atomic(); 8351e03f97eSMikulas Patocka wake_up_bit(&s->state_bits, RUNNING_MERGE); 8361e03f97eSMikulas Patocka } 8371e03f97eSMikulas Patocka 8389fe86254SMikulas Patocka static struct bio *__release_queued_bios_after_merge(struct dm_snapshot *s) 8399fe86254SMikulas Patocka { 8409fe86254SMikulas Patocka s->first_merging_chunk = 0; 8419fe86254SMikulas Patocka s->num_merging_chunks = 0; 8429fe86254SMikulas Patocka 8439fe86254SMikulas Patocka return bio_list_get(&s->bios_queued_during_merge); 8449fe86254SMikulas Patocka } 8459fe86254SMikulas Patocka 8461e03f97eSMikulas Patocka /* 8471e03f97eSMikulas Patocka * Remove one chunk from the index of completed exceptions. 8481e03f97eSMikulas Patocka */ 8491e03f97eSMikulas Patocka static int __remove_single_exception_chunk(struct dm_snapshot *s, 8501e03f97eSMikulas Patocka chunk_t old_chunk) 8511e03f97eSMikulas Patocka { 8521e03f97eSMikulas Patocka struct dm_exception *e; 8531e03f97eSMikulas Patocka 8541e03f97eSMikulas Patocka e = dm_lookup_exception(&s->complete, old_chunk); 8551e03f97eSMikulas Patocka if (!e) { 8561e03f97eSMikulas Patocka DMERR("Corruption detected: exception for block %llu is " 8571e03f97eSMikulas Patocka "on disk but not in memory", 8581e03f97eSMikulas Patocka (unsigned long long)old_chunk); 8591e03f97eSMikulas Patocka return -EINVAL; 8601e03f97eSMikulas Patocka } 8611e03f97eSMikulas Patocka 8621e03f97eSMikulas Patocka /* 8631e03f97eSMikulas Patocka * If this is the only chunk using this exception, remove exception. 8641e03f97eSMikulas Patocka */ 8651e03f97eSMikulas Patocka if (!dm_consecutive_chunk_count(e)) { 8661e03f97eSMikulas Patocka dm_remove_exception(e); 8671e03f97eSMikulas Patocka free_completed_exception(e); 8681e03f97eSMikulas Patocka return 0; 8691e03f97eSMikulas Patocka } 8701e03f97eSMikulas Patocka 8711e03f97eSMikulas Patocka /* 8721e03f97eSMikulas Patocka * The chunk may be either at the beginning or the end of a 8731e03f97eSMikulas Patocka * group of consecutive chunks - never in the middle. We are 8741e03f97eSMikulas Patocka * removing chunks in the opposite order to that in which they 8751e03f97eSMikulas Patocka * were added, so this should always be true. 8761e03f97eSMikulas Patocka * Decrement the consecutive chunk counter and adjust the 8771e03f97eSMikulas Patocka * starting point if necessary. 8781e03f97eSMikulas Patocka */ 8791e03f97eSMikulas Patocka if (old_chunk == e->old_chunk) { 8801e03f97eSMikulas Patocka e->old_chunk++; 8811e03f97eSMikulas Patocka e->new_chunk++; 8821e03f97eSMikulas Patocka } else if (old_chunk != e->old_chunk + 8831e03f97eSMikulas Patocka dm_consecutive_chunk_count(e)) { 8841e03f97eSMikulas Patocka DMERR("Attempt to merge block %llu from the " 8851e03f97eSMikulas Patocka "middle of a chunk range [%llu - %llu]", 8861e03f97eSMikulas Patocka (unsigned long long)old_chunk, 8871e03f97eSMikulas Patocka (unsigned long long)e->old_chunk, 8881e03f97eSMikulas Patocka (unsigned long long) 8891e03f97eSMikulas Patocka e->old_chunk + dm_consecutive_chunk_count(e)); 8901e03f97eSMikulas Patocka return -EINVAL; 8911e03f97eSMikulas Patocka } 8921e03f97eSMikulas Patocka 8931e03f97eSMikulas Patocka dm_consecutive_chunk_count_dec(e); 8941e03f97eSMikulas Patocka 8951e03f97eSMikulas Patocka return 0; 8961e03f97eSMikulas Patocka } 8971e03f97eSMikulas Patocka 8989fe86254SMikulas Patocka static void flush_bios(struct bio *bio); 8999fe86254SMikulas Patocka 9009fe86254SMikulas Patocka static int remove_single_exception_chunk(struct dm_snapshot *s) 9011e03f97eSMikulas Patocka { 9029fe86254SMikulas Patocka struct bio *b = NULL; 9039fe86254SMikulas Patocka int r; 9049fe86254SMikulas Patocka chunk_t old_chunk = s->first_merging_chunk + s->num_merging_chunks - 1; 9051e03f97eSMikulas Patocka 9061e03f97eSMikulas Patocka down_write(&s->lock); 9079fe86254SMikulas Patocka 9089fe86254SMikulas Patocka /* 9099fe86254SMikulas Patocka * Process chunks (and associated exceptions) in reverse order 9109fe86254SMikulas Patocka * so that dm_consecutive_chunk_count_dec() accounting works. 9119fe86254SMikulas Patocka */ 9129fe86254SMikulas Patocka do { 9131e03f97eSMikulas Patocka r = __remove_single_exception_chunk(s, old_chunk); 9149fe86254SMikulas Patocka if (r) 9159fe86254SMikulas Patocka goto out; 9169fe86254SMikulas Patocka } while (old_chunk-- > s->first_merging_chunk); 9179fe86254SMikulas Patocka 9189fe86254SMikulas Patocka b = __release_queued_bios_after_merge(s); 9199fe86254SMikulas Patocka 9209fe86254SMikulas Patocka out: 9211e03f97eSMikulas Patocka up_write(&s->lock); 9229fe86254SMikulas Patocka if (b) 9239fe86254SMikulas Patocka flush_bios(b); 9241e03f97eSMikulas Patocka 9251e03f97eSMikulas Patocka return r; 9261e03f97eSMikulas Patocka } 9271e03f97eSMikulas Patocka 92873dfd078SMikulas Patocka static int origin_write_extent(struct dm_snapshot *merging_snap, 92973dfd078SMikulas Patocka sector_t sector, unsigned chunk_size); 93073dfd078SMikulas Patocka 9311e03f97eSMikulas Patocka static void merge_callback(int read_err, unsigned long write_err, 9321e03f97eSMikulas Patocka void *context); 9331e03f97eSMikulas Patocka 93473dfd078SMikulas Patocka static uint64_t read_pending_exceptions_done_count(void) 93573dfd078SMikulas Patocka { 93673dfd078SMikulas Patocka uint64_t pending_exceptions_done; 93773dfd078SMikulas Patocka 93873dfd078SMikulas Patocka spin_lock(&_pending_exceptions_done_spinlock); 93973dfd078SMikulas Patocka pending_exceptions_done = _pending_exceptions_done_count; 94073dfd078SMikulas Patocka spin_unlock(&_pending_exceptions_done_spinlock); 94173dfd078SMikulas Patocka 94273dfd078SMikulas Patocka return pending_exceptions_done; 94373dfd078SMikulas Patocka } 94473dfd078SMikulas Patocka 94573dfd078SMikulas Patocka static void increment_pending_exceptions_done_count(void) 94673dfd078SMikulas Patocka { 94773dfd078SMikulas Patocka spin_lock(&_pending_exceptions_done_spinlock); 94873dfd078SMikulas Patocka _pending_exceptions_done_count++; 94973dfd078SMikulas Patocka spin_unlock(&_pending_exceptions_done_spinlock); 95073dfd078SMikulas Patocka 95173dfd078SMikulas Patocka wake_up_all(&_pending_exceptions_done); 95273dfd078SMikulas Patocka } 95373dfd078SMikulas Patocka 9541e03f97eSMikulas Patocka static void snapshot_merge_next_chunks(struct dm_snapshot *s) 9551e03f97eSMikulas Patocka { 9568a2d5286SMike Snitzer int i, linear_chunks; 9571e03f97eSMikulas Patocka chunk_t old_chunk, new_chunk; 9581e03f97eSMikulas Patocka struct dm_io_region src, dest; 9598a2d5286SMike Snitzer sector_t io_size; 96073dfd078SMikulas Patocka uint64_t previous_count; 9611e03f97eSMikulas Patocka 9621e03f97eSMikulas Patocka BUG_ON(!test_bit(RUNNING_MERGE, &s->state_bits)); 9631e03f97eSMikulas Patocka if (unlikely(test_bit(SHUTDOWN_MERGE, &s->state_bits))) 9641e03f97eSMikulas Patocka goto shut; 9651e03f97eSMikulas Patocka 9661e03f97eSMikulas Patocka /* 9671e03f97eSMikulas Patocka * valid flag never changes during merge, so no lock required. 9681e03f97eSMikulas Patocka */ 9691e03f97eSMikulas Patocka if (!s->valid) { 9701e03f97eSMikulas Patocka DMERR("Snapshot is invalid: can't merge"); 9711e03f97eSMikulas Patocka goto shut; 9721e03f97eSMikulas Patocka } 9731e03f97eSMikulas Patocka 9748a2d5286SMike Snitzer linear_chunks = s->store->type->prepare_merge(s->store, &old_chunk, 9758a2d5286SMike Snitzer &new_chunk); 9768a2d5286SMike Snitzer if (linear_chunks <= 0) { 977d8ddb1cfSMike Snitzer if (linear_chunks < 0) { 9781e03f97eSMikulas Patocka DMERR("Read error in exception store: " 9791e03f97eSMikulas Patocka "shutting down merge"); 980d8ddb1cfSMike Snitzer down_write(&s->lock); 981d8ddb1cfSMike Snitzer s->merge_failed = 1; 982d8ddb1cfSMike Snitzer up_write(&s->lock); 983d8ddb1cfSMike Snitzer } 9841e03f97eSMikulas Patocka goto shut; 9851e03f97eSMikulas Patocka } 9861e03f97eSMikulas Patocka 9878a2d5286SMike Snitzer /* Adjust old_chunk and new_chunk to reflect start of linear region */ 9888a2d5286SMike Snitzer old_chunk = old_chunk + 1 - linear_chunks; 9898a2d5286SMike Snitzer new_chunk = new_chunk + 1 - linear_chunks; 9908a2d5286SMike Snitzer 9918a2d5286SMike Snitzer /* 9928a2d5286SMike Snitzer * Use one (potentially large) I/O to copy all 'linear_chunks' 9938a2d5286SMike Snitzer * from the exception store to the origin 9948a2d5286SMike Snitzer */ 9958a2d5286SMike Snitzer io_size = linear_chunks * s->store->chunk_size; 9961e03f97eSMikulas Patocka 9971e03f97eSMikulas Patocka dest.bdev = s->origin->bdev; 9981e03f97eSMikulas Patocka dest.sector = chunk_to_sector(s->store, old_chunk); 9998a2d5286SMike Snitzer dest.count = min(io_size, get_dev_size(dest.bdev) - dest.sector); 10001e03f97eSMikulas Patocka 10011e03f97eSMikulas Patocka src.bdev = s->cow->bdev; 10021e03f97eSMikulas Patocka src.sector = chunk_to_sector(s->store, new_chunk); 10031e03f97eSMikulas Patocka src.count = dest.count; 10041e03f97eSMikulas Patocka 100573dfd078SMikulas Patocka /* 100673dfd078SMikulas Patocka * Reallocate any exceptions needed in other snapshots then 100773dfd078SMikulas Patocka * wait for the pending exceptions to complete. 100873dfd078SMikulas Patocka * Each time any pending exception (globally on the system) 100973dfd078SMikulas Patocka * completes we are woken and repeat the process to find out 101073dfd078SMikulas Patocka * if we can proceed. While this may not seem a particularly 101173dfd078SMikulas Patocka * efficient algorithm, it is not expected to have any 101273dfd078SMikulas Patocka * significant impact on performance. 101373dfd078SMikulas Patocka */ 101473dfd078SMikulas Patocka previous_count = read_pending_exceptions_done_count(); 10158a2d5286SMike Snitzer while (origin_write_extent(s, dest.sector, io_size)) { 101673dfd078SMikulas Patocka wait_event(_pending_exceptions_done, 101773dfd078SMikulas Patocka (read_pending_exceptions_done_count() != 101873dfd078SMikulas Patocka previous_count)); 101973dfd078SMikulas Patocka /* Retry after the wait, until all exceptions are done. */ 102073dfd078SMikulas Patocka previous_count = read_pending_exceptions_done_count(); 102173dfd078SMikulas Patocka } 102273dfd078SMikulas Patocka 10239fe86254SMikulas Patocka down_write(&s->lock); 10249fe86254SMikulas Patocka s->first_merging_chunk = old_chunk; 10258a2d5286SMike Snitzer s->num_merging_chunks = linear_chunks; 10269fe86254SMikulas Patocka up_write(&s->lock); 10279fe86254SMikulas Patocka 10288a2d5286SMike Snitzer /* Wait until writes to all 'linear_chunks' drain */ 10298a2d5286SMike Snitzer for (i = 0; i < linear_chunks; i++) 10308a2d5286SMike Snitzer __check_for_conflicting_io(s, old_chunk + i); 10319fe86254SMikulas Patocka 10321e03f97eSMikulas Patocka dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, merge_callback, s); 10331e03f97eSMikulas Patocka return; 10341e03f97eSMikulas Patocka 10351e03f97eSMikulas Patocka shut: 10361e03f97eSMikulas Patocka merge_shutdown(s); 10371e03f97eSMikulas Patocka } 10381e03f97eSMikulas Patocka 10399fe86254SMikulas Patocka static void error_bios(struct bio *bio); 10409fe86254SMikulas Patocka 10411e03f97eSMikulas Patocka static void merge_callback(int read_err, unsigned long write_err, void *context) 10421e03f97eSMikulas Patocka { 10431e03f97eSMikulas Patocka struct dm_snapshot *s = context; 10449fe86254SMikulas Patocka struct bio *b = NULL; 10451e03f97eSMikulas Patocka 10461e03f97eSMikulas Patocka if (read_err || write_err) { 10471e03f97eSMikulas Patocka if (read_err) 10481e03f97eSMikulas Patocka DMERR("Read error: shutting down merge."); 10491e03f97eSMikulas Patocka else 10501e03f97eSMikulas Patocka DMERR("Write error: shutting down merge."); 10511e03f97eSMikulas Patocka goto shut; 10521e03f97eSMikulas Patocka } 10531e03f97eSMikulas Patocka 10549fe86254SMikulas Patocka if (s->store->type->commit_merge(s->store, 10559fe86254SMikulas Patocka s->num_merging_chunks) < 0) { 10561e03f97eSMikulas Patocka DMERR("Write error in exception store: shutting down merge"); 10571e03f97eSMikulas Patocka goto shut; 10581e03f97eSMikulas Patocka } 10591e03f97eSMikulas Patocka 10609fe86254SMikulas Patocka if (remove_single_exception_chunk(s) < 0) 10619fe86254SMikulas Patocka goto shut; 10629fe86254SMikulas Patocka 10631e03f97eSMikulas Patocka snapshot_merge_next_chunks(s); 10641e03f97eSMikulas Patocka 10651e03f97eSMikulas Patocka return; 10661e03f97eSMikulas Patocka 10671e03f97eSMikulas Patocka shut: 10689fe86254SMikulas Patocka down_write(&s->lock); 1069d8ddb1cfSMike Snitzer s->merge_failed = 1; 10709fe86254SMikulas Patocka b = __release_queued_bios_after_merge(s); 10719fe86254SMikulas Patocka up_write(&s->lock); 10729fe86254SMikulas Patocka error_bios(b); 10739fe86254SMikulas Patocka 10741e03f97eSMikulas Patocka merge_shutdown(s); 10751e03f97eSMikulas Patocka } 10761e03f97eSMikulas Patocka 10771e03f97eSMikulas Patocka static void start_merge(struct dm_snapshot *s) 10781e03f97eSMikulas Patocka { 10791e03f97eSMikulas Patocka if (!test_and_set_bit(RUNNING_MERGE, &s->state_bits)) 10801e03f97eSMikulas Patocka snapshot_merge_next_chunks(s); 10811e03f97eSMikulas Patocka } 10821e03f97eSMikulas Patocka 10831e03f97eSMikulas Patocka /* 10841e03f97eSMikulas Patocka * Stop the merging process and wait until it finishes. 10851e03f97eSMikulas Patocka */ 10861e03f97eSMikulas Patocka static void stop_merge(struct dm_snapshot *s) 10871e03f97eSMikulas Patocka { 10881e03f97eSMikulas Patocka set_bit(SHUTDOWN_MERGE, &s->state_bits); 108974316201SNeilBrown wait_on_bit(&s->state_bits, RUNNING_MERGE, TASK_UNINTERRUPTIBLE); 10901e03f97eSMikulas Patocka clear_bit(SHUTDOWN_MERGE, &s->state_bits); 10911e03f97eSMikulas Patocka } 10921e03f97eSMikulas Patocka 10931da177e4SLinus Torvalds /* 10941da177e4SLinus Torvalds * Construct a snapshot mapping: <origin_dev> <COW-dev> <p/n> <chunk-size> 10951da177e4SLinus Torvalds */ 10961da177e4SLinus Torvalds static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv) 10971da177e4SLinus Torvalds { 10981da177e4SLinus Torvalds struct dm_snapshot *s; 1099cd45daffSMikulas Patocka int i; 11001da177e4SLinus Torvalds int r = -EINVAL; 1101fc56f6fbSMike Snitzer char *origin_path, *cow_path; 110255a62eefSAlasdair G Kergon unsigned args_used, num_flush_bios = 1; 110310b8106aSMike Snitzer fmode_t origin_mode = FMODE_READ; 11041da177e4SLinus Torvalds 11054c7e3bf4SMark McLoughlin if (argc != 4) { 110672d94861SAlasdair G Kergon ti->error = "requires exactly 4 arguments"; 11071da177e4SLinus Torvalds r = -EINVAL; 1108fc56f6fbSMike Snitzer goto bad; 11091da177e4SLinus Torvalds } 11101da177e4SLinus Torvalds 111110b8106aSMike Snitzer if (dm_target_is_snapshot_merge(ti)) { 111255a62eefSAlasdair G Kergon num_flush_bios = 2; 111310b8106aSMike Snitzer origin_mode = FMODE_WRITE; 111410b8106aSMike Snitzer } 111510b8106aSMike Snitzer 11161da177e4SLinus Torvalds s = kmalloc(sizeof(*s), GFP_KERNEL); 1117fee1998eSJonathan Brassow if (!s) { 1118a2d2b034SJonathan Brassow ti->error = "Cannot allocate private snapshot structure"; 11191da177e4SLinus Torvalds r = -ENOMEM; 1120fc56f6fbSMike Snitzer goto bad; 11211da177e4SLinus Torvalds } 11221da177e4SLinus Torvalds 1123c2411045SMikulas Patocka origin_path = argv[0]; 1124c2411045SMikulas Patocka argv++; 1125c2411045SMikulas Patocka argc--; 1126c2411045SMikulas Patocka 1127c2411045SMikulas Patocka r = dm_get_device(ti, origin_path, origin_mode, &s->origin); 1128c2411045SMikulas Patocka if (r) { 1129c2411045SMikulas Patocka ti->error = "Cannot get origin device"; 1130c2411045SMikulas Patocka goto bad_origin; 1131c2411045SMikulas Patocka } 1132c2411045SMikulas Patocka 1133fc56f6fbSMike Snitzer cow_path = argv[0]; 1134fc56f6fbSMike Snitzer argv++; 1135fc56f6fbSMike Snitzer argc--; 1136fc56f6fbSMike Snitzer 1137024d37e9SMilan Broz r = dm_get_device(ti, cow_path, dm_table_get_mode(ti->table), &s->cow); 1138fc56f6fbSMike Snitzer if (r) { 1139fc56f6fbSMike Snitzer ti->error = "Cannot get COW device"; 1140fc56f6fbSMike Snitzer goto bad_cow; 1141fc56f6fbSMike Snitzer } 1142fc56f6fbSMike Snitzer 1143fc56f6fbSMike Snitzer r = dm_exception_store_create(ti, argc, argv, s, &args_used, &s->store); 1144fc56f6fbSMike Snitzer if (r) { 1145fc56f6fbSMike Snitzer ti->error = "Couldn't create exception store"; 1146fc56f6fbSMike Snitzer r = -EINVAL; 1147fc56f6fbSMike Snitzer goto bad_store; 1148fc56f6fbSMike Snitzer } 1149fc56f6fbSMike Snitzer 1150fc56f6fbSMike Snitzer argv += args_used; 1151fc56f6fbSMike Snitzer argc -= args_used; 1152fc56f6fbSMike Snitzer 1153fc56f6fbSMike Snitzer s->ti = ti; 11541da177e4SLinus Torvalds s->valid = 1; 1155aa14edebSAlasdair G Kergon s->active = 0; 1156879129d2SMikulas Patocka atomic_set(&s->pending_exceptions_count, 0); 1157230c83afSMikulas Patocka s->exception_start_sequence = 0; 1158230c83afSMikulas Patocka s->exception_complete_sequence = 0; 1159230c83afSMikulas Patocka INIT_LIST_HEAD(&s->out_of_order_list); 11601da177e4SLinus Torvalds init_rwsem(&s->lock); 1161c1f0c183SMike Snitzer INIT_LIST_HEAD(&s->list); 1162ca3a931fSAlasdair G Kergon spin_lock_init(&s->pe_lock); 11631e03f97eSMikulas Patocka s->state_bits = 0; 1164d8ddb1cfSMike Snitzer s->merge_failed = 0; 11659fe86254SMikulas Patocka s->first_merging_chunk = 0; 11669fe86254SMikulas Patocka s->num_merging_chunks = 0; 11679fe86254SMikulas Patocka bio_list_init(&s->bios_queued_during_merge); 11681da177e4SLinus Torvalds 11691da177e4SLinus Torvalds /* Allocate hash table for COW data */ 1170fee1998eSJonathan Brassow if (init_hash_tables(s)) { 11711da177e4SLinus Torvalds ti->error = "Unable to allocate hash table space"; 11721da177e4SLinus Torvalds r = -ENOMEM; 1173fee1998eSJonathan Brassow goto bad_hash_tables; 11741da177e4SLinus Torvalds } 11751da177e4SLinus Torvalds 1176df5d2e90SMikulas Patocka s->kcopyd_client = dm_kcopyd_client_create(&dm_kcopyd_throttle); 1177fa34ce73SMikulas Patocka if (IS_ERR(s->kcopyd_client)) { 1178fa34ce73SMikulas Patocka r = PTR_ERR(s->kcopyd_client); 11791da177e4SLinus Torvalds ti->error = "Could not create kcopyd client"; 1180fee1998eSJonathan Brassow goto bad_kcopyd; 11811da177e4SLinus Torvalds } 11821da177e4SLinus Torvalds 118392e86812SMikulas Patocka s->pending_pool = mempool_create_slab_pool(MIN_IOS, pending_cache); 118492e86812SMikulas Patocka if (!s->pending_pool) { 118592e86812SMikulas Patocka ti->error = "Could not allocate mempool for pending exceptions"; 118609e8b813SWei Yongjun r = -ENOMEM; 1187fee1998eSJonathan Brassow goto bad_pending_pool; 118892e86812SMikulas Patocka } 118992e86812SMikulas Patocka 1190cd45daffSMikulas Patocka for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++) 1191cd45daffSMikulas Patocka INIT_HLIST_HEAD(&s->tracked_chunk_hash[i]); 1192cd45daffSMikulas Patocka 1193cd45daffSMikulas Patocka spin_lock_init(&s->tracked_chunk_lock); 1194cd45daffSMikulas Patocka 1195c1f0c183SMike Snitzer ti->private = s; 119655a62eefSAlasdair G Kergon ti->num_flush_bios = num_flush_bios; 119742bc954fSMikulas Patocka ti->per_bio_data_size = sizeof(struct dm_snap_tracked_chunk); 1198c1f0c183SMike Snitzer 1199c1f0c183SMike Snitzer /* Add snapshot to the list of snapshots for this origin */ 1200c1f0c183SMike Snitzer /* Exceptions aren't triggered till snapshot_resume() is called */ 1201c1f0c183SMike Snitzer r = register_snapshot(s); 1202c1f0c183SMike Snitzer if (r == -ENOMEM) { 1203c1f0c183SMike Snitzer ti->error = "Snapshot origin struct allocation failed"; 1204c1f0c183SMike Snitzer goto bad_load_and_register; 1205c1f0c183SMike Snitzer } else if (r < 0) { 1206c1f0c183SMike Snitzer /* invalid handover, register_snapshot has set ti->error */ 1207c1f0c183SMike Snitzer goto bad_load_and_register; 1208c1f0c183SMike Snitzer } 1209c1f0c183SMike Snitzer 1210c1f0c183SMike Snitzer /* 1211c1f0c183SMike Snitzer * Metadata must only be loaded into one table at once, so skip this 1212c1f0c183SMike Snitzer * if metadata will be handed over during resume. 1213c1f0c183SMike Snitzer * Chunk size will be set during the handover - set it to zero to 1214c1f0c183SMike Snitzer * ensure it's ignored. 1215c1f0c183SMike Snitzer */ 1216c1f0c183SMike Snitzer if (r > 0) { 1217c1f0c183SMike Snitzer s->store->chunk_size = 0; 1218c1f0c183SMike Snitzer return 0; 1219c1f0c183SMike Snitzer } 1220c1f0c183SMike Snitzer 1221493df71cSJonathan Brassow r = s->store->type->read_metadata(s->store, dm_add_exception, 1222493df71cSJonathan Brassow (void *)s); 12230764147bSMilan Broz if (r < 0) { 1224f9cea4f7SMark McLoughlin ti->error = "Failed to read snapshot metadata"; 1225c1f0c183SMike Snitzer goto bad_read_metadata; 12260764147bSMilan Broz } else if (r > 0) { 12270764147bSMilan Broz s->valid = 0; 12280764147bSMilan Broz DMWARN("Snapshot is marked invalid."); 1229f9cea4f7SMark McLoughlin } 1230aa14edebSAlasdair G Kergon 12313f2412dcSMikulas Patocka if (!s->store->chunk_size) { 12323f2412dcSMikulas Patocka ti->error = "Chunk size not set"; 1233c1f0c183SMike Snitzer goto bad_read_metadata; 12343f2412dcSMikulas Patocka } 1235542f9038SMike Snitzer 1236542f9038SMike Snitzer r = dm_set_target_max_io_len(ti, s->store->chunk_size); 1237542f9038SMike Snitzer if (r) 1238542f9038SMike Snitzer goto bad_read_metadata; 12391da177e4SLinus Torvalds 12401da177e4SLinus Torvalds return 0; 12411da177e4SLinus Torvalds 1242c1f0c183SMike Snitzer bad_read_metadata: 1243c1f0c183SMike Snitzer unregister_snapshot(s); 1244c1f0c183SMike Snitzer 1245cd45daffSMikulas Patocka bad_load_and_register: 124692e86812SMikulas Patocka mempool_destroy(s->pending_pool); 124792e86812SMikulas Patocka 1248fee1998eSJonathan Brassow bad_pending_pool: 1249eb69aca5SHeinz Mauelshagen dm_kcopyd_client_destroy(s->kcopyd_client); 12501da177e4SLinus Torvalds 1251fee1998eSJonathan Brassow bad_kcopyd: 12523510cb94SJon Brassow dm_exception_table_exit(&s->pending, pending_cache); 12533510cb94SJon Brassow dm_exception_table_exit(&s->complete, exception_cache); 12541da177e4SLinus Torvalds 1255fee1998eSJonathan Brassow bad_hash_tables: 1256fc56f6fbSMike Snitzer dm_exception_store_destroy(s->store); 1257fc56f6fbSMike Snitzer 1258fc56f6fbSMike Snitzer bad_store: 1259fc56f6fbSMike Snitzer dm_put_device(ti, s->cow); 1260fc56f6fbSMike Snitzer 1261fc56f6fbSMike Snitzer bad_cow: 1262c2411045SMikulas Patocka dm_put_device(ti, s->origin); 1263c2411045SMikulas Patocka 1264c2411045SMikulas Patocka bad_origin: 12651da177e4SLinus Torvalds kfree(s); 12661da177e4SLinus Torvalds 1267fc56f6fbSMike Snitzer bad: 12681da177e4SLinus Torvalds return r; 12691da177e4SLinus Torvalds } 12701da177e4SLinus Torvalds 127131c93a0cSMilan Broz static void __free_exceptions(struct dm_snapshot *s) 127231c93a0cSMilan Broz { 1273eb69aca5SHeinz Mauelshagen dm_kcopyd_client_destroy(s->kcopyd_client); 127431c93a0cSMilan Broz s->kcopyd_client = NULL; 127531c93a0cSMilan Broz 12763510cb94SJon Brassow dm_exception_table_exit(&s->pending, pending_cache); 12773510cb94SJon Brassow dm_exception_table_exit(&s->complete, exception_cache); 127831c93a0cSMilan Broz } 127931c93a0cSMilan Broz 1280c1f0c183SMike Snitzer static void __handover_exceptions(struct dm_snapshot *snap_src, 1281c1f0c183SMike Snitzer struct dm_snapshot *snap_dest) 1282c1f0c183SMike Snitzer { 1283c1f0c183SMike Snitzer union { 1284c1f0c183SMike Snitzer struct dm_exception_table table_swap; 1285c1f0c183SMike Snitzer struct dm_exception_store *store_swap; 1286c1f0c183SMike Snitzer } u; 1287c1f0c183SMike Snitzer 1288c1f0c183SMike Snitzer /* 1289c1f0c183SMike Snitzer * Swap all snapshot context information between the two instances. 1290c1f0c183SMike Snitzer */ 1291c1f0c183SMike Snitzer u.table_swap = snap_dest->complete; 1292c1f0c183SMike Snitzer snap_dest->complete = snap_src->complete; 1293c1f0c183SMike Snitzer snap_src->complete = u.table_swap; 1294c1f0c183SMike Snitzer 1295c1f0c183SMike Snitzer u.store_swap = snap_dest->store; 1296c1f0c183SMike Snitzer snap_dest->store = snap_src->store; 1297c1f0c183SMike Snitzer snap_src->store = u.store_swap; 1298c1f0c183SMike Snitzer 1299c1f0c183SMike Snitzer snap_dest->store->snap = snap_dest; 1300c1f0c183SMike Snitzer snap_src->store->snap = snap_src; 1301c1f0c183SMike Snitzer 1302542f9038SMike Snitzer snap_dest->ti->max_io_len = snap_dest->store->chunk_size; 1303c1f0c183SMike Snitzer snap_dest->valid = snap_src->valid; 1304c1f0c183SMike Snitzer 1305c1f0c183SMike Snitzer /* 1306c1f0c183SMike Snitzer * Set source invalid to ensure it receives no further I/O. 1307c1f0c183SMike Snitzer */ 1308c1f0c183SMike Snitzer snap_src->valid = 0; 1309c1f0c183SMike Snitzer } 1310c1f0c183SMike Snitzer 13111da177e4SLinus Torvalds static void snapshot_dtr(struct dm_target *ti) 13121da177e4SLinus Torvalds { 1313cd45daffSMikulas Patocka #ifdef CONFIG_DM_DEBUG 1314cd45daffSMikulas Patocka int i; 1315cd45daffSMikulas Patocka #endif 1316028867acSAlasdair G Kergon struct dm_snapshot *s = ti->private; 1317c1f0c183SMike Snitzer struct dm_snapshot *snap_src = NULL, *snap_dest = NULL; 13181da177e4SLinus Torvalds 1319c1f0c183SMike Snitzer down_read(&_origins_lock); 1320c1f0c183SMike Snitzer /* Check whether exception handover must be cancelled */ 13219d3b15c4SMikulas Patocka (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL); 1322c1f0c183SMike Snitzer if (snap_src && snap_dest && (s == snap_src)) { 1323c1f0c183SMike Snitzer down_write(&snap_dest->lock); 1324c1f0c183SMike Snitzer snap_dest->valid = 0; 1325c1f0c183SMike Snitzer up_write(&snap_dest->lock); 1326c1f0c183SMike Snitzer DMERR("Cancelling snapshot handover."); 1327c1f0c183SMike Snitzer } 1328c1f0c183SMike Snitzer up_read(&_origins_lock); 1329c1f0c183SMike Snitzer 13301e03f97eSMikulas Patocka if (dm_target_is_snapshot_merge(ti)) 13311e03f97eSMikulas Patocka stop_merge(s); 13321e03f97eSMikulas Patocka 1333138728dcSAlasdair G Kergon /* Prevent further origin writes from using this snapshot. */ 1334138728dcSAlasdair G Kergon /* After this returns there can be no new kcopyd jobs. */ 13351da177e4SLinus Torvalds unregister_snapshot(s); 13361da177e4SLinus Torvalds 1337879129d2SMikulas Patocka while (atomic_read(&s->pending_exceptions_count)) 133890fa1527SMikulas Patocka msleep(1); 1339879129d2SMikulas Patocka /* 1340879129d2SMikulas Patocka * Ensure instructions in mempool_destroy aren't reordered 1341879129d2SMikulas Patocka * before atomic_read. 1342879129d2SMikulas Patocka */ 1343879129d2SMikulas Patocka smp_mb(); 1344879129d2SMikulas Patocka 1345cd45daffSMikulas Patocka #ifdef CONFIG_DM_DEBUG 1346cd45daffSMikulas Patocka for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++) 1347cd45daffSMikulas Patocka BUG_ON(!hlist_empty(&s->tracked_chunk_hash[i])); 1348cd45daffSMikulas Patocka #endif 1349cd45daffSMikulas Patocka 135031c93a0cSMilan Broz __free_exceptions(s); 13511da177e4SLinus Torvalds 135292e86812SMikulas Patocka mempool_destroy(s->pending_pool); 135392e86812SMikulas Patocka 1354fee1998eSJonathan Brassow dm_exception_store_destroy(s->store); 1355138728dcSAlasdair G Kergon 1356fc56f6fbSMike Snitzer dm_put_device(ti, s->cow); 1357fc56f6fbSMike Snitzer 1358c2411045SMikulas Patocka dm_put_device(ti, s->origin); 1359c2411045SMikulas Patocka 13601da177e4SLinus Torvalds kfree(s); 13611da177e4SLinus Torvalds } 13621da177e4SLinus Torvalds 13631da177e4SLinus Torvalds /* 13641da177e4SLinus Torvalds * Flush a list of buffers. 13651da177e4SLinus Torvalds */ 13661da177e4SLinus Torvalds static void flush_bios(struct bio *bio) 13671da177e4SLinus Torvalds { 13681da177e4SLinus Torvalds struct bio *n; 13691da177e4SLinus Torvalds 13701da177e4SLinus Torvalds while (bio) { 13711da177e4SLinus Torvalds n = bio->bi_next; 13721da177e4SLinus Torvalds bio->bi_next = NULL; 13731da177e4SLinus Torvalds generic_make_request(bio); 13741da177e4SLinus Torvalds bio = n; 13751da177e4SLinus Torvalds } 13761da177e4SLinus Torvalds } 13771da177e4SLinus Torvalds 1378515ad66cSMikulas Patocka static int do_origin(struct dm_dev *origin, struct bio *bio); 1379515ad66cSMikulas Patocka 1380515ad66cSMikulas Patocka /* 1381515ad66cSMikulas Patocka * Flush a list of buffers. 1382515ad66cSMikulas Patocka */ 1383515ad66cSMikulas Patocka static void retry_origin_bios(struct dm_snapshot *s, struct bio *bio) 1384515ad66cSMikulas Patocka { 1385515ad66cSMikulas Patocka struct bio *n; 1386515ad66cSMikulas Patocka int r; 1387515ad66cSMikulas Patocka 1388515ad66cSMikulas Patocka while (bio) { 1389515ad66cSMikulas Patocka n = bio->bi_next; 1390515ad66cSMikulas Patocka bio->bi_next = NULL; 1391515ad66cSMikulas Patocka r = do_origin(s->origin, bio); 1392515ad66cSMikulas Patocka if (r == DM_MAPIO_REMAPPED) 1393515ad66cSMikulas Patocka generic_make_request(bio); 1394515ad66cSMikulas Patocka bio = n; 1395515ad66cSMikulas Patocka } 1396515ad66cSMikulas Patocka } 1397515ad66cSMikulas Patocka 13981da177e4SLinus Torvalds /* 13991da177e4SLinus Torvalds * Error a list of buffers. 14001da177e4SLinus Torvalds */ 14011da177e4SLinus Torvalds static void error_bios(struct bio *bio) 14021da177e4SLinus Torvalds { 14031da177e4SLinus Torvalds struct bio *n; 14041da177e4SLinus Torvalds 14051da177e4SLinus Torvalds while (bio) { 14061da177e4SLinus Torvalds n = bio->bi_next; 14071da177e4SLinus Torvalds bio->bi_next = NULL; 14086712ecf8SNeilBrown bio_io_error(bio); 14091da177e4SLinus Torvalds bio = n; 14101da177e4SLinus Torvalds } 14111da177e4SLinus Torvalds } 14121da177e4SLinus Torvalds 1413695368acSAlasdair G Kergon static void __invalidate_snapshot(struct dm_snapshot *s, int err) 141476df1c65SAlasdair G Kergon { 141576df1c65SAlasdair G Kergon if (!s->valid) 141676df1c65SAlasdair G Kergon return; 141776df1c65SAlasdair G Kergon 141876df1c65SAlasdair G Kergon if (err == -EIO) 141976df1c65SAlasdair G Kergon DMERR("Invalidating snapshot: Error reading/writing."); 142076df1c65SAlasdair G Kergon else if (err == -ENOMEM) 142176df1c65SAlasdair G Kergon DMERR("Invalidating snapshot: Unable to allocate exception."); 142276df1c65SAlasdair G Kergon 1423493df71cSJonathan Brassow if (s->store->type->drop_snapshot) 1424493df71cSJonathan Brassow s->store->type->drop_snapshot(s->store); 142576df1c65SAlasdair G Kergon 142676df1c65SAlasdair G Kergon s->valid = 0; 142776df1c65SAlasdair G Kergon 1428fc56f6fbSMike Snitzer dm_table_event(s->ti->table); 142976df1c65SAlasdair G Kergon } 143076df1c65SAlasdair G Kergon 1431028867acSAlasdair G Kergon static void pending_complete(struct dm_snap_pending_exception *pe, int success) 14321da177e4SLinus Torvalds { 14331d4989c8SJon Brassow struct dm_exception *e; 14341da177e4SLinus Torvalds struct dm_snapshot *s = pe->snap; 14359d493fa8SAlasdair G Kergon struct bio *origin_bios = NULL; 14369d493fa8SAlasdair G Kergon struct bio *snapshot_bios = NULL; 1437a6e50b40SMikulas Patocka struct bio *full_bio = NULL; 14389d493fa8SAlasdair G Kergon int error = 0; 14391da177e4SLinus Torvalds 144076df1c65SAlasdair G Kergon if (!success) { 144176df1c65SAlasdair G Kergon /* Read/write error - snapshot is unusable */ 14421da177e4SLinus Torvalds down_write(&s->lock); 1443695368acSAlasdair G Kergon __invalidate_snapshot(s, -EIO); 14449d493fa8SAlasdair G Kergon error = 1; 144576df1c65SAlasdair G Kergon goto out; 144676df1c65SAlasdair G Kergon } 144776df1c65SAlasdair G Kergon 1448119bc547SMikulas Patocka e = alloc_completed_exception(GFP_NOIO); 144976df1c65SAlasdair G Kergon if (!e) { 145076df1c65SAlasdair G Kergon down_write(&s->lock); 1451695368acSAlasdair G Kergon __invalidate_snapshot(s, -ENOMEM); 14529d493fa8SAlasdair G Kergon error = 1; 14531da177e4SLinus Torvalds goto out; 14541da177e4SLinus Torvalds } 14551da177e4SLinus Torvalds *e = pe->e; 14561da177e4SLinus Torvalds 14579d493fa8SAlasdair G Kergon down_write(&s->lock); 14589d493fa8SAlasdair G Kergon if (!s->valid) { 14593510cb94SJon Brassow free_completed_exception(e); 14609d493fa8SAlasdair G Kergon error = 1; 14619d493fa8SAlasdair G Kergon goto out; 14629d493fa8SAlasdair G Kergon } 14639d493fa8SAlasdair G Kergon 1464615d1eb9SMike Snitzer /* Check for conflicting reads */ 1465615d1eb9SMike Snitzer __check_for_conflicting_io(s, pe->e.old_chunk); 1466a8d41b59SMikulas Patocka 1467a8d41b59SMikulas Patocka /* 14681da177e4SLinus Torvalds * Add a proper exception, and remove the 14691da177e4SLinus Torvalds * in-flight exception from the list. 14701da177e4SLinus Torvalds */ 14713510cb94SJon Brassow dm_insert_exception(&s->complete, e); 14721da177e4SLinus Torvalds 14731da177e4SLinus Torvalds out: 14743510cb94SJon Brassow dm_remove_exception(&pe->e); 14759d493fa8SAlasdair G Kergon snapshot_bios = bio_list_get(&pe->snapshot_bios); 1476515ad66cSMikulas Patocka origin_bios = bio_list_get(&pe->origin_bios); 1477a6e50b40SMikulas Patocka full_bio = pe->full_bio; 1478a6e50b40SMikulas Patocka if (full_bio) { 1479a6e50b40SMikulas Patocka full_bio->bi_end_io = pe->full_bio_end_io; 1480a6e50b40SMikulas Patocka full_bio->bi_private = pe->full_bio_private; 1481196d38bcSKent Overstreet atomic_inc(&full_bio->bi_remaining); 1482a6e50b40SMikulas Patocka } 148373dfd078SMikulas Patocka increment_pending_exceptions_done_count(); 148473dfd078SMikulas Patocka 14859d493fa8SAlasdair G Kergon up_write(&s->lock); 14869d493fa8SAlasdair G Kergon 14879d493fa8SAlasdair G Kergon /* Submit any pending write bios */ 1488a6e50b40SMikulas Patocka if (error) { 1489a6e50b40SMikulas Patocka if (full_bio) 1490a6e50b40SMikulas Patocka bio_io_error(full_bio); 14919d493fa8SAlasdair G Kergon error_bios(snapshot_bios); 1492a6e50b40SMikulas Patocka } else { 1493a6e50b40SMikulas Patocka if (full_bio) 1494a6e50b40SMikulas Patocka bio_endio(full_bio, 0); 14959d493fa8SAlasdair G Kergon flush_bios(snapshot_bios); 1496a6e50b40SMikulas Patocka } 14979d493fa8SAlasdair G Kergon 1498515ad66cSMikulas Patocka retry_origin_bios(s, origin_bios); 149922aa66a3SMikulas Patocka 150022aa66a3SMikulas Patocka free_pending_exception(pe); 15011da177e4SLinus Torvalds } 15021da177e4SLinus Torvalds 15031da177e4SLinus Torvalds static void commit_callback(void *context, int success) 15041da177e4SLinus Torvalds { 1505028867acSAlasdair G Kergon struct dm_snap_pending_exception *pe = context; 1506028867acSAlasdair G Kergon 15071da177e4SLinus Torvalds pending_complete(pe, success); 15081da177e4SLinus Torvalds } 15091da177e4SLinus Torvalds 1510230c83afSMikulas Patocka static void complete_exception(struct dm_snap_pending_exception *pe) 1511230c83afSMikulas Patocka { 1512230c83afSMikulas Patocka struct dm_snapshot *s = pe->snap; 1513230c83afSMikulas Patocka 1514230c83afSMikulas Patocka if (unlikely(pe->copy_error)) 1515230c83afSMikulas Patocka pending_complete(pe, 0); 1516230c83afSMikulas Patocka 1517230c83afSMikulas Patocka else 1518230c83afSMikulas Patocka /* Update the metadata if we are persistent */ 1519230c83afSMikulas Patocka s->store->type->commit_exception(s->store, &pe->e, 1520230c83afSMikulas Patocka commit_callback, pe); 1521230c83afSMikulas Patocka } 1522230c83afSMikulas Patocka 15231da177e4SLinus Torvalds /* 15241da177e4SLinus Torvalds * Called when the copy I/O has finished. kcopyd actually runs 15251da177e4SLinus Torvalds * this code so don't block. 15261da177e4SLinus Torvalds */ 15274cdc1d1fSAlasdair G Kergon static void copy_callback(int read_err, unsigned long write_err, void *context) 15281da177e4SLinus Torvalds { 1529028867acSAlasdair G Kergon struct dm_snap_pending_exception *pe = context; 15301da177e4SLinus Torvalds struct dm_snapshot *s = pe->snap; 15311da177e4SLinus Torvalds 1532230c83afSMikulas Patocka pe->copy_error = read_err || write_err; 15331da177e4SLinus Torvalds 1534230c83afSMikulas Patocka if (pe->exception_sequence == s->exception_complete_sequence) { 1535230c83afSMikulas Patocka s->exception_complete_sequence++; 1536230c83afSMikulas Patocka complete_exception(pe); 1537230c83afSMikulas Patocka 1538230c83afSMikulas Patocka while (!list_empty(&s->out_of_order_list)) { 1539230c83afSMikulas Patocka pe = list_entry(s->out_of_order_list.next, 1540230c83afSMikulas Patocka struct dm_snap_pending_exception, out_of_order_entry); 1541230c83afSMikulas Patocka if (pe->exception_sequence != s->exception_complete_sequence) 1542230c83afSMikulas Patocka break; 1543230c83afSMikulas Patocka s->exception_complete_sequence++; 1544230c83afSMikulas Patocka list_del(&pe->out_of_order_entry); 1545230c83afSMikulas Patocka complete_exception(pe); 1546230c83afSMikulas Patocka } 1547230c83afSMikulas Patocka } else { 1548230c83afSMikulas Patocka struct list_head *lh; 1549230c83afSMikulas Patocka struct dm_snap_pending_exception *pe2; 1550230c83afSMikulas Patocka 1551230c83afSMikulas Patocka list_for_each_prev(lh, &s->out_of_order_list) { 1552230c83afSMikulas Patocka pe2 = list_entry(lh, struct dm_snap_pending_exception, out_of_order_entry); 1553230c83afSMikulas Patocka if (pe2->exception_sequence < pe->exception_sequence) 1554230c83afSMikulas Patocka break; 1555230c83afSMikulas Patocka } 1556230c83afSMikulas Patocka list_add(&pe->out_of_order_entry, lh); 1557230c83afSMikulas Patocka } 15581da177e4SLinus Torvalds } 15591da177e4SLinus Torvalds 15601da177e4SLinus Torvalds /* 15611da177e4SLinus Torvalds * Dispatches the copy operation to kcopyd. 15621da177e4SLinus Torvalds */ 1563028867acSAlasdair G Kergon static void start_copy(struct dm_snap_pending_exception *pe) 15641da177e4SLinus Torvalds { 15651da177e4SLinus Torvalds struct dm_snapshot *s = pe->snap; 156622a1ceb1SHeinz Mauelshagen struct dm_io_region src, dest; 15671da177e4SLinus Torvalds struct block_device *bdev = s->origin->bdev; 15681da177e4SLinus Torvalds sector_t dev_size; 15691da177e4SLinus Torvalds 15701da177e4SLinus Torvalds dev_size = get_dev_size(bdev); 15711da177e4SLinus Torvalds 15721da177e4SLinus Torvalds src.bdev = bdev; 157371fab00aSJonathan Brassow src.sector = chunk_to_sector(s->store, pe->e.old_chunk); 1574df96eee6SMikulas Patocka src.count = min((sector_t)s->store->chunk_size, dev_size - src.sector); 15751da177e4SLinus Torvalds 1576fc56f6fbSMike Snitzer dest.bdev = s->cow->bdev; 157771fab00aSJonathan Brassow dest.sector = chunk_to_sector(s->store, pe->e.new_chunk); 15781da177e4SLinus Torvalds dest.count = src.count; 15791da177e4SLinus Torvalds 15801da177e4SLinus Torvalds /* Hand over to kcopyd */ 1581a2d2b034SJonathan Brassow dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, copy_callback, pe); 15821da177e4SLinus Torvalds } 15831da177e4SLinus Torvalds 1584a6e50b40SMikulas Patocka static void full_bio_end_io(struct bio *bio, int error) 1585a6e50b40SMikulas Patocka { 1586a6e50b40SMikulas Patocka void *callback_data = bio->bi_private; 1587a6e50b40SMikulas Patocka 1588a6e50b40SMikulas Patocka dm_kcopyd_do_callback(callback_data, 0, error ? 1 : 0); 1589a6e50b40SMikulas Patocka } 1590a6e50b40SMikulas Patocka 1591a6e50b40SMikulas Patocka static void start_full_bio(struct dm_snap_pending_exception *pe, 1592a6e50b40SMikulas Patocka struct bio *bio) 1593a6e50b40SMikulas Patocka { 1594a6e50b40SMikulas Patocka struct dm_snapshot *s = pe->snap; 1595a6e50b40SMikulas Patocka void *callback_data; 1596a6e50b40SMikulas Patocka 1597a6e50b40SMikulas Patocka pe->full_bio = bio; 1598a6e50b40SMikulas Patocka pe->full_bio_end_io = bio->bi_end_io; 1599a6e50b40SMikulas Patocka pe->full_bio_private = bio->bi_private; 1600a6e50b40SMikulas Patocka 1601a6e50b40SMikulas Patocka callback_data = dm_kcopyd_prepare_callback(s->kcopyd_client, 1602a6e50b40SMikulas Patocka copy_callback, pe); 1603a6e50b40SMikulas Patocka 1604a6e50b40SMikulas Patocka bio->bi_end_io = full_bio_end_io; 1605a6e50b40SMikulas Patocka bio->bi_private = callback_data; 1606a6e50b40SMikulas Patocka 1607a6e50b40SMikulas Patocka generic_make_request(bio); 1608a6e50b40SMikulas Patocka } 1609a6e50b40SMikulas Patocka 16102913808eSMikulas Patocka static struct dm_snap_pending_exception * 16112913808eSMikulas Patocka __lookup_pending_exception(struct dm_snapshot *s, chunk_t chunk) 16122913808eSMikulas Patocka { 16133510cb94SJon Brassow struct dm_exception *e = dm_lookup_exception(&s->pending, chunk); 16142913808eSMikulas Patocka 16152913808eSMikulas Patocka if (!e) 16162913808eSMikulas Patocka return NULL; 16172913808eSMikulas Patocka 16182913808eSMikulas Patocka return container_of(e, struct dm_snap_pending_exception, e); 16192913808eSMikulas Patocka } 16202913808eSMikulas Patocka 16211da177e4SLinus Torvalds /* 16221da177e4SLinus Torvalds * Looks to see if this snapshot already has a pending exception 16231da177e4SLinus Torvalds * for this chunk, otherwise it allocates a new one and inserts 16241da177e4SLinus Torvalds * it into the pending table. 16251da177e4SLinus Torvalds * 16261da177e4SLinus Torvalds * NOTE: a write lock must be held on snap->lock before calling 16271da177e4SLinus Torvalds * this. 16281da177e4SLinus Torvalds */ 1629028867acSAlasdair G Kergon static struct dm_snap_pending_exception * 1630c6621392SMikulas Patocka __find_pending_exception(struct dm_snapshot *s, 1631c6621392SMikulas Patocka struct dm_snap_pending_exception *pe, chunk_t chunk) 16321da177e4SLinus Torvalds { 1633c6621392SMikulas Patocka struct dm_snap_pending_exception *pe2; 163476df1c65SAlasdair G Kergon 16352913808eSMikulas Patocka pe2 = __lookup_pending_exception(s, chunk); 16362913808eSMikulas Patocka if (pe2) { 16371da177e4SLinus Torvalds free_pending_exception(pe); 16382913808eSMikulas Patocka return pe2; 163976df1c65SAlasdair G Kergon } 164076df1c65SAlasdair G Kergon 16411da177e4SLinus Torvalds pe->e.old_chunk = chunk; 16421da177e4SLinus Torvalds bio_list_init(&pe->origin_bios); 16431da177e4SLinus Torvalds bio_list_init(&pe->snapshot_bios); 16441da177e4SLinus Torvalds pe->started = 0; 1645a6e50b40SMikulas Patocka pe->full_bio = NULL; 16461da177e4SLinus Torvalds 1647493df71cSJonathan Brassow if (s->store->type->prepare_exception(s->store, &pe->e)) { 16481da177e4SLinus Torvalds free_pending_exception(pe); 16491da177e4SLinus Torvalds return NULL; 16501da177e4SLinus Torvalds } 16511da177e4SLinus Torvalds 1652230c83afSMikulas Patocka pe->exception_sequence = s->exception_start_sequence++; 1653230c83afSMikulas Patocka 16543510cb94SJon Brassow dm_insert_exception(&s->pending, &pe->e); 16551da177e4SLinus Torvalds 16561da177e4SLinus Torvalds return pe; 16571da177e4SLinus Torvalds } 16581da177e4SLinus Torvalds 16591d4989c8SJon Brassow static void remap_exception(struct dm_snapshot *s, struct dm_exception *e, 1660d74f81f8SMilan Broz struct bio *bio, chunk_t chunk) 16611da177e4SLinus Torvalds { 1662fc56f6fbSMike Snitzer bio->bi_bdev = s->cow->bdev; 16634f024f37SKent Overstreet bio->bi_iter.bi_sector = 16644f024f37SKent Overstreet chunk_to_sector(s->store, dm_chunk_number(e->new_chunk) + 1665d74f81f8SMilan Broz (chunk - e->old_chunk)) + 16664f024f37SKent Overstreet (bio->bi_iter.bi_sector & s->store->chunk_mask); 16671da177e4SLinus Torvalds } 16681da177e4SLinus Torvalds 16697de3ee57SMikulas Patocka static int snapshot_map(struct dm_target *ti, struct bio *bio) 16701da177e4SLinus Torvalds { 16711d4989c8SJon Brassow struct dm_exception *e; 1672028867acSAlasdair G Kergon struct dm_snapshot *s = ti->private; 1673d2a7ad29SKiyoshi Ueda int r = DM_MAPIO_REMAPPED; 16741da177e4SLinus Torvalds chunk_t chunk; 1675028867acSAlasdair G Kergon struct dm_snap_pending_exception *pe = NULL; 16761da177e4SLinus Torvalds 1677ee18026aSMikulas Patocka init_tracked_chunk(bio); 1678ee18026aSMikulas Patocka 1679d87f4c14STejun Heo if (bio->bi_rw & REQ_FLUSH) { 1680fc56f6fbSMike Snitzer bio->bi_bdev = s->cow->bdev; 1681494b3ee7SMikulas Patocka return DM_MAPIO_REMAPPED; 1682494b3ee7SMikulas Patocka } 1683494b3ee7SMikulas Patocka 16844f024f37SKent Overstreet chunk = sector_to_chunk(s->store, bio->bi_iter.bi_sector); 16851da177e4SLinus Torvalds 16861da177e4SLinus Torvalds /* Full snapshots are not usable */ 168776df1c65SAlasdair G Kergon /* To get here the table must be live so s->active is always set. */ 16881da177e4SLinus Torvalds if (!s->valid) 1689f6a80ea8SAlasdair G Kergon return -EIO; 16901da177e4SLinus Torvalds 16911da177e4SLinus Torvalds /* FIXME: should only take write lock if we need 16921da177e4SLinus Torvalds * to copy an exception */ 16931da177e4SLinus Torvalds down_write(&s->lock); 16941da177e4SLinus Torvalds 169576df1c65SAlasdair G Kergon if (!s->valid) { 169676df1c65SAlasdair G Kergon r = -EIO; 169776df1c65SAlasdair G Kergon goto out_unlock; 169876df1c65SAlasdair G Kergon } 169976df1c65SAlasdair G Kergon 17001da177e4SLinus Torvalds /* If the block is already remapped - use that, else remap it */ 17013510cb94SJon Brassow e = dm_lookup_exception(&s->complete, chunk); 17021da177e4SLinus Torvalds if (e) { 1703d74f81f8SMilan Broz remap_exception(s, e, bio, chunk); 170476df1c65SAlasdair G Kergon goto out_unlock; 170576df1c65SAlasdair G Kergon } 17061da177e4SLinus Torvalds 1707ba40a2aaSAlasdair G Kergon /* 1708ba40a2aaSAlasdair G Kergon * Write to snapshot - higher level takes care of RW/RO 1709ba40a2aaSAlasdair G Kergon * flags so we should only get this if we are 1710ba40a2aaSAlasdair G Kergon * writeable. 1711ba40a2aaSAlasdair G Kergon */ 1712ba40a2aaSAlasdair G Kergon if (bio_rw(bio) == WRITE) { 17132913808eSMikulas Patocka pe = __lookup_pending_exception(s, chunk); 17142913808eSMikulas Patocka if (!pe) { 1715c6621392SMikulas Patocka up_write(&s->lock); 1716c6621392SMikulas Patocka pe = alloc_pending_exception(s); 1717c6621392SMikulas Patocka down_write(&s->lock); 1718c6621392SMikulas Patocka 1719c6621392SMikulas Patocka if (!s->valid) { 1720c6621392SMikulas Patocka free_pending_exception(pe); 1721c6621392SMikulas Patocka r = -EIO; 1722c6621392SMikulas Patocka goto out_unlock; 1723c6621392SMikulas Patocka } 1724c6621392SMikulas Patocka 17253510cb94SJon Brassow e = dm_lookup_exception(&s->complete, chunk); 172635bf659bSMikulas Patocka if (e) { 172735bf659bSMikulas Patocka free_pending_exception(pe); 172835bf659bSMikulas Patocka remap_exception(s, e, bio, chunk); 172935bf659bSMikulas Patocka goto out_unlock; 173035bf659bSMikulas Patocka } 173135bf659bSMikulas Patocka 1732c6621392SMikulas Patocka pe = __find_pending_exception(s, pe, chunk); 17331da177e4SLinus Torvalds if (!pe) { 1734695368acSAlasdair G Kergon __invalidate_snapshot(s, -ENOMEM); 17351da177e4SLinus Torvalds r = -EIO; 173676df1c65SAlasdair G Kergon goto out_unlock; 173776df1c65SAlasdair G Kergon } 17382913808eSMikulas Patocka } 173976df1c65SAlasdair G Kergon 1740d74f81f8SMilan Broz remap_exception(s, &pe->e, bio, chunk); 17411da177e4SLinus Torvalds 1742d2a7ad29SKiyoshi Ueda r = DM_MAPIO_SUBMITTED; 1743ba40a2aaSAlasdair G Kergon 1744a6e50b40SMikulas Patocka if (!pe->started && 17454f024f37SKent Overstreet bio->bi_iter.bi_size == 17464f024f37SKent Overstreet (s->store->chunk_size << SECTOR_SHIFT)) { 1747a6e50b40SMikulas Patocka pe->started = 1; 1748a6e50b40SMikulas Patocka up_write(&s->lock); 1749a6e50b40SMikulas Patocka start_full_bio(pe, bio); 1750a6e50b40SMikulas Patocka goto out; 1751a6e50b40SMikulas Patocka } 1752a6e50b40SMikulas Patocka 1753a6e50b40SMikulas Patocka bio_list_add(&pe->snapshot_bios, bio); 1754a6e50b40SMikulas Patocka 17551da177e4SLinus Torvalds if (!pe->started) { 17561da177e4SLinus Torvalds /* this is protected by snap->lock */ 17571da177e4SLinus Torvalds pe->started = 1; 175876df1c65SAlasdair G Kergon up_write(&s->lock); 175976df1c65SAlasdair G Kergon start_copy(pe); 1760ba40a2aaSAlasdair G Kergon goto out; 1761ba40a2aaSAlasdair G Kergon } 1762cd45daffSMikulas Patocka } else { 17631da177e4SLinus Torvalds bio->bi_bdev = s->origin->bdev; 1764ee18026aSMikulas Patocka track_chunk(s, bio, chunk); 1765cd45daffSMikulas Patocka } 17661da177e4SLinus Torvalds 1767ba40a2aaSAlasdair G Kergon out_unlock: 1768ba40a2aaSAlasdair G Kergon up_write(&s->lock); 1769ba40a2aaSAlasdair G Kergon out: 17701da177e4SLinus Torvalds return r; 17711da177e4SLinus Torvalds } 17721da177e4SLinus Torvalds 17733452c2a1SMikulas Patocka /* 17743452c2a1SMikulas Patocka * A snapshot-merge target behaves like a combination of a snapshot 17753452c2a1SMikulas Patocka * target and a snapshot-origin target. It only generates new 17763452c2a1SMikulas Patocka * exceptions in other snapshots and not in the one that is being 17773452c2a1SMikulas Patocka * merged. 17783452c2a1SMikulas Patocka * 17793452c2a1SMikulas Patocka * For each chunk, if there is an existing exception, it is used to 17803452c2a1SMikulas Patocka * redirect I/O to the cow device. Otherwise I/O is sent to the origin, 17813452c2a1SMikulas Patocka * which in turn might generate exceptions in other snapshots. 17829fe86254SMikulas Patocka * If merging is currently taking place on the chunk in question, the 17839fe86254SMikulas Patocka * I/O is deferred by adding it to s->bios_queued_during_merge. 17843452c2a1SMikulas Patocka */ 17857de3ee57SMikulas Patocka static int snapshot_merge_map(struct dm_target *ti, struct bio *bio) 17863452c2a1SMikulas Patocka { 17873452c2a1SMikulas Patocka struct dm_exception *e; 17883452c2a1SMikulas Patocka struct dm_snapshot *s = ti->private; 17893452c2a1SMikulas Patocka int r = DM_MAPIO_REMAPPED; 17903452c2a1SMikulas Patocka chunk_t chunk; 17913452c2a1SMikulas Patocka 1792ee18026aSMikulas Patocka init_tracked_chunk(bio); 1793ee18026aSMikulas Patocka 1794d87f4c14STejun Heo if (bio->bi_rw & REQ_FLUSH) { 179555a62eefSAlasdair G Kergon if (!dm_bio_get_target_bio_nr(bio)) 179610b8106aSMike Snitzer bio->bi_bdev = s->origin->bdev; 179710b8106aSMike Snitzer else 179810b8106aSMike Snitzer bio->bi_bdev = s->cow->bdev; 179910b8106aSMike Snitzer return DM_MAPIO_REMAPPED; 180010b8106aSMike Snitzer } 180110b8106aSMike Snitzer 18024f024f37SKent Overstreet chunk = sector_to_chunk(s->store, bio->bi_iter.bi_sector); 18033452c2a1SMikulas Patocka 18049fe86254SMikulas Patocka down_write(&s->lock); 18053452c2a1SMikulas Patocka 1806d2fdb776SMikulas Patocka /* Full merging snapshots are redirected to the origin */ 1807d2fdb776SMikulas Patocka if (!s->valid) 1808d2fdb776SMikulas Patocka goto redirect_to_origin; 18093452c2a1SMikulas Patocka 18103452c2a1SMikulas Patocka /* If the block is already remapped - use that */ 18113452c2a1SMikulas Patocka e = dm_lookup_exception(&s->complete, chunk); 18123452c2a1SMikulas Patocka if (e) { 18139fe86254SMikulas Patocka /* Queue writes overlapping with chunks being merged */ 18149fe86254SMikulas Patocka if (bio_rw(bio) == WRITE && 18159fe86254SMikulas Patocka chunk >= s->first_merging_chunk && 18169fe86254SMikulas Patocka chunk < (s->first_merging_chunk + 18179fe86254SMikulas Patocka s->num_merging_chunks)) { 18189fe86254SMikulas Patocka bio->bi_bdev = s->origin->bdev; 18199fe86254SMikulas Patocka bio_list_add(&s->bios_queued_during_merge, bio); 18209fe86254SMikulas Patocka r = DM_MAPIO_SUBMITTED; 18219fe86254SMikulas Patocka goto out_unlock; 18229fe86254SMikulas Patocka } 182317aa0332SMikulas Patocka 18243452c2a1SMikulas Patocka remap_exception(s, e, bio, chunk); 182517aa0332SMikulas Patocka 182617aa0332SMikulas Patocka if (bio_rw(bio) == WRITE) 1827ee18026aSMikulas Patocka track_chunk(s, bio, chunk); 18283452c2a1SMikulas Patocka goto out_unlock; 18293452c2a1SMikulas Patocka } 18303452c2a1SMikulas Patocka 1831d2fdb776SMikulas Patocka redirect_to_origin: 18323452c2a1SMikulas Patocka bio->bi_bdev = s->origin->bdev; 18333452c2a1SMikulas Patocka 18343452c2a1SMikulas Patocka if (bio_rw(bio) == WRITE) { 18359fe86254SMikulas Patocka up_write(&s->lock); 18363452c2a1SMikulas Patocka return do_origin(s->origin, bio); 18373452c2a1SMikulas Patocka } 18383452c2a1SMikulas Patocka 18393452c2a1SMikulas Patocka out_unlock: 18409fe86254SMikulas Patocka up_write(&s->lock); 18413452c2a1SMikulas Patocka 18423452c2a1SMikulas Patocka return r; 18433452c2a1SMikulas Patocka } 18443452c2a1SMikulas Patocka 18457de3ee57SMikulas Patocka static int snapshot_end_io(struct dm_target *ti, struct bio *bio, int error) 1846cd45daffSMikulas Patocka { 1847cd45daffSMikulas Patocka struct dm_snapshot *s = ti->private; 1848cd45daffSMikulas Patocka 1849ee18026aSMikulas Patocka if (is_bio_tracked(bio)) 1850ee18026aSMikulas Patocka stop_tracking_chunk(s, bio); 1851cd45daffSMikulas Patocka 1852cd45daffSMikulas Patocka return 0; 1853cd45daffSMikulas Patocka } 1854cd45daffSMikulas Patocka 18551e03f97eSMikulas Patocka static void snapshot_merge_presuspend(struct dm_target *ti) 18561e03f97eSMikulas Patocka { 18571e03f97eSMikulas Patocka struct dm_snapshot *s = ti->private; 18581e03f97eSMikulas Patocka 18591e03f97eSMikulas Patocka stop_merge(s); 18601e03f97eSMikulas Patocka } 18611e03f97eSMikulas Patocka 1862c1f0c183SMike Snitzer static int snapshot_preresume(struct dm_target *ti) 1863c1f0c183SMike Snitzer { 1864c1f0c183SMike Snitzer int r = 0; 1865c1f0c183SMike Snitzer struct dm_snapshot *s = ti->private; 1866c1f0c183SMike Snitzer struct dm_snapshot *snap_src = NULL, *snap_dest = NULL; 1867c1f0c183SMike Snitzer 1868c1f0c183SMike Snitzer down_read(&_origins_lock); 18699d3b15c4SMikulas Patocka (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL); 1870c1f0c183SMike Snitzer if (snap_src && snap_dest) { 1871c1f0c183SMike Snitzer down_read(&snap_src->lock); 1872c1f0c183SMike Snitzer if (s == snap_src) { 1873c1f0c183SMike Snitzer DMERR("Unable to resume snapshot source until " 1874c1f0c183SMike Snitzer "handover completes."); 1875c1f0c183SMike Snitzer r = -EINVAL; 1876b83b2f29SMike Snitzer } else if (!dm_suspended(snap_src->ti)) { 1877c1f0c183SMike Snitzer DMERR("Unable to perform snapshot handover until " 1878c1f0c183SMike Snitzer "source is suspended."); 1879c1f0c183SMike Snitzer r = -EINVAL; 1880c1f0c183SMike Snitzer } 1881c1f0c183SMike Snitzer up_read(&snap_src->lock); 1882c1f0c183SMike Snitzer } 1883c1f0c183SMike Snitzer up_read(&_origins_lock); 1884c1f0c183SMike Snitzer 1885c1f0c183SMike Snitzer return r; 1886c1f0c183SMike Snitzer } 1887c1f0c183SMike Snitzer 18881da177e4SLinus Torvalds static void snapshot_resume(struct dm_target *ti) 18891da177e4SLinus Torvalds { 1890028867acSAlasdair G Kergon struct dm_snapshot *s = ti->private; 189109ee96b2SMikulas Patocka struct dm_snapshot *snap_src = NULL, *snap_dest = NULL, *snap_merging = NULL; 1892b735fedeSMikulas Patocka struct dm_origin *o; 1893b735fedeSMikulas Patocka struct mapped_device *origin_md = NULL; 189409ee96b2SMikulas Patocka bool must_restart_merging = false; 1895c1f0c183SMike Snitzer 1896c1f0c183SMike Snitzer down_read(&_origins_lock); 1897b735fedeSMikulas Patocka 1898b735fedeSMikulas Patocka o = __lookup_dm_origin(s->origin->bdev); 1899b735fedeSMikulas Patocka if (o) 1900b735fedeSMikulas Patocka origin_md = dm_table_get_md(o->ti->table); 190109ee96b2SMikulas Patocka if (!origin_md) { 190209ee96b2SMikulas Patocka (void) __find_snapshots_sharing_cow(s, NULL, NULL, &snap_merging); 190309ee96b2SMikulas Patocka if (snap_merging) 190409ee96b2SMikulas Patocka origin_md = dm_table_get_md(snap_merging->ti->table); 190509ee96b2SMikulas Patocka } 1906b735fedeSMikulas Patocka if (origin_md == dm_table_get_md(ti->table)) 1907b735fedeSMikulas Patocka origin_md = NULL; 190809ee96b2SMikulas Patocka if (origin_md) { 190909ee96b2SMikulas Patocka if (dm_hold(origin_md)) 191009ee96b2SMikulas Patocka origin_md = NULL; 191109ee96b2SMikulas Patocka } 1912b735fedeSMikulas Patocka 191309ee96b2SMikulas Patocka up_read(&_origins_lock); 191409ee96b2SMikulas Patocka 191509ee96b2SMikulas Patocka if (origin_md) { 1916b735fedeSMikulas Patocka dm_internal_suspend_fast(origin_md); 191709ee96b2SMikulas Patocka if (snap_merging && test_bit(RUNNING_MERGE, &snap_merging->state_bits)) { 191809ee96b2SMikulas Patocka must_restart_merging = true; 191909ee96b2SMikulas Patocka stop_merge(snap_merging); 192009ee96b2SMikulas Patocka } 192109ee96b2SMikulas Patocka } 192209ee96b2SMikulas Patocka 192309ee96b2SMikulas Patocka down_read(&_origins_lock); 1924b735fedeSMikulas Patocka 19259d3b15c4SMikulas Patocka (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL); 1926c1f0c183SMike Snitzer if (snap_src && snap_dest) { 1927c1f0c183SMike Snitzer down_write(&snap_src->lock); 1928c1f0c183SMike Snitzer down_write_nested(&snap_dest->lock, SINGLE_DEPTH_NESTING); 1929c1f0c183SMike Snitzer __handover_exceptions(snap_src, snap_dest); 1930c1f0c183SMike Snitzer up_write(&snap_dest->lock); 1931c1f0c183SMike Snitzer up_write(&snap_src->lock); 1932c1f0c183SMike Snitzer } 1933b735fedeSMikulas Patocka 1934c1f0c183SMike Snitzer up_read(&_origins_lock); 1935c1f0c183SMike Snitzer 193609ee96b2SMikulas Patocka if (origin_md) { 193709ee96b2SMikulas Patocka if (must_restart_merging) 193809ee96b2SMikulas Patocka start_merge(snap_merging); 193909ee96b2SMikulas Patocka dm_internal_resume_fast(origin_md); 194009ee96b2SMikulas Patocka dm_put(origin_md); 194109ee96b2SMikulas Patocka } 194209ee96b2SMikulas Patocka 1943c1f0c183SMike Snitzer /* Now we have correct chunk size, reregister */ 1944c1f0c183SMike Snitzer reregister_snapshot(s); 19451da177e4SLinus Torvalds 1946aa14edebSAlasdair G Kergon down_write(&s->lock); 1947aa14edebSAlasdair G Kergon s->active = 1; 1948aa14edebSAlasdair G Kergon up_write(&s->lock); 19491da177e4SLinus Torvalds } 19501da177e4SLinus Torvalds 1951542f9038SMike Snitzer static uint32_t get_origin_minimum_chunksize(struct block_device *bdev) 19521e03f97eSMikulas Patocka { 1953542f9038SMike Snitzer uint32_t min_chunksize; 19541e03f97eSMikulas Patocka 19551e03f97eSMikulas Patocka down_read(&_origins_lock); 19561e03f97eSMikulas Patocka min_chunksize = __minimum_chunk_size(__lookup_origin(bdev)); 19571e03f97eSMikulas Patocka up_read(&_origins_lock); 19581e03f97eSMikulas Patocka 19591e03f97eSMikulas Patocka return min_chunksize; 19601e03f97eSMikulas Patocka } 19611e03f97eSMikulas Patocka 19621e03f97eSMikulas Patocka static void snapshot_merge_resume(struct dm_target *ti) 19631e03f97eSMikulas Patocka { 19641e03f97eSMikulas Patocka struct dm_snapshot *s = ti->private; 19651e03f97eSMikulas Patocka 19661e03f97eSMikulas Patocka /* 19671e03f97eSMikulas Patocka * Handover exceptions from existing snapshot. 19681e03f97eSMikulas Patocka */ 19691e03f97eSMikulas Patocka snapshot_resume(ti); 19701e03f97eSMikulas Patocka 19711e03f97eSMikulas Patocka /* 1972542f9038SMike Snitzer * snapshot-merge acts as an origin, so set ti->max_io_len 19731e03f97eSMikulas Patocka */ 1974542f9038SMike Snitzer ti->max_io_len = get_origin_minimum_chunksize(s->origin->bdev); 19751e03f97eSMikulas Patocka 19761e03f97eSMikulas Patocka start_merge(s); 19771e03f97eSMikulas Patocka } 19781e03f97eSMikulas Patocka 1979fd7c092eSMikulas Patocka static void snapshot_status(struct dm_target *ti, status_type_t type, 19801f4e0ff0SAlasdair G Kergon unsigned status_flags, char *result, unsigned maxlen) 19811da177e4SLinus Torvalds { 19822e4a31dfSJonathan Brassow unsigned sz = 0; 1983028867acSAlasdair G Kergon struct dm_snapshot *snap = ti->private; 19841da177e4SLinus Torvalds 19851da177e4SLinus Torvalds switch (type) { 19861da177e4SLinus Torvalds case STATUSTYPE_INFO: 198794e76572SMikulas Patocka 198894e76572SMikulas Patocka down_write(&snap->lock); 198994e76572SMikulas Patocka 19901da177e4SLinus Torvalds if (!snap->valid) 19912e4a31dfSJonathan Brassow DMEMIT("Invalid"); 1992d8ddb1cfSMike Snitzer else if (snap->merge_failed) 1993d8ddb1cfSMike Snitzer DMEMIT("Merge failed"); 19941da177e4SLinus Torvalds else { 1995985903bbSMike Snitzer if (snap->store->type->usage) { 1996985903bbSMike Snitzer sector_t total_sectors, sectors_allocated, 1997985903bbSMike Snitzer metadata_sectors; 1998985903bbSMike Snitzer snap->store->type->usage(snap->store, 1999985903bbSMike Snitzer &total_sectors, 2000985903bbSMike Snitzer §ors_allocated, 2001985903bbSMike Snitzer &metadata_sectors); 2002985903bbSMike Snitzer DMEMIT("%llu/%llu %llu", 2003985903bbSMike Snitzer (unsigned long long)sectors_allocated, 2004985903bbSMike Snitzer (unsigned long long)total_sectors, 2005985903bbSMike Snitzer (unsigned long long)metadata_sectors); 20061da177e4SLinus Torvalds } 20071da177e4SLinus Torvalds else 20082e4a31dfSJonathan Brassow DMEMIT("Unknown"); 20091da177e4SLinus Torvalds } 201094e76572SMikulas Patocka 201194e76572SMikulas Patocka up_write(&snap->lock); 201294e76572SMikulas Patocka 20131da177e4SLinus Torvalds break; 20141da177e4SLinus Torvalds 20151da177e4SLinus Torvalds case STATUSTYPE_TABLE: 20161da177e4SLinus Torvalds /* 20171da177e4SLinus Torvalds * kdevname returns a static pointer so we need 20181da177e4SLinus Torvalds * to make private copies if the output is to 20191da177e4SLinus Torvalds * make sense. 20201da177e4SLinus Torvalds */ 2021fc56f6fbSMike Snitzer DMEMIT("%s %s", snap->origin->name, snap->cow->name); 20221e302a92SJonathan Brassow snap->store->type->status(snap->store, type, result + sz, 20231e302a92SJonathan Brassow maxlen - sz); 20241da177e4SLinus Torvalds break; 20251da177e4SLinus Torvalds } 20261da177e4SLinus Torvalds } 20271da177e4SLinus Torvalds 20288811f46cSMike Snitzer static int snapshot_iterate_devices(struct dm_target *ti, 20298811f46cSMike Snitzer iterate_devices_callout_fn fn, void *data) 20308811f46cSMike Snitzer { 20318811f46cSMike Snitzer struct dm_snapshot *snap = ti->private; 20321e5554c8SMikulas Patocka int r; 20338811f46cSMike Snitzer 20341e5554c8SMikulas Patocka r = fn(ti, snap->origin, 0, ti->len, data); 20351e5554c8SMikulas Patocka 20361e5554c8SMikulas Patocka if (!r) 20371e5554c8SMikulas Patocka r = fn(ti, snap->cow, 0, get_dev_size(snap->cow->bdev), data); 20381e5554c8SMikulas Patocka 20391e5554c8SMikulas Patocka return r; 20408811f46cSMike Snitzer } 20418811f46cSMike Snitzer 20428811f46cSMike Snitzer 20431da177e4SLinus Torvalds /*----------------------------------------------------------------- 20441da177e4SLinus Torvalds * Origin methods 20451da177e4SLinus Torvalds *---------------------------------------------------------------*/ 20469eaae8ffSMikulas Patocka 20479eaae8ffSMikulas Patocka /* 20489eaae8ffSMikulas Patocka * If no exceptions need creating, DM_MAPIO_REMAPPED is returned and any 20499eaae8ffSMikulas Patocka * supplied bio was ignored. The caller may submit it immediately. 20509eaae8ffSMikulas Patocka * (No remapping actually occurs as the origin is always a direct linear 20519eaae8ffSMikulas Patocka * map.) 20529eaae8ffSMikulas Patocka * 20539eaae8ffSMikulas Patocka * If further exceptions are required, DM_MAPIO_SUBMITTED is returned 20549eaae8ffSMikulas Patocka * and any supplied bio is added to a list to be submitted once all 20559eaae8ffSMikulas Patocka * the necessary exceptions exist. 20569eaae8ffSMikulas Patocka */ 20579eaae8ffSMikulas Patocka static int __origin_write(struct list_head *snapshots, sector_t sector, 20589eaae8ffSMikulas Patocka struct bio *bio) 20591da177e4SLinus Torvalds { 2060515ad66cSMikulas Patocka int r = DM_MAPIO_REMAPPED; 20611da177e4SLinus Torvalds struct dm_snapshot *snap; 20621d4989c8SJon Brassow struct dm_exception *e; 2063515ad66cSMikulas Patocka struct dm_snap_pending_exception *pe; 2064515ad66cSMikulas Patocka struct dm_snap_pending_exception *pe_to_start_now = NULL; 2065515ad66cSMikulas Patocka struct dm_snap_pending_exception *pe_to_start_last = NULL; 20661da177e4SLinus Torvalds chunk_t chunk; 20671da177e4SLinus Torvalds 20681da177e4SLinus Torvalds /* Do all the snapshots on this origin */ 20691da177e4SLinus Torvalds list_for_each_entry (snap, snapshots, list) { 20703452c2a1SMikulas Patocka /* 20713452c2a1SMikulas Patocka * Don't make new exceptions in a merging snapshot 20723452c2a1SMikulas Patocka * because it has effectively been deleted 20733452c2a1SMikulas Patocka */ 20743452c2a1SMikulas Patocka if (dm_target_is_snapshot_merge(snap->ti)) 20753452c2a1SMikulas Patocka continue; 20763452c2a1SMikulas Patocka 207776df1c65SAlasdair G Kergon down_write(&snap->lock); 207876df1c65SAlasdair G Kergon 2079aa14edebSAlasdair G Kergon /* Only deal with valid and active snapshots */ 2080aa14edebSAlasdair G Kergon if (!snap->valid || !snap->active) 208176df1c65SAlasdair G Kergon goto next_snapshot; 20821da177e4SLinus Torvalds 2083d5e404c1SAlasdair G Kergon /* Nothing to do if writing beyond end of snapshot */ 20849eaae8ffSMikulas Patocka if (sector >= dm_table_get_size(snap->ti->table)) 208576df1c65SAlasdair G Kergon goto next_snapshot; 20861da177e4SLinus Torvalds 20871da177e4SLinus Torvalds /* 20881da177e4SLinus Torvalds * Remember, different snapshots can have 20891da177e4SLinus Torvalds * different chunk sizes. 20901da177e4SLinus Torvalds */ 20919eaae8ffSMikulas Patocka chunk = sector_to_chunk(snap->store, sector); 20921da177e4SLinus Torvalds 20931da177e4SLinus Torvalds /* 20941da177e4SLinus Torvalds * Check exception table to see if block 20951da177e4SLinus Torvalds * is already remapped in this snapshot 20961da177e4SLinus Torvalds * and trigger an exception if not. 20971da177e4SLinus Torvalds */ 20983510cb94SJon Brassow e = dm_lookup_exception(&snap->complete, chunk); 209976df1c65SAlasdair G Kergon if (e) 210076df1c65SAlasdair G Kergon goto next_snapshot; 210176df1c65SAlasdair G Kergon 21022913808eSMikulas Patocka pe = __lookup_pending_exception(snap, chunk); 21032913808eSMikulas Patocka if (!pe) { 2104c6621392SMikulas Patocka up_write(&snap->lock); 2105c6621392SMikulas Patocka pe = alloc_pending_exception(snap); 2106c6621392SMikulas Patocka down_write(&snap->lock); 2107c6621392SMikulas Patocka 2108c6621392SMikulas Patocka if (!snap->valid) { 2109c6621392SMikulas Patocka free_pending_exception(pe); 2110c6621392SMikulas Patocka goto next_snapshot; 2111c6621392SMikulas Patocka } 2112c6621392SMikulas Patocka 21133510cb94SJon Brassow e = dm_lookup_exception(&snap->complete, chunk); 211435bf659bSMikulas Patocka if (e) { 211535bf659bSMikulas Patocka free_pending_exception(pe); 211635bf659bSMikulas Patocka goto next_snapshot; 211735bf659bSMikulas Patocka } 211835bf659bSMikulas Patocka 2119c6621392SMikulas Patocka pe = __find_pending_exception(snap, pe, chunk); 21201da177e4SLinus Torvalds if (!pe) { 2121695368acSAlasdair G Kergon __invalidate_snapshot(snap, -ENOMEM); 212276df1c65SAlasdair G Kergon goto next_snapshot; 212376df1c65SAlasdair G Kergon } 21242913808eSMikulas Patocka } 21251da177e4SLinus Torvalds 2126d2a7ad29SKiyoshi Ueda r = DM_MAPIO_SUBMITTED; 212776df1c65SAlasdair G Kergon 2128515ad66cSMikulas Patocka /* 2129515ad66cSMikulas Patocka * If an origin bio was supplied, queue it to wait for the 2130515ad66cSMikulas Patocka * completion of this exception, and start this one last, 2131515ad66cSMikulas Patocka * at the end of the function. 2132515ad66cSMikulas Patocka */ 2133515ad66cSMikulas Patocka if (bio) { 2134515ad66cSMikulas Patocka bio_list_add(&pe->origin_bios, bio); 2135515ad66cSMikulas Patocka bio = NULL; 2136515ad66cSMikulas Patocka 2137515ad66cSMikulas Patocka if (!pe->started) { 2138515ad66cSMikulas Patocka pe->started = 1; 2139515ad66cSMikulas Patocka pe_to_start_last = pe; 2140515ad66cSMikulas Patocka } 2141b4b610f6SAlasdair G Kergon } 214276df1c65SAlasdair G Kergon 2143eccf0817SAlasdair G Kergon if (!pe->started) { 2144eccf0817SAlasdair G Kergon pe->started = 1; 2145515ad66cSMikulas Patocka pe_to_start_now = pe; 2146eccf0817SAlasdair G Kergon } 21471da177e4SLinus Torvalds 214876df1c65SAlasdair G Kergon next_snapshot: 21491da177e4SLinus Torvalds up_write(&snap->lock); 2150515ad66cSMikulas Patocka 2151515ad66cSMikulas Patocka if (pe_to_start_now) { 2152515ad66cSMikulas Patocka start_copy(pe_to_start_now); 2153515ad66cSMikulas Patocka pe_to_start_now = NULL; 21541da177e4SLinus Torvalds } 2155b4b610f6SAlasdair G Kergon } 2156b4b610f6SAlasdair G Kergon 21571da177e4SLinus Torvalds /* 2158515ad66cSMikulas Patocka * Submit the exception against which the bio is queued last, 2159515ad66cSMikulas Patocka * to give the other exceptions a head start. 21601da177e4SLinus Torvalds */ 2161515ad66cSMikulas Patocka if (pe_to_start_last) 2162515ad66cSMikulas Patocka start_copy(pe_to_start_last); 21631da177e4SLinus Torvalds 21641da177e4SLinus Torvalds return r; 21651da177e4SLinus Torvalds } 21661da177e4SLinus Torvalds 21671da177e4SLinus Torvalds /* 21681da177e4SLinus Torvalds * Called on a write from the origin driver. 21691da177e4SLinus Torvalds */ 21701da177e4SLinus Torvalds static int do_origin(struct dm_dev *origin, struct bio *bio) 21711da177e4SLinus Torvalds { 21721da177e4SLinus Torvalds struct origin *o; 2173d2a7ad29SKiyoshi Ueda int r = DM_MAPIO_REMAPPED; 21741da177e4SLinus Torvalds 21751da177e4SLinus Torvalds down_read(&_origins_lock); 21761da177e4SLinus Torvalds o = __lookup_origin(origin->bdev); 21771da177e4SLinus Torvalds if (o) 21784f024f37SKent Overstreet r = __origin_write(&o->snapshots, bio->bi_iter.bi_sector, bio); 21791da177e4SLinus Torvalds up_read(&_origins_lock); 21801da177e4SLinus Torvalds 21811da177e4SLinus Torvalds return r; 21821da177e4SLinus Torvalds } 21831da177e4SLinus Torvalds 21841da177e4SLinus Torvalds /* 218573dfd078SMikulas Patocka * Trigger exceptions in all non-merging snapshots. 218673dfd078SMikulas Patocka * 218773dfd078SMikulas Patocka * The chunk size of the merging snapshot may be larger than the chunk 218873dfd078SMikulas Patocka * size of some other snapshot so we may need to reallocate multiple 218973dfd078SMikulas Patocka * chunks in other snapshots. 219073dfd078SMikulas Patocka * 219173dfd078SMikulas Patocka * We scan all the overlapping exceptions in the other snapshots. 219273dfd078SMikulas Patocka * Returns 1 if anything was reallocated and must be waited for, 219373dfd078SMikulas Patocka * otherwise returns 0. 219473dfd078SMikulas Patocka * 219573dfd078SMikulas Patocka * size must be a multiple of merging_snap's chunk_size. 219673dfd078SMikulas Patocka */ 219773dfd078SMikulas Patocka static int origin_write_extent(struct dm_snapshot *merging_snap, 219873dfd078SMikulas Patocka sector_t sector, unsigned size) 219973dfd078SMikulas Patocka { 220073dfd078SMikulas Patocka int must_wait = 0; 220173dfd078SMikulas Patocka sector_t n; 220273dfd078SMikulas Patocka struct origin *o; 220373dfd078SMikulas Patocka 220473dfd078SMikulas Patocka /* 2205542f9038SMike Snitzer * The origin's __minimum_chunk_size() got stored in max_io_len 220673dfd078SMikulas Patocka * by snapshot_merge_resume(). 220773dfd078SMikulas Patocka */ 220873dfd078SMikulas Patocka down_read(&_origins_lock); 220973dfd078SMikulas Patocka o = __lookup_origin(merging_snap->origin->bdev); 2210542f9038SMike Snitzer for (n = 0; n < size; n += merging_snap->ti->max_io_len) 221173dfd078SMikulas Patocka if (__origin_write(&o->snapshots, sector + n, NULL) == 221273dfd078SMikulas Patocka DM_MAPIO_SUBMITTED) 221373dfd078SMikulas Patocka must_wait = 1; 221473dfd078SMikulas Patocka up_read(&_origins_lock); 221573dfd078SMikulas Patocka 221673dfd078SMikulas Patocka return must_wait; 221773dfd078SMikulas Patocka } 221873dfd078SMikulas Patocka 221973dfd078SMikulas Patocka /* 22201da177e4SLinus Torvalds * Origin: maps a linear range of a device, with hooks for snapshotting. 22211da177e4SLinus Torvalds */ 22221da177e4SLinus Torvalds 22231da177e4SLinus Torvalds /* 22241da177e4SLinus Torvalds * Construct an origin mapping: <dev_path> 22251da177e4SLinus Torvalds * The context for an origin is merely a 'struct dm_dev *' 22261da177e4SLinus Torvalds * pointing to the real device. 22271da177e4SLinus Torvalds */ 22281da177e4SLinus Torvalds static int origin_ctr(struct dm_target *ti, unsigned int argc, char **argv) 22291da177e4SLinus Torvalds { 22301da177e4SLinus Torvalds int r; 2231599cdf3bSMikulas Patocka struct dm_origin *o; 22321da177e4SLinus Torvalds 22331da177e4SLinus Torvalds if (argc != 1) { 223472d94861SAlasdair G Kergon ti->error = "origin: incorrect number of arguments"; 22351da177e4SLinus Torvalds return -EINVAL; 22361da177e4SLinus Torvalds } 22371da177e4SLinus Torvalds 2238599cdf3bSMikulas Patocka o = kmalloc(sizeof(struct dm_origin), GFP_KERNEL); 2239599cdf3bSMikulas Patocka if (!o) { 2240599cdf3bSMikulas Patocka ti->error = "Cannot allocate private origin structure"; 2241599cdf3bSMikulas Patocka r = -ENOMEM; 2242599cdf3bSMikulas Patocka goto bad_alloc; 22431da177e4SLinus Torvalds } 22441da177e4SLinus Torvalds 2245599cdf3bSMikulas Patocka r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &o->dev); 2246599cdf3bSMikulas Patocka if (r) { 2247599cdf3bSMikulas Patocka ti->error = "Cannot get target device"; 2248599cdf3bSMikulas Patocka goto bad_open; 2249599cdf3bSMikulas Patocka } 2250599cdf3bSMikulas Patocka 2251b735fedeSMikulas Patocka o->ti = ti; 2252599cdf3bSMikulas Patocka ti->private = o; 225355a62eefSAlasdair G Kergon ti->num_flush_bios = 1; 2254494b3ee7SMikulas Patocka 22551da177e4SLinus Torvalds return 0; 2256599cdf3bSMikulas Patocka 2257599cdf3bSMikulas Patocka bad_open: 2258599cdf3bSMikulas Patocka kfree(o); 2259599cdf3bSMikulas Patocka bad_alloc: 2260599cdf3bSMikulas Patocka return r; 22611da177e4SLinus Torvalds } 22621da177e4SLinus Torvalds 22631da177e4SLinus Torvalds static void origin_dtr(struct dm_target *ti) 22641da177e4SLinus Torvalds { 2265599cdf3bSMikulas Patocka struct dm_origin *o = ti->private; 2266b735fedeSMikulas Patocka 2267599cdf3bSMikulas Patocka dm_put_device(ti, o->dev); 2268599cdf3bSMikulas Patocka kfree(o); 22691da177e4SLinus Torvalds } 22701da177e4SLinus Torvalds 22717de3ee57SMikulas Patocka static int origin_map(struct dm_target *ti, struct bio *bio) 22721da177e4SLinus Torvalds { 2273599cdf3bSMikulas Patocka struct dm_origin *o = ti->private; 2274298eaa89SMikulas Patocka unsigned available_sectors; 22751da177e4SLinus Torvalds 2276599cdf3bSMikulas Patocka bio->bi_bdev = o->dev->bdev; 22771da177e4SLinus Torvalds 2278298eaa89SMikulas Patocka if (unlikely(bio->bi_rw & REQ_FLUSH)) 2279494b3ee7SMikulas Patocka return DM_MAPIO_REMAPPED; 2280494b3ee7SMikulas Patocka 2281298eaa89SMikulas Patocka if (bio_rw(bio) != WRITE) 2282298eaa89SMikulas Patocka return DM_MAPIO_REMAPPED; 2283298eaa89SMikulas Patocka 2284298eaa89SMikulas Patocka available_sectors = o->split_boundary - 2285298eaa89SMikulas Patocka ((unsigned)bio->bi_iter.bi_sector & (o->split_boundary - 1)); 2286298eaa89SMikulas Patocka 2287298eaa89SMikulas Patocka if (bio_sectors(bio) > available_sectors) 2288298eaa89SMikulas Patocka dm_accept_partial_bio(bio, available_sectors); 2289298eaa89SMikulas Patocka 22901da177e4SLinus Torvalds /* Only tell snapshots if this is a write */ 2291298eaa89SMikulas Patocka return do_origin(o->dev, bio); 22921da177e4SLinus Torvalds } 22931da177e4SLinus Torvalds 22941da177e4SLinus Torvalds /* 2295542f9038SMike Snitzer * Set the target "max_io_len" field to the minimum of all the snapshots' 22961da177e4SLinus Torvalds * chunk sizes. 22971da177e4SLinus Torvalds */ 22981da177e4SLinus Torvalds static void origin_resume(struct dm_target *ti) 22991da177e4SLinus Torvalds { 2300599cdf3bSMikulas Patocka struct dm_origin *o = ti->private; 23011da177e4SLinus Torvalds 2302298eaa89SMikulas Patocka o->split_boundary = get_origin_minimum_chunksize(o->dev->bdev); 2303b735fedeSMikulas Patocka 2304b735fedeSMikulas Patocka down_write(&_origins_lock); 2305b735fedeSMikulas Patocka __insert_dm_origin(o); 2306b735fedeSMikulas Patocka up_write(&_origins_lock); 2307b735fedeSMikulas Patocka } 2308b735fedeSMikulas Patocka 2309b735fedeSMikulas Patocka static void origin_postsuspend(struct dm_target *ti) 2310b735fedeSMikulas Patocka { 2311b735fedeSMikulas Patocka struct dm_origin *o = ti->private; 2312b735fedeSMikulas Patocka 2313b735fedeSMikulas Patocka down_write(&_origins_lock); 2314b735fedeSMikulas Patocka __remove_dm_origin(o); 2315b735fedeSMikulas Patocka up_write(&_origins_lock); 23161da177e4SLinus Torvalds } 23171da177e4SLinus Torvalds 2318fd7c092eSMikulas Patocka static void origin_status(struct dm_target *ti, status_type_t type, 23191f4e0ff0SAlasdair G Kergon unsigned status_flags, char *result, unsigned maxlen) 23201da177e4SLinus Torvalds { 2321599cdf3bSMikulas Patocka struct dm_origin *o = ti->private; 23221da177e4SLinus Torvalds 23231da177e4SLinus Torvalds switch (type) { 23241da177e4SLinus Torvalds case STATUSTYPE_INFO: 23251da177e4SLinus Torvalds result[0] = '\0'; 23261da177e4SLinus Torvalds break; 23271da177e4SLinus Torvalds 23281da177e4SLinus Torvalds case STATUSTYPE_TABLE: 2329599cdf3bSMikulas Patocka snprintf(result, maxlen, "%s", o->dev->name); 23301da177e4SLinus Torvalds break; 23311da177e4SLinus Torvalds } 23321da177e4SLinus Torvalds } 23331da177e4SLinus Torvalds 2334b1d55528SMikulas Patocka static int origin_merge(struct dm_target *ti, struct bvec_merge_data *bvm, 2335b1d55528SMikulas Patocka struct bio_vec *biovec, int max_size) 2336b1d55528SMikulas Patocka { 2337599cdf3bSMikulas Patocka struct dm_origin *o = ti->private; 2338599cdf3bSMikulas Patocka struct request_queue *q = bdev_get_queue(o->dev->bdev); 2339b1d55528SMikulas Patocka 2340b1d55528SMikulas Patocka if (!q->merge_bvec_fn) 2341b1d55528SMikulas Patocka return max_size; 2342b1d55528SMikulas Patocka 2343599cdf3bSMikulas Patocka bvm->bi_bdev = o->dev->bdev; 2344b1d55528SMikulas Patocka 2345b1d55528SMikulas Patocka return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); 2346b1d55528SMikulas Patocka } 2347b1d55528SMikulas Patocka 23488811f46cSMike Snitzer static int origin_iterate_devices(struct dm_target *ti, 23498811f46cSMike Snitzer iterate_devices_callout_fn fn, void *data) 23508811f46cSMike Snitzer { 2351599cdf3bSMikulas Patocka struct dm_origin *o = ti->private; 23528811f46cSMike Snitzer 2353599cdf3bSMikulas Patocka return fn(ti, o->dev, 0, ti->len, data); 23548811f46cSMike Snitzer } 23558811f46cSMike Snitzer 23561da177e4SLinus Torvalds static struct target_type origin_target = { 23571da177e4SLinus Torvalds .name = "snapshot-origin", 2358b735fedeSMikulas Patocka .version = {1, 9, 0}, 23591da177e4SLinus Torvalds .module = THIS_MODULE, 23601da177e4SLinus Torvalds .ctr = origin_ctr, 23611da177e4SLinus Torvalds .dtr = origin_dtr, 23621da177e4SLinus Torvalds .map = origin_map, 23631da177e4SLinus Torvalds .resume = origin_resume, 2364b735fedeSMikulas Patocka .postsuspend = origin_postsuspend, 23651da177e4SLinus Torvalds .status = origin_status, 2366b1d55528SMikulas Patocka .merge = origin_merge, 23678811f46cSMike Snitzer .iterate_devices = origin_iterate_devices, 23681da177e4SLinus Torvalds }; 23691da177e4SLinus Torvalds 23701da177e4SLinus Torvalds static struct target_type snapshot_target = { 23711da177e4SLinus Torvalds .name = "snapshot", 2372b735fedeSMikulas Patocka .version = {1, 13, 0}, 23731da177e4SLinus Torvalds .module = THIS_MODULE, 23741da177e4SLinus Torvalds .ctr = snapshot_ctr, 23751da177e4SLinus Torvalds .dtr = snapshot_dtr, 23761da177e4SLinus Torvalds .map = snapshot_map, 2377cd45daffSMikulas Patocka .end_io = snapshot_end_io, 2378c1f0c183SMike Snitzer .preresume = snapshot_preresume, 23791da177e4SLinus Torvalds .resume = snapshot_resume, 23801da177e4SLinus Torvalds .status = snapshot_status, 23818811f46cSMike Snitzer .iterate_devices = snapshot_iterate_devices, 23821da177e4SLinus Torvalds }; 23831da177e4SLinus Torvalds 2384d698aa45SMikulas Patocka static struct target_type merge_target = { 2385d698aa45SMikulas Patocka .name = dm_snapshot_merge_target_name, 238609ee96b2SMikulas Patocka .version = {1, 3, 0}, 2387d698aa45SMikulas Patocka .module = THIS_MODULE, 2388d698aa45SMikulas Patocka .ctr = snapshot_ctr, 2389d698aa45SMikulas Patocka .dtr = snapshot_dtr, 23903452c2a1SMikulas Patocka .map = snapshot_merge_map, 2391d698aa45SMikulas Patocka .end_io = snapshot_end_io, 23921e03f97eSMikulas Patocka .presuspend = snapshot_merge_presuspend, 2393d698aa45SMikulas Patocka .preresume = snapshot_preresume, 23941e03f97eSMikulas Patocka .resume = snapshot_merge_resume, 2395d698aa45SMikulas Patocka .status = snapshot_status, 2396d698aa45SMikulas Patocka .iterate_devices = snapshot_iterate_devices, 2397d698aa45SMikulas Patocka }; 2398d698aa45SMikulas Patocka 23991da177e4SLinus Torvalds static int __init dm_snapshot_init(void) 24001da177e4SLinus Torvalds { 24011da177e4SLinus Torvalds int r; 24021da177e4SLinus Torvalds 24034db6bfe0SAlasdair G Kergon r = dm_exception_store_init(); 24044db6bfe0SAlasdair G Kergon if (r) { 24054db6bfe0SAlasdair G Kergon DMERR("Failed to initialize exception stores"); 24064db6bfe0SAlasdair G Kergon return r; 24074db6bfe0SAlasdair G Kergon } 24084db6bfe0SAlasdair G Kergon 24091da177e4SLinus Torvalds r = dm_register_target(&snapshot_target); 2410d698aa45SMikulas Patocka if (r < 0) { 24111da177e4SLinus Torvalds DMERR("snapshot target register failed %d", r); 2412034a186dSJonathan Brassow goto bad_register_snapshot_target; 24131da177e4SLinus Torvalds } 24141da177e4SLinus Torvalds 24151da177e4SLinus Torvalds r = dm_register_target(&origin_target); 24161da177e4SLinus Torvalds if (r < 0) { 241772d94861SAlasdair G Kergon DMERR("Origin target register failed %d", r); 2418d698aa45SMikulas Patocka goto bad_register_origin_target; 2419d698aa45SMikulas Patocka } 2420d698aa45SMikulas Patocka 2421d698aa45SMikulas Patocka r = dm_register_target(&merge_target); 2422d698aa45SMikulas Patocka if (r < 0) { 2423d698aa45SMikulas Patocka DMERR("Merge target register failed %d", r); 2424d698aa45SMikulas Patocka goto bad_register_merge_target; 24251da177e4SLinus Torvalds } 24261da177e4SLinus Torvalds 24271da177e4SLinus Torvalds r = init_origin_hash(); 24281da177e4SLinus Torvalds if (r) { 24291da177e4SLinus Torvalds DMERR("init_origin_hash failed."); 2430d698aa45SMikulas Patocka goto bad_origin_hash; 24311da177e4SLinus Torvalds } 24321da177e4SLinus Torvalds 24331d4989c8SJon Brassow exception_cache = KMEM_CACHE(dm_exception, 0); 24341da177e4SLinus Torvalds if (!exception_cache) { 24351da177e4SLinus Torvalds DMERR("Couldn't create exception cache."); 24361da177e4SLinus Torvalds r = -ENOMEM; 2437d698aa45SMikulas Patocka goto bad_exception_cache; 24381da177e4SLinus Torvalds } 24391da177e4SLinus Torvalds 2440028867acSAlasdair G Kergon pending_cache = KMEM_CACHE(dm_snap_pending_exception, 0); 24411da177e4SLinus Torvalds if (!pending_cache) { 24421da177e4SLinus Torvalds DMERR("Couldn't create pending cache."); 24431da177e4SLinus Torvalds r = -ENOMEM; 2444d698aa45SMikulas Patocka goto bad_pending_cache; 24451da177e4SLinus Torvalds } 24461da177e4SLinus Torvalds 24471da177e4SLinus Torvalds return 0; 24481da177e4SLinus Torvalds 2449d698aa45SMikulas Patocka bad_pending_cache: 24501da177e4SLinus Torvalds kmem_cache_destroy(exception_cache); 2451d698aa45SMikulas Patocka bad_exception_cache: 24521da177e4SLinus Torvalds exit_origin_hash(); 2453d698aa45SMikulas Patocka bad_origin_hash: 2454d698aa45SMikulas Patocka dm_unregister_target(&merge_target); 2455d698aa45SMikulas Patocka bad_register_merge_target: 24561da177e4SLinus Torvalds dm_unregister_target(&origin_target); 2457d698aa45SMikulas Patocka bad_register_origin_target: 24581da177e4SLinus Torvalds dm_unregister_target(&snapshot_target); 2459034a186dSJonathan Brassow bad_register_snapshot_target: 2460034a186dSJonathan Brassow dm_exception_store_exit(); 2461d698aa45SMikulas Patocka 24621da177e4SLinus Torvalds return r; 24631da177e4SLinus Torvalds } 24641da177e4SLinus Torvalds 24651da177e4SLinus Torvalds static void __exit dm_snapshot_exit(void) 24661da177e4SLinus Torvalds { 246710d3bd09SMikulas Patocka dm_unregister_target(&snapshot_target); 246810d3bd09SMikulas Patocka dm_unregister_target(&origin_target); 2469d698aa45SMikulas Patocka dm_unregister_target(&merge_target); 24701da177e4SLinus Torvalds 24711da177e4SLinus Torvalds exit_origin_hash(); 24721da177e4SLinus Torvalds kmem_cache_destroy(pending_cache); 24731da177e4SLinus Torvalds kmem_cache_destroy(exception_cache); 24744db6bfe0SAlasdair G Kergon 24754db6bfe0SAlasdair G Kergon dm_exception_store_exit(); 24761da177e4SLinus Torvalds } 24771da177e4SLinus Torvalds 24781da177e4SLinus Torvalds /* Module hooks */ 24791da177e4SLinus Torvalds module_init(dm_snapshot_init); 24801da177e4SLinus Torvalds module_exit(dm_snapshot_exit); 24811da177e4SLinus Torvalds 24821da177e4SLinus Torvalds MODULE_DESCRIPTION(DM_NAME " snapshot target"); 24831da177e4SLinus Torvalds MODULE_AUTHOR("Joe Thornber"); 24841da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 248523cb2109SMikulas Patocka MODULE_ALIAS("dm-snapshot-origin"); 248623cb2109SMikulas Patocka MODULE_ALIAS("dm-snapshot-merge"); 2487