11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * dm-snapshot.c 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Copyright (C) 2001-2002 Sistina Software (UK) Limited. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * This file is released under the GPL. 71da177e4SLinus Torvalds */ 81da177e4SLinus Torvalds 91da177e4SLinus Torvalds #include <linux/blkdev.h> 101da177e4SLinus Torvalds #include <linux/device-mapper.h> 1190fa1527SMikulas Patocka #include <linux/delay.h> 121da177e4SLinus Torvalds #include <linux/fs.h> 131da177e4SLinus Torvalds #include <linux/init.h> 141da177e4SLinus Torvalds #include <linux/kdev_t.h> 151da177e4SLinus Torvalds #include <linux/list.h> 161da177e4SLinus Torvalds #include <linux/mempool.h> 171da177e4SLinus Torvalds #include <linux/module.h> 181da177e4SLinus Torvalds #include <linux/slab.h> 191da177e4SLinus Torvalds #include <linux/vmalloc.h> 206f3c3f0aSvignesh babu #include <linux/log2.h> 21a765e20eSAlasdair G Kergon #include <linux/dm-kcopyd.h> 221da177e4SLinus Torvalds 23b735fedeSMikulas Patocka #include "dm.h" 24b735fedeSMikulas Patocka 25aea53d92SJonathan Brassow #include "dm-exception-store.h" 261da177e4SLinus Torvalds 2772d94861SAlasdair G Kergon #define DM_MSG_PREFIX "snapshots" 2872d94861SAlasdair G Kergon 29d698aa45SMikulas Patocka static const char dm_snapshot_merge_target_name[] = "snapshot-merge"; 30d698aa45SMikulas Patocka 31d698aa45SMikulas Patocka #define dm_target_is_snapshot_merge(ti) \ 32d698aa45SMikulas Patocka ((ti)->type->name == dm_snapshot_merge_target_name) 33d698aa45SMikulas Patocka 341da177e4SLinus Torvalds /* 35cd45daffSMikulas Patocka * The size of the mempool used to track chunks in use. 36cd45daffSMikulas Patocka */ 37cd45daffSMikulas Patocka #define MIN_IOS 256 38cd45daffSMikulas Patocka 39ccc45ea8SJonathan Brassow #define DM_TRACKED_CHUNK_HASH_SIZE 16 40ccc45ea8SJonathan Brassow #define DM_TRACKED_CHUNK_HASH(x) ((unsigned long)(x) & \ 41ccc45ea8SJonathan Brassow (DM_TRACKED_CHUNK_HASH_SIZE - 1)) 42ccc45ea8SJonathan Brassow 43191437a5SJon Brassow struct dm_exception_table { 44ccc45ea8SJonathan Brassow uint32_t hash_mask; 45ccc45ea8SJonathan Brassow unsigned hash_shift; 46ccc45ea8SJonathan Brassow struct list_head *table; 47ccc45ea8SJonathan Brassow }; 48ccc45ea8SJonathan Brassow 49ccc45ea8SJonathan Brassow struct dm_snapshot { 50ccc45ea8SJonathan Brassow struct rw_semaphore lock; 51ccc45ea8SJonathan Brassow 52ccc45ea8SJonathan Brassow struct dm_dev *origin; 53fc56f6fbSMike Snitzer struct dm_dev *cow; 54fc56f6fbSMike Snitzer 55fc56f6fbSMike Snitzer struct dm_target *ti; 56ccc45ea8SJonathan Brassow 57ccc45ea8SJonathan Brassow /* List of snapshots per Origin */ 58ccc45ea8SJonathan Brassow struct list_head list; 59ccc45ea8SJonathan Brassow 60d8ddb1cfSMike Snitzer /* 61d8ddb1cfSMike Snitzer * You can't use a snapshot if this is 0 (e.g. if full). 62d8ddb1cfSMike Snitzer * A snapshot-merge target never clears this. 63d8ddb1cfSMike Snitzer */ 64ccc45ea8SJonathan Brassow int valid; 65ccc45ea8SJonathan Brassow 6676c44f6dSMikulas Patocka /* 6776c44f6dSMikulas Patocka * The snapshot overflowed because of a write to the snapshot device. 6876c44f6dSMikulas Patocka * We don't have to invalidate the snapshot in this case, but we need 6976c44f6dSMikulas Patocka * to prevent further writes. 7076c44f6dSMikulas Patocka */ 7176c44f6dSMikulas Patocka int snapshot_overflowed; 7276c44f6dSMikulas Patocka 73ccc45ea8SJonathan Brassow /* Origin writes don't trigger exceptions until this is set */ 74ccc45ea8SJonathan Brassow int active; 75ccc45ea8SJonathan Brassow 76ccc45ea8SJonathan Brassow atomic_t pending_exceptions_count; 77ccc45ea8SJonathan Brassow 78230c83afSMikulas Patocka /* Protected by "lock" */ 79230c83afSMikulas Patocka sector_t exception_start_sequence; 80230c83afSMikulas Patocka 81230c83afSMikulas Patocka /* Protected by kcopyd single-threaded callback */ 82230c83afSMikulas Patocka sector_t exception_complete_sequence; 83230c83afSMikulas Patocka 84230c83afSMikulas Patocka /* 85230c83afSMikulas Patocka * A list of pending exceptions that completed out of order. 86230c83afSMikulas Patocka * Protected by kcopyd single-threaded callback. 87230c83afSMikulas Patocka */ 88230c83afSMikulas Patocka struct list_head out_of_order_list; 89230c83afSMikulas Patocka 90924e600dSMike Snitzer mempool_t *pending_pool; 91924e600dSMike Snitzer 92191437a5SJon Brassow struct dm_exception_table pending; 93191437a5SJon Brassow struct dm_exception_table complete; 94ccc45ea8SJonathan Brassow 95ccc45ea8SJonathan Brassow /* 96ccc45ea8SJonathan Brassow * pe_lock protects all pending_exception operations and access 97ccc45ea8SJonathan Brassow * as well as the snapshot_bios list. 98ccc45ea8SJonathan Brassow */ 99ccc45ea8SJonathan Brassow spinlock_t pe_lock; 100ccc45ea8SJonathan Brassow 101924e600dSMike Snitzer /* Chunks with outstanding reads */ 102924e600dSMike Snitzer spinlock_t tracked_chunk_lock; 103924e600dSMike Snitzer struct hlist_head tracked_chunk_hash[DM_TRACKED_CHUNK_HASH_SIZE]; 104924e600dSMike Snitzer 105ccc45ea8SJonathan Brassow /* The on disk metadata handler */ 106ccc45ea8SJonathan Brassow struct dm_exception_store *store; 107ccc45ea8SJonathan Brassow 108ccc45ea8SJonathan Brassow struct dm_kcopyd_client *kcopyd_client; 109ccc45ea8SJonathan Brassow 110924e600dSMike Snitzer /* Wait for events based on state_bits */ 111924e600dSMike Snitzer unsigned long state_bits; 112924e600dSMike Snitzer 113924e600dSMike Snitzer /* Range of chunks currently being merged. */ 114924e600dSMike Snitzer chunk_t first_merging_chunk; 115924e600dSMike Snitzer int num_merging_chunks; 1161e03f97eSMikulas Patocka 117d8ddb1cfSMike Snitzer /* 118d8ddb1cfSMike Snitzer * The merge operation failed if this flag is set. 119d8ddb1cfSMike Snitzer * Failure modes are handled as follows: 120d8ddb1cfSMike Snitzer * - I/O error reading the header 121d8ddb1cfSMike Snitzer * => don't load the target; abort. 122d8ddb1cfSMike Snitzer * - Header does not have "valid" flag set 123d8ddb1cfSMike Snitzer * => use the origin; forget about the snapshot. 124d8ddb1cfSMike Snitzer * - I/O error when reading exceptions 125d8ddb1cfSMike Snitzer * => don't load the target; abort. 126d8ddb1cfSMike Snitzer * (We can't use the intermediate origin state.) 127d8ddb1cfSMike Snitzer * - I/O error while merging 128d8ddb1cfSMike Snitzer * => stop merging; set merge_failed; process I/O normally. 129d8ddb1cfSMike Snitzer */ 130d8ddb1cfSMike Snitzer int merge_failed; 131d8ddb1cfSMike Snitzer 1329fe86254SMikulas Patocka /* 1339fe86254SMikulas Patocka * Incoming bios that overlap with chunks being merged must wait 1349fe86254SMikulas Patocka * for them to be committed. 1359fe86254SMikulas Patocka */ 1369fe86254SMikulas Patocka struct bio_list bios_queued_during_merge; 137ccc45ea8SJonathan Brassow }; 138ccc45ea8SJonathan Brassow 1391e03f97eSMikulas Patocka /* 1401e03f97eSMikulas Patocka * state_bits: 1411e03f97eSMikulas Patocka * RUNNING_MERGE - Merge operation is in progress. 1421e03f97eSMikulas Patocka * SHUTDOWN_MERGE - Set to signal that merge needs to be stopped; 1431e03f97eSMikulas Patocka * cleared afterwards. 1441e03f97eSMikulas Patocka */ 1451e03f97eSMikulas Patocka #define RUNNING_MERGE 0 1461e03f97eSMikulas Patocka #define SHUTDOWN_MERGE 1 1471e03f97eSMikulas Patocka 148df5d2e90SMikulas Patocka DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(snapshot_copy_throttle, 149df5d2e90SMikulas Patocka "A percentage of time allocated for copy on write"); 150df5d2e90SMikulas Patocka 151c2411045SMikulas Patocka struct dm_dev *dm_snap_origin(struct dm_snapshot *s) 152c2411045SMikulas Patocka { 153c2411045SMikulas Patocka return s->origin; 154c2411045SMikulas Patocka } 155c2411045SMikulas Patocka EXPORT_SYMBOL(dm_snap_origin); 156c2411045SMikulas Patocka 157fc56f6fbSMike Snitzer struct dm_dev *dm_snap_cow(struct dm_snapshot *s) 158fc56f6fbSMike Snitzer { 159fc56f6fbSMike Snitzer return s->cow; 160fc56f6fbSMike Snitzer } 161fc56f6fbSMike Snitzer EXPORT_SYMBOL(dm_snap_cow); 162fc56f6fbSMike Snitzer 163ccc45ea8SJonathan Brassow static sector_t chunk_to_sector(struct dm_exception_store *store, 164ccc45ea8SJonathan Brassow chunk_t chunk) 165ccc45ea8SJonathan Brassow { 166ccc45ea8SJonathan Brassow return chunk << store->chunk_shift; 167ccc45ea8SJonathan Brassow } 168ccc45ea8SJonathan Brassow 169ccc45ea8SJonathan Brassow static int bdev_equal(struct block_device *lhs, struct block_device *rhs) 170ccc45ea8SJonathan Brassow { 171ccc45ea8SJonathan Brassow /* 172ccc45ea8SJonathan Brassow * There is only ever one instance of a particular block 173ccc45ea8SJonathan Brassow * device so we can compare pointers safely. 174ccc45ea8SJonathan Brassow */ 175ccc45ea8SJonathan Brassow return lhs == rhs; 176ccc45ea8SJonathan Brassow } 177ccc45ea8SJonathan Brassow 178028867acSAlasdair G Kergon struct dm_snap_pending_exception { 1791d4989c8SJon Brassow struct dm_exception e; 1801da177e4SLinus Torvalds 1811da177e4SLinus Torvalds /* 1821da177e4SLinus Torvalds * Origin buffers waiting for this to complete are held 1831da177e4SLinus Torvalds * in a bio list 1841da177e4SLinus Torvalds */ 1851da177e4SLinus Torvalds struct bio_list origin_bios; 1861da177e4SLinus Torvalds struct bio_list snapshot_bios; 1871da177e4SLinus Torvalds 1881da177e4SLinus Torvalds /* Pointer back to snapshot context */ 1891da177e4SLinus Torvalds struct dm_snapshot *snap; 1901da177e4SLinus Torvalds 1911da177e4SLinus Torvalds /* 1921da177e4SLinus Torvalds * 1 indicates the exception has already been sent to 1931da177e4SLinus Torvalds * kcopyd. 1941da177e4SLinus Torvalds */ 1951da177e4SLinus Torvalds int started; 196a6e50b40SMikulas Patocka 197230c83afSMikulas Patocka /* There was copying error. */ 198230c83afSMikulas Patocka int copy_error; 199230c83afSMikulas Patocka 200230c83afSMikulas Patocka /* A sequence number, it is used for in-order completion. */ 201230c83afSMikulas Patocka sector_t exception_sequence; 202230c83afSMikulas Patocka 203230c83afSMikulas Patocka struct list_head out_of_order_entry; 204230c83afSMikulas Patocka 205a6e50b40SMikulas Patocka /* 206a6e50b40SMikulas Patocka * For writing a complete chunk, bypassing the copy. 207a6e50b40SMikulas Patocka */ 208a6e50b40SMikulas Patocka struct bio *full_bio; 209a6e50b40SMikulas Patocka bio_end_io_t *full_bio_end_io; 2101da177e4SLinus Torvalds }; 2111da177e4SLinus Torvalds 2121da177e4SLinus Torvalds /* 2131da177e4SLinus Torvalds * Hash table mapping origin volumes to lists of snapshots and 2141da177e4SLinus Torvalds * a lock to protect it 2151da177e4SLinus Torvalds */ 216e18b890bSChristoph Lameter static struct kmem_cache *exception_cache; 217e18b890bSChristoph Lameter static struct kmem_cache *pending_cache; 2181da177e4SLinus Torvalds 219cd45daffSMikulas Patocka struct dm_snap_tracked_chunk { 220cd45daffSMikulas Patocka struct hlist_node node; 221cd45daffSMikulas Patocka chunk_t chunk; 222cd45daffSMikulas Patocka }; 223cd45daffSMikulas Patocka 224ee18026aSMikulas Patocka static void init_tracked_chunk(struct bio *bio) 225ee18026aSMikulas Patocka { 226ee18026aSMikulas Patocka struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk)); 227ee18026aSMikulas Patocka INIT_HLIST_NODE(&c->node); 228ee18026aSMikulas Patocka } 229ee18026aSMikulas Patocka 230ee18026aSMikulas Patocka static bool is_bio_tracked(struct bio *bio) 231ee18026aSMikulas Patocka { 232ee18026aSMikulas Patocka struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk)); 233ee18026aSMikulas Patocka return !hlist_unhashed(&c->node); 234ee18026aSMikulas Patocka } 235ee18026aSMikulas Patocka 236ee18026aSMikulas Patocka static void track_chunk(struct dm_snapshot *s, struct bio *bio, chunk_t chunk) 237cd45daffSMikulas Patocka { 23842bc954fSMikulas Patocka struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk)); 239cd45daffSMikulas Patocka 240cd45daffSMikulas Patocka c->chunk = chunk; 241cd45daffSMikulas Patocka 2429aa0c0e6SMikulas Patocka spin_lock_irq(&s->tracked_chunk_lock); 243cd45daffSMikulas Patocka hlist_add_head(&c->node, 244cd45daffSMikulas Patocka &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)]); 2459aa0c0e6SMikulas Patocka spin_unlock_irq(&s->tracked_chunk_lock); 246cd45daffSMikulas Patocka } 247cd45daffSMikulas Patocka 248ee18026aSMikulas Patocka static void stop_tracking_chunk(struct dm_snapshot *s, struct bio *bio) 249cd45daffSMikulas Patocka { 250ee18026aSMikulas Patocka struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk)); 251cd45daffSMikulas Patocka unsigned long flags; 252cd45daffSMikulas Patocka 253cd45daffSMikulas Patocka spin_lock_irqsave(&s->tracked_chunk_lock, flags); 254cd45daffSMikulas Patocka hlist_del(&c->node); 255cd45daffSMikulas Patocka spin_unlock_irqrestore(&s->tracked_chunk_lock, flags); 256cd45daffSMikulas Patocka } 257cd45daffSMikulas Patocka 258a8d41b59SMikulas Patocka static int __chunk_is_tracked(struct dm_snapshot *s, chunk_t chunk) 259a8d41b59SMikulas Patocka { 260a8d41b59SMikulas Patocka struct dm_snap_tracked_chunk *c; 261a8d41b59SMikulas Patocka int found = 0; 262a8d41b59SMikulas Patocka 263a8d41b59SMikulas Patocka spin_lock_irq(&s->tracked_chunk_lock); 264a8d41b59SMikulas Patocka 265b67bfe0dSSasha Levin hlist_for_each_entry(c, 266a8d41b59SMikulas Patocka &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)], node) { 267a8d41b59SMikulas Patocka if (c->chunk == chunk) { 268a8d41b59SMikulas Patocka found = 1; 269a8d41b59SMikulas Patocka break; 270a8d41b59SMikulas Patocka } 271a8d41b59SMikulas Patocka } 272a8d41b59SMikulas Patocka 273a8d41b59SMikulas Patocka spin_unlock_irq(&s->tracked_chunk_lock); 274a8d41b59SMikulas Patocka 275a8d41b59SMikulas Patocka return found; 276a8d41b59SMikulas Patocka } 277a8d41b59SMikulas Patocka 2781da177e4SLinus Torvalds /* 279615d1eb9SMike Snitzer * This conflicting I/O is extremely improbable in the caller, 280615d1eb9SMike Snitzer * so msleep(1) is sufficient and there is no need for a wait queue. 281615d1eb9SMike Snitzer */ 282615d1eb9SMike Snitzer static void __check_for_conflicting_io(struct dm_snapshot *s, chunk_t chunk) 283615d1eb9SMike Snitzer { 284615d1eb9SMike Snitzer while (__chunk_is_tracked(s, chunk)) 285615d1eb9SMike Snitzer msleep(1); 286615d1eb9SMike Snitzer } 287615d1eb9SMike Snitzer 288615d1eb9SMike Snitzer /* 2891da177e4SLinus Torvalds * One of these per registered origin, held in the snapshot_origins hash 2901da177e4SLinus Torvalds */ 2911da177e4SLinus Torvalds struct origin { 2921da177e4SLinus Torvalds /* The origin device */ 2931da177e4SLinus Torvalds struct block_device *bdev; 2941da177e4SLinus Torvalds 2951da177e4SLinus Torvalds struct list_head hash_list; 2961da177e4SLinus Torvalds 2971da177e4SLinus Torvalds /* List of snapshots for this origin */ 2981da177e4SLinus Torvalds struct list_head snapshots; 2991da177e4SLinus Torvalds }; 3001da177e4SLinus Torvalds 3011da177e4SLinus Torvalds /* 302b735fedeSMikulas Patocka * This structure is allocated for each origin target 303b735fedeSMikulas Patocka */ 304b735fedeSMikulas Patocka struct dm_origin { 305b735fedeSMikulas Patocka struct dm_dev *dev; 306b735fedeSMikulas Patocka struct dm_target *ti; 307b735fedeSMikulas Patocka unsigned split_boundary; 308b735fedeSMikulas Patocka struct list_head hash_list; 309b735fedeSMikulas Patocka }; 310b735fedeSMikulas Patocka 311b735fedeSMikulas Patocka /* 3121da177e4SLinus Torvalds * Size of the hash table for origin volumes. If we make this 3131da177e4SLinus Torvalds * the size of the minors list then it should be nearly perfect 3141da177e4SLinus Torvalds */ 3151da177e4SLinus Torvalds #define ORIGIN_HASH_SIZE 256 3161da177e4SLinus Torvalds #define ORIGIN_MASK 0xFF 3171da177e4SLinus Torvalds static struct list_head *_origins; 318b735fedeSMikulas Patocka static struct list_head *_dm_origins; 3191da177e4SLinus Torvalds static struct rw_semaphore _origins_lock; 3201da177e4SLinus Torvalds 32173dfd078SMikulas Patocka static DECLARE_WAIT_QUEUE_HEAD(_pending_exceptions_done); 32273dfd078SMikulas Patocka static DEFINE_SPINLOCK(_pending_exceptions_done_spinlock); 32373dfd078SMikulas Patocka static uint64_t _pending_exceptions_done_count; 32473dfd078SMikulas Patocka 3251da177e4SLinus Torvalds static int init_origin_hash(void) 3261da177e4SLinus Torvalds { 3271da177e4SLinus Torvalds int i; 3281da177e4SLinus Torvalds 3291da177e4SLinus Torvalds _origins = kmalloc(ORIGIN_HASH_SIZE * sizeof(struct list_head), 3301da177e4SLinus Torvalds GFP_KERNEL); 3311da177e4SLinus Torvalds if (!_origins) { 332b735fedeSMikulas Patocka DMERR("unable to allocate memory for _origins"); 3331da177e4SLinus Torvalds return -ENOMEM; 3341da177e4SLinus Torvalds } 3351da177e4SLinus Torvalds for (i = 0; i < ORIGIN_HASH_SIZE; i++) 3361da177e4SLinus Torvalds INIT_LIST_HEAD(_origins + i); 337b735fedeSMikulas Patocka 338b735fedeSMikulas Patocka _dm_origins = kmalloc(ORIGIN_HASH_SIZE * sizeof(struct list_head), 339b735fedeSMikulas Patocka GFP_KERNEL); 340b735fedeSMikulas Patocka if (!_dm_origins) { 341b735fedeSMikulas Patocka DMERR("unable to allocate memory for _dm_origins"); 342b735fedeSMikulas Patocka kfree(_origins); 343b735fedeSMikulas Patocka return -ENOMEM; 344b735fedeSMikulas Patocka } 345b735fedeSMikulas Patocka for (i = 0; i < ORIGIN_HASH_SIZE; i++) 346b735fedeSMikulas Patocka INIT_LIST_HEAD(_dm_origins + i); 347b735fedeSMikulas Patocka 3481da177e4SLinus Torvalds init_rwsem(&_origins_lock); 3491da177e4SLinus Torvalds 3501da177e4SLinus Torvalds return 0; 3511da177e4SLinus Torvalds } 3521da177e4SLinus Torvalds 3531da177e4SLinus Torvalds static void exit_origin_hash(void) 3541da177e4SLinus Torvalds { 3551da177e4SLinus Torvalds kfree(_origins); 356b735fedeSMikulas Patocka kfree(_dm_origins); 3571da177e4SLinus Torvalds } 3581da177e4SLinus Torvalds 359028867acSAlasdair G Kergon static unsigned origin_hash(struct block_device *bdev) 3601da177e4SLinus Torvalds { 3611da177e4SLinus Torvalds return bdev->bd_dev & ORIGIN_MASK; 3621da177e4SLinus Torvalds } 3631da177e4SLinus Torvalds 3641da177e4SLinus Torvalds static struct origin *__lookup_origin(struct block_device *origin) 3651da177e4SLinus Torvalds { 3661da177e4SLinus Torvalds struct list_head *ol; 3671da177e4SLinus Torvalds struct origin *o; 3681da177e4SLinus Torvalds 3691da177e4SLinus Torvalds ol = &_origins[origin_hash(origin)]; 3701da177e4SLinus Torvalds list_for_each_entry (o, ol, hash_list) 3711da177e4SLinus Torvalds if (bdev_equal(o->bdev, origin)) 3721da177e4SLinus Torvalds return o; 3731da177e4SLinus Torvalds 3741da177e4SLinus Torvalds return NULL; 3751da177e4SLinus Torvalds } 3761da177e4SLinus Torvalds 3771da177e4SLinus Torvalds static void __insert_origin(struct origin *o) 3781da177e4SLinus Torvalds { 3791da177e4SLinus Torvalds struct list_head *sl = &_origins[origin_hash(o->bdev)]; 3801da177e4SLinus Torvalds list_add_tail(&o->hash_list, sl); 3811da177e4SLinus Torvalds } 3821da177e4SLinus Torvalds 383b735fedeSMikulas Patocka static struct dm_origin *__lookup_dm_origin(struct block_device *origin) 384b735fedeSMikulas Patocka { 385b735fedeSMikulas Patocka struct list_head *ol; 386b735fedeSMikulas Patocka struct dm_origin *o; 387b735fedeSMikulas Patocka 388b735fedeSMikulas Patocka ol = &_dm_origins[origin_hash(origin)]; 389b735fedeSMikulas Patocka list_for_each_entry (o, ol, hash_list) 390b735fedeSMikulas Patocka if (bdev_equal(o->dev->bdev, origin)) 391b735fedeSMikulas Patocka return o; 392b735fedeSMikulas Patocka 393b735fedeSMikulas Patocka return NULL; 394b735fedeSMikulas Patocka } 395b735fedeSMikulas Patocka 396b735fedeSMikulas Patocka static void __insert_dm_origin(struct dm_origin *o) 397b735fedeSMikulas Patocka { 398b735fedeSMikulas Patocka struct list_head *sl = &_dm_origins[origin_hash(o->dev->bdev)]; 399b735fedeSMikulas Patocka list_add_tail(&o->hash_list, sl); 400b735fedeSMikulas Patocka } 401b735fedeSMikulas Patocka 402b735fedeSMikulas Patocka static void __remove_dm_origin(struct dm_origin *o) 403b735fedeSMikulas Patocka { 404b735fedeSMikulas Patocka list_del(&o->hash_list); 405b735fedeSMikulas Patocka } 406b735fedeSMikulas Patocka 4071da177e4SLinus Torvalds /* 408c1f0c183SMike Snitzer * _origins_lock must be held when calling this function. 409c1f0c183SMike Snitzer * Returns number of snapshots registered using the supplied cow device, plus: 410c1f0c183SMike Snitzer * snap_src - a snapshot suitable for use as a source of exception handover 411c1f0c183SMike Snitzer * snap_dest - a snapshot capable of receiving exception handover. 4129d3b15c4SMikulas Patocka * snap_merge - an existing snapshot-merge target linked to the same origin. 4139d3b15c4SMikulas Patocka * There can be at most one snapshot-merge target. The parameter is optional. 414c1f0c183SMike Snitzer * 4159d3b15c4SMikulas Patocka * Possible return values and states of snap_src and snap_dest. 416c1f0c183SMike Snitzer * 0: NULL, NULL - first new snapshot 417c1f0c183SMike Snitzer * 1: snap_src, NULL - normal snapshot 418c1f0c183SMike Snitzer * 2: snap_src, snap_dest - waiting for handover 419c1f0c183SMike Snitzer * 2: snap_src, NULL - handed over, waiting for old to be deleted 420c1f0c183SMike Snitzer * 1: NULL, snap_dest - source got destroyed without handover 421c1f0c183SMike Snitzer */ 422c1f0c183SMike Snitzer static int __find_snapshots_sharing_cow(struct dm_snapshot *snap, 423c1f0c183SMike Snitzer struct dm_snapshot **snap_src, 4249d3b15c4SMikulas Patocka struct dm_snapshot **snap_dest, 4259d3b15c4SMikulas Patocka struct dm_snapshot **snap_merge) 426c1f0c183SMike Snitzer { 427c1f0c183SMike Snitzer struct dm_snapshot *s; 428c1f0c183SMike Snitzer struct origin *o; 429c1f0c183SMike Snitzer int count = 0; 430c1f0c183SMike Snitzer int active; 431c1f0c183SMike Snitzer 432c1f0c183SMike Snitzer o = __lookup_origin(snap->origin->bdev); 433c1f0c183SMike Snitzer if (!o) 434c1f0c183SMike Snitzer goto out; 435c1f0c183SMike Snitzer 436c1f0c183SMike Snitzer list_for_each_entry(s, &o->snapshots, list) { 4379d3b15c4SMikulas Patocka if (dm_target_is_snapshot_merge(s->ti) && snap_merge) 4389d3b15c4SMikulas Patocka *snap_merge = s; 439c1f0c183SMike Snitzer if (!bdev_equal(s->cow->bdev, snap->cow->bdev)) 440c1f0c183SMike Snitzer continue; 441c1f0c183SMike Snitzer 442c1f0c183SMike Snitzer down_read(&s->lock); 443c1f0c183SMike Snitzer active = s->active; 444c1f0c183SMike Snitzer up_read(&s->lock); 445c1f0c183SMike Snitzer 446c1f0c183SMike Snitzer if (active) { 447c1f0c183SMike Snitzer if (snap_src) 448c1f0c183SMike Snitzer *snap_src = s; 449c1f0c183SMike Snitzer } else if (snap_dest) 450c1f0c183SMike Snitzer *snap_dest = s; 451c1f0c183SMike Snitzer 452c1f0c183SMike Snitzer count++; 453c1f0c183SMike Snitzer } 454c1f0c183SMike Snitzer 455c1f0c183SMike Snitzer out: 456c1f0c183SMike Snitzer return count; 457c1f0c183SMike Snitzer } 458c1f0c183SMike Snitzer 459c1f0c183SMike Snitzer /* 460c1f0c183SMike Snitzer * On success, returns 1 if this snapshot is a handover destination, 461c1f0c183SMike Snitzer * otherwise returns 0. 462c1f0c183SMike Snitzer */ 463c1f0c183SMike Snitzer static int __validate_exception_handover(struct dm_snapshot *snap) 464c1f0c183SMike Snitzer { 465c1f0c183SMike Snitzer struct dm_snapshot *snap_src = NULL, *snap_dest = NULL; 4669d3b15c4SMikulas Patocka struct dm_snapshot *snap_merge = NULL; 467c1f0c183SMike Snitzer 468c1f0c183SMike Snitzer /* Does snapshot need exceptions handed over to it? */ 4699d3b15c4SMikulas Patocka if ((__find_snapshots_sharing_cow(snap, &snap_src, &snap_dest, 4709d3b15c4SMikulas Patocka &snap_merge) == 2) || 471c1f0c183SMike Snitzer snap_dest) { 472c1f0c183SMike Snitzer snap->ti->error = "Snapshot cow pairing for exception " 473c1f0c183SMike Snitzer "table handover failed"; 474c1f0c183SMike Snitzer return -EINVAL; 475c1f0c183SMike Snitzer } 476c1f0c183SMike Snitzer 477c1f0c183SMike Snitzer /* 478c1f0c183SMike Snitzer * If no snap_src was found, snap cannot become a handover 479c1f0c183SMike Snitzer * destination. 480c1f0c183SMike Snitzer */ 481c1f0c183SMike Snitzer if (!snap_src) 482c1f0c183SMike Snitzer return 0; 483c1f0c183SMike Snitzer 4849d3b15c4SMikulas Patocka /* 4859d3b15c4SMikulas Patocka * Non-snapshot-merge handover? 4869d3b15c4SMikulas Patocka */ 4879d3b15c4SMikulas Patocka if (!dm_target_is_snapshot_merge(snap->ti)) 4889d3b15c4SMikulas Patocka return 1; 4899d3b15c4SMikulas Patocka 4909d3b15c4SMikulas Patocka /* 4919d3b15c4SMikulas Patocka * Do not allow more than one merging snapshot. 4929d3b15c4SMikulas Patocka */ 4939d3b15c4SMikulas Patocka if (snap_merge) { 4949d3b15c4SMikulas Patocka snap->ti->error = "A snapshot is already merging."; 4959d3b15c4SMikulas Patocka return -EINVAL; 4969d3b15c4SMikulas Patocka } 4979d3b15c4SMikulas Patocka 4981e03f97eSMikulas Patocka if (!snap_src->store->type->prepare_merge || 4991e03f97eSMikulas Patocka !snap_src->store->type->commit_merge) { 5001e03f97eSMikulas Patocka snap->ti->error = "Snapshot exception store does not " 5011e03f97eSMikulas Patocka "support snapshot-merge."; 5021e03f97eSMikulas Patocka return -EINVAL; 5031e03f97eSMikulas Patocka } 5041e03f97eSMikulas Patocka 505c1f0c183SMike Snitzer return 1; 506c1f0c183SMike Snitzer } 507c1f0c183SMike Snitzer 508c1f0c183SMike Snitzer static void __insert_snapshot(struct origin *o, struct dm_snapshot *s) 509c1f0c183SMike Snitzer { 510c1f0c183SMike Snitzer struct dm_snapshot *l; 511c1f0c183SMike Snitzer 512c1f0c183SMike Snitzer /* Sort the list according to chunk size, largest-first smallest-last */ 513c1f0c183SMike Snitzer list_for_each_entry(l, &o->snapshots, list) 514c1f0c183SMike Snitzer if (l->store->chunk_size < s->store->chunk_size) 515c1f0c183SMike Snitzer break; 516c1f0c183SMike Snitzer list_add_tail(&s->list, &l->list); 517c1f0c183SMike Snitzer } 518c1f0c183SMike Snitzer 519c1f0c183SMike Snitzer /* 5201da177e4SLinus Torvalds * Make a note of the snapshot and its origin so we can look it 5211da177e4SLinus Torvalds * up when the origin has a write on it. 522c1f0c183SMike Snitzer * 523c1f0c183SMike Snitzer * Also validate snapshot exception store handovers. 524c1f0c183SMike Snitzer * On success, returns 1 if this registration is a handover destination, 525c1f0c183SMike Snitzer * otherwise returns 0. 5261da177e4SLinus Torvalds */ 5271da177e4SLinus Torvalds static int register_snapshot(struct dm_snapshot *snap) 5281da177e4SLinus Torvalds { 529c1f0c183SMike Snitzer struct origin *o, *new_o = NULL; 5301da177e4SLinus Torvalds struct block_device *bdev = snap->origin->bdev; 531c1f0c183SMike Snitzer int r = 0; 5321da177e4SLinus Torvalds 53360c856c8SMikulas Patocka new_o = kmalloc(sizeof(*new_o), GFP_KERNEL); 53460c856c8SMikulas Patocka if (!new_o) 53560c856c8SMikulas Patocka return -ENOMEM; 53660c856c8SMikulas Patocka 5371da177e4SLinus Torvalds down_write(&_origins_lock); 5381da177e4SLinus Torvalds 539c1f0c183SMike Snitzer r = __validate_exception_handover(snap); 540c1f0c183SMike Snitzer if (r < 0) { 541c1f0c183SMike Snitzer kfree(new_o); 542c1f0c183SMike Snitzer goto out; 543c1f0c183SMike Snitzer } 544c1f0c183SMike Snitzer 545c1f0c183SMike Snitzer o = __lookup_origin(bdev); 54660c856c8SMikulas Patocka if (o) 54760c856c8SMikulas Patocka kfree(new_o); 54860c856c8SMikulas Patocka else { 5491da177e4SLinus Torvalds /* New origin */ 55060c856c8SMikulas Patocka o = new_o; 5511da177e4SLinus Torvalds 5521da177e4SLinus Torvalds /* Initialise the struct */ 5531da177e4SLinus Torvalds INIT_LIST_HEAD(&o->snapshots); 5541da177e4SLinus Torvalds o->bdev = bdev; 5551da177e4SLinus Torvalds 5561da177e4SLinus Torvalds __insert_origin(o); 5571da177e4SLinus Torvalds } 5581da177e4SLinus Torvalds 559c1f0c183SMike Snitzer __insert_snapshot(o, snap); 560c1f0c183SMike Snitzer 561c1f0c183SMike Snitzer out: 562c1f0c183SMike Snitzer up_write(&_origins_lock); 563c1f0c183SMike Snitzer 564c1f0c183SMike Snitzer return r; 565c1f0c183SMike Snitzer } 566c1f0c183SMike Snitzer 567c1f0c183SMike Snitzer /* 568c1f0c183SMike Snitzer * Move snapshot to correct place in list according to chunk size. 569c1f0c183SMike Snitzer */ 570c1f0c183SMike Snitzer static void reregister_snapshot(struct dm_snapshot *s) 571c1f0c183SMike Snitzer { 572c1f0c183SMike Snitzer struct block_device *bdev = s->origin->bdev; 573c1f0c183SMike Snitzer 574c1f0c183SMike Snitzer down_write(&_origins_lock); 575c1f0c183SMike Snitzer 576c1f0c183SMike Snitzer list_del(&s->list); 577c1f0c183SMike Snitzer __insert_snapshot(__lookup_origin(bdev), s); 5781da177e4SLinus Torvalds 5791da177e4SLinus Torvalds up_write(&_origins_lock); 5801da177e4SLinus Torvalds } 5811da177e4SLinus Torvalds 5821da177e4SLinus Torvalds static void unregister_snapshot(struct dm_snapshot *s) 5831da177e4SLinus Torvalds { 5841da177e4SLinus Torvalds struct origin *o; 5851da177e4SLinus Torvalds 5861da177e4SLinus Torvalds down_write(&_origins_lock); 5871da177e4SLinus Torvalds o = __lookup_origin(s->origin->bdev); 5881da177e4SLinus Torvalds 5891da177e4SLinus Torvalds list_del(&s->list); 590c1f0c183SMike Snitzer if (o && list_empty(&o->snapshots)) { 5911da177e4SLinus Torvalds list_del(&o->hash_list); 5921da177e4SLinus Torvalds kfree(o); 5931da177e4SLinus Torvalds } 5941da177e4SLinus Torvalds 5951da177e4SLinus Torvalds up_write(&_origins_lock); 5961da177e4SLinus Torvalds } 5971da177e4SLinus Torvalds 5981da177e4SLinus Torvalds /* 5991da177e4SLinus Torvalds * Implementation of the exception hash tables. 600d74f81f8SMilan Broz * The lowest hash_shift bits of the chunk number are ignored, allowing 601d74f81f8SMilan Broz * some consecutive chunks to be grouped together. 6021da177e4SLinus Torvalds */ 6033510cb94SJon Brassow static int dm_exception_table_init(struct dm_exception_table *et, 6043510cb94SJon Brassow uint32_t size, unsigned hash_shift) 6051da177e4SLinus Torvalds { 6061da177e4SLinus Torvalds unsigned int i; 6071da177e4SLinus Torvalds 608d74f81f8SMilan Broz et->hash_shift = hash_shift; 6091da177e4SLinus Torvalds et->hash_mask = size - 1; 6101da177e4SLinus Torvalds et->table = dm_vcalloc(size, sizeof(struct list_head)); 6111da177e4SLinus Torvalds if (!et->table) 6121da177e4SLinus Torvalds return -ENOMEM; 6131da177e4SLinus Torvalds 6141da177e4SLinus Torvalds for (i = 0; i < size; i++) 6151da177e4SLinus Torvalds INIT_LIST_HEAD(et->table + i); 6161da177e4SLinus Torvalds 6171da177e4SLinus Torvalds return 0; 6181da177e4SLinus Torvalds } 6191da177e4SLinus Torvalds 6203510cb94SJon Brassow static void dm_exception_table_exit(struct dm_exception_table *et, 621191437a5SJon Brassow struct kmem_cache *mem) 6221da177e4SLinus Torvalds { 6231da177e4SLinus Torvalds struct list_head *slot; 6241d4989c8SJon Brassow struct dm_exception *ex, *next; 6251da177e4SLinus Torvalds int i, size; 6261da177e4SLinus Torvalds 6271da177e4SLinus Torvalds size = et->hash_mask + 1; 6281da177e4SLinus Torvalds for (i = 0; i < size; i++) { 6291da177e4SLinus Torvalds slot = et->table + i; 6301da177e4SLinus Torvalds 6311da177e4SLinus Torvalds list_for_each_entry_safe (ex, next, slot, hash_list) 6321da177e4SLinus Torvalds kmem_cache_free(mem, ex); 6331da177e4SLinus Torvalds } 6341da177e4SLinus Torvalds 6351da177e4SLinus Torvalds vfree(et->table); 6361da177e4SLinus Torvalds } 6371da177e4SLinus Torvalds 638191437a5SJon Brassow static uint32_t exception_hash(struct dm_exception_table *et, chunk_t chunk) 6391da177e4SLinus Torvalds { 640d74f81f8SMilan Broz return (chunk >> et->hash_shift) & et->hash_mask; 6411da177e4SLinus Torvalds } 6421da177e4SLinus Torvalds 6433510cb94SJon Brassow static void dm_remove_exception(struct dm_exception *e) 6441da177e4SLinus Torvalds { 6451da177e4SLinus Torvalds list_del(&e->hash_list); 6461da177e4SLinus Torvalds } 6471da177e4SLinus Torvalds 6481da177e4SLinus Torvalds /* 6491da177e4SLinus Torvalds * Return the exception data for a sector, or NULL if not 6501da177e4SLinus Torvalds * remapped. 6511da177e4SLinus Torvalds */ 6523510cb94SJon Brassow static struct dm_exception *dm_lookup_exception(struct dm_exception_table *et, 6531da177e4SLinus Torvalds chunk_t chunk) 6541da177e4SLinus Torvalds { 6551da177e4SLinus Torvalds struct list_head *slot; 6561d4989c8SJon Brassow struct dm_exception *e; 6571da177e4SLinus Torvalds 6581da177e4SLinus Torvalds slot = &et->table[exception_hash(et, chunk)]; 6591da177e4SLinus Torvalds list_for_each_entry (e, slot, hash_list) 660d74f81f8SMilan Broz if (chunk >= e->old_chunk && 661d74f81f8SMilan Broz chunk <= e->old_chunk + dm_consecutive_chunk_count(e)) 6621da177e4SLinus Torvalds return e; 6631da177e4SLinus Torvalds 6641da177e4SLinus Torvalds return NULL; 6651da177e4SLinus Torvalds } 6661da177e4SLinus Torvalds 667119bc547SMikulas Patocka static struct dm_exception *alloc_completed_exception(gfp_t gfp) 6681da177e4SLinus Torvalds { 6691d4989c8SJon Brassow struct dm_exception *e; 6701da177e4SLinus Torvalds 671119bc547SMikulas Patocka e = kmem_cache_alloc(exception_cache, gfp); 672119bc547SMikulas Patocka if (!e && gfp == GFP_NOIO) 6731da177e4SLinus Torvalds e = kmem_cache_alloc(exception_cache, GFP_ATOMIC); 6741da177e4SLinus Torvalds 6751da177e4SLinus Torvalds return e; 6761da177e4SLinus Torvalds } 6771da177e4SLinus Torvalds 6783510cb94SJon Brassow static void free_completed_exception(struct dm_exception *e) 6791da177e4SLinus Torvalds { 6801da177e4SLinus Torvalds kmem_cache_free(exception_cache, e); 6811da177e4SLinus Torvalds } 6821da177e4SLinus Torvalds 68392e86812SMikulas Patocka static struct dm_snap_pending_exception *alloc_pending_exception(struct dm_snapshot *s) 6841da177e4SLinus Torvalds { 68592e86812SMikulas Patocka struct dm_snap_pending_exception *pe = mempool_alloc(s->pending_pool, 68692e86812SMikulas Patocka GFP_NOIO); 68792e86812SMikulas Patocka 688879129d2SMikulas Patocka atomic_inc(&s->pending_exceptions_count); 68992e86812SMikulas Patocka pe->snap = s; 69092e86812SMikulas Patocka 69192e86812SMikulas Patocka return pe; 6921da177e4SLinus Torvalds } 6931da177e4SLinus Torvalds 694028867acSAlasdair G Kergon static void free_pending_exception(struct dm_snap_pending_exception *pe) 6951da177e4SLinus Torvalds { 696879129d2SMikulas Patocka struct dm_snapshot *s = pe->snap; 697879129d2SMikulas Patocka 698879129d2SMikulas Patocka mempool_free(pe, s->pending_pool); 6994e857c58SPeter Zijlstra smp_mb__before_atomic(); 700879129d2SMikulas Patocka atomic_dec(&s->pending_exceptions_count); 7011da177e4SLinus Torvalds } 7021da177e4SLinus Torvalds 7033510cb94SJon Brassow static void dm_insert_exception(struct dm_exception_table *eh, 7041d4989c8SJon Brassow struct dm_exception *new_e) 705d74f81f8SMilan Broz { 706d74f81f8SMilan Broz struct list_head *l; 7071d4989c8SJon Brassow struct dm_exception *e = NULL; 708d74f81f8SMilan Broz 709d74f81f8SMilan Broz l = &eh->table[exception_hash(eh, new_e->old_chunk)]; 710d74f81f8SMilan Broz 711d74f81f8SMilan Broz /* Add immediately if this table doesn't support consecutive chunks */ 712d74f81f8SMilan Broz if (!eh->hash_shift) 713d74f81f8SMilan Broz goto out; 714d74f81f8SMilan Broz 715d74f81f8SMilan Broz /* List is ordered by old_chunk */ 716d74f81f8SMilan Broz list_for_each_entry_reverse(e, l, hash_list) { 717d74f81f8SMilan Broz /* Insert after an existing chunk? */ 718d74f81f8SMilan Broz if (new_e->old_chunk == (e->old_chunk + 719d74f81f8SMilan Broz dm_consecutive_chunk_count(e) + 1) && 720d74f81f8SMilan Broz new_e->new_chunk == (dm_chunk_number(e->new_chunk) + 721d74f81f8SMilan Broz dm_consecutive_chunk_count(e) + 1)) { 722d74f81f8SMilan Broz dm_consecutive_chunk_count_inc(e); 7233510cb94SJon Brassow free_completed_exception(new_e); 724d74f81f8SMilan Broz return; 725d74f81f8SMilan Broz } 726d74f81f8SMilan Broz 727d74f81f8SMilan Broz /* Insert before an existing chunk? */ 728d74f81f8SMilan Broz if (new_e->old_chunk == (e->old_chunk - 1) && 729d74f81f8SMilan Broz new_e->new_chunk == (dm_chunk_number(e->new_chunk) - 1)) { 730d74f81f8SMilan Broz dm_consecutive_chunk_count_inc(e); 731d74f81f8SMilan Broz e->old_chunk--; 732d74f81f8SMilan Broz e->new_chunk--; 7333510cb94SJon Brassow free_completed_exception(new_e); 734d74f81f8SMilan Broz return; 735d74f81f8SMilan Broz } 736d74f81f8SMilan Broz 737d74f81f8SMilan Broz if (new_e->old_chunk > e->old_chunk) 738d74f81f8SMilan Broz break; 739d74f81f8SMilan Broz } 740d74f81f8SMilan Broz 741d74f81f8SMilan Broz out: 742d74f81f8SMilan Broz list_add(&new_e->hash_list, e ? &e->hash_list : l); 743d74f81f8SMilan Broz } 744d74f81f8SMilan Broz 745a159c1acSJonathan Brassow /* 746a159c1acSJonathan Brassow * Callback used by the exception stores to load exceptions when 747a159c1acSJonathan Brassow * initialising. 748a159c1acSJonathan Brassow */ 749a159c1acSJonathan Brassow static int dm_add_exception(void *context, chunk_t old, chunk_t new) 7501da177e4SLinus Torvalds { 751a159c1acSJonathan Brassow struct dm_snapshot *s = context; 7521d4989c8SJon Brassow struct dm_exception *e; 7531da177e4SLinus Torvalds 754119bc547SMikulas Patocka e = alloc_completed_exception(GFP_KERNEL); 7551da177e4SLinus Torvalds if (!e) 7561da177e4SLinus Torvalds return -ENOMEM; 7571da177e4SLinus Torvalds 7581da177e4SLinus Torvalds e->old_chunk = old; 759d74f81f8SMilan Broz 760d74f81f8SMilan Broz /* Consecutive_count is implicitly initialised to zero */ 7611da177e4SLinus Torvalds e->new_chunk = new; 762d74f81f8SMilan Broz 7633510cb94SJon Brassow dm_insert_exception(&s->complete, e); 764d74f81f8SMilan Broz 7651da177e4SLinus Torvalds return 0; 7661da177e4SLinus Torvalds } 7671da177e4SLinus Torvalds 7687e201b35SMikulas Patocka /* 7697e201b35SMikulas Patocka * Return a minimum chunk size of all snapshots that have the specified origin. 7707e201b35SMikulas Patocka * Return zero if the origin has no snapshots. 7717e201b35SMikulas Patocka */ 772542f9038SMike Snitzer static uint32_t __minimum_chunk_size(struct origin *o) 7737e201b35SMikulas Patocka { 7747e201b35SMikulas Patocka struct dm_snapshot *snap; 7757e201b35SMikulas Patocka unsigned chunk_size = 0; 7767e201b35SMikulas Patocka 7777e201b35SMikulas Patocka if (o) 7787e201b35SMikulas Patocka list_for_each_entry(snap, &o->snapshots, list) 7797e201b35SMikulas Patocka chunk_size = min_not_zero(chunk_size, 7807e201b35SMikulas Patocka snap->store->chunk_size); 7817e201b35SMikulas Patocka 782542f9038SMike Snitzer return (uint32_t) chunk_size; 7837e201b35SMikulas Patocka } 7847e201b35SMikulas Patocka 7851da177e4SLinus Torvalds /* 7861da177e4SLinus Torvalds * Hard coded magic. 7871da177e4SLinus Torvalds */ 7881da177e4SLinus Torvalds static int calc_max_buckets(void) 7891da177e4SLinus Torvalds { 7901da177e4SLinus Torvalds /* use a fixed size of 2MB */ 7911da177e4SLinus Torvalds unsigned long mem = 2 * 1024 * 1024; 7921da177e4SLinus Torvalds mem /= sizeof(struct list_head); 7931da177e4SLinus Torvalds 7941da177e4SLinus Torvalds return mem; 7951da177e4SLinus Torvalds } 7961da177e4SLinus Torvalds 7971da177e4SLinus Torvalds /* 7981da177e4SLinus Torvalds * Allocate room for a suitable hash table. 7991da177e4SLinus Torvalds */ 800fee1998eSJonathan Brassow static int init_hash_tables(struct dm_snapshot *s) 8011da177e4SLinus Torvalds { 80260e356f3SMikulas Patocka sector_t hash_size, cow_dev_size, max_buckets; 8031da177e4SLinus Torvalds 8041da177e4SLinus Torvalds /* 8051da177e4SLinus Torvalds * Calculate based on the size of the original volume or 8061da177e4SLinus Torvalds * the COW volume... 8071da177e4SLinus Torvalds */ 808fc56f6fbSMike Snitzer cow_dev_size = get_dev_size(s->cow->bdev); 8091da177e4SLinus Torvalds max_buckets = calc_max_buckets(); 8101da177e4SLinus Torvalds 81160e356f3SMikulas Patocka hash_size = cow_dev_size >> s->store->chunk_shift; 8121da177e4SLinus Torvalds hash_size = min(hash_size, max_buckets); 8131da177e4SLinus Torvalds 8148e87b9b8SMikulas Patocka if (hash_size < 64) 8158e87b9b8SMikulas Patocka hash_size = 64; 8168defd830SRobert P. J. Day hash_size = rounddown_pow_of_two(hash_size); 8173510cb94SJon Brassow if (dm_exception_table_init(&s->complete, hash_size, 818d74f81f8SMilan Broz DM_CHUNK_CONSECUTIVE_BITS)) 8191da177e4SLinus Torvalds return -ENOMEM; 8201da177e4SLinus Torvalds 8211da177e4SLinus Torvalds /* 8221da177e4SLinus Torvalds * Allocate hash table for in-flight exceptions 8231da177e4SLinus Torvalds * Make this smaller than the real hash table 8241da177e4SLinus Torvalds */ 8251da177e4SLinus Torvalds hash_size >>= 3; 8261da177e4SLinus Torvalds if (hash_size < 64) 8271da177e4SLinus Torvalds hash_size = 64; 8281da177e4SLinus Torvalds 8293510cb94SJon Brassow if (dm_exception_table_init(&s->pending, hash_size, 0)) { 8303510cb94SJon Brassow dm_exception_table_exit(&s->complete, exception_cache); 8311da177e4SLinus Torvalds return -ENOMEM; 8321da177e4SLinus Torvalds } 8331da177e4SLinus Torvalds 8341da177e4SLinus Torvalds return 0; 8351da177e4SLinus Torvalds } 8361da177e4SLinus Torvalds 8371e03f97eSMikulas Patocka static void merge_shutdown(struct dm_snapshot *s) 8381e03f97eSMikulas Patocka { 8391e03f97eSMikulas Patocka clear_bit_unlock(RUNNING_MERGE, &s->state_bits); 8404e857c58SPeter Zijlstra smp_mb__after_atomic(); 8411e03f97eSMikulas Patocka wake_up_bit(&s->state_bits, RUNNING_MERGE); 8421e03f97eSMikulas Patocka } 8431e03f97eSMikulas Patocka 8449fe86254SMikulas Patocka static struct bio *__release_queued_bios_after_merge(struct dm_snapshot *s) 8459fe86254SMikulas Patocka { 8469fe86254SMikulas Patocka s->first_merging_chunk = 0; 8479fe86254SMikulas Patocka s->num_merging_chunks = 0; 8489fe86254SMikulas Patocka 8499fe86254SMikulas Patocka return bio_list_get(&s->bios_queued_during_merge); 8509fe86254SMikulas Patocka } 8519fe86254SMikulas Patocka 8521e03f97eSMikulas Patocka /* 8531e03f97eSMikulas Patocka * Remove one chunk from the index of completed exceptions. 8541e03f97eSMikulas Patocka */ 8551e03f97eSMikulas Patocka static int __remove_single_exception_chunk(struct dm_snapshot *s, 8561e03f97eSMikulas Patocka chunk_t old_chunk) 8571e03f97eSMikulas Patocka { 8581e03f97eSMikulas Patocka struct dm_exception *e; 8591e03f97eSMikulas Patocka 8601e03f97eSMikulas Patocka e = dm_lookup_exception(&s->complete, old_chunk); 8611e03f97eSMikulas Patocka if (!e) { 8621e03f97eSMikulas Patocka DMERR("Corruption detected: exception for block %llu is " 8631e03f97eSMikulas Patocka "on disk but not in memory", 8641e03f97eSMikulas Patocka (unsigned long long)old_chunk); 8651e03f97eSMikulas Patocka return -EINVAL; 8661e03f97eSMikulas Patocka } 8671e03f97eSMikulas Patocka 8681e03f97eSMikulas Patocka /* 8691e03f97eSMikulas Patocka * If this is the only chunk using this exception, remove exception. 8701e03f97eSMikulas Patocka */ 8711e03f97eSMikulas Patocka if (!dm_consecutive_chunk_count(e)) { 8721e03f97eSMikulas Patocka dm_remove_exception(e); 8731e03f97eSMikulas Patocka free_completed_exception(e); 8741e03f97eSMikulas Patocka return 0; 8751e03f97eSMikulas Patocka } 8761e03f97eSMikulas Patocka 8771e03f97eSMikulas Patocka /* 8781e03f97eSMikulas Patocka * The chunk may be either at the beginning or the end of a 8791e03f97eSMikulas Patocka * group of consecutive chunks - never in the middle. We are 8801e03f97eSMikulas Patocka * removing chunks in the opposite order to that in which they 8811e03f97eSMikulas Patocka * were added, so this should always be true. 8821e03f97eSMikulas Patocka * Decrement the consecutive chunk counter and adjust the 8831e03f97eSMikulas Patocka * starting point if necessary. 8841e03f97eSMikulas Patocka */ 8851e03f97eSMikulas Patocka if (old_chunk == e->old_chunk) { 8861e03f97eSMikulas Patocka e->old_chunk++; 8871e03f97eSMikulas Patocka e->new_chunk++; 8881e03f97eSMikulas Patocka } else if (old_chunk != e->old_chunk + 8891e03f97eSMikulas Patocka dm_consecutive_chunk_count(e)) { 8901e03f97eSMikulas Patocka DMERR("Attempt to merge block %llu from the " 8911e03f97eSMikulas Patocka "middle of a chunk range [%llu - %llu]", 8921e03f97eSMikulas Patocka (unsigned long long)old_chunk, 8931e03f97eSMikulas Patocka (unsigned long long)e->old_chunk, 8941e03f97eSMikulas Patocka (unsigned long long) 8951e03f97eSMikulas Patocka e->old_chunk + dm_consecutive_chunk_count(e)); 8961e03f97eSMikulas Patocka return -EINVAL; 8971e03f97eSMikulas Patocka } 8981e03f97eSMikulas Patocka 8991e03f97eSMikulas Patocka dm_consecutive_chunk_count_dec(e); 9001e03f97eSMikulas Patocka 9011e03f97eSMikulas Patocka return 0; 9021e03f97eSMikulas Patocka } 9031e03f97eSMikulas Patocka 9049fe86254SMikulas Patocka static void flush_bios(struct bio *bio); 9059fe86254SMikulas Patocka 9069fe86254SMikulas Patocka static int remove_single_exception_chunk(struct dm_snapshot *s) 9071e03f97eSMikulas Patocka { 9089fe86254SMikulas Patocka struct bio *b = NULL; 9099fe86254SMikulas Patocka int r; 9109fe86254SMikulas Patocka chunk_t old_chunk = s->first_merging_chunk + s->num_merging_chunks - 1; 9111e03f97eSMikulas Patocka 9121e03f97eSMikulas Patocka down_write(&s->lock); 9139fe86254SMikulas Patocka 9149fe86254SMikulas Patocka /* 9159fe86254SMikulas Patocka * Process chunks (and associated exceptions) in reverse order 9169fe86254SMikulas Patocka * so that dm_consecutive_chunk_count_dec() accounting works. 9179fe86254SMikulas Patocka */ 9189fe86254SMikulas Patocka do { 9191e03f97eSMikulas Patocka r = __remove_single_exception_chunk(s, old_chunk); 9209fe86254SMikulas Patocka if (r) 9219fe86254SMikulas Patocka goto out; 9229fe86254SMikulas Patocka } while (old_chunk-- > s->first_merging_chunk); 9239fe86254SMikulas Patocka 9249fe86254SMikulas Patocka b = __release_queued_bios_after_merge(s); 9259fe86254SMikulas Patocka 9269fe86254SMikulas Patocka out: 9271e03f97eSMikulas Patocka up_write(&s->lock); 9289fe86254SMikulas Patocka if (b) 9299fe86254SMikulas Patocka flush_bios(b); 9301e03f97eSMikulas Patocka 9311e03f97eSMikulas Patocka return r; 9321e03f97eSMikulas Patocka } 9331e03f97eSMikulas Patocka 93473dfd078SMikulas Patocka static int origin_write_extent(struct dm_snapshot *merging_snap, 93573dfd078SMikulas Patocka sector_t sector, unsigned chunk_size); 93673dfd078SMikulas Patocka 9371e03f97eSMikulas Patocka static void merge_callback(int read_err, unsigned long write_err, 9381e03f97eSMikulas Patocka void *context); 9391e03f97eSMikulas Patocka 94073dfd078SMikulas Patocka static uint64_t read_pending_exceptions_done_count(void) 94173dfd078SMikulas Patocka { 94273dfd078SMikulas Patocka uint64_t pending_exceptions_done; 94373dfd078SMikulas Patocka 94473dfd078SMikulas Patocka spin_lock(&_pending_exceptions_done_spinlock); 94573dfd078SMikulas Patocka pending_exceptions_done = _pending_exceptions_done_count; 94673dfd078SMikulas Patocka spin_unlock(&_pending_exceptions_done_spinlock); 94773dfd078SMikulas Patocka 94873dfd078SMikulas Patocka return pending_exceptions_done; 94973dfd078SMikulas Patocka } 95073dfd078SMikulas Patocka 95173dfd078SMikulas Patocka static void increment_pending_exceptions_done_count(void) 95273dfd078SMikulas Patocka { 95373dfd078SMikulas Patocka spin_lock(&_pending_exceptions_done_spinlock); 95473dfd078SMikulas Patocka _pending_exceptions_done_count++; 95573dfd078SMikulas Patocka spin_unlock(&_pending_exceptions_done_spinlock); 95673dfd078SMikulas Patocka 95773dfd078SMikulas Patocka wake_up_all(&_pending_exceptions_done); 95873dfd078SMikulas Patocka } 95973dfd078SMikulas Patocka 9601e03f97eSMikulas Patocka static void snapshot_merge_next_chunks(struct dm_snapshot *s) 9611e03f97eSMikulas Patocka { 9628a2d5286SMike Snitzer int i, linear_chunks; 9631e03f97eSMikulas Patocka chunk_t old_chunk, new_chunk; 9641e03f97eSMikulas Patocka struct dm_io_region src, dest; 9658a2d5286SMike Snitzer sector_t io_size; 96673dfd078SMikulas Patocka uint64_t previous_count; 9671e03f97eSMikulas Patocka 9681e03f97eSMikulas Patocka BUG_ON(!test_bit(RUNNING_MERGE, &s->state_bits)); 9691e03f97eSMikulas Patocka if (unlikely(test_bit(SHUTDOWN_MERGE, &s->state_bits))) 9701e03f97eSMikulas Patocka goto shut; 9711e03f97eSMikulas Patocka 9721e03f97eSMikulas Patocka /* 9731e03f97eSMikulas Patocka * valid flag never changes during merge, so no lock required. 9741e03f97eSMikulas Patocka */ 9751e03f97eSMikulas Patocka if (!s->valid) { 9761e03f97eSMikulas Patocka DMERR("Snapshot is invalid: can't merge"); 9771e03f97eSMikulas Patocka goto shut; 9781e03f97eSMikulas Patocka } 9791e03f97eSMikulas Patocka 9808a2d5286SMike Snitzer linear_chunks = s->store->type->prepare_merge(s->store, &old_chunk, 9818a2d5286SMike Snitzer &new_chunk); 9828a2d5286SMike Snitzer if (linear_chunks <= 0) { 983d8ddb1cfSMike Snitzer if (linear_chunks < 0) { 9841e03f97eSMikulas Patocka DMERR("Read error in exception store: " 9851e03f97eSMikulas Patocka "shutting down merge"); 986d8ddb1cfSMike Snitzer down_write(&s->lock); 987d8ddb1cfSMike Snitzer s->merge_failed = 1; 988d8ddb1cfSMike Snitzer up_write(&s->lock); 989d8ddb1cfSMike Snitzer } 9901e03f97eSMikulas Patocka goto shut; 9911e03f97eSMikulas Patocka } 9921e03f97eSMikulas Patocka 9938a2d5286SMike Snitzer /* Adjust old_chunk and new_chunk to reflect start of linear region */ 9948a2d5286SMike Snitzer old_chunk = old_chunk + 1 - linear_chunks; 9958a2d5286SMike Snitzer new_chunk = new_chunk + 1 - linear_chunks; 9968a2d5286SMike Snitzer 9978a2d5286SMike Snitzer /* 9988a2d5286SMike Snitzer * Use one (potentially large) I/O to copy all 'linear_chunks' 9998a2d5286SMike Snitzer * from the exception store to the origin 10008a2d5286SMike Snitzer */ 10018a2d5286SMike Snitzer io_size = linear_chunks * s->store->chunk_size; 10021e03f97eSMikulas Patocka 10031e03f97eSMikulas Patocka dest.bdev = s->origin->bdev; 10041e03f97eSMikulas Patocka dest.sector = chunk_to_sector(s->store, old_chunk); 10058a2d5286SMike Snitzer dest.count = min(io_size, get_dev_size(dest.bdev) - dest.sector); 10061e03f97eSMikulas Patocka 10071e03f97eSMikulas Patocka src.bdev = s->cow->bdev; 10081e03f97eSMikulas Patocka src.sector = chunk_to_sector(s->store, new_chunk); 10091e03f97eSMikulas Patocka src.count = dest.count; 10101e03f97eSMikulas Patocka 101173dfd078SMikulas Patocka /* 101273dfd078SMikulas Patocka * Reallocate any exceptions needed in other snapshots then 101373dfd078SMikulas Patocka * wait for the pending exceptions to complete. 101473dfd078SMikulas Patocka * Each time any pending exception (globally on the system) 101573dfd078SMikulas Patocka * completes we are woken and repeat the process to find out 101673dfd078SMikulas Patocka * if we can proceed. While this may not seem a particularly 101773dfd078SMikulas Patocka * efficient algorithm, it is not expected to have any 101873dfd078SMikulas Patocka * significant impact on performance. 101973dfd078SMikulas Patocka */ 102073dfd078SMikulas Patocka previous_count = read_pending_exceptions_done_count(); 10218a2d5286SMike Snitzer while (origin_write_extent(s, dest.sector, io_size)) { 102273dfd078SMikulas Patocka wait_event(_pending_exceptions_done, 102373dfd078SMikulas Patocka (read_pending_exceptions_done_count() != 102473dfd078SMikulas Patocka previous_count)); 102573dfd078SMikulas Patocka /* Retry after the wait, until all exceptions are done. */ 102673dfd078SMikulas Patocka previous_count = read_pending_exceptions_done_count(); 102773dfd078SMikulas Patocka } 102873dfd078SMikulas Patocka 10299fe86254SMikulas Patocka down_write(&s->lock); 10309fe86254SMikulas Patocka s->first_merging_chunk = old_chunk; 10318a2d5286SMike Snitzer s->num_merging_chunks = linear_chunks; 10329fe86254SMikulas Patocka up_write(&s->lock); 10339fe86254SMikulas Patocka 10348a2d5286SMike Snitzer /* Wait until writes to all 'linear_chunks' drain */ 10358a2d5286SMike Snitzer for (i = 0; i < linear_chunks; i++) 10368a2d5286SMike Snitzer __check_for_conflicting_io(s, old_chunk + i); 10379fe86254SMikulas Patocka 10381e03f97eSMikulas Patocka dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, merge_callback, s); 10391e03f97eSMikulas Patocka return; 10401e03f97eSMikulas Patocka 10411e03f97eSMikulas Patocka shut: 10421e03f97eSMikulas Patocka merge_shutdown(s); 10431e03f97eSMikulas Patocka } 10441e03f97eSMikulas Patocka 10459fe86254SMikulas Patocka static void error_bios(struct bio *bio); 10469fe86254SMikulas Patocka 10471e03f97eSMikulas Patocka static void merge_callback(int read_err, unsigned long write_err, void *context) 10481e03f97eSMikulas Patocka { 10491e03f97eSMikulas Patocka struct dm_snapshot *s = context; 10509fe86254SMikulas Patocka struct bio *b = NULL; 10511e03f97eSMikulas Patocka 10521e03f97eSMikulas Patocka if (read_err || write_err) { 10531e03f97eSMikulas Patocka if (read_err) 10541e03f97eSMikulas Patocka DMERR("Read error: shutting down merge."); 10551e03f97eSMikulas Patocka else 10561e03f97eSMikulas Patocka DMERR("Write error: shutting down merge."); 10571e03f97eSMikulas Patocka goto shut; 10581e03f97eSMikulas Patocka } 10591e03f97eSMikulas Patocka 10609fe86254SMikulas Patocka if (s->store->type->commit_merge(s->store, 10619fe86254SMikulas Patocka s->num_merging_chunks) < 0) { 10621e03f97eSMikulas Patocka DMERR("Write error in exception store: shutting down merge"); 10631e03f97eSMikulas Patocka goto shut; 10641e03f97eSMikulas Patocka } 10651e03f97eSMikulas Patocka 10669fe86254SMikulas Patocka if (remove_single_exception_chunk(s) < 0) 10679fe86254SMikulas Patocka goto shut; 10689fe86254SMikulas Patocka 10691e03f97eSMikulas Patocka snapshot_merge_next_chunks(s); 10701e03f97eSMikulas Patocka 10711e03f97eSMikulas Patocka return; 10721e03f97eSMikulas Patocka 10731e03f97eSMikulas Patocka shut: 10749fe86254SMikulas Patocka down_write(&s->lock); 1075d8ddb1cfSMike Snitzer s->merge_failed = 1; 10769fe86254SMikulas Patocka b = __release_queued_bios_after_merge(s); 10779fe86254SMikulas Patocka up_write(&s->lock); 10789fe86254SMikulas Patocka error_bios(b); 10799fe86254SMikulas Patocka 10801e03f97eSMikulas Patocka merge_shutdown(s); 10811e03f97eSMikulas Patocka } 10821e03f97eSMikulas Patocka 10831e03f97eSMikulas Patocka static void start_merge(struct dm_snapshot *s) 10841e03f97eSMikulas Patocka { 10851e03f97eSMikulas Patocka if (!test_and_set_bit(RUNNING_MERGE, &s->state_bits)) 10861e03f97eSMikulas Patocka snapshot_merge_next_chunks(s); 10871e03f97eSMikulas Patocka } 10881e03f97eSMikulas Patocka 10891e03f97eSMikulas Patocka /* 10901e03f97eSMikulas Patocka * Stop the merging process and wait until it finishes. 10911e03f97eSMikulas Patocka */ 10921e03f97eSMikulas Patocka static void stop_merge(struct dm_snapshot *s) 10931e03f97eSMikulas Patocka { 10941e03f97eSMikulas Patocka set_bit(SHUTDOWN_MERGE, &s->state_bits); 109574316201SNeilBrown wait_on_bit(&s->state_bits, RUNNING_MERGE, TASK_UNINTERRUPTIBLE); 10961e03f97eSMikulas Patocka clear_bit(SHUTDOWN_MERGE, &s->state_bits); 10971e03f97eSMikulas Patocka } 10981e03f97eSMikulas Patocka 10991da177e4SLinus Torvalds /* 1100b0d3cc01SMike Snitzer * Construct a snapshot mapping: <origin_dev> <COW-dev> <p|po|n> <chunk-size> 11011da177e4SLinus Torvalds */ 11021da177e4SLinus Torvalds static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv) 11031da177e4SLinus Torvalds { 11041da177e4SLinus Torvalds struct dm_snapshot *s; 1105cd45daffSMikulas Patocka int i; 11061da177e4SLinus Torvalds int r = -EINVAL; 1107fc56f6fbSMike Snitzer char *origin_path, *cow_path; 11084df2bf46SDingXiang dev_t origin_dev, cow_dev; 110955a62eefSAlasdair G Kergon unsigned args_used, num_flush_bios = 1; 111010b8106aSMike Snitzer fmode_t origin_mode = FMODE_READ; 11111da177e4SLinus Torvalds 11124c7e3bf4SMark McLoughlin if (argc != 4) { 111372d94861SAlasdair G Kergon ti->error = "requires exactly 4 arguments"; 11141da177e4SLinus Torvalds r = -EINVAL; 1115fc56f6fbSMike Snitzer goto bad; 11161da177e4SLinus Torvalds } 11171da177e4SLinus Torvalds 111810b8106aSMike Snitzer if (dm_target_is_snapshot_merge(ti)) { 111955a62eefSAlasdair G Kergon num_flush_bios = 2; 112010b8106aSMike Snitzer origin_mode = FMODE_WRITE; 112110b8106aSMike Snitzer } 112210b8106aSMike Snitzer 11231da177e4SLinus Torvalds s = kmalloc(sizeof(*s), GFP_KERNEL); 1124fee1998eSJonathan Brassow if (!s) { 1125a2d2b034SJonathan Brassow ti->error = "Cannot allocate private snapshot structure"; 11261da177e4SLinus Torvalds r = -ENOMEM; 1127fc56f6fbSMike Snitzer goto bad; 11281da177e4SLinus Torvalds } 11291da177e4SLinus Torvalds 1130c2411045SMikulas Patocka origin_path = argv[0]; 1131c2411045SMikulas Patocka argv++; 1132c2411045SMikulas Patocka argc--; 1133c2411045SMikulas Patocka 1134c2411045SMikulas Patocka r = dm_get_device(ti, origin_path, origin_mode, &s->origin); 1135c2411045SMikulas Patocka if (r) { 1136c2411045SMikulas Patocka ti->error = "Cannot get origin device"; 1137c2411045SMikulas Patocka goto bad_origin; 1138c2411045SMikulas Patocka } 11394df2bf46SDingXiang origin_dev = s->origin->bdev->bd_dev; 1140c2411045SMikulas Patocka 1141fc56f6fbSMike Snitzer cow_path = argv[0]; 1142fc56f6fbSMike Snitzer argv++; 1143fc56f6fbSMike Snitzer argc--; 1144fc56f6fbSMike Snitzer 11454df2bf46SDingXiang cow_dev = dm_get_dev_t(cow_path); 11464df2bf46SDingXiang if (cow_dev && cow_dev == origin_dev) { 11474df2bf46SDingXiang ti->error = "COW device cannot be the same as origin device"; 11484df2bf46SDingXiang r = -EINVAL; 11494df2bf46SDingXiang goto bad_cow; 11504df2bf46SDingXiang } 11514df2bf46SDingXiang 1152024d37e9SMilan Broz r = dm_get_device(ti, cow_path, dm_table_get_mode(ti->table), &s->cow); 1153fc56f6fbSMike Snitzer if (r) { 1154fc56f6fbSMike Snitzer ti->error = "Cannot get COW device"; 1155fc56f6fbSMike Snitzer goto bad_cow; 1156fc56f6fbSMike Snitzer } 1157fc56f6fbSMike Snitzer 1158fc56f6fbSMike Snitzer r = dm_exception_store_create(ti, argc, argv, s, &args_used, &s->store); 1159fc56f6fbSMike Snitzer if (r) { 1160fc56f6fbSMike Snitzer ti->error = "Couldn't create exception store"; 1161fc56f6fbSMike Snitzer r = -EINVAL; 1162fc56f6fbSMike Snitzer goto bad_store; 1163fc56f6fbSMike Snitzer } 1164fc56f6fbSMike Snitzer 1165fc56f6fbSMike Snitzer argv += args_used; 1166fc56f6fbSMike Snitzer argc -= args_used; 1167fc56f6fbSMike Snitzer 1168fc56f6fbSMike Snitzer s->ti = ti; 11691da177e4SLinus Torvalds s->valid = 1; 117076c44f6dSMikulas Patocka s->snapshot_overflowed = 0; 1171aa14edebSAlasdair G Kergon s->active = 0; 1172879129d2SMikulas Patocka atomic_set(&s->pending_exceptions_count, 0); 1173230c83afSMikulas Patocka s->exception_start_sequence = 0; 1174230c83afSMikulas Patocka s->exception_complete_sequence = 0; 1175230c83afSMikulas Patocka INIT_LIST_HEAD(&s->out_of_order_list); 11761da177e4SLinus Torvalds init_rwsem(&s->lock); 1177c1f0c183SMike Snitzer INIT_LIST_HEAD(&s->list); 1178ca3a931fSAlasdair G Kergon spin_lock_init(&s->pe_lock); 11791e03f97eSMikulas Patocka s->state_bits = 0; 1180d8ddb1cfSMike Snitzer s->merge_failed = 0; 11819fe86254SMikulas Patocka s->first_merging_chunk = 0; 11829fe86254SMikulas Patocka s->num_merging_chunks = 0; 11839fe86254SMikulas Patocka bio_list_init(&s->bios_queued_during_merge); 11841da177e4SLinus Torvalds 11851da177e4SLinus Torvalds /* Allocate hash table for COW data */ 1186fee1998eSJonathan Brassow if (init_hash_tables(s)) { 11871da177e4SLinus Torvalds ti->error = "Unable to allocate hash table space"; 11881da177e4SLinus Torvalds r = -ENOMEM; 1189fee1998eSJonathan Brassow goto bad_hash_tables; 11901da177e4SLinus Torvalds } 11911da177e4SLinus Torvalds 1192df5d2e90SMikulas Patocka s->kcopyd_client = dm_kcopyd_client_create(&dm_kcopyd_throttle); 1193fa34ce73SMikulas Patocka if (IS_ERR(s->kcopyd_client)) { 1194fa34ce73SMikulas Patocka r = PTR_ERR(s->kcopyd_client); 11951da177e4SLinus Torvalds ti->error = "Could not create kcopyd client"; 1196fee1998eSJonathan Brassow goto bad_kcopyd; 11971da177e4SLinus Torvalds } 11981da177e4SLinus Torvalds 119992e86812SMikulas Patocka s->pending_pool = mempool_create_slab_pool(MIN_IOS, pending_cache); 120092e86812SMikulas Patocka if (!s->pending_pool) { 120192e86812SMikulas Patocka ti->error = "Could not allocate mempool for pending exceptions"; 120209e8b813SWei Yongjun r = -ENOMEM; 1203fee1998eSJonathan Brassow goto bad_pending_pool; 120492e86812SMikulas Patocka } 120592e86812SMikulas Patocka 1206cd45daffSMikulas Patocka for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++) 1207cd45daffSMikulas Patocka INIT_HLIST_HEAD(&s->tracked_chunk_hash[i]); 1208cd45daffSMikulas Patocka 1209cd45daffSMikulas Patocka spin_lock_init(&s->tracked_chunk_lock); 1210cd45daffSMikulas Patocka 1211c1f0c183SMike Snitzer ti->private = s; 121255a62eefSAlasdair G Kergon ti->num_flush_bios = num_flush_bios; 121330187e1dSMike Snitzer ti->per_io_data_size = sizeof(struct dm_snap_tracked_chunk); 1214c1f0c183SMike Snitzer 1215c1f0c183SMike Snitzer /* Add snapshot to the list of snapshots for this origin */ 1216c1f0c183SMike Snitzer /* Exceptions aren't triggered till snapshot_resume() is called */ 1217c1f0c183SMike Snitzer r = register_snapshot(s); 1218c1f0c183SMike Snitzer if (r == -ENOMEM) { 1219c1f0c183SMike Snitzer ti->error = "Snapshot origin struct allocation failed"; 1220c1f0c183SMike Snitzer goto bad_load_and_register; 1221c1f0c183SMike Snitzer } else if (r < 0) { 1222c1f0c183SMike Snitzer /* invalid handover, register_snapshot has set ti->error */ 1223c1f0c183SMike Snitzer goto bad_load_and_register; 1224c1f0c183SMike Snitzer } 1225c1f0c183SMike Snitzer 1226c1f0c183SMike Snitzer /* 1227c1f0c183SMike Snitzer * Metadata must only be loaded into one table at once, so skip this 1228c1f0c183SMike Snitzer * if metadata will be handed over during resume. 1229c1f0c183SMike Snitzer * Chunk size will be set during the handover - set it to zero to 1230c1f0c183SMike Snitzer * ensure it's ignored. 1231c1f0c183SMike Snitzer */ 1232c1f0c183SMike Snitzer if (r > 0) { 1233c1f0c183SMike Snitzer s->store->chunk_size = 0; 1234c1f0c183SMike Snitzer return 0; 1235c1f0c183SMike Snitzer } 1236c1f0c183SMike Snitzer 1237493df71cSJonathan Brassow r = s->store->type->read_metadata(s->store, dm_add_exception, 1238493df71cSJonathan Brassow (void *)s); 12390764147bSMilan Broz if (r < 0) { 1240f9cea4f7SMark McLoughlin ti->error = "Failed to read snapshot metadata"; 1241c1f0c183SMike Snitzer goto bad_read_metadata; 12420764147bSMilan Broz } else if (r > 0) { 12430764147bSMilan Broz s->valid = 0; 12440764147bSMilan Broz DMWARN("Snapshot is marked invalid."); 1245f9cea4f7SMark McLoughlin } 1246aa14edebSAlasdair G Kergon 12473f2412dcSMikulas Patocka if (!s->store->chunk_size) { 12483f2412dcSMikulas Patocka ti->error = "Chunk size not set"; 1249c1f0c183SMike Snitzer goto bad_read_metadata; 12503f2412dcSMikulas Patocka } 1251542f9038SMike Snitzer 1252542f9038SMike Snitzer r = dm_set_target_max_io_len(ti, s->store->chunk_size); 1253542f9038SMike Snitzer if (r) 1254542f9038SMike Snitzer goto bad_read_metadata; 12551da177e4SLinus Torvalds 12561da177e4SLinus Torvalds return 0; 12571da177e4SLinus Torvalds 1258c1f0c183SMike Snitzer bad_read_metadata: 1259c1f0c183SMike Snitzer unregister_snapshot(s); 1260c1f0c183SMike Snitzer 1261cd45daffSMikulas Patocka bad_load_and_register: 126292e86812SMikulas Patocka mempool_destroy(s->pending_pool); 126392e86812SMikulas Patocka 1264fee1998eSJonathan Brassow bad_pending_pool: 1265eb69aca5SHeinz Mauelshagen dm_kcopyd_client_destroy(s->kcopyd_client); 12661da177e4SLinus Torvalds 1267fee1998eSJonathan Brassow bad_kcopyd: 12683510cb94SJon Brassow dm_exception_table_exit(&s->pending, pending_cache); 12693510cb94SJon Brassow dm_exception_table_exit(&s->complete, exception_cache); 12701da177e4SLinus Torvalds 1271fee1998eSJonathan Brassow bad_hash_tables: 1272fc56f6fbSMike Snitzer dm_exception_store_destroy(s->store); 1273fc56f6fbSMike Snitzer 1274fc56f6fbSMike Snitzer bad_store: 1275fc56f6fbSMike Snitzer dm_put_device(ti, s->cow); 1276fc56f6fbSMike Snitzer 1277fc56f6fbSMike Snitzer bad_cow: 1278c2411045SMikulas Patocka dm_put_device(ti, s->origin); 1279c2411045SMikulas Patocka 1280c2411045SMikulas Patocka bad_origin: 12811da177e4SLinus Torvalds kfree(s); 12821da177e4SLinus Torvalds 1283fc56f6fbSMike Snitzer bad: 12841da177e4SLinus Torvalds return r; 12851da177e4SLinus Torvalds } 12861da177e4SLinus Torvalds 128731c93a0cSMilan Broz static void __free_exceptions(struct dm_snapshot *s) 128831c93a0cSMilan Broz { 1289eb69aca5SHeinz Mauelshagen dm_kcopyd_client_destroy(s->kcopyd_client); 129031c93a0cSMilan Broz s->kcopyd_client = NULL; 129131c93a0cSMilan Broz 12923510cb94SJon Brassow dm_exception_table_exit(&s->pending, pending_cache); 12933510cb94SJon Brassow dm_exception_table_exit(&s->complete, exception_cache); 129431c93a0cSMilan Broz } 129531c93a0cSMilan Broz 1296c1f0c183SMike Snitzer static void __handover_exceptions(struct dm_snapshot *snap_src, 1297c1f0c183SMike Snitzer struct dm_snapshot *snap_dest) 1298c1f0c183SMike Snitzer { 1299c1f0c183SMike Snitzer union { 1300c1f0c183SMike Snitzer struct dm_exception_table table_swap; 1301c1f0c183SMike Snitzer struct dm_exception_store *store_swap; 1302c1f0c183SMike Snitzer } u; 1303c1f0c183SMike Snitzer 1304c1f0c183SMike Snitzer /* 1305c1f0c183SMike Snitzer * Swap all snapshot context information between the two instances. 1306c1f0c183SMike Snitzer */ 1307c1f0c183SMike Snitzer u.table_swap = snap_dest->complete; 1308c1f0c183SMike Snitzer snap_dest->complete = snap_src->complete; 1309c1f0c183SMike Snitzer snap_src->complete = u.table_swap; 1310c1f0c183SMike Snitzer 1311c1f0c183SMike Snitzer u.store_swap = snap_dest->store; 1312c1f0c183SMike Snitzer snap_dest->store = snap_src->store; 1313b0d3cc01SMike Snitzer snap_dest->store->userspace_supports_overflow = u.store_swap->userspace_supports_overflow; 1314c1f0c183SMike Snitzer snap_src->store = u.store_swap; 1315c1f0c183SMike Snitzer 1316c1f0c183SMike Snitzer snap_dest->store->snap = snap_dest; 1317c1f0c183SMike Snitzer snap_src->store->snap = snap_src; 1318c1f0c183SMike Snitzer 1319542f9038SMike Snitzer snap_dest->ti->max_io_len = snap_dest->store->chunk_size; 1320c1f0c183SMike Snitzer snap_dest->valid = snap_src->valid; 132176c44f6dSMikulas Patocka snap_dest->snapshot_overflowed = snap_src->snapshot_overflowed; 1322c1f0c183SMike Snitzer 1323c1f0c183SMike Snitzer /* 1324c1f0c183SMike Snitzer * Set source invalid to ensure it receives no further I/O. 1325c1f0c183SMike Snitzer */ 1326c1f0c183SMike Snitzer snap_src->valid = 0; 1327c1f0c183SMike Snitzer } 1328c1f0c183SMike Snitzer 13291da177e4SLinus Torvalds static void snapshot_dtr(struct dm_target *ti) 13301da177e4SLinus Torvalds { 1331cd45daffSMikulas Patocka #ifdef CONFIG_DM_DEBUG 1332cd45daffSMikulas Patocka int i; 1333cd45daffSMikulas Patocka #endif 1334028867acSAlasdair G Kergon struct dm_snapshot *s = ti->private; 1335c1f0c183SMike Snitzer struct dm_snapshot *snap_src = NULL, *snap_dest = NULL; 13361da177e4SLinus Torvalds 1337c1f0c183SMike Snitzer down_read(&_origins_lock); 1338c1f0c183SMike Snitzer /* Check whether exception handover must be cancelled */ 13399d3b15c4SMikulas Patocka (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL); 1340c1f0c183SMike Snitzer if (snap_src && snap_dest && (s == snap_src)) { 1341c1f0c183SMike Snitzer down_write(&snap_dest->lock); 1342c1f0c183SMike Snitzer snap_dest->valid = 0; 1343c1f0c183SMike Snitzer up_write(&snap_dest->lock); 1344c1f0c183SMike Snitzer DMERR("Cancelling snapshot handover."); 1345c1f0c183SMike Snitzer } 1346c1f0c183SMike Snitzer up_read(&_origins_lock); 1347c1f0c183SMike Snitzer 13481e03f97eSMikulas Patocka if (dm_target_is_snapshot_merge(ti)) 13491e03f97eSMikulas Patocka stop_merge(s); 13501e03f97eSMikulas Patocka 1351138728dcSAlasdair G Kergon /* Prevent further origin writes from using this snapshot. */ 1352138728dcSAlasdair G Kergon /* After this returns there can be no new kcopyd jobs. */ 13531da177e4SLinus Torvalds unregister_snapshot(s); 13541da177e4SLinus Torvalds 1355879129d2SMikulas Patocka while (atomic_read(&s->pending_exceptions_count)) 135690fa1527SMikulas Patocka msleep(1); 1357879129d2SMikulas Patocka /* 1358879129d2SMikulas Patocka * Ensure instructions in mempool_destroy aren't reordered 1359879129d2SMikulas Patocka * before atomic_read. 1360879129d2SMikulas Patocka */ 1361879129d2SMikulas Patocka smp_mb(); 1362879129d2SMikulas Patocka 1363cd45daffSMikulas Patocka #ifdef CONFIG_DM_DEBUG 1364cd45daffSMikulas Patocka for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++) 1365cd45daffSMikulas Patocka BUG_ON(!hlist_empty(&s->tracked_chunk_hash[i])); 1366cd45daffSMikulas Patocka #endif 1367cd45daffSMikulas Patocka 136831c93a0cSMilan Broz __free_exceptions(s); 13691da177e4SLinus Torvalds 137092e86812SMikulas Patocka mempool_destroy(s->pending_pool); 137192e86812SMikulas Patocka 1372fee1998eSJonathan Brassow dm_exception_store_destroy(s->store); 1373138728dcSAlasdair G Kergon 1374fc56f6fbSMike Snitzer dm_put_device(ti, s->cow); 1375fc56f6fbSMike Snitzer 1376c2411045SMikulas Patocka dm_put_device(ti, s->origin); 1377c2411045SMikulas Patocka 13781da177e4SLinus Torvalds kfree(s); 13791da177e4SLinus Torvalds } 13801da177e4SLinus Torvalds 13811da177e4SLinus Torvalds /* 13821da177e4SLinus Torvalds * Flush a list of buffers. 13831da177e4SLinus Torvalds */ 13841da177e4SLinus Torvalds static void flush_bios(struct bio *bio) 13851da177e4SLinus Torvalds { 13861da177e4SLinus Torvalds struct bio *n; 13871da177e4SLinus Torvalds 13881da177e4SLinus Torvalds while (bio) { 13891da177e4SLinus Torvalds n = bio->bi_next; 13901da177e4SLinus Torvalds bio->bi_next = NULL; 13911da177e4SLinus Torvalds generic_make_request(bio); 13921da177e4SLinus Torvalds bio = n; 13931da177e4SLinus Torvalds } 13941da177e4SLinus Torvalds } 13951da177e4SLinus Torvalds 1396515ad66cSMikulas Patocka static int do_origin(struct dm_dev *origin, struct bio *bio); 1397515ad66cSMikulas Patocka 1398515ad66cSMikulas Patocka /* 1399515ad66cSMikulas Patocka * Flush a list of buffers. 1400515ad66cSMikulas Patocka */ 1401515ad66cSMikulas Patocka static void retry_origin_bios(struct dm_snapshot *s, struct bio *bio) 1402515ad66cSMikulas Patocka { 1403515ad66cSMikulas Patocka struct bio *n; 1404515ad66cSMikulas Patocka int r; 1405515ad66cSMikulas Patocka 1406515ad66cSMikulas Patocka while (bio) { 1407515ad66cSMikulas Patocka n = bio->bi_next; 1408515ad66cSMikulas Patocka bio->bi_next = NULL; 1409515ad66cSMikulas Patocka r = do_origin(s->origin, bio); 1410515ad66cSMikulas Patocka if (r == DM_MAPIO_REMAPPED) 1411515ad66cSMikulas Patocka generic_make_request(bio); 1412515ad66cSMikulas Patocka bio = n; 1413515ad66cSMikulas Patocka } 1414515ad66cSMikulas Patocka } 1415515ad66cSMikulas Patocka 14161da177e4SLinus Torvalds /* 14171da177e4SLinus Torvalds * Error a list of buffers. 14181da177e4SLinus Torvalds */ 14191da177e4SLinus Torvalds static void error_bios(struct bio *bio) 14201da177e4SLinus Torvalds { 14211da177e4SLinus Torvalds struct bio *n; 14221da177e4SLinus Torvalds 14231da177e4SLinus Torvalds while (bio) { 14241da177e4SLinus Torvalds n = bio->bi_next; 14251da177e4SLinus Torvalds bio->bi_next = NULL; 14266712ecf8SNeilBrown bio_io_error(bio); 14271da177e4SLinus Torvalds bio = n; 14281da177e4SLinus Torvalds } 14291da177e4SLinus Torvalds } 14301da177e4SLinus Torvalds 1431695368acSAlasdair G Kergon static void __invalidate_snapshot(struct dm_snapshot *s, int err) 143276df1c65SAlasdair G Kergon { 143376df1c65SAlasdair G Kergon if (!s->valid) 143476df1c65SAlasdair G Kergon return; 143576df1c65SAlasdair G Kergon 143676df1c65SAlasdair G Kergon if (err == -EIO) 143776df1c65SAlasdair G Kergon DMERR("Invalidating snapshot: Error reading/writing."); 143876df1c65SAlasdair G Kergon else if (err == -ENOMEM) 143976df1c65SAlasdair G Kergon DMERR("Invalidating snapshot: Unable to allocate exception."); 144076df1c65SAlasdair G Kergon 1441493df71cSJonathan Brassow if (s->store->type->drop_snapshot) 1442493df71cSJonathan Brassow s->store->type->drop_snapshot(s->store); 144376df1c65SAlasdair G Kergon 144476df1c65SAlasdair G Kergon s->valid = 0; 144576df1c65SAlasdair G Kergon 1446fc56f6fbSMike Snitzer dm_table_event(s->ti->table); 144776df1c65SAlasdair G Kergon } 144876df1c65SAlasdair G Kergon 1449385277bfSMikulas Patocka static void pending_complete(void *context, int success) 14501da177e4SLinus Torvalds { 1451385277bfSMikulas Patocka struct dm_snap_pending_exception *pe = context; 14521d4989c8SJon Brassow struct dm_exception *e; 14531da177e4SLinus Torvalds struct dm_snapshot *s = pe->snap; 14549d493fa8SAlasdair G Kergon struct bio *origin_bios = NULL; 14559d493fa8SAlasdair G Kergon struct bio *snapshot_bios = NULL; 1456a6e50b40SMikulas Patocka struct bio *full_bio = NULL; 14579d493fa8SAlasdair G Kergon int error = 0; 14581da177e4SLinus Torvalds 145976df1c65SAlasdair G Kergon if (!success) { 146076df1c65SAlasdair G Kergon /* Read/write error - snapshot is unusable */ 14611da177e4SLinus Torvalds down_write(&s->lock); 1462695368acSAlasdair G Kergon __invalidate_snapshot(s, -EIO); 14639d493fa8SAlasdair G Kergon error = 1; 146476df1c65SAlasdair G Kergon goto out; 146576df1c65SAlasdair G Kergon } 146676df1c65SAlasdair G Kergon 1467119bc547SMikulas Patocka e = alloc_completed_exception(GFP_NOIO); 146876df1c65SAlasdair G Kergon if (!e) { 146976df1c65SAlasdair G Kergon down_write(&s->lock); 1470695368acSAlasdair G Kergon __invalidate_snapshot(s, -ENOMEM); 14719d493fa8SAlasdair G Kergon error = 1; 14721da177e4SLinus Torvalds goto out; 14731da177e4SLinus Torvalds } 14741da177e4SLinus Torvalds *e = pe->e; 14751da177e4SLinus Torvalds 14769d493fa8SAlasdair G Kergon down_write(&s->lock); 14779d493fa8SAlasdair G Kergon if (!s->valid) { 14783510cb94SJon Brassow free_completed_exception(e); 14799d493fa8SAlasdair G Kergon error = 1; 14809d493fa8SAlasdair G Kergon goto out; 14819d493fa8SAlasdair G Kergon } 14829d493fa8SAlasdair G Kergon 1483615d1eb9SMike Snitzer /* Check for conflicting reads */ 1484615d1eb9SMike Snitzer __check_for_conflicting_io(s, pe->e.old_chunk); 1485a8d41b59SMikulas Patocka 1486a8d41b59SMikulas Patocka /* 14871da177e4SLinus Torvalds * Add a proper exception, and remove the 14881da177e4SLinus Torvalds * in-flight exception from the list. 14891da177e4SLinus Torvalds */ 14903510cb94SJon Brassow dm_insert_exception(&s->complete, e); 14911da177e4SLinus Torvalds 14921da177e4SLinus Torvalds out: 14933510cb94SJon Brassow dm_remove_exception(&pe->e); 14949d493fa8SAlasdair G Kergon snapshot_bios = bio_list_get(&pe->snapshot_bios); 1495515ad66cSMikulas Patocka origin_bios = bio_list_get(&pe->origin_bios); 1496a6e50b40SMikulas Patocka full_bio = pe->full_bio; 1497fe3265b1SMikulas Patocka if (full_bio) 1498a6e50b40SMikulas Patocka full_bio->bi_end_io = pe->full_bio_end_io; 149973dfd078SMikulas Patocka increment_pending_exceptions_done_count(); 150073dfd078SMikulas Patocka 15019d493fa8SAlasdair G Kergon up_write(&s->lock); 15029d493fa8SAlasdair G Kergon 15039d493fa8SAlasdair G Kergon /* Submit any pending write bios */ 1504a6e50b40SMikulas Patocka if (error) { 1505a6e50b40SMikulas Patocka if (full_bio) 1506a6e50b40SMikulas Patocka bio_io_error(full_bio); 15079d493fa8SAlasdair G Kergon error_bios(snapshot_bios); 1508a6e50b40SMikulas Patocka } else { 1509a6e50b40SMikulas Patocka if (full_bio) 15104246a0b6SChristoph Hellwig bio_endio(full_bio); 15119d493fa8SAlasdair G Kergon flush_bios(snapshot_bios); 1512a6e50b40SMikulas Patocka } 15139d493fa8SAlasdair G Kergon 1514515ad66cSMikulas Patocka retry_origin_bios(s, origin_bios); 151522aa66a3SMikulas Patocka 151622aa66a3SMikulas Patocka free_pending_exception(pe); 15171da177e4SLinus Torvalds } 15181da177e4SLinus Torvalds 1519230c83afSMikulas Patocka static void complete_exception(struct dm_snap_pending_exception *pe) 1520230c83afSMikulas Patocka { 1521230c83afSMikulas Patocka struct dm_snapshot *s = pe->snap; 1522230c83afSMikulas Patocka 1523230c83afSMikulas Patocka /* Update the metadata if we are persistent */ 1524385277bfSMikulas Patocka s->store->type->commit_exception(s->store, &pe->e, !pe->copy_error, 1525385277bfSMikulas Patocka pending_complete, pe); 1526230c83afSMikulas Patocka } 1527230c83afSMikulas Patocka 15281da177e4SLinus Torvalds /* 15291da177e4SLinus Torvalds * Called when the copy I/O has finished. kcopyd actually runs 15301da177e4SLinus Torvalds * this code so don't block. 15311da177e4SLinus Torvalds */ 15324cdc1d1fSAlasdair G Kergon static void copy_callback(int read_err, unsigned long write_err, void *context) 15331da177e4SLinus Torvalds { 1534028867acSAlasdair G Kergon struct dm_snap_pending_exception *pe = context; 15351da177e4SLinus Torvalds struct dm_snapshot *s = pe->snap; 15361da177e4SLinus Torvalds 1537230c83afSMikulas Patocka pe->copy_error = read_err || write_err; 15381da177e4SLinus Torvalds 1539230c83afSMikulas Patocka if (pe->exception_sequence == s->exception_complete_sequence) { 1540230c83afSMikulas Patocka s->exception_complete_sequence++; 1541230c83afSMikulas Patocka complete_exception(pe); 1542230c83afSMikulas Patocka 1543230c83afSMikulas Patocka while (!list_empty(&s->out_of_order_list)) { 1544230c83afSMikulas Patocka pe = list_entry(s->out_of_order_list.next, 1545230c83afSMikulas Patocka struct dm_snap_pending_exception, out_of_order_entry); 1546230c83afSMikulas Patocka if (pe->exception_sequence != s->exception_complete_sequence) 1547230c83afSMikulas Patocka break; 1548230c83afSMikulas Patocka s->exception_complete_sequence++; 1549230c83afSMikulas Patocka list_del(&pe->out_of_order_entry); 1550230c83afSMikulas Patocka complete_exception(pe); 1551230c83afSMikulas Patocka } 1552230c83afSMikulas Patocka } else { 1553230c83afSMikulas Patocka struct list_head *lh; 1554230c83afSMikulas Patocka struct dm_snap_pending_exception *pe2; 1555230c83afSMikulas Patocka 1556230c83afSMikulas Patocka list_for_each_prev(lh, &s->out_of_order_list) { 1557230c83afSMikulas Patocka pe2 = list_entry(lh, struct dm_snap_pending_exception, out_of_order_entry); 1558230c83afSMikulas Patocka if (pe2->exception_sequence < pe->exception_sequence) 1559230c83afSMikulas Patocka break; 1560230c83afSMikulas Patocka } 1561230c83afSMikulas Patocka list_add(&pe->out_of_order_entry, lh); 1562230c83afSMikulas Patocka } 15631da177e4SLinus Torvalds } 15641da177e4SLinus Torvalds 15651da177e4SLinus Torvalds /* 15661da177e4SLinus Torvalds * Dispatches the copy operation to kcopyd. 15671da177e4SLinus Torvalds */ 1568028867acSAlasdair G Kergon static void start_copy(struct dm_snap_pending_exception *pe) 15691da177e4SLinus Torvalds { 15701da177e4SLinus Torvalds struct dm_snapshot *s = pe->snap; 157122a1ceb1SHeinz Mauelshagen struct dm_io_region src, dest; 15721da177e4SLinus Torvalds struct block_device *bdev = s->origin->bdev; 15731da177e4SLinus Torvalds sector_t dev_size; 15741da177e4SLinus Torvalds 15751da177e4SLinus Torvalds dev_size = get_dev_size(bdev); 15761da177e4SLinus Torvalds 15771da177e4SLinus Torvalds src.bdev = bdev; 157871fab00aSJonathan Brassow src.sector = chunk_to_sector(s->store, pe->e.old_chunk); 1579df96eee6SMikulas Patocka src.count = min((sector_t)s->store->chunk_size, dev_size - src.sector); 15801da177e4SLinus Torvalds 1581fc56f6fbSMike Snitzer dest.bdev = s->cow->bdev; 158271fab00aSJonathan Brassow dest.sector = chunk_to_sector(s->store, pe->e.new_chunk); 15831da177e4SLinus Torvalds dest.count = src.count; 15841da177e4SLinus Torvalds 15851da177e4SLinus Torvalds /* Hand over to kcopyd */ 1586a2d2b034SJonathan Brassow dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, copy_callback, pe); 15871da177e4SLinus Torvalds } 15881da177e4SLinus Torvalds 15894246a0b6SChristoph Hellwig static void full_bio_end_io(struct bio *bio) 1590a6e50b40SMikulas Patocka { 1591a6e50b40SMikulas Patocka void *callback_data = bio->bi_private; 1592a6e50b40SMikulas Patocka 15934246a0b6SChristoph Hellwig dm_kcopyd_do_callback(callback_data, 0, bio->bi_error ? 1 : 0); 1594a6e50b40SMikulas Patocka } 1595a6e50b40SMikulas Patocka 1596a6e50b40SMikulas Patocka static void start_full_bio(struct dm_snap_pending_exception *pe, 1597a6e50b40SMikulas Patocka struct bio *bio) 1598a6e50b40SMikulas Patocka { 1599a6e50b40SMikulas Patocka struct dm_snapshot *s = pe->snap; 1600a6e50b40SMikulas Patocka void *callback_data; 1601a6e50b40SMikulas Patocka 1602a6e50b40SMikulas Patocka pe->full_bio = bio; 1603a6e50b40SMikulas Patocka pe->full_bio_end_io = bio->bi_end_io; 1604a6e50b40SMikulas Patocka 1605a6e50b40SMikulas Patocka callback_data = dm_kcopyd_prepare_callback(s->kcopyd_client, 1606a6e50b40SMikulas Patocka copy_callback, pe); 1607a6e50b40SMikulas Patocka 1608a6e50b40SMikulas Patocka bio->bi_end_io = full_bio_end_io; 1609a6e50b40SMikulas Patocka bio->bi_private = callback_data; 1610a6e50b40SMikulas Patocka 1611a6e50b40SMikulas Patocka generic_make_request(bio); 1612a6e50b40SMikulas Patocka } 1613a6e50b40SMikulas Patocka 16142913808eSMikulas Patocka static struct dm_snap_pending_exception * 16152913808eSMikulas Patocka __lookup_pending_exception(struct dm_snapshot *s, chunk_t chunk) 16162913808eSMikulas Patocka { 16173510cb94SJon Brassow struct dm_exception *e = dm_lookup_exception(&s->pending, chunk); 16182913808eSMikulas Patocka 16192913808eSMikulas Patocka if (!e) 16202913808eSMikulas Patocka return NULL; 16212913808eSMikulas Patocka 16222913808eSMikulas Patocka return container_of(e, struct dm_snap_pending_exception, e); 16232913808eSMikulas Patocka } 16242913808eSMikulas Patocka 16251da177e4SLinus Torvalds /* 16261da177e4SLinus Torvalds * Looks to see if this snapshot already has a pending exception 16271da177e4SLinus Torvalds * for this chunk, otherwise it allocates a new one and inserts 16281da177e4SLinus Torvalds * it into the pending table. 16291da177e4SLinus Torvalds * 16301da177e4SLinus Torvalds * NOTE: a write lock must be held on snap->lock before calling 16311da177e4SLinus Torvalds * this. 16321da177e4SLinus Torvalds */ 1633028867acSAlasdair G Kergon static struct dm_snap_pending_exception * 1634c6621392SMikulas Patocka __find_pending_exception(struct dm_snapshot *s, 1635c6621392SMikulas Patocka struct dm_snap_pending_exception *pe, chunk_t chunk) 16361da177e4SLinus Torvalds { 1637c6621392SMikulas Patocka struct dm_snap_pending_exception *pe2; 163876df1c65SAlasdair G Kergon 16392913808eSMikulas Patocka pe2 = __lookup_pending_exception(s, chunk); 16402913808eSMikulas Patocka if (pe2) { 16411da177e4SLinus Torvalds free_pending_exception(pe); 16422913808eSMikulas Patocka return pe2; 164376df1c65SAlasdair G Kergon } 164476df1c65SAlasdair G Kergon 16451da177e4SLinus Torvalds pe->e.old_chunk = chunk; 16461da177e4SLinus Torvalds bio_list_init(&pe->origin_bios); 16471da177e4SLinus Torvalds bio_list_init(&pe->snapshot_bios); 16481da177e4SLinus Torvalds pe->started = 0; 1649a6e50b40SMikulas Patocka pe->full_bio = NULL; 16501da177e4SLinus Torvalds 1651493df71cSJonathan Brassow if (s->store->type->prepare_exception(s->store, &pe->e)) { 16521da177e4SLinus Torvalds free_pending_exception(pe); 16531da177e4SLinus Torvalds return NULL; 16541da177e4SLinus Torvalds } 16551da177e4SLinus Torvalds 1656230c83afSMikulas Patocka pe->exception_sequence = s->exception_start_sequence++; 1657230c83afSMikulas Patocka 16583510cb94SJon Brassow dm_insert_exception(&s->pending, &pe->e); 16591da177e4SLinus Torvalds 16601da177e4SLinus Torvalds return pe; 16611da177e4SLinus Torvalds } 16621da177e4SLinus Torvalds 16631d4989c8SJon Brassow static void remap_exception(struct dm_snapshot *s, struct dm_exception *e, 1664d74f81f8SMilan Broz struct bio *bio, chunk_t chunk) 16651da177e4SLinus Torvalds { 1666fc56f6fbSMike Snitzer bio->bi_bdev = s->cow->bdev; 16674f024f37SKent Overstreet bio->bi_iter.bi_sector = 16684f024f37SKent Overstreet chunk_to_sector(s->store, dm_chunk_number(e->new_chunk) + 1669d74f81f8SMilan Broz (chunk - e->old_chunk)) + 16704f024f37SKent Overstreet (bio->bi_iter.bi_sector & s->store->chunk_mask); 16711da177e4SLinus Torvalds } 16721da177e4SLinus Torvalds 16737de3ee57SMikulas Patocka static int snapshot_map(struct dm_target *ti, struct bio *bio) 16741da177e4SLinus Torvalds { 16751d4989c8SJon Brassow struct dm_exception *e; 1676028867acSAlasdair G Kergon struct dm_snapshot *s = ti->private; 1677d2a7ad29SKiyoshi Ueda int r = DM_MAPIO_REMAPPED; 16781da177e4SLinus Torvalds chunk_t chunk; 1679028867acSAlasdair G Kergon struct dm_snap_pending_exception *pe = NULL; 16801da177e4SLinus Torvalds 1681ee18026aSMikulas Patocka init_tracked_chunk(bio); 1682ee18026aSMikulas Patocka 16831eff9d32SJens Axboe if (bio->bi_opf & REQ_PREFLUSH) { 1684fc56f6fbSMike Snitzer bio->bi_bdev = s->cow->bdev; 1685494b3ee7SMikulas Patocka return DM_MAPIO_REMAPPED; 1686494b3ee7SMikulas Patocka } 1687494b3ee7SMikulas Patocka 16884f024f37SKent Overstreet chunk = sector_to_chunk(s->store, bio->bi_iter.bi_sector); 16891da177e4SLinus Torvalds 16901da177e4SLinus Torvalds /* Full snapshots are not usable */ 169176df1c65SAlasdair G Kergon /* To get here the table must be live so s->active is always set. */ 16921da177e4SLinus Torvalds if (!s->valid) 1693f6a80ea8SAlasdair G Kergon return -EIO; 16941da177e4SLinus Torvalds 16951da177e4SLinus Torvalds /* FIXME: should only take write lock if we need 16961da177e4SLinus Torvalds * to copy an exception */ 16971da177e4SLinus Torvalds down_write(&s->lock); 16981da177e4SLinus Torvalds 169970246286SChristoph Hellwig if (!s->valid || (unlikely(s->snapshot_overflowed) && 170070246286SChristoph Hellwig bio_data_dir(bio) == WRITE)) { 170176df1c65SAlasdair G Kergon r = -EIO; 170276df1c65SAlasdair G Kergon goto out_unlock; 170376df1c65SAlasdair G Kergon } 170476df1c65SAlasdair G Kergon 17051da177e4SLinus Torvalds /* If the block is already remapped - use that, else remap it */ 17063510cb94SJon Brassow e = dm_lookup_exception(&s->complete, chunk); 17071da177e4SLinus Torvalds if (e) { 1708d74f81f8SMilan Broz remap_exception(s, e, bio, chunk); 170976df1c65SAlasdair G Kergon goto out_unlock; 171076df1c65SAlasdair G Kergon } 17111da177e4SLinus Torvalds 1712ba40a2aaSAlasdair G Kergon /* 1713ba40a2aaSAlasdair G Kergon * Write to snapshot - higher level takes care of RW/RO 1714ba40a2aaSAlasdair G Kergon * flags so we should only get this if we are 1715ba40a2aaSAlasdair G Kergon * writeable. 1716ba40a2aaSAlasdair G Kergon */ 171770246286SChristoph Hellwig if (bio_data_dir(bio) == WRITE) { 17182913808eSMikulas Patocka pe = __lookup_pending_exception(s, chunk); 17192913808eSMikulas Patocka if (!pe) { 1720c6621392SMikulas Patocka up_write(&s->lock); 1721c6621392SMikulas Patocka pe = alloc_pending_exception(s); 1722c6621392SMikulas Patocka down_write(&s->lock); 1723c6621392SMikulas Patocka 172476c44f6dSMikulas Patocka if (!s->valid || s->snapshot_overflowed) { 1725c6621392SMikulas Patocka free_pending_exception(pe); 1726c6621392SMikulas Patocka r = -EIO; 1727c6621392SMikulas Patocka goto out_unlock; 1728c6621392SMikulas Patocka } 1729c6621392SMikulas Patocka 17303510cb94SJon Brassow e = dm_lookup_exception(&s->complete, chunk); 173135bf659bSMikulas Patocka if (e) { 173235bf659bSMikulas Patocka free_pending_exception(pe); 173335bf659bSMikulas Patocka remap_exception(s, e, bio, chunk); 173435bf659bSMikulas Patocka goto out_unlock; 173535bf659bSMikulas Patocka } 173635bf659bSMikulas Patocka 1737c6621392SMikulas Patocka pe = __find_pending_exception(s, pe, chunk); 17381da177e4SLinus Torvalds if (!pe) { 1739b0d3cc01SMike Snitzer if (s->store->userspace_supports_overflow) { 174076c44f6dSMikulas Patocka s->snapshot_overflowed = 1; 174176c44f6dSMikulas Patocka DMERR("Snapshot overflowed: Unable to allocate exception."); 1742b0d3cc01SMike Snitzer } else 1743b0d3cc01SMike Snitzer __invalidate_snapshot(s, -ENOMEM); 17441da177e4SLinus Torvalds r = -EIO; 174576df1c65SAlasdair G Kergon goto out_unlock; 174676df1c65SAlasdair G Kergon } 17472913808eSMikulas Patocka } 174876df1c65SAlasdair G Kergon 1749d74f81f8SMilan Broz remap_exception(s, &pe->e, bio, chunk); 17501da177e4SLinus Torvalds 1751d2a7ad29SKiyoshi Ueda r = DM_MAPIO_SUBMITTED; 1752ba40a2aaSAlasdair G Kergon 1753a6e50b40SMikulas Patocka if (!pe->started && 17544f024f37SKent Overstreet bio->bi_iter.bi_size == 17554f024f37SKent Overstreet (s->store->chunk_size << SECTOR_SHIFT)) { 1756a6e50b40SMikulas Patocka pe->started = 1; 1757a6e50b40SMikulas Patocka up_write(&s->lock); 1758a6e50b40SMikulas Patocka start_full_bio(pe, bio); 1759a6e50b40SMikulas Patocka goto out; 1760a6e50b40SMikulas Patocka } 1761a6e50b40SMikulas Patocka 1762a6e50b40SMikulas Patocka bio_list_add(&pe->snapshot_bios, bio); 1763a6e50b40SMikulas Patocka 17641da177e4SLinus Torvalds if (!pe->started) { 17651da177e4SLinus Torvalds /* this is protected by snap->lock */ 17661da177e4SLinus Torvalds pe->started = 1; 176776df1c65SAlasdair G Kergon up_write(&s->lock); 176876df1c65SAlasdair G Kergon start_copy(pe); 1769ba40a2aaSAlasdair G Kergon goto out; 1770ba40a2aaSAlasdair G Kergon } 1771cd45daffSMikulas Patocka } else { 17721da177e4SLinus Torvalds bio->bi_bdev = s->origin->bdev; 1773ee18026aSMikulas Patocka track_chunk(s, bio, chunk); 1774cd45daffSMikulas Patocka } 17751da177e4SLinus Torvalds 1776ba40a2aaSAlasdair G Kergon out_unlock: 1777ba40a2aaSAlasdair G Kergon up_write(&s->lock); 1778ba40a2aaSAlasdair G Kergon out: 17791da177e4SLinus Torvalds return r; 17801da177e4SLinus Torvalds } 17811da177e4SLinus Torvalds 17823452c2a1SMikulas Patocka /* 17833452c2a1SMikulas Patocka * A snapshot-merge target behaves like a combination of a snapshot 17843452c2a1SMikulas Patocka * target and a snapshot-origin target. It only generates new 17853452c2a1SMikulas Patocka * exceptions in other snapshots and not in the one that is being 17863452c2a1SMikulas Patocka * merged. 17873452c2a1SMikulas Patocka * 17883452c2a1SMikulas Patocka * For each chunk, if there is an existing exception, it is used to 17893452c2a1SMikulas Patocka * redirect I/O to the cow device. Otherwise I/O is sent to the origin, 17903452c2a1SMikulas Patocka * which in turn might generate exceptions in other snapshots. 17919fe86254SMikulas Patocka * If merging is currently taking place on the chunk in question, the 17929fe86254SMikulas Patocka * I/O is deferred by adding it to s->bios_queued_during_merge. 17933452c2a1SMikulas Patocka */ 17947de3ee57SMikulas Patocka static int snapshot_merge_map(struct dm_target *ti, struct bio *bio) 17953452c2a1SMikulas Patocka { 17963452c2a1SMikulas Patocka struct dm_exception *e; 17973452c2a1SMikulas Patocka struct dm_snapshot *s = ti->private; 17983452c2a1SMikulas Patocka int r = DM_MAPIO_REMAPPED; 17993452c2a1SMikulas Patocka chunk_t chunk; 18003452c2a1SMikulas Patocka 1801ee18026aSMikulas Patocka init_tracked_chunk(bio); 1802ee18026aSMikulas Patocka 18031eff9d32SJens Axboe if (bio->bi_opf & REQ_PREFLUSH) { 180455a62eefSAlasdair G Kergon if (!dm_bio_get_target_bio_nr(bio)) 180510b8106aSMike Snitzer bio->bi_bdev = s->origin->bdev; 180610b8106aSMike Snitzer else 180710b8106aSMike Snitzer bio->bi_bdev = s->cow->bdev; 180810b8106aSMike Snitzer return DM_MAPIO_REMAPPED; 180910b8106aSMike Snitzer } 181010b8106aSMike Snitzer 18114f024f37SKent Overstreet chunk = sector_to_chunk(s->store, bio->bi_iter.bi_sector); 18123452c2a1SMikulas Patocka 18139fe86254SMikulas Patocka down_write(&s->lock); 18143452c2a1SMikulas Patocka 1815d2fdb776SMikulas Patocka /* Full merging snapshots are redirected to the origin */ 1816d2fdb776SMikulas Patocka if (!s->valid) 1817d2fdb776SMikulas Patocka goto redirect_to_origin; 18183452c2a1SMikulas Patocka 18193452c2a1SMikulas Patocka /* If the block is already remapped - use that */ 18203452c2a1SMikulas Patocka e = dm_lookup_exception(&s->complete, chunk); 18213452c2a1SMikulas Patocka if (e) { 18229fe86254SMikulas Patocka /* Queue writes overlapping with chunks being merged */ 182370246286SChristoph Hellwig if (bio_data_dir(bio) == WRITE && 18249fe86254SMikulas Patocka chunk >= s->first_merging_chunk && 18259fe86254SMikulas Patocka chunk < (s->first_merging_chunk + 18269fe86254SMikulas Patocka s->num_merging_chunks)) { 18279fe86254SMikulas Patocka bio->bi_bdev = s->origin->bdev; 18289fe86254SMikulas Patocka bio_list_add(&s->bios_queued_during_merge, bio); 18299fe86254SMikulas Patocka r = DM_MAPIO_SUBMITTED; 18309fe86254SMikulas Patocka goto out_unlock; 18319fe86254SMikulas Patocka } 183217aa0332SMikulas Patocka 18333452c2a1SMikulas Patocka remap_exception(s, e, bio, chunk); 183417aa0332SMikulas Patocka 183570246286SChristoph Hellwig if (bio_data_dir(bio) == WRITE) 1836ee18026aSMikulas Patocka track_chunk(s, bio, chunk); 18373452c2a1SMikulas Patocka goto out_unlock; 18383452c2a1SMikulas Patocka } 18393452c2a1SMikulas Patocka 1840d2fdb776SMikulas Patocka redirect_to_origin: 18413452c2a1SMikulas Patocka bio->bi_bdev = s->origin->bdev; 18423452c2a1SMikulas Patocka 184370246286SChristoph Hellwig if (bio_data_dir(bio) == WRITE) { 18449fe86254SMikulas Patocka up_write(&s->lock); 18453452c2a1SMikulas Patocka return do_origin(s->origin, bio); 18463452c2a1SMikulas Patocka } 18473452c2a1SMikulas Patocka 18483452c2a1SMikulas Patocka out_unlock: 18499fe86254SMikulas Patocka up_write(&s->lock); 18503452c2a1SMikulas Patocka 18513452c2a1SMikulas Patocka return r; 18523452c2a1SMikulas Patocka } 18533452c2a1SMikulas Patocka 18547de3ee57SMikulas Patocka static int snapshot_end_io(struct dm_target *ti, struct bio *bio, int error) 1855cd45daffSMikulas Patocka { 1856cd45daffSMikulas Patocka struct dm_snapshot *s = ti->private; 1857cd45daffSMikulas Patocka 1858ee18026aSMikulas Patocka if (is_bio_tracked(bio)) 1859ee18026aSMikulas Patocka stop_tracking_chunk(s, bio); 1860cd45daffSMikulas Patocka 1861cd45daffSMikulas Patocka return 0; 1862cd45daffSMikulas Patocka } 1863cd45daffSMikulas Patocka 18641e03f97eSMikulas Patocka static void snapshot_merge_presuspend(struct dm_target *ti) 18651e03f97eSMikulas Patocka { 18661e03f97eSMikulas Patocka struct dm_snapshot *s = ti->private; 18671e03f97eSMikulas Patocka 18681e03f97eSMikulas Patocka stop_merge(s); 18691e03f97eSMikulas Patocka } 18701e03f97eSMikulas Patocka 1871c1f0c183SMike Snitzer static int snapshot_preresume(struct dm_target *ti) 1872c1f0c183SMike Snitzer { 1873c1f0c183SMike Snitzer int r = 0; 1874c1f0c183SMike Snitzer struct dm_snapshot *s = ti->private; 1875c1f0c183SMike Snitzer struct dm_snapshot *snap_src = NULL, *snap_dest = NULL; 1876c1f0c183SMike Snitzer 1877c1f0c183SMike Snitzer down_read(&_origins_lock); 18789d3b15c4SMikulas Patocka (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL); 1879c1f0c183SMike Snitzer if (snap_src && snap_dest) { 1880c1f0c183SMike Snitzer down_read(&snap_src->lock); 1881c1f0c183SMike Snitzer if (s == snap_src) { 1882c1f0c183SMike Snitzer DMERR("Unable to resume snapshot source until " 1883c1f0c183SMike Snitzer "handover completes."); 1884c1f0c183SMike Snitzer r = -EINVAL; 1885b83b2f29SMike Snitzer } else if (!dm_suspended(snap_src->ti)) { 1886c1f0c183SMike Snitzer DMERR("Unable to perform snapshot handover until " 1887c1f0c183SMike Snitzer "source is suspended."); 1888c1f0c183SMike Snitzer r = -EINVAL; 1889c1f0c183SMike Snitzer } 1890c1f0c183SMike Snitzer up_read(&snap_src->lock); 1891c1f0c183SMike Snitzer } 1892c1f0c183SMike Snitzer up_read(&_origins_lock); 1893c1f0c183SMike Snitzer 1894c1f0c183SMike Snitzer return r; 1895c1f0c183SMike Snitzer } 1896c1f0c183SMike Snitzer 18971da177e4SLinus Torvalds static void snapshot_resume(struct dm_target *ti) 18981da177e4SLinus Torvalds { 1899028867acSAlasdair G Kergon struct dm_snapshot *s = ti->private; 190009ee96b2SMikulas Patocka struct dm_snapshot *snap_src = NULL, *snap_dest = NULL, *snap_merging = NULL; 1901b735fedeSMikulas Patocka struct dm_origin *o; 1902b735fedeSMikulas Patocka struct mapped_device *origin_md = NULL; 190309ee96b2SMikulas Patocka bool must_restart_merging = false; 1904c1f0c183SMike Snitzer 1905c1f0c183SMike Snitzer down_read(&_origins_lock); 1906b735fedeSMikulas Patocka 1907b735fedeSMikulas Patocka o = __lookup_dm_origin(s->origin->bdev); 1908b735fedeSMikulas Patocka if (o) 1909b735fedeSMikulas Patocka origin_md = dm_table_get_md(o->ti->table); 191009ee96b2SMikulas Patocka if (!origin_md) { 191109ee96b2SMikulas Patocka (void) __find_snapshots_sharing_cow(s, NULL, NULL, &snap_merging); 191209ee96b2SMikulas Patocka if (snap_merging) 191309ee96b2SMikulas Patocka origin_md = dm_table_get_md(snap_merging->ti->table); 191409ee96b2SMikulas Patocka } 1915b735fedeSMikulas Patocka if (origin_md == dm_table_get_md(ti->table)) 1916b735fedeSMikulas Patocka origin_md = NULL; 191709ee96b2SMikulas Patocka if (origin_md) { 191809ee96b2SMikulas Patocka if (dm_hold(origin_md)) 191909ee96b2SMikulas Patocka origin_md = NULL; 192009ee96b2SMikulas Patocka } 1921b735fedeSMikulas Patocka 192209ee96b2SMikulas Patocka up_read(&_origins_lock); 192309ee96b2SMikulas Patocka 192409ee96b2SMikulas Patocka if (origin_md) { 1925b735fedeSMikulas Patocka dm_internal_suspend_fast(origin_md); 192609ee96b2SMikulas Patocka if (snap_merging && test_bit(RUNNING_MERGE, &snap_merging->state_bits)) { 192709ee96b2SMikulas Patocka must_restart_merging = true; 192809ee96b2SMikulas Patocka stop_merge(snap_merging); 192909ee96b2SMikulas Patocka } 193009ee96b2SMikulas Patocka } 193109ee96b2SMikulas Patocka 193209ee96b2SMikulas Patocka down_read(&_origins_lock); 1933b735fedeSMikulas Patocka 19349d3b15c4SMikulas Patocka (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL); 1935c1f0c183SMike Snitzer if (snap_src && snap_dest) { 1936c1f0c183SMike Snitzer down_write(&snap_src->lock); 1937c1f0c183SMike Snitzer down_write_nested(&snap_dest->lock, SINGLE_DEPTH_NESTING); 1938c1f0c183SMike Snitzer __handover_exceptions(snap_src, snap_dest); 1939c1f0c183SMike Snitzer up_write(&snap_dest->lock); 1940c1f0c183SMike Snitzer up_write(&snap_src->lock); 1941c1f0c183SMike Snitzer } 1942b735fedeSMikulas Patocka 1943c1f0c183SMike Snitzer up_read(&_origins_lock); 1944c1f0c183SMike Snitzer 194509ee96b2SMikulas Patocka if (origin_md) { 194609ee96b2SMikulas Patocka if (must_restart_merging) 194709ee96b2SMikulas Patocka start_merge(snap_merging); 194809ee96b2SMikulas Patocka dm_internal_resume_fast(origin_md); 194909ee96b2SMikulas Patocka dm_put(origin_md); 195009ee96b2SMikulas Patocka } 195109ee96b2SMikulas Patocka 1952c1f0c183SMike Snitzer /* Now we have correct chunk size, reregister */ 1953c1f0c183SMike Snitzer reregister_snapshot(s); 19541da177e4SLinus Torvalds 1955aa14edebSAlasdair G Kergon down_write(&s->lock); 1956aa14edebSAlasdair G Kergon s->active = 1; 1957aa14edebSAlasdair G Kergon up_write(&s->lock); 19581da177e4SLinus Torvalds } 19591da177e4SLinus Torvalds 1960542f9038SMike Snitzer static uint32_t get_origin_minimum_chunksize(struct block_device *bdev) 19611e03f97eSMikulas Patocka { 1962542f9038SMike Snitzer uint32_t min_chunksize; 19631e03f97eSMikulas Patocka 19641e03f97eSMikulas Patocka down_read(&_origins_lock); 19651e03f97eSMikulas Patocka min_chunksize = __minimum_chunk_size(__lookup_origin(bdev)); 19661e03f97eSMikulas Patocka up_read(&_origins_lock); 19671e03f97eSMikulas Patocka 19681e03f97eSMikulas Patocka return min_chunksize; 19691e03f97eSMikulas Patocka } 19701e03f97eSMikulas Patocka 19711e03f97eSMikulas Patocka static void snapshot_merge_resume(struct dm_target *ti) 19721e03f97eSMikulas Patocka { 19731e03f97eSMikulas Patocka struct dm_snapshot *s = ti->private; 19741e03f97eSMikulas Patocka 19751e03f97eSMikulas Patocka /* 19761e03f97eSMikulas Patocka * Handover exceptions from existing snapshot. 19771e03f97eSMikulas Patocka */ 19781e03f97eSMikulas Patocka snapshot_resume(ti); 19791e03f97eSMikulas Patocka 19801e03f97eSMikulas Patocka /* 1981542f9038SMike Snitzer * snapshot-merge acts as an origin, so set ti->max_io_len 19821e03f97eSMikulas Patocka */ 1983542f9038SMike Snitzer ti->max_io_len = get_origin_minimum_chunksize(s->origin->bdev); 19841e03f97eSMikulas Patocka 19851e03f97eSMikulas Patocka start_merge(s); 19861e03f97eSMikulas Patocka } 19871e03f97eSMikulas Patocka 1988fd7c092eSMikulas Patocka static void snapshot_status(struct dm_target *ti, status_type_t type, 19891f4e0ff0SAlasdair G Kergon unsigned status_flags, char *result, unsigned maxlen) 19901da177e4SLinus Torvalds { 19912e4a31dfSJonathan Brassow unsigned sz = 0; 1992028867acSAlasdair G Kergon struct dm_snapshot *snap = ti->private; 19931da177e4SLinus Torvalds 19941da177e4SLinus Torvalds switch (type) { 19951da177e4SLinus Torvalds case STATUSTYPE_INFO: 199694e76572SMikulas Patocka 199794e76572SMikulas Patocka down_write(&snap->lock); 199894e76572SMikulas Patocka 19991da177e4SLinus Torvalds if (!snap->valid) 20002e4a31dfSJonathan Brassow DMEMIT("Invalid"); 2001d8ddb1cfSMike Snitzer else if (snap->merge_failed) 2002d8ddb1cfSMike Snitzer DMEMIT("Merge failed"); 200376c44f6dSMikulas Patocka else if (snap->snapshot_overflowed) 200476c44f6dSMikulas Patocka DMEMIT("Overflow"); 20051da177e4SLinus Torvalds else { 2006985903bbSMike Snitzer if (snap->store->type->usage) { 2007985903bbSMike Snitzer sector_t total_sectors, sectors_allocated, 2008985903bbSMike Snitzer metadata_sectors; 2009985903bbSMike Snitzer snap->store->type->usage(snap->store, 2010985903bbSMike Snitzer &total_sectors, 2011985903bbSMike Snitzer §ors_allocated, 2012985903bbSMike Snitzer &metadata_sectors); 2013985903bbSMike Snitzer DMEMIT("%llu/%llu %llu", 2014985903bbSMike Snitzer (unsigned long long)sectors_allocated, 2015985903bbSMike Snitzer (unsigned long long)total_sectors, 2016985903bbSMike Snitzer (unsigned long long)metadata_sectors); 20171da177e4SLinus Torvalds } 20181da177e4SLinus Torvalds else 20192e4a31dfSJonathan Brassow DMEMIT("Unknown"); 20201da177e4SLinus Torvalds } 202194e76572SMikulas Patocka 202294e76572SMikulas Patocka up_write(&snap->lock); 202394e76572SMikulas Patocka 20241da177e4SLinus Torvalds break; 20251da177e4SLinus Torvalds 20261da177e4SLinus Torvalds case STATUSTYPE_TABLE: 20271da177e4SLinus Torvalds /* 20281da177e4SLinus Torvalds * kdevname returns a static pointer so we need 20291da177e4SLinus Torvalds * to make private copies if the output is to 20301da177e4SLinus Torvalds * make sense. 20311da177e4SLinus Torvalds */ 2032fc56f6fbSMike Snitzer DMEMIT("%s %s", snap->origin->name, snap->cow->name); 20331e302a92SJonathan Brassow snap->store->type->status(snap->store, type, result + sz, 20341e302a92SJonathan Brassow maxlen - sz); 20351da177e4SLinus Torvalds break; 20361da177e4SLinus Torvalds } 20371da177e4SLinus Torvalds } 20381da177e4SLinus Torvalds 20398811f46cSMike Snitzer static int snapshot_iterate_devices(struct dm_target *ti, 20408811f46cSMike Snitzer iterate_devices_callout_fn fn, void *data) 20418811f46cSMike Snitzer { 20428811f46cSMike Snitzer struct dm_snapshot *snap = ti->private; 20431e5554c8SMikulas Patocka int r; 20448811f46cSMike Snitzer 20451e5554c8SMikulas Patocka r = fn(ti, snap->origin, 0, ti->len, data); 20461e5554c8SMikulas Patocka 20471e5554c8SMikulas Patocka if (!r) 20481e5554c8SMikulas Patocka r = fn(ti, snap->cow, 0, get_dev_size(snap->cow->bdev), data); 20491e5554c8SMikulas Patocka 20501e5554c8SMikulas Patocka return r; 20518811f46cSMike Snitzer } 20528811f46cSMike Snitzer 20538811f46cSMike Snitzer 20541da177e4SLinus Torvalds /*----------------------------------------------------------------- 20551da177e4SLinus Torvalds * Origin methods 20561da177e4SLinus Torvalds *---------------------------------------------------------------*/ 20579eaae8ffSMikulas Patocka 20589eaae8ffSMikulas Patocka /* 20599eaae8ffSMikulas Patocka * If no exceptions need creating, DM_MAPIO_REMAPPED is returned and any 20609eaae8ffSMikulas Patocka * supplied bio was ignored. The caller may submit it immediately. 20619eaae8ffSMikulas Patocka * (No remapping actually occurs as the origin is always a direct linear 20629eaae8ffSMikulas Patocka * map.) 20639eaae8ffSMikulas Patocka * 20649eaae8ffSMikulas Patocka * If further exceptions are required, DM_MAPIO_SUBMITTED is returned 20659eaae8ffSMikulas Patocka * and any supplied bio is added to a list to be submitted once all 20669eaae8ffSMikulas Patocka * the necessary exceptions exist. 20679eaae8ffSMikulas Patocka */ 20689eaae8ffSMikulas Patocka static int __origin_write(struct list_head *snapshots, sector_t sector, 20699eaae8ffSMikulas Patocka struct bio *bio) 20701da177e4SLinus Torvalds { 2071515ad66cSMikulas Patocka int r = DM_MAPIO_REMAPPED; 20721da177e4SLinus Torvalds struct dm_snapshot *snap; 20731d4989c8SJon Brassow struct dm_exception *e; 2074515ad66cSMikulas Patocka struct dm_snap_pending_exception *pe; 2075515ad66cSMikulas Patocka struct dm_snap_pending_exception *pe_to_start_now = NULL; 2076515ad66cSMikulas Patocka struct dm_snap_pending_exception *pe_to_start_last = NULL; 20771da177e4SLinus Torvalds chunk_t chunk; 20781da177e4SLinus Torvalds 20791da177e4SLinus Torvalds /* Do all the snapshots on this origin */ 20801da177e4SLinus Torvalds list_for_each_entry (snap, snapshots, list) { 20813452c2a1SMikulas Patocka /* 20823452c2a1SMikulas Patocka * Don't make new exceptions in a merging snapshot 20833452c2a1SMikulas Patocka * because it has effectively been deleted 20843452c2a1SMikulas Patocka */ 20853452c2a1SMikulas Patocka if (dm_target_is_snapshot_merge(snap->ti)) 20863452c2a1SMikulas Patocka continue; 20873452c2a1SMikulas Patocka 208876df1c65SAlasdair G Kergon down_write(&snap->lock); 208976df1c65SAlasdair G Kergon 2090aa14edebSAlasdair G Kergon /* Only deal with valid and active snapshots */ 2091aa14edebSAlasdair G Kergon if (!snap->valid || !snap->active) 209276df1c65SAlasdair G Kergon goto next_snapshot; 20931da177e4SLinus Torvalds 2094d5e404c1SAlasdair G Kergon /* Nothing to do if writing beyond end of snapshot */ 20959eaae8ffSMikulas Patocka if (sector >= dm_table_get_size(snap->ti->table)) 209676df1c65SAlasdair G Kergon goto next_snapshot; 20971da177e4SLinus Torvalds 20981da177e4SLinus Torvalds /* 20991da177e4SLinus Torvalds * Remember, different snapshots can have 21001da177e4SLinus Torvalds * different chunk sizes. 21011da177e4SLinus Torvalds */ 21029eaae8ffSMikulas Patocka chunk = sector_to_chunk(snap->store, sector); 21031da177e4SLinus Torvalds 21041da177e4SLinus Torvalds /* 21051da177e4SLinus Torvalds * Check exception table to see if block 21061da177e4SLinus Torvalds * is already remapped in this snapshot 21071da177e4SLinus Torvalds * and trigger an exception if not. 21081da177e4SLinus Torvalds */ 21093510cb94SJon Brassow e = dm_lookup_exception(&snap->complete, chunk); 211076df1c65SAlasdair G Kergon if (e) 211176df1c65SAlasdair G Kergon goto next_snapshot; 211276df1c65SAlasdair G Kergon 21132913808eSMikulas Patocka pe = __lookup_pending_exception(snap, chunk); 21142913808eSMikulas Patocka if (!pe) { 2115c6621392SMikulas Patocka up_write(&snap->lock); 2116c6621392SMikulas Patocka pe = alloc_pending_exception(snap); 2117c6621392SMikulas Patocka down_write(&snap->lock); 2118c6621392SMikulas Patocka 2119c6621392SMikulas Patocka if (!snap->valid) { 2120c6621392SMikulas Patocka free_pending_exception(pe); 2121c6621392SMikulas Patocka goto next_snapshot; 2122c6621392SMikulas Patocka } 2123c6621392SMikulas Patocka 21243510cb94SJon Brassow e = dm_lookup_exception(&snap->complete, chunk); 212535bf659bSMikulas Patocka if (e) { 212635bf659bSMikulas Patocka free_pending_exception(pe); 212735bf659bSMikulas Patocka goto next_snapshot; 212835bf659bSMikulas Patocka } 212935bf659bSMikulas Patocka 2130c6621392SMikulas Patocka pe = __find_pending_exception(snap, pe, chunk); 21311da177e4SLinus Torvalds if (!pe) { 2132695368acSAlasdair G Kergon __invalidate_snapshot(snap, -ENOMEM); 213376df1c65SAlasdair G Kergon goto next_snapshot; 213476df1c65SAlasdair G Kergon } 21352913808eSMikulas Patocka } 21361da177e4SLinus Torvalds 2137d2a7ad29SKiyoshi Ueda r = DM_MAPIO_SUBMITTED; 213876df1c65SAlasdair G Kergon 2139515ad66cSMikulas Patocka /* 2140515ad66cSMikulas Patocka * If an origin bio was supplied, queue it to wait for the 2141515ad66cSMikulas Patocka * completion of this exception, and start this one last, 2142515ad66cSMikulas Patocka * at the end of the function. 2143515ad66cSMikulas Patocka */ 2144515ad66cSMikulas Patocka if (bio) { 2145515ad66cSMikulas Patocka bio_list_add(&pe->origin_bios, bio); 2146515ad66cSMikulas Patocka bio = NULL; 2147515ad66cSMikulas Patocka 2148515ad66cSMikulas Patocka if (!pe->started) { 2149515ad66cSMikulas Patocka pe->started = 1; 2150515ad66cSMikulas Patocka pe_to_start_last = pe; 2151515ad66cSMikulas Patocka } 2152b4b610f6SAlasdair G Kergon } 215376df1c65SAlasdair G Kergon 2154eccf0817SAlasdair G Kergon if (!pe->started) { 2155eccf0817SAlasdair G Kergon pe->started = 1; 2156515ad66cSMikulas Patocka pe_to_start_now = pe; 2157eccf0817SAlasdair G Kergon } 21581da177e4SLinus Torvalds 215976df1c65SAlasdair G Kergon next_snapshot: 21601da177e4SLinus Torvalds up_write(&snap->lock); 2161515ad66cSMikulas Patocka 2162515ad66cSMikulas Patocka if (pe_to_start_now) { 2163515ad66cSMikulas Patocka start_copy(pe_to_start_now); 2164515ad66cSMikulas Patocka pe_to_start_now = NULL; 21651da177e4SLinus Torvalds } 2166b4b610f6SAlasdair G Kergon } 2167b4b610f6SAlasdair G Kergon 21681da177e4SLinus Torvalds /* 2169515ad66cSMikulas Patocka * Submit the exception against which the bio is queued last, 2170515ad66cSMikulas Patocka * to give the other exceptions a head start. 21711da177e4SLinus Torvalds */ 2172515ad66cSMikulas Patocka if (pe_to_start_last) 2173515ad66cSMikulas Patocka start_copy(pe_to_start_last); 21741da177e4SLinus Torvalds 21751da177e4SLinus Torvalds return r; 21761da177e4SLinus Torvalds } 21771da177e4SLinus Torvalds 21781da177e4SLinus Torvalds /* 21791da177e4SLinus Torvalds * Called on a write from the origin driver. 21801da177e4SLinus Torvalds */ 21811da177e4SLinus Torvalds static int do_origin(struct dm_dev *origin, struct bio *bio) 21821da177e4SLinus Torvalds { 21831da177e4SLinus Torvalds struct origin *o; 2184d2a7ad29SKiyoshi Ueda int r = DM_MAPIO_REMAPPED; 21851da177e4SLinus Torvalds 21861da177e4SLinus Torvalds down_read(&_origins_lock); 21871da177e4SLinus Torvalds o = __lookup_origin(origin->bdev); 21881da177e4SLinus Torvalds if (o) 21894f024f37SKent Overstreet r = __origin_write(&o->snapshots, bio->bi_iter.bi_sector, bio); 21901da177e4SLinus Torvalds up_read(&_origins_lock); 21911da177e4SLinus Torvalds 21921da177e4SLinus Torvalds return r; 21931da177e4SLinus Torvalds } 21941da177e4SLinus Torvalds 21951da177e4SLinus Torvalds /* 219673dfd078SMikulas Patocka * Trigger exceptions in all non-merging snapshots. 219773dfd078SMikulas Patocka * 219873dfd078SMikulas Patocka * The chunk size of the merging snapshot may be larger than the chunk 219973dfd078SMikulas Patocka * size of some other snapshot so we may need to reallocate multiple 220073dfd078SMikulas Patocka * chunks in other snapshots. 220173dfd078SMikulas Patocka * 220273dfd078SMikulas Patocka * We scan all the overlapping exceptions in the other snapshots. 220373dfd078SMikulas Patocka * Returns 1 if anything was reallocated and must be waited for, 220473dfd078SMikulas Patocka * otherwise returns 0. 220573dfd078SMikulas Patocka * 220673dfd078SMikulas Patocka * size must be a multiple of merging_snap's chunk_size. 220773dfd078SMikulas Patocka */ 220873dfd078SMikulas Patocka static int origin_write_extent(struct dm_snapshot *merging_snap, 220973dfd078SMikulas Patocka sector_t sector, unsigned size) 221073dfd078SMikulas Patocka { 221173dfd078SMikulas Patocka int must_wait = 0; 221273dfd078SMikulas Patocka sector_t n; 221373dfd078SMikulas Patocka struct origin *o; 221473dfd078SMikulas Patocka 221573dfd078SMikulas Patocka /* 2216542f9038SMike Snitzer * The origin's __minimum_chunk_size() got stored in max_io_len 221773dfd078SMikulas Patocka * by snapshot_merge_resume(). 221873dfd078SMikulas Patocka */ 221973dfd078SMikulas Patocka down_read(&_origins_lock); 222073dfd078SMikulas Patocka o = __lookup_origin(merging_snap->origin->bdev); 2221542f9038SMike Snitzer for (n = 0; n < size; n += merging_snap->ti->max_io_len) 222273dfd078SMikulas Patocka if (__origin_write(&o->snapshots, sector + n, NULL) == 222373dfd078SMikulas Patocka DM_MAPIO_SUBMITTED) 222473dfd078SMikulas Patocka must_wait = 1; 222573dfd078SMikulas Patocka up_read(&_origins_lock); 222673dfd078SMikulas Patocka 222773dfd078SMikulas Patocka return must_wait; 222873dfd078SMikulas Patocka } 222973dfd078SMikulas Patocka 223073dfd078SMikulas Patocka /* 22311da177e4SLinus Torvalds * Origin: maps a linear range of a device, with hooks for snapshotting. 22321da177e4SLinus Torvalds */ 22331da177e4SLinus Torvalds 22341da177e4SLinus Torvalds /* 22351da177e4SLinus Torvalds * Construct an origin mapping: <dev_path> 22361da177e4SLinus Torvalds * The context for an origin is merely a 'struct dm_dev *' 22371da177e4SLinus Torvalds * pointing to the real device. 22381da177e4SLinus Torvalds */ 22391da177e4SLinus Torvalds static int origin_ctr(struct dm_target *ti, unsigned int argc, char **argv) 22401da177e4SLinus Torvalds { 22411da177e4SLinus Torvalds int r; 2242599cdf3bSMikulas Patocka struct dm_origin *o; 22431da177e4SLinus Torvalds 22441da177e4SLinus Torvalds if (argc != 1) { 224572d94861SAlasdair G Kergon ti->error = "origin: incorrect number of arguments"; 22461da177e4SLinus Torvalds return -EINVAL; 22471da177e4SLinus Torvalds } 22481da177e4SLinus Torvalds 2249599cdf3bSMikulas Patocka o = kmalloc(sizeof(struct dm_origin), GFP_KERNEL); 2250599cdf3bSMikulas Patocka if (!o) { 2251599cdf3bSMikulas Patocka ti->error = "Cannot allocate private origin structure"; 2252599cdf3bSMikulas Patocka r = -ENOMEM; 2253599cdf3bSMikulas Patocka goto bad_alloc; 22541da177e4SLinus Torvalds } 22551da177e4SLinus Torvalds 2256599cdf3bSMikulas Patocka r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &o->dev); 2257599cdf3bSMikulas Patocka if (r) { 2258599cdf3bSMikulas Patocka ti->error = "Cannot get target device"; 2259599cdf3bSMikulas Patocka goto bad_open; 2260599cdf3bSMikulas Patocka } 2261599cdf3bSMikulas Patocka 2262b735fedeSMikulas Patocka o->ti = ti; 2263599cdf3bSMikulas Patocka ti->private = o; 226455a62eefSAlasdair G Kergon ti->num_flush_bios = 1; 2265494b3ee7SMikulas Patocka 22661da177e4SLinus Torvalds return 0; 2267599cdf3bSMikulas Patocka 2268599cdf3bSMikulas Patocka bad_open: 2269599cdf3bSMikulas Patocka kfree(o); 2270599cdf3bSMikulas Patocka bad_alloc: 2271599cdf3bSMikulas Patocka return r; 22721da177e4SLinus Torvalds } 22731da177e4SLinus Torvalds 22741da177e4SLinus Torvalds static void origin_dtr(struct dm_target *ti) 22751da177e4SLinus Torvalds { 2276599cdf3bSMikulas Patocka struct dm_origin *o = ti->private; 2277b735fedeSMikulas Patocka 2278599cdf3bSMikulas Patocka dm_put_device(ti, o->dev); 2279599cdf3bSMikulas Patocka kfree(o); 22801da177e4SLinus Torvalds } 22811da177e4SLinus Torvalds 22827de3ee57SMikulas Patocka static int origin_map(struct dm_target *ti, struct bio *bio) 22831da177e4SLinus Torvalds { 2284599cdf3bSMikulas Patocka struct dm_origin *o = ti->private; 2285298eaa89SMikulas Patocka unsigned available_sectors; 22861da177e4SLinus Torvalds 2287599cdf3bSMikulas Patocka bio->bi_bdev = o->dev->bdev; 22881da177e4SLinus Torvalds 22891eff9d32SJens Axboe if (unlikely(bio->bi_opf & REQ_PREFLUSH)) 2290494b3ee7SMikulas Patocka return DM_MAPIO_REMAPPED; 2291494b3ee7SMikulas Patocka 229270246286SChristoph Hellwig if (bio_data_dir(bio) != WRITE) 2293298eaa89SMikulas Patocka return DM_MAPIO_REMAPPED; 2294298eaa89SMikulas Patocka 2295298eaa89SMikulas Patocka available_sectors = o->split_boundary - 2296298eaa89SMikulas Patocka ((unsigned)bio->bi_iter.bi_sector & (o->split_boundary - 1)); 2297298eaa89SMikulas Patocka 2298298eaa89SMikulas Patocka if (bio_sectors(bio) > available_sectors) 2299298eaa89SMikulas Patocka dm_accept_partial_bio(bio, available_sectors); 2300298eaa89SMikulas Patocka 23011da177e4SLinus Torvalds /* Only tell snapshots if this is a write */ 2302298eaa89SMikulas Patocka return do_origin(o->dev, bio); 23031da177e4SLinus Torvalds } 23041da177e4SLinus Torvalds 2305f6e629bdSToshi Kani static long origin_direct_access(struct dm_target *ti, sector_t sector, 2306f0c98ebcSLinus Torvalds void **kaddr, pfn_t *pfn, long size) 2307f6e629bdSToshi Kani { 2308f6e629bdSToshi Kani DMWARN("device does not support dax."); 2309f6e629bdSToshi Kani return -EIO; 2310f6e629bdSToshi Kani } 2311f6e629bdSToshi Kani 23121da177e4SLinus Torvalds /* 2313542f9038SMike Snitzer * Set the target "max_io_len" field to the minimum of all the snapshots' 23141da177e4SLinus Torvalds * chunk sizes. 23151da177e4SLinus Torvalds */ 23161da177e4SLinus Torvalds static void origin_resume(struct dm_target *ti) 23171da177e4SLinus Torvalds { 2318599cdf3bSMikulas Patocka struct dm_origin *o = ti->private; 23191da177e4SLinus Torvalds 2320298eaa89SMikulas Patocka o->split_boundary = get_origin_minimum_chunksize(o->dev->bdev); 2321b735fedeSMikulas Patocka 2322b735fedeSMikulas Patocka down_write(&_origins_lock); 2323b735fedeSMikulas Patocka __insert_dm_origin(o); 2324b735fedeSMikulas Patocka up_write(&_origins_lock); 2325b735fedeSMikulas Patocka } 2326b735fedeSMikulas Patocka 2327b735fedeSMikulas Patocka static void origin_postsuspend(struct dm_target *ti) 2328b735fedeSMikulas Patocka { 2329b735fedeSMikulas Patocka struct dm_origin *o = ti->private; 2330b735fedeSMikulas Patocka 2331b735fedeSMikulas Patocka down_write(&_origins_lock); 2332b735fedeSMikulas Patocka __remove_dm_origin(o); 2333b735fedeSMikulas Patocka up_write(&_origins_lock); 23341da177e4SLinus Torvalds } 23351da177e4SLinus Torvalds 2336fd7c092eSMikulas Patocka static void origin_status(struct dm_target *ti, status_type_t type, 23371f4e0ff0SAlasdair G Kergon unsigned status_flags, char *result, unsigned maxlen) 23381da177e4SLinus Torvalds { 2339599cdf3bSMikulas Patocka struct dm_origin *o = ti->private; 23401da177e4SLinus Torvalds 23411da177e4SLinus Torvalds switch (type) { 23421da177e4SLinus Torvalds case STATUSTYPE_INFO: 23431da177e4SLinus Torvalds result[0] = '\0'; 23441da177e4SLinus Torvalds break; 23451da177e4SLinus Torvalds 23461da177e4SLinus Torvalds case STATUSTYPE_TABLE: 2347599cdf3bSMikulas Patocka snprintf(result, maxlen, "%s", o->dev->name); 23481da177e4SLinus Torvalds break; 23491da177e4SLinus Torvalds } 23501da177e4SLinus Torvalds } 23511da177e4SLinus Torvalds 23528811f46cSMike Snitzer static int origin_iterate_devices(struct dm_target *ti, 23538811f46cSMike Snitzer iterate_devices_callout_fn fn, void *data) 23548811f46cSMike Snitzer { 2355599cdf3bSMikulas Patocka struct dm_origin *o = ti->private; 23568811f46cSMike Snitzer 2357599cdf3bSMikulas Patocka return fn(ti, o->dev, 0, ti->len, data); 23588811f46cSMike Snitzer } 23598811f46cSMike Snitzer 23601da177e4SLinus Torvalds static struct target_type origin_target = { 23611da177e4SLinus Torvalds .name = "snapshot-origin", 2362b735fedeSMikulas Patocka .version = {1, 9, 0}, 23631da177e4SLinus Torvalds .module = THIS_MODULE, 23641da177e4SLinus Torvalds .ctr = origin_ctr, 23651da177e4SLinus Torvalds .dtr = origin_dtr, 23661da177e4SLinus Torvalds .map = origin_map, 23671da177e4SLinus Torvalds .resume = origin_resume, 2368b735fedeSMikulas Patocka .postsuspend = origin_postsuspend, 23691da177e4SLinus Torvalds .status = origin_status, 23708811f46cSMike Snitzer .iterate_devices = origin_iterate_devices, 2371f6e629bdSToshi Kani .direct_access = origin_direct_access, 23721da177e4SLinus Torvalds }; 23731da177e4SLinus Torvalds 23741da177e4SLinus Torvalds static struct target_type snapshot_target = { 23751da177e4SLinus Torvalds .name = "snapshot", 2376b0d3cc01SMike Snitzer .version = {1, 15, 0}, 23771da177e4SLinus Torvalds .module = THIS_MODULE, 23781da177e4SLinus Torvalds .ctr = snapshot_ctr, 23791da177e4SLinus Torvalds .dtr = snapshot_dtr, 23801da177e4SLinus Torvalds .map = snapshot_map, 2381cd45daffSMikulas Patocka .end_io = snapshot_end_io, 2382c1f0c183SMike Snitzer .preresume = snapshot_preresume, 23831da177e4SLinus Torvalds .resume = snapshot_resume, 23841da177e4SLinus Torvalds .status = snapshot_status, 23858811f46cSMike Snitzer .iterate_devices = snapshot_iterate_devices, 23861da177e4SLinus Torvalds }; 23871da177e4SLinus Torvalds 2388d698aa45SMikulas Patocka static struct target_type merge_target = { 2389d698aa45SMikulas Patocka .name = dm_snapshot_merge_target_name, 2390b0d3cc01SMike Snitzer .version = {1, 4, 0}, 2391d698aa45SMikulas Patocka .module = THIS_MODULE, 2392d698aa45SMikulas Patocka .ctr = snapshot_ctr, 2393d698aa45SMikulas Patocka .dtr = snapshot_dtr, 23943452c2a1SMikulas Patocka .map = snapshot_merge_map, 2395d698aa45SMikulas Patocka .end_io = snapshot_end_io, 23961e03f97eSMikulas Patocka .presuspend = snapshot_merge_presuspend, 2397d698aa45SMikulas Patocka .preresume = snapshot_preresume, 23981e03f97eSMikulas Patocka .resume = snapshot_merge_resume, 2399d698aa45SMikulas Patocka .status = snapshot_status, 2400d698aa45SMikulas Patocka .iterate_devices = snapshot_iterate_devices, 2401d698aa45SMikulas Patocka }; 2402d698aa45SMikulas Patocka 24031da177e4SLinus Torvalds static int __init dm_snapshot_init(void) 24041da177e4SLinus Torvalds { 24051da177e4SLinus Torvalds int r; 24061da177e4SLinus Torvalds 24074db6bfe0SAlasdair G Kergon r = dm_exception_store_init(); 24084db6bfe0SAlasdair G Kergon if (r) { 24094db6bfe0SAlasdair G Kergon DMERR("Failed to initialize exception stores"); 24104db6bfe0SAlasdair G Kergon return r; 24114db6bfe0SAlasdair G Kergon } 24124db6bfe0SAlasdair G Kergon 24131da177e4SLinus Torvalds r = dm_register_target(&snapshot_target); 2414d698aa45SMikulas Patocka if (r < 0) { 24151da177e4SLinus Torvalds DMERR("snapshot target register failed %d", r); 2416034a186dSJonathan Brassow goto bad_register_snapshot_target; 24171da177e4SLinus Torvalds } 24181da177e4SLinus Torvalds 24191da177e4SLinus Torvalds r = dm_register_target(&origin_target); 24201da177e4SLinus Torvalds if (r < 0) { 242172d94861SAlasdair G Kergon DMERR("Origin target register failed %d", r); 2422d698aa45SMikulas Patocka goto bad_register_origin_target; 2423d698aa45SMikulas Patocka } 2424d698aa45SMikulas Patocka 2425d698aa45SMikulas Patocka r = dm_register_target(&merge_target); 2426d698aa45SMikulas Patocka if (r < 0) { 2427d698aa45SMikulas Patocka DMERR("Merge target register failed %d", r); 2428d698aa45SMikulas Patocka goto bad_register_merge_target; 24291da177e4SLinus Torvalds } 24301da177e4SLinus Torvalds 24311da177e4SLinus Torvalds r = init_origin_hash(); 24321da177e4SLinus Torvalds if (r) { 24331da177e4SLinus Torvalds DMERR("init_origin_hash failed."); 2434d698aa45SMikulas Patocka goto bad_origin_hash; 24351da177e4SLinus Torvalds } 24361da177e4SLinus Torvalds 24371d4989c8SJon Brassow exception_cache = KMEM_CACHE(dm_exception, 0); 24381da177e4SLinus Torvalds if (!exception_cache) { 24391da177e4SLinus Torvalds DMERR("Couldn't create exception cache."); 24401da177e4SLinus Torvalds r = -ENOMEM; 2441d698aa45SMikulas Patocka goto bad_exception_cache; 24421da177e4SLinus Torvalds } 24431da177e4SLinus Torvalds 2444028867acSAlasdair G Kergon pending_cache = KMEM_CACHE(dm_snap_pending_exception, 0); 24451da177e4SLinus Torvalds if (!pending_cache) { 24461da177e4SLinus Torvalds DMERR("Couldn't create pending cache."); 24471da177e4SLinus Torvalds r = -ENOMEM; 2448d698aa45SMikulas Patocka goto bad_pending_cache; 24491da177e4SLinus Torvalds } 24501da177e4SLinus Torvalds 24511da177e4SLinus Torvalds return 0; 24521da177e4SLinus Torvalds 2453d698aa45SMikulas Patocka bad_pending_cache: 24541da177e4SLinus Torvalds kmem_cache_destroy(exception_cache); 2455d698aa45SMikulas Patocka bad_exception_cache: 24561da177e4SLinus Torvalds exit_origin_hash(); 2457d698aa45SMikulas Patocka bad_origin_hash: 2458d698aa45SMikulas Patocka dm_unregister_target(&merge_target); 2459d698aa45SMikulas Patocka bad_register_merge_target: 24601da177e4SLinus Torvalds dm_unregister_target(&origin_target); 2461d698aa45SMikulas Patocka bad_register_origin_target: 24621da177e4SLinus Torvalds dm_unregister_target(&snapshot_target); 2463034a186dSJonathan Brassow bad_register_snapshot_target: 2464034a186dSJonathan Brassow dm_exception_store_exit(); 2465d698aa45SMikulas Patocka 24661da177e4SLinus Torvalds return r; 24671da177e4SLinus Torvalds } 24681da177e4SLinus Torvalds 24691da177e4SLinus Torvalds static void __exit dm_snapshot_exit(void) 24701da177e4SLinus Torvalds { 247110d3bd09SMikulas Patocka dm_unregister_target(&snapshot_target); 247210d3bd09SMikulas Patocka dm_unregister_target(&origin_target); 2473d698aa45SMikulas Patocka dm_unregister_target(&merge_target); 24741da177e4SLinus Torvalds 24751da177e4SLinus Torvalds exit_origin_hash(); 24761da177e4SLinus Torvalds kmem_cache_destroy(pending_cache); 24771da177e4SLinus Torvalds kmem_cache_destroy(exception_cache); 24784db6bfe0SAlasdair G Kergon 24794db6bfe0SAlasdair G Kergon dm_exception_store_exit(); 24801da177e4SLinus Torvalds } 24811da177e4SLinus Torvalds 24821da177e4SLinus Torvalds /* Module hooks */ 24831da177e4SLinus Torvalds module_init(dm_snapshot_init); 24841da177e4SLinus Torvalds module_exit(dm_snapshot_exit); 24851da177e4SLinus Torvalds 24861da177e4SLinus Torvalds MODULE_DESCRIPTION(DM_NAME " snapshot target"); 24871da177e4SLinus Torvalds MODULE_AUTHOR("Joe Thornber"); 24881da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 248923cb2109SMikulas Patocka MODULE_ALIAS("dm-snapshot-origin"); 249023cb2109SMikulas Patocka MODULE_ALIAS("dm-snapshot-merge"); 2491