11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * dm-snapshot.c 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Copyright (C) 2001-2002 Sistina Software (UK) Limited. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * This file is released under the GPL. 71da177e4SLinus Torvalds */ 81da177e4SLinus Torvalds 91da177e4SLinus Torvalds #include <linux/blkdev.h> 101da177e4SLinus Torvalds #include <linux/device-mapper.h> 1190fa1527SMikulas Patocka #include <linux/delay.h> 121da177e4SLinus Torvalds #include <linux/fs.h> 131da177e4SLinus Torvalds #include <linux/init.h> 141da177e4SLinus Torvalds #include <linux/kdev_t.h> 151da177e4SLinus Torvalds #include <linux/list.h> 161da177e4SLinus Torvalds #include <linux/mempool.h> 171da177e4SLinus Torvalds #include <linux/module.h> 181da177e4SLinus Torvalds #include <linux/slab.h> 191da177e4SLinus Torvalds #include <linux/vmalloc.h> 206f3c3f0aSvignesh babu #include <linux/log2.h> 21a765e20eSAlasdair G Kergon #include <linux/dm-kcopyd.h> 221da177e4SLinus Torvalds 23aea53d92SJonathan Brassow #include "dm-exception-store.h" 241da177e4SLinus Torvalds 2572d94861SAlasdair G Kergon #define DM_MSG_PREFIX "snapshots" 2672d94861SAlasdair G Kergon 27d698aa45SMikulas Patocka static const char dm_snapshot_merge_target_name[] = "snapshot-merge"; 28d698aa45SMikulas Patocka 29d698aa45SMikulas Patocka #define dm_target_is_snapshot_merge(ti) \ 30d698aa45SMikulas Patocka ((ti)->type->name == dm_snapshot_merge_target_name) 31d698aa45SMikulas Patocka 321da177e4SLinus Torvalds /* 33cd45daffSMikulas Patocka * The size of the mempool used to track chunks in use. 34cd45daffSMikulas Patocka */ 35cd45daffSMikulas Patocka #define MIN_IOS 256 36cd45daffSMikulas Patocka 37ccc45ea8SJonathan Brassow #define DM_TRACKED_CHUNK_HASH_SIZE 16 38ccc45ea8SJonathan Brassow #define DM_TRACKED_CHUNK_HASH(x) ((unsigned long)(x) & \ 39ccc45ea8SJonathan Brassow (DM_TRACKED_CHUNK_HASH_SIZE - 1)) 40ccc45ea8SJonathan Brassow 41191437a5SJon Brassow struct dm_exception_table { 42ccc45ea8SJonathan Brassow uint32_t hash_mask; 43ccc45ea8SJonathan Brassow unsigned hash_shift; 44ccc45ea8SJonathan Brassow struct list_head *table; 45ccc45ea8SJonathan Brassow }; 46ccc45ea8SJonathan Brassow 47ccc45ea8SJonathan Brassow struct dm_snapshot { 48ccc45ea8SJonathan Brassow struct rw_semaphore lock; 49ccc45ea8SJonathan Brassow 50ccc45ea8SJonathan Brassow struct dm_dev *origin; 51fc56f6fbSMike Snitzer struct dm_dev *cow; 52fc56f6fbSMike Snitzer 53fc56f6fbSMike Snitzer struct dm_target *ti; 54ccc45ea8SJonathan Brassow 55ccc45ea8SJonathan Brassow /* List of snapshots per Origin */ 56ccc45ea8SJonathan Brassow struct list_head list; 57ccc45ea8SJonathan Brassow 58d8ddb1cfSMike Snitzer /* 59d8ddb1cfSMike Snitzer * You can't use a snapshot if this is 0 (e.g. if full). 60d8ddb1cfSMike Snitzer * A snapshot-merge target never clears this. 61d8ddb1cfSMike Snitzer */ 62ccc45ea8SJonathan Brassow int valid; 63ccc45ea8SJonathan Brassow 64ccc45ea8SJonathan Brassow /* Origin writes don't trigger exceptions until this is set */ 65ccc45ea8SJonathan Brassow int active; 66ccc45ea8SJonathan Brassow 67ccc45ea8SJonathan Brassow atomic_t pending_exceptions_count; 68ccc45ea8SJonathan Brassow 69230c83afSMikulas Patocka /* Protected by "lock" */ 70230c83afSMikulas Patocka sector_t exception_start_sequence; 71230c83afSMikulas Patocka 72230c83afSMikulas Patocka /* Protected by kcopyd single-threaded callback */ 73230c83afSMikulas Patocka sector_t exception_complete_sequence; 74230c83afSMikulas Patocka 75230c83afSMikulas Patocka /* 76230c83afSMikulas Patocka * A list of pending exceptions that completed out of order. 77230c83afSMikulas Patocka * Protected by kcopyd single-threaded callback. 78230c83afSMikulas Patocka */ 79230c83afSMikulas Patocka struct list_head out_of_order_list; 80230c83afSMikulas Patocka 81924e600dSMike Snitzer mempool_t *pending_pool; 82924e600dSMike Snitzer 83191437a5SJon Brassow struct dm_exception_table pending; 84191437a5SJon Brassow struct dm_exception_table complete; 85ccc45ea8SJonathan Brassow 86ccc45ea8SJonathan Brassow /* 87ccc45ea8SJonathan Brassow * pe_lock protects all pending_exception operations and access 88ccc45ea8SJonathan Brassow * as well as the snapshot_bios list. 89ccc45ea8SJonathan Brassow */ 90ccc45ea8SJonathan Brassow spinlock_t pe_lock; 91ccc45ea8SJonathan Brassow 92924e600dSMike Snitzer /* Chunks with outstanding reads */ 93924e600dSMike Snitzer spinlock_t tracked_chunk_lock; 94924e600dSMike Snitzer struct hlist_head tracked_chunk_hash[DM_TRACKED_CHUNK_HASH_SIZE]; 95924e600dSMike Snitzer 96ccc45ea8SJonathan Brassow /* The on disk metadata handler */ 97ccc45ea8SJonathan Brassow struct dm_exception_store *store; 98ccc45ea8SJonathan Brassow 99ccc45ea8SJonathan Brassow struct dm_kcopyd_client *kcopyd_client; 100ccc45ea8SJonathan Brassow 101924e600dSMike Snitzer /* Wait for events based on state_bits */ 102924e600dSMike Snitzer unsigned long state_bits; 103924e600dSMike Snitzer 104924e600dSMike Snitzer /* Range of chunks currently being merged. */ 105924e600dSMike Snitzer chunk_t first_merging_chunk; 106924e600dSMike Snitzer int num_merging_chunks; 1071e03f97eSMikulas Patocka 108d8ddb1cfSMike Snitzer /* 109d8ddb1cfSMike Snitzer * The merge operation failed if this flag is set. 110d8ddb1cfSMike Snitzer * Failure modes are handled as follows: 111d8ddb1cfSMike Snitzer * - I/O error reading the header 112d8ddb1cfSMike Snitzer * => don't load the target; abort. 113d8ddb1cfSMike Snitzer * - Header does not have "valid" flag set 114d8ddb1cfSMike Snitzer * => use the origin; forget about the snapshot. 115d8ddb1cfSMike Snitzer * - I/O error when reading exceptions 116d8ddb1cfSMike Snitzer * => don't load the target; abort. 117d8ddb1cfSMike Snitzer * (We can't use the intermediate origin state.) 118d8ddb1cfSMike Snitzer * - I/O error while merging 119d8ddb1cfSMike Snitzer * => stop merging; set merge_failed; process I/O normally. 120d8ddb1cfSMike Snitzer */ 121d8ddb1cfSMike Snitzer int merge_failed; 122d8ddb1cfSMike Snitzer 1239fe86254SMikulas Patocka /* 1249fe86254SMikulas Patocka * Incoming bios that overlap with chunks being merged must wait 1259fe86254SMikulas Patocka * for them to be committed. 1269fe86254SMikulas Patocka */ 1279fe86254SMikulas Patocka struct bio_list bios_queued_during_merge; 128ccc45ea8SJonathan Brassow }; 129ccc45ea8SJonathan Brassow 1301e03f97eSMikulas Patocka /* 1311e03f97eSMikulas Patocka * state_bits: 1321e03f97eSMikulas Patocka * RUNNING_MERGE - Merge operation is in progress. 1331e03f97eSMikulas Patocka * SHUTDOWN_MERGE - Set to signal that merge needs to be stopped; 1341e03f97eSMikulas Patocka * cleared afterwards. 1351e03f97eSMikulas Patocka */ 1361e03f97eSMikulas Patocka #define RUNNING_MERGE 0 1371e03f97eSMikulas Patocka #define SHUTDOWN_MERGE 1 1381e03f97eSMikulas Patocka 139df5d2e90SMikulas Patocka DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(snapshot_copy_throttle, 140df5d2e90SMikulas Patocka "A percentage of time allocated for copy on write"); 141df5d2e90SMikulas Patocka 142c2411045SMikulas Patocka struct dm_dev *dm_snap_origin(struct dm_snapshot *s) 143c2411045SMikulas Patocka { 144c2411045SMikulas Patocka return s->origin; 145c2411045SMikulas Patocka } 146c2411045SMikulas Patocka EXPORT_SYMBOL(dm_snap_origin); 147c2411045SMikulas Patocka 148fc56f6fbSMike Snitzer struct dm_dev *dm_snap_cow(struct dm_snapshot *s) 149fc56f6fbSMike Snitzer { 150fc56f6fbSMike Snitzer return s->cow; 151fc56f6fbSMike Snitzer } 152fc56f6fbSMike Snitzer EXPORT_SYMBOL(dm_snap_cow); 153fc56f6fbSMike Snitzer 154ccc45ea8SJonathan Brassow static sector_t chunk_to_sector(struct dm_exception_store *store, 155ccc45ea8SJonathan Brassow chunk_t chunk) 156ccc45ea8SJonathan Brassow { 157ccc45ea8SJonathan Brassow return chunk << store->chunk_shift; 158ccc45ea8SJonathan Brassow } 159ccc45ea8SJonathan Brassow 160ccc45ea8SJonathan Brassow static int bdev_equal(struct block_device *lhs, struct block_device *rhs) 161ccc45ea8SJonathan Brassow { 162ccc45ea8SJonathan Brassow /* 163ccc45ea8SJonathan Brassow * There is only ever one instance of a particular block 164ccc45ea8SJonathan Brassow * device so we can compare pointers safely. 165ccc45ea8SJonathan Brassow */ 166ccc45ea8SJonathan Brassow return lhs == rhs; 167ccc45ea8SJonathan Brassow } 168ccc45ea8SJonathan Brassow 169028867acSAlasdair G Kergon struct dm_snap_pending_exception { 1701d4989c8SJon Brassow struct dm_exception e; 1711da177e4SLinus Torvalds 1721da177e4SLinus Torvalds /* 1731da177e4SLinus Torvalds * Origin buffers waiting for this to complete are held 1741da177e4SLinus Torvalds * in a bio list 1751da177e4SLinus Torvalds */ 1761da177e4SLinus Torvalds struct bio_list origin_bios; 1771da177e4SLinus Torvalds struct bio_list snapshot_bios; 1781da177e4SLinus Torvalds 1791da177e4SLinus Torvalds /* Pointer back to snapshot context */ 1801da177e4SLinus Torvalds struct dm_snapshot *snap; 1811da177e4SLinus Torvalds 1821da177e4SLinus Torvalds /* 1831da177e4SLinus Torvalds * 1 indicates the exception has already been sent to 1841da177e4SLinus Torvalds * kcopyd. 1851da177e4SLinus Torvalds */ 1861da177e4SLinus Torvalds int started; 187a6e50b40SMikulas Patocka 188230c83afSMikulas Patocka /* There was copying error. */ 189230c83afSMikulas Patocka int copy_error; 190230c83afSMikulas Patocka 191230c83afSMikulas Patocka /* A sequence number, it is used for in-order completion. */ 192230c83afSMikulas Patocka sector_t exception_sequence; 193230c83afSMikulas Patocka 194230c83afSMikulas Patocka struct list_head out_of_order_entry; 195230c83afSMikulas Patocka 196a6e50b40SMikulas Patocka /* 197a6e50b40SMikulas Patocka * For writing a complete chunk, bypassing the copy. 198a6e50b40SMikulas Patocka */ 199a6e50b40SMikulas Patocka struct bio *full_bio; 200a6e50b40SMikulas Patocka bio_end_io_t *full_bio_end_io; 201a6e50b40SMikulas Patocka void *full_bio_private; 2021da177e4SLinus Torvalds }; 2031da177e4SLinus Torvalds 2041da177e4SLinus Torvalds /* 2051da177e4SLinus Torvalds * Hash table mapping origin volumes to lists of snapshots and 2061da177e4SLinus Torvalds * a lock to protect it 2071da177e4SLinus Torvalds */ 208e18b890bSChristoph Lameter static struct kmem_cache *exception_cache; 209e18b890bSChristoph Lameter static struct kmem_cache *pending_cache; 2101da177e4SLinus Torvalds 211cd45daffSMikulas Patocka struct dm_snap_tracked_chunk { 212cd45daffSMikulas Patocka struct hlist_node node; 213cd45daffSMikulas Patocka chunk_t chunk; 214cd45daffSMikulas Patocka }; 215cd45daffSMikulas Patocka 216ee18026aSMikulas Patocka static void init_tracked_chunk(struct bio *bio) 217ee18026aSMikulas Patocka { 218ee18026aSMikulas Patocka struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk)); 219ee18026aSMikulas Patocka INIT_HLIST_NODE(&c->node); 220ee18026aSMikulas Patocka } 221ee18026aSMikulas Patocka 222ee18026aSMikulas Patocka static bool is_bio_tracked(struct bio *bio) 223ee18026aSMikulas Patocka { 224ee18026aSMikulas Patocka struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk)); 225ee18026aSMikulas Patocka return !hlist_unhashed(&c->node); 226ee18026aSMikulas Patocka } 227ee18026aSMikulas Patocka 228ee18026aSMikulas Patocka static void track_chunk(struct dm_snapshot *s, struct bio *bio, chunk_t chunk) 229cd45daffSMikulas Patocka { 23042bc954fSMikulas Patocka struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk)); 231cd45daffSMikulas Patocka 232cd45daffSMikulas Patocka c->chunk = chunk; 233cd45daffSMikulas Patocka 2349aa0c0e6SMikulas Patocka spin_lock_irq(&s->tracked_chunk_lock); 235cd45daffSMikulas Patocka hlist_add_head(&c->node, 236cd45daffSMikulas Patocka &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)]); 2379aa0c0e6SMikulas Patocka spin_unlock_irq(&s->tracked_chunk_lock); 238cd45daffSMikulas Patocka } 239cd45daffSMikulas Patocka 240ee18026aSMikulas Patocka static void stop_tracking_chunk(struct dm_snapshot *s, struct bio *bio) 241cd45daffSMikulas Patocka { 242ee18026aSMikulas Patocka struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk)); 243cd45daffSMikulas Patocka unsigned long flags; 244cd45daffSMikulas Patocka 245cd45daffSMikulas Patocka spin_lock_irqsave(&s->tracked_chunk_lock, flags); 246cd45daffSMikulas Patocka hlist_del(&c->node); 247cd45daffSMikulas Patocka spin_unlock_irqrestore(&s->tracked_chunk_lock, flags); 248cd45daffSMikulas Patocka } 249cd45daffSMikulas Patocka 250a8d41b59SMikulas Patocka static int __chunk_is_tracked(struct dm_snapshot *s, chunk_t chunk) 251a8d41b59SMikulas Patocka { 252a8d41b59SMikulas Patocka struct dm_snap_tracked_chunk *c; 253a8d41b59SMikulas Patocka int found = 0; 254a8d41b59SMikulas Patocka 255a8d41b59SMikulas Patocka spin_lock_irq(&s->tracked_chunk_lock); 256a8d41b59SMikulas Patocka 257b67bfe0dSSasha Levin hlist_for_each_entry(c, 258a8d41b59SMikulas Patocka &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)], node) { 259a8d41b59SMikulas Patocka if (c->chunk == chunk) { 260a8d41b59SMikulas Patocka found = 1; 261a8d41b59SMikulas Patocka break; 262a8d41b59SMikulas Patocka } 263a8d41b59SMikulas Patocka } 264a8d41b59SMikulas Patocka 265a8d41b59SMikulas Patocka spin_unlock_irq(&s->tracked_chunk_lock); 266a8d41b59SMikulas Patocka 267a8d41b59SMikulas Patocka return found; 268a8d41b59SMikulas Patocka } 269a8d41b59SMikulas Patocka 2701da177e4SLinus Torvalds /* 271615d1eb9SMike Snitzer * This conflicting I/O is extremely improbable in the caller, 272615d1eb9SMike Snitzer * so msleep(1) is sufficient and there is no need for a wait queue. 273615d1eb9SMike Snitzer */ 274615d1eb9SMike Snitzer static void __check_for_conflicting_io(struct dm_snapshot *s, chunk_t chunk) 275615d1eb9SMike Snitzer { 276615d1eb9SMike Snitzer while (__chunk_is_tracked(s, chunk)) 277615d1eb9SMike Snitzer msleep(1); 278615d1eb9SMike Snitzer } 279615d1eb9SMike Snitzer 280615d1eb9SMike Snitzer /* 2811da177e4SLinus Torvalds * One of these per registered origin, held in the snapshot_origins hash 2821da177e4SLinus Torvalds */ 2831da177e4SLinus Torvalds struct origin { 2841da177e4SLinus Torvalds /* The origin device */ 2851da177e4SLinus Torvalds struct block_device *bdev; 2861da177e4SLinus Torvalds 2871da177e4SLinus Torvalds struct list_head hash_list; 2881da177e4SLinus Torvalds 2891da177e4SLinus Torvalds /* List of snapshots for this origin */ 2901da177e4SLinus Torvalds struct list_head snapshots; 2911da177e4SLinus Torvalds }; 2921da177e4SLinus Torvalds 2931da177e4SLinus Torvalds /* 2941da177e4SLinus Torvalds * Size of the hash table for origin volumes. If we make this 2951da177e4SLinus Torvalds * the size of the minors list then it should be nearly perfect 2961da177e4SLinus Torvalds */ 2971da177e4SLinus Torvalds #define ORIGIN_HASH_SIZE 256 2981da177e4SLinus Torvalds #define ORIGIN_MASK 0xFF 2991da177e4SLinus Torvalds static struct list_head *_origins; 3001da177e4SLinus Torvalds static struct rw_semaphore _origins_lock; 3011da177e4SLinus Torvalds 30273dfd078SMikulas Patocka static DECLARE_WAIT_QUEUE_HEAD(_pending_exceptions_done); 30373dfd078SMikulas Patocka static DEFINE_SPINLOCK(_pending_exceptions_done_spinlock); 30473dfd078SMikulas Patocka static uint64_t _pending_exceptions_done_count; 30573dfd078SMikulas Patocka 3061da177e4SLinus Torvalds static int init_origin_hash(void) 3071da177e4SLinus Torvalds { 3081da177e4SLinus Torvalds int i; 3091da177e4SLinus Torvalds 3101da177e4SLinus Torvalds _origins = kmalloc(ORIGIN_HASH_SIZE * sizeof(struct list_head), 3111da177e4SLinus Torvalds GFP_KERNEL); 3121da177e4SLinus Torvalds if (!_origins) { 31372d94861SAlasdair G Kergon DMERR("unable to allocate memory"); 3141da177e4SLinus Torvalds return -ENOMEM; 3151da177e4SLinus Torvalds } 3161da177e4SLinus Torvalds 3171da177e4SLinus Torvalds for (i = 0; i < ORIGIN_HASH_SIZE; i++) 3181da177e4SLinus Torvalds INIT_LIST_HEAD(_origins + i); 3191da177e4SLinus Torvalds init_rwsem(&_origins_lock); 3201da177e4SLinus Torvalds 3211da177e4SLinus Torvalds return 0; 3221da177e4SLinus Torvalds } 3231da177e4SLinus Torvalds 3241da177e4SLinus Torvalds static void exit_origin_hash(void) 3251da177e4SLinus Torvalds { 3261da177e4SLinus Torvalds kfree(_origins); 3271da177e4SLinus Torvalds } 3281da177e4SLinus Torvalds 329028867acSAlasdair G Kergon static unsigned origin_hash(struct block_device *bdev) 3301da177e4SLinus Torvalds { 3311da177e4SLinus Torvalds return bdev->bd_dev & ORIGIN_MASK; 3321da177e4SLinus Torvalds } 3331da177e4SLinus Torvalds 3341da177e4SLinus Torvalds static struct origin *__lookup_origin(struct block_device *origin) 3351da177e4SLinus Torvalds { 3361da177e4SLinus Torvalds struct list_head *ol; 3371da177e4SLinus Torvalds struct origin *o; 3381da177e4SLinus Torvalds 3391da177e4SLinus Torvalds ol = &_origins[origin_hash(origin)]; 3401da177e4SLinus Torvalds list_for_each_entry (o, ol, hash_list) 3411da177e4SLinus Torvalds if (bdev_equal(o->bdev, origin)) 3421da177e4SLinus Torvalds return o; 3431da177e4SLinus Torvalds 3441da177e4SLinus Torvalds return NULL; 3451da177e4SLinus Torvalds } 3461da177e4SLinus Torvalds 3471da177e4SLinus Torvalds static void __insert_origin(struct origin *o) 3481da177e4SLinus Torvalds { 3491da177e4SLinus Torvalds struct list_head *sl = &_origins[origin_hash(o->bdev)]; 3501da177e4SLinus Torvalds list_add_tail(&o->hash_list, sl); 3511da177e4SLinus Torvalds } 3521da177e4SLinus Torvalds 3531da177e4SLinus Torvalds /* 354c1f0c183SMike Snitzer * _origins_lock must be held when calling this function. 355c1f0c183SMike Snitzer * Returns number of snapshots registered using the supplied cow device, plus: 356c1f0c183SMike Snitzer * snap_src - a snapshot suitable for use as a source of exception handover 357c1f0c183SMike Snitzer * snap_dest - a snapshot capable of receiving exception handover. 3589d3b15c4SMikulas Patocka * snap_merge - an existing snapshot-merge target linked to the same origin. 3599d3b15c4SMikulas Patocka * There can be at most one snapshot-merge target. The parameter is optional. 360c1f0c183SMike Snitzer * 3619d3b15c4SMikulas Patocka * Possible return values and states of snap_src and snap_dest. 362c1f0c183SMike Snitzer * 0: NULL, NULL - first new snapshot 363c1f0c183SMike Snitzer * 1: snap_src, NULL - normal snapshot 364c1f0c183SMike Snitzer * 2: snap_src, snap_dest - waiting for handover 365c1f0c183SMike Snitzer * 2: snap_src, NULL - handed over, waiting for old to be deleted 366c1f0c183SMike Snitzer * 1: NULL, snap_dest - source got destroyed without handover 367c1f0c183SMike Snitzer */ 368c1f0c183SMike Snitzer static int __find_snapshots_sharing_cow(struct dm_snapshot *snap, 369c1f0c183SMike Snitzer struct dm_snapshot **snap_src, 3709d3b15c4SMikulas Patocka struct dm_snapshot **snap_dest, 3719d3b15c4SMikulas Patocka struct dm_snapshot **snap_merge) 372c1f0c183SMike Snitzer { 373c1f0c183SMike Snitzer struct dm_snapshot *s; 374c1f0c183SMike Snitzer struct origin *o; 375c1f0c183SMike Snitzer int count = 0; 376c1f0c183SMike Snitzer int active; 377c1f0c183SMike Snitzer 378c1f0c183SMike Snitzer o = __lookup_origin(snap->origin->bdev); 379c1f0c183SMike Snitzer if (!o) 380c1f0c183SMike Snitzer goto out; 381c1f0c183SMike Snitzer 382c1f0c183SMike Snitzer list_for_each_entry(s, &o->snapshots, list) { 3839d3b15c4SMikulas Patocka if (dm_target_is_snapshot_merge(s->ti) && snap_merge) 3849d3b15c4SMikulas Patocka *snap_merge = s; 385c1f0c183SMike Snitzer if (!bdev_equal(s->cow->bdev, snap->cow->bdev)) 386c1f0c183SMike Snitzer continue; 387c1f0c183SMike Snitzer 388c1f0c183SMike Snitzer down_read(&s->lock); 389c1f0c183SMike Snitzer active = s->active; 390c1f0c183SMike Snitzer up_read(&s->lock); 391c1f0c183SMike Snitzer 392c1f0c183SMike Snitzer if (active) { 393c1f0c183SMike Snitzer if (snap_src) 394c1f0c183SMike Snitzer *snap_src = s; 395c1f0c183SMike Snitzer } else if (snap_dest) 396c1f0c183SMike Snitzer *snap_dest = s; 397c1f0c183SMike Snitzer 398c1f0c183SMike Snitzer count++; 399c1f0c183SMike Snitzer } 400c1f0c183SMike Snitzer 401c1f0c183SMike Snitzer out: 402c1f0c183SMike Snitzer return count; 403c1f0c183SMike Snitzer } 404c1f0c183SMike Snitzer 405c1f0c183SMike Snitzer /* 406c1f0c183SMike Snitzer * On success, returns 1 if this snapshot is a handover destination, 407c1f0c183SMike Snitzer * otherwise returns 0. 408c1f0c183SMike Snitzer */ 409c1f0c183SMike Snitzer static int __validate_exception_handover(struct dm_snapshot *snap) 410c1f0c183SMike Snitzer { 411c1f0c183SMike Snitzer struct dm_snapshot *snap_src = NULL, *snap_dest = NULL; 4129d3b15c4SMikulas Patocka struct dm_snapshot *snap_merge = NULL; 413c1f0c183SMike Snitzer 414c1f0c183SMike Snitzer /* Does snapshot need exceptions handed over to it? */ 4159d3b15c4SMikulas Patocka if ((__find_snapshots_sharing_cow(snap, &snap_src, &snap_dest, 4169d3b15c4SMikulas Patocka &snap_merge) == 2) || 417c1f0c183SMike Snitzer snap_dest) { 418c1f0c183SMike Snitzer snap->ti->error = "Snapshot cow pairing for exception " 419c1f0c183SMike Snitzer "table handover failed"; 420c1f0c183SMike Snitzer return -EINVAL; 421c1f0c183SMike Snitzer } 422c1f0c183SMike Snitzer 423c1f0c183SMike Snitzer /* 424c1f0c183SMike Snitzer * If no snap_src was found, snap cannot become a handover 425c1f0c183SMike Snitzer * destination. 426c1f0c183SMike Snitzer */ 427c1f0c183SMike Snitzer if (!snap_src) 428c1f0c183SMike Snitzer return 0; 429c1f0c183SMike Snitzer 4309d3b15c4SMikulas Patocka /* 4319d3b15c4SMikulas Patocka * Non-snapshot-merge handover? 4329d3b15c4SMikulas Patocka */ 4339d3b15c4SMikulas Patocka if (!dm_target_is_snapshot_merge(snap->ti)) 4349d3b15c4SMikulas Patocka return 1; 4359d3b15c4SMikulas Patocka 4369d3b15c4SMikulas Patocka /* 4379d3b15c4SMikulas Patocka * Do not allow more than one merging snapshot. 4389d3b15c4SMikulas Patocka */ 4399d3b15c4SMikulas Patocka if (snap_merge) { 4409d3b15c4SMikulas Patocka snap->ti->error = "A snapshot is already merging."; 4419d3b15c4SMikulas Patocka return -EINVAL; 4429d3b15c4SMikulas Patocka } 4439d3b15c4SMikulas Patocka 4441e03f97eSMikulas Patocka if (!snap_src->store->type->prepare_merge || 4451e03f97eSMikulas Patocka !snap_src->store->type->commit_merge) { 4461e03f97eSMikulas Patocka snap->ti->error = "Snapshot exception store does not " 4471e03f97eSMikulas Patocka "support snapshot-merge."; 4481e03f97eSMikulas Patocka return -EINVAL; 4491e03f97eSMikulas Patocka } 4501e03f97eSMikulas Patocka 451c1f0c183SMike Snitzer return 1; 452c1f0c183SMike Snitzer } 453c1f0c183SMike Snitzer 454c1f0c183SMike Snitzer static void __insert_snapshot(struct origin *o, struct dm_snapshot *s) 455c1f0c183SMike Snitzer { 456c1f0c183SMike Snitzer struct dm_snapshot *l; 457c1f0c183SMike Snitzer 458c1f0c183SMike Snitzer /* Sort the list according to chunk size, largest-first smallest-last */ 459c1f0c183SMike Snitzer list_for_each_entry(l, &o->snapshots, list) 460c1f0c183SMike Snitzer if (l->store->chunk_size < s->store->chunk_size) 461c1f0c183SMike Snitzer break; 462c1f0c183SMike Snitzer list_add_tail(&s->list, &l->list); 463c1f0c183SMike Snitzer } 464c1f0c183SMike Snitzer 465c1f0c183SMike Snitzer /* 4661da177e4SLinus Torvalds * Make a note of the snapshot and its origin so we can look it 4671da177e4SLinus Torvalds * up when the origin has a write on it. 468c1f0c183SMike Snitzer * 469c1f0c183SMike Snitzer * Also validate snapshot exception store handovers. 470c1f0c183SMike Snitzer * On success, returns 1 if this registration is a handover destination, 471c1f0c183SMike Snitzer * otherwise returns 0. 4721da177e4SLinus Torvalds */ 4731da177e4SLinus Torvalds static int register_snapshot(struct dm_snapshot *snap) 4741da177e4SLinus Torvalds { 475c1f0c183SMike Snitzer struct origin *o, *new_o = NULL; 4761da177e4SLinus Torvalds struct block_device *bdev = snap->origin->bdev; 477c1f0c183SMike Snitzer int r = 0; 4781da177e4SLinus Torvalds 47960c856c8SMikulas Patocka new_o = kmalloc(sizeof(*new_o), GFP_KERNEL); 48060c856c8SMikulas Patocka if (!new_o) 48160c856c8SMikulas Patocka return -ENOMEM; 48260c856c8SMikulas Patocka 4831da177e4SLinus Torvalds down_write(&_origins_lock); 4841da177e4SLinus Torvalds 485c1f0c183SMike Snitzer r = __validate_exception_handover(snap); 486c1f0c183SMike Snitzer if (r < 0) { 487c1f0c183SMike Snitzer kfree(new_o); 488c1f0c183SMike Snitzer goto out; 489c1f0c183SMike Snitzer } 490c1f0c183SMike Snitzer 491c1f0c183SMike Snitzer o = __lookup_origin(bdev); 49260c856c8SMikulas Patocka if (o) 49360c856c8SMikulas Patocka kfree(new_o); 49460c856c8SMikulas Patocka else { 4951da177e4SLinus Torvalds /* New origin */ 49660c856c8SMikulas Patocka o = new_o; 4971da177e4SLinus Torvalds 4981da177e4SLinus Torvalds /* Initialise the struct */ 4991da177e4SLinus Torvalds INIT_LIST_HEAD(&o->snapshots); 5001da177e4SLinus Torvalds o->bdev = bdev; 5011da177e4SLinus Torvalds 5021da177e4SLinus Torvalds __insert_origin(o); 5031da177e4SLinus Torvalds } 5041da177e4SLinus Torvalds 505c1f0c183SMike Snitzer __insert_snapshot(o, snap); 506c1f0c183SMike Snitzer 507c1f0c183SMike Snitzer out: 508c1f0c183SMike Snitzer up_write(&_origins_lock); 509c1f0c183SMike Snitzer 510c1f0c183SMike Snitzer return r; 511c1f0c183SMike Snitzer } 512c1f0c183SMike Snitzer 513c1f0c183SMike Snitzer /* 514c1f0c183SMike Snitzer * Move snapshot to correct place in list according to chunk size. 515c1f0c183SMike Snitzer */ 516c1f0c183SMike Snitzer static void reregister_snapshot(struct dm_snapshot *s) 517c1f0c183SMike Snitzer { 518c1f0c183SMike Snitzer struct block_device *bdev = s->origin->bdev; 519c1f0c183SMike Snitzer 520c1f0c183SMike Snitzer down_write(&_origins_lock); 521c1f0c183SMike Snitzer 522c1f0c183SMike Snitzer list_del(&s->list); 523c1f0c183SMike Snitzer __insert_snapshot(__lookup_origin(bdev), s); 5241da177e4SLinus Torvalds 5251da177e4SLinus Torvalds up_write(&_origins_lock); 5261da177e4SLinus Torvalds } 5271da177e4SLinus Torvalds 5281da177e4SLinus Torvalds static void unregister_snapshot(struct dm_snapshot *s) 5291da177e4SLinus Torvalds { 5301da177e4SLinus Torvalds struct origin *o; 5311da177e4SLinus Torvalds 5321da177e4SLinus Torvalds down_write(&_origins_lock); 5331da177e4SLinus Torvalds o = __lookup_origin(s->origin->bdev); 5341da177e4SLinus Torvalds 5351da177e4SLinus Torvalds list_del(&s->list); 536c1f0c183SMike Snitzer if (o && list_empty(&o->snapshots)) { 5371da177e4SLinus Torvalds list_del(&o->hash_list); 5381da177e4SLinus Torvalds kfree(o); 5391da177e4SLinus Torvalds } 5401da177e4SLinus Torvalds 5411da177e4SLinus Torvalds up_write(&_origins_lock); 5421da177e4SLinus Torvalds } 5431da177e4SLinus Torvalds 5441da177e4SLinus Torvalds /* 5451da177e4SLinus Torvalds * Implementation of the exception hash tables. 546d74f81f8SMilan Broz * The lowest hash_shift bits of the chunk number are ignored, allowing 547d74f81f8SMilan Broz * some consecutive chunks to be grouped together. 5481da177e4SLinus Torvalds */ 5493510cb94SJon Brassow static int dm_exception_table_init(struct dm_exception_table *et, 5503510cb94SJon Brassow uint32_t size, unsigned hash_shift) 5511da177e4SLinus Torvalds { 5521da177e4SLinus Torvalds unsigned int i; 5531da177e4SLinus Torvalds 554d74f81f8SMilan Broz et->hash_shift = hash_shift; 5551da177e4SLinus Torvalds et->hash_mask = size - 1; 5561da177e4SLinus Torvalds et->table = dm_vcalloc(size, sizeof(struct list_head)); 5571da177e4SLinus Torvalds if (!et->table) 5581da177e4SLinus Torvalds return -ENOMEM; 5591da177e4SLinus Torvalds 5601da177e4SLinus Torvalds for (i = 0; i < size; i++) 5611da177e4SLinus Torvalds INIT_LIST_HEAD(et->table + i); 5621da177e4SLinus Torvalds 5631da177e4SLinus Torvalds return 0; 5641da177e4SLinus Torvalds } 5651da177e4SLinus Torvalds 5663510cb94SJon Brassow static void dm_exception_table_exit(struct dm_exception_table *et, 567191437a5SJon Brassow struct kmem_cache *mem) 5681da177e4SLinus Torvalds { 5691da177e4SLinus Torvalds struct list_head *slot; 5701d4989c8SJon Brassow struct dm_exception *ex, *next; 5711da177e4SLinus Torvalds int i, size; 5721da177e4SLinus Torvalds 5731da177e4SLinus Torvalds size = et->hash_mask + 1; 5741da177e4SLinus Torvalds for (i = 0; i < size; i++) { 5751da177e4SLinus Torvalds slot = et->table + i; 5761da177e4SLinus Torvalds 5771da177e4SLinus Torvalds list_for_each_entry_safe (ex, next, slot, hash_list) 5781da177e4SLinus Torvalds kmem_cache_free(mem, ex); 5791da177e4SLinus Torvalds } 5801da177e4SLinus Torvalds 5811da177e4SLinus Torvalds vfree(et->table); 5821da177e4SLinus Torvalds } 5831da177e4SLinus Torvalds 584191437a5SJon Brassow static uint32_t exception_hash(struct dm_exception_table *et, chunk_t chunk) 5851da177e4SLinus Torvalds { 586d74f81f8SMilan Broz return (chunk >> et->hash_shift) & et->hash_mask; 5871da177e4SLinus Torvalds } 5881da177e4SLinus Torvalds 5893510cb94SJon Brassow static void dm_remove_exception(struct dm_exception *e) 5901da177e4SLinus Torvalds { 5911da177e4SLinus Torvalds list_del(&e->hash_list); 5921da177e4SLinus Torvalds } 5931da177e4SLinus Torvalds 5941da177e4SLinus Torvalds /* 5951da177e4SLinus Torvalds * Return the exception data for a sector, or NULL if not 5961da177e4SLinus Torvalds * remapped. 5971da177e4SLinus Torvalds */ 5983510cb94SJon Brassow static struct dm_exception *dm_lookup_exception(struct dm_exception_table *et, 5991da177e4SLinus Torvalds chunk_t chunk) 6001da177e4SLinus Torvalds { 6011da177e4SLinus Torvalds struct list_head *slot; 6021d4989c8SJon Brassow struct dm_exception *e; 6031da177e4SLinus Torvalds 6041da177e4SLinus Torvalds slot = &et->table[exception_hash(et, chunk)]; 6051da177e4SLinus Torvalds list_for_each_entry (e, slot, hash_list) 606d74f81f8SMilan Broz if (chunk >= e->old_chunk && 607d74f81f8SMilan Broz chunk <= e->old_chunk + dm_consecutive_chunk_count(e)) 6081da177e4SLinus Torvalds return e; 6091da177e4SLinus Torvalds 6101da177e4SLinus Torvalds return NULL; 6111da177e4SLinus Torvalds } 6121da177e4SLinus Torvalds 613119bc547SMikulas Patocka static struct dm_exception *alloc_completed_exception(gfp_t gfp) 6141da177e4SLinus Torvalds { 6151d4989c8SJon Brassow struct dm_exception *e; 6161da177e4SLinus Torvalds 617119bc547SMikulas Patocka e = kmem_cache_alloc(exception_cache, gfp); 618119bc547SMikulas Patocka if (!e && gfp == GFP_NOIO) 6191da177e4SLinus Torvalds e = kmem_cache_alloc(exception_cache, GFP_ATOMIC); 6201da177e4SLinus Torvalds 6211da177e4SLinus Torvalds return e; 6221da177e4SLinus Torvalds } 6231da177e4SLinus Torvalds 6243510cb94SJon Brassow static void free_completed_exception(struct dm_exception *e) 6251da177e4SLinus Torvalds { 6261da177e4SLinus Torvalds kmem_cache_free(exception_cache, e); 6271da177e4SLinus Torvalds } 6281da177e4SLinus Torvalds 62992e86812SMikulas Patocka static struct dm_snap_pending_exception *alloc_pending_exception(struct dm_snapshot *s) 6301da177e4SLinus Torvalds { 63192e86812SMikulas Patocka struct dm_snap_pending_exception *pe = mempool_alloc(s->pending_pool, 63292e86812SMikulas Patocka GFP_NOIO); 63392e86812SMikulas Patocka 634879129d2SMikulas Patocka atomic_inc(&s->pending_exceptions_count); 63592e86812SMikulas Patocka pe->snap = s; 63692e86812SMikulas Patocka 63792e86812SMikulas Patocka return pe; 6381da177e4SLinus Torvalds } 6391da177e4SLinus Torvalds 640028867acSAlasdair G Kergon static void free_pending_exception(struct dm_snap_pending_exception *pe) 6411da177e4SLinus Torvalds { 642879129d2SMikulas Patocka struct dm_snapshot *s = pe->snap; 643879129d2SMikulas Patocka 644879129d2SMikulas Patocka mempool_free(pe, s->pending_pool); 6454e857c58SPeter Zijlstra smp_mb__before_atomic(); 646879129d2SMikulas Patocka atomic_dec(&s->pending_exceptions_count); 6471da177e4SLinus Torvalds } 6481da177e4SLinus Torvalds 6493510cb94SJon Brassow static void dm_insert_exception(struct dm_exception_table *eh, 6501d4989c8SJon Brassow struct dm_exception *new_e) 651d74f81f8SMilan Broz { 652d74f81f8SMilan Broz struct list_head *l; 6531d4989c8SJon Brassow struct dm_exception *e = NULL; 654d74f81f8SMilan Broz 655d74f81f8SMilan Broz l = &eh->table[exception_hash(eh, new_e->old_chunk)]; 656d74f81f8SMilan Broz 657d74f81f8SMilan Broz /* Add immediately if this table doesn't support consecutive chunks */ 658d74f81f8SMilan Broz if (!eh->hash_shift) 659d74f81f8SMilan Broz goto out; 660d74f81f8SMilan Broz 661d74f81f8SMilan Broz /* List is ordered by old_chunk */ 662d74f81f8SMilan Broz list_for_each_entry_reverse(e, l, hash_list) { 663d74f81f8SMilan Broz /* Insert after an existing chunk? */ 664d74f81f8SMilan Broz if (new_e->old_chunk == (e->old_chunk + 665d74f81f8SMilan Broz dm_consecutive_chunk_count(e) + 1) && 666d74f81f8SMilan Broz new_e->new_chunk == (dm_chunk_number(e->new_chunk) + 667d74f81f8SMilan Broz dm_consecutive_chunk_count(e) + 1)) { 668d74f81f8SMilan Broz dm_consecutive_chunk_count_inc(e); 6693510cb94SJon Brassow free_completed_exception(new_e); 670d74f81f8SMilan Broz return; 671d74f81f8SMilan Broz } 672d74f81f8SMilan Broz 673d74f81f8SMilan Broz /* Insert before an existing chunk? */ 674d74f81f8SMilan Broz if (new_e->old_chunk == (e->old_chunk - 1) && 675d74f81f8SMilan Broz new_e->new_chunk == (dm_chunk_number(e->new_chunk) - 1)) { 676d74f81f8SMilan Broz dm_consecutive_chunk_count_inc(e); 677d74f81f8SMilan Broz e->old_chunk--; 678d74f81f8SMilan Broz e->new_chunk--; 6793510cb94SJon Brassow free_completed_exception(new_e); 680d74f81f8SMilan Broz return; 681d74f81f8SMilan Broz } 682d74f81f8SMilan Broz 683d74f81f8SMilan Broz if (new_e->old_chunk > e->old_chunk) 684d74f81f8SMilan Broz break; 685d74f81f8SMilan Broz } 686d74f81f8SMilan Broz 687d74f81f8SMilan Broz out: 688d74f81f8SMilan Broz list_add(&new_e->hash_list, e ? &e->hash_list : l); 689d74f81f8SMilan Broz } 690d74f81f8SMilan Broz 691a159c1acSJonathan Brassow /* 692a159c1acSJonathan Brassow * Callback used by the exception stores to load exceptions when 693a159c1acSJonathan Brassow * initialising. 694a159c1acSJonathan Brassow */ 695a159c1acSJonathan Brassow static int dm_add_exception(void *context, chunk_t old, chunk_t new) 6961da177e4SLinus Torvalds { 697a159c1acSJonathan Brassow struct dm_snapshot *s = context; 6981d4989c8SJon Brassow struct dm_exception *e; 6991da177e4SLinus Torvalds 700119bc547SMikulas Patocka e = alloc_completed_exception(GFP_KERNEL); 7011da177e4SLinus Torvalds if (!e) 7021da177e4SLinus Torvalds return -ENOMEM; 7031da177e4SLinus Torvalds 7041da177e4SLinus Torvalds e->old_chunk = old; 705d74f81f8SMilan Broz 706d74f81f8SMilan Broz /* Consecutive_count is implicitly initialised to zero */ 7071da177e4SLinus Torvalds e->new_chunk = new; 708d74f81f8SMilan Broz 7093510cb94SJon Brassow dm_insert_exception(&s->complete, e); 710d74f81f8SMilan Broz 7111da177e4SLinus Torvalds return 0; 7121da177e4SLinus Torvalds } 7131da177e4SLinus Torvalds 7147e201b35SMikulas Patocka /* 7157e201b35SMikulas Patocka * Return a minimum chunk size of all snapshots that have the specified origin. 7167e201b35SMikulas Patocka * Return zero if the origin has no snapshots. 7177e201b35SMikulas Patocka */ 718542f9038SMike Snitzer static uint32_t __minimum_chunk_size(struct origin *o) 7197e201b35SMikulas Patocka { 7207e201b35SMikulas Patocka struct dm_snapshot *snap; 7217e201b35SMikulas Patocka unsigned chunk_size = 0; 7227e201b35SMikulas Patocka 7237e201b35SMikulas Patocka if (o) 7247e201b35SMikulas Patocka list_for_each_entry(snap, &o->snapshots, list) 7257e201b35SMikulas Patocka chunk_size = min_not_zero(chunk_size, 7267e201b35SMikulas Patocka snap->store->chunk_size); 7277e201b35SMikulas Patocka 728542f9038SMike Snitzer return (uint32_t) chunk_size; 7297e201b35SMikulas Patocka } 7307e201b35SMikulas Patocka 7311da177e4SLinus Torvalds /* 7321da177e4SLinus Torvalds * Hard coded magic. 7331da177e4SLinus Torvalds */ 7341da177e4SLinus Torvalds static int calc_max_buckets(void) 7351da177e4SLinus Torvalds { 7361da177e4SLinus Torvalds /* use a fixed size of 2MB */ 7371da177e4SLinus Torvalds unsigned long mem = 2 * 1024 * 1024; 7381da177e4SLinus Torvalds mem /= sizeof(struct list_head); 7391da177e4SLinus Torvalds 7401da177e4SLinus Torvalds return mem; 7411da177e4SLinus Torvalds } 7421da177e4SLinus Torvalds 7431da177e4SLinus Torvalds /* 7441da177e4SLinus Torvalds * Allocate room for a suitable hash table. 7451da177e4SLinus Torvalds */ 746fee1998eSJonathan Brassow static int init_hash_tables(struct dm_snapshot *s) 7471da177e4SLinus Torvalds { 74860e356f3SMikulas Patocka sector_t hash_size, cow_dev_size, max_buckets; 7491da177e4SLinus Torvalds 7501da177e4SLinus Torvalds /* 7511da177e4SLinus Torvalds * Calculate based on the size of the original volume or 7521da177e4SLinus Torvalds * the COW volume... 7531da177e4SLinus Torvalds */ 754fc56f6fbSMike Snitzer cow_dev_size = get_dev_size(s->cow->bdev); 7551da177e4SLinus Torvalds max_buckets = calc_max_buckets(); 7561da177e4SLinus Torvalds 75760e356f3SMikulas Patocka hash_size = cow_dev_size >> s->store->chunk_shift; 7581da177e4SLinus Torvalds hash_size = min(hash_size, max_buckets); 7591da177e4SLinus Torvalds 7608e87b9b8SMikulas Patocka if (hash_size < 64) 7618e87b9b8SMikulas Patocka hash_size = 64; 7628defd830SRobert P. J. Day hash_size = rounddown_pow_of_two(hash_size); 7633510cb94SJon Brassow if (dm_exception_table_init(&s->complete, hash_size, 764d74f81f8SMilan Broz DM_CHUNK_CONSECUTIVE_BITS)) 7651da177e4SLinus Torvalds return -ENOMEM; 7661da177e4SLinus Torvalds 7671da177e4SLinus Torvalds /* 7681da177e4SLinus Torvalds * Allocate hash table for in-flight exceptions 7691da177e4SLinus Torvalds * Make this smaller than the real hash table 7701da177e4SLinus Torvalds */ 7711da177e4SLinus Torvalds hash_size >>= 3; 7721da177e4SLinus Torvalds if (hash_size < 64) 7731da177e4SLinus Torvalds hash_size = 64; 7741da177e4SLinus Torvalds 7753510cb94SJon Brassow if (dm_exception_table_init(&s->pending, hash_size, 0)) { 7763510cb94SJon Brassow dm_exception_table_exit(&s->complete, exception_cache); 7771da177e4SLinus Torvalds return -ENOMEM; 7781da177e4SLinus Torvalds } 7791da177e4SLinus Torvalds 7801da177e4SLinus Torvalds return 0; 7811da177e4SLinus Torvalds } 7821da177e4SLinus Torvalds 7831e03f97eSMikulas Patocka static void merge_shutdown(struct dm_snapshot *s) 7841e03f97eSMikulas Patocka { 7851e03f97eSMikulas Patocka clear_bit_unlock(RUNNING_MERGE, &s->state_bits); 7864e857c58SPeter Zijlstra smp_mb__after_atomic(); 7871e03f97eSMikulas Patocka wake_up_bit(&s->state_bits, RUNNING_MERGE); 7881e03f97eSMikulas Patocka } 7891e03f97eSMikulas Patocka 7909fe86254SMikulas Patocka static struct bio *__release_queued_bios_after_merge(struct dm_snapshot *s) 7919fe86254SMikulas Patocka { 7929fe86254SMikulas Patocka s->first_merging_chunk = 0; 7939fe86254SMikulas Patocka s->num_merging_chunks = 0; 7949fe86254SMikulas Patocka 7959fe86254SMikulas Patocka return bio_list_get(&s->bios_queued_during_merge); 7969fe86254SMikulas Patocka } 7979fe86254SMikulas Patocka 7981e03f97eSMikulas Patocka /* 7991e03f97eSMikulas Patocka * Remove one chunk from the index of completed exceptions. 8001e03f97eSMikulas Patocka */ 8011e03f97eSMikulas Patocka static int __remove_single_exception_chunk(struct dm_snapshot *s, 8021e03f97eSMikulas Patocka chunk_t old_chunk) 8031e03f97eSMikulas Patocka { 8041e03f97eSMikulas Patocka struct dm_exception *e; 8051e03f97eSMikulas Patocka 8061e03f97eSMikulas Patocka e = dm_lookup_exception(&s->complete, old_chunk); 8071e03f97eSMikulas Patocka if (!e) { 8081e03f97eSMikulas Patocka DMERR("Corruption detected: exception for block %llu is " 8091e03f97eSMikulas Patocka "on disk but not in memory", 8101e03f97eSMikulas Patocka (unsigned long long)old_chunk); 8111e03f97eSMikulas Patocka return -EINVAL; 8121e03f97eSMikulas Patocka } 8131e03f97eSMikulas Patocka 8141e03f97eSMikulas Patocka /* 8151e03f97eSMikulas Patocka * If this is the only chunk using this exception, remove exception. 8161e03f97eSMikulas Patocka */ 8171e03f97eSMikulas Patocka if (!dm_consecutive_chunk_count(e)) { 8181e03f97eSMikulas Patocka dm_remove_exception(e); 8191e03f97eSMikulas Patocka free_completed_exception(e); 8201e03f97eSMikulas Patocka return 0; 8211e03f97eSMikulas Patocka } 8221e03f97eSMikulas Patocka 8231e03f97eSMikulas Patocka /* 8241e03f97eSMikulas Patocka * The chunk may be either at the beginning or the end of a 8251e03f97eSMikulas Patocka * group of consecutive chunks - never in the middle. We are 8261e03f97eSMikulas Patocka * removing chunks in the opposite order to that in which they 8271e03f97eSMikulas Patocka * were added, so this should always be true. 8281e03f97eSMikulas Patocka * Decrement the consecutive chunk counter and adjust the 8291e03f97eSMikulas Patocka * starting point if necessary. 8301e03f97eSMikulas Patocka */ 8311e03f97eSMikulas Patocka if (old_chunk == e->old_chunk) { 8321e03f97eSMikulas Patocka e->old_chunk++; 8331e03f97eSMikulas Patocka e->new_chunk++; 8341e03f97eSMikulas Patocka } else if (old_chunk != e->old_chunk + 8351e03f97eSMikulas Patocka dm_consecutive_chunk_count(e)) { 8361e03f97eSMikulas Patocka DMERR("Attempt to merge block %llu from the " 8371e03f97eSMikulas Patocka "middle of a chunk range [%llu - %llu]", 8381e03f97eSMikulas Patocka (unsigned long long)old_chunk, 8391e03f97eSMikulas Patocka (unsigned long long)e->old_chunk, 8401e03f97eSMikulas Patocka (unsigned long long) 8411e03f97eSMikulas Patocka e->old_chunk + dm_consecutive_chunk_count(e)); 8421e03f97eSMikulas Patocka return -EINVAL; 8431e03f97eSMikulas Patocka } 8441e03f97eSMikulas Patocka 8451e03f97eSMikulas Patocka dm_consecutive_chunk_count_dec(e); 8461e03f97eSMikulas Patocka 8471e03f97eSMikulas Patocka return 0; 8481e03f97eSMikulas Patocka } 8491e03f97eSMikulas Patocka 8509fe86254SMikulas Patocka static void flush_bios(struct bio *bio); 8519fe86254SMikulas Patocka 8529fe86254SMikulas Patocka static int remove_single_exception_chunk(struct dm_snapshot *s) 8531e03f97eSMikulas Patocka { 8549fe86254SMikulas Patocka struct bio *b = NULL; 8559fe86254SMikulas Patocka int r; 8569fe86254SMikulas Patocka chunk_t old_chunk = s->first_merging_chunk + s->num_merging_chunks - 1; 8571e03f97eSMikulas Patocka 8581e03f97eSMikulas Patocka down_write(&s->lock); 8599fe86254SMikulas Patocka 8609fe86254SMikulas Patocka /* 8619fe86254SMikulas Patocka * Process chunks (and associated exceptions) in reverse order 8629fe86254SMikulas Patocka * so that dm_consecutive_chunk_count_dec() accounting works. 8639fe86254SMikulas Patocka */ 8649fe86254SMikulas Patocka do { 8651e03f97eSMikulas Patocka r = __remove_single_exception_chunk(s, old_chunk); 8669fe86254SMikulas Patocka if (r) 8679fe86254SMikulas Patocka goto out; 8689fe86254SMikulas Patocka } while (old_chunk-- > s->first_merging_chunk); 8699fe86254SMikulas Patocka 8709fe86254SMikulas Patocka b = __release_queued_bios_after_merge(s); 8719fe86254SMikulas Patocka 8729fe86254SMikulas Patocka out: 8731e03f97eSMikulas Patocka up_write(&s->lock); 8749fe86254SMikulas Patocka if (b) 8759fe86254SMikulas Patocka flush_bios(b); 8761e03f97eSMikulas Patocka 8771e03f97eSMikulas Patocka return r; 8781e03f97eSMikulas Patocka } 8791e03f97eSMikulas Patocka 88073dfd078SMikulas Patocka static int origin_write_extent(struct dm_snapshot *merging_snap, 88173dfd078SMikulas Patocka sector_t sector, unsigned chunk_size); 88273dfd078SMikulas Patocka 8831e03f97eSMikulas Patocka static void merge_callback(int read_err, unsigned long write_err, 8841e03f97eSMikulas Patocka void *context); 8851e03f97eSMikulas Patocka 88673dfd078SMikulas Patocka static uint64_t read_pending_exceptions_done_count(void) 88773dfd078SMikulas Patocka { 88873dfd078SMikulas Patocka uint64_t pending_exceptions_done; 88973dfd078SMikulas Patocka 89073dfd078SMikulas Patocka spin_lock(&_pending_exceptions_done_spinlock); 89173dfd078SMikulas Patocka pending_exceptions_done = _pending_exceptions_done_count; 89273dfd078SMikulas Patocka spin_unlock(&_pending_exceptions_done_spinlock); 89373dfd078SMikulas Patocka 89473dfd078SMikulas Patocka return pending_exceptions_done; 89573dfd078SMikulas Patocka } 89673dfd078SMikulas Patocka 89773dfd078SMikulas Patocka static void increment_pending_exceptions_done_count(void) 89873dfd078SMikulas Patocka { 89973dfd078SMikulas Patocka spin_lock(&_pending_exceptions_done_spinlock); 90073dfd078SMikulas Patocka _pending_exceptions_done_count++; 90173dfd078SMikulas Patocka spin_unlock(&_pending_exceptions_done_spinlock); 90273dfd078SMikulas Patocka 90373dfd078SMikulas Patocka wake_up_all(&_pending_exceptions_done); 90473dfd078SMikulas Patocka } 90573dfd078SMikulas Patocka 9061e03f97eSMikulas Patocka static void snapshot_merge_next_chunks(struct dm_snapshot *s) 9071e03f97eSMikulas Patocka { 9088a2d5286SMike Snitzer int i, linear_chunks; 9091e03f97eSMikulas Patocka chunk_t old_chunk, new_chunk; 9101e03f97eSMikulas Patocka struct dm_io_region src, dest; 9118a2d5286SMike Snitzer sector_t io_size; 91273dfd078SMikulas Patocka uint64_t previous_count; 9131e03f97eSMikulas Patocka 9141e03f97eSMikulas Patocka BUG_ON(!test_bit(RUNNING_MERGE, &s->state_bits)); 9151e03f97eSMikulas Patocka if (unlikely(test_bit(SHUTDOWN_MERGE, &s->state_bits))) 9161e03f97eSMikulas Patocka goto shut; 9171e03f97eSMikulas Patocka 9181e03f97eSMikulas Patocka /* 9191e03f97eSMikulas Patocka * valid flag never changes during merge, so no lock required. 9201e03f97eSMikulas Patocka */ 9211e03f97eSMikulas Patocka if (!s->valid) { 9221e03f97eSMikulas Patocka DMERR("Snapshot is invalid: can't merge"); 9231e03f97eSMikulas Patocka goto shut; 9241e03f97eSMikulas Patocka } 9251e03f97eSMikulas Patocka 9268a2d5286SMike Snitzer linear_chunks = s->store->type->prepare_merge(s->store, &old_chunk, 9278a2d5286SMike Snitzer &new_chunk); 9288a2d5286SMike Snitzer if (linear_chunks <= 0) { 929d8ddb1cfSMike Snitzer if (linear_chunks < 0) { 9301e03f97eSMikulas Patocka DMERR("Read error in exception store: " 9311e03f97eSMikulas Patocka "shutting down merge"); 932d8ddb1cfSMike Snitzer down_write(&s->lock); 933d8ddb1cfSMike Snitzer s->merge_failed = 1; 934d8ddb1cfSMike Snitzer up_write(&s->lock); 935d8ddb1cfSMike Snitzer } 9361e03f97eSMikulas Patocka goto shut; 9371e03f97eSMikulas Patocka } 9381e03f97eSMikulas Patocka 9398a2d5286SMike Snitzer /* Adjust old_chunk and new_chunk to reflect start of linear region */ 9408a2d5286SMike Snitzer old_chunk = old_chunk + 1 - linear_chunks; 9418a2d5286SMike Snitzer new_chunk = new_chunk + 1 - linear_chunks; 9428a2d5286SMike Snitzer 9438a2d5286SMike Snitzer /* 9448a2d5286SMike Snitzer * Use one (potentially large) I/O to copy all 'linear_chunks' 9458a2d5286SMike Snitzer * from the exception store to the origin 9468a2d5286SMike Snitzer */ 9478a2d5286SMike Snitzer io_size = linear_chunks * s->store->chunk_size; 9481e03f97eSMikulas Patocka 9491e03f97eSMikulas Patocka dest.bdev = s->origin->bdev; 9501e03f97eSMikulas Patocka dest.sector = chunk_to_sector(s->store, old_chunk); 9518a2d5286SMike Snitzer dest.count = min(io_size, get_dev_size(dest.bdev) - dest.sector); 9521e03f97eSMikulas Patocka 9531e03f97eSMikulas Patocka src.bdev = s->cow->bdev; 9541e03f97eSMikulas Patocka src.sector = chunk_to_sector(s->store, new_chunk); 9551e03f97eSMikulas Patocka src.count = dest.count; 9561e03f97eSMikulas Patocka 95773dfd078SMikulas Patocka /* 95873dfd078SMikulas Patocka * Reallocate any exceptions needed in other snapshots then 95973dfd078SMikulas Patocka * wait for the pending exceptions to complete. 96073dfd078SMikulas Patocka * Each time any pending exception (globally on the system) 96173dfd078SMikulas Patocka * completes we are woken and repeat the process to find out 96273dfd078SMikulas Patocka * if we can proceed. While this may not seem a particularly 96373dfd078SMikulas Patocka * efficient algorithm, it is not expected to have any 96473dfd078SMikulas Patocka * significant impact on performance. 96573dfd078SMikulas Patocka */ 96673dfd078SMikulas Patocka previous_count = read_pending_exceptions_done_count(); 9678a2d5286SMike Snitzer while (origin_write_extent(s, dest.sector, io_size)) { 96873dfd078SMikulas Patocka wait_event(_pending_exceptions_done, 96973dfd078SMikulas Patocka (read_pending_exceptions_done_count() != 97073dfd078SMikulas Patocka previous_count)); 97173dfd078SMikulas Patocka /* Retry after the wait, until all exceptions are done. */ 97273dfd078SMikulas Patocka previous_count = read_pending_exceptions_done_count(); 97373dfd078SMikulas Patocka } 97473dfd078SMikulas Patocka 9759fe86254SMikulas Patocka down_write(&s->lock); 9769fe86254SMikulas Patocka s->first_merging_chunk = old_chunk; 9778a2d5286SMike Snitzer s->num_merging_chunks = linear_chunks; 9789fe86254SMikulas Patocka up_write(&s->lock); 9799fe86254SMikulas Patocka 9808a2d5286SMike Snitzer /* Wait until writes to all 'linear_chunks' drain */ 9818a2d5286SMike Snitzer for (i = 0; i < linear_chunks; i++) 9828a2d5286SMike Snitzer __check_for_conflicting_io(s, old_chunk + i); 9839fe86254SMikulas Patocka 9841e03f97eSMikulas Patocka dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, merge_callback, s); 9851e03f97eSMikulas Patocka return; 9861e03f97eSMikulas Patocka 9871e03f97eSMikulas Patocka shut: 9881e03f97eSMikulas Patocka merge_shutdown(s); 9891e03f97eSMikulas Patocka } 9901e03f97eSMikulas Patocka 9919fe86254SMikulas Patocka static void error_bios(struct bio *bio); 9929fe86254SMikulas Patocka 9931e03f97eSMikulas Patocka static void merge_callback(int read_err, unsigned long write_err, void *context) 9941e03f97eSMikulas Patocka { 9951e03f97eSMikulas Patocka struct dm_snapshot *s = context; 9969fe86254SMikulas Patocka struct bio *b = NULL; 9971e03f97eSMikulas Patocka 9981e03f97eSMikulas Patocka if (read_err || write_err) { 9991e03f97eSMikulas Patocka if (read_err) 10001e03f97eSMikulas Patocka DMERR("Read error: shutting down merge."); 10011e03f97eSMikulas Patocka else 10021e03f97eSMikulas Patocka DMERR("Write error: shutting down merge."); 10031e03f97eSMikulas Patocka goto shut; 10041e03f97eSMikulas Patocka } 10051e03f97eSMikulas Patocka 10069fe86254SMikulas Patocka if (s->store->type->commit_merge(s->store, 10079fe86254SMikulas Patocka s->num_merging_chunks) < 0) { 10081e03f97eSMikulas Patocka DMERR("Write error in exception store: shutting down merge"); 10091e03f97eSMikulas Patocka goto shut; 10101e03f97eSMikulas Patocka } 10111e03f97eSMikulas Patocka 10129fe86254SMikulas Patocka if (remove_single_exception_chunk(s) < 0) 10139fe86254SMikulas Patocka goto shut; 10149fe86254SMikulas Patocka 10151e03f97eSMikulas Patocka snapshot_merge_next_chunks(s); 10161e03f97eSMikulas Patocka 10171e03f97eSMikulas Patocka return; 10181e03f97eSMikulas Patocka 10191e03f97eSMikulas Patocka shut: 10209fe86254SMikulas Patocka down_write(&s->lock); 1021d8ddb1cfSMike Snitzer s->merge_failed = 1; 10229fe86254SMikulas Patocka b = __release_queued_bios_after_merge(s); 10239fe86254SMikulas Patocka up_write(&s->lock); 10249fe86254SMikulas Patocka error_bios(b); 10259fe86254SMikulas Patocka 10261e03f97eSMikulas Patocka merge_shutdown(s); 10271e03f97eSMikulas Patocka } 10281e03f97eSMikulas Patocka 10291e03f97eSMikulas Patocka static void start_merge(struct dm_snapshot *s) 10301e03f97eSMikulas Patocka { 10311e03f97eSMikulas Patocka if (!test_and_set_bit(RUNNING_MERGE, &s->state_bits)) 10321e03f97eSMikulas Patocka snapshot_merge_next_chunks(s); 10331e03f97eSMikulas Patocka } 10341e03f97eSMikulas Patocka 10351e03f97eSMikulas Patocka /* 10361e03f97eSMikulas Patocka * Stop the merging process and wait until it finishes. 10371e03f97eSMikulas Patocka */ 10381e03f97eSMikulas Patocka static void stop_merge(struct dm_snapshot *s) 10391e03f97eSMikulas Patocka { 10401e03f97eSMikulas Patocka set_bit(SHUTDOWN_MERGE, &s->state_bits); 104174316201SNeilBrown wait_on_bit(&s->state_bits, RUNNING_MERGE, TASK_UNINTERRUPTIBLE); 10421e03f97eSMikulas Patocka clear_bit(SHUTDOWN_MERGE, &s->state_bits); 10431e03f97eSMikulas Patocka } 10441e03f97eSMikulas Patocka 10451da177e4SLinus Torvalds /* 10461da177e4SLinus Torvalds * Construct a snapshot mapping: <origin_dev> <COW-dev> <p/n> <chunk-size> 10471da177e4SLinus Torvalds */ 10481da177e4SLinus Torvalds static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv) 10491da177e4SLinus Torvalds { 10501da177e4SLinus Torvalds struct dm_snapshot *s; 1051cd45daffSMikulas Patocka int i; 10521da177e4SLinus Torvalds int r = -EINVAL; 1053fc56f6fbSMike Snitzer char *origin_path, *cow_path; 105455a62eefSAlasdair G Kergon unsigned args_used, num_flush_bios = 1; 105510b8106aSMike Snitzer fmode_t origin_mode = FMODE_READ; 10561da177e4SLinus Torvalds 10574c7e3bf4SMark McLoughlin if (argc != 4) { 105872d94861SAlasdair G Kergon ti->error = "requires exactly 4 arguments"; 10591da177e4SLinus Torvalds r = -EINVAL; 1060fc56f6fbSMike Snitzer goto bad; 10611da177e4SLinus Torvalds } 10621da177e4SLinus Torvalds 106310b8106aSMike Snitzer if (dm_target_is_snapshot_merge(ti)) { 106455a62eefSAlasdair G Kergon num_flush_bios = 2; 106510b8106aSMike Snitzer origin_mode = FMODE_WRITE; 106610b8106aSMike Snitzer } 106710b8106aSMike Snitzer 10681da177e4SLinus Torvalds s = kmalloc(sizeof(*s), GFP_KERNEL); 1069fee1998eSJonathan Brassow if (!s) { 1070a2d2b034SJonathan Brassow ti->error = "Cannot allocate private snapshot structure"; 10711da177e4SLinus Torvalds r = -ENOMEM; 1072fc56f6fbSMike Snitzer goto bad; 10731da177e4SLinus Torvalds } 10741da177e4SLinus Torvalds 1075c2411045SMikulas Patocka origin_path = argv[0]; 1076c2411045SMikulas Patocka argv++; 1077c2411045SMikulas Patocka argc--; 1078c2411045SMikulas Patocka 1079c2411045SMikulas Patocka r = dm_get_device(ti, origin_path, origin_mode, &s->origin); 1080c2411045SMikulas Patocka if (r) { 1081c2411045SMikulas Patocka ti->error = "Cannot get origin device"; 1082c2411045SMikulas Patocka goto bad_origin; 1083c2411045SMikulas Patocka } 1084c2411045SMikulas Patocka 1085fc56f6fbSMike Snitzer cow_path = argv[0]; 1086fc56f6fbSMike Snitzer argv++; 1087fc56f6fbSMike Snitzer argc--; 1088fc56f6fbSMike Snitzer 1089024d37e9SMilan Broz r = dm_get_device(ti, cow_path, dm_table_get_mode(ti->table), &s->cow); 1090fc56f6fbSMike Snitzer if (r) { 1091fc56f6fbSMike Snitzer ti->error = "Cannot get COW device"; 1092fc56f6fbSMike Snitzer goto bad_cow; 1093fc56f6fbSMike Snitzer } 1094fc56f6fbSMike Snitzer 1095fc56f6fbSMike Snitzer r = dm_exception_store_create(ti, argc, argv, s, &args_used, &s->store); 1096fc56f6fbSMike Snitzer if (r) { 1097fc56f6fbSMike Snitzer ti->error = "Couldn't create exception store"; 1098fc56f6fbSMike Snitzer r = -EINVAL; 1099fc56f6fbSMike Snitzer goto bad_store; 1100fc56f6fbSMike Snitzer } 1101fc56f6fbSMike Snitzer 1102fc56f6fbSMike Snitzer argv += args_used; 1103fc56f6fbSMike Snitzer argc -= args_used; 1104fc56f6fbSMike Snitzer 1105fc56f6fbSMike Snitzer s->ti = ti; 11061da177e4SLinus Torvalds s->valid = 1; 1107aa14edebSAlasdair G Kergon s->active = 0; 1108879129d2SMikulas Patocka atomic_set(&s->pending_exceptions_count, 0); 1109230c83afSMikulas Patocka s->exception_start_sequence = 0; 1110230c83afSMikulas Patocka s->exception_complete_sequence = 0; 1111230c83afSMikulas Patocka INIT_LIST_HEAD(&s->out_of_order_list); 11121da177e4SLinus Torvalds init_rwsem(&s->lock); 1113c1f0c183SMike Snitzer INIT_LIST_HEAD(&s->list); 1114ca3a931fSAlasdair G Kergon spin_lock_init(&s->pe_lock); 11151e03f97eSMikulas Patocka s->state_bits = 0; 1116d8ddb1cfSMike Snitzer s->merge_failed = 0; 11179fe86254SMikulas Patocka s->first_merging_chunk = 0; 11189fe86254SMikulas Patocka s->num_merging_chunks = 0; 11199fe86254SMikulas Patocka bio_list_init(&s->bios_queued_during_merge); 11201da177e4SLinus Torvalds 11211da177e4SLinus Torvalds /* Allocate hash table for COW data */ 1122fee1998eSJonathan Brassow if (init_hash_tables(s)) { 11231da177e4SLinus Torvalds ti->error = "Unable to allocate hash table space"; 11241da177e4SLinus Torvalds r = -ENOMEM; 1125fee1998eSJonathan Brassow goto bad_hash_tables; 11261da177e4SLinus Torvalds } 11271da177e4SLinus Torvalds 1128df5d2e90SMikulas Patocka s->kcopyd_client = dm_kcopyd_client_create(&dm_kcopyd_throttle); 1129fa34ce73SMikulas Patocka if (IS_ERR(s->kcopyd_client)) { 1130fa34ce73SMikulas Patocka r = PTR_ERR(s->kcopyd_client); 11311da177e4SLinus Torvalds ti->error = "Could not create kcopyd client"; 1132fee1998eSJonathan Brassow goto bad_kcopyd; 11331da177e4SLinus Torvalds } 11341da177e4SLinus Torvalds 113592e86812SMikulas Patocka s->pending_pool = mempool_create_slab_pool(MIN_IOS, pending_cache); 113692e86812SMikulas Patocka if (!s->pending_pool) { 113792e86812SMikulas Patocka ti->error = "Could not allocate mempool for pending exceptions"; 113809e8b813SWei Yongjun r = -ENOMEM; 1139fee1998eSJonathan Brassow goto bad_pending_pool; 114092e86812SMikulas Patocka } 114192e86812SMikulas Patocka 1142cd45daffSMikulas Patocka for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++) 1143cd45daffSMikulas Patocka INIT_HLIST_HEAD(&s->tracked_chunk_hash[i]); 1144cd45daffSMikulas Patocka 1145cd45daffSMikulas Patocka spin_lock_init(&s->tracked_chunk_lock); 1146cd45daffSMikulas Patocka 1147c1f0c183SMike Snitzer ti->private = s; 114855a62eefSAlasdair G Kergon ti->num_flush_bios = num_flush_bios; 114942bc954fSMikulas Patocka ti->per_bio_data_size = sizeof(struct dm_snap_tracked_chunk); 1150c1f0c183SMike Snitzer 1151c1f0c183SMike Snitzer /* Add snapshot to the list of snapshots for this origin */ 1152c1f0c183SMike Snitzer /* Exceptions aren't triggered till snapshot_resume() is called */ 1153c1f0c183SMike Snitzer r = register_snapshot(s); 1154c1f0c183SMike Snitzer if (r == -ENOMEM) { 1155c1f0c183SMike Snitzer ti->error = "Snapshot origin struct allocation failed"; 1156c1f0c183SMike Snitzer goto bad_load_and_register; 1157c1f0c183SMike Snitzer } else if (r < 0) { 1158c1f0c183SMike Snitzer /* invalid handover, register_snapshot has set ti->error */ 1159c1f0c183SMike Snitzer goto bad_load_and_register; 1160c1f0c183SMike Snitzer } 1161c1f0c183SMike Snitzer 1162c1f0c183SMike Snitzer /* 1163c1f0c183SMike Snitzer * Metadata must only be loaded into one table at once, so skip this 1164c1f0c183SMike Snitzer * if metadata will be handed over during resume. 1165c1f0c183SMike Snitzer * Chunk size will be set during the handover - set it to zero to 1166c1f0c183SMike Snitzer * ensure it's ignored. 1167c1f0c183SMike Snitzer */ 1168c1f0c183SMike Snitzer if (r > 0) { 1169c1f0c183SMike Snitzer s->store->chunk_size = 0; 1170c1f0c183SMike Snitzer return 0; 1171c1f0c183SMike Snitzer } 1172c1f0c183SMike Snitzer 1173493df71cSJonathan Brassow r = s->store->type->read_metadata(s->store, dm_add_exception, 1174493df71cSJonathan Brassow (void *)s); 11750764147bSMilan Broz if (r < 0) { 1176f9cea4f7SMark McLoughlin ti->error = "Failed to read snapshot metadata"; 1177c1f0c183SMike Snitzer goto bad_read_metadata; 11780764147bSMilan Broz } else if (r > 0) { 11790764147bSMilan Broz s->valid = 0; 11800764147bSMilan Broz DMWARN("Snapshot is marked invalid."); 1181f9cea4f7SMark McLoughlin } 1182aa14edebSAlasdair G Kergon 11833f2412dcSMikulas Patocka if (!s->store->chunk_size) { 11843f2412dcSMikulas Patocka ti->error = "Chunk size not set"; 1185c1f0c183SMike Snitzer goto bad_read_metadata; 11863f2412dcSMikulas Patocka } 1187542f9038SMike Snitzer 1188542f9038SMike Snitzer r = dm_set_target_max_io_len(ti, s->store->chunk_size); 1189542f9038SMike Snitzer if (r) 1190542f9038SMike Snitzer goto bad_read_metadata; 11911da177e4SLinus Torvalds 11921da177e4SLinus Torvalds return 0; 11931da177e4SLinus Torvalds 1194c1f0c183SMike Snitzer bad_read_metadata: 1195c1f0c183SMike Snitzer unregister_snapshot(s); 1196c1f0c183SMike Snitzer 1197cd45daffSMikulas Patocka bad_load_and_register: 119892e86812SMikulas Patocka mempool_destroy(s->pending_pool); 119992e86812SMikulas Patocka 1200fee1998eSJonathan Brassow bad_pending_pool: 1201eb69aca5SHeinz Mauelshagen dm_kcopyd_client_destroy(s->kcopyd_client); 12021da177e4SLinus Torvalds 1203fee1998eSJonathan Brassow bad_kcopyd: 12043510cb94SJon Brassow dm_exception_table_exit(&s->pending, pending_cache); 12053510cb94SJon Brassow dm_exception_table_exit(&s->complete, exception_cache); 12061da177e4SLinus Torvalds 1207fee1998eSJonathan Brassow bad_hash_tables: 1208fc56f6fbSMike Snitzer dm_exception_store_destroy(s->store); 1209fc56f6fbSMike Snitzer 1210fc56f6fbSMike Snitzer bad_store: 1211fc56f6fbSMike Snitzer dm_put_device(ti, s->cow); 1212fc56f6fbSMike Snitzer 1213fc56f6fbSMike Snitzer bad_cow: 1214c2411045SMikulas Patocka dm_put_device(ti, s->origin); 1215c2411045SMikulas Patocka 1216c2411045SMikulas Patocka bad_origin: 12171da177e4SLinus Torvalds kfree(s); 12181da177e4SLinus Torvalds 1219fc56f6fbSMike Snitzer bad: 12201da177e4SLinus Torvalds return r; 12211da177e4SLinus Torvalds } 12221da177e4SLinus Torvalds 122331c93a0cSMilan Broz static void __free_exceptions(struct dm_snapshot *s) 122431c93a0cSMilan Broz { 1225eb69aca5SHeinz Mauelshagen dm_kcopyd_client_destroy(s->kcopyd_client); 122631c93a0cSMilan Broz s->kcopyd_client = NULL; 122731c93a0cSMilan Broz 12283510cb94SJon Brassow dm_exception_table_exit(&s->pending, pending_cache); 12293510cb94SJon Brassow dm_exception_table_exit(&s->complete, exception_cache); 123031c93a0cSMilan Broz } 123131c93a0cSMilan Broz 1232c1f0c183SMike Snitzer static void __handover_exceptions(struct dm_snapshot *snap_src, 1233c1f0c183SMike Snitzer struct dm_snapshot *snap_dest) 1234c1f0c183SMike Snitzer { 1235c1f0c183SMike Snitzer union { 1236c1f0c183SMike Snitzer struct dm_exception_table table_swap; 1237c1f0c183SMike Snitzer struct dm_exception_store *store_swap; 1238c1f0c183SMike Snitzer } u; 1239c1f0c183SMike Snitzer 1240c1f0c183SMike Snitzer /* 1241c1f0c183SMike Snitzer * Swap all snapshot context information between the two instances. 1242c1f0c183SMike Snitzer */ 1243c1f0c183SMike Snitzer u.table_swap = snap_dest->complete; 1244c1f0c183SMike Snitzer snap_dest->complete = snap_src->complete; 1245c1f0c183SMike Snitzer snap_src->complete = u.table_swap; 1246c1f0c183SMike Snitzer 1247c1f0c183SMike Snitzer u.store_swap = snap_dest->store; 1248c1f0c183SMike Snitzer snap_dest->store = snap_src->store; 1249c1f0c183SMike Snitzer snap_src->store = u.store_swap; 1250c1f0c183SMike Snitzer 1251c1f0c183SMike Snitzer snap_dest->store->snap = snap_dest; 1252c1f0c183SMike Snitzer snap_src->store->snap = snap_src; 1253c1f0c183SMike Snitzer 1254542f9038SMike Snitzer snap_dest->ti->max_io_len = snap_dest->store->chunk_size; 1255c1f0c183SMike Snitzer snap_dest->valid = snap_src->valid; 1256c1f0c183SMike Snitzer 1257c1f0c183SMike Snitzer /* 1258c1f0c183SMike Snitzer * Set source invalid to ensure it receives no further I/O. 1259c1f0c183SMike Snitzer */ 1260c1f0c183SMike Snitzer snap_src->valid = 0; 1261c1f0c183SMike Snitzer } 1262c1f0c183SMike Snitzer 12631da177e4SLinus Torvalds static void snapshot_dtr(struct dm_target *ti) 12641da177e4SLinus Torvalds { 1265cd45daffSMikulas Patocka #ifdef CONFIG_DM_DEBUG 1266cd45daffSMikulas Patocka int i; 1267cd45daffSMikulas Patocka #endif 1268028867acSAlasdair G Kergon struct dm_snapshot *s = ti->private; 1269c1f0c183SMike Snitzer struct dm_snapshot *snap_src = NULL, *snap_dest = NULL; 12701da177e4SLinus Torvalds 1271c1f0c183SMike Snitzer down_read(&_origins_lock); 1272c1f0c183SMike Snitzer /* Check whether exception handover must be cancelled */ 12739d3b15c4SMikulas Patocka (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL); 1274c1f0c183SMike Snitzer if (snap_src && snap_dest && (s == snap_src)) { 1275c1f0c183SMike Snitzer down_write(&snap_dest->lock); 1276c1f0c183SMike Snitzer snap_dest->valid = 0; 1277c1f0c183SMike Snitzer up_write(&snap_dest->lock); 1278c1f0c183SMike Snitzer DMERR("Cancelling snapshot handover."); 1279c1f0c183SMike Snitzer } 1280c1f0c183SMike Snitzer up_read(&_origins_lock); 1281c1f0c183SMike Snitzer 12821e03f97eSMikulas Patocka if (dm_target_is_snapshot_merge(ti)) 12831e03f97eSMikulas Patocka stop_merge(s); 12841e03f97eSMikulas Patocka 1285138728dcSAlasdair G Kergon /* Prevent further origin writes from using this snapshot. */ 1286138728dcSAlasdair G Kergon /* After this returns there can be no new kcopyd jobs. */ 12871da177e4SLinus Torvalds unregister_snapshot(s); 12881da177e4SLinus Torvalds 1289879129d2SMikulas Patocka while (atomic_read(&s->pending_exceptions_count)) 129090fa1527SMikulas Patocka msleep(1); 1291879129d2SMikulas Patocka /* 1292879129d2SMikulas Patocka * Ensure instructions in mempool_destroy aren't reordered 1293879129d2SMikulas Patocka * before atomic_read. 1294879129d2SMikulas Patocka */ 1295879129d2SMikulas Patocka smp_mb(); 1296879129d2SMikulas Patocka 1297cd45daffSMikulas Patocka #ifdef CONFIG_DM_DEBUG 1298cd45daffSMikulas Patocka for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++) 1299cd45daffSMikulas Patocka BUG_ON(!hlist_empty(&s->tracked_chunk_hash[i])); 1300cd45daffSMikulas Patocka #endif 1301cd45daffSMikulas Patocka 130231c93a0cSMilan Broz __free_exceptions(s); 13031da177e4SLinus Torvalds 130492e86812SMikulas Patocka mempool_destroy(s->pending_pool); 130592e86812SMikulas Patocka 1306fee1998eSJonathan Brassow dm_exception_store_destroy(s->store); 1307138728dcSAlasdair G Kergon 1308fc56f6fbSMike Snitzer dm_put_device(ti, s->cow); 1309fc56f6fbSMike Snitzer 1310c2411045SMikulas Patocka dm_put_device(ti, s->origin); 1311c2411045SMikulas Patocka 13121da177e4SLinus Torvalds kfree(s); 13131da177e4SLinus Torvalds } 13141da177e4SLinus Torvalds 13151da177e4SLinus Torvalds /* 13161da177e4SLinus Torvalds * Flush a list of buffers. 13171da177e4SLinus Torvalds */ 13181da177e4SLinus Torvalds static void flush_bios(struct bio *bio) 13191da177e4SLinus Torvalds { 13201da177e4SLinus Torvalds struct bio *n; 13211da177e4SLinus Torvalds 13221da177e4SLinus Torvalds while (bio) { 13231da177e4SLinus Torvalds n = bio->bi_next; 13241da177e4SLinus Torvalds bio->bi_next = NULL; 13251da177e4SLinus Torvalds generic_make_request(bio); 13261da177e4SLinus Torvalds bio = n; 13271da177e4SLinus Torvalds } 13281da177e4SLinus Torvalds } 13291da177e4SLinus Torvalds 1330515ad66cSMikulas Patocka static int do_origin(struct dm_dev *origin, struct bio *bio); 1331515ad66cSMikulas Patocka 1332515ad66cSMikulas Patocka /* 1333515ad66cSMikulas Patocka * Flush a list of buffers. 1334515ad66cSMikulas Patocka */ 1335515ad66cSMikulas Patocka static void retry_origin_bios(struct dm_snapshot *s, struct bio *bio) 1336515ad66cSMikulas Patocka { 1337515ad66cSMikulas Patocka struct bio *n; 1338515ad66cSMikulas Patocka int r; 1339515ad66cSMikulas Patocka 1340515ad66cSMikulas Patocka while (bio) { 1341515ad66cSMikulas Patocka n = bio->bi_next; 1342515ad66cSMikulas Patocka bio->bi_next = NULL; 1343515ad66cSMikulas Patocka r = do_origin(s->origin, bio); 1344515ad66cSMikulas Patocka if (r == DM_MAPIO_REMAPPED) 1345515ad66cSMikulas Patocka generic_make_request(bio); 1346515ad66cSMikulas Patocka bio = n; 1347515ad66cSMikulas Patocka } 1348515ad66cSMikulas Patocka } 1349515ad66cSMikulas Patocka 13501da177e4SLinus Torvalds /* 13511da177e4SLinus Torvalds * Error a list of buffers. 13521da177e4SLinus Torvalds */ 13531da177e4SLinus Torvalds static void error_bios(struct bio *bio) 13541da177e4SLinus Torvalds { 13551da177e4SLinus Torvalds struct bio *n; 13561da177e4SLinus Torvalds 13571da177e4SLinus Torvalds while (bio) { 13581da177e4SLinus Torvalds n = bio->bi_next; 13591da177e4SLinus Torvalds bio->bi_next = NULL; 13606712ecf8SNeilBrown bio_io_error(bio); 13611da177e4SLinus Torvalds bio = n; 13621da177e4SLinus Torvalds } 13631da177e4SLinus Torvalds } 13641da177e4SLinus Torvalds 1365695368acSAlasdair G Kergon static void __invalidate_snapshot(struct dm_snapshot *s, int err) 136676df1c65SAlasdair G Kergon { 136776df1c65SAlasdair G Kergon if (!s->valid) 136876df1c65SAlasdair G Kergon return; 136976df1c65SAlasdair G Kergon 137076df1c65SAlasdair G Kergon if (err == -EIO) 137176df1c65SAlasdair G Kergon DMERR("Invalidating snapshot: Error reading/writing."); 137276df1c65SAlasdair G Kergon else if (err == -ENOMEM) 137376df1c65SAlasdair G Kergon DMERR("Invalidating snapshot: Unable to allocate exception."); 137476df1c65SAlasdair G Kergon 1375493df71cSJonathan Brassow if (s->store->type->drop_snapshot) 1376493df71cSJonathan Brassow s->store->type->drop_snapshot(s->store); 137776df1c65SAlasdair G Kergon 137876df1c65SAlasdair G Kergon s->valid = 0; 137976df1c65SAlasdair G Kergon 1380fc56f6fbSMike Snitzer dm_table_event(s->ti->table); 138176df1c65SAlasdair G Kergon } 138276df1c65SAlasdair G Kergon 1383028867acSAlasdair G Kergon static void pending_complete(struct dm_snap_pending_exception *pe, int success) 13841da177e4SLinus Torvalds { 13851d4989c8SJon Brassow struct dm_exception *e; 13861da177e4SLinus Torvalds struct dm_snapshot *s = pe->snap; 13879d493fa8SAlasdair G Kergon struct bio *origin_bios = NULL; 13889d493fa8SAlasdair G Kergon struct bio *snapshot_bios = NULL; 1389a6e50b40SMikulas Patocka struct bio *full_bio = NULL; 13909d493fa8SAlasdair G Kergon int error = 0; 13911da177e4SLinus Torvalds 139276df1c65SAlasdair G Kergon if (!success) { 139376df1c65SAlasdair G Kergon /* Read/write error - snapshot is unusable */ 13941da177e4SLinus Torvalds down_write(&s->lock); 1395695368acSAlasdair G Kergon __invalidate_snapshot(s, -EIO); 13969d493fa8SAlasdair G Kergon error = 1; 139776df1c65SAlasdair G Kergon goto out; 139876df1c65SAlasdair G Kergon } 139976df1c65SAlasdair G Kergon 1400119bc547SMikulas Patocka e = alloc_completed_exception(GFP_NOIO); 140176df1c65SAlasdair G Kergon if (!e) { 140276df1c65SAlasdair G Kergon down_write(&s->lock); 1403695368acSAlasdair G Kergon __invalidate_snapshot(s, -ENOMEM); 14049d493fa8SAlasdair G Kergon error = 1; 14051da177e4SLinus Torvalds goto out; 14061da177e4SLinus Torvalds } 14071da177e4SLinus Torvalds *e = pe->e; 14081da177e4SLinus Torvalds 14099d493fa8SAlasdair G Kergon down_write(&s->lock); 14109d493fa8SAlasdair G Kergon if (!s->valid) { 14113510cb94SJon Brassow free_completed_exception(e); 14129d493fa8SAlasdair G Kergon error = 1; 14139d493fa8SAlasdair G Kergon goto out; 14149d493fa8SAlasdair G Kergon } 14159d493fa8SAlasdair G Kergon 1416615d1eb9SMike Snitzer /* Check for conflicting reads */ 1417615d1eb9SMike Snitzer __check_for_conflicting_io(s, pe->e.old_chunk); 1418a8d41b59SMikulas Patocka 1419a8d41b59SMikulas Patocka /* 14201da177e4SLinus Torvalds * Add a proper exception, and remove the 14211da177e4SLinus Torvalds * in-flight exception from the list. 14221da177e4SLinus Torvalds */ 14233510cb94SJon Brassow dm_insert_exception(&s->complete, e); 14241da177e4SLinus Torvalds 14251da177e4SLinus Torvalds out: 14263510cb94SJon Brassow dm_remove_exception(&pe->e); 14279d493fa8SAlasdair G Kergon snapshot_bios = bio_list_get(&pe->snapshot_bios); 1428515ad66cSMikulas Patocka origin_bios = bio_list_get(&pe->origin_bios); 1429a6e50b40SMikulas Patocka full_bio = pe->full_bio; 1430a6e50b40SMikulas Patocka if (full_bio) { 1431a6e50b40SMikulas Patocka full_bio->bi_end_io = pe->full_bio_end_io; 1432a6e50b40SMikulas Patocka full_bio->bi_private = pe->full_bio_private; 1433196d38bcSKent Overstreet atomic_inc(&full_bio->bi_remaining); 1434a6e50b40SMikulas Patocka } 1435515ad66cSMikulas Patocka free_pending_exception(pe); 1436b4b610f6SAlasdair G Kergon 143773dfd078SMikulas Patocka increment_pending_exceptions_done_count(); 143873dfd078SMikulas Patocka 14399d493fa8SAlasdair G Kergon up_write(&s->lock); 14409d493fa8SAlasdair G Kergon 14419d493fa8SAlasdair G Kergon /* Submit any pending write bios */ 1442a6e50b40SMikulas Patocka if (error) { 1443a6e50b40SMikulas Patocka if (full_bio) 1444a6e50b40SMikulas Patocka bio_io_error(full_bio); 14459d493fa8SAlasdair G Kergon error_bios(snapshot_bios); 1446a6e50b40SMikulas Patocka } else { 1447a6e50b40SMikulas Patocka if (full_bio) 1448a6e50b40SMikulas Patocka bio_endio(full_bio, 0); 14499d493fa8SAlasdair G Kergon flush_bios(snapshot_bios); 1450a6e50b40SMikulas Patocka } 14519d493fa8SAlasdair G Kergon 1452515ad66cSMikulas Patocka retry_origin_bios(s, origin_bios); 14531da177e4SLinus Torvalds } 14541da177e4SLinus Torvalds 14551da177e4SLinus Torvalds static void commit_callback(void *context, int success) 14561da177e4SLinus Torvalds { 1457028867acSAlasdair G Kergon struct dm_snap_pending_exception *pe = context; 1458028867acSAlasdair G Kergon 14591da177e4SLinus Torvalds pending_complete(pe, success); 14601da177e4SLinus Torvalds } 14611da177e4SLinus Torvalds 1462230c83afSMikulas Patocka static void complete_exception(struct dm_snap_pending_exception *pe) 1463230c83afSMikulas Patocka { 1464230c83afSMikulas Patocka struct dm_snapshot *s = pe->snap; 1465230c83afSMikulas Patocka 1466230c83afSMikulas Patocka if (unlikely(pe->copy_error)) 1467230c83afSMikulas Patocka pending_complete(pe, 0); 1468230c83afSMikulas Patocka 1469230c83afSMikulas Patocka else 1470230c83afSMikulas Patocka /* Update the metadata if we are persistent */ 1471230c83afSMikulas Patocka s->store->type->commit_exception(s->store, &pe->e, 1472230c83afSMikulas Patocka commit_callback, pe); 1473230c83afSMikulas Patocka } 1474230c83afSMikulas Patocka 14751da177e4SLinus Torvalds /* 14761da177e4SLinus Torvalds * Called when the copy I/O has finished. kcopyd actually runs 14771da177e4SLinus Torvalds * this code so don't block. 14781da177e4SLinus Torvalds */ 14794cdc1d1fSAlasdair G Kergon static void copy_callback(int read_err, unsigned long write_err, void *context) 14801da177e4SLinus Torvalds { 1481028867acSAlasdair G Kergon struct dm_snap_pending_exception *pe = context; 14821da177e4SLinus Torvalds struct dm_snapshot *s = pe->snap; 14831da177e4SLinus Torvalds 1484230c83afSMikulas Patocka pe->copy_error = read_err || write_err; 14851da177e4SLinus Torvalds 1486230c83afSMikulas Patocka if (pe->exception_sequence == s->exception_complete_sequence) { 1487230c83afSMikulas Patocka s->exception_complete_sequence++; 1488230c83afSMikulas Patocka complete_exception(pe); 1489230c83afSMikulas Patocka 1490230c83afSMikulas Patocka while (!list_empty(&s->out_of_order_list)) { 1491230c83afSMikulas Patocka pe = list_entry(s->out_of_order_list.next, 1492230c83afSMikulas Patocka struct dm_snap_pending_exception, out_of_order_entry); 1493230c83afSMikulas Patocka if (pe->exception_sequence != s->exception_complete_sequence) 1494230c83afSMikulas Patocka break; 1495230c83afSMikulas Patocka s->exception_complete_sequence++; 1496230c83afSMikulas Patocka list_del(&pe->out_of_order_entry); 1497230c83afSMikulas Patocka complete_exception(pe); 1498230c83afSMikulas Patocka } 1499230c83afSMikulas Patocka } else { 1500230c83afSMikulas Patocka struct list_head *lh; 1501230c83afSMikulas Patocka struct dm_snap_pending_exception *pe2; 1502230c83afSMikulas Patocka 1503230c83afSMikulas Patocka list_for_each_prev(lh, &s->out_of_order_list) { 1504230c83afSMikulas Patocka pe2 = list_entry(lh, struct dm_snap_pending_exception, out_of_order_entry); 1505230c83afSMikulas Patocka if (pe2->exception_sequence < pe->exception_sequence) 1506230c83afSMikulas Patocka break; 1507230c83afSMikulas Patocka } 1508230c83afSMikulas Patocka list_add(&pe->out_of_order_entry, lh); 1509230c83afSMikulas Patocka } 15101da177e4SLinus Torvalds } 15111da177e4SLinus Torvalds 15121da177e4SLinus Torvalds /* 15131da177e4SLinus Torvalds * Dispatches the copy operation to kcopyd. 15141da177e4SLinus Torvalds */ 1515028867acSAlasdair G Kergon static void start_copy(struct dm_snap_pending_exception *pe) 15161da177e4SLinus Torvalds { 15171da177e4SLinus Torvalds struct dm_snapshot *s = pe->snap; 151822a1ceb1SHeinz Mauelshagen struct dm_io_region src, dest; 15191da177e4SLinus Torvalds struct block_device *bdev = s->origin->bdev; 15201da177e4SLinus Torvalds sector_t dev_size; 15211da177e4SLinus Torvalds 15221da177e4SLinus Torvalds dev_size = get_dev_size(bdev); 15231da177e4SLinus Torvalds 15241da177e4SLinus Torvalds src.bdev = bdev; 152571fab00aSJonathan Brassow src.sector = chunk_to_sector(s->store, pe->e.old_chunk); 1526df96eee6SMikulas Patocka src.count = min((sector_t)s->store->chunk_size, dev_size - src.sector); 15271da177e4SLinus Torvalds 1528fc56f6fbSMike Snitzer dest.bdev = s->cow->bdev; 152971fab00aSJonathan Brassow dest.sector = chunk_to_sector(s->store, pe->e.new_chunk); 15301da177e4SLinus Torvalds dest.count = src.count; 15311da177e4SLinus Torvalds 15321da177e4SLinus Torvalds /* Hand over to kcopyd */ 1533a2d2b034SJonathan Brassow dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, copy_callback, pe); 15341da177e4SLinus Torvalds } 15351da177e4SLinus Torvalds 1536a6e50b40SMikulas Patocka static void full_bio_end_io(struct bio *bio, int error) 1537a6e50b40SMikulas Patocka { 1538a6e50b40SMikulas Patocka void *callback_data = bio->bi_private; 1539a6e50b40SMikulas Patocka 1540a6e50b40SMikulas Patocka dm_kcopyd_do_callback(callback_data, 0, error ? 1 : 0); 1541a6e50b40SMikulas Patocka } 1542a6e50b40SMikulas Patocka 1543a6e50b40SMikulas Patocka static void start_full_bio(struct dm_snap_pending_exception *pe, 1544a6e50b40SMikulas Patocka struct bio *bio) 1545a6e50b40SMikulas Patocka { 1546a6e50b40SMikulas Patocka struct dm_snapshot *s = pe->snap; 1547a6e50b40SMikulas Patocka void *callback_data; 1548a6e50b40SMikulas Patocka 1549a6e50b40SMikulas Patocka pe->full_bio = bio; 1550a6e50b40SMikulas Patocka pe->full_bio_end_io = bio->bi_end_io; 1551a6e50b40SMikulas Patocka pe->full_bio_private = bio->bi_private; 1552a6e50b40SMikulas Patocka 1553a6e50b40SMikulas Patocka callback_data = dm_kcopyd_prepare_callback(s->kcopyd_client, 1554a6e50b40SMikulas Patocka copy_callback, pe); 1555a6e50b40SMikulas Patocka 1556a6e50b40SMikulas Patocka bio->bi_end_io = full_bio_end_io; 1557a6e50b40SMikulas Patocka bio->bi_private = callback_data; 1558a6e50b40SMikulas Patocka 1559a6e50b40SMikulas Patocka generic_make_request(bio); 1560a6e50b40SMikulas Patocka } 1561a6e50b40SMikulas Patocka 15622913808eSMikulas Patocka static struct dm_snap_pending_exception * 15632913808eSMikulas Patocka __lookup_pending_exception(struct dm_snapshot *s, chunk_t chunk) 15642913808eSMikulas Patocka { 15653510cb94SJon Brassow struct dm_exception *e = dm_lookup_exception(&s->pending, chunk); 15662913808eSMikulas Patocka 15672913808eSMikulas Patocka if (!e) 15682913808eSMikulas Patocka return NULL; 15692913808eSMikulas Patocka 15702913808eSMikulas Patocka return container_of(e, struct dm_snap_pending_exception, e); 15712913808eSMikulas Patocka } 15722913808eSMikulas Patocka 15731da177e4SLinus Torvalds /* 15741da177e4SLinus Torvalds * Looks to see if this snapshot already has a pending exception 15751da177e4SLinus Torvalds * for this chunk, otherwise it allocates a new one and inserts 15761da177e4SLinus Torvalds * it into the pending table. 15771da177e4SLinus Torvalds * 15781da177e4SLinus Torvalds * NOTE: a write lock must be held on snap->lock before calling 15791da177e4SLinus Torvalds * this. 15801da177e4SLinus Torvalds */ 1581028867acSAlasdair G Kergon static struct dm_snap_pending_exception * 1582c6621392SMikulas Patocka __find_pending_exception(struct dm_snapshot *s, 1583c6621392SMikulas Patocka struct dm_snap_pending_exception *pe, chunk_t chunk) 15841da177e4SLinus Torvalds { 1585c6621392SMikulas Patocka struct dm_snap_pending_exception *pe2; 158676df1c65SAlasdair G Kergon 15872913808eSMikulas Patocka pe2 = __lookup_pending_exception(s, chunk); 15882913808eSMikulas Patocka if (pe2) { 15891da177e4SLinus Torvalds free_pending_exception(pe); 15902913808eSMikulas Patocka return pe2; 159176df1c65SAlasdair G Kergon } 159276df1c65SAlasdair G Kergon 15931da177e4SLinus Torvalds pe->e.old_chunk = chunk; 15941da177e4SLinus Torvalds bio_list_init(&pe->origin_bios); 15951da177e4SLinus Torvalds bio_list_init(&pe->snapshot_bios); 15961da177e4SLinus Torvalds pe->started = 0; 1597a6e50b40SMikulas Patocka pe->full_bio = NULL; 15981da177e4SLinus Torvalds 1599493df71cSJonathan Brassow if (s->store->type->prepare_exception(s->store, &pe->e)) { 16001da177e4SLinus Torvalds free_pending_exception(pe); 16011da177e4SLinus Torvalds return NULL; 16021da177e4SLinus Torvalds } 16031da177e4SLinus Torvalds 1604230c83afSMikulas Patocka pe->exception_sequence = s->exception_start_sequence++; 1605230c83afSMikulas Patocka 16063510cb94SJon Brassow dm_insert_exception(&s->pending, &pe->e); 16071da177e4SLinus Torvalds 16081da177e4SLinus Torvalds return pe; 16091da177e4SLinus Torvalds } 16101da177e4SLinus Torvalds 16111d4989c8SJon Brassow static void remap_exception(struct dm_snapshot *s, struct dm_exception *e, 1612d74f81f8SMilan Broz struct bio *bio, chunk_t chunk) 16131da177e4SLinus Torvalds { 1614fc56f6fbSMike Snitzer bio->bi_bdev = s->cow->bdev; 16154f024f37SKent Overstreet bio->bi_iter.bi_sector = 16164f024f37SKent Overstreet chunk_to_sector(s->store, dm_chunk_number(e->new_chunk) + 1617d74f81f8SMilan Broz (chunk - e->old_chunk)) + 16184f024f37SKent Overstreet (bio->bi_iter.bi_sector & s->store->chunk_mask); 16191da177e4SLinus Torvalds } 16201da177e4SLinus Torvalds 16217de3ee57SMikulas Patocka static int snapshot_map(struct dm_target *ti, struct bio *bio) 16221da177e4SLinus Torvalds { 16231d4989c8SJon Brassow struct dm_exception *e; 1624028867acSAlasdair G Kergon struct dm_snapshot *s = ti->private; 1625d2a7ad29SKiyoshi Ueda int r = DM_MAPIO_REMAPPED; 16261da177e4SLinus Torvalds chunk_t chunk; 1627028867acSAlasdair G Kergon struct dm_snap_pending_exception *pe = NULL; 16281da177e4SLinus Torvalds 1629ee18026aSMikulas Patocka init_tracked_chunk(bio); 1630ee18026aSMikulas Patocka 1631d87f4c14STejun Heo if (bio->bi_rw & REQ_FLUSH) { 1632fc56f6fbSMike Snitzer bio->bi_bdev = s->cow->bdev; 1633494b3ee7SMikulas Patocka return DM_MAPIO_REMAPPED; 1634494b3ee7SMikulas Patocka } 1635494b3ee7SMikulas Patocka 16364f024f37SKent Overstreet chunk = sector_to_chunk(s->store, bio->bi_iter.bi_sector); 16371da177e4SLinus Torvalds 16381da177e4SLinus Torvalds /* Full snapshots are not usable */ 163976df1c65SAlasdair G Kergon /* To get here the table must be live so s->active is always set. */ 16401da177e4SLinus Torvalds if (!s->valid) 1641f6a80ea8SAlasdair G Kergon return -EIO; 16421da177e4SLinus Torvalds 16431da177e4SLinus Torvalds /* FIXME: should only take write lock if we need 16441da177e4SLinus Torvalds * to copy an exception */ 16451da177e4SLinus Torvalds down_write(&s->lock); 16461da177e4SLinus Torvalds 164776df1c65SAlasdair G Kergon if (!s->valid) { 164876df1c65SAlasdair G Kergon r = -EIO; 164976df1c65SAlasdair G Kergon goto out_unlock; 165076df1c65SAlasdair G Kergon } 165176df1c65SAlasdair G Kergon 16521da177e4SLinus Torvalds /* If the block is already remapped - use that, else remap it */ 16533510cb94SJon Brassow e = dm_lookup_exception(&s->complete, chunk); 16541da177e4SLinus Torvalds if (e) { 1655d74f81f8SMilan Broz remap_exception(s, e, bio, chunk); 165676df1c65SAlasdair G Kergon goto out_unlock; 165776df1c65SAlasdair G Kergon } 16581da177e4SLinus Torvalds 1659ba40a2aaSAlasdair G Kergon /* 1660ba40a2aaSAlasdair G Kergon * Write to snapshot - higher level takes care of RW/RO 1661ba40a2aaSAlasdair G Kergon * flags so we should only get this if we are 1662ba40a2aaSAlasdair G Kergon * writeable. 1663ba40a2aaSAlasdair G Kergon */ 1664ba40a2aaSAlasdair G Kergon if (bio_rw(bio) == WRITE) { 16652913808eSMikulas Patocka pe = __lookup_pending_exception(s, chunk); 16662913808eSMikulas Patocka if (!pe) { 1667c6621392SMikulas Patocka up_write(&s->lock); 1668c6621392SMikulas Patocka pe = alloc_pending_exception(s); 1669c6621392SMikulas Patocka down_write(&s->lock); 1670c6621392SMikulas Patocka 1671c6621392SMikulas Patocka if (!s->valid) { 1672c6621392SMikulas Patocka free_pending_exception(pe); 1673c6621392SMikulas Patocka r = -EIO; 1674c6621392SMikulas Patocka goto out_unlock; 1675c6621392SMikulas Patocka } 1676c6621392SMikulas Patocka 16773510cb94SJon Brassow e = dm_lookup_exception(&s->complete, chunk); 167835bf659bSMikulas Patocka if (e) { 167935bf659bSMikulas Patocka free_pending_exception(pe); 168035bf659bSMikulas Patocka remap_exception(s, e, bio, chunk); 168135bf659bSMikulas Patocka goto out_unlock; 168235bf659bSMikulas Patocka } 168335bf659bSMikulas Patocka 1684c6621392SMikulas Patocka pe = __find_pending_exception(s, pe, chunk); 16851da177e4SLinus Torvalds if (!pe) { 1686695368acSAlasdair G Kergon __invalidate_snapshot(s, -ENOMEM); 16871da177e4SLinus Torvalds r = -EIO; 168876df1c65SAlasdair G Kergon goto out_unlock; 168976df1c65SAlasdair G Kergon } 16902913808eSMikulas Patocka } 169176df1c65SAlasdair G Kergon 1692d74f81f8SMilan Broz remap_exception(s, &pe->e, bio, chunk); 16931da177e4SLinus Torvalds 1694d2a7ad29SKiyoshi Ueda r = DM_MAPIO_SUBMITTED; 1695ba40a2aaSAlasdair G Kergon 1696a6e50b40SMikulas Patocka if (!pe->started && 16974f024f37SKent Overstreet bio->bi_iter.bi_size == 16984f024f37SKent Overstreet (s->store->chunk_size << SECTOR_SHIFT)) { 1699a6e50b40SMikulas Patocka pe->started = 1; 1700a6e50b40SMikulas Patocka up_write(&s->lock); 1701a6e50b40SMikulas Patocka start_full_bio(pe, bio); 1702a6e50b40SMikulas Patocka goto out; 1703a6e50b40SMikulas Patocka } 1704a6e50b40SMikulas Patocka 1705a6e50b40SMikulas Patocka bio_list_add(&pe->snapshot_bios, bio); 1706a6e50b40SMikulas Patocka 17071da177e4SLinus Torvalds if (!pe->started) { 17081da177e4SLinus Torvalds /* this is protected by snap->lock */ 17091da177e4SLinus Torvalds pe->started = 1; 171076df1c65SAlasdair G Kergon up_write(&s->lock); 171176df1c65SAlasdair G Kergon start_copy(pe); 1712ba40a2aaSAlasdair G Kergon goto out; 1713ba40a2aaSAlasdair G Kergon } 1714cd45daffSMikulas Patocka } else { 17151da177e4SLinus Torvalds bio->bi_bdev = s->origin->bdev; 1716ee18026aSMikulas Patocka track_chunk(s, bio, chunk); 1717cd45daffSMikulas Patocka } 17181da177e4SLinus Torvalds 1719ba40a2aaSAlasdair G Kergon out_unlock: 1720ba40a2aaSAlasdair G Kergon up_write(&s->lock); 1721ba40a2aaSAlasdair G Kergon out: 17221da177e4SLinus Torvalds return r; 17231da177e4SLinus Torvalds } 17241da177e4SLinus Torvalds 17253452c2a1SMikulas Patocka /* 17263452c2a1SMikulas Patocka * A snapshot-merge target behaves like a combination of a snapshot 17273452c2a1SMikulas Patocka * target and a snapshot-origin target. It only generates new 17283452c2a1SMikulas Patocka * exceptions in other snapshots and not in the one that is being 17293452c2a1SMikulas Patocka * merged. 17303452c2a1SMikulas Patocka * 17313452c2a1SMikulas Patocka * For each chunk, if there is an existing exception, it is used to 17323452c2a1SMikulas Patocka * redirect I/O to the cow device. Otherwise I/O is sent to the origin, 17333452c2a1SMikulas Patocka * which in turn might generate exceptions in other snapshots. 17349fe86254SMikulas Patocka * If merging is currently taking place on the chunk in question, the 17359fe86254SMikulas Patocka * I/O is deferred by adding it to s->bios_queued_during_merge. 17363452c2a1SMikulas Patocka */ 17377de3ee57SMikulas Patocka static int snapshot_merge_map(struct dm_target *ti, struct bio *bio) 17383452c2a1SMikulas Patocka { 17393452c2a1SMikulas Patocka struct dm_exception *e; 17403452c2a1SMikulas Patocka struct dm_snapshot *s = ti->private; 17413452c2a1SMikulas Patocka int r = DM_MAPIO_REMAPPED; 17423452c2a1SMikulas Patocka chunk_t chunk; 17433452c2a1SMikulas Patocka 1744ee18026aSMikulas Patocka init_tracked_chunk(bio); 1745ee18026aSMikulas Patocka 1746d87f4c14STejun Heo if (bio->bi_rw & REQ_FLUSH) { 174755a62eefSAlasdair G Kergon if (!dm_bio_get_target_bio_nr(bio)) 174810b8106aSMike Snitzer bio->bi_bdev = s->origin->bdev; 174910b8106aSMike Snitzer else 175010b8106aSMike Snitzer bio->bi_bdev = s->cow->bdev; 175110b8106aSMike Snitzer return DM_MAPIO_REMAPPED; 175210b8106aSMike Snitzer } 175310b8106aSMike Snitzer 17544f024f37SKent Overstreet chunk = sector_to_chunk(s->store, bio->bi_iter.bi_sector); 17553452c2a1SMikulas Patocka 17569fe86254SMikulas Patocka down_write(&s->lock); 17573452c2a1SMikulas Patocka 1758d2fdb776SMikulas Patocka /* Full merging snapshots are redirected to the origin */ 1759d2fdb776SMikulas Patocka if (!s->valid) 1760d2fdb776SMikulas Patocka goto redirect_to_origin; 17613452c2a1SMikulas Patocka 17623452c2a1SMikulas Patocka /* If the block is already remapped - use that */ 17633452c2a1SMikulas Patocka e = dm_lookup_exception(&s->complete, chunk); 17643452c2a1SMikulas Patocka if (e) { 17659fe86254SMikulas Patocka /* Queue writes overlapping with chunks being merged */ 17669fe86254SMikulas Patocka if (bio_rw(bio) == WRITE && 17679fe86254SMikulas Patocka chunk >= s->first_merging_chunk && 17689fe86254SMikulas Patocka chunk < (s->first_merging_chunk + 17699fe86254SMikulas Patocka s->num_merging_chunks)) { 17709fe86254SMikulas Patocka bio->bi_bdev = s->origin->bdev; 17719fe86254SMikulas Patocka bio_list_add(&s->bios_queued_during_merge, bio); 17729fe86254SMikulas Patocka r = DM_MAPIO_SUBMITTED; 17739fe86254SMikulas Patocka goto out_unlock; 17749fe86254SMikulas Patocka } 177517aa0332SMikulas Patocka 17763452c2a1SMikulas Patocka remap_exception(s, e, bio, chunk); 177717aa0332SMikulas Patocka 177817aa0332SMikulas Patocka if (bio_rw(bio) == WRITE) 1779ee18026aSMikulas Patocka track_chunk(s, bio, chunk); 17803452c2a1SMikulas Patocka goto out_unlock; 17813452c2a1SMikulas Patocka } 17823452c2a1SMikulas Patocka 1783d2fdb776SMikulas Patocka redirect_to_origin: 17843452c2a1SMikulas Patocka bio->bi_bdev = s->origin->bdev; 17853452c2a1SMikulas Patocka 17863452c2a1SMikulas Patocka if (bio_rw(bio) == WRITE) { 17879fe86254SMikulas Patocka up_write(&s->lock); 17883452c2a1SMikulas Patocka return do_origin(s->origin, bio); 17893452c2a1SMikulas Patocka } 17903452c2a1SMikulas Patocka 17913452c2a1SMikulas Patocka out_unlock: 17929fe86254SMikulas Patocka up_write(&s->lock); 17933452c2a1SMikulas Patocka 17943452c2a1SMikulas Patocka return r; 17953452c2a1SMikulas Patocka } 17963452c2a1SMikulas Patocka 17977de3ee57SMikulas Patocka static int snapshot_end_io(struct dm_target *ti, struct bio *bio, int error) 1798cd45daffSMikulas Patocka { 1799cd45daffSMikulas Patocka struct dm_snapshot *s = ti->private; 1800cd45daffSMikulas Patocka 1801ee18026aSMikulas Patocka if (is_bio_tracked(bio)) 1802ee18026aSMikulas Patocka stop_tracking_chunk(s, bio); 1803cd45daffSMikulas Patocka 1804cd45daffSMikulas Patocka return 0; 1805cd45daffSMikulas Patocka } 1806cd45daffSMikulas Patocka 18071e03f97eSMikulas Patocka static void snapshot_merge_presuspend(struct dm_target *ti) 18081e03f97eSMikulas Patocka { 18091e03f97eSMikulas Patocka struct dm_snapshot *s = ti->private; 18101e03f97eSMikulas Patocka 18111e03f97eSMikulas Patocka stop_merge(s); 18121e03f97eSMikulas Patocka } 18131e03f97eSMikulas Patocka 1814c1f0c183SMike Snitzer static int snapshot_preresume(struct dm_target *ti) 1815c1f0c183SMike Snitzer { 1816c1f0c183SMike Snitzer int r = 0; 1817c1f0c183SMike Snitzer struct dm_snapshot *s = ti->private; 1818c1f0c183SMike Snitzer struct dm_snapshot *snap_src = NULL, *snap_dest = NULL; 1819c1f0c183SMike Snitzer 1820c1f0c183SMike Snitzer down_read(&_origins_lock); 18219d3b15c4SMikulas Patocka (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL); 1822c1f0c183SMike Snitzer if (snap_src && snap_dest) { 1823c1f0c183SMike Snitzer down_read(&snap_src->lock); 1824c1f0c183SMike Snitzer if (s == snap_src) { 1825c1f0c183SMike Snitzer DMERR("Unable to resume snapshot source until " 1826c1f0c183SMike Snitzer "handover completes."); 1827c1f0c183SMike Snitzer r = -EINVAL; 1828b83b2f29SMike Snitzer } else if (!dm_suspended(snap_src->ti)) { 1829c1f0c183SMike Snitzer DMERR("Unable to perform snapshot handover until " 1830c1f0c183SMike Snitzer "source is suspended."); 1831c1f0c183SMike Snitzer r = -EINVAL; 1832c1f0c183SMike Snitzer } 1833c1f0c183SMike Snitzer up_read(&snap_src->lock); 1834c1f0c183SMike Snitzer } 1835c1f0c183SMike Snitzer up_read(&_origins_lock); 1836c1f0c183SMike Snitzer 1837c1f0c183SMike Snitzer return r; 1838c1f0c183SMike Snitzer } 1839c1f0c183SMike Snitzer 18401da177e4SLinus Torvalds static void snapshot_resume(struct dm_target *ti) 18411da177e4SLinus Torvalds { 1842028867acSAlasdair G Kergon struct dm_snapshot *s = ti->private; 1843c1f0c183SMike Snitzer struct dm_snapshot *snap_src = NULL, *snap_dest = NULL; 1844c1f0c183SMike Snitzer 1845c1f0c183SMike Snitzer down_read(&_origins_lock); 18469d3b15c4SMikulas Patocka (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL); 1847c1f0c183SMike Snitzer if (snap_src && snap_dest) { 1848c1f0c183SMike Snitzer down_write(&snap_src->lock); 1849c1f0c183SMike Snitzer down_write_nested(&snap_dest->lock, SINGLE_DEPTH_NESTING); 1850c1f0c183SMike Snitzer __handover_exceptions(snap_src, snap_dest); 1851c1f0c183SMike Snitzer up_write(&snap_dest->lock); 1852c1f0c183SMike Snitzer up_write(&snap_src->lock); 1853c1f0c183SMike Snitzer } 1854c1f0c183SMike Snitzer up_read(&_origins_lock); 1855c1f0c183SMike Snitzer 1856c1f0c183SMike Snitzer /* Now we have correct chunk size, reregister */ 1857c1f0c183SMike Snitzer reregister_snapshot(s); 18581da177e4SLinus Torvalds 1859aa14edebSAlasdair G Kergon down_write(&s->lock); 1860aa14edebSAlasdair G Kergon s->active = 1; 1861aa14edebSAlasdair G Kergon up_write(&s->lock); 18621da177e4SLinus Torvalds } 18631da177e4SLinus Torvalds 1864542f9038SMike Snitzer static uint32_t get_origin_minimum_chunksize(struct block_device *bdev) 18651e03f97eSMikulas Patocka { 1866542f9038SMike Snitzer uint32_t min_chunksize; 18671e03f97eSMikulas Patocka 18681e03f97eSMikulas Patocka down_read(&_origins_lock); 18691e03f97eSMikulas Patocka min_chunksize = __minimum_chunk_size(__lookup_origin(bdev)); 18701e03f97eSMikulas Patocka up_read(&_origins_lock); 18711e03f97eSMikulas Patocka 18721e03f97eSMikulas Patocka return min_chunksize; 18731e03f97eSMikulas Patocka } 18741e03f97eSMikulas Patocka 18751e03f97eSMikulas Patocka static void snapshot_merge_resume(struct dm_target *ti) 18761e03f97eSMikulas Patocka { 18771e03f97eSMikulas Patocka struct dm_snapshot *s = ti->private; 18781e03f97eSMikulas Patocka 18791e03f97eSMikulas Patocka /* 18801e03f97eSMikulas Patocka * Handover exceptions from existing snapshot. 18811e03f97eSMikulas Patocka */ 18821e03f97eSMikulas Patocka snapshot_resume(ti); 18831e03f97eSMikulas Patocka 18841e03f97eSMikulas Patocka /* 1885542f9038SMike Snitzer * snapshot-merge acts as an origin, so set ti->max_io_len 18861e03f97eSMikulas Patocka */ 1887542f9038SMike Snitzer ti->max_io_len = get_origin_minimum_chunksize(s->origin->bdev); 18881e03f97eSMikulas Patocka 18891e03f97eSMikulas Patocka start_merge(s); 18901e03f97eSMikulas Patocka } 18911e03f97eSMikulas Patocka 1892fd7c092eSMikulas Patocka static void snapshot_status(struct dm_target *ti, status_type_t type, 18931f4e0ff0SAlasdair G Kergon unsigned status_flags, char *result, unsigned maxlen) 18941da177e4SLinus Torvalds { 18952e4a31dfSJonathan Brassow unsigned sz = 0; 1896028867acSAlasdair G Kergon struct dm_snapshot *snap = ti->private; 18971da177e4SLinus Torvalds 18981da177e4SLinus Torvalds switch (type) { 18991da177e4SLinus Torvalds case STATUSTYPE_INFO: 190094e76572SMikulas Patocka 190194e76572SMikulas Patocka down_write(&snap->lock); 190294e76572SMikulas Patocka 19031da177e4SLinus Torvalds if (!snap->valid) 19042e4a31dfSJonathan Brassow DMEMIT("Invalid"); 1905d8ddb1cfSMike Snitzer else if (snap->merge_failed) 1906d8ddb1cfSMike Snitzer DMEMIT("Merge failed"); 19071da177e4SLinus Torvalds else { 1908985903bbSMike Snitzer if (snap->store->type->usage) { 1909985903bbSMike Snitzer sector_t total_sectors, sectors_allocated, 1910985903bbSMike Snitzer metadata_sectors; 1911985903bbSMike Snitzer snap->store->type->usage(snap->store, 1912985903bbSMike Snitzer &total_sectors, 1913985903bbSMike Snitzer §ors_allocated, 1914985903bbSMike Snitzer &metadata_sectors); 1915985903bbSMike Snitzer DMEMIT("%llu/%llu %llu", 1916985903bbSMike Snitzer (unsigned long long)sectors_allocated, 1917985903bbSMike Snitzer (unsigned long long)total_sectors, 1918985903bbSMike Snitzer (unsigned long long)metadata_sectors); 19191da177e4SLinus Torvalds } 19201da177e4SLinus Torvalds else 19212e4a31dfSJonathan Brassow DMEMIT("Unknown"); 19221da177e4SLinus Torvalds } 192394e76572SMikulas Patocka 192494e76572SMikulas Patocka up_write(&snap->lock); 192594e76572SMikulas Patocka 19261da177e4SLinus Torvalds break; 19271da177e4SLinus Torvalds 19281da177e4SLinus Torvalds case STATUSTYPE_TABLE: 19291da177e4SLinus Torvalds /* 19301da177e4SLinus Torvalds * kdevname returns a static pointer so we need 19311da177e4SLinus Torvalds * to make private copies if the output is to 19321da177e4SLinus Torvalds * make sense. 19331da177e4SLinus Torvalds */ 1934fc56f6fbSMike Snitzer DMEMIT("%s %s", snap->origin->name, snap->cow->name); 19351e302a92SJonathan Brassow snap->store->type->status(snap->store, type, result + sz, 19361e302a92SJonathan Brassow maxlen - sz); 19371da177e4SLinus Torvalds break; 19381da177e4SLinus Torvalds } 19391da177e4SLinus Torvalds } 19401da177e4SLinus Torvalds 19418811f46cSMike Snitzer static int snapshot_iterate_devices(struct dm_target *ti, 19428811f46cSMike Snitzer iterate_devices_callout_fn fn, void *data) 19438811f46cSMike Snitzer { 19448811f46cSMike Snitzer struct dm_snapshot *snap = ti->private; 19451e5554c8SMikulas Patocka int r; 19468811f46cSMike Snitzer 19471e5554c8SMikulas Patocka r = fn(ti, snap->origin, 0, ti->len, data); 19481e5554c8SMikulas Patocka 19491e5554c8SMikulas Patocka if (!r) 19501e5554c8SMikulas Patocka r = fn(ti, snap->cow, 0, get_dev_size(snap->cow->bdev), data); 19511e5554c8SMikulas Patocka 19521e5554c8SMikulas Patocka return r; 19538811f46cSMike Snitzer } 19548811f46cSMike Snitzer 19558811f46cSMike Snitzer 19561da177e4SLinus Torvalds /*----------------------------------------------------------------- 19571da177e4SLinus Torvalds * Origin methods 19581da177e4SLinus Torvalds *---------------------------------------------------------------*/ 19599eaae8ffSMikulas Patocka 19609eaae8ffSMikulas Patocka /* 19619eaae8ffSMikulas Patocka * If no exceptions need creating, DM_MAPIO_REMAPPED is returned and any 19629eaae8ffSMikulas Patocka * supplied bio was ignored. The caller may submit it immediately. 19639eaae8ffSMikulas Patocka * (No remapping actually occurs as the origin is always a direct linear 19649eaae8ffSMikulas Patocka * map.) 19659eaae8ffSMikulas Patocka * 19669eaae8ffSMikulas Patocka * If further exceptions are required, DM_MAPIO_SUBMITTED is returned 19679eaae8ffSMikulas Patocka * and any supplied bio is added to a list to be submitted once all 19689eaae8ffSMikulas Patocka * the necessary exceptions exist. 19699eaae8ffSMikulas Patocka */ 19709eaae8ffSMikulas Patocka static int __origin_write(struct list_head *snapshots, sector_t sector, 19719eaae8ffSMikulas Patocka struct bio *bio) 19721da177e4SLinus Torvalds { 1973515ad66cSMikulas Patocka int r = DM_MAPIO_REMAPPED; 19741da177e4SLinus Torvalds struct dm_snapshot *snap; 19751d4989c8SJon Brassow struct dm_exception *e; 1976515ad66cSMikulas Patocka struct dm_snap_pending_exception *pe; 1977515ad66cSMikulas Patocka struct dm_snap_pending_exception *pe_to_start_now = NULL; 1978515ad66cSMikulas Patocka struct dm_snap_pending_exception *pe_to_start_last = NULL; 19791da177e4SLinus Torvalds chunk_t chunk; 19801da177e4SLinus Torvalds 19811da177e4SLinus Torvalds /* Do all the snapshots on this origin */ 19821da177e4SLinus Torvalds list_for_each_entry (snap, snapshots, list) { 19833452c2a1SMikulas Patocka /* 19843452c2a1SMikulas Patocka * Don't make new exceptions in a merging snapshot 19853452c2a1SMikulas Patocka * because it has effectively been deleted 19863452c2a1SMikulas Patocka */ 19873452c2a1SMikulas Patocka if (dm_target_is_snapshot_merge(snap->ti)) 19883452c2a1SMikulas Patocka continue; 19893452c2a1SMikulas Patocka 199076df1c65SAlasdair G Kergon down_write(&snap->lock); 199176df1c65SAlasdair G Kergon 1992aa14edebSAlasdair G Kergon /* Only deal with valid and active snapshots */ 1993aa14edebSAlasdair G Kergon if (!snap->valid || !snap->active) 199476df1c65SAlasdair G Kergon goto next_snapshot; 19951da177e4SLinus Torvalds 1996d5e404c1SAlasdair G Kergon /* Nothing to do if writing beyond end of snapshot */ 19979eaae8ffSMikulas Patocka if (sector >= dm_table_get_size(snap->ti->table)) 199876df1c65SAlasdair G Kergon goto next_snapshot; 19991da177e4SLinus Torvalds 20001da177e4SLinus Torvalds /* 20011da177e4SLinus Torvalds * Remember, different snapshots can have 20021da177e4SLinus Torvalds * different chunk sizes. 20031da177e4SLinus Torvalds */ 20049eaae8ffSMikulas Patocka chunk = sector_to_chunk(snap->store, sector); 20051da177e4SLinus Torvalds 20061da177e4SLinus Torvalds /* 20071da177e4SLinus Torvalds * Check exception table to see if block 20081da177e4SLinus Torvalds * is already remapped in this snapshot 20091da177e4SLinus Torvalds * and trigger an exception if not. 20101da177e4SLinus Torvalds */ 20113510cb94SJon Brassow e = dm_lookup_exception(&snap->complete, chunk); 201276df1c65SAlasdair G Kergon if (e) 201376df1c65SAlasdair G Kergon goto next_snapshot; 201476df1c65SAlasdair G Kergon 20152913808eSMikulas Patocka pe = __lookup_pending_exception(snap, chunk); 20162913808eSMikulas Patocka if (!pe) { 2017c6621392SMikulas Patocka up_write(&snap->lock); 2018c6621392SMikulas Patocka pe = alloc_pending_exception(snap); 2019c6621392SMikulas Patocka down_write(&snap->lock); 2020c6621392SMikulas Patocka 2021c6621392SMikulas Patocka if (!snap->valid) { 2022c6621392SMikulas Patocka free_pending_exception(pe); 2023c6621392SMikulas Patocka goto next_snapshot; 2024c6621392SMikulas Patocka } 2025c6621392SMikulas Patocka 20263510cb94SJon Brassow e = dm_lookup_exception(&snap->complete, chunk); 202735bf659bSMikulas Patocka if (e) { 202835bf659bSMikulas Patocka free_pending_exception(pe); 202935bf659bSMikulas Patocka goto next_snapshot; 203035bf659bSMikulas Patocka } 203135bf659bSMikulas Patocka 2032c6621392SMikulas Patocka pe = __find_pending_exception(snap, pe, chunk); 20331da177e4SLinus Torvalds if (!pe) { 2034695368acSAlasdair G Kergon __invalidate_snapshot(snap, -ENOMEM); 203576df1c65SAlasdair G Kergon goto next_snapshot; 203676df1c65SAlasdair G Kergon } 20372913808eSMikulas Patocka } 20381da177e4SLinus Torvalds 2039d2a7ad29SKiyoshi Ueda r = DM_MAPIO_SUBMITTED; 204076df1c65SAlasdair G Kergon 2041515ad66cSMikulas Patocka /* 2042515ad66cSMikulas Patocka * If an origin bio was supplied, queue it to wait for the 2043515ad66cSMikulas Patocka * completion of this exception, and start this one last, 2044515ad66cSMikulas Patocka * at the end of the function. 2045515ad66cSMikulas Patocka */ 2046515ad66cSMikulas Patocka if (bio) { 2047515ad66cSMikulas Patocka bio_list_add(&pe->origin_bios, bio); 2048515ad66cSMikulas Patocka bio = NULL; 2049515ad66cSMikulas Patocka 2050515ad66cSMikulas Patocka if (!pe->started) { 2051515ad66cSMikulas Patocka pe->started = 1; 2052515ad66cSMikulas Patocka pe_to_start_last = pe; 2053515ad66cSMikulas Patocka } 2054b4b610f6SAlasdair G Kergon } 205576df1c65SAlasdair G Kergon 2056eccf0817SAlasdair G Kergon if (!pe->started) { 2057eccf0817SAlasdair G Kergon pe->started = 1; 2058515ad66cSMikulas Patocka pe_to_start_now = pe; 2059eccf0817SAlasdair G Kergon } 20601da177e4SLinus Torvalds 206176df1c65SAlasdair G Kergon next_snapshot: 20621da177e4SLinus Torvalds up_write(&snap->lock); 2063515ad66cSMikulas Patocka 2064515ad66cSMikulas Patocka if (pe_to_start_now) { 2065515ad66cSMikulas Patocka start_copy(pe_to_start_now); 2066515ad66cSMikulas Patocka pe_to_start_now = NULL; 20671da177e4SLinus Torvalds } 2068b4b610f6SAlasdair G Kergon } 2069b4b610f6SAlasdair G Kergon 20701da177e4SLinus Torvalds /* 2071515ad66cSMikulas Patocka * Submit the exception against which the bio is queued last, 2072515ad66cSMikulas Patocka * to give the other exceptions a head start. 20731da177e4SLinus Torvalds */ 2074515ad66cSMikulas Patocka if (pe_to_start_last) 2075515ad66cSMikulas Patocka start_copy(pe_to_start_last); 20761da177e4SLinus Torvalds 20771da177e4SLinus Torvalds return r; 20781da177e4SLinus Torvalds } 20791da177e4SLinus Torvalds 20801da177e4SLinus Torvalds /* 20811da177e4SLinus Torvalds * Called on a write from the origin driver. 20821da177e4SLinus Torvalds */ 20831da177e4SLinus Torvalds static int do_origin(struct dm_dev *origin, struct bio *bio) 20841da177e4SLinus Torvalds { 20851da177e4SLinus Torvalds struct origin *o; 2086d2a7ad29SKiyoshi Ueda int r = DM_MAPIO_REMAPPED; 20871da177e4SLinus Torvalds 20881da177e4SLinus Torvalds down_read(&_origins_lock); 20891da177e4SLinus Torvalds o = __lookup_origin(origin->bdev); 20901da177e4SLinus Torvalds if (o) 20914f024f37SKent Overstreet r = __origin_write(&o->snapshots, bio->bi_iter.bi_sector, bio); 20921da177e4SLinus Torvalds up_read(&_origins_lock); 20931da177e4SLinus Torvalds 20941da177e4SLinus Torvalds return r; 20951da177e4SLinus Torvalds } 20961da177e4SLinus Torvalds 20971da177e4SLinus Torvalds /* 209873dfd078SMikulas Patocka * Trigger exceptions in all non-merging snapshots. 209973dfd078SMikulas Patocka * 210073dfd078SMikulas Patocka * The chunk size of the merging snapshot may be larger than the chunk 210173dfd078SMikulas Patocka * size of some other snapshot so we may need to reallocate multiple 210273dfd078SMikulas Patocka * chunks in other snapshots. 210373dfd078SMikulas Patocka * 210473dfd078SMikulas Patocka * We scan all the overlapping exceptions in the other snapshots. 210573dfd078SMikulas Patocka * Returns 1 if anything was reallocated and must be waited for, 210673dfd078SMikulas Patocka * otherwise returns 0. 210773dfd078SMikulas Patocka * 210873dfd078SMikulas Patocka * size must be a multiple of merging_snap's chunk_size. 210973dfd078SMikulas Patocka */ 211073dfd078SMikulas Patocka static int origin_write_extent(struct dm_snapshot *merging_snap, 211173dfd078SMikulas Patocka sector_t sector, unsigned size) 211273dfd078SMikulas Patocka { 211373dfd078SMikulas Patocka int must_wait = 0; 211473dfd078SMikulas Patocka sector_t n; 211573dfd078SMikulas Patocka struct origin *o; 211673dfd078SMikulas Patocka 211773dfd078SMikulas Patocka /* 2118542f9038SMike Snitzer * The origin's __minimum_chunk_size() got stored in max_io_len 211973dfd078SMikulas Patocka * by snapshot_merge_resume(). 212073dfd078SMikulas Patocka */ 212173dfd078SMikulas Patocka down_read(&_origins_lock); 212273dfd078SMikulas Patocka o = __lookup_origin(merging_snap->origin->bdev); 2123542f9038SMike Snitzer for (n = 0; n < size; n += merging_snap->ti->max_io_len) 212473dfd078SMikulas Patocka if (__origin_write(&o->snapshots, sector + n, NULL) == 212573dfd078SMikulas Patocka DM_MAPIO_SUBMITTED) 212673dfd078SMikulas Patocka must_wait = 1; 212773dfd078SMikulas Patocka up_read(&_origins_lock); 212873dfd078SMikulas Patocka 212973dfd078SMikulas Patocka return must_wait; 213073dfd078SMikulas Patocka } 213173dfd078SMikulas Patocka 213273dfd078SMikulas Patocka /* 21331da177e4SLinus Torvalds * Origin: maps a linear range of a device, with hooks for snapshotting. 21341da177e4SLinus Torvalds */ 21351da177e4SLinus Torvalds 2136599cdf3bSMikulas Patocka struct dm_origin { 2137599cdf3bSMikulas Patocka struct dm_dev *dev; 2138298eaa89SMikulas Patocka unsigned split_boundary; 2139599cdf3bSMikulas Patocka }; 2140599cdf3bSMikulas Patocka 21411da177e4SLinus Torvalds /* 21421da177e4SLinus Torvalds * Construct an origin mapping: <dev_path> 21431da177e4SLinus Torvalds * The context for an origin is merely a 'struct dm_dev *' 21441da177e4SLinus Torvalds * pointing to the real device. 21451da177e4SLinus Torvalds */ 21461da177e4SLinus Torvalds static int origin_ctr(struct dm_target *ti, unsigned int argc, char **argv) 21471da177e4SLinus Torvalds { 21481da177e4SLinus Torvalds int r; 2149599cdf3bSMikulas Patocka struct dm_origin *o; 21501da177e4SLinus Torvalds 21511da177e4SLinus Torvalds if (argc != 1) { 215272d94861SAlasdair G Kergon ti->error = "origin: incorrect number of arguments"; 21531da177e4SLinus Torvalds return -EINVAL; 21541da177e4SLinus Torvalds } 21551da177e4SLinus Torvalds 2156599cdf3bSMikulas Patocka o = kmalloc(sizeof(struct dm_origin), GFP_KERNEL); 2157599cdf3bSMikulas Patocka if (!o) { 2158599cdf3bSMikulas Patocka ti->error = "Cannot allocate private origin structure"; 2159599cdf3bSMikulas Patocka r = -ENOMEM; 2160599cdf3bSMikulas Patocka goto bad_alloc; 21611da177e4SLinus Torvalds } 21621da177e4SLinus Torvalds 2163599cdf3bSMikulas Patocka r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &o->dev); 2164599cdf3bSMikulas Patocka if (r) { 2165599cdf3bSMikulas Patocka ti->error = "Cannot get target device"; 2166599cdf3bSMikulas Patocka goto bad_open; 2167599cdf3bSMikulas Patocka } 2168599cdf3bSMikulas Patocka 2169599cdf3bSMikulas Patocka ti->private = o; 217055a62eefSAlasdair G Kergon ti->num_flush_bios = 1; 2171494b3ee7SMikulas Patocka 21721da177e4SLinus Torvalds return 0; 2173599cdf3bSMikulas Patocka 2174599cdf3bSMikulas Patocka bad_open: 2175599cdf3bSMikulas Patocka kfree(o); 2176599cdf3bSMikulas Patocka bad_alloc: 2177599cdf3bSMikulas Patocka return r; 21781da177e4SLinus Torvalds } 21791da177e4SLinus Torvalds 21801da177e4SLinus Torvalds static void origin_dtr(struct dm_target *ti) 21811da177e4SLinus Torvalds { 2182599cdf3bSMikulas Patocka struct dm_origin *o = ti->private; 2183599cdf3bSMikulas Patocka dm_put_device(ti, o->dev); 2184599cdf3bSMikulas Patocka kfree(o); 21851da177e4SLinus Torvalds } 21861da177e4SLinus Torvalds 21877de3ee57SMikulas Patocka static int origin_map(struct dm_target *ti, struct bio *bio) 21881da177e4SLinus Torvalds { 2189599cdf3bSMikulas Patocka struct dm_origin *o = ti->private; 2190298eaa89SMikulas Patocka unsigned available_sectors; 21911da177e4SLinus Torvalds 2192599cdf3bSMikulas Patocka bio->bi_bdev = o->dev->bdev; 21931da177e4SLinus Torvalds 2194298eaa89SMikulas Patocka if (unlikely(bio->bi_rw & REQ_FLUSH)) 2195494b3ee7SMikulas Patocka return DM_MAPIO_REMAPPED; 2196494b3ee7SMikulas Patocka 2197298eaa89SMikulas Patocka if (bio_rw(bio) != WRITE) 2198298eaa89SMikulas Patocka return DM_MAPIO_REMAPPED; 2199298eaa89SMikulas Patocka 2200298eaa89SMikulas Patocka available_sectors = o->split_boundary - 2201298eaa89SMikulas Patocka ((unsigned)bio->bi_iter.bi_sector & (o->split_boundary - 1)); 2202298eaa89SMikulas Patocka 2203298eaa89SMikulas Patocka if (bio_sectors(bio) > available_sectors) 2204298eaa89SMikulas Patocka dm_accept_partial_bio(bio, available_sectors); 2205298eaa89SMikulas Patocka 22061da177e4SLinus Torvalds /* Only tell snapshots if this is a write */ 2207298eaa89SMikulas Patocka return do_origin(o->dev, bio); 22081da177e4SLinus Torvalds } 22091da177e4SLinus Torvalds 22101da177e4SLinus Torvalds /* 2211542f9038SMike Snitzer * Set the target "max_io_len" field to the minimum of all the snapshots' 22121da177e4SLinus Torvalds * chunk sizes. 22131da177e4SLinus Torvalds */ 22141da177e4SLinus Torvalds static void origin_resume(struct dm_target *ti) 22151da177e4SLinus Torvalds { 2216599cdf3bSMikulas Patocka struct dm_origin *o = ti->private; 22171da177e4SLinus Torvalds 2218298eaa89SMikulas Patocka o->split_boundary = get_origin_minimum_chunksize(o->dev->bdev); 22191da177e4SLinus Torvalds } 22201da177e4SLinus Torvalds 2221fd7c092eSMikulas Patocka static void origin_status(struct dm_target *ti, status_type_t type, 22221f4e0ff0SAlasdair G Kergon unsigned status_flags, char *result, unsigned maxlen) 22231da177e4SLinus Torvalds { 2224599cdf3bSMikulas Patocka struct dm_origin *o = ti->private; 22251da177e4SLinus Torvalds 22261da177e4SLinus Torvalds switch (type) { 22271da177e4SLinus Torvalds case STATUSTYPE_INFO: 22281da177e4SLinus Torvalds result[0] = '\0'; 22291da177e4SLinus Torvalds break; 22301da177e4SLinus Torvalds 22311da177e4SLinus Torvalds case STATUSTYPE_TABLE: 2232599cdf3bSMikulas Patocka snprintf(result, maxlen, "%s", o->dev->name); 22331da177e4SLinus Torvalds break; 22341da177e4SLinus Torvalds } 22351da177e4SLinus Torvalds } 22361da177e4SLinus Torvalds 2237b1d55528SMikulas Patocka static int origin_merge(struct dm_target *ti, struct bvec_merge_data *bvm, 2238b1d55528SMikulas Patocka struct bio_vec *biovec, int max_size) 2239b1d55528SMikulas Patocka { 2240599cdf3bSMikulas Patocka struct dm_origin *o = ti->private; 2241599cdf3bSMikulas Patocka struct request_queue *q = bdev_get_queue(o->dev->bdev); 2242b1d55528SMikulas Patocka 2243b1d55528SMikulas Patocka if (!q->merge_bvec_fn) 2244b1d55528SMikulas Patocka return max_size; 2245b1d55528SMikulas Patocka 2246599cdf3bSMikulas Patocka bvm->bi_bdev = o->dev->bdev; 2247b1d55528SMikulas Patocka 2248b1d55528SMikulas Patocka return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); 2249b1d55528SMikulas Patocka } 2250b1d55528SMikulas Patocka 22518811f46cSMike Snitzer static int origin_iterate_devices(struct dm_target *ti, 22528811f46cSMike Snitzer iterate_devices_callout_fn fn, void *data) 22538811f46cSMike Snitzer { 2254599cdf3bSMikulas Patocka struct dm_origin *o = ti->private; 22558811f46cSMike Snitzer 2256599cdf3bSMikulas Patocka return fn(ti, o->dev, 0, ti->len, data); 22578811f46cSMike Snitzer } 22588811f46cSMike Snitzer 22591da177e4SLinus Torvalds static struct target_type origin_target = { 22601da177e4SLinus Torvalds .name = "snapshot-origin", 2261fd7c092eSMikulas Patocka .version = {1, 8, 1}, 22621da177e4SLinus Torvalds .module = THIS_MODULE, 22631da177e4SLinus Torvalds .ctr = origin_ctr, 22641da177e4SLinus Torvalds .dtr = origin_dtr, 22651da177e4SLinus Torvalds .map = origin_map, 22661da177e4SLinus Torvalds .resume = origin_resume, 22671da177e4SLinus Torvalds .status = origin_status, 2268b1d55528SMikulas Patocka .merge = origin_merge, 22698811f46cSMike Snitzer .iterate_devices = origin_iterate_devices, 22701da177e4SLinus Torvalds }; 22711da177e4SLinus Torvalds 22721da177e4SLinus Torvalds static struct target_type snapshot_target = { 22731da177e4SLinus Torvalds .name = "snapshot", 2274230c83afSMikulas Patocka .version = {1, 12, 0}, 22751da177e4SLinus Torvalds .module = THIS_MODULE, 22761da177e4SLinus Torvalds .ctr = snapshot_ctr, 22771da177e4SLinus Torvalds .dtr = snapshot_dtr, 22781da177e4SLinus Torvalds .map = snapshot_map, 2279cd45daffSMikulas Patocka .end_io = snapshot_end_io, 2280c1f0c183SMike Snitzer .preresume = snapshot_preresume, 22811da177e4SLinus Torvalds .resume = snapshot_resume, 22821da177e4SLinus Torvalds .status = snapshot_status, 22838811f46cSMike Snitzer .iterate_devices = snapshot_iterate_devices, 22841da177e4SLinus Torvalds }; 22851da177e4SLinus Torvalds 2286d698aa45SMikulas Patocka static struct target_type merge_target = { 2287d698aa45SMikulas Patocka .name = dm_snapshot_merge_target_name, 228842bc954fSMikulas Patocka .version = {1, 2, 0}, 2289d698aa45SMikulas Patocka .module = THIS_MODULE, 2290d698aa45SMikulas Patocka .ctr = snapshot_ctr, 2291d698aa45SMikulas Patocka .dtr = snapshot_dtr, 22923452c2a1SMikulas Patocka .map = snapshot_merge_map, 2293d698aa45SMikulas Patocka .end_io = snapshot_end_io, 22941e03f97eSMikulas Patocka .presuspend = snapshot_merge_presuspend, 2295d698aa45SMikulas Patocka .preresume = snapshot_preresume, 22961e03f97eSMikulas Patocka .resume = snapshot_merge_resume, 2297d698aa45SMikulas Patocka .status = snapshot_status, 2298d698aa45SMikulas Patocka .iterate_devices = snapshot_iterate_devices, 2299d698aa45SMikulas Patocka }; 2300d698aa45SMikulas Patocka 23011da177e4SLinus Torvalds static int __init dm_snapshot_init(void) 23021da177e4SLinus Torvalds { 23031da177e4SLinus Torvalds int r; 23041da177e4SLinus Torvalds 23054db6bfe0SAlasdair G Kergon r = dm_exception_store_init(); 23064db6bfe0SAlasdair G Kergon if (r) { 23074db6bfe0SAlasdair G Kergon DMERR("Failed to initialize exception stores"); 23084db6bfe0SAlasdair G Kergon return r; 23094db6bfe0SAlasdair G Kergon } 23104db6bfe0SAlasdair G Kergon 23111da177e4SLinus Torvalds r = dm_register_target(&snapshot_target); 2312d698aa45SMikulas Patocka if (r < 0) { 23131da177e4SLinus Torvalds DMERR("snapshot target register failed %d", r); 2314034a186dSJonathan Brassow goto bad_register_snapshot_target; 23151da177e4SLinus Torvalds } 23161da177e4SLinus Torvalds 23171da177e4SLinus Torvalds r = dm_register_target(&origin_target); 23181da177e4SLinus Torvalds if (r < 0) { 231972d94861SAlasdair G Kergon DMERR("Origin target register failed %d", r); 2320d698aa45SMikulas Patocka goto bad_register_origin_target; 2321d698aa45SMikulas Patocka } 2322d698aa45SMikulas Patocka 2323d698aa45SMikulas Patocka r = dm_register_target(&merge_target); 2324d698aa45SMikulas Patocka if (r < 0) { 2325d698aa45SMikulas Patocka DMERR("Merge target register failed %d", r); 2326d698aa45SMikulas Patocka goto bad_register_merge_target; 23271da177e4SLinus Torvalds } 23281da177e4SLinus Torvalds 23291da177e4SLinus Torvalds r = init_origin_hash(); 23301da177e4SLinus Torvalds if (r) { 23311da177e4SLinus Torvalds DMERR("init_origin_hash failed."); 2332d698aa45SMikulas Patocka goto bad_origin_hash; 23331da177e4SLinus Torvalds } 23341da177e4SLinus Torvalds 23351d4989c8SJon Brassow exception_cache = KMEM_CACHE(dm_exception, 0); 23361da177e4SLinus Torvalds if (!exception_cache) { 23371da177e4SLinus Torvalds DMERR("Couldn't create exception cache."); 23381da177e4SLinus Torvalds r = -ENOMEM; 2339d698aa45SMikulas Patocka goto bad_exception_cache; 23401da177e4SLinus Torvalds } 23411da177e4SLinus Torvalds 2342028867acSAlasdair G Kergon pending_cache = KMEM_CACHE(dm_snap_pending_exception, 0); 23431da177e4SLinus Torvalds if (!pending_cache) { 23441da177e4SLinus Torvalds DMERR("Couldn't create pending cache."); 23451da177e4SLinus Torvalds r = -ENOMEM; 2346d698aa45SMikulas Patocka goto bad_pending_cache; 23471da177e4SLinus Torvalds } 23481da177e4SLinus Torvalds 23491da177e4SLinus Torvalds return 0; 23501da177e4SLinus Torvalds 2351d698aa45SMikulas Patocka bad_pending_cache: 23521da177e4SLinus Torvalds kmem_cache_destroy(exception_cache); 2353d698aa45SMikulas Patocka bad_exception_cache: 23541da177e4SLinus Torvalds exit_origin_hash(); 2355d698aa45SMikulas Patocka bad_origin_hash: 2356d698aa45SMikulas Patocka dm_unregister_target(&merge_target); 2357d698aa45SMikulas Patocka bad_register_merge_target: 23581da177e4SLinus Torvalds dm_unregister_target(&origin_target); 2359d698aa45SMikulas Patocka bad_register_origin_target: 23601da177e4SLinus Torvalds dm_unregister_target(&snapshot_target); 2361034a186dSJonathan Brassow bad_register_snapshot_target: 2362034a186dSJonathan Brassow dm_exception_store_exit(); 2363d698aa45SMikulas Patocka 23641da177e4SLinus Torvalds return r; 23651da177e4SLinus Torvalds } 23661da177e4SLinus Torvalds 23671da177e4SLinus Torvalds static void __exit dm_snapshot_exit(void) 23681da177e4SLinus Torvalds { 236910d3bd09SMikulas Patocka dm_unregister_target(&snapshot_target); 237010d3bd09SMikulas Patocka dm_unregister_target(&origin_target); 2371d698aa45SMikulas Patocka dm_unregister_target(&merge_target); 23721da177e4SLinus Torvalds 23731da177e4SLinus Torvalds exit_origin_hash(); 23741da177e4SLinus Torvalds kmem_cache_destroy(pending_cache); 23751da177e4SLinus Torvalds kmem_cache_destroy(exception_cache); 23764db6bfe0SAlasdair G Kergon 23774db6bfe0SAlasdair G Kergon dm_exception_store_exit(); 23781da177e4SLinus Torvalds } 23791da177e4SLinus Torvalds 23801da177e4SLinus Torvalds /* Module hooks */ 23811da177e4SLinus Torvalds module_init(dm_snapshot_init); 23821da177e4SLinus Torvalds module_exit(dm_snapshot_exit); 23831da177e4SLinus Torvalds 23841da177e4SLinus Torvalds MODULE_DESCRIPTION(DM_NAME " snapshot target"); 23851da177e4SLinus Torvalds MODULE_AUTHOR("Joe Thornber"); 23861da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 238723cb2109SMikulas Patocka MODULE_ALIAS("dm-snapshot-origin"); 238823cb2109SMikulas Patocka MODULE_ALIAS("dm-snapshot-merge"); 2389