11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * dm-snapshot.c 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Copyright (C) 2001-2002 Sistina Software (UK) Limited. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * This file is released under the GPL. 71da177e4SLinus Torvalds */ 81da177e4SLinus Torvalds 91da177e4SLinus Torvalds #include <linux/blkdev.h> 101da177e4SLinus Torvalds #include <linux/ctype.h> 111da177e4SLinus Torvalds #include <linux/device-mapper.h> 1290fa1527SMikulas Patocka #include <linux/delay.h> 131da177e4SLinus Torvalds #include <linux/fs.h> 141da177e4SLinus Torvalds #include <linux/init.h> 151da177e4SLinus Torvalds #include <linux/kdev_t.h> 161da177e4SLinus Torvalds #include <linux/list.h> 171da177e4SLinus Torvalds #include <linux/mempool.h> 181da177e4SLinus Torvalds #include <linux/module.h> 191da177e4SLinus Torvalds #include <linux/slab.h> 201da177e4SLinus Torvalds #include <linux/vmalloc.h> 216f3c3f0aSvignesh babu #include <linux/log2.h> 22a765e20eSAlasdair G Kergon #include <linux/dm-kcopyd.h> 231da177e4SLinus Torvalds 24aea53d92SJonathan Brassow #include "dm-exception-store.h" 251da177e4SLinus Torvalds #include "dm-snap.h" 261da177e4SLinus Torvalds #include "dm-bio-list.h" 271da177e4SLinus Torvalds 2872d94861SAlasdair G Kergon #define DM_MSG_PREFIX "snapshots" 2972d94861SAlasdair G Kergon 301da177e4SLinus Torvalds /* 311da177e4SLinus Torvalds * The percentage increment we will wake up users at 321da177e4SLinus Torvalds */ 331da177e4SLinus Torvalds #define WAKE_UP_PERCENT 5 341da177e4SLinus Torvalds 351da177e4SLinus Torvalds /* 361da177e4SLinus Torvalds * kcopyd priority of snapshot operations 371da177e4SLinus Torvalds */ 381da177e4SLinus Torvalds #define SNAPSHOT_COPY_PRIORITY 2 391da177e4SLinus Torvalds 401da177e4SLinus Torvalds /* 418ee2767aSMilan Broz * Reserve 1MB for each snapshot initially (with minimum of 1 page). 421da177e4SLinus Torvalds */ 438ee2767aSMilan Broz #define SNAPSHOT_PAGES (((1UL << 20) >> PAGE_SHIFT) ? : 1) 441da177e4SLinus Torvalds 45cd45daffSMikulas Patocka /* 46cd45daffSMikulas Patocka * The size of the mempool used to track chunks in use. 47cd45daffSMikulas Patocka */ 48cd45daffSMikulas Patocka #define MIN_IOS 256 49cd45daffSMikulas Patocka 50c642f9e0SAdrian Bunk static struct workqueue_struct *ksnapd; 51c4028958SDavid Howells static void flush_queued_bios(struct work_struct *work); 52ca3a931fSAlasdair G Kergon 53028867acSAlasdair G Kergon struct dm_snap_pending_exception { 54028867acSAlasdair G Kergon struct dm_snap_exception e; 551da177e4SLinus Torvalds 561da177e4SLinus Torvalds /* 571da177e4SLinus Torvalds * Origin buffers waiting for this to complete are held 581da177e4SLinus Torvalds * in a bio list 591da177e4SLinus Torvalds */ 601da177e4SLinus Torvalds struct bio_list origin_bios; 611da177e4SLinus Torvalds struct bio_list snapshot_bios; 621da177e4SLinus Torvalds 631da177e4SLinus Torvalds /* 64eccf0817SAlasdair G Kergon * Short-term queue of pending exceptions prior to submission. 65eccf0817SAlasdair G Kergon */ 66eccf0817SAlasdair G Kergon struct list_head list; 67eccf0817SAlasdair G Kergon 68eccf0817SAlasdair G Kergon /* 69b4b610f6SAlasdair G Kergon * The primary pending_exception is the one that holds 704b832e8dSAlasdair G Kergon * the ref_count and the list of origin_bios for a 71b4b610f6SAlasdair G Kergon * group of pending_exceptions. It is always last to get freed. 72b4b610f6SAlasdair G Kergon * These fields get set up when writing to the origin. 731da177e4SLinus Torvalds */ 74028867acSAlasdair G Kergon struct dm_snap_pending_exception *primary_pe; 75b4b610f6SAlasdair G Kergon 76b4b610f6SAlasdair G Kergon /* 77b4b610f6SAlasdair G Kergon * Number of pending_exceptions processing this chunk. 78b4b610f6SAlasdair G Kergon * When this drops to zero we must complete the origin bios. 79b4b610f6SAlasdair G Kergon * If incrementing or decrementing this, hold pe->snap->lock for 80b4b610f6SAlasdair G Kergon * the sibling concerned and not pe->primary_pe->snap->lock unless 81b4b610f6SAlasdair G Kergon * they are the same. 82b4b610f6SAlasdair G Kergon */ 834b832e8dSAlasdair G Kergon atomic_t ref_count; 841da177e4SLinus Torvalds 851da177e4SLinus Torvalds /* Pointer back to snapshot context */ 861da177e4SLinus Torvalds struct dm_snapshot *snap; 871da177e4SLinus Torvalds 881da177e4SLinus Torvalds /* 891da177e4SLinus Torvalds * 1 indicates the exception has already been sent to 901da177e4SLinus Torvalds * kcopyd. 911da177e4SLinus Torvalds */ 921da177e4SLinus Torvalds int started; 931da177e4SLinus Torvalds }; 941da177e4SLinus Torvalds 951da177e4SLinus Torvalds /* 961da177e4SLinus Torvalds * Hash table mapping origin volumes to lists of snapshots and 971da177e4SLinus Torvalds * a lock to protect it 981da177e4SLinus Torvalds */ 99e18b890bSChristoph Lameter static struct kmem_cache *exception_cache; 100e18b890bSChristoph Lameter static struct kmem_cache *pending_cache; 1011da177e4SLinus Torvalds 102cd45daffSMikulas Patocka struct dm_snap_tracked_chunk { 103cd45daffSMikulas Patocka struct hlist_node node; 104cd45daffSMikulas Patocka chunk_t chunk; 105cd45daffSMikulas Patocka }; 106cd45daffSMikulas Patocka 107cd45daffSMikulas Patocka static struct kmem_cache *tracked_chunk_cache; 108cd45daffSMikulas Patocka 109cd45daffSMikulas Patocka static struct dm_snap_tracked_chunk *track_chunk(struct dm_snapshot *s, 110cd45daffSMikulas Patocka chunk_t chunk) 111cd45daffSMikulas Patocka { 112cd45daffSMikulas Patocka struct dm_snap_tracked_chunk *c = mempool_alloc(s->tracked_chunk_pool, 113cd45daffSMikulas Patocka GFP_NOIO); 114cd45daffSMikulas Patocka unsigned long flags; 115cd45daffSMikulas Patocka 116cd45daffSMikulas Patocka c->chunk = chunk; 117cd45daffSMikulas Patocka 118cd45daffSMikulas Patocka spin_lock_irqsave(&s->tracked_chunk_lock, flags); 119cd45daffSMikulas Patocka hlist_add_head(&c->node, 120cd45daffSMikulas Patocka &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)]); 121cd45daffSMikulas Patocka spin_unlock_irqrestore(&s->tracked_chunk_lock, flags); 122cd45daffSMikulas Patocka 123cd45daffSMikulas Patocka return c; 124cd45daffSMikulas Patocka } 125cd45daffSMikulas Patocka 126cd45daffSMikulas Patocka static void stop_tracking_chunk(struct dm_snapshot *s, 127cd45daffSMikulas Patocka struct dm_snap_tracked_chunk *c) 128cd45daffSMikulas Patocka { 129cd45daffSMikulas Patocka unsigned long flags; 130cd45daffSMikulas Patocka 131cd45daffSMikulas Patocka spin_lock_irqsave(&s->tracked_chunk_lock, flags); 132cd45daffSMikulas Patocka hlist_del(&c->node); 133cd45daffSMikulas Patocka spin_unlock_irqrestore(&s->tracked_chunk_lock, flags); 134cd45daffSMikulas Patocka 135cd45daffSMikulas Patocka mempool_free(c, s->tracked_chunk_pool); 136cd45daffSMikulas Patocka } 137cd45daffSMikulas Patocka 138a8d41b59SMikulas Patocka static int __chunk_is_tracked(struct dm_snapshot *s, chunk_t chunk) 139a8d41b59SMikulas Patocka { 140a8d41b59SMikulas Patocka struct dm_snap_tracked_chunk *c; 141a8d41b59SMikulas Patocka struct hlist_node *hn; 142a8d41b59SMikulas Patocka int found = 0; 143a8d41b59SMikulas Patocka 144a8d41b59SMikulas Patocka spin_lock_irq(&s->tracked_chunk_lock); 145a8d41b59SMikulas Patocka 146a8d41b59SMikulas Patocka hlist_for_each_entry(c, hn, 147a8d41b59SMikulas Patocka &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)], node) { 148a8d41b59SMikulas Patocka if (c->chunk == chunk) { 149a8d41b59SMikulas Patocka found = 1; 150a8d41b59SMikulas Patocka break; 151a8d41b59SMikulas Patocka } 152a8d41b59SMikulas Patocka } 153a8d41b59SMikulas Patocka 154a8d41b59SMikulas Patocka spin_unlock_irq(&s->tracked_chunk_lock); 155a8d41b59SMikulas Patocka 156a8d41b59SMikulas Patocka return found; 157a8d41b59SMikulas Patocka } 158a8d41b59SMikulas Patocka 1591da177e4SLinus Torvalds /* 1601da177e4SLinus Torvalds * One of these per registered origin, held in the snapshot_origins hash 1611da177e4SLinus Torvalds */ 1621da177e4SLinus Torvalds struct origin { 1631da177e4SLinus Torvalds /* The origin device */ 1641da177e4SLinus Torvalds struct block_device *bdev; 1651da177e4SLinus Torvalds 1661da177e4SLinus Torvalds struct list_head hash_list; 1671da177e4SLinus Torvalds 1681da177e4SLinus Torvalds /* List of snapshots for this origin */ 1691da177e4SLinus Torvalds struct list_head snapshots; 1701da177e4SLinus Torvalds }; 1711da177e4SLinus Torvalds 1721da177e4SLinus Torvalds /* 1731da177e4SLinus Torvalds * Size of the hash table for origin volumes. If we make this 1741da177e4SLinus Torvalds * the size of the minors list then it should be nearly perfect 1751da177e4SLinus Torvalds */ 1761da177e4SLinus Torvalds #define ORIGIN_HASH_SIZE 256 1771da177e4SLinus Torvalds #define ORIGIN_MASK 0xFF 1781da177e4SLinus Torvalds static struct list_head *_origins; 1791da177e4SLinus Torvalds static struct rw_semaphore _origins_lock; 1801da177e4SLinus Torvalds 1811da177e4SLinus Torvalds static int init_origin_hash(void) 1821da177e4SLinus Torvalds { 1831da177e4SLinus Torvalds int i; 1841da177e4SLinus Torvalds 1851da177e4SLinus Torvalds _origins = kmalloc(ORIGIN_HASH_SIZE * sizeof(struct list_head), 1861da177e4SLinus Torvalds GFP_KERNEL); 1871da177e4SLinus Torvalds if (!_origins) { 18872d94861SAlasdair G Kergon DMERR("unable to allocate memory"); 1891da177e4SLinus Torvalds return -ENOMEM; 1901da177e4SLinus Torvalds } 1911da177e4SLinus Torvalds 1921da177e4SLinus Torvalds for (i = 0; i < ORIGIN_HASH_SIZE; i++) 1931da177e4SLinus Torvalds INIT_LIST_HEAD(_origins + i); 1941da177e4SLinus Torvalds init_rwsem(&_origins_lock); 1951da177e4SLinus Torvalds 1961da177e4SLinus Torvalds return 0; 1971da177e4SLinus Torvalds } 1981da177e4SLinus Torvalds 1991da177e4SLinus Torvalds static void exit_origin_hash(void) 2001da177e4SLinus Torvalds { 2011da177e4SLinus Torvalds kfree(_origins); 2021da177e4SLinus Torvalds } 2031da177e4SLinus Torvalds 204028867acSAlasdair G Kergon static unsigned origin_hash(struct block_device *bdev) 2051da177e4SLinus Torvalds { 2061da177e4SLinus Torvalds return bdev->bd_dev & ORIGIN_MASK; 2071da177e4SLinus Torvalds } 2081da177e4SLinus Torvalds 2091da177e4SLinus Torvalds static struct origin *__lookup_origin(struct block_device *origin) 2101da177e4SLinus Torvalds { 2111da177e4SLinus Torvalds struct list_head *ol; 2121da177e4SLinus Torvalds struct origin *o; 2131da177e4SLinus Torvalds 2141da177e4SLinus Torvalds ol = &_origins[origin_hash(origin)]; 2151da177e4SLinus Torvalds list_for_each_entry (o, ol, hash_list) 2161da177e4SLinus Torvalds if (bdev_equal(o->bdev, origin)) 2171da177e4SLinus Torvalds return o; 2181da177e4SLinus Torvalds 2191da177e4SLinus Torvalds return NULL; 2201da177e4SLinus Torvalds } 2211da177e4SLinus Torvalds 2221da177e4SLinus Torvalds static void __insert_origin(struct origin *o) 2231da177e4SLinus Torvalds { 2241da177e4SLinus Torvalds struct list_head *sl = &_origins[origin_hash(o->bdev)]; 2251da177e4SLinus Torvalds list_add_tail(&o->hash_list, sl); 2261da177e4SLinus Torvalds } 2271da177e4SLinus Torvalds 2281da177e4SLinus Torvalds /* 2291da177e4SLinus Torvalds * Make a note of the snapshot and its origin so we can look it 2301da177e4SLinus Torvalds * up when the origin has a write on it. 2311da177e4SLinus Torvalds */ 2321da177e4SLinus Torvalds static int register_snapshot(struct dm_snapshot *snap) 2331da177e4SLinus Torvalds { 23460c856c8SMikulas Patocka struct origin *o, *new_o; 2351da177e4SLinus Torvalds struct block_device *bdev = snap->origin->bdev; 2361da177e4SLinus Torvalds 23760c856c8SMikulas Patocka new_o = kmalloc(sizeof(*new_o), GFP_KERNEL); 23860c856c8SMikulas Patocka if (!new_o) 23960c856c8SMikulas Patocka return -ENOMEM; 24060c856c8SMikulas Patocka 2411da177e4SLinus Torvalds down_write(&_origins_lock); 2421da177e4SLinus Torvalds o = __lookup_origin(bdev); 2431da177e4SLinus Torvalds 24460c856c8SMikulas Patocka if (o) 24560c856c8SMikulas Patocka kfree(new_o); 24660c856c8SMikulas Patocka else { 2471da177e4SLinus Torvalds /* New origin */ 24860c856c8SMikulas Patocka o = new_o; 2491da177e4SLinus Torvalds 2501da177e4SLinus Torvalds /* Initialise the struct */ 2511da177e4SLinus Torvalds INIT_LIST_HEAD(&o->snapshots); 2521da177e4SLinus Torvalds o->bdev = bdev; 2531da177e4SLinus Torvalds 2541da177e4SLinus Torvalds __insert_origin(o); 2551da177e4SLinus Torvalds } 2561da177e4SLinus Torvalds 2571da177e4SLinus Torvalds list_add_tail(&snap->list, &o->snapshots); 2581da177e4SLinus Torvalds 2591da177e4SLinus Torvalds up_write(&_origins_lock); 2601da177e4SLinus Torvalds return 0; 2611da177e4SLinus Torvalds } 2621da177e4SLinus Torvalds 2631da177e4SLinus Torvalds static void unregister_snapshot(struct dm_snapshot *s) 2641da177e4SLinus Torvalds { 2651da177e4SLinus Torvalds struct origin *o; 2661da177e4SLinus Torvalds 2671da177e4SLinus Torvalds down_write(&_origins_lock); 2681da177e4SLinus Torvalds o = __lookup_origin(s->origin->bdev); 2691da177e4SLinus Torvalds 2701da177e4SLinus Torvalds list_del(&s->list); 2711da177e4SLinus Torvalds if (list_empty(&o->snapshots)) { 2721da177e4SLinus Torvalds list_del(&o->hash_list); 2731da177e4SLinus Torvalds kfree(o); 2741da177e4SLinus Torvalds } 2751da177e4SLinus Torvalds 2761da177e4SLinus Torvalds up_write(&_origins_lock); 2771da177e4SLinus Torvalds } 2781da177e4SLinus Torvalds 2791da177e4SLinus Torvalds /* 2801da177e4SLinus Torvalds * Implementation of the exception hash tables. 281d74f81f8SMilan Broz * The lowest hash_shift bits of the chunk number are ignored, allowing 282d74f81f8SMilan Broz * some consecutive chunks to be grouped together. 2831da177e4SLinus Torvalds */ 284d74f81f8SMilan Broz static int init_exception_table(struct exception_table *et, uint32_t size, 285d74f81f8SMilan Broz unsigned hash_shift) 2861da177e4SLinus Torvalds { 2871da177e4SLinus Torvalds unsigned int i; 2881da177e4SLinus Torvalds 289d74f81f8SMilan Broz et->hash_shift = hash_shift; 2901da177e4SLinus Torvalds et->hash_mask = size - 1; 2911da177e4SLinus Torvalds et->table = dm_vcalloc(size, sizeof(struct list_head)); 2921da177e4SLinus Torvalds if (!et->table) 2931da177e4SLinus Torvalds return -ENOMEM; 2941da177e4SLinus Torvalds 2951da177e4SLinus Torvalds for (i = 0; i < size; i++) 2961da177e4SLinus Torvalds INIT_LIST_HEAD(et->table + i); 2971da177e4SLinus Torvalds 2981da177e4SLinus Torvalds return 0; 2991da177e4SLinus Torvalds } 3001da177e4SLinus Torvalds 301e18b890bSChristoph Lameter static void exit_exception_table(struct exception_table *et, struct kmem_cache *mem) 3021da177e4SLinus Torvalds { 3031da177e4SLinus Torvalds struct list_head *slot; 304028867acSAlasdair G Kergon struct dm_snap_exception *ex, *next; 3051da177e4SLinus Torvalds int i, size; 3061da177e4SLinus Torvalds 3071da177e4SLinus Torvalds size = et->hash_mask + 1; 3081da177e4SLinus Torvalds for (i = 0; i < size; i++) { 3091da177e4SLinus Torvalds slot = et->table + i; 3101da177e4SLinus Torvalds 3111da177e4SLinus Torvalds list_for_each_entry_safe (ex, next, slot, hash_list) 3121da177e4SLinus Torvalds kmem_cache_free(mem, ex); 3131da177e4SLinus Torvalds } 3141da177e4SLinus Torvalds 3151da177e4SLinus Torvalds vfree(et->table); 3161da177e4SLinus Torvalds } 3171da177e4SLinus Torvalds 318028867acSAlasdair G Kergon static uint32_t exception_hash(struct exception_table *et, chunk_t chunk) 3191da177e4SLinus Torvalds { 320d74f81f8SMilan Broz return (chunk >> et->hash_shift) & et->hash_mask; 3211da177e4SLinus Torvalds } 3221da177e4SLinus Torvalds 323028867acSAlasdair G Kergon static void insert_exception(struct exception_table *eh, 324028867acSAlasdair G Kergon struct dm_snap_exception *e) 3251da177e4SLinus Torvalds { 3261da177e4SLinus Torvalds struct list_head *l = &eh->table[exception_hash(eh, e->old_chunk)]; 3271da177e4SLinus Torvalds list_add(&e->hash_list, l); 3281da177e4SLinus Torvalds } 3291da177e4SLinus Torvalds 330028867acSAlasdair G Kergon static void remove_exception(struct dm_snap_exception *e) 3311da177e4SLinus Torvalds { 3321da177e4SLinus Torvalds list_del(&e->hash_list); 3331da177e4SLinus Torvalds } 3341da177e4SLinus Torvalds 3351da177e4SLinus Torvalds /* 3361da177e4SLinus Torvalds * Return the exception data for a sector, or NULL if not 3371da177e4SLinus Torvalds * remapped. 3381da177e4SLinus Torvalds */ 339028867acSAlasdair G Kergon static struct dm_snap_exception *lookup_exception(struct exception_table *et, 3401da177e4SLinus Torvalds chunk_t chunk) 3411da177e4SLinus Torvalds { 3421da177e4SLinus Torvalds struct list_head *slot; 343028867acSAlasdair G Kergon struct dm_snap_exception *e; 3441da177e4SLinus Torvalds 3451da177e4SLinus Torvalds slot = &et->table[exception_hash(et, chunk)]; 3461da177e4SLinus Torvalds list_for_each_entry (e, slot, hash_list) 347d74f81f8SMilan Broz if (chunk >= e->old_chunk && 348d74f81f8SMilan Broz chunk <= e->old_chunk + dm_consecutive_chunk_count(e)) 3491da177e4SLinus Torvalds return e; 3501da177e4SLinus Torvalds 3511da177e4SLinus Torvalds return NULL; 3521da177e4SLinus Torvalds } 3531da177e4SLinus Torvalds 354028867acSAlasdair G Kergon static struct dm_snap_exception *alloc_exception(void) 3551da177e4SLinus Torvalds { 356028867acSAlasdair G Kergon struct dm_snap_exception *e; 3571da177e4SLinus Torvalds 3581da177e4SLinus Torvalds e = kmem_cache_alloc(exception_cache, GFP_NOIO); 3591da177e4SLinus Torvalds if (!e) 3601da177e4SLinus Torvalds e = kmem_cache_alloc(exception_cache, GFP_ATOMIC); 3611da177e4SLinus Torvalds 3621da177e4SLinus Torvalds return e; 3631da177e4SLinus Torvalds } 3641da177e4SLinus Torvalds 365028867acSAlasdair G Kergon static void free_exception(struct dm_snap_exception *e) 3661da177e4SLinus Torvalds { 3671da177e4SLinus Torvalds kmem_cache_free(exception_cache, e); 3681da177e4SLinus Torvalds } 3691da177e4SLinus Torvalds 37092e86812SMikulas Patocka static struct dm_snap_pending_exception *alloc_pending_exception(struct dm_snapshot *s) 3711da177e4SLinus Torvalds { 37292e86812SMikulas Patocka struct dm_snap_pending_exception *pe = mempool_alloc(s->pending_pool, 37392e86812SMikulas Patocka GFP_NOIO); 37492e86812SMikulas Patocka 375879129d2SMikulas Patocka atomic_inc(&s->pending_exceptions_count); 37692e86812SMikulas Patocka pe->snap = s; 37792e86812SMikulas Patocka 37892e86812SMikulas Patocka return pe; 3791da177e4SLinus Torvalds } 3801da177e4SLinus Torvalds 381028867acSAlasdair G Kergon static void free_pending_exception(struct dm_snap_pending_exception *pe) 3821da177e4SLinus Torvalds { 383879129d2SMikulas Patocka struct dm_snapshot *s = pe->snap; 384879129d2SMikulas Patocka 385879129d2SMikulas Patocka mempool_free(pe, s->pending_pool); 386879129d2SMikulas Patocka smp_mb__before_atomic_dec(); 387879129d2SMikulas Patocka atomic_dec(&s->pending_exceptions_count); 3881da177e4SLinus Torvalds } 3891da177e4SLinus Torvalds 390d74f81f8SMilan Broz static void insert_completed_exception(struct dm_snapshot *s, 391d74f81f8SMilan Broz struct dm_snap_exception *new_e) 392d74f81f8SMilan Broz { 393d74f81f8SMilan Broz struct exception_table *eh = &s->complete; 394d74f81f8SMilan Broz struct list_head *l; 395d74f81f8SMilan Broz struct dm_snap_exception *e = NULL; 396d74f81f8SMilan Broz 397d74f81f8SMilan Broz l = &eh->table[exception_hash(eh, new_e->old_chunk)]; 398d74f81f8SMilan Broz 399d74f81f8SMilan Broz /* Add immediately if this table doesn't support consecutive chunks */ 400d74f81f8SMilan Broz if (!eh->hash_shift) 401d74f81f8SMilan Broz goto out; 402d74f81f8SMilan Broz 403d74f81f8SMilan Broz /* List is ordered by old_chunk */ 404d74f81f8SMilan Broz list_for_each_entry_reverse(e, l, hash_list) { 405d74f81f8SMilan Broz /* Insert after an existing chunk? */ 406d74f81f8SMilan Broz if (new_e->old_chunk == (e->old_chunk + 407d74f81f8SMilan Broz dm_consecutive_chunk_count(e) + 1) && 408d74f81f8SMilan Broz new_e->new_chunk == (dm_chunk_number(e->new_chunk) + 409d74f81f8SMilan Broz dm_consecutive_chunk_count(e) + 1)) { 410d74f81f8SMilan Broz dm_consecutive_chunk_count_inc(e); 411d74f81f8SMilan Broz free_exception(new_e); 412d74f81f8SMilan Broz return; 413d74f81f8SMilan Broz } 414d74f81f8SMilan Broz 415d74f81f8SMilan Broz /* Insert before an existing chunk? */ 416d74f81f8SMilan Broz if (new_e->old_chunk == (e->old_chunk - 1) && 417d74f81f8SMilan Broz new_e->new_chunk == (dm_chunk_number(e->new_chunk) - 1)) { 418d74f81f8SMilan Broz dm_consecutive_chunk_count_inc(e); 419d74f81f8SMilan Broz e->old_chunk--; 420d74f81f8SMilan Broz e->new_chunk--; 421d74f81f8SMilan Broz free_exception(new_e); 422d74f81f8SMilan Broz return; 423d74f81f8SMilan Broz } 424d74f81f8SMilan Broz 425d74f81f8SMilan Broz if (new_e->old_chunk > e->old_chunk) 426d74f81f8SMilan Broz break; 427d74f81f8SMilan Broz } 428d74f81f8SMilan Broz 429d74f81f8SMilan Broz out: 430d74f81f8SMilan Broz list_add(&new_e->hash_list, e ? &e->hash_list : l); 431d74f81f8SMilan Broz } 432d74f81f8SMilan Broz 433a159c1acSJonathan Brassow /* 434a159c1acSJonathan Brassow * Callback used by the exception stores to load exceptions when 435a159c1acSJonathan Brassow * initialising. 436a159c1acSJonathan Brassow */ 437a159c1acSJonathan Brassow static int dm_add_exception(void *context, chunk_t old, chunk_t new) 4381da177e4SLinus Torvalds { 439a159c1acSJonathan Brassow struct dm_snapshot *s = context; 440028867acSAlasdair G Kergon struct dm_snap_exception *e; 4411da177e4SLinus Torvalds 4421da177e4SLinus Torvalds e = alloc_exception(); 4431da177e4SLinus Torvalds if (!e) 4441da177e4SLinus Torvalds return -ENOMEM; 4451da177e4SLinus Torvalds 4461da177e4SLinus Torvalds e->old_chunk = old; 447d74f81f8SMilan Broz 448d74f81f8SMilan Broz /* Consecutive_count is implicitly initialised to zero */ 4491da177e4SLinus Torvalds e->new_chunk = new; 450d74f81f8SMilan Broz 451d74f81f8SMilan Broz insert_completed_exception(s, e); 452d74f81f8SMilan Broz 4531da177e4SLinus Torvalds return 0; 4541da177e4SLinus Torvalds } 4551da177e4SLinus Torvalds 4561da177e4SLinus Torvalds /* 4571da177e4SLinus Torvalds * Hard coded magic. 4581da177e4SLinus Torvalds */ 4591da177e4SLinus Torvalds static int calc_max_buckets(void) 4601da177e4SLinus Torvalds { 4611da177e4SLinus Torvalds /* use a fixed size of 2MB */ 4621da177e4SLinus Torvalds unsigned long mem = 2 * 1024 * 1024; 4631da177e4SLinus Torvalds mem /= sizeof(struct list_head); 4641da177e4SLinus Torvalds 4651da177e4SLinus Torvalds return mem; 4661da177e4SLinus Torvalds } 4671da177e4SLinus Torvalds 4681da177e4SLinus Torvalds /* 4691da177e4SLinus Torvalds * Allocate room for a suitable hash table. 4701da177e4SLinus Torvalds */ 4711da177e4SLinus Torvalds static int init_hash_tables(struct dm_snapshot *s) 4721da177e4SLinus Torvalds { 4731da177e4SLinus Torvalds sector_t hash_size, cow_dev_size, origin_dev_size, max_buckets; 4741da177e4SLinus Torvalds 4751da177e4SLinus Torvalds /* 4761da177e4SLinus Torvalds * Calculate based on the size of the original volume or 4771da177e4SLinus Torvalds * the COW volume... 4781da177e4SLinus Torvalds */ 4791da177e4SLinus Torvalds cow_dev_size = get_dev_size(s->cow->bdev); 4801da177e4SLinus Torvalds origin_dev_size = get_dev_size(s->origin->bdev); 4811da177e4SLinus Torvalds max_buckets = calc_max_buckets(); 4821da177e4SLinus Torvalds 4831da177e4SLinus Torvalds hash_size = min(origin_dev_size, cow_dev_size) >> s->chunk_shift; 4841da177e4SLinus Torvalds hash_size = min(hash_size, max_buckets); 4851da177e4SLinus Torvalds 4868defd830SRobert P. J. Day hash_size = rounddown_pow_of_two(hash_size); 487d74f81f8SMilan Broz if (init_exception_table(&s->complete, hash_size, 488d74f81f8SMilan Broz DM_CHUNK_CONSECUTIVE_BITS)) 4891da177e4SLinus Torvalds return -ENOMEM; 4901da177e4SLinus Torvalds 4911da177e4SLinus Torvalds /* 4921da177e4SLinus Torvalds * Allocate hash table for in-flight exceptions 4931da177e4SLinus Torvalds * Make this smaller than the real hash table 4941da177e4SLinus Torvalds */ 4951da177e4SLinus Torvalds hash_size >>= 3; 4961da177e4SLinus Torvalds if (hash_size < 64) 4971da177e4SLinus Torvalds hash_size = 64; 4981da177e4SLinus Torvalds 499d74f81f8SMilan Broz if (init_exception_table(&s->pending, hash_size, 0)) { 5001da177e4SLinus Torvalds exit_exception_table(&s->complete, exception_cache); 5011da177e4SLinus Torvalds return -ENOMEM; 5021da177e4SLinus Torvalds } 5031da177e4SLinus Torvalds 5041da177e4SLinus Torvalds return 0; 5051da177e4SLinus Torvalds } 5061da177e4SLinus Torvalds 5071da177e4SLinus Torvalds /* 5081da177e4SLinus Torvalds * Round a number up to the nearest 'size' boundary. size must 5091da177e4SLinus Torvalds * be a power of 2. 5101da177e4SLinus Torvalds */ 511028867acSAlasdair G Kergon static ulong round_up(ulong n, ulong size) 5121da177e4SLinus Torvalds { 5131da177e4SLinus Torvalds size--; 5141da177e4SLinus Torvalds return (n + size) & ~size; 5151da177e4SLinus Torvalds } 5161da177e4SLinus Torvalds 5174c7e3bf4SMark McLoughlin static int set_chunk_size(struct dm_snapshot *s, const char *chunk_size_arg, 5184c7e3bf4SMark McLoughlin char **error) 5194c7e3bf4SMark McLoughlin { 5204c7e3bf4SMark McLoughlin unsigned long chunk_size; 5214c7e3bf4SMark McLoughlin char *value; 5224c7e3bf4SMark McLoughlin 5234c7e3bf4SMark McLoughlin chunk_size = simple_strtoul(chunk_size_arg, &value, 10); 5244c7e3bf4SMark McLoughlin if (*chunk_size_arg == '\0' || *value != '\0') { 5254c7e3bf4SMark McLoughlin *error = "Invalid chunk size"; 5264c7e3bf4SMark McLoughlin return -EINVAL; 5274c7e3bf4SMark McLoughlin } 5284c7e3bf4SMark McLoughlin 5294c7e3bf4SMark McLoughlin if (!chunk_size) { 5304c7e3bf4SMark McLoughlin s->chunk_size = s->chunk_mask = s->chunk_shift = 0; 5314c7e3bf4SMark McLoughlin return 0; 5324c7e3bf4SMark McLoughlin } 5334c7e3bf4SMark McLoughlin 5344c7e3bf4SMark McLoughlin /* 5354c7e3bf4SMark McLoughlin * Chunk size must be multiple of page size. Silently 5364c7e3bf4SMark McLoughlin * round up if it's not. 5374c7e3bf4SMark McLoughlin */ 5384c7e3bf4SMark McLoughlin chunk_size = round_up(chunk_size, PAGE_SIZE >> 9); 5394c7e3bf4SMark McLoughlin 5404c7e3bf4SMark McLoughlin /* Check chunk_size is a power of 2 */ 5416f3c3f0aSvignesh babu if (!is_power_of_2(chunk_size)) { 5424c7e3bf4SMark McLoughlin *error = "Chunk size is not a power of 2"; 5434c7e3bf4SMark McLoughlin return -EINVAL; 5444c7e3bf4SMark McLoughlin } 5454c7e3bf4SMark McLoughlin 5464c7e3bf4SMark McLoughlin /* Validate the chunk size against the device block size */ 5474c7e3bf4SMark McLoughlin if (chunk_size % (bdev_hardsect_size(s->cow->bdev) >> 9)) { 5484c7e3bf4SMark McLoughlin *error = "Chunk size is not a multiple of device blocksize"; 5494c7e3bf4SMark McLoughlin return -EINVAL; 5504c7e3bf4SMark McLoughlin } 5514c7e3bf4SMark McLoughlin 5524c7e3bf4SMark McLoughlin s->chunk_size = chunk_size; 5534c7e3bf4SMark McLoughlin s->chunk_mask = chunk_size - 1; 5544c7e3bf4SMark McLoughlin s->chunk_shift = ffs(chunk_size) - 1; 5554c7e3bf4SMark McLoughlin 5564c7e3bf4SMark McLoughlin return 0; 5574c7e3bf4SMark McLoughlin } 5584c7e3bf4SMark McLoughlin 5591da177e4SLinus Torvalds /* 5601da177e4SLinus Torvalds * Construct a snapshot mapping: <origin_dev> <COW-dev> <p/n> <chunk-size> 5611da177e4SLinus Torvalds */ 5621da177e4SLinus Torvalds static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv) 5631da177e4SLinus Torvalds { 5641da177e4SLinus Torvalds struct dm_snapshot *s; 565cd45daffSMikulas Patocka int i; 5661da177e4SLinus Torvalds int r = -EINVAL; 5671da177e4SLinus Torvalds char persistent; 5681da177e4SLinus Torvalds char *origin_path; 5691da177e4SLinus Torvalds char *cow_path; 5701da177e4SLinus Torvalds 5714c7e3bf4SMark McLoughlin if (argc != 4) { 57272d94861SAlasdair G Kergon ti->error = "requires exactly 4 arguments"; 5731da177e4SLinus Torvalds r = -EINVAL; 5741da177e4SLinus Torvalds goto bad1; 5751da177e4SLinus Torvalds } 5761da177e4SLinus Torvalds 5771da177e4SLinus Torvalds origin_path = argv[0]; 5781da177e4SLinus Torvalds cow_path = argv[1]; 5791da177e4SLinus Torvalds persistent = toupper(*argv[2]); 5801da177e4SLinus Torvalds 5811da177e4SLinus Torvalds if (persistent != 'P' && persistent != 'N') { 5821da177e4SLinus Torvalds ti->error = "Persistent flag is not P or N"; 5831da177e4SLinus Torvalds r = -EINVAL; 5841da177e4SLinus Torvalds goto bad1; 5851da177e4SLinus Torvalds } 5861da177e4SLinus Torvalds 5871da177e4SLinus Torvalds s = kmalloc(sizeof(*s), GFP_KERNEL); 5881da177e4SLinus Torvalds if (s == NULL) { 5891da177e4SLinus Torvalds ti->error = "Cannot allocate snapshot context private " 5901da177e4SLinus Torvalds "structure"; 5911da177e4SLinus Torvalds r = -ENOMEM; 5921da177e4SLinus Torvalds goto bad1; 5931da177e4SLinus Torvalds } 5941da177e4SLinus Torvalds 5951da177e4SLinus Torvalds r = dm_get_device(ti, origin_path, 0, ti->len, FMODE_READ, &s->origin); 5961da177e4SLinus Torvalds if (r) { 5971da177e4SLinus Torvalds ti->error = "Cannot get origin device"; 5981da177e4SLinus Torvalds goto bad2; 5991da177e4SLinus Torvalds } 6001da177e4SLinus Torvalds 6011da177e4SLinus Torvalds r = dm_get_device(ti, cow_path, 0, 0, 6021da177e4SLinus Torvalds FMODE_READ | FMODE_WRITE, &s->cow); 6031da177e4SLinus Torvalds if (r) { 6041da177e4SLinus Torvalds dm_put_device(ti, s->origin); 6051da177e4SLinus Torvalds ti->error = "Cannot get COW device"; 6061da177e4SLinus Torvalds goto bad2; 6071da177e4SLinus Torvalds } 6081da177e4SLinus Torvalds 6094c7e3bf4SMark McLoughlin r = set_chunk_size(s, argv[3], &ti->error); 6104c7e3bf4SMark McLoughlin if (r) 6111da177e4SLinus Torvalds goto bad3; 6121da177e4SLinus Torvalds 6131da177e4SLinus Torvalds s->type = persistent; 6141da177e4SLinus Torvalds 6151da177e4SLinus Torvalds s->valid = 1; 616aa14edebSAlasdair G Kergon s->active = 0; 617879129d2SMikulas Patocka atomic_set(&s->pending_exceptions_count, 0); 6181da177e4SLinus Torvalds init_rwsem(&s->lock); 619ca3a931fSAlasdair G Kergon spin_lock_init(&s->pe_lock); 62072727badSMikulas Patocka s->ti = ti; 6211da177e4SLinus Torvalds 6221da177e4SLinus Torvalds /* Allocate hash table for COW data */ 6231da177e4SLinus Torvalds if (init_hash_tables(s)) { 6241da177e4SLinus Torvalds ti->error = "Unable to allocate hash table space"; 6251da177e4SLinus Torvalds r = -ENOMEM; 6261da177e4SLinus Torvalds goto bad3; 6271da177e4SLinus Torvalds } 6281da177e4SLinus Torvalds 6291da177e4SLinus Torvalds s->store.snap = s; 6301da177e4SLinus Torvalds 6311da177e4SLinus Torvalds if (persistent == 'P') 6324c7e3bf4SMark McLoughlin r = dm_create_persistent(&s->store); 6331da177e4SLinus Torvalds else 6344c7e3bf4SMark McLoughlin r = dm_create_transient(&s->store); 6351da177e4SLinus Torvalds 6361da177e4SLinus Torvalds if (r) { 6371da177e4SLinus Torvalds ti->error = "Couldn't create exception store"; 6381da177e4SLinus Torvalds r = -EINVAL; 6391da177e4SLinus Torvalds goto bad4; 6401da177e4SLinus Torvalds } 6411da177e4SLinus Torvalds 642eb69aca5SHeinz Mauelshagen r = dm_kcopyd_client_create(SNAPSHOT_PAGES, &s->kcopyd_client); 6431da177e4SLinus Torvalds if (r) { 6441da177e4SLinus Torvalds ti->error = "Could not create kcopyd client"; 6451da177e4SLinus Torvalds goto bad5; 6461da177e4SLinus Torvalds } 6471da177e4SLinus Torvalds 64892e86812SMikulas Patocka s->pending_pool = mempool_create_slab_pool(MIN_IOS, pending_cache); 64992e86812SMikulas Patocka if (!s->pending_pool) { 65092e86812SMikulas Patocka ti->error = "Could not allocate mempool for pending exceptions"; 65192e86812SMikulas Patocka goto bad6; 65292e86812SMikulas Patocka } 65392e86812SMikulas Patocka 654cd45daffSMikulas Patocka s->tracked_chunk_pool = mempool_create_slab_pool(MIN_IOS, 655cd45daffSMikulas Patocka tracked_chunk_cache); 656cd45daffSMikulas Patocka if (!s->tracked_chunk_pool) { 657cd45daffSMikulas Patocka ti->error = "Could not allocate tracked_chunk mempool for " 658cd45daffSMikulas Patocka "tracking reads"; 65992e86812SMikulas Patocka goto bad_tracked_chunk_pool; 660cd45daffSMikulas Patocka } 661cd45daffSMikulas Patocka 662cd45daffSMikulas Patocka for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++) 663cd45daffSMikulas Patocka INIT_HLIST_HEAD(&s->tracked_chunk_hash[i]); 664cd45daffSMikulas Patocka 665cd45daffSMikulas Patocka spin_lock_init(&s->tracked_chunk_lock); 666cd45daffSMikulas Patocka 667aa14edebSAlasdair G Kergon /* Metadata must only be loaded into one table at once */ 668a159c1acSJonathan Brassow r = s->store.read_metadata(&s->store, dm_add_exception, (void *)s); 6690764147bSMilan Broz if (r < 0) { 670f9cea4f7SMark McLoughlin ti->error = "Failed to read snapshot metadata"; 671cd45daffSMikulas Patocka goto bad_load_and_register; 6720764147bSMilan Broz } else if (r > 0) { 6730764147bSMilan Broz s->valid = 0; 6740764147bSMilan Broz DMWARN("Snapshot is marked invalid."); 675f9cea4f7SMark McLoughlin } 676aa14edebSAlasdair G Kergon 677ca3a931fSAlasdair G Kergon bio_list_init(&s->queued_bios); 678c4028958SDavid Howells INIT_WORK(&s->queued_bios_work, flush_queued_bios); 679ca3a931fSAlasdair G Kergon 6801da177e4SLinus Torvalds /* Add snapshot to the list of snapshots for this origin */ 681aa14edebSAlasdair G Kergon /* Exceptions aren't triggered till snapshot_resume() is called */ 6821da177e4SLinus Torvalds if (register_snapshot(s)) { 6831da177e4SLinus Torvalds r = -EINVAL; 6841da177e4SLinus Torvalds ti->error = "Cannot register snapshot origin"; 685cd45daffSMikulas Patocka goto bad_load_and_register; 6861da177e4SLinus Torvalds } 6871da177e4SLinus Torvalds 6881da177e4SLinus Torvalds ti->private = s; 689c51c2752SAlasdair G Kergon ti->split_io = s->chunk_size; 6901da177e4SLinus Torvalds 6911da177e4SLinus Torvalds return 0; 6921da177e4SLinus Torvalds 693cd45daffSMikulas Patocka bad_load_and_register: 694cd45daffSMikulas Patocka mempool_destroy(s->tracked_chunk_pool); 695cd45daffSMikulas Patocka 69692e86812SMikulas Patocka bad_tracked_chunk_pool: 69792e86812SMikulas Patocka mempool_destroy(s->pending_pool); 69892e86812SMikulas Patocka 6991da177e4SLinus Torvalds bad6: 700eb69aca5SHeinz Mauelshagen dm_kcopyd_client_destroy(s->kcopyd_client); 7011da177e4SLinus Torvalds 7021da177e4SLinus Torvalds bad5: 7031da177e4SLinus Torvalds s->store.destroy(&s->store); 7041da177e4SLinus Torvalds 7051da177e4SLinus Torvalds bad4: 7061da177e4SLinus Torvalds exit_exception_table(&s->pending, pending_cache); 7071da177e4SLinus Torvalds exit_exception_table(&s->complete, exception_cache); 7081da177e4SLinus Torvalds 7091da177e4SLinus Torvalds bad3: 7101da177e4SLinus Torvalds dm_put_device(ti, s->cow); 7111da177e4SLinus Torvalds dm_put_device(ti, s->origin); 7121da177e4SLinus Torvalds 7131da177e4SLinus Torvalds bad2: 7141da177e4SLinus Torvalds kfree(s); 7151da177e4SLinus Torvalds 7161da177e4SLinus Torvalds bad1: 7171da177e4SLinus Torvalds return r; 7181da177e4SLinus Torvalds } 7191da177e4SLinus Torvalds 72031c93a0cSMilan Broz static void __free_exceptions(struct dm_snapshot *s) 72131c93a0cSMilan Broz { 722eb69aca5SHeinz Mauelshagen dm_kcopyd_client_destroy(s->kcopyd_client); 72331c93a0cSMilan Broz s->kcopyd_client = NULL; 72431c93a0cSMilan Broz 72531c93a0cSMilan Broz exit_exception_table(&s->pending, pending_cache); 72631c93a0cSMilan Broz exit_exception_table(&s->complete, exception_cache); 72731c93a0cSMilan Broz 72831c93a0cSMilan Broz s->store.destroy(&s->store); 72931c93a0cSMilan Broz } 73031c93a0cSMilan Broz 7311da177e4SLinus Torvalds static void snapshot_dtr(struct dm_target *ti) 7321da177e4SLinus Torvalds { 733cd45daffSMikulas Patocka #ifdef CONFIG_DM_DEBUG 734cd45daffSMikulas Patocka int i; 735cd45daffSMikulas Patocka #endif 736028867acSAlasdair G Kergon struct dm_snapshot *s = ti->private; 7371da177e4SLinus Torvalds 738ca3a931fSAlasdair G Kergon flush_workqueue(ksnapd); 739ca3a931fSAlasdair G Kergon 740138728dcSAlasdair G Kergon /* Prevent further origin writes from using this snapshot. */ 741138728dcSAlasdair G Kergon /* After this returns there can be no new kcopyd jobs. */ 7421da177e4SLinus Torvalds unregister_snapshot(s); 7431da177e4SLinus Torvalds 744879129d2SMikulas Patocka while (atomic_read(&s->pending_exceptions_count)) 74590fa1527SMikulas Patocka msleep(1); 746879129d2SMikulas Patocka /* 747879129d2SMikulas Patocka * Ensure instructions in mempool_destroy aren't reordered 748879129d2SMikulas Patocka * before atomic_read. 749879129d2SMikulas Patocka */ 750879129d2SMikulas Patocka smp_mb(); 751879129d2SMikulas Patocka 752cd45daffSMikulas Patocka #ifdef CONFIG_DM_DEBUG 753cd45daffSMikulas Patocka for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++) 754cd45daffSMikulas Patocka BUG_ON(!hlist_empty(&s->tracked_chunk_hash[i])); 755cd45daffSMikulas Patocka #endif 756cd45daffSMikulas Patocka 757cd45daffSMikulas Patocka mempool_destroy(s->tracked_chunk_pool); 758cd45daffSMikulas Patocka 75931c93a0cSMilan Broz __free_exceptions(s); 7601da177e4SLinus Torvalds 76192e86812SMikulas Patocka mempool_destroy(s->pending_pool); 76292e86812SMikulas Patocka 7631da177e4SLinus Torvalds dm_put_device(ti, s->origin); 7641da177e4SLinus Torvalds dm_put_device(ti, s->cow); 765138728dcSAlasdair G Kergon 7661da177e4SLinus Torvalds kfree(s); 7671da177e4SLinus Torvalds } 7681da177e4SLinus Torvalds 7691da177e4SLinus Torvalds /* 7701da177e4SLinus Torvalds * Flush a list of buffers. 7711da177e4SLinus Torvalds */ 7721da177e4SLinus Torvalds static void flush_bios(struct bio *bio) 7731da177e4SLinus Torvalds { 7741da177e4SLinus Torvalds struct bio *n; 7751da177e4SLinus Torvalds 7761da177e4SLinus Torvalds while (bio) { 7771da177e4SLinus Torvalds n = bio->bi_next; 7781da177e4SLinus Torvalds bio->bi_next = NULL; 7791da177e4SLinus Torvalds generic_make_request(bio); 7801da177e4SLinus Torvalds bio = n; 7811da177e4SLinus Torvalds } 7821da177e4SLinus Torvalds } 7831da177e4SLinus Torvalds 784c4028958SDavid Howells static void flush_queued_bios(struct work_struct *work) 785ca3a931fSAlasdair G Kergon { 786c4028958SDavid Howells struct dm_snapshot *s = 787c4028958SDavid Howells container_of(work, struct dm_snapshot, queued_bios_work); 788ca3a931fSAlasdair G Kergon struct bio *queued_bios; 789ca3a931fSAlasdair G Kergon unsigned long flags; 790ca3a931fSAlasdair G Kergon 791ca3a931fSAlasdair G Kergon spin_lock_irqsave(&s->pe_lock, flags); 792ca3a931fSAlasdair G Kergon queued_bios = bio_list_get(&s->queued_bios); 793ca3a931fSAlasdair G Kergon spin_unlock_irqrestore(&s->pe_lock, flags); 794ca3a931fSAlasdair G Kergon 795ca3a931fSAlasdair G Kergon flush_bios(queued_bios); 796ca3a931fSAlasdair G Kergon } 797ca3a931fSAlasdair G Kergon 7981da177e4SLinus Torvalds /* 7991da177e4SLinus Torvalds * Error a list of buffers. 8001da177e4SLinus Torvalds */ 8011da177e4SLinus Torvalds static void error_bios(struct bio *bio) 8021da177e4SLinus Torvalds { 8031da177e4SLinus Torvalds struct bio *n; 8041da177e4SLinus Torvalds 8051da177e4SLinus Torvalds while (bio) { 8061da177e4SLinus Torvalds n = bio->bi_next; 8071da177e4SLinus Torvalds bio->bi_next = NULL; 8086712ecf8SNeilBrown bio_io_error(bio); 8091da177e4SLinus Torvalds bio = n; 8101da177e4SLinus Torvalds } 8111da177e4SLinus Torvalds } 8121da177e4SLinus Torvalds 813695368acSAlasdair G Kergon static void __invalidate_snapshot(struct dm_snapshot *s, int err) 81476df1c65SAlasdair G Kergon { 81576df1c65SAlasdair G Kergon if (!s->valid) 81676df1c65SAlasdair G Kergon return; 81776df1c65SAlasdair G Kergon 81876df1c65SAlasdair G Kergon if (err == -EIO) 81976df1c65SAlasdair G Kergon DMERR("Invalidating snapshot: Error reading/writing."); 82076df1c65SAlasdair G Kergon else if (err == -ENOMEM) 82176df1c65SAlasdair G Kergon DMERR("Invalidating snapshot: Unable to allocate exception."); 82276df1c65SAlasdair G Kergon 82376df1c65SAlasdair G Kergon if (s->store.drop_snapshot) 82476df1c65SAlasdair G Kergon s->store.drop_snapshot(&s->store); 82576df1c65SAlasdair G Kergon 82676df1c65SAlasdair G Kergon s->valid = 0; 82776df1c65SAlasdair G Kergon 82872727badSMikulas Patocka dm_table_event(s->ti->table); 82976df1c65SAlasdair G Kergon } 83076df1c65SAlasdair G Kergon 831028867acSAlasdair G Kergon static void get_pending_exception(struct dm_snap_pending_exception *pe) 8324b832e8dSAlasdair G Kergon { 8334b832e8dSAlasdair G Kergon atomic_inc(&pe->ref_count); 8344b832e8dSAlasdair G Kergon } 8354b832e8dSAlasdair G Kergon 836028867acSAlasdair G Kergon static struct bio *put_pending_exception(struct dm_snap_pending_exception *pe) 8374b832e8dSAlasdair G Kergon { 838028867acSAlasdair G Kergon struct dm_snap_pending_exception *primary_pe; 8394b832e8dSAlasdair G Kergon struct bio *origin_bios = NULL; 8404b832e8dSAlasdair G Kergon 8414b832e8dSAlasdair G Kergon primary_pe = pe->primary_pe; 8424b832e8dSAlasdair G Kergon 8434b832e8dSAlasdair G Kergon /* 8444b832e8dSAlasdair G Kergon * If this pe is involved in a write to the origin and 8454b832e8dSAlasdair G Kergon * it is the last sibling to complete then release 8464b832e8dSAlasdair G Kergon * the bios for the original write to the origin. 8474b832e8dSAlasdair G Kergon */ 8484b832e8dSAlasdair G Kergon if (primary_pe && 8497c5f78b9SMikulas Patocka atomic_dec_and_test(&primary_pe->ref_count)) { 8504b832e8dSAlasdair G Kergon origin_bios = bio_list_get(&primary_pe->origin_bios); 8517c5f78b9SMikulas Patocka free_pending_exception(primary_pe); 8527c5f78b9SMikulas Patocka } 8534b832e8dSAlasdair G Kergon 8544b832e8dSAlasdair G Kergon /* 8554b832e8dSAlasdair G Kergon * Free the pe if it's not linked to an origin write or if 8564b832e8dSAlasdair G Kergon * it's not itself a primary pe. 8574b832e8dSAlasdair G Kergon */ 8584b832e8dSAlasdair G Kergon if (!primary_pe || primary_pe != pe) 8594b832e8dSAlasdair G Kergon free_pending_exception(pe); 8604b832e8dSAlasdair G Kergon 8614b832e8dSAlasdair G Kergon return origin_bios; 8624b832e8dSAlasdair G Kergon } 8634b832e8dSAlasdair G Kergon 864028867acSAlasdair G Kergon static void pending_complete(struct dm_snap_pending_exception *pe, int success) 8651da177e4SLinus Torvalds { 866028867acSAlasdair G Kergon struct dm_snap_exception *e; 8671da177e4SLinus Torvalds struct dm_snapshot *s = pe->snap; 8689d493fa8SAlasdair G Kergon struct bio *origin_bios = NULL; 8699d493fa8SAlasdair G Kergon struct bio *snapshot_bios = NULL; 8709d493fa8SAlasdair G Kergon int error = 0; 8711da177e4SLinus Torvalds 87276df1c65SAlasdair G Kergon if (!success) { 87376df1c65SAlasdair G Kergon /* Read/write error - snapshot is unusable */ 8741da177e4SLinus Torvalds down_write(&s->lock); 875695368acSAlasdair G Kergon __invalidate_snapshot(s, -EIO); 8769d493fa8SAlasdair G Kergon error = 1; 87776df1c65SAlasdair G Kergon goto out; 87876df1c65SAlasdair G Kergon } 87976df1c65SAlasdair G Kergon 88076df1c65SAlasdair G Kergon e = alloc_exception(); 88176df1c65SAlasdair G Kergon if (!e) { 88276df1c65SAlasdair G Kergon down_write(&s->lock); 883695368acSAlasdair G Kergon __invalidate_snapshot(s, -ENOMEM); 8849d493fa8SAlasdair G Kergon error = 1; 8851da177e4SLinus Torvalds goto out; 8861da177e4SLinus Torvalds } 8871da177e4SLinus Torvalds *e = pe->e; 8881da177e4SLinus Torvalds 8899d493fa8SAlasdair G Kergon down_write(&s->lock); 8909d493fa8SAlasdair G Kergon if (!s->valid) { 8919d493fa8SAlasdair G Kergon free_exception(e); 8929d493fa8SAlasdair G Kergon error = 1; 8939d493fa8SAlasdair G Kergon goto out; 8949d493fa8SAlasdair G Kergon } 8959d493fa8SAlasdair G Kergon 8961da177e4SLinus Torvalds /* 897a8d41b59SMikulas Patocka * Check for conflicting reads. This is extremely improbable, 89890fa1527SMikulas Patocka * so msleep(1) is sufficient and there is no need for a wait queue. 899a8d41b59SMikulas Patocka */ 900a8d41b59SMikulas Patocka while (__chunk_is_tracked(s, pe->e.old_chunk)) 90190fa1527SMikulas Patocka msleep(1); 902a8d41b59SMikulas Patocka 903a8d41b59SMikulas Patocka /* 9041da177e4SLinus Torvalds * Add a proper exception, and remove the 9051da177e4SLinus Torvalds * in-flight exception from the list. 9061da177e4SLinus Torvalds */ 907d74f81f8SMilan Broz insert_completed_exception(s, e); 9081da177e4SLinus Torvalds 9091da177e4SLinus Torvalds out: 910695368acSAlasdair G Kergon remove_exception(&pe->e); 9119d493fa8SAlasdair G Kergon snapshot_bios = bio_list_get(&pe->snapshot_bios); 9124b832e8dSAlasdair G Kergon origin_bios = put_pending_exception(pe); 913b4b610f6SAlasdair G Kergon 9149d493fa8SAlasdair G Kergon up_write(&s->lock); 9159d493fa8SAlasdair G Kergon 9169d493fa8SAlasdair G Kergon /* Submit any pending write bios */ 9179d493fa8SAlasdair G Kergon if (error) 9189d493fa8SAlasdair G Kergon error_bios(snapshot_bios); 9199d493fa8SAlasdair G Kergon else 9209d493fa8SAlasdair G Kergon flush_bios(snapshot_bios); 9219d493fa8SAlasdair G Kergon 9229d493fa8SAlasdair G Kergon flush_bios(origin_bios); 9231da177e4SLinus Torvalds } 9241da177e4SLinus Torvalds 9251da177e4SLinus Torvalds static void commit_callback(void *context, int success) 9261da177e4SLinus Torvalds { 927028867acSAlasdair G Kergon struct dm_snap_pending_exception *pe = context; 928028867acSAlasdair G Kergon 9291da177e4SLinus Torvalds pending_complete(pe, success); 9301da177e4SLinus Torvalds } 9311da177e4SLinus Torvalds 9321da177e4SLinus Torvalds /* 9331da177e4SLinus Torvalds * Called when the copy I/O has finished. kcopyd actually runs 9341da177e4SLinus Torvalds * this code so don't block. 9351da177e4SLinus Torvalds */ 9364cdc1d1fSAlasdair G Kergon static void copy_callback(int read_err, unsigned long write_err, void *context) 9371da177e4SLinus Torvalds { 938028867acSAlasdair G Kergon struct dm_snap_pending_exception *pe = context; 9391da177e4SLinus Torvalds struct dm_snapshot *s = pe->snap; 9401da177e4SLinus Torvalds 9411da177e4SLinus Torvalds if (read_err || write_err) 9421da177e4SLinus Torvalds pending_complete(pe, 0); 9431da177e4SLinus Torvalds 9441da177e4SLinus Torvalds else 9451da177e4SLinus Torvalds /* Update the metadata if we are persistent */ 9461da177e4SLinus Torvalds s->store.commit_exception(&s->store, &pe->e, commit_callback, 9471da177e4SLinus Torvalds pe); 9481da177e4SLinus Torvalds } 9491da177e4SLinus Torvalds 9501da177e4SLinus Torvalds /* 9511da177e4SLinus Torvalds * Dispatches the copy operation to kcopyd. 9521da177e4SLinus Torvalds */ 953028867acSAlasdair G Kergon static void start_copy(struct dm_snap_pending_exception *pe) 9541da177e4SLinus Torvalds { 9551da177e4SLinus Torvalds struct dm_snapshot *s = pe->snap; 95622a1ceb1SHeinz Mauelshagen struct dm_io_region src, dest; 9571da177e4SLinus Torvalds struct block_device *bdev = s->origin->bdev; 9581da177e4SLinus Torvalds sector_t dev_size; 9591da177e4SLinus Torvalds 9601da177e4SLinus Torvalds dev_size = get_dev_size(bdev); 9611da177e4SLinus Torvalds 9621da177e4SLinus Torvalds src.bdev = bdev; 9631da177e4SLinus Torvalds src.sector = chunk_to_sector(s, pe->e.old_chunk); 9641da177e4SLinus Torvalds src.count = min(s->chunk_size, dev_size - src.sector); 9651da177e4SLinus Torvalds 9661da177e4SLinus Torvalds dest.bdev = s->cow->bdev; 9671da177e4SLinus Torvalds dest.sector = chunk_to_sector(s, pe->e.new_chunk); 9681da177e4SLinus Torvalds dest.count = src.count; 9691da177e4SLinus Torvalds 9701da177e4SLinus Torvalds /* Hand over to kcopyd */ 971eb69aca5SHeinz Mauelshagen dm_kcopyd_copy(s->kcopyd_client, 9721da177e4SLinus Torvalds &src, 1, &dest, 0, copy_callback, pe); 9731da177e4SLinus Torvalds } 9741da177e4SLinus Torvalds 9752913808eSMikulas Patocka static struct dm_snap_pending_exception * 9762913808eSMikulas Patocka __lookup_pending_exception(struct dm_snapshot *s, chunk_t chunk) 9772913808eSMikulas Patocka { 9782913808eSMikulas Patocka struct dm_snap_exception *e = lookup_exception(&s->pending, chunk); 9792913808eSMikulas Patocka 9802913808eSMikulas Patocka if (!e) 9812913808eSMikulas Patocka return NULL; 9822913808eSMikulas Patocka 9832913808eSMikulas Patocka return container_of(e, struct dm_snap_pending_exception, e); 9842913808eSMikulas Patocka } 9852913808eSMikulas Patocka 9861da177e4SLinus Torvalds /* 9871da177e4SLinus Torvalds * Looks to see if this snapshot already has a pending exception 9881da177e4SLinus Torvalds * for this chunk, otherwise it allocates a new one and inserts 9891da177e4SLinus Torvalds * it into the pending table. 9901da177e4SLinus Torvalds * 9911da177e4SLinus Torvalds * NOTE: a write lock must be held on snap->lock before calling 9921da177e4SLinus Torvalds * this. 9931da177e4SLinus Torvalds */ 994028867acSAlasdair G Kergon static struct dm_snap_pending_exception * 9951da177e4SLinus Torvalds __find_pending_exception(struct dm_snapshot *s, struct bio *bio) 9961da177e4SLinus Torvalds { 9972913808eSMikulas Patocka struct dm_snap_pending_exception *pe, *pe2; 9981da177e4SLinus Torvalds chunk_t chunk = sector_to_chunk(s, bio->bi_sector); 9991da177e4SLinus Torvalds 10001da177e4SLinus Torvalds /* 10011da177e4SLinus Torvalds * Create a new pending exception, we don't want 10021da177e4SLinus Torvalds * to hold the lock while we do this. 10031da177e4SLinus Torvalds */ 10041da177e4SLinus Torvalds up_write(&s->lock); 100592e86812SMikulas Patocka pe = alloc_pending_exception(s); 10061da177e4SLinus Torvalds down_write(&s->lock); 10071da177e4SLinus Torvalds 100876df1c65SAlasdair G Kergon if (!s->valid) { 100976df1c65SAlasdair G Kergon free_pending_exception(pe); 101076df1c65SAlasdair G Kergon return NULL; 101176df1c65SAlasdair G Kergon } 101276df1c65SAlasdair G Kergon 10132913808eSMikulas Patocka pe2 = __lookup_pending_exception(s, chunk); 10142913808eSMikulas Patocka if (pe2) { 10151da177e4SLinus Torvalds free_pending_exception(pe); 10162913808eSMikulas Patocka return pe2; 101776df1c65SAlasdair G Kergon } 101876df1c65SAlasdair G Kergon 10191da177e4SLinus Torvalds pe->e.old_chunk = chunk; 10201da177e4SLinus Torvalds bio_list_init(&pe->origin_bios); 10211da177e4SLinus Torvalds bio_list_init(&pe->snapshot_bios); 1022b4b610f6SAlasdair G Kergon pe->primary_pe = NULL; 10234b832e8dSAlasdair G Kergon atomic_set(&pe->ref_count, 0); 10241da177e4SLinus Torvalds pe->started = 0; 10251da177e4SLinus Torvalds 10261da177e4SLinus Torvalds if (s->store.prepare_exception(&s->store, &pe->e)) { 10271da177e4SLinus Torvalds free_pending_exception(pe); 10281da177e4SLinus Torvalds return NULL; 10291da177e4SLinus Torvalds } 10301da177e4SLinus Torvalds 10314b832e8dSAlasdair G Kergon get_pending_exception(pe); 10321da177e4SLinus Torvalds insert_exception(&s->pending, &pe->e); 10331da177e4SLinus Torvalds 10341da177e4SLinus Torvalds return pe; 10351da177e4SLinus Torvalds } 10361da177e4SLinus Torvalds 1037028867acSAlasdair G Kergon static void remap_exception(struct dm_snapshot *s, struct dm_snap_exception *e, 1038d74f81f8SMilan Broz struct bio *bio, chunk_t chunk) 10391da177e4SLinus Torvalds { 10401da177e4SLinus Torvalds bio->bi_bdev = s->cow->bdev; 1041d74f81f8SMilan Broz bio->bi_sector = chunk_to_sector(s, dm_chunk_number(e->new_chunk) + 1042d74f81f8SMilan Broz (chunk - e->old_chunk)) + 10431da177e4SLinus Torvalds (bio->bi_sector & s->chunk_mask); 10441da177e4SLinus Torvalds } 10451da177e4SLinus Torvalds 10461da177e4SLinus Torvalds static int snapshot_map(struct dm_target *ti, struct bio *bio, 10471da177e4SLinus Torvalds union map_info *map_context) 10481da177e4SLinus Torvalds { 1049028867acSAlasdair G Kergon struct dm_snap_exception *e; 1050028867acSAlasdair G Kergon struct dm_snapshot *s = ti->private; 1051d2a7ad29SKiyoshi Ueda int r = DM_MAPIO_REMAPPED; 10521da177e4SLinus Torvalds chunk_t chunk; 1053028867acSAlasdair G Kergon struct dm_snap_pending_exception *pe = NULL; 10541da177e4SLinus Torvalds 10551da177e4SLinus Torvalds chunk = sector_to_chunk(s, bio->bi_sector); 10561da177e4SLinus Torvalds 10571da177e4SLinus Torvalds /* Full snapshots are not usable */ 105876df1c65SAlasdair G Kergon /* To get here the table must be live so s->active is always set. */ 10591da177e4SLinus Torvalds if (!s->valid) 1060f6a80ea8SAlasdair G Kergon return -EIO; 10611da177e4SLinus Torvalds 10621da177e4SLinus Torvalds /* FIXME: should only take write lock if we need 10631da177e4SLinus Torvalds * to copy an exception */ 10641da177e4SLinus Torvalds down_write(&s->lock); 10651da177e4SLinus Torvalds 106676df1c65SAlasdair G Kergon if (!s->valid) { 106776df1c65SAlasdair G Kergon r = -EIO; 106876df1c65SAlasdair G Kergon goto out_unlock; 106976df1c65SAlasdair G Kergon } 107076df1c65SAlasdair G Kergon 10711da177e4SLinus Torvalds /* If the block is already remapped - use that, else remap it */ 10721da177e4SLinus Torvalds e = lookup_exception(&s->complete, chunk); 10731da177e4SLinus Torvalds if (e) { 1074d74f81f8SMilan Broz remap_exception(s, e, bio, chunk); 107576df1c65SAlasdair G Kergon goto out_unlock; 107676df1c65SAlasdair G Kergon } 10771da177e4SLinus Torvalds 1078ba40a2aaSAlasdair G Kergon /* 1079ba40a2aaSAlasdair G Kergon * Write to snapshot - higher level takes care of RW/RO 1080ba40a2aaSAlasdair G Kergon * flags so we should only get this if we are 1081ba40a2aaSAlasdair G Kergon * writeable. 1082ba40a2aaSAlasdair G Kergon */ 1083ba40a2aaSAlasdair G Kergon if (bio_rw(bio) == WRITE) { 10842913808eSMikulas Patocka pe = __lookup_pending_exception(s, chunk); 10852913808eSMikulas Patocka if (!pe) { 10861da177e4SLinus Torvalds pe = __find_pending_exception(s, bio); 10871da177e4SLinus Torvalds if (!pe) { 1088695368acSAlasdair G Kergon __invalidate_snapshot(s, -ENOMEM); 10891da177e4SLinus Torvalds r = -EIO; 109076df1c65SAlasdair G Kergon goto out_unlock; 109176df1c65SAlasdair G Kergon } 10922913808eSMikulas Patocka } 109376df1c65SAlasdair G Kergon 1094d74f81f8SMilan Broz remap_exception(s, &pe->e, bio, chunk); 10951da177e4SLinus Torvalds bio_list_add(&pe->snapshot_bios, bio); 10961da177e4SLinus Torvalds 1097d2a7ad29SKiyoshi Ueda r = DM_MAPIO_SUBMITTED; 1098ba40a2aaSAlasdair G Kergon 10991da177e4SLinus Torvalds if (!pe->started) { 11001da177e4SLinus Torvalds /* this is protected by snap->lock */ 11011da177e4SLinus Torvalds pe->started = 1; 110276df1c65SAlasdair G Kergon up_write(&s->lock); 110376df1c65SAlasdair G Kergon start_copy(pe); 1104ba40a2aaSAlasdair G Kergon goto out; 1105ba40a2aaSAlasdair G Kergon } 1106cd45daffSMikulas Patocka } else { 11071da177e4SLinus Torvalds bio->bi_bdev = s->origin->bdev; 1108cd45daffSMikulas Patocka map_context->ptr = track_chunk(s, chunk); 1109cd45daffSMikulas Patocka } 11101da177e4SLinus Torvalds 1111ba40a2aaSAlasdair G Kergon out_unlock: 1112ba40a2aaSAlasdair G Kergon up_write(&s->lock); 1113ba40a2aaSAlasdair G Kergon out: 11141da177e4SLinus Torvalds return r; 11151da177e4SLinus Torvalds } 11161da177e4SLinus Torvalds 1117cd45daffSMikulas Patocka static int snapshot_end_io(struct dm_target *ti, struct bio *bio, 1118cd45daffSMikulas Patocka int error, union map_info *map_context) 1119cd45daffSMikulas Patocka { 1120cd45daffSMikulas Patocka struct dm_snapshot *s = ti->private; 1121cd45daffSMikulas Patocka struct dm_snap_tracked_chunk *c = map_context->ptr; 1122cd45daffSMikulas Patocka 1123cd45daffSMikulas Patocka if (c) 1124cd45daffSMikulas Patocka stop_tracking_chunk(s, c); 1125cd45daffSMikulas Patocka 1126cd45daffSMikulas Patocka return 0; 1127cd45daffSMikulas Patocka } 1128cd45daffSMikulas Patocka 11291da177e4SLinus Torvalds static void snapshot_resume(struct dm_target *ti) 11301da177e4SLinus Torvalds { 1131028867acSAlasdair G Kergon struct dm_snapshot *s = ti->private; 11321da177e4SLinus Torvalds 1133aa14edebSAlasdair G Kergon down_write(&s->lock); 1134aa14edebSAlasdair G Kergon s->active = 1; 1135aa14edebSAlasdair G Kergon up_write(&s->lock); 11361da177e4SLinus Torvalds } 11371da177e4SLinus Torvalds 11381da177e4SLinus Torvalds static int snapshot_status(struct dm_target *ti, status_type_t type, 11391da177e4SLinus Torvalds char *result, unsigned int maxlen) 11401da177e4SLinus Torvalds { 1141028867acSAlasdair G Kergon struct dm_snapshot *snap = ti->private; 11421da177e4SLinus Torvalds 11431da177e4SLinus Torvalds switch (type) { 11441da177e4SLinus Torvalds case STATUSTYPE_INFO: 11451da177e4SLinus Torvalds if (!snap->valid) 11461da177e4SLinus Torvalds snprintf(result, maxlen, "Invalid"); 11471da177e4SLinus Torvalds else { 11481da177e4SLinus Torvalds if (snap->store.fraction_full) { 11491da177e4SLinus Torvalds sector_t numerator, denominator; 11501da177e4SLinus Torvalds snap->store.fraction_full(&snap->store, 11511da177e4SLinus Torvalds &numerator, 11521da177e4SLinus Torvalds &denominator); 11534ee218cdSAndrew Morton snprintf(result, maxlen, "%llu/%llu", 11544ee218cdSAndrew Morton (unsigned long long)numerator, 11554ee218cdSAndrew Morton (unsigned long long)denominator); 11561da177e4SLinus Torvalds } 11571da177e4SLinus Torvalds else 11581da177e4SLinus Torvalds snprintf(result, maxlen, "Unknown"); 11591da177e4SLinus Torvalds } 11601da177e4SLinus Torvalds break; 11611da177e4SLinus Torvalds 11621da177e4SLinus Torvalds case STATUSTYPE_TABLE: 11631da177e4SLinus Torvalds /* 11641da177e4SLinus Torvalds * kdevname returns a static pointer so we need 11651da177e4SLinus Torvalds * to make private copies if the output is to 11661da177e4SLinus Torvalds * make sense. 11671da177e4SLinus Torvalds */ 11684ee218cdSAndrew Morton snprintf(result, maxlen, "%s %s %c %llu", 11691da177e4SLinus Torvalds snap->origin->name, snap->cow->name, 11704ee218cdSAndrew Morton snap->type, 11714ee218cdSAndrew Morton (unsigned long long)snap->chunk_size); 11721da177e4SLinus Torvalds break; 11731da177e4SLinus Torvalds } 11741da177e4SLinus Torvalds 11751da177e4SLinus Torvalds return 0; 11761da177e4SLinus Torvalds } 11771da177e4SLinus Torvalds 11781da177e4SLinus Torvalds /*----------------------------------------------------------------- 11791da177e4SLinus Torvalds * Origin methods 11801da177e4SLinus Torvalds *---------------------------------------------------------------*/ 11811da177e4SLinus Torvalds static int __origin_write(struct list_head *snapshots, struct bio *bio) 11821da177e4SLinus Torvalds { 1183d2a7ad29SKiyoshi Ueda int r = DM_MAPIO_REMAPPED, first = 0; 11841da177e4SLinus Torvalds struct dm_snapshot *snap; 1185028867acSAlasdair G Kergon struct dm_snap_exception *e; 1186028867acSAlasdair G Kergon struct dm_snap_pending_exception *pe, *next_pe, *primary_pe = NULL; 11871da177e4SLinus Torvalds chunk_t chunk; 1188eccf0817SAlasdair G Kergon LIST_HEAD(pe_queue); 11891da177e4SLinus Torvalds 11901da177e4SLinus Torvalds /* Do all the snapshots on this origin */ 11911da177e4SLinus Torvalds list_for_each_entry (snap, snapshots, list) { 11921da177e4SLinus Torvalds 119376df1c65SAlasdair G Kergon down_write(&snap->lock); 119476df1c65SAlasdair G Kergon 1195aa14edebSAlasdair G Kergon /* Only deal with valid and active snapshots */ 1196aa14edebSAlasdair G Kergon if (!snap->valid || !snap->active) 119776df1c65SAlasdair G Kergon goto next_snapshot; 11981da177e4SLinus Torvalds 1199d5e404c1SAlasdair G Kergon /* Nothing to do if writing beyond end of snapshot */ 120072727badSMikulas Patocka if (bio->bi_sector >= dm_table_get_size(snap->ti->table)) 120176df1c65SAlasdair G Kergon goto next_snapshot; 12021da177e4SLinus Torvalds 12031da177e4SLinus Torvalds /* 12041da177e4SLinus Torvalds * Remember, different snapshots can have 12051da177e4SLinus Torvalds * different chunk sizes. 12061da177e4SLinus Torvalds */ 12071da177e4SLinus Torvalds chunk = sector_to_chunk(snap, bio->bi_sector); 12081da177e4SLinus Torvalds 12091da177e4SLinus Torvalds /* 12101da177e4SLinus Torvalds * Check exception table to see if block 12111da177e4SLinus Torvalds * is already remapped in this snapshot 12121da177e4SLinus Torvalds * and trigger an exception if not. 1213b4b610f6SAlasdair G Kergon * 12144b832e8dSAlasdair G Kergon * ref_count is initialised to 1 so pending_complete() 1215b4b610f6SAlasdair G Kergon * won't destroy the primary_pe while we're inside this loop. 12161da177e4SLinus Torvalds */ 12171da177e4SLinus Torvalds e = lookup_exception(&snap->complete, chunk); 121876df1c65SAlasdair G Kergon if (e) 121976df1c65SAlasdair G Kergon goto next_snapshot; 122076df1c65SAlasdair G Kergon 12212913808eSMikulas Patocka pe = __lookup_pending_exception(snap, chunk); 12222913808eSMikulas Patocka if (!pe) { 12231da177e4SLinus Torvalds pe = __find_pending_exception(snap, bio); 12241da177e4SLinus Torvalds if (!pe) { 1225695368acSAlasdair G Kergon __invalidate_snapshot(snap, -ENOMEM); 122676df1c65SAlasdair G Kergon goto next_snapshot; 122776df1c65SAlasdair G Kergon } 12282913808eSMikulas Patocka } 12291da177e4SLinus Torvalds 1230b4b610f6SAlasdair G Kergon if (!primary_pe) { 1231b4b610f6SAlasdair G Kergon /* 1232b4b610f6SAlasdair G Kergon * Either every pe here has same 1233b4b610f6SAlasdair G Kergon * primary_pe or none has one yet. 1234b4b610f6SAlasdair G Kergon */ 1235b4b610f6SAlasdair G Kergon if (pe->primary_pe) 1236b4b610f6SAlasdair G Kergon primary_pe = pe->primary_pe; 1237b4b610f6SAlasdair G Kergon else { 1238b4b610f6SAlasdair G Kergon primary_pe = pe; 1239b4b610f6SAlasdair G Kergon first = 1; 1240eccf0817SAlasdair G Kergon } 1241b4b610f6SAlasdair G Kergon 124276df1c65SAlasdair G Kergon bio_list_add(&primary_pe->origin_bios, bio); 124376df1c65SAlasdair G Kergon 1244d2a7ad29SKiyoshi Ueda r = DM_MAPIO_SUBMITTED; 1245b4b610f6SAlasdair G Kergon } 124676df1c65SAlasdair G Kergon 1247b4b610f6SAlasdair G Kergon if (!pe->primary_pe) { 1248b4b610f6SAlasdair G Kergon pe->primary_pe = primary_pe; 12494b832e8dSAlasdair G Kergon get_pending_exception(primary_pe); 1250b4b610f6SAlasdair G Kergon } 125176df1c65SAlasdair G Kergon 1252eccf0817SAlasdair G Kergon if (!pe->started) { 1253eccf0817SAlasdair G Kergon pe->started = 1; 1254eccf0817SAlasdair G Kergon list_add_tail(&pe->list, &pe_queue); 1255eccf0817SAlasdair G Kergon } 12561da177e4SLinus Torvalds 125776df1c65SAlasdair G Kergon next_snapshot: 12581da177e4SLinus Torvalds up_write(&snap->lock); 12591da177e4SLinus Torvalds } 12601da177e4SLinus Torvalds 1261b4b610f6SAlasdair G Kergon if (!primary_pe) 12624b832e8dSAlasdair G Kergon return r; 1263b4b610f6SAlasdair G Kergon 1264b4b610f6SAlasdair G Kergon /* 1265b4b610f6SAlasdair G Kergon * If this is the first time we're processing this chunk and 12664b832e8dSAlasdair G Kergon * ref_count is now 1 it means all the pending exceptions 1267b4b610f6SAlasdair G Kergon * got completed while we were in the loop above, so it falls to 1268b4b610f6SAlasdair G Kergon * us here to remove the primary_pe and submit any origin_bios. 1269b4b610f6SAlasdair G Kergon */ 1270b4b610f6SAlasdair G Kergon 12714b832e8dSAlasdair G Kergon if (first && atomic_dec_and_test(&primary_pe->ref_count)) { 1272b4b610f6SAlasdair G Kergon flush_bios(bio_list_get(&primary_pe->origin_bios)); 1273b4b610f6SAlasdair G Kergon free_pending_exception(primary_pe); 1274b4b610f6SAlasdair G Kergon /* If we got here, pe_queue is necessarily empty. */ 12754b832e8dSAlasdair G Kergon return r; 1276b4b610f6SAlasdair G Kergon } 1277b4b610f6SAlasdair G Kergon 12781da177e4SLinus Torvalds /* 12791da177e4SLinus Torvalds * Now that we have a complete pe list we can start the copying. 12801da177e4SLinus Torvalds */ 1281eccf0817SAlasdair G Kergon list_for_each_entry_safe(pe, next_pe, &pe_queue, list) 12821da177e4SLinus Torvalds start_copy(pe); 12831da177e4SLinus Torvalds 12841da177e4SLinus Torvalds return r; 12851da177e4SLinus Torvalds } 12861da177e4SLinus Torvalds 12871da177e4SLinus Torvalds /* 12881da177e4SLinus Torvalds * Called on a write from the origin driver. 12891da177e4SLinus Torvalds */ 12901da177e4SLinus Torvalds static int do_origin(struct dm_dev *origin, struct bio *bio) 12911da177e4SLinus Torvalds { 12921da177e4SLinus Torvalds struct origin *o; 1293d2a7ad29SKiyoshi Ueda int r = DM_MAPIO_REMAPPED; 12941da177e4SLinus Torvalds 12951da177e4SLinus Torvalds down_read(&_origins_lock); 12961da177e4SLinus Torvalds o = __lookup_origin(origin->bdev); 12971da177e4SLinus Torvalds if (o) 12981da177e4SLinus Torvalds r = __origin_write(&o->snapshots, bio); 12991da177e4SLinus Torvalds up_read(&_origins_lock); 13001da177e4SLinus Torvalds 13011da177e4SLinus Torvalds return r; 13021da177e4SLinus Torvalds } 13031da177e4SLinus Torvalds 13041da177e4SLinus Torvalds /* 13051da177e4SLinus Torvalds * Origin: maps a linear range of a device, with hooks for snapshotting. 13061da177e4SLinus Torvalds */ 13071da177e4SLinus Torvalds 13081da177e4SLinus Torvalds /* 13091da177e4SLinus Torvalds * Construct an origin mapping: <dev_path> 13101da177e4SLinus Torvalds * The context for an origin is merely a 'struct dm_dev *' 13111da177e4SLinus Torvalds * pointing to the real device. 13121da177e4SLinus Torvalds */ 13131da177e4SLinus Torvalds static int origin_ctr(struct dm_target *ti, unsigned int argc, char **argv) 13141da177e4SLinus Torvalds { 13151da177e4SLinus Torvalds int r; 13161da177e4SLinus Torvalds struct dm_dev *dev; 13171da177e4SLinus Torvalds 13181da177e4SLinus Torvalds if (argc != 1) { 131972d94861SAlasdair G Kergon ti->error = "origin: incorrect number of arguments"; 13201da177e4SLinus Torvalds return -EINVAL; 13211da177e4SLinus Torvalds } 13221da177e4SLinus Torvalds 13231da177e4SLinus Torvalds r = dm_get_device(ti, argv[0], 0, ti->len, 13241da177e4SLinus Torvalds dm_table_get_mode(ti->table), &dev); 13251da177e4SLinus Torvalds if (r) { 13261da177e4SLinus Torvalds ti->error = "Cannot get target device"; 13271da177e4SLinus Torvalds return r; 13281da177e4SLinus Torvalds } 13291da177e4SLinus Torvalds 13301da177e4SLinus Torvalds ti->private = dev; 13311da177e4SLinus Torvalds return 0; 13321da177e4SLinus Torvalds } 13331da177e4SLinus Torvalds 13341da177e4SLinus Torvalds static void origin_dtr(struct dm_target *ti) 13351da177e4SLinus Torvalds { 1336028867acSAlasdair G Kergon struct dm_dev *dev = ti->private; 13371da177e4SLinus Torvalds dm_put_device(ti, dev); 13381da177e4SLinus Torvalds } 13391da177e4SLinus Torvalds 13401da177e4SLinus Torvalds static int origin_map(struct dm_target *ti, struct bio *bio, 13411da177e4SLinus Torvalds union map_info *map_context) 13421da177e4SLinus Torvalds { 1343028867acSAlasdair G Kergon struct dm_dev *dev = ti->private; 13441da177e4SLinus Torvalds bio->bi_bdev = dev->bdev; 13451da177e4SLinus Torvalds 13461da177e4SLinus Torvalds /* Only tell snapshots if this is a write */ 1347d2a7ad29SKiyoshi Ueda return (bio_rw(bio) == WRITE) ? do_origin(dev, bio) : DM_MAPIO_REMAPPED; 13481da177e4SLinus Torvalds } 13491da177e4SLinus Torvalds 13501da177e4SLinus Torvalds #define min_not_zero(l, r) (l == 0) ? r : ((r == 0) ? l : min(l, r)) 13511da177e4SLinus Torvalds 13521da177e4SLinus Torvalds /* 13531da177e4SLinus Torvalds * Set the target "split_io" field to the minimum of all the snapshots' 13541da177e4SLinus Torvalds * chunk sizes. 13551da177e4SLinus Torvalds */ 13561da177e4SLinus Torvalds static void origin_resume(struct dm_target *ti) 13571da177e4SLinus Torvalds { 1358028867acSAlasdair G Kergon struct dm_dev *dev = ti->private; 13591da177e4SLinus Torvalds struct dm_snapshot *snap; 13601da177e4SLinus Torvalds struct origin *o; 13611da177e4SLinus Torvalds chunk_t chunk_size = 0; 13621da177e4SLinus Torvalds 13631da177e4SLinus Torvalds down_read(&_origins_lock); 13641da177e4SLinus Torvalds o = __lookup_origin(dev->bdev); 13651da177e4SLinus Torvalds if (o) 13661da177e4SLinus Torvalds list_for_each_entry (snap, &o->snapshots, list) 13671da177e4SLinus Torvalds chunk_size = min_not_zero(chunk_size, snap->chunk_size); 13681da177e4SLinus Torvalds up_read(&_origins_lock); 13691da177e4SLinus Torvalds 13701da177e4SLinus Torvalds ti->split_io = chunk_size; 13711da177e4SLinus Torvalds } 13721da177e4SLinus Torvalds 13731da177e4SLinus Torvalds static int origin_status(struct dm_target *ti, status_type_t type, char *result, 13741da177e4SLinus Torvalds unsigned int maxlen) 13751da177e4SLinus Torvalds { 1376028867acSAlasdair G Kergon struct dm_dev *dev = ti->private; 13771da177e4SLinus Torvalds 13781da177e4SLinus Torvalds switch (type) { 13791da177e4SLinus Torvalds case STATUSTYPE_INFO: 13801da177e4SLinus Torvalds result[0] = '\0'; 13811da177e4SLinus Torvalds break; 13821da177e4SLinus Torvalds 13831da177e4SLinus Torvalds case STATUSTYPE_TABLE: 13841da177e4SLinus Torvalds snprintf(result, maxlen, "%s", dev->name); 13851da177e4SLinus Torvalds break; 13861da177e4SLinus Torvalds } 13871da177e4SLinus Torvalds 13881da177e4SLinus Torvalds return 0; 13891da177e4SLinus Torvalds } 13901da177e4SLinus Torvalds 13911da177e4SLinus Torvalds static struct target_type origin_target = { 13921da177e4SLinus Torvalds .name = "snapshot-origin", 1393d74f81f8SMilan Broz .version = {1, 6, 0}, 13941da177e4SLinus Torvalds .module = THIS_MODULE, 13951da177e4SLinus Torvalds .ctr = origin_ctr, 13961da177e4SLinus Torvalds .dtr = origin_dtr, 13971da177e4SLinus Torvalds .map = origin_map, 13981da177e4SLinus Torvalds .resume = origin_resume, 13991da177e4SLinus Torvalds .status = origin_status, 14001da177e4SLinus Torvalds }; 14011da177e4SLinus Torvalds 14021da177e4SLinus Torvalds static struct target_type snapshot_target = { 14031da177e4SLinus Torvalds .name = "snapshot", 1404d74f81f8SMilan Broz .version = {1, 6, 0}, 14051da177e4SLinus Torvalds .module = THIS_MODULE, 14061da177e4SLinus Torvalds .ctr = snapshot_ctr, 14071da177e4SLinus Torvalds .dtr = snapshot_dtr, 14081da177e4SLinus Torvalds .map = snapshot_map, 1409cd45daffSMikulas Patocka .end_io = snapshot_end_io, 14101da177e4SLinus Torvalds .resume = snapshot_resume, 14111da177e4SLinus Torvalds .status = snapshot_status, 14121da177e4SLinus Torvalds }; 14131da177e4SLinus Torvalds 14141da177e4SLinus Torvalds static int __init dm_snapshot_init(void) 14151da177e4SLinus Torvalds { 14161da177e4SLinus Torvalds int r; 14171da177e4SLinus Torvalds 14184db6bfe0SAlasdair G Kergon r = dm_exception_store_init(); 14194db6bfe0SAlasdair G Kergon if (r) { 14204db6bfe0SAlasdair G Kergon DMERR("Failed to initialize exception stores"); 14214db6bfe0SAlasdair G Kergon return r; 14224db6bfe0SAlasdair G Kergon } 14234db6bfe0SAlasdair G Kergon 14241da177e4SLinus Torvalds r = dm_register_target(&snapshot_target); 14251da177e4SLinus Torvalds if (r) { 14261da177e4SLinus Torvalds DMERR("snapshot target register failed %d", r); 14271da177e4SLinus Torvalds return r; 14281da177e4SLinus Torvalds } 14291da177e4SLinus Torvalds 14301da177e4SLinus Torvalds r = dm_register_target(&origin_target); 14311da177e4SLinus Torvalds if (r < 0) { 143272d94861SAlasdair G Kergon DMERR("Origin target register failed %d", r); 14331da177e4SLinus Torvalds goto bad1; 14341da177e4SLinus Torvalds } 14351da177e4SLinus Torvalds 14361da177e4SLinus Torvalds r = init_origin_hash(); 14371da177e4SLinus Torvalds if (r) { 14381da177e4SLinus Torvalds DMERR("init_origin_hash failed."); 14391da177e4SLinus Torvalds goto bad2; 14401da177e4SLinus Torvalds } 14411da177e4SLinus Torvalds 1442028867acSAlasdair G Kergon exception_cache = KMEM_CACHE(dm_snap_exception, 0); 14431da177e4SLinus Torvalds if (!exception_cache) { 14441da177e4SLinus Torvalds DMERR("Couldn't create exception cache."); 14451da177e4SLinus Torvalds r = -ENOMEM; 14461da177e4SLinus Torvalds goto bad3; 14471da177e4SLinus Torvalds } 14481da177e4SLinus Torvalds 1449028867acSAlasdair G Kergon pending_cache = KMEM_CACHE(dm_snap_pending_exception, 0); 14501da177e4SLinus Torvalds if (!pending_cache) { 14511da177e4SLinus Torvalds DMERR("Couldn't create pending cache."); 14521da177e4SLinus Torvalds r = -ENOMEM; 14531da177e4SLinus Torvalds goto bad4; 14541da177e4SLinus Torvalds } 14551da177e4SLinus Torvalds 1456cd45daffSMikulas Patocka tracked_chunk_cache = KMEM_CACHE(dm_snap_tracked_chunk, 0); 1457cd45daffSMikulas Patocka if (!tracked_chunk_cache) { 1458cd45daffSMikulas Patocka DMERR("Couldn't create cache to track chunks in use."); 1459cd45daffSMikulas Patocka r = -ENOMEM; 1460cd45daffSMikulas Patocka goto bad5; 1461cd45daffSMikulas Patocka } 1462cd45daffSMikulas Patocka 1463ca3a931fSAlasdair G Kergon ksnapd = create_singlethread_workqueue("ksnapd"); 1464ca3a931fSAlasdair G Kergon if (!ksnapd) { 1465ca3a931fSAlasdair G Kergon DMERR("Failed to create ksnapd workqueue."); 1466ca3a931fSAlasdair G Kergon r = -ENOMEM; 146792e86812SMikulas Patocka goto bad_pending_pool; 1468ca3a931fSAlasdair G Kergon } 1469ca3a931fSAlasdair G Kergon 14701da177e4SLinus Torvalds return 0; 14711da177e4SLinus Torvalds 1472cd45daffSMikulas Patocka bad_pending_pool: 1473cd45daffSMikulas Patocka kmem_cache_destroy(tracked_chunk_cache); 14741da177e4SLinus Torvalds bad5: 14751da177e4SLinus Torvalds kmem_cache_destroy(pending_cache); 14761da177e4SLinus Torvalds bad4: 14771da177e4SLinus Torvalds kmem_cache_destroy(exception_cache); 14781da177e4SLinus Torvalds bad3: 14791da177e4SLinus Torvalds exit_origin_hash(); 14801da177e4SLinus Torvalds bad2: 14811da177e4SLinus Torvalds dm_unregister_target(&origin_target); 14821da177e4SLinus Torvalds bad1: 14831da177e4SLinus Torvalds dm_unregister_target(&snapshot_target); 14841da177e4SLinus Torvalds return r; 14851da177e4SLinus Torvalds } 14861da177e4SLinus Torvalds 14871da177e4SLinus Torvalds static void __exit dm_snapshot_exit(void) 14881da177e4SLinus Torvalds { 1489ca3a931fSAlasdair G Kergon destroy_workqueue(ksnapd); 1490ca3a931fSAlasdair G Kergon 149110d3bd09SMikulas Patocka dm_unregister_target(&snapshot_target); 149210d3bd09SMikulas Patocka dm_unregister_target(&origin_target); 14931da177e4SLinus Torvalds 14941da177e4SLinus Torvalds exit_origin_hash(); 14951da177e4SLinus Torvalds kmem_cache_destroy(pending_cache); 14961da177e4SLinus Torvalds kmem_cache_destroy(exception_cache); 1497cd45daffSMikulas Patocka kmem_cache_destroy(tracked_chunk_cache); 14984db6bfe0SAlasdair G Kergon 14994db6bfe0SAlasdair G Kergon dm_exception_store_exit(); 15001da177e4SLinus Torvalds } 15011da177e4SLinus Torvalds 15021da177e4SLinus Torvalds /* Module hooks */ 15031da177e4SLinus Torvalds module_init(dm_snapshot_init); 15041da177e4SLinus Torvalds module_exit(dm_snapshot_exit); 15051da177e4SLinus Torvalds 15061da177e4SLinus Torvalds MODULE_DESCRIPTION(DM_NAME " snapshot target"); 15071da177e4SLinus Torvalds MODULE_AUTHOR("Joe Thornber"); 15081da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 1509