xref: /openbmc/linux/drivers/md/dm.c (revision d5cb9783536a41df9f9cba5b0a1d78047ed787f7)
1 /*
2  * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3  * Copyright (C) 2004 Red Hat, Inc. All rights reserved.
4  *
5  * This file is released under the GPL.
6  */
7 
8 #include "dm.h"
9 #include "dm-bio-list.h"
10 
11 #include <linux/init.h>
12 #include <linux/module.h>
13 #include <linux/moduleparam.h>
14 #include <linux/blkpg.h>
15 #include <linux/bio.h>
16 #include <linux/buffer_head.h>
17 #include <linux/mempool.h>
18 #include <linux/slab.h>
19 #include <linux/idr.h>
20 
21 static const char *_name = DM_NAME;
22 
23 static unsigned int major = 0;
24 static unsigned int _major = 0;
25 
26 /*
27  * One of these is allocated per bio.
28  */
29 struct dm_io {
30 	struct mapped_device *md;
31 	int error;
32 	struct bio *bio;
33 	atomic_t io_count;
34 };
35 
36 /*
37  * One of these is allocated per target within a bio.  Hopefully
38  * this will be simplified out one day.
39  */
40 struct target_io {
41 	struct dm_io *io;
42 	struct dm_target *ti;
43 	union map_info info;
44 };
45 
46 union map_info *dm_get_mapinfo(struct bio *bio)
47 {
48         if (bio && bio->bi_private)
49                 return &((struct target_io *)bio->bi_private)->info;
50         return NULL;
51 }
52 
53 /*
54  * Bits for the md->flags field.
55  */
56 #define DMF_BLOCK_IO 0
57 #define DMF_SUSPENDED 1
58 
59 struct mapped_device {
60 	struct rw_semaphore io_lock;
61 	struct semaphore suspend_lock;
62 	rwlock_t map_lock;
63 	atomic_t holders;
64 
65 	unsigned long flags;
66 
67 	request_queue_t *queue;
68 	struct gendisk *disk;
69 
70 	void *interface_ptr;
71 
72 	/*
73 	 * A list of ios that arrived while we were suspended.
74 	 */
75 	atomic_t pending;
76 	wait_queue_head_t wait;
77  	struct bio_list deferred;
78 
79 	/*
80 	 * The current mapping.
81 	 */
82 	struct dm_table *map;
83 
84 	/*
85 	 * io objects are allocated from here.
86 	 */
87 	mempool_t *io_pool;
88 	mempool_t *tio_pool;
89 
90 	/*
91 	 * Event handling.
92 	 */
93 	atomic_t event_nr;
94 	wait_queue_head_t eventq;
95 
96 	/*
97 	 * freeze/thaw support require holding onto a super block
98 	 */
99 	struct super_block *frozen_sb;
100 	struct block_device *frozen_bdev;
101 };
102 
103 #define MIN_IOS 256
104 static kmem_cache_t *_io_cache;
105 static kmem_cache_t *_tio_cache;
106 
107 static struct bio_set *dm_set;
108 
109 static int __init local_init(void)
110 {
111 	int r;
112 
113 	dm_set = bioset_create(16, 16, 4);
114 	if (!dm_set)
115 		return -ENOMEM;
116 
117 	/* allocate a slab for the dm_ios */
118 	_io_cache = kmem_cache_create("dm_io",
119 				      sizeof(struct dm_io), 0, 0, NULL, NULL);
120 	if (!_io_cache)
121 		return -ENOMEM;
122 
123 	/* allocate a slab for the target ios */
124 	_tio_cache = kmem_cache_create("dm_tio", sizeof(struct target_io),
125 				       0, 0, NULL, NULL);
126 	if (!_tio_cache) {
127 		kmem_cache_destroy(_io_cache);
128 		return -ENOMEM;
129 	}
130 
131 	_major = major;
132 	r = register_blkdev(_major, _name);
133 	if (r < 0) {
134 		kmem_cache_destroy(_tio_cache);
135 		kmem_cache_destroy(_io_cache);
136 		return r;
137 	}
138 
139 	if (!_major)
140 		_major = r;
141 
142 	return 0;
143 }
144 
145 static void local_exit(void)
146 {
147 	kmem_cache_destroy(_tio_cache);
148 	kmem_cache_destroy(_io_cache);
149 
150 	bioset_free(dm_set);
151 
152 	if (unregister_blkdev(_major, _name) < 0)
153 		DMERR("devfs_unregister_blkdev failed");
154 
155 	_major = 0;
156 
157 	DMINFO("cleaned up");
158 }
159 
160 int (*_inits[])(void) __initdata = {
161 	local_init,
162 	dm_target_init,
163 	dm_linear_init,
164 	dm_stripe_init,
165 	dm_interface_init,
166 };
167 
168 void (*_exits[])(void) = {
169 	local_exit,
170 	dm_target_exit,
171 	dm_linear_exit,
172 	dm_stripe_exit,
173 	dm_interface_exit,
174 };
175 
176 static int __init dm_init(void)
177 {
178 	const int count = ARRAY_SIZE(_inits);
179 
180 	int r, i;
181 
182 	for (i = 0; i < count; i++) {
183 		r = _inits[i]();
184 		if (r)
185 			goto bad;
186 	}
187 
188 	return 0;
189 
190       bad:
191 	while (i--)
192 		_exits[i]();
193 
194 	return r;
195 }
196 
197 static void __exit dm_exit(void)
198 {
199 	int i = ARRAY_SIZE(_exits);
200 
201 	while (i--)
202 		_exits[i]();
203 }
204 
205 /*
206  * Block device functions
207  */
208 static int dm_blk_open(struct inode *inode, struct file *file)
209 {
210 	struct mapped_device *md;
211 
212 	md = inode->i_bdev->bd_disk->private_data;
213 	dm_get(md);
214 	return 0;
215 }
216 
217 static int dm_blk_close(struct inode *inode, struct file *file)
218 {
219 	struct mapped_device *md;
220 
221 	md = inode->i_bdev->bd_disk->private_data;
222 	dm_put(md);
223 	return 0;
224 }
225 
226 static inline struct dm_io *alloc_io(struct mapped_device *md)
227 {
228 	return mempool_alloc(md->io_pool, GFP_NOIO);
229 }
230 
231 static inline void free_io(struct mapped_device *md, struct dm_io *io)
232 {
233 	mempool_free(io, md->io_pool);
234 }
235 
236 static inline struct target_io *alloc_tio(struct mapped_device *md)
237 {
238 	return mempool_alloc(md->tio_pool, GFP_NOIO);
239 }
240 
241 static inline void free_tio(struct mapped_device *md, struct target_io *tio)
242 {
243 	mempool_free(tio, md->tio_pool);
244 }
245 
246 /*
247  * Add the bio to the list of deferred io.
248  */
249 static int queue_io(struct mapped_device *md, struct bio *bio)
250 {
251 	down_write(&md->io_lock);
252 
253 	if (!test_bit(DMF_BLOCK_IO, &md->flags)) {
254 		up_write(&md->io_lock);
255 		return 1;
256 	}
257 
258 	bio_list_add(&md->deferred, bio);
259 
260 	up_write(&md->io_lock);
261 	return 0;		/* deferred successfully */
262 }
263 
264 /*
265  * Everyone (including functions in this file), should use this
266  * function to access the md->map field, and make sure they call
267  * dm_table_put() when finished.
268  */
269 struct dm_table *dm_get_table(struct mapped_device *md)
270 {
271 	struct dm_table *t;
272 
273 	read_lock(&md->map_lock);
274 	t = md->map;
275 	if (t)
276 		dm_table_get(t);
277 	read_unlock(&md->map_lock);
278 
279 	return t;
280 }
281 
282 /*-----------------------------------------------------------------
283  * CRUD START:
284  *   A more elegant soln is in the works that uses the queue
285  *   merge fn, unfortunately there are a couple of changes to
286  *   the block layer that I want to make for this.  So in the
287  *   interests of getting something for people to use I give
288  *   you this clearly demarcated crap.
289  *---------------------------------------------------------------*/
290 
291 /*
292  * Decrements the number of outstanding ios that a bio has been
293  * cloned into, completing the original io if necc.
294  */
295 static inline void dec_pending(struct dm_io *io, int error)
296 {
297 	if (error)
298 		io->error = error;
299 
300 	if (atomic_dec_and_test(&io->io_count)) {
301 		if (atomic_dec_and_test(&io->md->pending))
302 			/* nudge anyone waiting on suspend queue */
303 			wake_up(&io->md->wait);
304 
305 		bio_endio(io->bio, io->bio->bi_size, io->error);
306 		free_io(io->md, io);
307 	}
308 }
309 
310 static int clone_endio(struct bio *bio, unsigned int done, int error)
311 {
312 	int r = 0;
313 	struct target_io *tio = bio->bi_private;
314 	struct dm_io *io = tio->io;
315 	dm_endio_fn endio = tio->ti->type->end_io;
316 
317 	if (bio->bi_size)
318 		return 1;
319 
320 	if (!bio_flagged(bio, BIO_UPTODATE) && !error)
321 		error = -EIO;
322 
323 	if (endio) {
324 		r = endio(tio->ti, bio, error, &tio->info);
325 		if (r < 0)
326 			error = r;
327 
328 		else if (r > 0)
329 			/* the target wants another shot at the io */
330 			return 1;
331 	}
332 
333 	free_tio(io->md, tio);
334 	dec_pending(io, error);
335 	bio_put(bio);
336 	return r;
337 }
338 
339 static sector_t max_io_len(struct mapped_device *md,
340 			   sector_t sector, struct dm_target *ti)
341 {
342 	sector_t offset = sector - ti->begin;
343 	sector_t len = ti->len - offset;
344 
345 	/*
346 	 * Does the target need to split even further ?
347 	 */
348 	if (ti->split_io) {
349 		sector_t boundary;
350 		boundary = ((offset + ti->split_io) & ~(ti->split_io - 1))
351 			   - offset;
352 		if (len > boundary)
353 			len = boundary;
354 	}
355 
356 	return len;
357 }
358 
359 static void __map_bio(struct dm_target *ti, struct bio *clone,
360 		      struct target_io *tio)
361 {
362 	int r;
363 
364 	/*
365 	 * Sanity checks.
366 	 */
367 	BUG_ON(!clone->bi_size);
368 
369 	clone->bi_end_io = clone_endio;
370 	clone->bi_private = tio;
371 
372 	/*
373 	 * Map the clone.  If r == 0 we don't need to do
374 	 * anything, the target has assumed ownership of
375 	 * this io.
376 	 */
377 	atomic_inc(&tio->io->io_count);
378 	r = ti->type->map(ti, clone, &tio->info);
379 	if (r > 0)
380 		/* the bio has been remapped so dispatch it */
381 		generic_make_request(clone);
382 
383 	else if (r < 0) {
384 		/* error the io and bail out */
385 		struct dm_io *io = tio->io;
386 		free_tio(tio->io->md, tio);
387 		dec_pending(io, r);
388 		bio_put(clone);
389 	}
390 }
391 
392 struct clone_info {
393 	struct mapped_device *md;
394 	struct dm_table *map;
395 	struct bio *bio;
396 	struct dm_io *io;
397 	sector_t sector;
398 	sector_t sector_count;
399 	unsigned short idx;
400 };
401 
402 static void dm_bio_destructor(struct bio *bio)
403 {
404 	bio_free(bio, dm_set);
405 }
406 
407 /*
408  * Creates a little bio that is just does part of a bvec.
409  */
410 static struct bio *split_bvec(struct bio *bio, sector_t sector,
411 			      unsigned short idx, unsigned int offset,
412 			      unsigned int len)
413 {
414 	struct bio *clone;
415 	struct bio_vec *bv = bio->bi_io_vec + idx;
416 
417 	clone = bio_alloc_bioset(GFP_NOIO, 1, dm_set);
418 	clone->bi_destructor = dm_bio_destructor;
419 	*clone->bi_io_vec = *bv;
420 
421 	clone->bi_sector = sector;
422 	clone->bi_bdev = bio->bi_bdev;
423 	clone->bi_rw = bio->bi_rw;
424 	clone->bi_vcnt = 1;
425 	clone->bi_size = to_bytes(len);
426 	clone->bi_io_vec->bv_offset = offset;
427 	clone->bi_io_vec->bv_len = clone->bi_size;
428 
429 	return clone;
430 }
431 
432 /*
433  * Creates a bio that consists of range of complete bvecs.
434  */
435 static struct bio *clone_bio(struct bio *bio, sector_t sector,
436 			     unsigned short idx, unsigned short bv_count,
437 			     unsigned int len)
438 {
439 	struct bio *clone;
440 
441 	clone = bio_clone(bio, GFP_NOIO);
442 	clone->bi_sector = sector;
443 	clone->bi_idx = idx;
444 	clone->bi_vcnt = idx + bv_count;
445 	clone->bi_size = to_bytes(len);
446 	clone->bi_flags &= ~(1 << BIO_SEG_VALID);
447 
448 	return clone;
449 }
450 
451 static void __clone_and_map(struct clone_info *ci)
452 {
453 	struct bio *clone, *bio = ci->bio;
454 	struct dm_target *ti = dm_table_find_target(ci->map, ci->sector);
455 	sector_t len = 0, max = max_io_len(ci->md, ci->sector, ti);
456 	struct target_io *tio;
457 
458 	/*
459 	 * Allocate a target io object.
460 	 */
461 	tio = alloc_tio(ci->md);
462 	tio->io = ci->io;
463 	tio->ti = ti;
464 	memset(&tio->info, 0, sizeof(tio->info));
465 
466 	if (ci->sector_count <= max) {
467 		/*
468 		 * Optimise for the simple case where we can do all of
469 		 * the remaining io with a single clone.
470 		 */
471 		clone = clone_bio(bio, ci->sector, ci->idx,
472 				  bio->bi_vcnt - ci->idx, ci->sector_count);
473 		__map_bio(ti, clone, tio);
474 		ci->sector_count = 0;
475 
476 	} else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) {
477 		/*
478 		 * There are some bvecs that don't span targets.
479 		 * Do as many of these as possible.
480 		 */
481 		int i;
482 		sector_t remaining = max;
483 		sector_t bv_len;
484 
485 		for (i = ci->idx; remaining && (i < bio->bi_vcnt); i++) {
486 			bv_len = to_sector(bio->bi_io_vec[i].bv_len);
487 
488 			if (bv_len > remaining)
489 				break;
490 
491 			remaining -= bv_len;
492 			len += bv_len;
493 		}
494 
495 		clone = clone_bio(bio, ci->sector, ci->idx, i - ci->idx, len);
496 		__map_bio(ti, clone, tio);
497 
498 		ci->sector += len;
499 		ci->sector_count -= len;
500 		ci->idx = i;
501 
502 	} else {
503 		/*
504 		 * Create two copy bios to deal with io that has
505 		 * been split across a target.
506 		 */
507 		struct bio_vec *bv = bio->bi_io_vec + ci->idx;
508 
509 		clone = split_bvec(bio, ci->sector, ci->idx,
510 				   bv->bv_offset, max);
511 		__map_bio(ti, clone, tio);
512 
513 		ci->sector += max;
514 		ci->sector_count -= max;
515 		ti = dm_table_find_target(ci->map, ci->sector);
516 
517 		len = to_sector(bv->bv_len) - max;
518 		clone = split_bvec(bio, ci->sector, ci->idx,
519 				   bv->bv_offset + to_bytes(max), len);
520 		tio = alloc_tio(ci->md);
521 		tio->io = ci->io;
522 		tio->ti = ti;
523 		memset(&tio->info, 0, sizeof(tio->info));
524 		__map_bio(ti, clone, tio);
525 
526 		ci->sector += len;
527 		ci->sector_count -= len;
528 		ci->idx++;
529 	}
530 }
531 
532 /*
533  * Split the bio into several clones.
534  */
535 static void __split_bio(struct mapped_device *md, struct bio *bio)
536 {
537 	struct clone_info ci;
538 
539 	ci.map = dm_get_table(md);
540 	if (!ci.map) {
541 		bio_io_error(bio, bio->bi_size);
542 		return;
543 	}
544 
545 	ci.md = md;
546 	ci.bio = bio;
547 	ci.io = alloc_io(md);
548 	ci.io->error = 0;
549 	atomic_set(&ci.io->io_count, 1);
550 	ci.io->bio = bio;
551 	ci.io->md = md;
552 	ci.sector = bio->bi_sector;
553 	ci.sector_count = bio_sectors(bio);
554 	ci.idx = bio->bi_idx;
555 
556 	atomic_inc(&md->pending);
557 	while (ci.sector_count)
558 		__clone_and_map(&ci);
559 
560 	/* drop the extra reference count */
561 	dec_pending(ci.io, 0);
562 	dm_table_put(ci.map);
563 }
564 /*-----------------------------------------------------------------
565  * CRUD END
566  *---------------------------------------------------------------*/
567 
568 /*
569  * The request function that just remaps the bio built up by
570  * dm_merge_bvec.
571  */
572 static int dm_request(request_queue_t *q, struct bio *bio)
573 {
574 	int r;
575 	struct mapped_device *md = q->queuedata;
576 
577 	down_read(&md->io_lock);
578 
579 	/*
580 	 * If we're suspended we have to queue
581 	 * this io for later.
582 	 */
583 	while (test_bit(DMF_BLOCK_IO, &md->flags)) {
584 		up_read(&md->io_lock);
585 
586 		if (bio_rw(bio) == READA) {
587 			bio_io_error(bio, bio->bi_size);
588 			return 0;
589 		}
590 
591 		r = queue_io(md, bio);
592 		if (r < 0) {
593 			bio_io_error(bio, bio->bi_size);
594 			return 0;
595 
596 		} else if (r == 0)
597 			return 0;	/* deferred successfully */
598 
599 		/*
600 		 * We're in a while loop, because someone could suspend
601 		 * before we get to the following read lock.
602 		 */
603 		down_read(&md->io_lock);
604 	}
605 
606 	__split_bio(md, bio);
607 	up_read(&md->io_lock);
608 	return 0;
609 }
610 
611 static int dm_flush_all(request_queue_t *q, struct gendisk *disk,
612 			sector_t *error_sector)
613 {
614 	struct mapped_device *md = q->queuedata;
615 	struct dm_table *map = dm_get_table(md);
616 	int ret = -ENXIO;
617 
618 	if (map) {
619 		ret = dm_table_flush_all(map);
620 		dm_table_put(map);
621 	}
622 
623 	return ret;
624 }
625 
626 static void dm_unplug_all(request_queue_t *q)
627 {
628 	struct mapped_device *md = q->queuedata;
629 	struct dm_table *map = dm_get_table(md);
630 
631 	if (map) {
632 		dm_table_unplug_all(map);
633 		dm_table_put(map);
634 	}
635 }
636 
637 static int dm_any_congested(void *congested_data, int bdi_bits)
638 {
639 	int r;
640 	struct mapped_device *md = (struct mapped_device *) congested_data;
641 	struct dm_table *map = dm_get_table(md);
642 
643 	if (!map || test_bit(DMF_BLOCK_IO, &md->flags))
644 		r = bdi_bits;
645 	else
646 		r = dm_table_any_congested(map, bdi_bits);
647 
648 	dm_table_put(map);
649 	return r;
650 }
651 
652 /*-----------------------------------------------------------------
653  * An IDR is used to keep track of allocated minor numbers.
654  *---------------------------------------------------------------*/
655 static DECLARE_MUTEX(_minor_lock);
656 static DEFINE_IDR(_minor_idr);
657 
658 static void free_minor(unsigned int minor)
659 {
660 	down(&_minor_lock);
661 	idr_remove(&_minor_idr, minor);
662 	up(&_minor_lock);
663 }
664 
665 /*
666  * See if the device with a specific minor # is free.
667  */
668 static int specific_minor(struct mapped_device *md, unsigned int minor)
669 {
670 	int r, m;
671 
672 	if (minor >= (1 << MINORBITS))
673 		return -EINVAL;
674 
675 	down(&_minor_lock);
676 
677 	if (idr_find(&_minor_idr, minor)) {
678 		r = -EBUSY;
679 		goto out;
680 	}
681 
682 	r = idr_pre_get(&_minor_idr, GFP_KERNEL);
683 	if (!r) {
684 		r = -ENOMEM;
685 		goto out;
686 	}
687 
688 	r = idr_get_new_above(&_minor_idr, md, minor, &m);
689 	if (r) {
690 		goto out;
691 	}
692 
693 	if (m != minor) {
694 		idr_remove(&_minor_idr, m);
695 		r = -EBUSY;
696 		goto out;
697 	}
698 
699 out:
700 	up(&_minor_lock);
701 	return r;
702 }
703 
704 static int next_free_minor(struct mapped_device *md, unsigned int *minor)
705 {
706 	int r;
707 	unsigned int m;
708 
709 	down(&_minor_lock);
710 
711 	r = idr_pre_get(&_minor_idr, GFP_KERNEL);
712 	if (!r) {
713 		r = -ENOMEM;
714 		goto out;
715 	}
716 
717 	r = idr_get_new(&_minor_idr, md, &m);
718 	if (r) {
719 		goto out;
720 	}
721 
722 	if (m >= (1 << MINORBITS)) {
723 		idr_remove(&_minor_idr, m);
724 		r = -ENOSPC;
725 		goto out;
726 	}
727 
728 	*minor = m;
729 
730 out:
731 	up(&_minor_lock);
732 	return r;
733 }
734 
735 static struct block_device_operations dm_blk_dops;
736 
737 /*
738  * Allocate and initialise a blank device with a given minor.
739  */
740 static struct mapped_device *alloc_dev(unsigned int minor, int persistent)
741 {
742 	int r;
743 	struct mapped_device *md = kmalloc(sizeof(*md), GFP_KERNEL);
744 
745 	if (!md) {
746 		DMWARN("unable to allocate device, out of memory.");
747 		return NULL;
748 	}
749 
750 	/* get a minor number for the dev */
751 	r = persistent ? specific_minor(md, minor) : next_free_minor(md, &minor);
752 	if (r < 0)
753 		goto bad1;
754 
755 	memset(md, 0, sizeof(*md));
756 	init_rwsem(&md->io_lock);
757 	init_MUTEX(&md->suspend_lock);
758 	rwlock_init(&md->map_lock);
759 	atomic_set(&md->holders, 1);
760 	atomic_set(&md->event_nr, 0);
761 
762 	md->queue = blk_alloc_queue(GFP_KERNEL);
763 	if (!md->queue)
764 		goto bad1;
765 
766 	md->queue->queuedata = md;
767 	md->queue->backing_dev_info.congested_fn = dm_any_congested;
768 	md->queue->backing_dev_info.congested_data = md;
769 	blk_queue_make_request(md->queue, dm_request);
770 	md->queue->unplug_fn = dm_unplug_all;
771 	md->queue->issue_flush_fn = dm_flush_all;
772 
773 	md->io_pool = mempool_create(MIN_IOS, mempool_alloc_slab,
774 				     mempool_free_slab, _io_cache);
775  	if (!md->io_pool)
776  		goto bad2;
777 
778 	md->tio_pool = mempool_create(MIN_IOS, mempool_alloc_slab,
779 				      mempool_free_slab, _tio_cache);
780 	if (!md->tio_pool)
781 		goto bad3;
782 
783 	md->disk = alloc_disk(1);
784 	if (!md->disk)
785 		goto bad4;
786 
787 	md->disk->major = _major;
788 	md->disk->first_minor = minor;
789 	md->disk->fops = &dm_blk_dops;
790 	md->disk->queue = md->queue;
791 	md->disk->private_data = md;
792 	sprintf(md->disk->disk_name, "dm-%d", minor);
793 	add_disk(md->disk);
794 
795 	atomic_set(&md->pending, 0);
796 	init_waitqueue_head(&md->wait);
797 	init_waitqueue_head(&md->eventq);
798 
799 	return md;
800 
801  bad4:
802 	mempool_destroy(md->tio_pool);
803  bad3:
804 	mempool_destroy(md->io_pool);
805  bad2:
806 	blk_put_queue(md->queue);
807 	free_minor(minor);
808  bad1:
809 	kfree(md);
810 	return NULL;
811 }
812 
813 static void free_dev(struct mapped_device *md)
814 {
815 	free_minor(md->disk->first_minor);
816 	mempool_destroy(md->tio_pool);
817 	mempool_destroy(md->io_pool);
818 	del_gendisk(md->disk);
819 	put_disk(md->disk);
820 	blk_put_queue(md->queue);
821 	kfree(md);
822 }
823 
824 /*
825  * Bind a table to the device.
826  */
827 static void event_callback(void *context)
828 {
829 	struct mapped_device *md = (struct mapped_device *) context;
830 
831 	atomic_inc(&md->event_nr);
832 	wake_up(&md->eventq);
833 }
834 
835 static void __set_size(struct mapped_device *md, sector_t size)
836 {
837 	set_capacity(md->disk, size);
838 
839 	down(&md->frozen_bdev->bd_inode->i_sem);
840 	i_size_write(md->frozen_bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
841 	up(&md->frozen_bdev->bd_inode->i_sem);
842 }
843 
844 static int __bind(struct mapped_device *md, struct dm_table *t)
845 {
846 	request_queue_t *q = md->queue;
847 	sector_t size;
848 
849 	size = dm_table_get_size(t);
850 	__set_size(md, size);
851 	if (size == 0)
852 		return 0;
853 
854 	dm_table_get(t);
855 	dm_table_event_callback(t, event_callback, md);
856 
857 	write_lock(&md->map_lock);
858 	md->map = t;
859 	dm_table_set_restrictions(t, q);
860 	write_unlock(&md->map_lock);
861 
862 	return 0;
863 }
864 
865 static void __unbind(struct mapped_device *md)
866 {
867 	struct dm_table *map = md->map;
868 
869 	if (!map)
870 		return;
871 
872 	dm_table_event_callback(map, NULL, NULL);
873 	write_lock(&md->map_lock);
874 	md->map = NULL;
875 	write_unlock(&md->map_lock);
876 	dm_table_put(map);
877 }
878 
879 /*
880  * Constructor for a new device.
881  */
882 static int create_aux(unsigned int minor, int persistent,
883 		      struct mapped_device **result)
884 {
885 	struct mapped_device *md;
886 
887 	md = alloc_dev(minor, persistent);
888 	if (!md)
889 		return -ENXIO;
890 
891 	*result = md;
892 	return 0;
893 }
894 
895 int dm_create(struct mapped_device **result)
896 {
897 	return create_aux(0, 0, result);
898 }
899 
900 int dm_create_with_minor(unsigned int minor, struct mapped_device **result)
901 {
902 	return create_aux(minor, 1, result);
903 }
904 
905 void *dm_get_mdptr(dev_t dev)
906 {
907 	struct mapped_device *md;
908 	void *mdptr = NULL;
909 	unsigned minor = MINOR(dev);
910 
911 	if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
912 		return NULL;
913 
914 	down(&_minor_lock);
915 
916 	md = idr_find(&_minor_idr, minor);
917 
918 	if (md && (dm_disk(md)->first_minor == minor))
919 		mdptr = md->interface_ptr;
920 
921 	up(&_minor_lock);
922 
923 	return mdptr;
924 }
925 
926 void dm_set_mdptr(struct mapped_device *md, void *ptr)
927 {
928 	md->interface_ptr = ptr;
929 }
930 
931 void dm_get(struct mapped_device *md)
932 {
933 	atomic_inc(&md->holders);
934 }
935 
936 void dm_put(struct mapped_device *md)
937 {
938 	struct dm_table *map = dm_get_table(md);
939 
940 	if (atomic_dec_and_test(&md->holders)) {
941 		if (!dm_suspended(md)) {
942 			dm_table_presuspend_targets(map);
943 			dm_table_postsuspend_targets(map);
944 		}
945 		__unbind(md);
946 		free_dev(md);
947 	}
948 
949 	dm_table_put(map);
950 }
951 
952 /*
953  * Process the deferred bios
954  */
955 static void __flush_deferred_io(struct mapped_device *md, struct bio *c)
956 {
957 	struct bio *n;
958 
959 	while (c) {
960 		n = c->bi_next;
961 		c->bi_next = NULL;
962 		__split_bio(md, c);
963 		c = n;
964 	}
965 }
966 
967 /*
968  * Swap in a new table (destroying old one).
969  */
970 int dm_swap_table(struct mapped_device *md, struct dm_table *table)
971 {
972 	int r = -EINVAL;
973 
974 	down(&md->suspend_lock);
975 
976 	/* device must be suspended */
977 	if (!dm_suspended(md))
978 		goto out;
979 
980 	__unbind(md);
981 	r = __bind(md, table);
982 
983 out:
984 	up(&md->suspend_lock);
985 	return r;
986 }
987 
988 /*
989  * Functions to lock and unlock any filesystem running on the
990  * device.
991  */
992 static int lock_fs(struct mapped_device *md)
993 {
994 	int r = -ENOMEM;
995 
996 	md->frozen_bdev = bdget_disk(md->disk, 0);
997 	if (!md->frozen_bdev) {
998 		DMWARN("bdget failed in lock_fs");
999 		goto out;
1000 	}
1001 
1002 	WARN_ON(md->frozen_sb);
1003 
1004 	md->frozen_sb = freeze_bdev(md->frozen_bdev);
1005 	if (IS_ERR(md->frozen_sb)) {
1006 		r = PTR_ERR(md->frozen_sb);
1007 		goto out_bdput;
1008 	}
1009 
1010 	/* don't bdput right now, we don't want the bdev
1011 	 * to go away while it is locked.  We'll bdput
1012 	 * in unlock_fs
1013 	 */
1014 	return 0;
1015 
1016 out_bdput:
1017 	bdput(md->frozen_bdev);
1018 	md->frozen_sb = NULL;
1019 	md->frozen_bdev = NULL;
1020 out:
1021 	return r;
1022 }
1023 
1024 static void unlock_fs(struct mapped_device *md)
1025 {
1026 	thaw_bdev(md->frozen_bdev, md->frozen_sb);
1027 	bdput(md->frozen_bdev);
1028 
1029 	md->frozen_sb = NULL;
1030 	md->frozen_bdev = NULL;
1031 }
1032 
1033 /*
1034  * We need to be able to change a mapping table under a mounted
1035  * filesystem.  For example we might want to move some data in
1036  * the background.  Before the table can be swapped with
1037  * dm_bind_table, dm_suspend must be called to flush any in
1038  * flight bios and ensure that any further io gets deferred.
1039  */
1040 int dm_suspend(struct mapped_device *md)
1041 {
1042 	struct dm_table *map = NULL;
1043 	DECLARE_WAITQUEUE(wait, current);
1044 	int r = -EINVAL;
1045 
1046 	down(&md->suspend_lock);
1047 
1048 	if (dm_suspended(md))
1049 		goto out;
1050 
1051 	map = dm_get_table(md);
1052 
1053 	/* This does not get reverted if there's an error later. */
1054 	dm_table_presuspend_targets(map);
1055 
1056 	/* Flush I/O to the device. */
1057 	r = lock_fs(md);
1058 	if (r)
1059 		goto out;
1060 
1061 	/*
1062 	 * First we set the BLOCK_IO flag so no more ios will be mapped.
1063 	 */
1064 	down_write(&md->io_lock);
1065 	set_bit(DMF_BLOCK_IO, &md->flags);
1066 
1067 	add_wait_queue(&md->wait, &wait);
1068 	up_write(&md->io_lock);
1069 
1070 	/* unplug */
1071 	if (map)
1072 		dm_table_unplug_all(map);
1073 
1074 	/*
1075 	 * Then we wait for the already mapped ios to
1076 	 * complete.
1077 	 */
1078 	while (1) {
1079 		set_current_state(TASK_INTERRUPTIBLE);
1080 
1081 		if (!atomic_read(&md->pending) || signal_pending(current))
1082 			break;
1083 
1084 		io_schedule();
1085 	}
1086 	set_current_state(TASK_RUNNING);
1087 
1088 	down_write(&md->io_lock);
1089 	remove_wait_queue(&md->wait, &wait);
1090 
1091 	/* were we interrupted ? */
1092 	r = -EINTR;
1093 	if (atomic_read(&md->pending)) {
1094 		up_write(&md->io_lock);
1095 		unlock_fs(md);
1096 		clear_bit(DMF_BLOCK_IO, &md->flags);
1097 		goto out;
1098 	}
1099 	up_write(&md->io_lock);
1100 
1101 	dm_table_postsuspend_targets(map);
1102 
1103 	set_bit(DMF_SUSPENDED, &md->flags);
1104 
1105 	r = 0;
1106 
1107 out:
1108 	dm_table_put(map);
1109 	up(&md->suspend_lock);
1110 	return r;
1111 }
1112 
1113 int dm_resume(struct mapped_device *md)
1114 {
1115 	int r = -EINVAL;
1116 	struct bio *def;
1117 	struct dm_table *map = NULL;
1118 
1119 	down(&md->suspend_lock);
1120 	if (!dm_suspended(md))
1121 		goto out;
1122 
1123 	map = dm_get_table(md);
1124 	if (!map || !dm_table_get_size(map))
1125 		goto out;
1126 
1127 	dm_table_resume_targets(map);
1128 
1129 	down_write(&md->io_lock);
1130 	clear_bit(DMF_BLOCK_IO, &md->flags);
1131 
1132 	def = bio_list_get(&md->deferred);
1133 	__flush_deferred_io(md, def);
1134 	up_write(&md->io_lock);
1135 
1136 	unlock_fs(md);
1137 
1138 	clear_bit(DMF_SUSPENDED, &md->flags);
1139 
1140 	dm_table_unplug_all(map);
1141 
1142 	r = 0;
1143 
1144 out:
1145 	dm_table_put(map);
1146 	up(&md->suspend_lock);
1147 
1148 	return r;
1149 }
1150 
1151 /*-----------------------------------------------------------------
1152  * Event notification.
1153  *---------------------------------------------------------------*/
1154 uint32_t dm_get_event_nr(struct mapped_device *md)
1155 {
1156 	return atomic_read(&md->event_nr);
1157 }
1158 
1159 int dm_wait_event(struct mapped_device *md, int event_nr)
1160 {
1161 	return wait_event_interruptible(md->eventq,
1162 			(event_nr != atomic_read(&md->event_nr)));
1163 }
1164 
1165 /*
1166  * The gendisk is only valid as long as you have a reference
1167  * count on 'md'.
1168  */
1169 struct gendisk *dm_disk(struct mapped_device *md)
1170 {
1171 	return md->disk;
1172 }
1173 
1174 int dm_suspended(struct mapped_device *md)
1175 {
1176 	return test_bit(DMF_SUSPENDED, &md->flags);
1177 }
1178 
1179 static struct block_device_operations dm_blk_dops = {
1180 	.open = dm_blk_open,
1181 	.release = dm_blk_close,
1182 	.owner = THIS_MODULE
1183 };
1184 
1185 EXPORT_SYMBOL(dm_get_mapinfo);
1186 
1187 /*
1188  * module hooks
1189  */
1190 module_init(dm_init);
1191 module_exit(dm_exit);
1192 
1193 module_param(major, uint, 0);
1194 MODULE_PARM_DESC(major, "The major number of the device mapper");
1195 MODULE_DESCRIPTION(DM_NAME " driver");
1196 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
1197 MODULE_LICENSE("GPL");
1198