1 /* 2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited. 3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. 4 * 5 * This file is released under the GPL. 6 */ 7 8 #include "dm.h" 9 #include "dm-bio-list.h" 10 #include "dm-uevent.h" 11 12 #include <linux/init.h> 13 #include <linux/module.h> 14 #include <linux/mutex.h> 15 #include <linux/moduleparam.h> 16 #include <linux/blkpg.h> 17 #include <linux/bio.h> 18 #include <linux/buffer_head.h> 19 #include <linux/mempool.h> 20 #include <linux/slab.h> 21 #include <linux/idr.h> 22 #include <linux/hdreg.h> 23 #include <linux/blktrace_api.h> 24 #include <trace/block.h> 25 26 #define DM_MSG_PREFIX "core" 27 28 static const char *_name = DM_NAME; 29 30 static unsigned int major = 0; 31 static unsigned int _major = 0; 32 33 static DEFINE_SPINLOCK(_minor_lock); 34 /* 35 * For bio-based dm. 36 * One of these is allocated per bio. 37 */ 38 struct dm_io { 39 struct mapped_device *md; 40 int error; 41 atomic_t io_count; 42 struct bio *bio; 43 unsigned long start_time; 44 }; 45 46 /* 47 * For bio-based dm. 48 * One of these is allocated per target within a bio. Hopefully 49 * this will be simplified out one day. 50 */ 51 struct dm_target_io { 52 struct dm_io *io; 53 struct dm_target *ti; 54 union map_info info; 55 }; 56 57 DEFINE_TRACE(block_bio_complete); 58 59 /* 60 * For request-based dm. 61 * One of these is allocated per request. 62 */ 63 struct dm_rq_target_io { 64 struct mapped_device *md; 65 struct dm_target *ti; 66 struct request *orig, clone; 67 int error; 68 union map_info info; 69 }; 70 71 /* 72 * For request-based dm. 73 * One of these is allocated per bio. 74 */ 75 struct dm_rq_clone_bio_info { 76 struct bio *orig; 77 struct request *rq; 78 }; 79 80 union map_info *dm_get_mapinfo(struct bio *bio) 81 { 82 if (bio && bio->bi_private) 83 return &((struct dm_target_io *)bio->bi_private)->info; 84 return NULL; 85 } 86 87 #define MINOR_ALLOCED ((void *)-1) 88 89 /* 90 * Bits for the md->flags field. 91 */ 92 #define DMF_BLOCK_IO 0 93 #define DMF_SUSPENDED 1 94 #define DMF_FROZEN 2 95 #define DMF_FREEING 3 96 #define DMF_DELETING 4 97 #define DMF_NOFLUSH_SUSPENDING 5 98 99 /* 100 * Work processed by per-device workqueue. 101 */ 102 struct mapped_device { 103 struct rw_semaphore io_lock; 104 struct mutex suspend_lock; 105 rwlock_t map_lock; 106 atomic_t holders; 107 atomic_t open_count; 108 109 unsigned long flags; 110 111 struct request_queue *queue; 112 struct gendisk *disk; 113 char name[16]; 114 115 void *interface_ptr; 116 117 /* 118 * A list of ios that arrived while we were suspended. 119 */ 120 atomic_t pending; 121 wait_queue_head_t wait; 122 struct work_struct work; 123 struct bio_list deferred; 124 spinlock_t deferred_lock; 125 126 /* 127 * Processing queue (flush/barriers) 128 */ 129 struct workqueue_struct *wq; 130 131 /* 132 * The current mapping. 133 */ 134 struct dm_table *map; 135 136 /* 137 * io objects are allocated from here. 138 */ 139 mempool_t *io_pool; 140 mempool_t *tio_pool; 141 142 struct bio_set *bs; 143 144 /* 145 * Event handling. 146 */ 147 atomic_t event_nr; 148 wait_queue_head_t eventq; 149 atomic_t uevent_seq; 150 struct list_head uevent_list; 151 spinlock_t uevent_lock; /* Protect access to uevent_list */ 152 153 /* 154 * freeze/thaw support require holding onto a super block 155 */ 156 struct super_block *frozen_sb; 157 struct block_device *suspended_bdev; 158 159 /* forced geometry settings */ 160 struct hd_geometry geometry; 161 162 /* sysfs handle */ 163 struct kobject kobj; 164 }; 165 166 #define MIN_IOS 256 167 static struct kmem_cache *_io_cache; 168 static struct kmem_cache *_tio_cache; 169 static struct kmem_cache *_rq_tio_cache; 170 static struct kmem_cache *_rq_bio_info_cache; 171 172 static int __init local_init(void) 173 { 174 int r = -ENOMEM; 175 176 /* allocate a slab for the dm_ios */ 177 _io_cache = KMEM_CACHE(dm_io, 0); 178 if (!_io_cache) 179 return r; 180 181 /* allocate a slab for the target ios */ 182 _tio_cache = KMEM_CACHE(dm_target_io, 0); 183 if (!_tio_cache) 184 goto out_free_io_cache; 185 186 _rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0); 187 if (!_rq_tio_cache) 188 goto out_free_tio_cache; 189 190 _rq_bio_info_cache = KMEM_CACHE(dm_rq_clone_bio_info, 0); 191 if (!_rq_bio_info_cache) 192 goto out_free_rq_tio_cache; 193 194 r = dm_uevent_init(); 195 if (r) 196 goto out_free_rq_bio_info_cache; 197 198 _major = major; 199 r = register_blkdev(_major, _name); 200 if (r < 0) 201 goto out_uevent_exit; 202 203 if (!_major) 204 _major = r; 205 206 return 0; 207 208 out_uevent_exit: 209 dm_uevent_exit(); 210 out_free_rq_bio_info_cache: 211 kmem_cache_destroy(_rq_bio_info_cache); 212 out_free_rq_tio_cache: 213 kmem_cache_destroy(_rq_tio_cache); 214 out_free_tio_cache: 215 kmem_cache_destroy(_tio_cache); 216 out_free_io_cache: 217 kmem_cache_destroy(_io_cache); 218 219 return r; 220 } 221 222 static void local_exit(void) 223 { 224 kmem_cache_destroy(_rq_bio_info_cache); 225 kmem_cache_destroy(_rq_tio_cache); 226 kmem_cache_destroy(_tio_cache); 227 kmem_cache_destroy(_io_cache); 228 unregister_blkdev(_major, _name); 229 dm_uevent_exit(); 230 231 _major = 0; 232 233 DMINFO("cleaned up"); 234 } 235 236 static int (*_inits[])(void) __initdata = { 237 local_init, 238 dm_target_init, 239 dm_linear_init, 240 dm_stripe_init, 241 dm_kcopyd_init, 242 dm_interface_init, 243 }; 244 245 static void (*_exits[])(void) = { 246 local_exit, 247 dm_target_exit, 248 dm_linear_exit, 249 dm_stripe_exit, 250 dm_kcopyd_exit, 251 dm_interface_exit, 252 }; 253 254 static int __init dm_init(void) 255 { 256 const int count = ARRAY_SIZE(_inits); 257 258 int r, i; 259 260 for (i = 0; i < count; i++) { 261 r = _inits[i](); 262 if (r) 263 goto bad; 264 } 265 266 return 0; 267 268 bad: 269 while (i--) 270 _exits[i](); 271 272 return r; 273 } 274 275 static void __exit dm_exit(void) 276 { 277 int i = ARRAY_SIZE(_exits); 278 279 while (i--) 280 _exits[i](); 281 } 282 283 /* 284 * Block device functions 285 */ 286 static int dm_blk_open(struct block_device *bdev, fmode_t mode) 287 { 288 struct mapped_device *md; 289 290 spin_lock(&_minor_lock); 291 292 md = bdev->bd_disk->private_data; 293 if (!md) 294 goto out; 295 296 if (test_bit(DMF_FREEING, &md->flags) || 297 test_bit(DMF_DELETING, &md->flags)) { 298 md = NULL; 299 goto out; 300 } 301 302 dm_get(md); 303 atomic_inc(&md->open_count); 304 305 out: 306 spin_unlock(&_minor_lock); 307 308 return md ? 0 : -ENXIO; 309 } 310 311 static int dm_blk_close(struct gendisk *disk, fmode_t mode) 312 { 313 struct mapped_device *md = disk->private_data; 314 atomic_dec(&md->open_count); 315 dm_put(md); 316 return 0; 317 } 318 319 int dm_open_count(struct mapped_device *md) 320 { 321 return atomic_read(&md->open_count); 322 } 323 324 /* 325 * Guarantees nothing is using the device before it's deleted. 326 */ 327 int dm_lock_for_deletion(struct mapped_device *md) 328 { 329 int r = 0; 330 331 spin_lock(&_minor_lock); 332 333 if (dm_open_count(md)) 334 r = -EBUSY; 335 else 336 set_bit(DMF_DELETING, &md->flags); 337 338 spin_unlock(&_minor_lock); 339 340 return r; 341 } 342 343 static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo) 344 { 345 struct mapped_device *md = bdev->bd_disk->private_data; 346 347 return dm_get_geometry(md, geo); 348 } 349 350 static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode, 351 unsigned int cmd, unsigned long arg) 352 { 353 struct mapped_device *md = bdev->bd_disk->private_data; 354 struct dm_table *map = dm_get_table(md); 355 struct dm_target *tgt; 356 int r = -ENOTTY; 357 358 if (!map || !dm_table_get_size(map)) 359 goto out; 360 361 /* We only support devices that have a single target */ 362 if (dm_table_get_num_targets(map) != 1) 363 goto out; 364 365 tgt = dm_table_get_target(map, 0); 366 367 if (dm_suspended(md)) { 368 r = -EAGAIN; 369 goto out; 370 } 371 372 if (tgt->type->ioctl) 373 r = tgt->type->ioctl(tgt, cmd, arg); 374 375 out: 376 dm_table_put(map); 377 378 return r; 379 } 380 381 static struct dm_io *alloc_io(struct mapped_device *md) 382 { 383 return mempool_alloc(md->io_pool, GFP_NOIO); 384 } 385 386 static void free_io(struct mapped_device *md, struct dm_io *io) 387 { 388 mempool_free(io, md->io_pool); 389 } 390 391 static struct dm_target_io *alloc_tio(struct mapped_device *md) 392 { 393 return mempool_alloc(md->tio_pool, GFP_NOIO); 394 } 395 396 static void free_tio(struct mapped_device *md, struct dm_target_io *tio) 397 { 398 mempool_free(tio, md->tio_pool); 399 } 400 401 static void start_io_acct(struct dm_io *io) 402 { 403 struct mapped_device *md = io->md; 404 int cpu; 405 406 io->start_time = jiffies; 407 408 cpu = part_stat_lock(); 409 part_round_stats(cpu, &dm_disk(md)->part0); 410 part_stat_unlock(); 411 dm_disk(md)->part0.in_flight = atomic_inc_return(&md->pending); 412 } 413 414 static void end_io_acct(struct dm_io *io) 415 { 416 struct mapped_device *md = io->md; 417 struct bio *bio = io->bio; 418 unsigned long duration = jiffies - io->start_time; 419 int pending, cpu; 420 int rw = bio_data_dir(bio); 421 422 cpu = part_stat_lock(); 423 part_round_stats(cpu, &dm_disk(md)->part0); 424 part_stat_add(cpu, &dm_disk(md)->part0, ticks[rw], duration); 425 part_stat_unlock(); 426 427 dm_disk(md)->part0.in_flight = pending = 428 atomic_dec_return(&md->pending); 429 430 /* nudge anyone waiting on suspend queue */ 431 if (!pending) 432 wake_up(&md->wait); 433 } 434 435 /* 436 * Add the bio to the list of deferred io. 437 */ 438 static int queue_io(struct mapped_device *md, struct bio *bio) 439 { 440 down_write(&md->io_lock); 441 442 if (!test_bit(DMF_BLOCK_IO, &md->flags)) { 443 up_write(&md->io_lock); 444 return 1; 445 } 446 447 spin_lock_irq(&md->deferred_lock); 448 bio_list_add(&md->deferred, bio); 449 spin_unlock_irq(&md->deferred_lock); 450 451 up_write(&md->io_lock); 452 return 0; /* deferred successfully */ 453 } 454 455 /* 456 * Everyone (including functions in this file), should use this 457 * function to access the md->map field, and make sure they call 458 * dm_table_put() when finished. 459 */ 460 struct dm_table *dm_get_table(struct mapped_device *md) 461 { 462 struct dm_table *t; 463 464 read_lock(&md->map_lock); 465 t = md->map; 466 if (t) 467 dm_table_get(t); 468 read_unlock(&md->map_lock); 469 470 return t; 471 } 472 473 /* 474 * Get the geometry associated with a dm device 475 */ 476 int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo) 477 { 478 *geo = md->geometry; 479 480 return 0; 481 } 482 483 /* 484 * Set the geometry of a device. 485 */ 486 int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo) 487 { 488 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors; 489 490 if (geo->start > sz) { 491 DMWARN("Start sector is beyond the geometry limits."); 492 return -EINVAL; 493 } 494 495 md->geometry = *geo; 496 497 return 0; 498 } 499 500 /*----------------------------------------------------------------- 501 * CRUD START: 502 * A more elegant soln is in the works that uses the queue 503 * merge fn, unfortunately there are a couple of changes to 504 * the block layer that I want to make for this. So in the 505 * interests of getting something for people to use I give 506 * you this clearly demarcated crap. 507 *---------------------------------------------------------------*/ 508 509 static int __noflush_suspending(struct mapped_device *md) 510 { 511 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 512 } 513 514 /* 515 * Decrements the number of outstanding ios that a bio has been 516 * cloned into, completing the original io if necc. 517 */ 518 static void dec_pending(struct dm_io *io, int error) 519 { 520 unsigned long flags; 521 int io_error; 522 struct bio *bio; 523 struct mapped_device *md = io->md; 524 525 /* Push-back supersedes any I/O errors */ 526 if (error && !(io->error > 0 && __noflush_suspending(md))) 527 io->error = error; 528 529 if (atomic_dec_and_test(&io->io_count)) { 530 if (io->error == DM_ENDIO_REQUEUE) { 531 /* 532 * Target requested pushing back the I/O. 533 */ 534 spin_lock_irqsave(&md->deferred_lock, flags); 535 if (__noflush_suspending(md)) 536 bio_list_add(&md->deferred, io->bio); 537 else 538 /* noflush suspend was interrupted. */ 539 io->error = -EIO; 540 spin_unlock_irqrestore(&md->deferred_lock, flags); 541 } 542 543 end_io_acct(io); 544 545 io_error = io->error; 546 bio = io->bio; 547 548 free_io(md, io); 549 550 if (io_error != DM_ENDIO_REQUEUE) { 551 trace_block_bio_complete(md->queue, bio); 552 553 bio_endio(bio, io_error); 554 } 555 } 556 } 557 558 static void clone_endio(struct bio *bio, int error) 559 { 560 int r = 0; 561 struct dm_target_io *tio = bio->bi_private; 562 struct dm_io *io = tio->io; 563 struct mapped_device *md = tio->io->md; 564 dm_endio_fn endio = tio->ti->type->end_io; 565 566 if (!bio_flagged(bio, BIO_UPTODATE) && !error) 567 error = -EIO; 568 569 if (endio) { 570 r = endio(tio->ti, bio, error, &tio->info); 571 if (r < 0 || r == DM_ENDIO_REQUEUE) 572 /* 573 * error and requeue request are handled 574 * in dec_pending(). 575 */ 576 error = r; 577 else if (r == DM_ENDIO_INCOMPLETE) 578 /* The target will handle the io */ 579 return; 580 else if (r) { 581 DMWARN("unimplemented target endio return value: %d", r); 582 BUG(); 583 } 584 } 585 586 /* 587 * Store md for cleanup instead of tio which is about to get freed. 588 */ 589 bio->bi_private = md->bs; 590 591 free_tio(md, tio); 592 bio_put(bio); 593 dec_pending(io, error); 594 } 595 596 static sector_t max_io_len(struct mapped_device *md, 597 sector_t sector, struct dm_target *ti) 598 { 599 sector_t offset = sector - ti->begin; 600 sector_t len = ti->len - offset; 601 602 /* 603 * Does the target need to split even further ? 604 */ 605 if (ti->split_io) { 606 sector_t boundary; 607 boundary = ((offset + ti->split_io) & ~(ti->split_io - 1)) 608 - offset; 609 if (len > boundary) 610 len = boundary; 611 } 612 613 return len; 614 } 615 616 static void __map_bio(struct dm_target *ti, struct bio *clone, 617 struct dm_target_io *tio) 618 { 619 int r; 620 sector_t sector; 621 struct mapped_device *md; 622 623 /* 624 * Sanity checks. 625 */ 626 BUG_ON(!clone->bi_size); 627 628 clone->bi_end_io = clone_endio; 629 clone->bi_private = tio; 630 631 /* 632 * Map the clone. If r == 0 we don't need to do 633 * anything, the target has assumed ownership of 634 * this io. 635 */ 636 atomic_inc(&tio->io->io_count); 637 sector = clone->bi_sector; 638 r = ti->type->map(ti, clone, &tio->info); 639 if (r == DM_MAPIO_REMAPPED) { 640 /* the bio has been remapped so dispatch it */ 641 642 trace_block_remap(bdev_get_queue(clone->bi_bdev), clone, 643 tio->io->bio->bi_bdev->bd_dev, 644 clone->bi_sector, sector); 645 646 generic_make_request(clone); 647 } else if (r < 0 || r == DM_MAPIO_REQUEUE) { 648 /* error the io and bail out, or requeue it if needed */ 649 md = tio->io->md; 650 dec_pending(tio->io, r); 651 /* 652 * Store bio_set for cleanup. 653 */ 654 clone->bi_private = md->bs; 655 bio_put(clone); 656 free_tio(md, tio); 657 } else if (r) { 658 DMWARN("unimplemented target map return value: %d", r); 659 BUG(); 660 } 661 } 662 663 struct clone_info { 664 struct mapped_device *md; 665 struct dm_table *map; 666 struct bio *bio; 667 struct dm_io *io; 668 sector_t sector; 669 sector_t sector_count; 670 unsigned short idx; 671 }; 672 673 static void dm_bio_destructor(struct bio *bio) 674 { 675 struct bio_set *bs = bio->bi_private; 676 677 bio_free(bio, bs); 678 } 679 680 /* 681 * Creates a little bio that is just does part of a bvec. 682 */ 683 static struct bio *split_bvec(struct bio *bio, sector_t sector, 684 unsigned short idx, unsigned int offset, 685 unsigned int len, struct bio_set *bs) 686 { 687 struct bio *clone; 688 struct bio_vec *bv = bio->bi_io_vec + idx; 689 690 clone = bio_alloc_bioset(GFP_NOIO, 1, bs); 691 clone->bi_destructor = dm_bio_destructor; 692 *clone->bi_io_vec = *bv; 693 694 clone->bi_sector = sector; 695 clone->bi_bdev = bio->bi_bdev; 696 clone->bi_rw = bio->bi_rw; 697 clone->bi_vcnt = 1; 698 clone->bi_size = to_bytes(len); 699 clone->bi_io_vec->bv_offset = offset; 700 clone->bi_io_vec->bv_len = clone->bi_size; 701 clone->bi_flags |= 1 << BIO_CLONED; 702 703 if (bio_integrity(bio)) { 704 bio_integrity_clone(clone, bio, GFP_NOIO); 705 bio_integrity_trim(clone, 706 bio_sector_offset(bio, idx, offset), len); 707 } 708 709 return clone; 710 } 711 712 /* 713 * Creates a bio that consists of range of complete bvecs. 714 */ 715 static struct bio *clone_bio(struct bio *bio, sector_t sector, 716 unsigned short idx, unsigned short bv_count, 717 unsigned int len, struct bio_set *bs) 718 { 719 struct bio *clone; 720 721 clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs); 722 __bio_clone(clone, bio); 723 clone->bi_destructor = dm_bio_destructor; 724 clone->bi_sector = sector; 725 clone->bi_idx = idx; 726 clone->bi_vcnt = idx + bv_count; 727 clone->bi_size = to_bytes(len); 728 clone->bi_flags &= ~(1 << BIO_SEG_VALID); 729 730 if (bio_integrity(bio)) { 731 bio_integrity_clone(clone, bio, GFP_NOIO); 732 733 if (idx != bio->bi_idx || clone->bi_size < bio->bi_size) 734 bio_integrity_trim(clone, 735 bio_sector_offset(bio, idx, 0), len); 736 } 737 738 return clone; 739 } 740 741 static int __clone_and_map(struct clone_info *ci) 742 { 743 struct bio *clone, *bio = ci->bio; 744 struct dm_target *ti; 745 sector_t len = 0, max; 746 struct dm_target_io *tio; 747 748 ti = dm_table_find_target(ci->map, ci->sector); 749 if (!dm_target_is_valid(ti)) 750 return -EIO; 751 752 max = max_io_len(ci->md, ci->sector, ti); 753 754 /* 755 * Allocate a target io object. 756 */ 757 tio = alloc_tio(ci->md); 758 tio->io = ci->io; 759 tio->ti = ti; 760 memset(&tio->info, 0, sizeof(tio->info)); 761 762 if (ci->sector_count <= max) { 763 /* 764 * Optimise for the simple case where we can do all of 765 * the remaining io with a single clone. 766 */ 767 clone = clone_bio(bio, ci->sector, ci->idx, 768 bio->bi_vcnt - ci->idx, ci->sector_count, 769 ci->md->bs); 770 __map_bio(ti, clone, tio); 771 ci->sector_count = 0; 772 773 } else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) { 774 /* 775 * There are some bvecs that don't span targets. 776 * Do as many of these as possible. 777 */ 778 int i; 779 sector_t remaining = max; 780 sector_t bv_len; 781 782 for (i = ci->idx; remaining && (i < bio->bi_vcnt); i++) { 783 bv_len = to_sector(bio->bi_io_vec[i].bv_len); 784 785 if (bv_len > remaining) 786 break; 787 788 remaining -= bv_len; 789 len += bv_len; 790 } 791 792 clone = clone_bio(bio, ci->sector, ci->idx, i - ci->idx, len, 793 ci->md->bs); 794 __map_bio(ti, clone, tio); 795 796 ci->sector += len; 797 ci->sector_count -= len; 798 ci->idx = i; 799 800 } else { 801 /* 802 * Handle a bvec that must be split between two or more targets. 803 */ 804 struct bio_vec *bv = bio->bi_io_vec + ci->idx; 805 sector_t remaining = to_sector(bv->bv_len); 806 unsigned int offset = 0; 807 808 do { 809 if (offset) { 810 ti = dm_table_find_target(ci->map, ci->sector); 811 if (!dm_target_is_valid(ti)) 812 return -EIO; 813 814 max = max_io_len(ci->md, ci->sector, ti); 815 816 tio = alloc_tio(ci->md); 817 tio->io = ci->io; 818 tio->ti = ti; 819 memset(&tio->info, 0, sizeof(tio->info)); 820 } 821 822 len = min(remaining, max); 823 824 clone = split_bvec(bio, ci->sector, ci->idx, 825 bv->bv_offset + offset, len, 826 ci->md->bs); 827 828 __map_bio(ti, clone, tio); 829 830 ci->sector += len; 831 ci->sector_count -= len; 832 offset += to_bytes(len); 833 } while (remaining -= len); 834 835 ci->idx++; 836 } 837 838 return 0; 839 } 840 841 /* 842 * Split the bio into several clones and submit it to targets. 843 */ 844 static void __split_and_process_bio(struct mapped_device *md, struct bio *bio) 845 { 846 struct clone_info ci; 847 int error = 0; 848 849 ci.map = dm_get_table(md); 850 if (unlikely(!ci.map)) { 851 bio_io_error(bio); 852 return; 853 } 854 855 ci.md = md; 856 ci.bio = bio; 857 ci.io = alloc_io(md); 858 ci.io->error = 0; 859 atomic_set(&ci.io->io_count, 1); 860 ci.io->bio = bio; 861 ci.io->md = md; 862 ci.sector = bio->bi_sector; 863 ci.sector_count = bio_sectors(bio); 864 ci.idx = bio->bi_idx; 865 866 start_io_acct(ci.io); 867 while (ci.sector_count && !error) 868 error = __clone_and_map(&ci); 869 870 /* drop the extra reference count */ 871 dec_pending(ci.io, error); 872 dm_table_put(ci.map); 873 } 874 /*----------------------------------------------------------------- 875 * CRUD END 876 *---------------------------------------------------------------*/ 877 878 static int dm_merge_bvec(struct request_queue *q, 879 struct bvec_merge_data *bvm, 880 struct bio_vec *biovec) 881 { 882 struct mapped_device *md = q->queuedata; 883 struct dm_table *map = dm_get_table(md); 884 struct dm_target *ti; 885 sector_t max_sectors; 886 int max_size = 0; 887 888 if (unlikely(!map)) 889 goto out; 890 891 ti = dm_table_find_target(map, bvm->bi_sector); 892 if (!dm_target_is_valid(ti)) 893 goto out_table; 894 895 /* 896 * Find maximum amount of I/O that won't need splitting 897 */ 898 max_sectors = min(max_io_len(md, bvm->bi_sector, ti), 899 (sector_t) BIO_MAX_SECTORS); 900 max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size; 901 if (max_size < 0) 902 max_size = 0; 903 904 /* 905 * merge_bvec_fn() returns number of bytes 906 * it can accept at this offset 907 * max is precomputed maximal io size 908 */ 909 if (max_size && ti->type->merge) 910 max_size = ti->type->merge(ti, bvm, biovec, max_size); 911 912 out_table: 913 dm_table_put(map); 914 915 out: 916 /* 917 * Always allow an entire first page 918 */ 919 if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT)) 920 max_size = biovec->bv_len; 921 922 return max_size; 923 } 924 925 /* 926 * The request function that just remaps the bio built up by 927 * dm_merge_bvec. 928 */ 929 static int dm_request(struct request_queue *q, struct bio *bio) 930 { 931 int r = -EIO; 932 int rw = bio_data_dir(bio); 933 struct mapped_device *md = q->queuedata; 934 int cpu; 935 936 /* 937 * There is no use in forwarding any barrier request since we can't 938 * guarantee it is (or can be) handled by the targets correctly. 939 */ 940 if (unlikely(bio_barrier(bio))) { 941 bio_endio(bio, -EOPNOTSUPP); 942 return 0; 943 } 944 945 down_read(&md->io_lock); 946 947 cpu = part_stat_lock(); 948 part_stat_inc(cpu, &dm_disk(md)->part0, ios[rw]); 949 part_stat_add(cpu, &dm_disk(md)->part0, sectors[rw], bio_sectors(bio)); 950 part_stat_unlock(); 951 952 /* 953 * If we're suspended we have to queue 954 * this io for later. 955 */ 956 while (test_bit(DMF_BLOCK_IO, &md->flags)) { 957 up_read(&md->io_lock); 958 959 if (bio_rw(bio) != READA) 960 r = queue_io(md, bio); 961 962 if (r <= 0) 963 goto out_req; 964 965 /* 966 * We're in a while loop, because someone could suspend 967 * before we get to the following read lock. 968 */ 969 down_read(&md->io_lock); 970 } 971 972 __split_and_process_bio(md, bio); 973 up_read(&md->io_lock); 974 return 0; 975 976 out_req: 977 if (r < 0) 978 bio_io_error(bio); 979 980 return 0; 981 } 982 983 static void dm_unplug_all(struct request_queue *q) 984 { 985 struct mapped_device *md = q->queuedata; 986 struct dm_table *map = dm_get_table(md); 987 988 if (map) { 989 dm_table_unplug_all(map); 990 dm_table_put(map); 991 } 992 } 993 994 static int dm_any_congested(void *congested_data, int bdi_bits) 995 { 996 int r = bdi_bits; 997 struct mapped_device *md = congested_data; 998 struct dm_table *map; 999 1000 if (!test_bit(DMF_BLOCK_IO, &md->flags)) { 1001 map = dm_get_table(md); 1002 if (map) { 1003 r = dm_table_any_congested(map, bdi_bits); 1004 dm_table_put(map); 1005 } 1006 } 1007 1008 return r; 1009 } 1010 1011 /*----------------------------------------------------------------- 1012 * An IDR is used to keep track of allocated minor numbers. 1013 *---------------------------------------------------------------*/ 1014 static DEFINE_IDR(_minor_idr); 1015 1016 static void free_minor(int minor) 1017 { 1018 spin_lock(&_minor_lock); 1019 idr_remove(&_minor_idr, minor); 1020 spin_unlock(&_minor_lock); 1021 } 1022 1023 /* 1024 * See if the device with a specific minor # is free. 1025 */ 1026 static int specific_minor(int minor) 1027 { 1028 int r, m; 1029 1030 if (minor >= (1 << MINORBITS)) 1031 return -EINVAL; 1032 1033 r = idr_pre_get(&_minor_idr, GFP_KERNEL); 1034 if (!r) 1035 return -ENOMEM; 1036 1037 spin_lock(&_minor_lock); 1038 1039 if (idr_find(&_minor_idr, minor)) { 1040 r = -EBUSY; 1041 goto out; 1042 } 1043 1044 r = idr_get_new_above(&_minor_idr, MINOR_ALLOCED, minor, &m); 1045 if (r) 1046 goto out; 1047 1048 if (m != minor) { 1049 idr_remove(&_minor_idr, m); 1050 r = -EBUSY; 1051 goto out; 1052 } 1053 1054 out: 1055 spin_unlock(&_minor_lock); 1056 return r; 1057 } 1058 1059 static int next_free_minor(int *minor) 1060 { 1061 int r, m; 1062 1063 r = idr_pre_get(&_minor_idr, GFP_KERNEL); 1064 if (!r) 1065 return -ENOMEM; 1066 1067 spin_lock(&_minor_lock); 1068 1069 r = idr_get_new(&_minor_idr, MINOR_ALLOCED, &m); 1070 if (r) 1071 goto out; 1072 1073 if (m >= (1 << MINORBITS)) { 1074 idr_remove(&_minor_idr, m); 1075 r = -ENOSPC; 1076 goto out; 1077 } 1078 1079 *minor = m; 1080 1081 out: 1082 spin_unlock(&_minor_lock); 1083 return r; 1084 } 1085 1086 static struct block_device_operations dm_blk_dops; 1087 1088 static void dm_wq_work(struct work_struct *work); 1089 1090 /* 1091 * Allocate and initialise a blank device with a given minor. 1092 */ 1093 static struct mapped_device *alloc_dev(int minor) 1094 { 1095 int r; 1096 struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL); 1097 void *old_md; 1098 1099 if (!md) { 1100 DMWARN("unable to allocate device, out of memory."); 1101 return NULL; 1102 } 1103 1104 if (!try_module_get(THIS_MODULE)) 1105 goto bad_module_get; 1106 1107 /* get a minor number for the dev */ 1108 if (minor == DM_ANY_MINOR) 1109 r = next_free_minor(&minor); 1110 else 1111 r = specific_minor(minor); 1112 if (r < 0) 1113 goto bad_minor; 1114 1115 init_rwsem(&md->io_lock); 1116 mutex_init(&md->suspend_lock); 1117 spin_lock_init(&md->deferred_lock); 1118 rwlock_init(&md->map_lock); 1119 atomic_set(&md->holders, 1); 1120 atomic_set(&md->open_count, 0); 1121 atomic_set(&md->event_nr, 0); 1122 atomic_set(&md->uevent_seq, 0); 1123 INIT_LIST_HEAD(&md->uevent_list); 1124 spin_lock_init(&md->uevent_lock); 1125 1126 md->queue = blk_alloc_queue(GFP_KERNEL); 1127 if (!md->queue) 1128 goto bad_queue; 1129 1130 md->queue->queuedata = md; 1131 md->queue->backing_dev_info.congested_fn = dm_any_congested; 1132 md->queue->backing_dev_info.congested_data = md; 1133 blk_queue_make_request(md->queue, dm_request); 1134 blk_queue_ordered(md->queue, QUEUE_ORDERED_DRAIN, NULL); 1135 blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY); 1136 md->queue->unplug_fn = dm_unplug_all; 1137 blk_queue_merge_bvec(md->queue, dm_merge_bvec); 1138 1139 md->io_pool = mempool_create_slab_pool(MIN_IOS, _io_cache); 1140 if (!md->io_pool) 1141 goto bad_io_pool; 1142 1143 md->tio_pool = mempool_create_slab_pool(MIN_IOS, _tio_cache); 1144 if (!md->tio_pool) 1145 goto bad_tio_pool; 1146 1147 md->bs = bioset_create(16, 0); 1148 if (!md->bs) 1149 goto bad_no_bioset; 1150 1151 md->disk = alloc_disk(1); 1152 if (!md->disk) 1153 goto bad_disk; 1154 1155 atomic_set(&md->pending, 0); 1156 init_waitqueue_head(&md->wait); 1157 INIT_WORK(&md->work, dm_wq_work); 1158 init_waitqueue_head(&md->eventq); 1159 1160 md->disk->major = _major; 1161 md->disk->first_minor = minor; 1162 md->disk->fops = &dm_blk_dops; 1163 md->disk->queue = md->queue; 1164 md->disk->private_data = md; 1165 sprintf(md->disk->disk_name, "dm-%d", minor); 1166 add_disk(md->disk); 1167 format_dev_t(md->name, MKDEV(_major, minor)); 1168 1169 md->wq = create_singlethread_workqueue("kdmflush"); 1170 if (!md->wq) 1171 goto bad_thread; 1172 1173 /* Populate the mapping, nobody knows we exist yet */ 1174 spin_lock(&_minor_lock); 1175 old_md = idr_replace(&_minor_idr, md, minor); 1176 spin_unlock(&_minor_lock); 1177 1178 BUG_ON(old_md != MINOR_ALLOCED); 1179 1180 return md; 1181 1182 bad_thread: 1183 put_disk(md->disk); 1184 bad_disk: 1185 bioset_free(md->bs); 1186 bad_no_bioset: 1187 mempool_destroy(md->tio_pool); 1188 bad_tio_pool: 1189 mempool_destroy(md->io_pool); 1190 bad_io_pool: 1191 blk_cleanup_queue(md->queue); 1192 bad_queue: 1193 free_minor(minor); 1194 bad_minor: 1195 module_put(THIS_MODULE); 1196 bad_module_get: 1197 kfree(md); 1198 return NULL; 1199 } 1200 1201 static void unlock_fs(struct mapped_device *md); 1202 1203 static void free_dev(struct mapped_device *md) 1204 { 1205 int minor = MINOR(disk_devt(md->disk)); 1206 1207 if (md->suspended_bdev) { 1208 unlock_fs(md); 1209 bdput(md->suspended_bdev); 1210 } 1211 destroy_workqueue(md->wq); 1212 mempool_destroy(md->tio_pool); 1213 mempool_destroy(md->io_pool); 1214 bioset_free(md->bs); 1215 blk_integrity_unregister(md->disk); 1216 del_gendisk(md->disk); 1217 free_minor(minor); 1218 1219 spin_lock(&_minor_lock); 1220 md->disk->private_data = NULL; 1221 spin_unlock(&_minor_lock); 1222 1223 put_disk(md->disk); 1224 blk_cleanup_queue(md->queue); 1225 module_put(THIS_MODULE); 1226 kfree(md); 1227 } 1228 1229 /* 1230 * Bind a table to the device. 1231 */ 1232 static void event_callback(void *context) 1233 { 1234 unsigned long flags; 1235 LIST_HEAD(uevents); 1236 struct mapped_device *md = (struct mapped_device *) context; 1237 1238 spin_lock_irqsave(&md->uevent_lock, flags); 1239 list_splice_init(&md->uevent_list, &uevents); 1240 spin_unlock_irqrestore(&md->uevent_lock, flags); 1241 1242 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj); 1243 1244 atomic_inc(&md->event_nr); 1245 wake_up(&md->eventq); 1246 } 1247 1248 static void __set_size(struct mapped_device *md, sector_t size) 1249 { 1250 set_capacity(md->disk, size); 1251 1252 mutex_lock(&md->suspended_bdev->bd_inode->i_mutex); 1253 i_size_write(md->suspended_bdev->bd_inode, (loff_t)size << SECTOR_SHIFT); 1254 mutex_unlock(&md->suspended_bdev->bd_inode->i_mutex); 1255 } 1256 1257 static int __bind(struct mapped_device *md, struct dm_table *t) 1258 { 1259 struct request_queue *q = md->queue; 1260 sector_t size; 1261 1262 size = dm_table_get_size(t); 1263 1264 /* 1265 * Wipe any geometry if the size of the table changed. 1266 */ 1267 if (size != get_capacity(md->disk)) 1268 memset(&md->geometry, 0, sizeof(md->geometry)); 1269 1270 if (md->suspended_bdev) 1271 __set_size(md, size); 1272 1273 if (!size) { 1274 dm_table_destroy(t); 1275 return 0; 1276 } 1277 1278 dm_table_event_callback(t, event_callback, md); 1279 1280 write_lock(&md->map_lock); 1281 md->map = t; 1282 dm_table_set_restrictions(t, q); 1283 write_unlock(&md->map_lock); 1284 1285 return 0; 1286 } 1287 1288 static void __unbind(struct mapped_device *md) 1289 { 1290 struct dm_table *map = md->map; 1291 1292 if (!map) 1293 return; 1294 1295 dm_table_event_callback(map, NULL, NULL); 1296 write_lock(&md->map_lock); 1297 md->map = NULL; 1298 write_unlock(&md->map_lock); 1299 dm_table_destroy(map); 1300 } 1301 1302 /* 1303 * Constructor for a new device. 1304 */ 1305 int dm_create(int minor, struct mapped_device **result) 1306 { 1307 struct mapped_device *md; 1308 1309 md = alloc_dev(minor); 1310 if (!md) 1311 return -ENXIO; 1312 1313 dm_sysfs_init(md); 1314 1315 *result = md; 1316 return 0; 1317 } 1318 1319 static struct mapped_device *dm_find_md(dev_t dev) 1320 { 1321 struct mapped_device *md; 1322 unsigned minor = MINOR(dev); 1323 1324 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS)) 1325 return NULL; 1326 1327 spin_lock(&_minor_lock); 1328 1329 md = idr_find(&_minor_idr, minor); 1330 if (md && (md == MINOR_ALLOCED || 1331 (MINOR(disk_devt(dm_disk(md))) != minor) || 1332 test_bit(DMF_FREEING, &md->flags))) { 1333 md = NULL; 1334 goto out; 1335 } 1336 1337 out: 1338 spin_unlock(&_minor_lock); 1339 1340 return md; 1341 } 1342 1343 struct mapped_device *dm_get_md(dev_t dev) 1344 { 1345 struct mapped_device *md = dm_find_md(dev); 1346 1347 if (md) 1348 dm_get(md); 1349 1350 return md; 1351 } 1352 1353 void *dm_get_mdptr(struct mapped_device *md) 1354 { 1355 return md->interface_ptr; 1356 } 1357 1358 void dm_set_mdptr(struct mapped_device *md, void *ptr) 1359 { 1360 md->interface_ptr = ptr; 1361 } 1362 1363 void dm_get(struct mapped_device *md) 1364 { 1365 atomic_inc(&md->holders); 1366 } 1367 1368 const char *dm_device_name(struct mapped_device *md) 1369 { 1370 return md->name; 1371 } 1372 EXPORT_SYMBOL_GPL(dm_device_name); 1373 1374 void dm_put(struct mapped_device *md) 1375 { 1376 struct dm_table *map; 1377 1378 BUG_ON(test_bit(DMF_FREEING, &md->flags)); 1379 1380 if (atomic_dec_and_lock(&md->holders, &_minor_lock)) { 1381 map = dm_get_table(md); 1382 idr_replace(&_minor_idr, MINOR_ALLOCED, 1383 MINOR(disk_devt(dm_disk(md)))); 1384 set_bit(DMF_FREEING, &md->flags); 1385 spin_unlock(&_minor_lock); 1386 if (!dm_suspended(md)) { 1387 dm_table_presuspend_targets(map); 1388 dm_table_postsuspend_targets(map); 1389 } 1390 dm_sysfs_exit(md); 1391 dm_table_put(map); 1392 __unbind(md); 1393 free_dev(md); 1394 } 1395 } 1396 EXPORT_SYMBOL_GPL(dm_put); 1397 1398 static int dm_wait_for_completion(struct mapped_device *md, int interruptible) 1399 { 1400 int r = 0; 1401 DECLARE_WAITQUEUE(wait, current); 1402 1403 dm_unplug_all(md->queue); 1404 1405 add_wait_queue(&md->wait, &wait); 1406 1407 while (1) { 1408 set_current_state(interruptible); 1409 1410 smp_mb(); 1411 if (!atomic_read(&md->pending)) 1412 break; 1413 1414 if (interruptible == TASK_INTERRUPTIBLE && 1415 signal_pending(current)) { 1416 r = -EINTR; 1417 break; 1418 } 1419 1420 io_schedule(); 1421 } 1422 set_current_state(TASK_RUNNING); 1423 1424 remove_wait_queue(&md->wait, &wait); 1425 1426 return r; 1427 } 1428 1429 /* 1430 * Process the deferred bios 1431 */ 1432 static void dm_wq_work(struct work_struct *work) 1433 { 1434 struct mapped_device *md = container_of(work, struct mapped_device, 1435 work); 1436 struct bio *c; 1437 1438 down_write(&md->io_lock); 1439 1440 next_bio: 1441 spin_lock_irq(&md->deferred_lock); 1442 c = bio_list_pop(&md->deferred); 1443 spin_unlock_irq(&md->deferred_lock); 1444 1445 if (c) { 1446 __split_and_process_bio(md, c); 1447 goto next_bio; 1448 } 1449 1450 clear_bit(DMF_BLOCK_IO, &md->flags); 1451 1452 up_write(&md->io_lock); 1453 } 1454 1455 static void dm_queue_flush(struct mapped_device *md) 1456 { 1457 queue_work(md->wq, &md->work); 1458 flush_workqueue(md->wq); 1459 } 1460 1461 /* 1462 * Swap in a new table (destroying old one). 1463 */ 1464 int dm_swap_table(struct mapped_device *md, struct dm_table *table) 1465 { 1466 int r = -EINVAL; 1467 1468 mutex_lock(&md->suspend_lock); 1469 1470 /* device must be suspended */ 1471 if (!dm_suspended(md)) 1472 goto out; 1473 1474 /* without bdev, the device size cannot be changed */ 1475 if (!md->suspended_bdev) 1476 if (get_capacity(md->disk) != dm_table_get_size(table)) 1477 goto out; 1478 1479 __unbind(md); 1480 r = __bind(md, table); 1481 1482 out: 1483 mutex_unlock(&md->suspend_lock); 1484 return r; 1485 } 1486 1487 /* 1488 * Functions to lock and unlock any filesystem running on the 1489 * device. 1490 */ 1491 static int lock_fs(struct mapped_device *md) 1492 { 1493 int r; 1494 1495 WARN_ON(md->frozen_sb); 1496 1497 md->frozen_sb = freeze_bdev(md->suspended_bdev); 1498 if (IS_ERR(md->frozen_sb)) { 1499 r = PTR_ERR(md->frozen_sb); 1500 md->frozen_sb = NULL; 1501 return r; 1502 } 1503 1504 set_bit(DMF_FROZEN, &md->flags); 1505 1506 /* don't bdput right now, we don't want the bdev 1507 * to go away while it is locked. 1508 */ 1509 return 0; 1510 } 1511 1512 static void unlock_fs(struct mapped_device *md) 1513 { 1514 if (!test_bit(DMF_FROZEN, &md->flags)) 1515 return; 1516 1517 thaw_bdev(md->suspended_bdev, md->frozen_sb); 1518 md->frozen_sb = NULL; 1519 clear_bit(DMF_FROZEN, &md->flags); 1520 } 1521 1522 /* 1523 * We need to be able to change a mapping table under a mounted 1524 * filesystem. For example we might want to move some data in 1525 * the background. Before the table can be swapped with 1526 * dm_bind_table, dm_suspend must be called to flush any in 1527 * flight bios and ensure that any further io gets deferred. 1528 */ 1529 int dm_suspend(struct mapped_device *md, unsigned suspend_flags) 1530 { 1531 struct dm_table *map = NULL; 1532 int r = 0; 1533 int do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG ? 1 : 0; 1534 int noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG ? 1 : 0; 1535 1536 mutex_lock(&md->suspend_lock); 1537 1538 if (dm_suspended(md)) { 1539 r = -EINVAL; 1540 goto out_unlock; 1541 } 1542 1543 map = dm_get_table(md); 1544 1545 /* 1546 * DMF_NOFLUSH_SUSPENDING must be set before presuspend. 1547 * This flag is cleared before dm_suspend returns. 1548 */ 1549 if (noflush) 1550 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 1551 1552 /* This does not get reverted if there's an error later. */ 1553 dm_table_presuspend_targets(map); 1554 1555 /* bdget() can stall if the pending I/Os are not flushed */ 1556 if (!noflush) { 1557 md->suspended_bdev = bdget_disk(md->disk, 0); 1558 if (!md->suspended_bdev) { 1559 DMWARN("bdget failed in dm_suspend"); 1560 r = -ENOMEM; 1561 goto out; 1562 } 1563 1564 /* 1565 * Flush I/O to the device. noflush supersedes do_lockfs, 1566 * because lock_fs() needs to flush I/Os. 1567 */ 1568 if (do_lockfs) { 1569 r = lock_fs(md); 1570 if (r) 1571 goto out; 1572 } 1573 } 1574 1575 /* 1576 * First we set the BLOCK_IO flag so no more ios will be mapped. 1577 */ 1578 down_write(&md->io_lock); 1579 set_bit(DMF_BLOCK_IO, &md->flags); 1580 1581 up_write(&md->io_lock); 1582 1583 /* 1584 * Wait for the already-mapped ios to complete. 1585 */ 1586 r = dm_wait_for_completion(md, TASK_INTERRUPTIBLE); 1587 1588 down_write(&md->io_lock); 1589 1590 if (noflush) 1591 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 1592 up_write(&md->io_lock); 1593 1594 /* were we interrupted ? */ 1595 if (r < 0) { 1596 dm_queue_flush(md); 1597 1598 unlock_fs(md); 1599 goto out; /* pushback list is already flushed, so skip flush */ 1600 } 1601 1602 dm_table_postsuspend_targets(map); 1603 1604 set_bit(DMF_SUSPENDED, &md->flags); 1605 1606 out: 1607 if (r && md->suspended_bdev) { 1608 bdput(md->suspended_bdev); 1609 md->suspended_bdev = NULL; 1610 } 1611 1612 dm_table_put(map); 1613 1614 out_unlock: 1615 mutex_unlock(&md->suspend_lock); 1616 return r; 1617 } 1618 1619 int dm_resume(struct mapped_device *md) 1620 { 1621 int r = -EINVAL; 1622 struct dm_table *map = NULL; 1623 1624 mutex_lock(&md->suspend_lock); 1625 if (!dm_suspended(md)) 1626 goto out; 1627 1628 map = dm_get_table(md); 1629 if (!map || !dm_table_get_size(map)) 1630 goto out; 1631 1632 r = dm_table_resume_targets(map); 1633 if (r) 1634 goto out; 1635 1636 dm_queue_flush(md); 1637 1638 unlock_fs(md); 1639 1640 if (md->suspended_bdev) { 1641 bdput(md->suspended_bdev); 1642 md->suspended_bdev = NULL; 1643 } 1644 1645 clear_bit(DMF_SUSPENDED, &md->flags); 1646 1647 dm_table_unplug_all(map); 1648 1649 dm_kobject_uevent(md); 1650 1651 r = 0; 1652 1653 out: 1654 dm_table_put(map); 1655 mutex_unlock(&md->suspend_lock); 1656 1657 return r; 1658 } 1659 1660 /*----------------------------------------------------------------- 1661 * Event notification. 1662 *---------------------------------------------------------------*/ 1663 void dm_kobject_uevent(struct mapped_device *md) 1664 { 1665 kobject_uevent(&disk_to_dev(md->disk)->kobj, KOBJ_CHANGE); 1666 } 1667 1668 uint32_t dm_next_uevent_seq(struct mapped_device *md) 1669 { 1670 return atomic_add_return(1, &md->uevent_seq); 1671 } 1672 1673 uint32_t dm_get_event_nr(struct mapped_device *md) 1674 { 1675 return atomic_read(&md->event_nr); 1676 } 1677 1678 int dm_wait_event(struct mapped_device *md, int event_nr) 1679 { 1680 return wait_event_interruptible(md->eventq, 1681 (event_nr != atomic_read(&md->event_nr))); 1682 } 1683 1684 void dm_uevent_add(struct mapped_device *md, struct list_head *elist) 1685 { 1686 unsigned long flags; 1687 1688 spin_lock_irqsave(&md->uevent_lock, flags); 1689 list_add(elist, &md->uevent_list); 1690 spin_unlock_irqrestore(&md->uevent_lock, flags); 1691 } 1692 1693 /* 1694 * The gendisk is only valid as long as you have a reference 1695 * count on 'md'. 1696 */ 1697 struct gendisk *dm_disk(struct mapped_device *md) 1698 { 1699 return md->disk; 1700 } 1701 1702 struct kobject *dm_kobject(struct mapped_device *md) 1703 { 1704 return &md->kobj; 1705 } 1706 1707 /* 1708 * struct mapped_device should not be exported outside of dm.c 1709 * so use this check to verify that kobj is part of md structure 1710 */ 1711 struct mapped_device *dm_get_from_kobject(struct kobject *kobj) 1712 { 1713 struct mapped_device *md; 1714 1715 md = container_of(kobj, struct mapped_device, kobj); 1716 if (&md->kobj != kobj) 1717 return NULL; 1718 1719 dm_get(md); 1720 return md; 1721 } 1722 1723 int dm_suspended(struct mapped_device *md) 1724 { 1725 return test_bit(DMF_SUSPENDED, &md->flags); 1726 } 1727 1728 int dm_noflush_suspending(struct dm_target *ti) 1729 { 1730 struct mapped_device *md = dm_table_get_md(ti->table); 1731 int r = __noflush_suspending(md); 1732 1733 dm_put(md); 1734 1735 return r; 1736 } 1737 EXPORT_SYMBOL_GPL(dm_noflush_suspending); 1738 1739 static struct block_device_operations dm_blk_dops = { 1740 .open = dm_blk_open, 1741 .release = dm_blk_close, 1742 .ioctl = dm_blk_ioctl, 1743 .getgeo = dm_blk_getgeo, 1744 .owner = THIS_MODULE 1745 }; 1746 1747 EXPORT_SYMBOL(dm_get_mapinfo); 1748 1749 /* 1750 * module hooks 1751 */ 1752 module_init(dm_init); 1753 module_exit(dm_exit); 1754 1755 module_param(major, uint, 0); 1756 MODULE_PARM_DESC(major, "The major number of the device mapper"); 1757 MODULE_DESCRIPTION(DM_NAME " driver"); 1758 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>"); 1759 MODULE_LICENSE("GPL"); 1760