1 /* 2 * Copyright (C) 2011-2012 Red Hat UK. 3 * 4 * This file is released under the GPL. 5 */ 6 7 #include "dm-thin-metadata.h" 8 #include "dm-bio-prison.h" 9 #include "dm.h" 10 11 #include <linux/device-mapper.h> 12 #include <linux/dm-io.h> 13 #include <linux/dm-kcopyd.h> 14 #include <linux/jiffies.h> 15 #include <linux/log2.h> 16 #include <linux/list.h> 17 #include <linux/rculist.h> 18 #include <linux/init.h> 19 #include <linux/module.h> 20 #include <linux/slab.h> 21 #include <linux/vmalloc.h> 22 #include <linux/sort.h> 23 #include <linux/rbtree.h> 24 25 #define DM_MSG_PREFIX "thin" 26 27 /* 28 * Tunable constants 29 */ 30 #define ENDIO_HOOK_POOL_SIZE 1024 31 #define MAPPING_POOL_SIZE 1024 32 #define COMMIT_PERIOD HZ 33 #define NO_SPACE_TIMEOUT_SECS 60 34 35 static unsigned no_space_timeout_secs = NO_SPACE_TIMEOUT_SECS; 36 37 DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(snapshot_copy_throttle, 38 "A percentage of time allocated for copy on write"); 39 40 /* 41 * The block size of the device holding pool data must be 42 * between 64KB and 1GB. 43 */ 44 #define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (64 * 1024 >> SECTOR_SHIFT) 45 #define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT) 46 47 /* 48 * Device id is restricted to 24 bits. 49 */ 50 #define MAX_DEV_ID ((1 << 24) - 1) 51 52 /* 53 * How do we handle breaking sharing of data blocks? 54 * ================================================= 55 * 56 * We use a standard copy-on-write btree to store the mappings for the 57 * devices (note I'm talking about copy-on-write of the metadata here, not 58 * the data). When you take an internal snapshot you clone the root node 59 * of the origin btree. After this there is no concept of an origin or a 60 * snapshot. They are just two device trees that happen to point to the 61 * same data blocks. 62 * 63 * When we get a write in we decide if it's to a shared data block using 64 * some timestamp magic. If it is, we have to break sharing. 65 * 66 * Let's say we write to a shared block in what was the origin. The 67 * steps are: 68 * 69 * i) plug io further to this physical block. (see bio_prison code). 70 * 71 * ii) quiesce any read io to that shared data block. Obviously 72 * including all devices that share this block. (see dm_deferred_set code) 73 * 74 * iii) copy the data block to a newly allocate block. This step can be 75 * missed out if the io covers the block. (schedule_copy). 76 * 77 * iv) insert the new mapping into the origin's btree 78 * (process_prepared_mapping). This act of inserting breaks some 79 * sharing of btree nodes between the two devices. Breaking sharing only 80 * effects the btree of that specific device. Btrees for the other 81 * devices that share the block never change. The btree for the origin 82 * device as it was after the last commit is untouched, ie. we're using 83 * persistent data structures in the functional programming sense. 84 * 85 * v) unplug io to this physical block, including the io that triggered 86 * the breaking of sharing. 87 * 88 * Steps (ii) and (iii) occur in parallel. 89 * 90 * The metadata _doesn't_ need to be committed before the io continues. We 91 * get away with this because the io is always written to a _new_ block. 92 * If there's a crash, then: 93 * 94 * - The origin mapping will point to the old origin block (the shared 95 * one). This will contain the data as it was before the io that triggered 96 * the breaking of sharing came in. 97 * 98 * - The snap mapping still points to the old block. As it would after 99 * the commit. 100 * 101 * The downside of this scheme is the timestamp magic isn't perfect, and 102 * will continue to think that data block in the snapshot device is shared 103 * even after the write to the origin has broken sharing. I suspect data 104 * blocks will typically be shared by many different devices, so we're 105 * breaking sharing n + 1 times, rather than n, where n is the number of 106 * devices that reference this data block. At the moment I think the 107 * benefits far, far outweigh the disadvantages. 108 */ 109 110 /*----------------------------------------------------------------*/ 111 112 /* 113 * Key building. 114 */ 115 enum lock_space { 116 VIRTUAL, 117 PHYSICAL 118 }; 119 120 static void build_key(struct dm_thin_device *td, enum lock_space ls, 121 dm_block_t b, dm_block_t e, struct dm_cell_key *key) 122 { 123 key->virtual = (ls == VIRTUAL); 124 key->dev = dm_thin_dev_id(td); 125 key->block_begin = b; 126 key->block_end = e; 127 } 128 129 static void build_data_key(struct dm_thin_device *td, dm_block_t b, 130 struct dm_cell_key *key) 131 { 132 build_key(td, PHYSICAL, b, b + 1llu, key); 133 } 134 135 static void build_virtual_key(struct dm_thin_device *td, dm_block_t b, 136 struct dm_cell_key *key) 137 { 138 build_key(td, VIRTUAL, b, b + 1llu, key); 139 } 140 141 /*----------------------------------------------------------------*/ 142 143 #define THROTTLE_THRESHOLD (1 * HZ) 144 145 struct throttle { 146 struct rw_semaphore lock; 147 unsigned long threshold; 148 bool throttle_applied; 149 }; 150 151 static void throttle_init(struct throttle *t) 152 { 153 init_rwsem(&t->lock); 154 t->throttle_applied = false; 155 } 156 157 static void throttle_work_start(struct throttle *t) 158 { 159 t->threshold = jiffies + THROTTLE_THRESHOLD; 160 } 161 162 static void throttle_work_update(struct throttle *t) 163 { 164 if (!t->throttle_applied && jiffies > t->threshold) { 165 down_write(&t->lock); 166 t->throttle_applied = true; 167 } 168 } 169 170 static void throttle_work_complete(struct throttle *t) 171 { 172 if (t->throttle_applied) { 173 t->throttle_applied = false; 174 up_write(&t->lock); 175 } 176 } 177 178 static void throttle_lock(struct throttle *t) 179 { 180 down_read(&t->lock); 181 } 182 183 static void throttle_unlock(struct throttle *t) 184 { 185 up_read(&t->lock); 186 } 187 188 /*----------------------------------------------------------------*/ 189 190 /* 191 * A pool device ties together a metadata device and a data device. It 192 * also provides the interface for creating and destroying internal 193 * devices. 194 */ 195 struct dm_thin_new_mapping; 196 197 /* 198 * The pool runs in 4 modes. Ordered in degraded order for comparisons. 199 */ 200 enum pool_mode { 201 PM_WRITE, /* metadata may be changed */ 202 PM_OUT_OF_DATA_SPACE, /* metadata may be changed, though data may not be allocated */ 203 PM_READ_ONLY, /* metadata may not be changed */ 204 PM_FAIL, /* all I/O fails */ 205 }; 206 207 struct pool_features { 208 enum pool_mode mode; 209 210 bool zero_new_blocks:1; 211 bool discard_enabled:1; 212 bool discard_passdown:1; 213 bool error_if_no_space:1; 214 }; 215 216 struct thin_c; 217 typedef void (*process_bio_fn)(struct thin_c *tc, struct bio *bio); 218 typedef void (*process_cell_fn)(struct thin_c *tc, struct dm_bio_prison_cell *cell); 219 typedef void (*process_mapping_fn)(struct dm_thin_new_mapping *m); 220 221 #define CELL_SORT_ARRAY_SIZE 8192 222 223 struct pool { 224 struct list_head list; 225 struct dm_target *ti; /* Only set if a pool target is bound */ 226 227 struct mapped_device *pool_md; 228 struct block_device *md_dev; 229 struct dm_pool_metadata *pmd; 230 231 dm_block_t low_water_blocks; 232 uint32_t sectors_per_block; 233 int sectors_per_block_shift; 234 235 struct pool_features pf; 236 bool low_water_triggered:1; /* A dm event has been sent */ 237 bool suspended:1; 238 239 struct dm_bio_prison *prison; 240 struct dm_kcopyd_client *copier; 241 242 struct workqueue_struct *wq; 243 struct throttle throttle; 244 struct work_struct worker; 245 struct delayed_work waker; 246 struct delayed_work no_space_timeout; 247 248 unsigned long last_commit_jiffies; 249 unsigned ref_count; 250 251 spinlock_t lock; 252 struct bio_list deferred_flush_bios; 253 struct list_head prepared_mappings; 254 struct list_head prepared_discards; 255 struct list_head active_thins; 256 257 struct dm_deferred_set *shared_read_ds; 258 struct dm_deferred_set *all_io_ds; 259 260 struct dm_thin_new_mapping *next_mapping; 261 mempool_t *mapping_pool; 262 263 process_bio_fn process_bio; 264 process_bio_fn process_discard; 265 266 process_cell_fn process_cell; 267 process_cell_fn process_discard_cell; 268 269 process_mapping_fn process_prepared_mapping; 270 process_mapping_fn process_prepared_discard; 271 272 struct dm_bio_prison_cell **cell_sort_array; 273 }; 274 275 static enum pool_mode get_pool_mode(struct pool *pool); 276 static void metadata_operation_failed(struct pool *pool, const char *op, int r); 277 278 /* 279 * Target context for a pool. 280 */ 281 struct pool_c { 282 struct dm_target *ti; 283 struct pool *pool; 284 struct dm_dev *data_dev; 285 struct dm_dev *metadata_dev; 286 struct dm_target_callbacks callbacks; 287 288 dm_block_t low_water_blocks; 289 struct pool_features requested_pf; /* Features requested during table load */ 290 struct pool_features adjusted_pf; /* Features used after adjusting for constituent devices */ 291 }; 292 293 /* 294 * Target context for a thin. 295 */ 296 struct thin_c { 297 struct list_head list; 298 struct dm_dev *pool_dev; 299 struct dm_dev *origin_dev; 300 sector_t origin_size; 301 dm_thin_id dev_id; 302 303 struct pool *pool; 304 struct dm_thin_device *td; 305 struct mapped_device *thin_md; 306 307 bool requeue_mode:1; 308 spinlock_t lock; 309 struct list_head deferred_cells; 310 struct bio_list deferred_bio_list; 311 struct bio_list retry_on_resume_list; 312 struct rb_root sort_bio_list; /* sorted list of deferred bios */ 313 314 /* 315 * Ensures the thin is not destroyed until the worker has finished 316 * iterating the active_thins list. 317 */ 318 atomic_t refcount; 319 struct completion can_destroy; 320 }; 321 322 /*----------------------------------------------------------------*/ 323 324 /** 325 * __blkdev_issue_discard_async - queue a discard with async completion 326 * @bdev: blockdev to issue discard for 327 * @sector: start sector 328 * @nr_sects: number of sectors to discard 329 * @gfp_mask: memory allocation flags (for bio_alloc) 330 * @flags: BLKDEV_IFL_* flags to control behaviour 331 * @parent_bio: parent discard bio that all sub discards get chained to 332 * 333 * Description: 334 * Asynchronously issue a discard request for the sectors in question. 335 * NOTE: this variant of blk-core's blkdev_issue_discard() is a stop-gap 336 * that is being kept local to DM thinp until the block changes to allow 337 * late bio splitting land upstream. 338 */ 339 static int __blkdev_issue_discard_async(struct block_device *bdev, sector_t sector, 340 sector_t nr_sects, gfp_t gfp_mask, unsigned long flags, 341 struct bio *parent_bio) 342 { 343 struct request_queue *q = bdev_get_queue(bdev); 344 int type = REQ_WRITE | REQ_DISCARD; 345 unsigned int max_discard_sectors, granularity; 346 int alignment; 347 struct bio *bio; 348 int ret = 0; 349 struct blk_plug plug; 350 351 if (!q) 352 return -ENXIO; 353 354 if (!blk_queue_discard(q)) 355 return -EOPNOTSUPP; 356 357 /* Zero-sector (unknown) and one-sector granularities are the same. */ 358 granularity = max(q->limits.discard_granularity >> 9, 1U); 359 alignment = (bdev_discard_alignment(bdev) >> 9) % granularity; 360 361 /* 362 * Ensure that max_discard_sectors is of the proper 363 * granularity, so that requests stay aligned after a split. 364 */ 365 max_discard_sectors = min(q->limits.max_discard_sectors, UINT_MAX >> 9); 366 max_discard_sectors -= max_discard_sectors % granularity; 367 if (unlikely(!max_discard_sectors)) { 368 /* Avoid infinite loop below. Being cautious never hurts. */ 369 return -EOPNOTSUPP; 370 } 371 372 if (flags & BLKDEV_DISCARD_SECURE) { 373 if (!blk_queue_secdiscard(q)) 374 return -EOPNOTSUPP; 375 type |= REQ_SECURE; 376 } 377 378 blk_start_plug(&plug); 379 while (nr_sects) { 380 unsigned int req_sects; 381 sector_t end_sect, tmp; 382 383 /* 384 * Required bio_put occurs in bio_endio thanks to bio_chain below 385 */ 386 bio = bio_alloc(gfp_mask, 1); 387 if (!bio) { 388 ret = -ENOMEM; 389 break; 390 } 391 392 req_sects = min_t(sector_t, nr_sects, max_discard_sectors); 393 394 /* 395 * If splitting a request, and the next starting sector would be 396 * misaligned, stop the discard at the previous aligned sector. 397 */ 398 end_sect = sector + req_sects; 399 tmp = end_sect; 400 if (req_sects < nr_sects && 401 sector_div(tmp, granularity) != alignment) { 402 end_sect = end_sect - alignment; 403 sector_div(end_sect, granularity); 404 end_sect = end_sect * granularity + alignment; 405 req_sects = end_sect - sector; 406 } 407 408 bio_chain(bio, parent_bio); 409 410 bio->bi_iter.bi_sector = sector; 411 bio->bi_bdev = bdev; 412 413 bio->bi_iter.bi_size = req_sects << 9; 414 nr_sects -= req_sects; 415 sector = end_sect; 416 417 submit_bio(type, bio); 418 419 /* 420 * We can loop for a long time in here, if someone does 421 * full device discards (like mkfs). Be nice and allow 422 * us to schedule out to avoid softlocking if preempt 423 * is disabled. 424 */ 425 cond_resched(); 426 } 427 blk_finish_plug(&plug); 428 429 return ret; 430 } 431 432 static bool block_size_is_power_of_two(struct pool *pool) 433 { 434 return pool->sectors_per_block_shift >= 0; 435 } 436 437 static sector_t block_to_sectors(struct pool *pool, dm_block_t b) 438 { 439 return block_size_is_power_of_two(pool) ? 440 (b << pool->sectors_per_block_shift) : 441 (b * pool->sectors_per_block); 442 } 443 444 static int issue_discard(struct thin_c *tc, dm_block_t data_b, dm_block_t data_e, 445 struct bio *parent_bio) 446 { 447 sector_t s = block_to_sectors(tc->pool, data_b); 448 sector_t len = block_to_sectors(tc->pool, data_e - data_b); 449 450 return __blkdev_issue_discard_async(tc->pool_dev->bdev, s, len, 451 GFP_NOWAIT, 0, parent_bio); 452 } 453 454 /*----------------------------------------------------------------*/ 455 456 /* 457 * wake_worker() is used when new work is queued and when pool_resume is 458 * ready to continue deferred IO processing. 459 */ 460 static void wake_worker(struct pool *pool) 461 { 462 queue_work(pool->wq, &pool->worker); 463 } 464 465 /*----------------------------------------------------------------*/ 466 467 static int bio_detain(struct pool *pool, struct dm_cell_key *key, struct bio *bio, 468 struct dm_bio_prison_cell **cell_result) 469 { 470 int r; 471 struct dm_bio_prison_cell *cell_prealloc; 472 473 /* 474 * Allocate a cell from the prison's mempool. 475 * This might block but it can't fail. 476 */ 477 cell_prealloc = dm_bio_prison_alloc_cell(pool->prison, GFP_NOIO); 478 479 r = dm_bio_detain(pool->prison, key, bio, cell_prealloc, cell_result); 480 if (r) 481 /* 482 * We reused an old cell; we can get rid of 483 * the new one. 484 */ 485 dm_bio_prison_free_cell(pool->prison, cell_prealloc); 486 487 return r; 488 } 489 490 static void cell_release(struct pool *pool, 491 struct dm_bio_prison_cell *cell, 492 struct bio_list *bios) 493 { 494 dm_cell_release(pool->prison, cell, bios); 495 dm_bio_prison_free_cell(pool->prison, cell); 496 } 497 498 static void cell_visit_release(struct pool *pool, 499 void (*fn)(void *, struct dm_bio_prison_cell *), 500 void *context, 501 struct dm_bio_prison_cell *cell) 502 { 503 dm_cell_visit_release(pool->prison, fn, context, cell); 504 dm_bio_prison_free_cell(pool->prison, cell); 505 } 506 507 static void cell_release_no_holder(struct pool *pool, 508 struct dm_bio_prison_cell *cell, 509 struct bio_list *bios) 510 { 511 dm_cell_release_no_holder(pool->prison, cell, bios); 512 dm_bio_prison_free_cell(pool->prison, cell); 513 } 514 515 static void cell_error_with_code(struct pool *pool, 516 struct dm_bio_prison_cell *cell, int error_code) 517 { 518 dm_cell_error(pool->prison, cell, error_code); 519 dm_bio_prison_free_cell(pool->prison, cell); 520 } 521 522 static void cell_error(struct pool *pool, struct dm_bio_prison_cell *cell) 523 { 524 cell_error_with_code(pool, cell, -EIO); 525 } 526 527 static void cell_success(struct pool *pool, struct dm_bio_prison_cell *cell) 528 { 529 cell_error_with_code(pool, cell, 0); 530 } 531 532 static void cell_requeue(struct pool *pool, struct dm_bio_prison_cell *cell) 533 { 534 cell_error_with_code(pool, cell, DM_ENDIO_REQUEUE); 535 } 536 537 /*----------------------------------------------------------------*/ 538 539 /* 540 * A global list of pools that uses a struct mapped_device as a key. 541 */ 542 static struct dm_thin_pool_table { 543 struct mutex mutex; 544 struct list_head pools; 545 } dm_thin_pool_table; 546 547 static void pool_table_init(void) 548 { 549 mutex_init(&dm_thin_pool_table.mutex); 550 INIT_LIST_HEAD(&dm_thin_pool_table.pools); 551 } 552 553 static void __pool_table_insert(struct pool *pool) 554 { 555 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex)); 556 list_add(&pool->list, &dm_thin_pool_table.pools); 557 } 558 559 static void __pool_table_remove(struct pool *pool) 560 { 561 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex)); 562 list_del(&pool->list); 563 } 564 565 static struct pool *__pool_table_lookup(struct mapped_device *md) 566 { 567 struct pool *pool = NULL, *tmp; 568 569 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex)); 570 571 list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) { 572 if (tmp->pool_md == md) { 573 pool = tmp; 574 break; 575 } 576 } 577 578 return pool; 579 } 580 581 static struct pool *__pool_table_lookup_metadata_dev(struct block_device *md_dev) 582 { 583 struct pool *pool = NULL, *tmp; 584 585 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex)); 586 587 list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) { 588 if (tmp->md_dev == md_dev) { 589 pool = tmp; 590 break; 591 } 592 } 593 594 return pool; 595 } 596 597 /*----------------------------------------------------------------*/ 598 599 struct dm_thin_endio_hook { 600 struct thin_c *tc; 601 struct dm_deferred_entry *shared_read_entry; 602 struct dm_deferred_entry *all_io_entry; 603 struct dm_thin_new_mapping *overwrite_mapping; 604 struct rb_node rb_node; 605 struct dm_bio_prison_cell *cell; 606 }; 607 608 static void __merge_bio_list(struct bio_list *bios, struct bio_list *master) 609 { 610 bio_list_merge(bios, master); 611 bio_list_init(master); 612 } 613 614 static void error_bio_list(struct bio_list *bios, int error) 615 { 616 struct bio *bio; 617 618 while ((bio = bio_list_pop(bios))) 619 bio_endio(bio, error); 620 } 621 622 static void error_thin_bio_list(struct thin_c *tc, struct bio_list *master, int error) 623 { 624 struct bio_list bios; 625 unsigned long flags; 626 627 bio_list_init(&bios); 628 629 spin_lock_irqsave(&tc->lock, flags); 630 __merge_bio_list(&bios, master); 631 spin_unlock_irqrestore(&tc->lock, flags); 632 633 error_bio_list(&bios, error); 634 } 635 636 static void requeue_deferred_cells(struct thin_c *tc) 637 { 638 struct pool *pool = tc->pool; 639 unsigned long flags; 640 struct list_head cells; 641 struct dm_bio_prison_cell *cell, *tmp; 642 643 INIT_LIST_HEAD(&cells); 644 645 spin_lock_irqsave(&tc->lock, flags); 646 list_splice_init(&tc->deferred_cells, &cells); 647 spin_unlock_irqrestore(&tc->lock, flags); 648 649 list_for_each_entry_safe(cell, tmp, &cells, user_list) 650 cell_requeue(pool, cell); 651 } 652 653 static void requeue_io(struct thin_c *tc) 654 { 655 struct bio_list bios; 656 unsigned long flags; 657 658 bio_list_init(&bios); 659 660 spin_lock_irqsave(&tc->lock, flags); 661 __merge_bio_list(&bios, &tc->deferred_bio_list); 662 __merge_bio_list(&bios, &tc->retry_on_resume_list); 663 spin_unlock_irqrestore(&tc->lock, flags); 664 665 error_bio_list(&bios, DM_ENDIO_REQUEUE); 666 requeue_deferred_cells(tc); 667 } 668 669 static void error_retry_list(struct pool *pool) 670 { 671 struct thin_c *tc; 672 673 rcu_read_lock(); 674 list_for_each_entry_rcu(tc, &pool->active_thins, list) 675 error_thin_bio_list(tc, &tc->retry_on_resume_list, -EIO); 676 rcu_read_unlock(); 677 } 678 679 /* 680 * This section of code contains the logic for processing a thin device's IO. 681 * Much of the code depends on pool object resources (lists, workqueues, etc) 682 * but most is exclusively called from the thin target rather than the thin-pool 683 * target. 684 */ 685 686 static dm_block_t get_bio_block(struct thin_c *tc, struct bio *bio) 687 { 688 struct pool *pool = tc->pool; 689 sector_t block_nr = bio->bi_iter.bi_sector; 690 691 if (block_size_is_power_of_two(pool)) 692 block_nr >>= pool->sectors_per_block_shift; 693 else 694 (void) sector_div(block_nr, pool->sectors_per_block); 695 696 return block_nr; 697 } 698 699 /* 700 * Returns the _complete_ blocks that this bio covers. 701 */ 702 static void get_bio_block_range(struct thin_c *tc, struct bio *bio, 703 dm_block_t *begin, dm_block_t *end) 704 { 705 struct pool *pool = tc->pool; 706 sector_t b = bio->bi_iter.bi_sector; 707 sector_t e = b + (bio->bi_iter.bi_size >> SECTOR_SHIFT); 708 709 b += pool->sectors_per_block - 1ull; /* so we round up */ 710 711 if (block_size_is_power_of_two(pool)) { 712 b >>= pool->sectors_per_block_shift; 713 e >>= pool->sectors_per_block_shift; 714 } else { 715 (void) sector_div(b, pool->sectors_per_block); 716 (void) sector_div(e, pool->sectors_per_block); 717 } 718 719 if (e < b) 720 /* Can happen if the bio is within a single block. */ 721 e = b; 722 723 *begin = b; 724 *end = e; 725 } 726 727 static void remap(struct thin_c *tc, struct bio *bio, dm_block_t block) 728 { 729 struct pool *pool = tc->pool; 730 sector_t bi_sector = bio->bi_iter.bi_sector; 731 732 bio->bi_bdev = tc->pool_dev->bdev; 733 if (block_size_is_power_of_two(pool)) 734 bio->bi_iter.bi_sector = 735 (block << pool->sectors_per_block_shift) | 736 (bi_sector & (pool->sectors_per_block - 1)); 737 else 738 bio->bi_iter.bi_sector = (block * pool->sectors_per_block) + 739 sector_div(bi_sector, pool->sectors_per_block); 740 } 741 742 static void remap_to_origin(struct thin_c *tc, struct bio *bio) 743 { 744 bio->bi_bdev = tc->origin_dev->bdev; 745 } 746 747 static int bio_triggers_commit(struct thin_c *tc, struct bio *bio) 748 { 749 return (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) && 750 dm_thin_changed_this_transaction(tc->td); 751 } 752 753 static void inc_all_io_entry(struct pool *pool, struct bio *bio) 754 { 755 struct dm_thin_endio_hook *h; 756 757 if (bio->bi_rw & REQ_DISCARD) 758 return; 759 760 h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 761 h->all_io_entry = dm_deferred_entry_inc(pool->all_io_ds); 762 } 763 764 static void issue(struct thin_c *tc, struct bio *bio) 765 { 766 struct pool *pool = tc->pool; 767 unsigned long flags; 768 769 if (!bio_triggers_commit(tc, bio)) { 770 generic_make_request(bio); 771 return; 772 } 773 774 /* 775 * Complete bio with an error if earlier I/O caused changes to 776 * the metadata that can't be committed e.g, due to I/O errors 777 * on the metadata device. 778 */ 779 if (dm_thin_aborted_changes(tc->td)) { 780 bio_io_error(bio); 781 return; 782 } 783 784 /* 785 * Batch together any bios that trigger commits and then issue a 786 * single commit for them in process_deferred_bios(). 787 */ 788 spin_lock_irqsave(&pool->lock, flags); 789 bio_list_add(&pool->deferred_flush_bios, bio); 790 spin_unlock_irqrestore(&pool->lock, flags); 791 } 792 793 static void remap_to_origin_and_issue(struct thin_c *tc, struct bio *bio) 794 { 795 remap_to_origin(tc, bio); 796 issue(tc, bio); 797 } 798 799 static void remap_and_issue(struct thin_c *tc, struct bio *bio, 800 dm_block_t block) 801 { 802 remap(tc, bio, block); 803 issue(tc, bio); 804 } 805 806 /*----------------------------------------------------------------*/ 807 808 /* 809 * Bio endio functions. 810 */ 811 struct dm_thin_new_mapping { 812 struct list_head list; 813 814 bool pass_discard:1; 815 bool maybe_shared:1; 816 817 /* 818 * Track quiescing, copying and zeroing preparation actions. When this 819 * counter hits zero the block is prepared and can be inserted into the 820 * btree. 821 */ 822 atomic_t prepare_actions; 823 824 int err; 825 struct thin_c *tc; 826 dm_block_t virt_begin, virt_end; 827 dm_block_t data_block; 828 struct dm_bio_prison_cell *cell; 829 830 /* 831 * If the bio covers the whole area of a block then we can avoid 832 * zeroing or copying. Instead this bio is hooked. The bio will 833 * still be in the cell, so care has to be taken to avoid issuing 834 * the bio twice. 835 */ 836 struct bio *bio; 837 bio_end_io_t *saved_bi_end_io; 838 }; 839 840 static void __complete_mapping_preparation(struct dm_thin_new_mapping *m) 841 { 842 struct pool *pool = m->tc->pool; 843 844 if (atomic_dec_and_test(&m->prepare_actions)) { 845 list_add_tail(&m->list, &pool->prepared_mappings); 846 wake_worker(pool); 847 } 848 } 849 850 static void complete_mapping_preparation(struct dm_thin_new_mapping *m) 851 { 852 unsigned long flags; 853 struct pool *pool = m->tc->pool; 854 855 spin_lock_irqsave(&pool->lock, flags); 856 __complete_mapping_preparation(m); 857 spin_unlock_irqrestore(&pool->lock, flags); 858 } 859 860 static void copy_complete(int read_err, unsigned long write_err, void *context) 861 { 862 struct dm_thin_new_mapping *m = context; 863 864 m->err = read_err || write_err ? -EIO : 0; 865 complete_mapping_preparation(m); 866 } 867 868 static void overwrite_endio(struct bio *bio, int err) 869 { 870 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 871 struct dm_thin_new_mapping *m = h->overwrite_mapping; 872 873 bio->bi_end_io = m->saved_bi_end_io; 874 875 m->err = err; 876 complete_mapping_preparation(m); 877 } 878 879 /*----------------------------------------------------------------*/ 880 881 /* 882 * Workqueue. 883 */ 884 885 /* 886 * Prepared mapping jobs. 887 */ 888 889 /* 890 * This sends the bios in the cell, except the original holder, back 891 * to the deferred_bios list. 892 */ 893 static void cell_defer_no_holder(struct thin_c *tc, struct dm_bio_prison_cell *cell) 894 { 895 struct pool *pool = tc->pool; 896 unsigned long flags; 897 898 spin_lock_irqsave(&tc->lock, flags); 899 cell_release_no_holder(pool, cell, &tc->deferred_bio_list); 900 spin_unlock_irqrestore(&tc->lock, flags); 901 902 wake_worker(pool); 903 } 904 905 static void thin_defer_bio(struct thin_c *tc, struct bio *bio); 906 907 struct remap_info { 908 struct thin_c *tc; 909 struct bio_list defer_bios; 910 struct bio_list issue_bios; 911 }; 912 913 static void __inc_remap_and_issue_cell(void *context, 914 struct dm_bio_prison_cell *cell) 915 { 916 struct remap_info *info = context; 917 struct bio *bio; 918 919 while ((bio = bio_list_pop(&cell->bios))) { 920 if (bio->bi_rw & (REQ_DISCARD | REQ_FLUSH | REQ_FUA)) 921 bio_list_add(&info->defer_bios, bio); 922 else { 923 inc_all_io_entry(info->tc->pool, bio); 924 925 /* 926 * We can't issue the bios with the bio prison lock 927 * held, so we add them to a list to issue on 928 * return from this function. 929 */ 930 bio_list_add(&info->issue_bios, bio); 931 } 932 } 933 } 934 935 static void inc_remap_and_issue_cell(struct thin_c *tc, 936 struct dm_bio_prison_cell *cell, 937 dm_block_t block) 938 { 939 struct bio *bio; 940 struct remap_info info; 941 942 info.tc = tc; 943 bio_list_init(&info.defer_bios); 944 bio_list_init(&info.issue_bios); 945 946 /* 947 * We have to be careful to inc any bios we're about to issue 948 * before the cell is released, and avoid a race with new bios 949 * being added to the cell. 950 */ 951 cell_visit_release(tc->pool, __inc_remap_and_issue_cell, 952 &info, cell); 953 954 while ((bio = bio_list_pop(&info.defer_bios))) 955 thin_defer_bio(tc, bio); 956 957 while ((bio = bio_list_pop(&info.issue_bios))) 958 remap_and_issue(info.tc, bio, block); 959 } 960 961 static void process_prepared_mapping_fail(struct dm_thin_new_mapping *m) 962 { 963 cell_error(m->tc->pool, m->cell); 964 list_del(&m->list); 965 mempool_free(m, m->tc->pool->mapping_pool); 966 } 967 968 static void process_prepared_mapping(struct dm_thin_new_mapping *m) 969 { 970 struct thin_c *tc = m->tc; 971 struct pool *pool = tc->pool; 972 struct bio *bio = m->bio; 973 int r; 974 975 if (m->err) { 976 cell_error(pool, m->cell); 977 goto out; 978 } 979 980 /* 981 * Commit the prepared block into the mapping btree. 982 * Any I/O for this block arriving after this point will get 983 * remapped to it directly. 984 */ 985 r = dm_thin_insert_block(tc->td, m->virt_begin, m->data_block); 986 if (r) { 987 metadata_operation_failed(pool, "dm_thin_insert_block", r); 988 cell_error(pool, m->cell); 989 goto out; 990 } 991 992 /* 993 * Release any bios held while the block was being provisioned. 994 * If we are processing a write bio that completely covers the block, 995 * we already processed it so can ignore it now when processing 996 * the bios in the cell. 997 */ 998 if (bio) { 999 inc_remap_and_issue_cell(tc, m->cell, m->data_block); 1000 bio_endio(bio, 0); 1001 } else { 1002 inc_all_io_entry(tc->pool, m->cell->holder); 1003 remap_and_issue(tc, m->cell->holder, m->data_block); 1004 inc_remap_and_issue_cell(tc, m->cell, m->data_block); 1005 } 1006 1007 out: 1008 list_del(&m->list); 1009 mempool_free(m, pool->mapping_pool); 1010 } 1011 1012 /*----------------------------------------------------------------*/ 1013 1014 static void free_discard_mapping(struct dm_thin_new_mapping *m) 1015 { 1016 struct thin_c *tc = m->tc; 1017 if (m->cell) 1018 cell_defer_no_holder(tc, m->cell); 1019 mempool_free(m, tc->pool->mapping_pool); 1020 } 1021 1022 static void process_prepared_discard_fail(struct dm_thin_new_mapping *m) 1023 { 1024 bio_io_error(m->bio); 1025 free_discard_mapping(m); 1026 } 1027 1028 static void process_prepared_discard_success(struct dm_thin_new_mapping *m) 1029 { 1030 bio_endio(m->bio, 0); 1031 free_discard_mapping(m); 1032 } 1033 1034 static void process_prepared_discard_no_passdown(struct dm_thin_new_mapping *m) 1035 { 1036 int r; 1037 struct thin_c *tc = m->tc; 1038 1039 r = dm_thin_remove_range(tc->td, m->cell->key.block_begin, m->cell->key.block_end); 1040 if (r) { 1041 metadata_operation_failed(tc->pool, "dm_thin_remove_range", r); 1042 bio_io_error(m->bio); 1043 } else 1044 bio_endio(m->bio, 0); 1045 1046 cell_defer_no_holder(tc, m->cell); 1047 mempool_free(m, tc->pool->mapping_pool); 1048 } 1049 1050 static int passdown_double_checking_shared_status(struct dm_thin_new_mapping *m) 1051 { 1052 /* 1053 * We've already unmapped this range of blocks, but before we 1054 * passdown we have to check that these blocks are now unused. 1055 */ 1056 int r; 1057 bool used = true; 1058 struct thin_c *tc = m->tc; 1059 struct pool *pool = tc->pool; 1060 dm_block_t b = m->data_block, e, end = m->data_block + m->virt_end - m->virt_begin; 1061 1062 while (b != end) { 1063 /* find start of unmapped run */ 1064 for (; b < end; b++) { 1065 r = dm_pool_block_is_used(pool->pmd, b, &used); 1066 if (r) 1067 return r; 1068 1069 if (!used) 1070 break; 1071 } 1072 1073 if (b == end) 1074 break; 1075 1076 /* find end of run */ 1077 for (e = b + 1; e != end; e++) { 1078 r = dm_pool_block_is_used(pool->pmd, e, &used); 1079 if (r) 1080 return r; 1081 1082 if (used) 1083 break; 1084 } 1085 1086 r = issue_discard(tc, b, e, m->bio); 1087 if (r) 1088 return r; 1089 1090 b = e; 1091 } 1092 1093 return 0; 1094 } 1095 1096 static void process_prepared_discard_passdown(struct dm_thin_new_mapping *m) 1097 { 1098 int r; 1099 struct thin_c *tc = m->tc; 1100 struct pool *pool = tc->pool; 1101 1102 r = dm_thin_remove_range(tc->td, m->virt_begin, m->virt_end); 1103 if (r) 1104 metadata_operation_failed(pool, "dm_thin_remove_range", r); 1105 1106 else if (m->maybe_shared) 1107 r = passdown_double_checking_shared_status(m); 1108 else 1109 r = issue_discard(tc, m->data_block, m->data_block + (m->virt_end - m->virt_begin), m->bio); 1110 1111 /* 1112 * Even if r is set, there could be sub discards in flight that we 1113 * need to wait for. 1114 */ 1115 bio_endio(m->bio, r); 1116 cell_defer_no_holder(tc, m->cell); 1117 mempool_free(m, pool->mapping_pool); 1118 } 1119 1120 static void process_prepared(struct pool *pool, struct list_head *head, 1121 process_mapping_fn *fn) 1122 { 1123 unsigned long flags; 1124 struct list_head maps; 1125 struct dm_thin_new_mapping *m, *tmp; 1126 1127 INIT_LIST_HEAD(&maps); 1128 spin_lock_irqsave(&pool->lock, flags); 1129 list_splice_init(head, &maps); 1130 spin_unlock_irqrestore(&pool->lock, flags); 1131 1132 list_for_each_entry_safe(m, tmp, &maps, list) 1133 (*fn)(m); 1134 } 1135 1136 /* 1137 * Deferred bio jobs. 1138 */ 1139 static int io_overlaps_block(struct pool *pool, struct bio *bio) 1140 { 1141 return bio->bi_iter.bi_size == 1142 (pool->sectors_per_block << SECTOR_SHIFT); 1143 } 1144 1145 static int io_overwrites_block(struct pool *pool, struct bio *bio) 1146 { 1147 return (bio_data_dir(bio) == WRITE) && 1148 io_overlaps_block(pool, bio); 1149 } 1150 1151 static void save_and_set_endio(struct bio *bio, bio_end_io_t **save, 1152 bio_end_io_t *fn) 1153 { 1154 *save = bio->bi_end_io; 1155 bio->bi_end_io = fn; 1156 } 1157 1158 static int ensure_next_mapping(struct pool *pool) 1159 { 1160 if (pool->next_mapping) 1161 return 0; 1162 1163 pool->next_mapping = mempool_alloc(pool->mapping_pool, GFP_ATOMIC); 1164 1165 return pool->next_mapping ? 0 : -ENOMEM; 1166 } 1167 1168 static struct dm_thin_new_mapping *get_next_mapping(struct pool *pool) 1169 { 1170 struct dm_thin_new_mapping *m = pool->next_mapping; 1171 1172 BUG_ON(!pool->next_mapping); 1173 1174 memset(m, 0, sizeof(struct dm_thin_new_mapping)); 1175 INIT_LIST_HEAD(&m->list); 1176 m->bio = NULL; 1177 1178 pool->next_mapping = NULL; 1179 1180 return m; 1181 } 1182 1183 static void ll_zero(struct thin_c *tc, struct dm_thin_new_mapping *m, 1184 sector_t begin, sector_t end) 1185 { 1186 int r; 1187 struct dm_io_region to; 1188 1189 to.bdev = tc->pool_dev->bdev; 1190 to.sector = begin; 1191 to.count = end - begin; 1192 1193 r = dm_kcopyd_zero(tc->pool->copier, 1, &to, 0, copy_complete, m); 1194 if (r < 0) { 1195 DMERR_LIMIT("dm_kcopyd_zero() failed"); 1196 copy_complete(1, 1, m); 1197 } 1198 } 1199 1200 static void remap_and_issue_overwrite(struct thin_c *tc, struct bio *bio, 1201 dm_block_t data_begin, 1202 struct dm_thin_new_mapping *m) 1203 { 1204 struct pool *pool = tc->pool; 1205 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 1206 1207 h->overwrite_mapping = m; 1208 m->bio = bio; 1209 save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio); 1210 inc_all_io_entry(pool, bio); 1211 remap_and_issue(tc, bio, data_begin); 1212 } 1213 1214 /* 1215 * A partial copy also needs to zero the uncopied region. 1216 */ 1217 static void schedule_copy(struct thin_c *tc, dm_block_t virt_block, 1218 struct dm_dev *origin, dm_block_t data_origin, 1219 dm_block_t data_dest, 1220 struct dm_bio_prison_cell *cell, struct bio *bio, 1221 sector_t len) 1222 { 1223 int r; 1224 struct pool *pool = tc->pool; 1225 struct dm_thin_new_mapping *m = get_next_mapping(pool); 1226 1227 m->tc = tc; 1228 m->virt_begin = virt_block; 1229 m->virt_end = virt_block + 1u; 1230 m->data_block = data_dest; 1231 m->cell = cell; 1232 1233 /* 1234 * quiesce action + copy action + an extra reference held for the 1235 * duration of this function (we may need to inc later for a 1236 * partial zero). 1237 */ 1238 atomic_set(&m->prepare_actions, 3); 1239 1240 if (!dm_deferred_set_add_work(pool->shared_read_ds, &m->list)) 1241 complete_mapping_preparation(m); /* already quiesced */ 1242 1243 /* 1244 * IO to pool_dev remaps to the pool target's data_dev. 1245 * 1246 * If the whole block of data is being overwritten, we can issue the 1247 * bio immediately. Otherwise we use kcopyd to clone the data first. 1248 */ 1249 if (io_overwrites_block(pool, bio)) 1250 remap_and_issue_overwrite(tc, bio, data_dest, m); 1251 else { 1252 struct dm_io_region from, to; 1253 1254 from.bdev = origin->bdev; 1255 from.sector = data_origin * pool->sectors_per_block; 1256 from.count = len; 1257 1258 to.bdev = tc->pool_dev->bdev; 1259 to.sector = data_dest * pool->sectors_per_block; 1260 to.count = len; 1261 1262 r = dm_kcopyd_copy(pool->copier, &from, 1, &to, 1263 0, copy_complete, m); 1264 if (r < 0) { 1265 DMERR_LIMIT("dm_kcopyd_copy() failed"); 1266 copy_complete(1, 1, m); 1267 1268 /* 1269 * We allow the zero to be issued, to simplify the 1270 * error path. Otherwise we'd need to start 1271 * worrying about decrementing the prepare_actions 1272 * counter. 1273 */ 1274 } 1275 1276 /* 1277 * Do we need to zero a tail region? 1278 */ 1279 if (len < pool->sectors_per_block && pool->pf.zero_new_blocks) { 1280 atomic_inc(&m->prepare_actions); 1281 ll_zero(tc, m, 1282 data_dest * pool->sectors_per_block + len, 1283 (data_dest + 1) * pool->sectors_per_block); 1284 } 1285 } 1286 1287 complete_mapping_preparation(m); /* drop our ref */ 1288 } 1289 1290 static void schedule_internal_copy(struct thin_c *tc, dm_block_t virt_block, 1291 dm_block_t data_origin, dm_block_t data_dest, 1292 struct dm_bio_prison_cell *cell, struct bio *bio) 1293 { 1294 schedule_copy(tc, virt_block, tc->pool_dev, 1295 data_origin, data_dest, cell, bio, 1296 tc->pool->sectors_per_block); 1297 } 1298 1299 static void schedule_zero(struct thin_c *tc, dm_block_t virt_block, 1300 dm_block_t data_block, struct dm_bio_prison_cell *cell, 1301 struct bio *bio) 1302 { 1303 struct pool *pool = tc->pool; 1304 struct dm_thin_new_mapping *m = get_next_mapping(pool); 1305 1306 atomic_set(&m->prepare_actions, 1); /* no need to quiesce */ 1307 m->tc = tc; 1308 m->virt_begin = virt_block; 1309 m->virt_end = virt_block + 1u; 1310 m->data_block = data_block; 1311 m->cell = cell; 1312 1313 /* 1314 * If the whole block of data is being overwritten or we are not 1315 * zeroing pre-existing data, we can issue the bio immediately. 1316 * Otherwise we use kcopyd to zero the data first. 1317 */ 1318 if (pool->pf.zero_new_blocks) { 1319 if (io_overwrites_block(pool, bio)) 1320 remap_and_issue_overwrite(tc, bio, data_block, m); 1321 else 1322 ll_zero(tc, m, data_block * pool->sectors_per_block, 1323 (data_block + 1) * pool->sectors_per_block); 1324 } else 1325 process_prepared_mapping(m); 1326 } 1327 1328 static void schedule_external_copy(struct thin_c *tc, dm_block_t virt_block, 1329 dm_block_t data_dest, 1330 struct dm_bio_prison_cell *cell, struct bio *bio) 1331 { 1332 struct pool *pool = tc->pool; 1333 sector_t virt_block_begin = virt_block * pool->sectors_per_block; 1334 sector_t virt_block_end = (virt_block + 1) * pool->sectors_per_block; 1335 1336 if (virt_block_end <= tc->origin_size) 1337 schedule_copy(tc, virt_block, tc->origin_dev, 1338 virt_block, data_dest, cell, bio, 1339 pool->sectors_per_block); 1340 1341 else if (virt_block_begin < tc->origin_size) 1342 schedule_copy(tc, virt_block, tc->origin_dev, 1343 virt_block, data_dest, cell, bio, 1344 tc->origin_size - virt_block_begin); 1345 1346 else 1347 schedule_zero(tc, virt_block, data_dest, cell, bio); 1348 } 1349 1350 static void set_pool_mode(struct pool *pool, enum pool_mode new_mode); 1351 1352 static void check_for_space(struct pool *pool) 1353 { 1354 int r; 1355 dm_block_t nr_free; 1356 1357 if (get_pool_mode(pool) != PM_OUT_OF_DATA_SPACE) 1358 return; 1359 1360 r = dm_pool_get_free_block_count(pool->pmd, &nr_free); 1361 if (r) 1362 return; 1363 1364 if (nr_free) 1365 set_pool_mode(pool, PM_WRITE); 1366 } 1367 1368 /* 1369 * A non-zero return indicates read_only or fail_io mode. 1370 * Many callers don't care about the return value. 1371 */ 1372 static int commit(struct pool *pool) 1373 { 1374 int r; 1375 1376 if (get_pool_mode(pool) >= PM_READ_ONLY) 1377 return -EINVAL; 1378 1379 r = dm_pool_commit_metadata(pool->pmd); 1380 if (r) 1381 metadata_operation_failed(pool, "dm_pool_commit_metadata", r); 1382 else 1383 check_for_space(pool); 1384 1385 return r; 1386 } 1387 1388 static void check_low_water_mark(struct pool *pool, dm_block_t free_blocks) 1389 { 1390 unsigned long flags; 1391 1392 if (free_blocks <= pool->low_water_blocks && !pool->low_water_triggered) { 1393 DMWARN("%s: reached low water mark for data device: sending event.", 1394 dm_device_name(pool->pool_md)); 1395 spin_lock_irqsave(&pool->lock, flags); 1396 pool->low_water_triggered = true; 1397 spin_unlock_irqrestore(&pool->lock, flags); 1398 dm_table_event(pool->ti->table); 1399 } 1400 } 1401 1402 static int alloc_data_block(struct thin_c *tc, dm_block_t *result) 1403 { 1404 int r; 1405 dm_block_t free_blocks; 1406 struct pool *pool = tc->pool; 1407 1408 if (WARN_ON(get_pool_mode(pool) != PM_WRITE)) 1409 return -EINVAL; 1410 1411 r = dm_pool_get_free_block_count(pool->pmd, &free_blocks); 1412 if (r) { 1413 metadata_operation_failed(pool, "dm_pool_get_free_block_count", r); 1414 return r; 1415 } 1416 1417 check_low_water_mark(pool, free_blocks); 1418 1419 if (!free_blocks) { 1420 /* 1421 * Try to commit to see if that will free up some 1422 * more space. 1423 */ 1424 r = commit(pool); 1425 if (r) 1426 return r; 1427 1428 r = dm_pool_get_free_block_count(pool->pmd, &free_blocks); 1429 if (r) { 1430 metadata_operation_failed(pool, "dm_pool_get_free_block_count", r); 1431 return r; 1432 } 1433 1434 if (!free_blocks) { 1435 set_pool_mode(pool, PM_OUT_OF_DATA_SPACE); 1436 return -ENOSPC; 1437 } 1438 } 1439 1440 r = dm_pool_alloc_data_block(pool->pmd, result); 1441 if (r) { 1442 metadata_operation_failed(pool, "dm_pool_alloc_data_block", r); 1443 return r; 1444 } 1445 1446 return 0; 1447 } 1448 1449 /* 1450 * If we have run out of space, queue bios until the device is 1451 * resumed, presumably after having been reloaded with more space. 1452 */ 1453 static void retry_on_resume(struct bio *bio) 1454 { 1455 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 1456 struct thin_c *tc = h->tc; 1457 unsigned long flags; 1458 1459 spin_lock_irqsave(&tc->lock, flags); 1460 bio_list_add(&tc->retry_on_resume_list, bio); 1461 spin_unlock_irqrestore(&tc->lock, flags); 1462 } 1463 1464 static int should_error_unserviceable_bio(struct pool *pool) 1465 { 1466 enum pool_mode m = get_pool_mode(pool); 1467 1468 switch (m) { 1469 case PM_WRITE: 1470 /* Shouldn't get here */ 1471 DMERR_LIMIT("bio unserviceable, yet pool is in PM_WRITE mode"); 1472 return -EIO; 1473 1474 case PM_OUT_OF_DATA_SPACE: 1475 return pool->pf.error_if_no_space ? -ENOSPC : 0; 1476 1477 case PM_READ_ONLY: 1478 case PM_FAIL: 1479 return -EIO; 1480 default: 1481 /* Shouldn't get here */ 1482 DMERR_LIMIT("bio unserviceable, yet pool has an unknown mode"); 1483 return -EIO; 1484 } 1485 } 1486 1487 static void handle_unserviceable_bio(struct pool *pool, struct bio *bio) 1488 { 1489 int error = should_error_unserviceable_bio(pool); 1490 1491 if (error) 1492 bio_endio(bio, error); 1493 else 1494 retry_on_resume(bio); 1495 } 1496 1497 static void retry_bios_on_resume(struct pool *pool, struct dm_bio_prison_cell *cell) 1498 { 1499 struct bio *bio; 1500 struct bio_list bios; 1501 int error; 1502 1503 error = should_error_unserviceable_bio(pool); 1504 if (error) { 1505 cell_error_with_code(pool, cell, error); 1506 return; 1507 } 1508 1509 bio_list_init(&bios); 1510 cell_release(pool, cell, &bios); 1511 1512 while ((bio = bio_list_pop(&bios))) 1513 retry_on_resume(bio); 1514 } 1515 1516 static void process_discard_cell_no_passdown(struct thin_c *tc, 1517 struct dm_bio_prison_cell *virt_cell) 1518 { 1519 struct pool *pool = tc->pool; 1520 struct dm_thin_new_mapping *m = get_next_mapping(pool); 1521 1522 /* 1523 * We don't need to lock the data blocks, since there's no 1524 * passdown. We only lock data blocks for allocation and breaking sharing. 1525 */ 1526 m->tc = tc; 1527 m->virt_begin = virt_cell->key.block_begin; 1528 m->virt_end = virt_cell->key.block_end; 1529 m->cell = virt_cell; 1530 m->bio = virt_cell->holder; 1531 1532 if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list)) 1533 pool->process_prepared_discard(m); 1534 } 1535 1536 /* 1537 * FIXME: DM local hack to defer parent bios's end_io until we 1538 * _know_ all chained sub range discard bios have completed. 1539 * Will go away once late bio splitting lands upstream! 1540 */ 1541 static inline void __bio_inc_remaining(struct bio *bio) 1542 { 1543 bio->bi_flags |= (1 << BIO_CHAIN); 1544 smp_mb__before_atomic(); 1545 atomic_inc(&bio->__bi_remaining); 1546 } 1547 1548 static void break_up_discard_bio(struct thin_c *tc, dm_block_t begin, dm_block_t end, 1549 struct bio *bio) 1550 { 1551 struct pool *pool = tc->pool; 1552 1553 int r; 1554 bool maybe_shared; 1555 struct dm_cell_key data_key; 1556 struct dm_bio_prison_cell *data_cell; 1557 struct dm_thin_new_mapping *m; 1558 dm_block_t virt_begin, virt_end, data_begin; 1559 1560 while (begin != end) { 1561 r = ensure_next_mapping(pool); 1562 if (r) 1563 /* we did our best */ 1564 return; 1565 1566 r = dm_thin_find_mapped_range(tc->td, begin, end, &virt_begin, &virt_end, 1567 &data_begin, &maybe_shared); 1568 if (r) 1569 /* 1570 * Silently fail, letting any mappings we've 1571 * created complete. 1572 */ 1573 break; 1574 1575 build_key(tc->td, PHYSICAL, data_begin, data_begin + (virt_end - virt_begin), &data_key); 1576 if (bio_detain(tc->pool, &data_key, NULL, &data_cell)) { 1577 /* contention, we'll give up with this range */ 1578 begin = virt_end; 1579 continue; 1580 } 1581 1582 /* 1583 * IO may still be going to the destination block. We must 1584 * quiesce before we can do the removal. 1585 */ 1586 m = get_next_mapping(pool); 1587 m->tc = tc; 1588 m->maybe_shared = maybe_shared; 1589 m->virt_begin = virt_begin; 1590 m->virt_end = virt_end; 1591 m->data_block = data_begin; 1592 m->cell = data_cell; 1593 m->bio = bio; 1594 1595 /* 1596 * The parent bio must not complete before sub discard bios are 1597 * chained to it (see __blkdev_issue_discard_async's bio_chain)! 1598 * 1599 * This per-mapping bi_remaining increment is paired with 1600 * the implicit decrement that occurs via bio_endio() in 1601 * process_prepared_discard_{passdown,no_passdown}. 1602 */ 1603 __bio_inc_remaining(bio); 1604 if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list)) 1605 pool->process_prepared_discard(m); 1606 1607 begin = virt_end; 1608 } 1609 } 1610 1611 static void process_discard_cell_passdown(struct thin_c *tc, struct dm_bio_prison_cell *virt_cell) 1612 { 1613 struct bio *bio = virt_cell->holder; 1614 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 1615 1616 /* 1617 * The virt_cell will only get freed once the origin bio completes. 1618 * This means it will remain locked while all the individual 1619 * passdown bios are in flight. 1620 */ 1621 h->cell = virt_cell; 1622 break_up_discard_bio(tc, virt_cell->key.block_begin, virt_cell->key.block_end, bio); 1623 1624 /* 1625 * We complete the bio now, knowing that the bi_remaining field 1626 * will prevent completion until the sub range discards have 1627 * completed. 1628 */ 1629 bio_endio(bio, 0); 1630 } 1631 1632 static void process_discard_bio(struct thin_c *tc, struct bio *bio) 1633 { 1634 dm_block_t begin, end; 1635 struct dm_cell_key virt_key; 1636 struct dm_bio_prison_cell *virt_cell; 1637 1638 get_bio_block_range(tc, bio, &begin, &end); 1639 if (begin == end) { 1640 /* 1641 * The discard covers less than a block. 1642 */ 1643 bio_endio(bio, 0); 1644 return; 1645 } 1646 1647 build_key(tc->td, VIRTUAL, begin, end, &virt_key); 1648 if (bio_detain(tc->pool, &virt_key, bio, &virt_cell)) 1649 /* 1650 * Potential starvation issue: We're relying on the 1651 * fs/application being well behaved, and not trying to 1652 * send IO to a region at the same time as discarding it. 1653 * If they do this persistently then it's possible this 1654 * cell will never be granted. 1655 */ 1656 return; 1657 1658 tc->pool->process_discard_cell(tc, virt_cell); 1659 } 1660 1661 static void break_sharing(struct thin_c *tc, struct bio *bio, dm_block_t block, 1662 struct dm_cell_key *key, 1663 struct dm_thin_lookup_result *lookup_result, 1664 struct dm_bio_prison_cell *cell) 1665 { 1666 int r; 1667 dm_block_t data_block; 1668 struct pool *pool = tc->pool; 1669 1670 r = alloc_data_block(tc, &data_block); 1671 switch (r) { 1672 case 0: 1673 schedule_internal_copy(tc, block, lookup_result->block, 1674 data_block, cell, bio); 1675 break; 1676 1677 case -ENOSPC: 1678 retry_bios_on_resume(pool, cell); 1679 break; 1680 1681 default: 1682 DMERR_LIMIT("%s: alloc_data_block() failed: error = %d", 1683 __func__, r); 1684 cell_error(pool, cell); 1685 break; 1686 } 1687 } 1688 1689 static void __remap_and_issue_shared_cell(void *context, 1690 struct dm_bio_prison_cell *cell) 1691 { 1692 struct remap_info *info = context; 1693 struct bio *bio; 1694 1695 while ((bio = bio_list_pop(&cell->bios))) { 1696 if ((bio_data_dir(bio) == WRITE) || 1697 (bio->bi_rw & (REQ_DISCARD | REQ_FLUSH | REQ_FUA))) 1698 bio_list_add(&info->defer_bios, bio); 1699 else { 1700 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));; 1701 1702 h->shared_read_entry = dm_deferred_entry_inc(info->tc->pool->shared_read_ds); 1703 inc_all_io_entry(info->tc->pool, bio); 1704 bio_list_add(&info->issue_bios, bio); 1705 } 1706 } 1707 } 1708 1709 static void remap_and_issue_shared_cell(struct thin_c *tc, 1710 struct dm_bio_prison_cell *cell, 1711 dm_block_t block) 1712 { 1713 struct bio *bio; 1714 struct remap_info info; 1715 1716 info.tc = tc; 1717 bio_list_init(&info.defer_bios); 1718 bio_list_init(&info.issue_bios); 1719 1720 cell_visit_release(tc->pool, __remap_and_issue_shared_cell, 1721 &info, cell); 1722 1723 while ((bio = bio_list_pop(&info.defer_bios))) 1724 thin_defer_bio(tc, bio); 1725 1726 while ((bio = bio_list_pop(&info.issue_bios))) 1727 remap_and_issue(tc, bio, block); 1728 } 1729 1730 static void process_shared_bio(struct thin_c *tc, struct bio *bio, 1731 dm_block_t block, 1732 struct dm_thin_lookup_result *lookup_result, 1733 struct dm_bio_prison_cell *virt_cell) 1734 { 1735 struct dm_bio_prison_cell *data_cell; 1736 struct pool *pool = tc->pool; 1737 struct dm_cell_key key; 1738 1739 /* 1740 * If cell is already occupied, then sharing is already in the process 1741 * of being broken so we have nothing further to do here. 1742 */ 1743 build_data_key(tc->td, lookup_result->block, &key); 1744 if (bio_detain(pool, &key, bio, &data_cell)) { 1745 cell_defer_no_holder(tc, virt_cell); 1746 return; 1747 } 1748 1749 if (bio_data_dir(bio) == WRITE && bio->bi_iter.bi_size) { 1750 break_sharing(tc, bio, block, &key, lookup_result, data_cell); 1751 cell_defer_no_holder(tc, virt_cell); 1752 } else { 1753 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 1754 1755 h->shared_read_entry = dm_deferred_entry_inc(pool->shared_read_ds); 1756 inc_all_io_entry(pool, bio); 1757 remap_and_issue(tc, bio, lookup_result->block); 1758 1759 remap_and_issue_shared_cell(tc, data_cell, lookup_result->block); 1760 remap_and_issue_shared_cell(tc, virt_cell, lookup_result->block); 1761 } 1762 } 1763 1764 static void provision_block(struct thin_c *tc, struct bio *bio, dm_block_t block, 1765 struct dm_bio_prison_cell *cell) 1766 { 1767 int r; 1768 dm_block_t data_block; 1769 struct pool *pool = tc->pool; 1770 1771 /* 1772 * Remap empty bios (flushes) immediately, without provisioning. 1773 */ 1774 if (!bio->bi_iter.bi_size) { 1775 inc_all_io_entry(pool, bio); 1776 cell_defer_no_holder(tc, cell); 1777 1778 remap_and_issue(tc, bio, 0); 1779 return; 1780 } 1781 1782 /* 1783 * Fill read bios with zeroes and complete them immediately. 1784 */ 1785 if (bio_data_dir(bio) == READ) { 1786 zero_fill_bio(bio); 1787 cell_defer_no_holder(tc, cell); 1788 bio_endio(bio, 0); 1789 return; 1790 } 1791 1792 r = alloc_data_block(tc, &data_block); 1793 switch (r) { 1794 case 0: 1795 if (tc->origin_dev) 1796 schedule_external_copy(tc, block, data_block, cell, bio); 1797 else 1798 schedule_zero(tc, block, data_block, cell, bio); 1799 break; 1800 1801 case -ENOSPC: 1802 retry_bios_on_resume(pool, cell); 1803 break; 1804 1805 default: 1806 DMERR_LIMIT("%s: alloc_data_block() failed: error = %d", 1807 __func__, r); 1808 cell_error(pool, cell); 1809 break; 1810 } 1811 } 1812 1813 static void process_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell) 1814 { 1815 int r; 1816 struct pool *pool = tc->pool; 1817 struct bio *bio = cell->holder; 1818 dm_block_t block = get_bio_block(tc, bio); 1819 struct dm_thin_lookup_result lookup_result; 1820 1821 if (tc->requeue_mode) { 1822 cell_requeue(pool, cell); 1823 return; 1824 } 1825 1826 r = dm_thin_find_block(tc->td, block, 1, &lookup_result); 1827 switch (r) { 1828 case 0: 1829 if (lookup_result.shared) 1830 process_shared_bio(tc, bio, block, &lookup_result, cell); 1831 else { 1832 inc_all_io_entry(pool, bio); 1833 remap_and_issue(tc, bio, lookup_result.block); 1834 inc_remap_and_issue_cell(tc, cell, lookup_result.block); 1835 } 1836 break; 1837 1838 case -ENODATA: 1839 if (bio_data_dir(bio) == READ && tc->origin_dev) { 1840 inc_all_io_entry(pool, bio); 1841 cell_defer_no_holder(tc, cell); 1842 1843 if (bio_end_sector(bio) <= tc->origin_size) 1844 remap_to_origin_and_issue(tc, bio); 1845 1846 else if (bio->bi_iter.bi_sector < tc->origin_size) { 1847 zero_fill_bio(bio); 1848 bio->bi_iter.bi_size = (tc->origin_size - bio->bi_iter.bi_sector) << SECTOR_SHIFT; 1849 remap_to_origin_and_issue(tc, bio); 1850 1851 } else { 1852 zero_fill_bio(bio); 1853 bio_endio(bio, 0); 1854 } 1855 } else 1856 provision_block(tc, bio, block, cell); 1857 break; 1858 1859 default: 1860 DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d", 1861 __func__, r); 1862 cell_defer_no_holder(tc, cell); 1863 bio_io_error(bio); 1864 break; 1865 } 1866 } 1867 1868 static void process_bio(struct thin_c *tc, struct bio *bio) 1869 { 1870 struct pool *pool = tc->pool; 1871 dm_block_t block = get_bio_block(tc, bio); 1872 struct dm_bio_prison_cell *cell; 1873 struct dm_cell_key key; 1874 1875 /* 1876 * If cell is already occupied, then the block is already 1877 * being provisioned so we have nothing further to do here. 1878 */ 1879 build_virtual_key(tc->td, block, &key); 1880 if (bio_detain(pool, &key, bio, &cell)) 1881 return; 1882 1883 process_cell(tc, cell); 1884 } 1885 1886 static void __process_bio_read_only(struct thin_c *tc, struct bio *bio, 1887 struct dm_bio_prison_cell *cell) 1888 { 1889 int r; 1890 int rw = bio_data_dir(bio); 1891 dm_block_t block = get_bio_block(tc, bio); 1892 struct dm_thin_lookup_result lookup_result; 1893 1894 r = dm_thin_find_block(tc->td, block, 1, &lookup_result); 1895 switch (r) { 1896 case 0: 1897 if (lookup_result.shared && (rw == WRITE) && bio->bi_iter.bi_size) { 1898 handle_unserviceable_bio(tc->pool, bio); 1899 if (cell) 1900 cell_defer_no_holder(tc, cell); 1901 } else { 1902 inc_all_io_entry(tc->pool, bio); 1903 remap_and_issue(tc, bio, lookup_result.block); 1904 if (cell) 1905 inc_remap_and_issue_cell(tc, cell, lookup_result.block); 1906 } 1907 break; 1908 1909 case -ENODATA: 1910 if (cell) 1911 cell_defer_no_holder(tc, cell); 1912 if (rw != READ) { 1913 handle_unserviceable_bio(tc->pool, bio); 1914 break; 1915 } 1916 1917 if (tc->origin_dev) { 1918 inc_all_io_entry(tc->pool, bio); 1919 remap_to_origin_and_issue(tc, bio); 1920 break; 1921 } 1922 1923 zero_fill_bio(bio); 1924 bio_endio(bio, 0); 1925 break; 1926 1927 default: 1928 DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d", 1929 __func__, r); 1930 if (cell) 1931 cell_defer_no_holder(tc, cell); 1932 bio_io_error(bio); 1933 break; 1934 } 1935 } 1936 1937 static void process_bio_read_only(struct thin_c *tc, struct bio *bio) 1938 { 1939 __process_bio_read_only(tc, bio, NULL); 1940 } 1941 1942 static void process_cell_read_only(struct thin_c *tc, struct dm_bio_prison_cell *cell) 1943 { 1944 __process_bio_read_only(tc, cell->holder, cell); 1945 } 1946 1947 static void process_bio_success(struct thin_c *tc, struct bio *bio) 1948 { 1949 bio_endio(bio, 0); 1950 } 1951 1952 static void process_bio_fail(struct thin_c *tc, struct bio *bio) 1953 { 1954 bio_io_error(bio); 1955 } 1956 1957 static void process_cell_success(struct thin_c *tc, struct dm_bio_prison_cell *cell) 1958 { 1959 cell_success(tc->pool, cell); 1960 } 1961 1962 static void process_cell_fail(struct thin_c *tc, struct dm_bio_prison_cell *cell) 1963 { 1964 cell_error(tc->pool, cell); 1965 } 1966 1967 /* 1968 * FIXME: should we also commit due to size of transaction, measured in 1969 * metadata blocks? 1970 */ 1971 static int need_commit_due_to_time(struct pool *pool) 1972 { 1973 return !time_in_range(jiffies, pool->last_commit_jiffies, 1974 pool->last_commit_jiffies + COMMIT_PERIOD); 1975 } 1976 1977 #define thin_pbd(node) rb_entry((node), struct dm_thin_endio_hook, rb_node) 1978 #define thin_bio(pbd) dm_bio_from_per_bio_data((pbd), sizeof(struct dm_thin_endio_hook)) 1979 1980 static void __thin_bio_rb_add(struct thin_c *tc, struct bio *bio) 1981 { 1982 struct rb_node **rbp, *parent; 1983 struct dm_thin_endio_hook *pbd; 1984 sector_t bi_sector = bio->bi_iter.bi_sector; 1985 1986 rbp = &tc->sort_bio_list.rb_node; 1987 parent = NULL; 1988 while (*rbp) { 1989 parent = *rbp; 1990 pbd = thin_pbd(parent); 1991 1992 if (bi_sector < thin_bio(pbd)->bi_iter.bi_sector) 1993 rbp = &(*rbp)->rb_left; 1994 else 1995 rbp = &(*rbp)->rb_right; 1996 } 1997 1998 pbd = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 1999 rb_link_node(&pbd->rb_node, parent, rbp); 2000 rb_insert_color(&pbd->rb_node, &tc->sort_bio_list); 2001 } 2002 2003 static void __extract_sorted_bios(struct thin_c *tc) 2004 { 2005 struct rb_node *node; 2006 struct dm_thin_endio_hook *pbd; 2007 struct bio *bio; 2008 2009 for (node = rb_first(&tc->sort_bio_list); node; node = rb_next(node)) { 2010 pbd = thin_pbd(node); 2011 bio = thin_bio(pbd); 2012 2013 bio_list_add(&tc->deferred_bio_list, bio); 2014 rb_erase(&pbd->rb_node, &tc->sort_bio_list); 2015 } 2016 2017 WARN_ON(!RB_EMPTY_ROOT(&tc->sort_bio_list)); 2018 } 2019 2020 static void __sort_thin_deferred_bios(struct thin_c *tc) 2021 { 2022 struct bio *bio; 2023 struct bio_list bios; 2024 2025 bio_list_init(&bios); 2026 bio_list_merge(&bios, &tc->deferred_bio_list); 2027 bio_list_init(&tc->deferred_bio_list); 2028 2029 /* Sort deferred_bio_list using rb-tree */ 2030 while ((bio = bio_list_pop(&bios))) 2031 __thin_bio_rb_add(tc, bio); 2032 2033 /* 2034 * Transfer the sorted bios in sort_bio_list back to 2035 * deferred_bio_list to allow lockless submission of 2036 * all bios. 2037 */ 2038 __extract_sorted_bios(tc); 2039 } 2040 2041 static void process_thin_deferred_bios(struct thin_c *tc) 2042 { 2043 struct pool *pool = tc->pool; 2044 unsigned long flags; 2045 struct bio *bio; 2046 struct bio_list bios; 2047 struct blk_plug plug; 2048 unsigned count = 0; 2049 2050 if (tc->requeue_mode) { 2051 error_thin_bio_list(tc, &tc->deferred_bio_list, DM_ENDIO_REQUEUE); 2052 return; 2053 } 2054 2055 bio_list_init(&bios); 2056 2057 spin_lock_irqsave(&tc->lock, flags); 2058 2059 if (bio_list_empty(&tc->deferred_bio_list)) { 2060 spin_unlock_irqrestore(&tc->lock, flags); 2061 return; 2062 } 2063 2064 __sort_thin_deferred_bios(tc); 2065 2066 bio_list_merge(&bios, &tc->deferred_bio_list); 2067 bio_list_init(&tc->deferred_bio_list); 2068 2069 spin_unlock_irqrestore(&tc->lock, flags); 2070 2071 blk_start_plug(&plug); 2072 while ((bio = bio_list_pop(&bios))) { 2073 /* 2074 * If we've got no free new_mapping structs, and processing 2075 * this bio might require one, we pause until there are some 2076 * prepared mappings to process. 2077 */ 2078 if (ensure_next_mapping(pool)) { 2079 spin_lock_irqsave(&tc->lock, flags); 2080 bio_list_add(&tc->deferred_bio_list, bio); 2081 bio_list_merge(&tc->deferred_bio_list, &bios); 2082 spin_unlock_irqrestore(&tc->lock, flags); 2083 break; 2084 } 2085 2086 if (bio->bi_rw & REQ_DISCARD) 2087 pool->process_discard(tc, bio); 2088 else 2089 pool->process_bio(tc, bio); 2090 2091 if ((count++ & 127) == 0) { 2092 throttle_work_update(&pool->throttle); 2093 dm_pool_issue_prefetches(pool->pmd); 2094 } 2095 } 2096 blk_finish_plug(&plug); 2097 } 2098 2099 static int cmp_cells(const void *lhs, const void *rhs) 2100 { 2101 struct dm_bio_prison_cell *lhs_cell = *((struct dm_bio_prison_cell **) lhs); 2102 struct dm_bio_prison_cell *rhs_cell = *((struct dm_bio_prison_cell **) rhs); 2103 2104 BUG_ON(!lhs_cell->holder); 2105 BUG_ON(!rhs_cell->holder); 2106 2107 if (lhs_cell->holder->bi_iter.bi_sector < rhs_cell->holder->bi_iter.bi_sector) 2108 return -1; 2109 2110 if (lhs_cell->holder->bi_iter.bi_sector > rhs_cell->holder->bi_iter.bi_sector) 2111 return 1; 2112 2113 return 0; 2114 } 2115 2116 static unsigned sort_cells(struct pool *pool, struct list_head *cells) 2117 { 2118 unsigned count = 0; 2119 struct dm_bio_prison_cell *cell, *tmp; 2120 2121 list_for_each_entry_safe(cell, tmp, cells, user_list) { 2122 if (count >= CELL_SORT_ARRAY_SIZE) 2123 break; 2124 2125 pool->cell_sort_array[count++] = cell; 2126 list_del(&cell->user_list); 2127 } 2128 2129 sort(pool->cell_sort_array, count, sizeof(cell), cmp_cells, NULL); 2130 2131 return count; 2132 } 2133 2134 static void process_thin_deferred_cells(struct thin_c *tc) 2135 { 2136 struct pool *pool = tc->pool; 2137 unsigned long flags; 2138 struct list_head cells; 2139 struct dm_bio_prison_cell *cell; 2140 unsigned i, j, count; 2141 2142 INIT_LIST_HEAD(&cells); 2143 2144 spin_lock_irqsave(&tc->lock, flags); 2145 list_splice_init(&tc->deferred_cells, &cells); 2146 spin_unlock_irqrestore(&tc->lock, flags); 2147 2148 if (list_empty(&cells)) 2149 return; 2150 2151 do { 2152 count = sort_cells(tc->pool, &cells); 2153 2154 for (i = 0; i < count; i++) { 2155 cell = pool->cell_sort_array[i]; 2156 BUG_ON(!cell->holder); 2157 2158 /* 2159 * If we've got no free new_mapping structs, and processing 2160 * this bio might require one, we pause until there are some 2161 * prepared mappings to process. 2162 */ 2163 if (ensure_next_mapping(pool)) { 2164 for (j = i; j < count; j++) 2165 list_add(&pool->cell_sort_array[j]->user_list, &cells); 2166 2167 spin_lock_irqsave(&tc->lock, flags); 2168 list_splice(&cells, &tc->deferred_cells); 2169 spin_unlock_irqrestore(&tc->lock, flags); 2170 return; 2171 } 2172 2173 if (cell->holder->bi_rw & REQ_DISCARD) 2174 pool->process_discard_cell(tc, cell); 2175 else 2176 pool->process_cell(tc, cell); 2177 } 2178 } while (!list_empty(&cells)); 2179 } 2180 2181 static void thin_get(struct thin_c *tc); 2182 static void thin_put(struct thin_c *tc); 2183 2184 /* 2185 * We can't hold rcu_read_lock() around code that can block. So we 2186 * find a thin with the rcu lock held; bump a refcount; then drop 2187 * the lock. 2188 */ 2189 static struct thin_c *get_first_thin(struct pool *pool) 2190 { 2191 struct thin_c *tc = NULL; 2192 2193 rcu_read_lock(); 2194 if (!list_empty(&pool->active_thins)) { 2195 tc = list_entry_rcu(pool->active_thins.next, struct thin_c, list); 2196 thin_get(tc); 2197 } 2198 rcu_read_unlock(); 2199 2200 return tc; 2201 } 2202 2203 static struct thin_c *get_next_thin(struct pool *pool, struct thin_c *tc) 2204 { 2205 struct thin_c *old_tc = tc; 2206 2207 rcu_read_lock(); 2208 list_for_each_entry_continue_rcu(tc, &pool->active_thins, list) { 2209 thin_get(tc); 2210 thin_put(old_tc); 2211 rcu_read_unlock(); 2212 return tc; 2213 } 2214 thin_put(old_tc); 2215 rcu_read_unlock(); 2216 2217 return NULL; 2218 } 2219 2220 static void process_deferred_bios(struct pool *pool) 2221 { 2222 unsigned long flags; 2223 struct bio *bio; 2224 struct bio_list bios; 2225 struct thin_c *tc; 2226 2227 tc = get_first_thin(pool); 2228 while (tc) { 2229 process_thin_deferred_cells(tc); 2230 process_thin_deferred_bios(tc); 2231 tc = get_next_thin(pool, tc); 2232 } 2233 2234 /* 2235 * If there are any deferred flush bios, we must commit 2236 * the metadata before issuing them. 2237 */ 2238 bio_list_init(&bios); 2239 spin_lock_irqsave(&pool->lock, flags); 2240 bio_list_merge(&bios, &pool->deferred_flush_bios); 2241 bio_list_init(&pool->deferred_flush_bios); 2242 spin_unlock_irqrestore(&pool->lock, flags); 2243 2244 if (bio_list_empty(&bios) && 2245 !(dm_pool_changed_this_transaction(pool->pmd) && need_commit_due_to_time(pool))) 2246 return; 2247 2248 if (commit(pool)) { 2249 while ((bio = bio_list_pop(&bios))) 2250 bio_io_error(bio); 2251 return; 2252 } 2253 pool->last_commit_jiffies = jiffies; 2254 2255 while ((bio = bio_list_pop(&bios))) 2256 generic_make_request(bio); 2257 } 2258 2259 static void do_worker(struct work_struct *ws) 2260 { 2261 struct pool *pool = container_of(ws, struct pool, worker); 2262 2263 throttle_work_start(&pool->throttle); 2264 dm_pool_issue_prefetches(pool->pmd); 2265 throttle_work_update(&pool->throttle); 2266 process_prepared(pool, &pool->prepared_mappings, &pool->process_prepared_mapping); 2267 throttle_work_update(&pool->throttle); 2268 process_prepared(pool, &pool->prepared_discards, &pool->process_prepared_discard); 2269 throttle_work_update(&pool->throttle); 2270 process_deferred_bios(pool); 2271 throttle_work_complete(&pool->throttle); 2272 } 2273 2274 /* 2275 * We want to commit periodically so that not too much 2276 * unwritten data builds up. 2277 */ 2278 static void do_waker(struct work_struct *ws) 2279 { 2280 struct pool *pool = container_of(to_delayed_work(ws), struct pool, waker); 2281 wake_worker(pool); 2282 queue_delayed_work(pool->wq, &pool->waker, COMMIT_PERIOD); 2283 } 2284 2285 static void notify_of_pool_mode_change_to_oods(struct pool *pool); 2286 2287 /* 2288 * We're holding onto IO to allow userland time to react. After the 2289 * timeout either the pool will have been resized (and thus back in 2290 * PM_WRITE mode), or we degrade to PM_OUT_OF_DATA_SPACE w/ error_if_no_space. 2291 */ 2292 static void do_no_space_timeout(struct work_struct *ws) 2293 { 2294 struct pool *pool = container_of(to_delayed_work(ws), struct pool, 2295 no_space_timeout); 2296 2297 if (get_pool_mode(pool) == PM_OUT_OF_DATA_SPACE && !pool->pf.error_if_no_space) { 2298 pool->pf.error_if_no_space = true; 2299 notify_of_pool_mode_change_to_oods(pool); 2300 error_retry_list(pool); 2301 } 2302 } 2303 2304 /*----------------------------------------------------------------*/ 2305 2306 struct pool_work { 2307 struct work_struct worker; 2308 struct completion complete; 2309 }; 2310 2311 static struct pool_work *to_pool_work(struct work_struct *ws) 2312 { 2313 return container_of(ws, struct pool_work, worker); 2314 } 2315 2316 static void pool_work_complete(struct pool_work *pw) 2317 { 2318 complete(&pw->complete); 2319 } 2320 2321 static void pool_work_wait(struct pool_work *pw, struct pool *pool, 2322 void (*fn)(struct work_struct *)) 2323 { 2324 INIT_WORK_ONSTACK(&pw->worker, fn); 2325 init_completion(&pw->complete); 2326 queue_work(pool->wq, &pw->worker); 2327 wait_for_completion(&pw->complete); 2328 } 2329 2330 /*----------------------------------------------------------------*/ 2331 2332 struct noflush_work { 2333 struct pool_work pw; 2334 struct thin_c *tc; 2335 }; 2336 2337 static struct noflush_work *to_noflush(struct work_struct *ws) 2338 { 2339 return container_of(to_pool_work(ws), struct noflush_work, pw); 2340 } 2341 2342 static void do_noflush_start(struct work_struct *ws) 2343 { 2344 struct noflush_work *w = to_noflush(ws); 2345 w->tc->requeue_mode = true; 2346 requeue_io(w->tc); 2347 pool_work_complete(&w->pw); 2348 } 2349 2350 static void do_noflush_stop(struct work_struct *ws) 2351 { 2352 struct noflush_work *w = to_noflush(ws); 2353 w->tc->requeue_mode = false; 2354 pool_work_complete(&w->pw); 2355 } 2356 2357 static void noflush_work(struct thin_c *tc, void (*fn)(struct work_struct *)) 2358 { 2359 struct noflush_work w; 2360 2361 w.tc = tc; 2362 pool_work_wait(&w.pw, tc->pool, fn); 2363 } 2364 2365 /*----------------------------------------------------------------*/ 2366 2367 static enum pool_mode get_pool_mode(struct pool *pool) 2368 { 2369 return pool->pf.mode; 2370 } 2371 2372 static void notify_of_pool_mode_change(struct pool *pool, const char *new_mode) 2373 { 2374 dm_table_event(pool->ti->table); 2375 DMINFO("%s: switching pool to %s mode", 2376 dm_device_name(pool->pool_md), new_mode); 2377 } 2378 2379 static void notify_of_pool_mode_change_to_oods(struct pool *pool) 2380 { 2381 if (!pool->pf.error_if_no_space) 2382 notify_of_pool_mode_change(pool, "out-of-data-space (queue IO)"); 2383 else 2384 notify_of_pool_mode_change(pool, "out-of-data-space (error IO)"); 2385 } 2386 2387 static bool passdown_enabled(struct pool_c *pt) 2388 { 2389 return pt->adjusted_pf.discard_passdown; 2390 } 2391 2392 static void set_discard_callbacks(struct pool *pool) 2393 { 2394 struct pool_c *pt = pool->ti->private; 2395 2396 if (passdown_enabled(pt)) { 2397 pool->process_discard_cell = process_discard_cell_passdown; 2398 pool->process_prepared_discard = process_prepared_discard_passdown; 2399 } else { 2400 pool->process_discard_cell = process_discard_cell_no_passdown; 2401 pool->process_prepared_discard = process_prepared_discard_no_passdown; 2402 } 2403 } 2404 2405 static void set_pool_mode(struct pool *pool, enum pool_mode new_mode) 2406 { 2407 struct pool_c *pt = pool->ti->private; 2408 bool needs_check = dm_pool_metadata_needs_check(pool->pmd); 2409 enum pool_mode old_mode = get_pool_mode(pool); 2410 unsigned long no_space_timeout = ACCESS_ONCE(no_space_timeout_secs) * HZ; 2411 2412 /* 2413 * Never allow the pool to transition to PM_WRITE mode if user 2414 * intervention is required to verify metadata and data consistency. 2415 */ 2416 if (new_mode == PM_WRITE && needs_check) { 2417 DMERR("%s: unable to switch pool to write mode until repaired.", 2418 dm_device_name(pool->pool_md)); 2419 if (old_mode != new_mode) 2420 new_mode = old_mode; 2421 else 2422 new_mode = PM_READ_ONLY; 2423 } 2424 /* 2425 * If we were in PM_FAIL mode, rollback of metadata failed. We're 2426 * not going to recover without a thin_repair. So we never let the 2427 * pool move out of the old mode. 2428 */ 2429 if (old_mode == PM_FAIL) 2430 new_mode = old_mode; 2431 2432 switch (new_mode) { 2433 case PM_FAIL: 2434 if (old_mode != new_mode) 2435 notify_of_pool_mode_change(pool, "failure"); 2436 dm_pool_metadata_read_only(pool->pmd); 2437 pool->process_bio = process_bio_fail; 2438 pool->process_discard = process_bio_fail; 2439 pool->process_cell = process_cell_fail; 2440 pool->process_discard_cell = process_cell_fail; 2441 pool->process_prepared_mapping = process_prepared_mapping_fail; 2442 pool->process_prepared_discard = process_prepared_discard_fail; 2443 2444 error_retry_list(pool); 2445 break; 2446 2447 case PM_READ_ONLY: 2448 if (old_mode != new_mode) 2449 notify_of_pool_mode_change(pool, "read-only"); 2450 dm_pool_metadata_read_only(pool->pmd); 2451 pool->process_bio = process_bio_read_only; 2452 pool->process_discard = process_bio_success; 2453 pool->process_cell = process_cell_read_only; 2454 pool->process_discard_cell = process_cell_success; 2455 pool->process_prepared_mapping = process_prepared_mapping_fail; 2456 pool->process_prepared_discard = process_prepared_discard_success; 2457 2458 error_retry_list(pool); 2459 break; 2460 2461 case PM_OUT_OF_DATA_SPACE: 2462 /* 2463 * Ideally we'd never hit this state; the low water mark 2464 * would trigger userland to extend the pool before we 2465 * completely run out of data space. However, many small 2466 * IOs to unprovisioned space can consume data space at an 2467 * alarming rate. Adjust your low water mark if you're 2468 * frequently seeing this mode. 2469 */ 2470 if (old_mode != new_mode) 2471 notify_of_pool_mode_change_to_oods(pool); 2472 pool->process_bio = process_bio_read_only; 2473 pool->process_discard = process_discard_bio; 2474 pool->process_cell = process_cell_read_only; 2475 pool->process_prepared_mapping = process_prepared_mapping; 2476 set_discard_callbacks(pool); 2477 2478 if (!pool->pf.error_if_no_space && no_space_timeout) 2479 queue_delayed_work(pool->wq, &pool->no_space_timeout, no_space_timeout); 2480 break; 2481 2482 case PM_WRITE: 2483 if (old_mode != new_mode) 2484 notify_of_pool_mode_change(pool, "write"); 2485 dm_pool_metadata_read_write(pool->pmd); 2486 pool->process_bio = process_bio; 2487 pool->process_discard = process_discard_bio; 2488 pool->process_cell = process_cell; 2489 pool->process_prepared_mapping = process_prepared_mapping; 2490 set_discard_callbacks(pool); 2491 break; 2492 } 2493 2494 pool->pf.mode = new_mode; 2495 /* 2496 * The pool mode may have changed, sync it so bind_control_target() 2497 * doesn't cause an unexpected mode transition on resume. 2498 */ 2499 pt->adjusted_pf.mode = new_mode; 2500 } 2501 2502 static void abort_transaction(struct pool *pool) 2503 { 2504 const char *dev_name = dm_device_name(pool->pool_md); 2505 2506 DMERR_LIMIT("%s: aborting current metadata transaction", dev_name); 2507 if (dm_pool_abort_metadata(pool->pmd)) { 2508 DMERR("%s: failed to abort metadata transaction", dev_name); 2509 set_pool_mode(pool, PM_FAIL); 2510 } 2511 2512 if (dm_pool_metadata_set_needs_check(pool->pmd)) { 2513 DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name); 2514 set_pool_mode(pool, PM_FAIL); 2515 } 2516 } 2517 2518 static void metadata_operation_failed(struct pool *pool, const char *op, int r) 2519 { 2520 DMERR_LIMIT("%s: metadata operation '%s' failed: error = %d", 2521 dm_device_name(pool->pool_md), op, r); 2522 2523 abort_transaction(pool); 2524 set_pool_mode(pool, PM_READ_ONLY); 2525 } 2526 2527 /*----------------------------------------------------------------*/ 2528 2529 /* 2530 * Mapping functions. 2531 */ 2532 2533 /* 2534 * Called only while mapping a thin bio to hand it over to the workqueue. 2535 */ 2536 static void thin_defer_bio(struct thin_c *tc, struct bio *bio) 2537 { 2538 unsigned long flags; 2539 struct pool *pool = tc->pool; 2540 2541 spin_lock_irqsave(&tc->lock, flags); 2542 bio_list_add(&tc->deferred_bio_list, bio); 2543 spin_unlock_irqrestore(&tc->lock, flags); 2544 2545 wake_worker(pool); 2546 } 2547 2548 static void thin_defer_bio_with_throttle(struct thin_c *tc, struct bio *bio) 2549 { 2550 struct pool *pool = tc->pool; 2551 2552 throttle_lock(&pool->throttle); 2553 thin_defer_bio(tc, bio); 2554 throttle_unlock(&pool->throttle); 2555 } 2556 2557 static void thin_defer_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell) 2558 { 2559 unsigned long flags; 2560 struct pool *pool = tc->pool; 2561 2562 throttle_lock(&pool->throttle); 2563 spin_lock_irqsave(&tc->lock, flags); 2564 list_add_tail(&cell->user_list, &tc->deferred_cells); 2565 spin_unlock_irqrestore(&tc->lock, flags); 2566 throttle_unlock(&pool->throttle); 2567 2568 wake_worker(pool); 2569 } 2570 2571 static void thin_hook_bio(struct thin_c *tc, struct bio *bio) 2572 { 2573 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 2574 2575 h->tc = tc; 2576 h->shared_read_entry = NULL; 2577 h->all_io_entry = NULL; 2578 h->overwrite_mapping = NULL; 2579 h->cell = NULL; 2580 } 2581 2582 /* 2583 * Non-blocking function called from the thin target's map function. 2584 */ 2585 static int thin_bio_map(struct dm_target *ti, struct bio *bio) 2586 { 2587 int r; 2588 struct thin_c *tc = ti->private; 2589 dm_block_t block = get_bio_block(tc, bio); 2590 struct dm_thin_device *td = tc->td; 2591 struct dm_thin_lookup_result result; 2592 struct dm_bio_prison_cell *virt_cell, *data_cell; 2593 struct dm_cell_key key; 2594 2595 thin_hook_bio(tc, bio); 2596 2597 if (tc->requeue_mode) { 2598 bio_endio(bio, DM_ENDIO_REQUEUE); 2599 return DM_MAPIO_SUBMITTED; 2600 } 2601 2602 if (get_pool_mode(tc->pool) == PM_FAIL) { 2603 bio_io_error(bio); 2604 return DM_MAPIO_SUBMITTED; 2605 } 2606 2607 if (bio->bi_rw & (REQ_DISCARD | REQ_FLUSH | REQ_FUA)) { 2608 thin_defer_bio_with_throttle(tc, bio); 2609 return DM_MAPIO_SUBMITTED; 2610 } 2611 2612 /* 2613 * We must hold the virtual cell before doing the lookup, otherwise 2614 * there's a race with discard. 2615 */ 2616 build_virtual_key(tc->td, block, &key); 2617 if (bio_detain(tc->pool, &key, bio, &virt_cell)) 2618 return DM_MAPIO_SUBMITTED; 2619 2620 r = dm_thin_find_block(td, block, 0, &result); 2621 2622 /* 2623 * Note that we defer readahead too. 2624 */ 2625 switch (r) { 2626 case 0: 2627 if (unlikely(result.shared)) { 2628 /* 2629 * We have a race condition here between the 2630 * result.shared value returned by the lookup and 2631 * snapshot creation, which may cause new 2632 * sharing. 2633 * 2634 * To avoid this always quiesce the origin before 2635 * taking the snap. You want to do this anyway to 2636 * ensure a consistent application view 2637 * (i.e. lockfs). 2638 * 2639 * More distant ancestors are irrelevant. The 2640 * shared flag will be set in their case. 2641 */ 2642 thin_defer_cell(tc, virt_cell); 2643 return DM_MAPIO_SUBMITTED; 2644 } 2645 2646 build_data_key(tc->td, result.block, &key); 2647 if (bio_detain(tc->pool, &key, bio, &data_cell)) { 2648 cell_defer_no_holder(tc, virt_cell); 2649 return DM_MAPIO_SUBMITTED; 2650 } 2651 2652 inc_all_io_entry(tc->pool, bio); 2653 cell_defer_no_holder(tc, data_cell); 2654 cell_defer_no_holder(tc, virt_cell); 2655 2656 remap(tc, bio, result.block); 2657 return DM_MAPIO_REMAPPED; 2658 2659 case -ENODATA: 2660 case -EWOULDBLOCK: 2661 thin_defer_cell(tc, virt_cell); 2662 return DM_MAPIO_SUBMITTED; 2663 2664 default: 2665 /* 2666 * Must always call bio_io_error on failure. 2667 * dm_thin_find_block can fail with -EINVAL if the 2668 * pool is switched to fail-io mode. 2669 */ 2670 bio_io_error(bio); 2671 cell_defer_no_holder(tc, virt_cell); 2672 return DM_MAPIO_SUBMITTED; 2673 } 2674 } 2675 2676 static int pool_is_congested(struct dm_target_callbacks *cb, int bdi_bits) 2677 { 2678 struct pool_c *pt = container_of(cb, struct pool_c, callbacks); 2679 struct request_queue *q; 2680 2681 if (get_pool_mode(pt->pool) == PM_OUT_OF_DATA_SPACE) 2682 return 1; 2683 2684 q = bdev_get_queue(pt->data_dev->bdev); 2685 return bdi_congested(&q->backing_dev_info, bdi_bits); 2686 } 2687 2688 static void requeue_bios(struct pool *pool) 2689 { 2690 unsigned long flags; 2691 struct thin_c *tc; 2692 2693 rcu_read_lock(); 2694 list_for_each_entry_rcu(tc, &pool->active_thins, list) { 2695 spin_lock_irqsave(&tc->lock, flags); 2696 bio_list_merge(&tc->deferred_bio_list, &tc->retry_on_resume_list); 2697 bio_list_init(&tc->retry_on_resume_list); 2698 spin_unlock_irqrestore(&tc->lock, flags); 2699 } 2700 rcu_read_unlock(); 2701 } 2702 2703 /*---------------------------------------------------------------- 2704 * Binding of control targets to a pool object 2705 *--------------------------------------------------------------*/ 2706 static bool data_dev_supports_discard(struct pool_c *pt) 2707 { 2708 struct request_queue *q = bdev_get_queue(pt->data_dev->bdev); 2709 2710 return q && blk_queue_discard(q); 2711 } 2712 2713 static bool is_factor(sector_t block_size, uint32_t n) 2714 { 2715 return !sector_div(block_size, n); 2716 } 2717 2718 /* 2719 * If discard_passdown was enabled verify that the data device 2720 * supports discards. Disable discard_passdown if not. 2721 */ 2722 static void disable_passdown_if_not_supported(struct pool_c *pt) 2723 { 2724 struct pool *pool = pt->pool; 2725 struct block_device *data_bdev = pt->data_dev->bdev; 2726 struct queue_limits *data_limits = &bdev_get_queue(data_bdev)->limits; 2727 const char *reason = NULL; 2728 char buf[BDEVNAME_SIZE]; 2729 2730 if (!pt->adjusted_pf.discard_passdown) 2731 return; 2732 2733 if (!data_dev_supports_discard(pt)) 2734 reason = "discard unsupported"; 2735 2736 else if (data_limits->max_discard_sectors < pool->sectors_per_block) 2737 reason = "max discard sectors smaller than a block"; 2738 2739 if (reason) { 2740 DMWARN("Data device (%s) %s: Disabling discard passdown.", bdevname(data_bdev, buf), reason); 2741 pt->adjusted_pf.discard_passdown = false; 2742 } 2743 } 2744 2745 static int bind_control_target(struct pool *pool, struct dm_target *ti) 2746 { 2747 struct pool_c *pt = ti->private; 2748 2749 /* 2750 * We want to make sure that a pool in PM_FAIL mode is never upgraded. 2751 */ 2752 enum pool_mode old_mode = get_pool_mode(pool); 2753 enum pool_mode new_mode = pt->adjusted_pf.mode; 2754 2755 /* 2756 * Don't change the pool's mode until set_pool_mode() below. 2757 * Otherwise the pool's process_* function pointers may 2758 * not match the desired pool mode. 2759 */ 2760 pt->adjusted_pf.mode = old_mode; 2761 2762 pool->ti = ti; 2763 pool->pf = pt->adjusted_pf; 2764 pool->low_water_blocks = pt->low_water_blocks; 2765 2766 set_pool_mode(pool, new_mode); 2767 2768 return 0; 2769 } 2770 2771 static void unbind_control_target(struct pool *pool, struct dm_target *ti) 2772 { 2773 if (pool->ti == ti) 2774 pool->ti = NULL; 2775 } 2776 2777 /*---------------------------------------------------------------- 2778 * Pool creation 2779 *--------------------------------------------------------------*/ 2780 /* Initialize pool features. */ 2781 static void pool_features_init(struct pool_features *pf) 2782 { 2783 pf->mode = PM_WRITE; 2784 pf->zero_new_blocks = true; 2785 pf->discard_enabled = true; 2786 pf->discard_passdown = true; 2787 pf->error_if_no_space = false; 2788 } 2789 2790 static void __pool_destroy(struct pool *pool) 2791 { 2792 __pool_table_remove(pool); 2793 2794 vfree(pool->cell_sort_array); 2795 if (dm_pool_metadata_close(pool->pmd) < 0) 2796 DMWARN("%s: dm_pool_metadata_close() failed.", __func__); 2797 2798 dm_bio_prison_destroy(pool->prison); 2799 dm_kcopyd_client_destroy(pool->copier); 2800 2801 if (pool->wq) 2802 destroy_workqueue(pool->wq); 2803 2804 if (pool->next_mapping) 2805 mempool_free(pool->next_mapping, pool->mapping_pool); 2806 mempool_destroy(pool->mapping_pool); 2807 dm_deferred_set_destroy(pool->shared_read_ds); 2808 dm_deferred_set_destroy(pool->all_io_ds); 2809 kfree(pool); 2810 } 2811 2812 static struct kmem_cache *_new_mapping_cache; 2813 2814 static struct pool *pool_create(struct mapped_device *pool_md, 2815 struct block_device *metadata_dev, 2816 unsigned long block_size, 2817 int read_only, char **error) 2818 { 2819 int r; 2820 void *err_p; 2821 struct pool *pool; 2822 struct dm_pool_metadata *pmd; 2823 bool format_device = read_only ? false : true; 2824 2825 pmd = dm_pool_metadata_open(metadata_dev, block_size, format_device); 2826 if (IS_ERR(pmd)) { 2827 *error = "Error creating metadata object"; 2828 return (struct pool *)pmd; 2829 } 2830 2831 pool = kmalloc(sizeof(*pool), GFP_KERNEL); 2832 if (!pool) { 2833 *error = "Error allocating memory for pool"; 2834 err_p = ERR_PTR(-ENOMEM); 2835 goto bad_pool; 2836 } 2837 2838 pool->pmd = pmd; 2839 pool->sectors_per_block = block_size; 2840 if (block_size & (block_size - 1)) 2841 pool->sectors_per_block_shift = -1; 2842 else 2843 pool->sectors_per_block_shift = __ffs(block_size); 2844 pool->low_water_blocks = 0; 2845 pool_features_init(&pool->pf); 2846 pool->prison = dm_bio_prison_create(); 2847 if (!pool->prison) { 2848 *error = "Error creating pool's bio prison"; 2849 err_p = ERR_PTR(-ENOMEM); 2850 goto bad_prison; 2851 } 2852 2853 pool->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle); 2854 if (IS_ERR(pool->copier)) { 2855 r = PTR_ERR(pool->copier); 2856 *error = "Error creating pool's kcopyd client"; 2857 err_p = ERR_PTR(r); 2858 goto bad_kcopyd_client; 2859 } 2860 2861 /* 2862 * Create singlethreaded workqueue that will service all devices 2863 * that use this metadata. 2864 */ 2865 pool->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM); 2866 if (!pool->wq) { 2867 *error = "Error creating pool's workqueue"; 2868 err_p = ERR_PTR(-ENOMEM); 2869 goto bad_wq; 2870 } 2871 2872 throttle_init(&pool->throttle); 2873 INIT_WORK(&pool->worker, do_worker); 2874 INIT_DELAYED_WORK(&pool->waker, do_waker); 2875 INIT_DELAYED_WORK(&pool->no_space_timeout, do_no_space_timeout); 2876 spin_lock_init(&pool->lock); 2877 bio_list_init(&pool->deferred_flush_bios); 2878 INIT_LIST_HEAD(&pool->prepared_mappings); 2879 INIT_LIST_HEAD(&pool->prepared_discards); 2880 INIT_LIST_HEAD(&pool->active_thins); 2881 pool->low_water_triggered = false; 2882 pool->suspended = true; 2883 2884 pool->shared_read_ds = dm_deferred_set_create(); 2885 if (!pool->shared_read_ds) { 2886 *error = "Error creating pool's shared read deferred set"; 2887 err_p = ERR_PTR(-ENOMEM); 2888 goto bad_shared_read_ds; 2889 } 2890 2891 pool->all_io_ds = dm_deferred_set_create(); 2892 if (!pool->all_io_ds) { 2893 *error = "Error creating pool's all io deferred set"; 2894 err_p = ERR_PTR(-ENOMEM); 2895 goto bad_all_io_ds; 2896 } 2897 2898 pool->next_mapping = NULL; 2899 pool->mapping_pool = mempool_create_slab_pool(MAPPING_POOL_SIZE, 2900 _new_mapping_cache); 2901 if (!pool->mapping_pool) { 2902 *error = "Error creating pool's mapping mempool"; 2903 err_p = ERR_PTR(-ENOMEM); 2904 goto bad_mapping_pool; 2905 } 2906 2907 pool->cell_sort_array = vmalloc(sizeof(*pool->cell_sort_array) * CELL_SORT_ARRAY_SIZE); 2908 if (!pool->cell_sort_array) { 2909 *error = "Error allocating cell sort array"; 2910 err_p = ERR_PTR(-ENOMEM); 2911 goto bad_sort_array; 2912 } 2913 2914 pool->ref_count = 1; 2915 pool->last_commit_jiffies = jiffies; 2916 pool->pool_md = pool_md; 2917 pool->md_dev = metadata_dev; 2918 __pool_table_insert(pool); 2919 2920 return pool; 2921 2922 bad_sort_array: 2923 mempool_destroy(pool->mapping_pool); 2924 bad_mapping_pool: 2925 dm_deferred_set_destroy(pool->all_io_ds); 2926 bad_all_io_ds: 2927 dm_deferred_set_destroy(pool->shared_read_ds); 2928 bad_shared_read_ds: 2929 destroy_workqueue(pool->wq); 2930 bad_wq: 2931 dm_kcopyd_client_destroy(pool->copier); 2932 bad_kcopyd_client: 2933 dm_bio_prison_destroy(pool->prison); 2934 bad_prison: 2935 kfree(pool); 2936 bad_pool: 2937 if (dm_pool_metadata_close(pmd)) 2938 DMWARN("%s: dm_pool_metadata_close() failed.", __func__); 2939 2940 return err_p; 2941 } 2942 2943 static void __pool_inc(struct pool *pool) 2944 { 2945 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex)); 2946 pool->ref_count++; 2947 } 2948 2949 static void __pool_dec(struct pool *pool) 2950 { 2951 BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex)); 2952 BUG_ON(!pool->ref_count); 2953 if (!--pool->ref_count) 2954 __pool_destroy(pool); 2955 } 2956 2957 static struct pool *__pool_find(struct mapped_device *pool_md, 2958 struct block_device *metadata_dev, 2959 unsigned long block_size, int read_only, 2960 char **error, int *created) 2961 { 2962 struct pool *pool = __pool_table_lookup_metadata_dev(metadata_dev); 2963 2964 if (pool) { 2965 if (pool->pool_md != pool_md) { 2966 *error = "metadata device already in use by a pool"; 2967 return ERR_PTR(-EBUSY); 2968 } 2969 __pool_inc(pool); 2970 2971 } else { 2972 pool = __pool_table_lookup(pool_md); 2973 if (pool) { 2974 if (pool->md_dev != metadata_dev) { 2975 *error = "different pool cannot replace a pool"; 2976 return ERR_PTR(-EINVAL); 2977 } 2978 __pool_inc(pool); 2979 2980 } else { 2981 pool = pool_create(pool_md, metadata_dev, block_size, read_only, error); 2982 *created = 1; 2983 } 2984 } 2985 2986 return pool; 2987 } 2988 2989 /*---------------------------------------------------------------- 2990 * Pool target methods 2991 *--------------------------------------------------------------*/ 2992 static void pool_dtr(struct dm_target *ti) 2993 { 2994 struct pool_c *pt = ti->private; 2995 2996 mutex_lock(&dm_thin_pool_table.mutex); 2997 2998 unbind_control_target(pt->pool, ti); 2999 __pool_dec(pt->pool); 3000 dm_put_device(ti, pt->metadata_dev); 3001 dm_put_device(ti, pt->data_dev); 3002 kfree(pt); 3003 3004 mutex_unlock(&dm_thin_pool_table.mutex); 3005 } 3006 3007 static int parse_pool_features(struct dm_arg_set *as, struct pool_features *pf, 3008 struct dm_target *ti) 3009 { 3010 int r; 3011 unsigned argc; 3012 const char *arg_name; 3013 3014 static struct dm_arg _args[] = { 3015 {0, 4, "Invalid number of pool feature arguments"}, 3016 }; 3017 3018 /* 3019 * No feature arguments supplied. 3020 */ 3021 if (!as->argc) 3022 return 0; 3023 3024 r = dm_read_arg_group(_args, as, &argc, &ti->error); 3025 if (r) 3026 return -EINVAL; 3027 3028 while (argc && !r) { 3029 arg_name = dm_shift_arg(as); 3030 argc--; 3031 3032 if (!strcasecmp(arg_name, "skip_block_zeroing")) 3033 pf->zero_new_blocks = false; 3034 3035 else if (!strcasecmp(arg_name, "ignore_discard")) 3036 pf->discard_enabled = false; 3037 3038 else if (!strcasecmp(arg_name, "no_discard_passdown")) 3039 pf->discard_passdown = false; 3040 3041 else if (!strcasecmp(arg_name, "read_only")) 3042 pf->mode = PM_READ_ONLY; 3043 3044 else if (!strcasecmp(arg_name, "error_if_no_space")) 3045 pf->error_if_no_space = true; 3046 3047 else { 3048 ti->error = "Unrecognised pool feature requested"; 3049 r = -EINVAL; 3050 break; 3051 } 3052 } 3053 3054 return r; 3055 } 3056 3057 static void metadata_low_callback(void *context) 3058 { 3059 struct pool *pool = context; 3060 3061 DMWARN("%s: reached low water mark for metadata device: sending event.", 3062 dm_device_name(pool->pool_md)); 3063 3064 dm_table_event(pool->ti->table); 3065 } 3066 3067 static sector_t get_dev_size(struct block_device *bdev) 3068 { 3069 return i_size_read(bdev->bd_inode) >> SECTOR_SHIFT; 3070 } 3071 3072 static void warn_if_metadata_device_too_big(struct block_device *bdev) 3073 { 3074 sector_t metadata_dev_size = get_dev_size(bdev); 3075 char buffer[BDEVNAME_SIZE]; 3076 3077 if (metadata_dev_size > THIN_METADATA_MAX_SECTORS_WARNING) 3078 DMWARN("Metadata device %s is larger than %u sectors: excess space will not be used.", 3079 bdevname(bdev, buffer), THIN_METADATA_MAX_SECTORS); 3080 } 3081 3082 static sector_t get_metadata_dev_size(struct block_device *bdev) 3083 { 3084 sector_t metadata_dev_size = get_dev_size(bdev); 3085 3086 if (metadata_dev_size > THIN_METADATA_MAX_SECTORS) 3087 metadata_dev_size = THIN_METADATA_MAX_SECTORS; 3088 3089 return metadata_dev_size; 3090 } 3091 3092 static dm_block_t get_metadata_dev_size_in_blocks(struct block_device *bdev) 3093 { 3094 sector_t metadata_dev_size = get_metadata_dev_size(bdev); 3095 3096 sector_div(metadata_dev_size, THIN_METADATA_BLOCK_SIZE); 3097 3098 return metadata_dev_size; 3099 } 3100 3101 /* 3102 * When a metadata threshold is crossed a dm event is triggered, and 3103 * userland should respond by growing the metadata device. We could let 3104 * userland set the threshold, like we do with the data threshold, but I'm 3105 * not sure they know enough to do this well. 3106 */ 3107 static dm_block_t calc_metadata_threshold(struct pool_c *pt) 3108 { 3109 /* 3110 * 4M is ample for all ops with the possible exception of thin 3111 * device deletion which is harmless if it fails (just retry the 3112 * delete after you've grown the device). 3113 */ 3114 dm_block_t quarter = get_metadata_dev_size_in_blocks(pt->metadata_dev->bdev) / 4; 3115 return min((dm_block_t)1024ULL /* 4M */, quarter); 3116 } 3117 3118 /* 3119 * thin-pool <metadata dev> <data dev> 3120 * <data block size (sectors)> 3121 * <low water mark (blocks)> 3122 * [<#feature args> [<arg>]*] 3123 * 3124 * Optional feature arguments are: 3125 * skip_block_zeroing: skips the zeroing of newly-provisioned blocks. 3126 * ignore_discard: disable discard 3127 * no_discard_passdown: don't pass discards down to the data device 3128 * read_only: Don't allow any changes to be made to the pool metadata. 3129 * error_if_no_space: error IOs, instead of queueing, if no space. 3130 */ 3131 static int pool_ctr(struct dm_target *ti, unsigned argc, char **argv) 3132 { 3133 int r, pool_created = 0; 3134 struct pool_c *pt; 3135 struct pool *pool; 3136 struct pool_features pf; 3137 struct dm_arg_set as; 3138 struct dm_dev *data_dev; 3139 unsigned long block_size; 3140 dm_block_t low_water_blocks; 3141 struct dm_dev *metadata_dev; 3142 fmode_t metadata_mode; 3143 3144 /* 3145 * FIXME Remove validation from scope of lock. 3146 */ 3147 mutex_lock(&dm_thin_pool_table.mutex); 3148 3149 if (argc < 4) { 3150 ti->error = "Invalid argument count"; 3151 r = -EINVAL; 3152 goto out_unlock; 3153 } 3154 3155 as.argc = argc; 3156 as.argv = argv; 3157 3158 /* 3159 * Set default pool features. 3160 */ 3161 pool_features_init(&pf); 3162 3163 dm_consume_args(&as, 4); 3164 r = parse_pool_features(&as, &pf, ti); 3165 if (r) 3166 goto out_unlock; 3167 3168 metadata_mode = FMODE_READ | ((pf.mode == PM_READ_ONLY) ? 0 : FMODE_WRITE); 3169 r = dm_get_device(ti, argv[0], metadata_mode, &metadata_dev); 3170 if (r) { 3171 ti->error = "Error opening metadata block device"; 3172 goto out_unlock; 3173 } 3174 warn_if_metadata_device_too_big(metadata_dev->bdev); 3175 3176 r = dm_get_device(ti, argv[1], FMODE_READ | FMODE_WRITE, &data_dev); 3177 if (r) { 3178 ti->error = "Error getting data device"; 3179 goto out_metadata; 3180 } 3181 3182 if (kstrtoul(argv[2], 10, &block_size) || !block_size || 3183 block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS || 3184 block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS || 3185 block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) { 3186 ti->error = "Invalid block size"; 3187 r = -EINVAL; 3188 goto out; 3189 } 3190 3191 if (kstrtoull(argv[3], 10, (unsigned long long *)&low_water_blocks)) { 3192 ti->error = "Invalid low water mark"; 3193 r = -EINVAL; 3194 goto out; 3195 } 3196 3197 pt = kzalloc(sizeof(*pt), GFP_KERNEL); 3198 if (!pt) { 3199 r = -ENOMEM; 3200 goto out; 3201 } 3202 3203 pool = __pool_find(dm_table_get_md(ti->table), metadata_dev->bdev, 3204 block_size, pf.mode == PM_READ_ONLY, &ti->error, &pool_created); 3205 if (IS_ERR(pool)) { 3206 r = PTR_ERR(pool); 3207 goto out_free_pt; 3208 } 3209 3210 /* 3211 * 'pool_created' reflects whether this is the first table load. 3212 * Top level discard support is not allowed to be changed after 3213 * initial load. This would require a pool reload to trigger thin 3214 * device changes. 3215 */ 3216 if (!pool_created && pf.discard_enabled != pool->pf.discard_enabled) { 3217 ti->error = "Discard support cannot be disabled once enabled"; 3218 r = -EINVAL; 3219 goto out_flags_changed; 3220 } 3221 3222 pt->pool = pool; 3223 pt->ti = ti; 3224 pt->metadata_dev = metadata_dev; 3225 pt->data_dev = data_dev; 3226 pt->low_water_blocks = low_water_blocks; 3227 pt->adjusted_pf = pt->requested_pf = pf; 3228 ti->num_flush_bios = 1; 3229 3230 /* 3231 * Only need to enable discards if the pool should pass 3232 * them down to the data device. The thin device's discard 3233 * processing will cause mappings to be removed from the btree. 3234 */ 3235 ti->discard_zeroes_data_unsupported = true; 3236 if (pf.discard_enabled && pf.discard_passdown) { 3237 ti->num_discard_bios = 1; 3238 3239 /* 3240 * Setting 'discards_supported' circumvents the normal 3241 * stacking of discard limits (this keeps the pool and 3242 * thin devices' discard limits consistent). 3243 */ 3244 ti->discards_supported = true; 3245 } 3246 ti->private = pt; 3247 3248 r = dm_pool_register_metadata_threshold(pt->pool->pmd, 3249 calc_metadata_threshold(pt), 3250 metadata_low_callback, 3251 pool); 3252 if (r) 3253 goto out_free_pt; 3254 3255 pt->callbacks.congested_fn = pool_is_congested; 3256 dm_table_add_target_callbacks(ti->table, &pt->callbacks); 3257 3258 mutex_unlock(&dm_thin_pool_table.mutex); 3259 3260 return 0; 3261 3262 out_flags_changed: 3263 __pool_dec(pool); 3264 out_free_pt: 3265 kfree(pt); 3266 out: 3267 dm_put_device(ti, data_dev); 3268 out_metadata: 3269 dm_put_device(ti, metadata_dev); 3270 out_unlock: 3271 mutex_unlock(&dm_thin_pool_table.mutex); 3272 3273 return r; 3274 } 3275 3276 static int pool_map(struct dm_target *ti, struct bio *bio) 3277 { 3278 int r; 3279 struct pool_c *pt = ti->private; 3280 struct pool *pool = pt->pool; 3281 unsigned long flags; 3282 3283 /* 3284 * As this is a singleton target, ti->begin is always zero. 3285 */ 3286 spin_lock_irqsave(&pool->lock, flags); 3287 bio->bi_bdev = pt->data_dev->bdev; 3288 r = DM_MAPIO_REMAPPED; 3289 spin_unlock_irqrestore(&pool->lock, flags); 3290 3291 return r; 3292 } 3293 3294 static int maybe_resize_data_dev(struct dm_target *ti, bool *need_commit) 3295 { 3296 int r; 3297 struct pool_c *pt = ti->private; 3298 struct pool *pool = pt->pool; 3299 sector_t data_size = ti->len; 3300 dm_block_t sb_data_size; 3301 3302 *need_commit = false; 3303 3304 (void) sector_div(data_size, pool->sectors_per_block); 3305 3306 r = dm_pool_get_data_dev_size(pool->pmd, &sb_data_size); 3307 if (r) { 3308 DMERR("%s: failed to retrieve data device size", 3309 dm_device_name(pool->pool_md)); 3310 return r; 3311 } 3312 3313 if (data_size < sb_data_size) { 3314 DMERR("%s: pool target (%llu blocks) too small: expected %llu", 3315 dm_device_name(pool->pool_md), 3316 (unsigned long long)data_size, sb_data_size); 3317 return -EINVAL; 3318 3319 } else if (data_size > sb_data_size) { 3320 if (dm_pool_metadata_needs_check(pool->pmd)) { 3321 DMERR("%s: unable to grow the data device until repaired.", 3322 dm_device_name(pool->pool_md)); 3323 return 0; 3324 } 3325 3326 if (sb_data_size) 3327 DMINFO("%s: growing the data device from %llu to %llu blocks", 3328 dm_device_name(pool->pool_md), 3329 sb_data_size, (unsigned long long)data_size); 3330 r = dm_pool_resize_data_dev(pool->pmd, data_size); 3331 if (r) { 3332 metadata_operation_failed(pool, "dm_pool_resize_data_dev", r); 3333 return r; 3334 } 3335 3336 *need_commit = true; 3337 } 3338 3339 return 0; 3340 } 3341 3342 static int maybe_resize_metadata_dev(struct dm_target *ti, bool *need_commit) 3343 { 3344 int r; 3345 struct pool_c *pt = ti->private; 3346 struct pool *pool = pt->pool; 3347 dm_block_t metadata_dev_size, sb_metadata_dev_size; 3348 3349 *need_commit = false; 3350 3351 metadata_dev_size = get_metadata_dev_size_in_blocks(pool->md_dev); 3352 3353 r = dm_pool_get_metadata_dev_size(pool->pmd, &sb_metadata_dev_size); 3354 if (r) { 3355 DMERR("%s: failed to retrieve metadata device size", 3356 dm_device_name(pool->pool_md)); 3357 return r; 3358 } 3359 3360 if (metadata_dev_size < sb_metadata_dev_size) { 3361 DMERR("%s: metadata device (%llu blocks) too small: expected %llu", 3362 dm_device_name(pool->pool_md), 3363 metadata_dev_size, sb_metadata_dev_size); 3364 return -EINVAL; 3365 3366 } else if (metadata_dev_size > sb_metadata_dev_size) { 3367 if (dm_pool_metadata_needs_check(pool->pmd)) { 3368 DMERR("%s: unable to grow the metadata device until repaired.", 3369 dm_device_name(pool->pool_md)); 3370 return 0; 3371 } 3372 3373 warn_if_metadata_device_too_big(pool->md_dev); 3374 DMINFO("%s: growing the metadata device from %llu to %llu blocks", 3375 dm_device_name(pool->pool_md), 3376 sb_metadata_dev_size, metadata_dev_size); 3377 r = dm_pool_resize_metadata_dev(pool->pmd, metadata_dev_size); 3378 if (r) { 3379 metadata_operation_failed(pool, "dm_pool_resize_metadata_dev", r); 3380 return r; 3381 } 3382 3383 *need_commit = true; 3384 } 3385 3386 return 0; 3387 } 3388 3389 /* 3390 * Retrieves the number of blocks of the data device from 3391 * the superblock and compares it to the actual device size, 3392 * thus resizing the data device in case it has grown. 3393 * 3394 * This both copes with opening preallocated data devices in the ctr 3395 * being followed by a resume 3396 * -and- 3397 * calling the resume method individually after userspace has 3398 * grown the data device in reaction to a table event. 3399 */ 3400 static int pool_preresume(struct dm_target *ti) 3401 { 3402 int r; 3403 bool need_commit1, need_commit2; 3404 struct pool_c *pt = ti->private; 3405 struct pool *pool = pt->pool; 3406 3407 /* 3408 * Take control of the pool object. 3409 */ 3410 r = bind_control_target(pool, ti); 3411 if (r) 3412 return r; 3413 3414 r = maybe_resize_data_dev(ti, &need_commit1); 3415 if (r) 3416 return r; 3417 3418 r = maybe_resize_metadata_dev(ti, &need_commit2); 3419 if (r) 3420 return r; 3421 3422 if (need_commit1 || need_commit2) 3423 (void) commit(pool); 3424 3425 return 0; 3426 } 3427 3428 static void pool_suspend_active_thins(struct pool *pool) 3429 { 3430 struct thin_c *tc; 3431 3432 /* Suspend all active thin devices */ 3433 tc = get_first_thin(pool); 3434 while (tc) { 3435 dm_internal_suspend_noflush(tc->thin_md); 3436 tc = get_next_thin(pool, tc); 3437 } 3438 } 3439 3440 static void pool_resume_active_thins(struct pool *pool) 3441 { 3442 struct thin_c *tc; 3443 3444 /* Resume all active thin devices */ 3445 tc = get_first_thin(pool); 3446 while (tc) { 3447 dm_internal_resume(tc->thin_md); 3448 tc = get_next_thin(pool, tc); 3449 } 3450 } 3451 3452 static void pool_resume(struct dm_target *ti) 3453 { 3454 struct pool_c *pt = ti->private; 3455 struct pool *pool = pt->pool; 3456 unsigned long flags; 3457 3458 /* 3459 * Must requeue active_thins' bios and then resume 3460 * active_thins _before_ clearing 'suspend' flag. 3461 */ 3462 requeue_bios(pool); 3463 pool_resume_active_thins(pool); 3464 3465 spin_lock_irqsave(&pool->lock, flags); 3466 pool->low_water_triggered = false; 3467 pool->suspended = false; 3468 spin_unlock_irqrestore(&pool->lock, flags); 3469 3470 do_waker(&pool->waker.work); 3471 } 3472 3473 static void pool_presuspend(struct dm_target *ti) 3474 { 3475 struct pool_c *pt = ti->private; 3476 struct pool *pool = pt->pool; 3477 unsigned long flags; 3478 3479 spin_lock_irqsave(&pool->lock, flags); 3480 pool->suspended = true; 3481 spin_unlock_irqrestore(&pool->lock, flags); 3482 3483 pool_suspend_active_thins(pool); 3484 } 3485 3486 static void pool_presuspend_undo(struct dm_target *ti) 3487 { 3488 struct pool_c *pt = ti->private; 3489 struct pool *pool = pt->pool; 3490 unsigned long flags; 3491 3492 pool_resume_active_thins(pool); 3493 3494 spin_lock_irqsave(&pool->lock, flags); 3495 pool->suspended = false; 3496 spin_unlock_irqrestore(&pool->lock, flags); 3497 } 3498 3499 static void pool_postsuspend(struct dm_target *ti) 3500 { 3501 struct pool_c *pt = ti->private; 3502 struct pool *pool = pt->pool; 3503 3504 cancel_delayed_work(&pool->waker); 3505 cancel_delayed_work(&pool->no_space_timeout); 3506 flush_workqueue(pool->wq); 3507 (void) commit(pool); 3508 } 3509 3510 static int check_arg_count(unsigned argc, unsigned args_required) 3511 { 3512 if (argc != args_required) { 3513 DMWARN("Message received with %u arguments instead of %u.", 3514 argc, args_required); 3515 return -EINVAL; 3516 } 3517 3518 return 0; 3519 } 3520 3521 static int read_dev_id(char *arg, dm_thin_id *dev_id, int warning) 3522 { 3523 if (!kstrtoull(arg, 10, (unsigned long long *)dev_id) && 3524 *dev_id <= MAX_DEV_ID) 3525 return 0; 3526 3527 if (warning) 3528 DMWARN("Message received with invalid device id: %s", arg); 3529 3530 return -EINVAL; 3531 } 3532 3533 static int process_create_thin_mesg(unsigned argc, char **argv, struct pool *pool) 3534 { 3535 dm_thin_id dev_id; 3536 int r; 3537 3538 r = check_arg_count(argc, 2); 3539 if (r) 3540 return r; 3541 3542 r = read_dev_id(argv[1], &dev_id, 1); 3543 if (r) 3544 return r; 3545 3546 r = dm_pool_create_thin(pool->pmd, dev_id); 3547 if (r) { 3548 DMWARN("Creation of new thinly-provisioned device with id %s failed.", 3549 argv[1]); 3550 return r; 3551 } 3552 3553 return 0; 3554 } 3555 3556 static int process_create_snap_mesg(unsigned argc, char **argv, struct pool *pool) 3557 { 3558 dm_thin_id dev_id; 3559 dm_thin_id origin_dev_id; 3560 int r; 3561 3562 r = check_arg_count(argc, 3); 3563 if (r) 3564 return r; 3565 3566 r = read_dev_id(argv[1], &dev_id, 1); 3567 if (r) 3568 return r; 3569 3570 r = read_dev_id(argv[2], &origin_dev_id, 1); 3571 if (r) 3572 return r; 3573 3574 r = dm_pool_create_snap(pool->pmd, dev_id, origin_dev_id); 3575 if (r) { 3576 DMWARN("Creation of new snapshot %s of device %s failed.", 3577 argv[1], argv[2]); 3578 return r; 3579 } 3580 3581 return 0; 3582 } 3583 3584 static int process_delete_mesg(unsigned argc, char **argv, struct pool *pool) 3585 { 3586 dm_thin_id dev_id; 3587 int r; 3588 3589 r = check_arg_count(argc, 2); 3590 if (r) 3591 return r; 3592 3593 r = read_dev_id(argv[1], &dev_id, 1); 3594 if (r) 3595 return r; 3596 3597 r = dm_pool_delete_thin_device(pool->pmd, dev_id); 3598 if (r) 3599 DMWARN("Deletion of thin device %s failed.", argv[1]); 3600 3601 return r; 3602 } 3603 3604 static int process_set_transaction_id_mesg(unsigned argc, char **argv, struct pool *pool) 3605 { 3606 dm_thin_id old_id, new_id; 3607 int r; 3608 3609 r = check_arg_count(argc, 3); 3610 if (r) 3611 return r; 3612 3613 if (kstrtoull(argv[1], 10, (unsigned long long *)&old_id)) { 3614 DMWARN("set_transaction_id message: Unrecognised id %s.", argv[1]); 3615 return -EINVAL; 3616 } 3617 3618 if (kstrtoull(argv[2], 10, (unsigned long long *)&new_id)) { 3619 DMWARN("set_transaction_id message: Unrecognised new id %s.", argv[2]); 3620 return -EINVAL; 3621 } 3622 3623 r = dm_pool_set_metadata_transaction_id(pool->pmd, old_id, new_id); 3624 if (r) { 3625 DMWARN("Failed to change transaction id from %s to %s.", 3626 argv[1], argv[2]); 3627 return r; 3628 } 3629 3630 return 0; 3631 } 3632 3633 static int process_reserve_metadata_snap_mesg(unsigned argc, char **argv, struct pool *pool) 3634 { 3635 int r; 3636 3637 r = check_arg_count(argc, 1); 3638 if (r) 3639 return r; 3640 3641 (void) commit(pool); 3642 3643 r = dm_pool_reserve_metadata_snap(pool->pmd); 3644 if (r) 3645 DMWARN("reserve_metadata_snap message failed."); 3646 3647 return r; 3648 } 3649 3650 static int process_release_metadata_snap_mesg(unsigned argc, char **argv, struct pool *pool) 3651 { 3652 int r; 3653 3654 r = check_arg_count(argc, 1); 3655 if (r) 3656 return r; 3657 3658 r = dm_pool_release_metadata_snap(pool->pmd); 3659 if (r) 3660 DMWARN("release_metadata_snap message failed."); 3661 3662 return r; 3663 } 3664 3665 /* 3666 * Messages supported: 3667 * create_thin <dev_id> 3668 * create_snap <dev_id> <origin_id> 3669 * delete <dev_id> 3670 * set_transaction_id <current_trans_id> <new_trans_id> 3671 * reserve_metadata_snap 3672 * release_metadata_snap 3673 */ 3674 static int pool_message(struct dm_target *ti, unsigned argc, char **argv) 3675 { 3676 int r = -EINVAL; 3677 struct pool_c *pt = ti->private; 3678 struct pool *pool = pt->pool; 3679 3680 if (get_pool_mode(pool) >= PM_READ_ONLY) { 3681 DMERR("%s: unable to service pool target messages in READ_ONLY or FAIL mode", 3682 dm_device_name(pool->pool_md)); 3683 return -EOPNOTSUPP; 3684 } 3685 3686 if (!strcasecmp(argv[0], "create_thin")) 3687 r = process_create_thin_mesg(argc, argv, pool); 3688 3689 else if (!strcasecmp(argv[0], "create_snap")) 3690 r = process_create_snap_mesg(argc, argv, pool); 3691 3692 else if (!strcasecmp(argv[0], "delete")) 3693 r = process_delete_mesg(argc, argv, pool); 3694 3695 else if (!strcasecmp(argv[0], "set_transaction_id")) 3696 r = process_set_transaction_id_mesg(argc, argv, pool); 3697 3698 else if (!strcasecmp(argv[0], "reserve_metadata_snap")) 3699 r = process_reserve_metadata_snap_mesg(argc, argv, pool); 3700 3701 else if (!strcasecmp(argv[0], "release_metadata_snap")) 3702 r = process_release_metadata_snap_mesg(argc, argv, pool); 3703 3704 else 3705 DMWARN("Unrecognised thin pool target message received: %s", argv[0]); 3706 3707 if (!r) 3708 (void) commit(pool); 3709 3710 return r; 3711 } 3712 3713 static void emit_flags(struct pool_features *pf, char *result, 3714 unsigned sz, unsigned maxlen) 3715 { 3716 unsigned count = !pf->zero_new_blocks + !pf->discard_enabled + 3717 !pf->discard_passdown + (pf->mode == PM_READ_ONLY) + 3718 pf->error_if_no_space; 3719 DMEMIT("%u ", count); 3720 3721 if (!pf->zero_new_blocks) 3722 DMEMIT("skip_block_zeroing "); 3723 3724 if (!pf->discard_enabled) 3725 DMEMIT("ignore_discard "); 3726 3727 if (!pf->discard_passdown) 3728 DMEMIT("no_discard_passdown "); 3729 3730 if (pf->mode == PM_READ_ONLY) 3731 DMEMIT("read_only "); 3732 3733 if (pf->error_if_no_space) 3734 DMEMIT("error_if_no_space "); 3735 } 3736 3737 /* 3738 * Status line is: 3739 * <transaction id> <used metadata sectors>/<total metadata sectors> 3740 * <used data sectors>/<total data sectors> <held metadata root> 3741 * <pool mode> <discard config> <no space config> <needs_check> 3742 */ 3743 static void pool_status(struct dm_target *ti, status_type_t type, 3744 unsigned status_flags, char *result, unsigned maxlen) 3745 { 3746 int r; 3747 unsigned sz = 0; 3748 uint64_t transaction_id; 3749 dm_block_t nr_free_blocks_data; 3750 dm_block_t nr_free_blocks_metadata; 3751 dm_block_t nr_blocks_data; 3752 dm_block_t nr_blocks_metadata; 3753 dm_block_t held_root; 3754 char buf[BDEVNAME_SIZE]; 3755 char buf2[BDEVNAME_SIZE]; 3756 struct pool_c *pt = ti->private; 3757 struct pool *pool = pt->pool; 3758 3759 switch (type) { 3760 case STATUSTYPE_INFO: 3761 if (get_pool_mode(pool) == PM_FAIL) { 3762 DMEMIT("Fail"); 3763 break; 3764 } 3765 3766 /* Commit to ensure statistics aren't out-of-date */ 3767 if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti)) 3768 (void) commit(pool); 3769 3770 r = dm_pool_get_metadata_transaction_id(pool->pmd, &transaction_id); 3771 if (r) { 3772 DMERR("%s: dm_pool_get_metadata_transaction_id returned %d", 3773 dm_device_name(pool->pool_md), r); 3774 goto err; 3775 } 3776 3777 r = dm_pool_get_free_metadata_block_count(pool->pmd, &nr_free_blocks_metadata); 3778 if (r) { 3779 DMERR("%s: dm_pool_get_free_metadata_block_count returned %d", 3780 dm_device_name(pool->pool_md), r); 3781 goto err; 3782 } 3783 3784 r = dm_pool_get_metadata_dev_size(pool->pmd, &nr_blocks_metadata); 3785 if (r) { 3786 DMERR("%s: dm_pool_get_metadata_dev_size returned %d", 3787 dm_device_name(pool->pool_md), r); 3788 goto err; 3789 } 3790 3791 r = dm_pool_get_free_block_count(pool->pmd, &nr_free_blocks_data); 3792 if (r) { 3793 DMERR("%s: dm_pool_get_free_block_count returned %d", 3794 dm_device_name(pool->pool_md), r); 3795 goto err; 3796 } 3797 3798 r = dm_pool_get_data_dev_size(pool->pmd, &nr_blocks_data); 3799 if (r) { 3800 DMERR("%s: dm_pool_get_data_dev_size returned %d", 3801 dm_device_name(pool->pool_md), r); 3802 goto err; 3803 } 3804 3805 r = dm_pool_get_metadata_snap(pool->pmd, &held_root); 3806 if (r) { 3807 DMERR("%s: dm_pool_get_metadata_snap returned %d", 3808 dm_device_name(pool->pool_md), r); 3809 goto err; 3810 } 3811 3812 DMEMIT("%llu %llu/%llu %llu/%llu ", 3813 (unsigned long long)transaction_id, 3814 (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata), 3815 (unsigned long long)nr_blocks_metadata, 3816 (unsigned long long)(nr_blocks_data - nr_free_blocks_data), 3817 (unsigned long long)nr_blocks_data); 3818 3819 if (held_root) 3820 DMEMIT("%llu ", held_root); 3821 else 3822 DMEMIT("- "); 3823 3824 if (pool->pf.mode == PM_OUT_OF_DATA_SPACE) 3825 DMEMIT("out_of_data_space "); 3826 else if (pool->pf.mode == PM_READ_ONLY) 3827 DMEMIT("ro "); 3828 else 3829 DMEMIT("rw "); 3830 3831 if (!pool->pf.discard_enabled) 3832 DMEMIT("ignore_discard "); 3833 else if (pool->pf.discard_passdown) 3834 DMEMIT("discard_passdown "); 3835 else 3836 DMEMIT("no_discard_passdown "); 3837 3838 if (pool->pf.error_if_no_space) 3839 DMEMIT("error_if_no_space "); 3840 else 3841 DMEMIT("queue_if_no_space "); 3842 3843 if (dm_pool_metadata_needs_check(pool->pmd)) 3844 DMEMIT("needs_check "); 3845 else 3846 DMEMIT("- "); 3847 3848 break; 3849 3850 case STATUSTYPE_TABLE: 3851 DMEMIT("%s %s %lu %llu ", 3852 format_dev_t(buf, pt->metadata_dev->bdev->bd_dev), 3853 format_dev_t(buf2, pt->data_dev->bdev->bd_dev), 3854 (unsigned long)pool->sectors_per_block, 3855 (unsigned long long)pt->low_water_blocks); 3856 emit_flags(&pt->requested_pf, result, sz, maxlen); 3857 break; 3858 } 3859 return; 3860 3861 err: 3862 DMEMIT("Error"); 3863 } 3864 3865 static int pool_iterate_devices(struct dm_target *ti, 3866 iterate_devices_callout_fn fn, void *data) 3867 { 3868 struct pool_c *pt = ti->private; 3869 3870 return fn(ti, pt->data_dev, 0, ti->len, data); 3871 } 3872 3873 static int pool_merge(struct dm_target *ti, struct bvec_merge_data *bvm, 3874 struct bio_vec *biovec, int max_size) 3875 { 3876 struct pool_c *pt = ti->private; 3877 struct request_queue *q = bdev_get_queue(pt->data_dev->bdev); 3878 3879 if (!q->merge_bvec_fn) 3880 return max_size; 3881 3882 bvm->bi_bdev = pt->data_dev->bdev; 3883 3884 return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); 3885 } 3886 3887 static void pool_io_hints(struct dm_target *ti, struct queue_limits *limits) 3888 { 3889 struct pool_c *pt = ti->private; 3890 struct pool *pool = pt->pool; 3891 sector_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT; 3892 3893 /* 3894 * If max_sectors is smaller than pool->sectors_per_block adjust it 3895 * to the highest possible power-of-2 factor of pool->sectors_per_block. 3896 * This is especially beneficial when the pool's data device is a RAID 3897 * device that has a full stripe width that matches pool->sectors_per_block 3898 * -- because even though partial RAID stripe-sized IOs will be issued to a 3899 * single RAID stripe; when aggregated they will end on a full RAID stripe 3900 * boundary.. which avoids additional partial RAID stripe writes cascading 3901 */ 3902 if (limits->max_sectors < pool->sectors_per_block) { 3903 while (!is_factor(pool->sectors_per_block, limits->max_sectors)) { 3904 if ((limits->max_sectors & (limits->max_sectors - 1)) == 0) 3905 limits->max_sectors--; 3906 limits->max_sectors = rounddown_pow_of_two(limits->max_sectors); 3907 } 3908 } 3909 3910 /* 3911 * If the system-determined stacked limits are compatible with the 3912 * pool's blocksize (io_opt is a factor) do not override them. 3913 */ 3914 if (io_opt_sectors < pool->sectors_per_block || 3915 !is_factor(io_opt_sectors, pool->sectors_per_block)) { 3916 if (is_factor(pool->sectors_per_block, limits->max_sectors)) 3917 blk_limits_io_min(limits, limits->max_sectors << SECTOR_SHIFT); 3918 else 3919 blk_limits_io_min(limits, pool->sectors_per_block << SECTOR_SHIFT); 3920 blk_limits_io_opt(limits, pool->sectors_per_block << SECTOR_SHIFT); 3921 } 3922 3923 /* 3924 * pt->adjusted_pf is a staging area for the actual features to use. 3925 * They get transferred to the live pool in bind_control_target() 3926 * called from pool_preresume(). 3927 */ 3928 if (!pt->adjusted_pf.discard_enabled) { 3929 /* 3930 * Must explicitly disallow stacking discard limits otherwise the 3931 * block layer will stack them if pool's data device has support. 3932 * QUEUE_FLAG_DISCARD wouldn't be set but there is no way for the 3933 * user to see that, so make sure to set all discard limits to 0. 3934 */ 3935 limits->discard_granularity = 0; 3936 return; 3937 } 3938 3939 disable_passdown_if_not_supported(pt); 3940 3941 /* 3942 * The pool uses the same discard limits as the underlying data 3943 * device. DM core has already set this up. 3944 */ 3945 } 3946 3947 static struct target_type pool_target = { 3948 .name = "thin-pool", 3949 .features = DM_TARGET_SINGLETON | DM_TARGET_ALWAYS_WRITEABLE | 3950 DM_TARGET_IMMUTABLE, 3951 .version = {1, 16, 0}, 3952 .module = THIS_MODULE, 3953 .ctr = pool_ctr, 3954 .dtr = pool_dtr, 3955 .map = pool_map, 3956 .presuspend = pool_presuspend, 3957 .presuspend_undo = pool_presuspend_undo, 3958 .postsuspend = pool_postsuspend, 3959 .preresume = pool_preresume, 3960 .resume = pool_resume, 3961 .message = pool_message, 3962 .status = pool_status, 3963 .merge = pool_merge, 3964 .iterate_devices = pool_iterate_devices, 3965 .io_hints = pool_io_hints, 3966 }; 3967 3968 /*---------------------------------------------------------------- 3969 * Thin target methods 3970 *--------------------------------------------------------------*/ 3971 static void thin_get(struct thin_c *tc) 3972 { 3973 atomic_inc(&tc->refcount); 3974 } 3975 3976 static void thin_put(struct thin_c *tc) 3977 { 3978 if (atomic_dec_and_test(&tc->refcount)) 3979 complete(&tc->can_destroy); 3980 } 3981 3982 static void thin_dtr(struct dm_target *ti) 3983 { 3984 struct thin_c *tc = ti->private; 3985 unsigned long flags; 3986 3987 spin_lock_irqsave(&tc->pool->lock, flags); 3988 list_del_rcu(&tc->list); 3989 spin_unlock_irqrestore(&tc->pool->lock, flags); 3990 synchronize_rcu(); 3991 3992 thin_put(tc); 3993 wait_for_completion(&tc->can_destroy); 3994 3995 mutex_lock(&dm_thin_pool_table.mutex); 3996 3997 __pool_dec(tc->pool); 3998 dm_pool_close_thin_device(tc->td); 3999 dm_put_device(ti, tc->pool_dev); 4000 if (tc->origin_dev) 4001 dm_put_device(ti, tc->origin_dev); 4002 kfree(tc); 4003 4004 mutex_unlock(&dm_thin_pool_table.mutex); 4005 } 4006 4007 /* 4008 * Thin target parameters: 4009 * 4010 * <pool_dev> <dev_id> [origin_dev] 4011 * 4012 * pool_dev: the path to the pool (eg, /dev/mapper/my_pool) 4013 * dev_id: the internal device identifier 4014 * origin_dev: a device external to the pool that should act as the origin 4015 * 4016 * If the pool device has discards disabled, they get disabled for the thin 4017 * device as well. 4018 */ 4019 static int thin_ctr(struct dm_target *ti, unsigned argc, char **argv) 4020 { 4021 int r; 4022 struct thin_c *tc; 4023 struct dm_dev *pool_dev, *origin_dev; 4024 struct mapped_device *pool_md; 4025 unsigned long flags; 4026 4027 mutex_lock(&dm_thin_pool_table.mutex); 4028 4029 if (argc != 2 && argc != 3) { 4030 ti->error = "Invalid argument count"; 4031 r = -EINVAL; 4032 goto out_unlock; 4033 } 4034 4035 tc = ti->private = kzalloc(sizeof(*tc), GFP_KERNEL); 4036 if (!tc) { 4037 ti->error = "Out of memory"; 4038 r = -ENOMEM; 4039 goto out_unlock; 4040 } 4041 tc->thin_md = dm_table_get_md(ti->table); 4042 spin_lock_init(&tc->lock); 4043 INIT_LIST_HEAD(&tc->deferred_cells); 4044 bio_list_init(&tc->deferred_bio_list); 4045 bio_list_init(&tc->retry_on_resume_list); 4046 tc->sort_bio_list = RB_ROOT; 4047 4048 if (argc == 3) { 4049 r = dm_get_device(ti, argv[2], FMODE_READ, &origin_dev); 4050 if (r) { 4051 ti->error = "Error opening origin device"; 4052 goto bad_origin_dev; 4053 } 4054 tc->origin_dev = origin_dev; 4055 } 4056 4057 r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &pool_dev); 4058 if (r) { 4059 ti->error = "Error opening pool device"; 4060 goto bad_pool_dev; 4061 } 4062 tc->pool_dev = pool_dev; 4063 4064 if (read_dev_id(argv[1], (unsigned long long *)&tc->dev_id, 0)) { 4065 ti->error = "Invalid device id"; 4066 r = -EINVAL; 4067 goto bad_common; 4068 } 4069 4070 pool_md = dm_get_md(tc->pool_dev->bdev->bd_dev); 4071 if (!pool_md) { 4072 ti->error = "Couldn't get pool mapped device"; 4073 r = -EINVAL; 4074 goto bad_common; 4075 } 4076 4077 tc->pool = __pool_table_lookup(pool_md); 4078 if (!tc->pool) { 4079 ti->error = "Couldn't find pool object"; 4080 r = -EINVAL; 4081 goto bad_pool_lookup; 4082 } 4083 __pool_inc(tc->pool); 4084 4085 if (get_pool_mode(tc->pool) == PM_FAIL) { 4086 ti->error = "Couldn't open thin device, Pool is in fail mode"; 4087 r = -EINVAL; 4088 goto bad_pool; 4089 } 4090 4091 r = dm_pool_open_thin_device(tc->pool->pmd, tc->dev_id, &tc->td); 4092 if (r) { 4093 ti->error = "Couldn't open thin internal device"; 4094 goto bad_pool; 4095 } 4096 4097 r = dm_set_target_max_io_len(ti, tc->pool->sectors_per_block); 4098 if (r) 4099 goto bad; 4100 4101 ti->num_flush_bios = 1; 4102 ti->flush_supported = true; 4103 ti->per_bio_data_size = sizeof(struct dm_thin_endio_hook); 4104 4105 /* In case the pool supports discards, pass them on. */ 4106 ti->discard_zeroes_data_unsupported = true; 4107 if (tc->pool->pf.discard_enabled) { 4108 ti->discards_supported = true; 4109 ti->num_discard_bios = 1; 4110 ti->split_discard_bios = false; 4111 } 4112 4113 mutex_unlock(&dm_thin_pool_table.mutex); 4114 4115 spin_lock_irqsave(&tc->pool->lock, flags); 4116 if (tc->pool->suspended) { 4117 spin_unlock_irqrestore(&tc->pool->lock, flags); 4118 mutex_lock(&dm_thin_pool_table.mutex); /* reacquire for __pool_dec */ 4119 ti->error = "Unable to activate thin device while pool is suspended"; 4120 r = -EINVAL; 4121 goto bad; 4122 } 4123 atomic_set(&tc->refcount, 1); 4124 init_completion(&tc->can_destroy); 4125 list_add_tail_rcu(&tc->list, &tc->pool->active_thins); 4126 spin_unlock_irqrestore(&tc->pool->lock, flags); 4127 /* 4128 * This synchronize_rcu() call is needed here otherwise we risk a 4129 * wake_worker() call finding no bios to process (because the newly 4130 * added tc isn't yet visible). So this reduces latency since we 4131 * aren't then dependent on the periodic commit to wake_worker(). 4132 */ 4133 synchronize_rcu(); 4134 4135 dm_put(pool_md); 4136 4137 return 0; 4138 4139 bad: 4140 dm_pool_close_thin_device(tc->td); 4141 bad_pool: 4142 __pool_dec(tc->pool); 4143 bad_pool_lookup: 4144 dm_put(pool_md); 4145 bad_common: 4146 dm_put_device(ti, tc->pool_dev); 4147 bad_pool_dev: 4148 if (tc->origin_dev) 4149 dm_put_device(ti, tc->origin_dev); 4150 bad_origin_dev: 4151 kfree(tc); 4152 out_unlock: 4153 mutex_unlock(&dm_thin_pool_table.mutex); 4154 4155 return r; 4156 } 4157 4158 static int thin_map(struct dm_target *ti, struct bio *bio) 4159 { 4160 bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector); 4161 4162 return thin_bio_map(ti, bio); 4163 } 4164 4165 static int thin_endio(struct dm_target *ti, struct bio *bio, int err) 4166 { 4167 unsigned long flags; 4168 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 4169 struct list_head work; 4170 struct dm_thin_new_mapping *m, *tmp; 4171 struct pool *pool = h->tc->pool; 4172 4173 if (h->shared_read_entry) { 4174 INIT_LIST_HEAD(&work); 4175 dm_deferred_entry_dec(h->shared_read_entry, &work); 4176 4177 spin_lock_irqsave(&pool->lock, flags); 4178 list_for_each_entry_safe(m, tmp, &work, list) { 4179 list_del(&m->list); 4180 __complete_mapping_preparation(m); 4181 } 4182 spin_unlock_irqrestore(&pool->lock, flags); 4183 } 4184 4185 if (h->all_io_entry) { 4186 INIT_LIST_HEAD(&work); 4187 dm_deferred_entry_dec(h->all_io_entry, &work); 4188 if (!list_empty(&work)) { 4189 spin_lock_irqsave(&pool->lock, flags); 4190 list_for_each_entry_safe(m, tmp, &work, list) 4191 list_add_tail(&m->list, &pool->prepared_discards); 4192 spin_unlock_irqrestore(&pool->lock, flags); 4193 wake_worker(pool); 4194 } 4195 } 4196 4197 if (h->cell) 4198 cell_defer_no_holder(h->tc, h->cell); 4199 4200 return 0; 4201 } 4202 4203 static void thin_presuspend(struct dm_target *ti) 4204 { 4205 struct thin_c *tc = ti->private; 4206 4207 if (dm_noflush_suspending(ti)) 4208 noflush_work(tc, do_noflush_start); 4209 } 4210 4211 static void thin_postsuspend(struct dm_target *ti) 4212 { 4213 struct thin_c *tc = ti->private; 4214 4215 /* 4216 * The dm_noflush_suspending flag has been cleared by now, so 4217 * unfortunately we must always run this. 4218 */ 4219 noflush_work(tc, do_noflush_stop); 4220 } 4221 4222 static int thin_preresume(struct dm_target *ti) 4223 { 4224 struct thin_c *tc = ti->private; 4225 4226 if (tc->origin_dev) 4227 tc->origin_size = get_dev_size(tc->origin_dev->bdev); 4228 4229 return 0; 4230 } 4231 4232 /* 4233 * <nr mapped sectors> <highest mapped sector> 4234 */ 4235 static void thin_status(struct dm_target *ti, status_type_t type, 4236 unsigned status_flags, char *result, unsigned maxlen) 4237 { 4238 int r; 4239 ssize_t sz = 0; 4240 dm_block_t mapped, highest; 4241 char buf[BDEVNAME_SIZE]; 4242 struct thin_c *tc = ti->private; 4243 4244 if (get_pool_mode(tc->pool) == PM_FAIL) { 4245 DMEMIT("Fail"); 4246 return; 4247 } 4248 4249 if (!tc->td) 4250 DMEMIT("-"); 4251 else { 4252 switch (type) { 4253 case STATUSTYPE_INFO: 4254 r = dm_thin_get_mapped_count(tc->td, &mapped); 4255 if (r) { 4256 DMERR("dm_thin_get_mapped_count returned %d", r); 4257 goto err; 4258 } 4259 4260 r = dm_thin_get_highest_mapped_block(tc->td, &highest); 4261 if (r < 0) { 4262 DMERR("dm_thin_get_highest_mapped_block returned %d", r); 4263 goto err; 4264 } 4265 4266 DMEMIT("%llu ", mapped * tc->pool->sectors_per_block); 4267 if (r) 4268 DMEMIT("%llu", ((highest + 1) * 4269 tc->pool->sectors_per_block) - 1); 4270 else 4271 DMEMIT("-"); 4272 break; 4273 4274 case STATUSTYPE_TABLE: 4275 DMEMIT("%s %lu", 4276 format_dev_t(buf, tc->pool_dev->bdev->bd_dev), 4277 (unsigned long) tc->dev_id); 4278 if (tc->origin_dev) 4279 DMEMIT(" %s", format_dev_t(buf, tc->origin_dev->bdev->bd_dev)); 4280 break; 4281 } 4282 } 4283 4284 return; 4285 4286 err: 4287 DMEMIT("Error"); 4288 } 4289 4290 static int thin_merge(struct dm_target *ti, struct bvec_merge_data *bvm, 4291 struct bio_vec *biovec, int max_size) 4292 { 4293 struct thin_c *tc = ti->private; 4294 struct request_queue *q = bdev_get_queue(tc->pool_dev->bdev); 4295 4296 if (!q->merge_bvec_fn) 4297 return max_size; 4298 4299 bvm->bi_bdev = tc->pool_dev->bdev; 4300 bvm->bi_sector = dm_target_offset(ti, bvm->bi_sector); 4301 4302 return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); 4303 } 4304 4305 static int thin_iterate_devices(struct dm_target *ti, 4306 iterate_devices_callout_fn fn, void *data) 4307 { 4308 sector_t blocks; 4309 struct thin_c *tc = ti->private; 4310 struct pool *pool = tc->pool; 4311 4312 /* 4313 * We can't call dm_pool_get_data_dev_size() since that blocks. So 4314 * we follow a more convoluted path through to the pool's target. 4315 */ 4316 if (!pool->ti) 4317 return 0; /* nothing is bound */ 4318 4319 blocks = pool->ti->len; 4320 (void) sector_div(blocks, pool->sectors_per_block); 4321 if (blocks) 4322 return fn(ti, tc->pool_dev, 0, pool->sectors_per_block * blocks, data); 4323 4324 return 0; 4325 } 4326 4327 static void thin_io_hints(struct dm_target *ti, struct queue_limits *limits) 4328 { 4329 struct thin_c *tc = ti->private; 4330 struct pool *pool = tc->pool; 4331 4332 limits->discard_granularity = pool->sectors_per_block << SECTOR_SHIFT; 4333 limits->max_discard_sectors = 2048 * 1024 * 16; /* 16G */ 4334 } 4335 4336 static struct target_type thin_target = { 4337 .name = "thin", 4338 .version = {1, 16, 0}, 4339 .module = THIS_MODULE, 4340 .ctr = thin_ctr, 4341 .dtr = thin_dtr, 4342 .map = thin_map, 4343 .end_io = thin_endio, 4344 .preresume = thin_preresume, 4345 .presuspend = thin_presuspend, 4346 .postsuspend = thin_postsuspend, 4347 .status = thin_status, 4348 .merge = thin_merge, 4349 .iterate_devices = thin_iterate_devices, 4350 .io_hints = thin_io_hints, 4351 }; 4352 4353 /*----------------------------------------------------------------*/ 4354 4355 static int __init dm_thin_init(void) 4356 { 4357 int r; 4358 4359 pool_table_init(); 4360 4361 r = dm_register_target(&thin_target); 4362 if (r) 4363 return r; 4364 4365 r = dm_register_target(&pool_target); 4366 if (r) 4367 goto bad_pool_target; 4368 4369 r = -ENOMEM; 4370 4371 _new_mapping_cache = KMEM_CACHE(dm_thin_new_mapping, 0); 4372 if (!_new_mapping_cache) 4373 goto bad_new_mapping_cache; 4374 4375 return 0; 4376 4377 bad_new_mapping_cache: 4378 dm_unregister_target(&pool_target); 4379 bad_pool_target: 4380 dm_unregister_target(&thin_target); 4381 4382 return r; 4383 } 4384 4385 static void dm_thin_exit(void) 4386 { 4387 dm_unregister_target(&thin_target); 4388 dm_unregister_target(&pool_target); 4389 4390 kmem_cache_destroy(_new_mapping_cache); 4391 } 4392 4393 module_init(dm_thin_init); 4394 module_exit(dm_thin_exit); 4395 4396 module_param_named(no_space_timeout, no_space_timeout_secs, uint, S_IRUGO | S_IWUSR); 4397 MODULE_PARM_DESC(no_space_timeout, "Out of data space queue IO timeout in seconds"); 4398 4399 MODULE_DESCRIPTION(DM_NAME " thin provisioning target"); 4400 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>"); 4401 MODULE_LICENSE("GPL"); 4402