1 /* 2 * Copyright (C) 2012 Red Hat. All rights reserved. 3 * 4 * This file is released under the GPL. 5 */ 6 7 #include "dm.h" 8 #include "dm-bio-prison-v2.h" 9 #include "dm-bio-record.h" 10 #include "dm-cache-metadata.h" 11 12 #include <linux/dm-io.h> 13 #include <linux/dm-kcopyd.h> 14 #include <linux/jiffies.h> 15 #include <linux/init.h> 16 #include <linux/mempool.h> 17 #include <linux/module.h> 18 #include <linux/rwsem.h> 19 #include <linux/slab.h> 20 #include <linux/vmalloc.h> 21 22 #define DM_MSG_PREFIX "cache" 23 24 DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(cache_copy_throttle, 25 "A percentage of time allocated for copying to and/or from cache"); 26 27 /*----------------------------------------------------------------*/ 28 29 /* 30 * Glossary: 31 * 32 * oblock: index of an origin block 33 * cblock: index of a cache block 34 * promotion: movement of a block from origin to cache 35 * demotion: movement of a block from cache to origin 36 * migration: movement of a block between the origin and cache device, 37 * either direction 38 */ 39 40 /*----------------------------------------------------------------*/ 41 42 struct io_tracker { 43 spinlock_t lock; 44 45 /* 46 * Sectors of in-flight IO. 47 */ 48 sector_t in_flight; 49 50 /* 51 * The time, in jiffies, when this device became idle (if it is 52 * indeed idle). 53 */ 54 unsigned long idle_time; 55 unsigned long last_update_time; 56 }; 57 58 static void iot_init(struct io_tracker *iot) 59 { 60 spin_lock_init(&iot->lock); 61 iot->in_flight = 0ul; 62 iot->idle_time = 0ul; 63 iot->last_update_time = jiffies; 64 } 65 66 static bool __iot_idle_for(struct io_tracker *iot, unsigned long jifs) 67 { 68 if (iot->in_flight) 69 return false; 70 71 return time_after(jiffies, iot->idle_time + jifs); 72 } 73 74 static bool iot_idle_for(struct io_tracker *iot, unsigned long jifs) 75 { 76 bool r; 77 unsigned long flags; 78 79 spin_lock_irqsave(&iot->lock, flags); 80 r = __iot_idle_for(iot, jifs); 81 spin_unlock_irqrestore(&iot->lock, flags); 82 83 return r; 84 } 85 86 static void iot_io_begin(struct io_tracker *iot, sector_t len) 87 { 88 unsigned long flags; 89 90 spin_lock_irqsave(&iot->lock, flags); 91 iot->in_flight += len; 92 spin_unlock_irqrestore(&iot->lock, flags); 93 } 94 95 static void __iot_io_end(struct io_tracker *iot, sector_t len) 96 { 97 if (!len) 98 return; 99 100 iot->in_flight -= len; 101 if (!iot->in_flight) 102 iot->idle_time = jiffies; 103 } 104 105 static void iot_io_end(struct io_tracker *iot, sector_t len) 106 { 107 unsigned long flags; 108 109 spin_lock_irqsave(&iot->lock, flags); 110 __iot_io_end(iot, len); 111 spin_unlock_irqrestore(&iot->lock, flags); 112 } 113 114 /*----------------------------------------------------------------*/ 115 116 /* 117 * Represents a chunk of future work. 'input' allows continuations to pass 118 * values between themselves, typically error values. 119 */ 120 struct continuation { 121 struct work_struct ws; 122 blk_status_t input; 123 }; 124 125 static inline void init_continuation(struct continuation *k, 126 void (*fn)(struct work_struct *)) 127 { 128 INIT_WORK(&k->ws, fn); 129 k->input = 0; 130 } 131 132 static inline void queue_continuation(struct workqueue_struct *wq, 133 struct continuation *k) 134 { 135 queue_work(wq, &k->ws); 136 } 137 138 /*----------------------------------------------------------------*/ 139 140 /* 141 * The batcher collects together pieces of work that need a particular 142 * operation to occur before they can proceed (typically a commit). 143 */ 144 struct batcher { 145 /* 146 * The operation that everyone is waiting for. 147 */ 148 blk_status_t (*commit_op)(void *context); 149 void *commit_context; 150 151 /* 152 * This is how bios should be issued once the commit op is complete 153 * (accounted_request). 154 */ 155 void (*issue_op)(struct bio *bio, void *context); 156 void *issue_context; 157 158 /* 159 * Queued work gets put on here after commit. 160 */ 161 struct workqueue_struct *wq; 162 163 spinlock_t lock; 164 struct list_head work_items; 165 struct bio_list bios; 166 struct work_struct commit_work; 167 168 bool commit_scheduled; 169 }; 170 171 static void __commit(struct work_struct *_ws) 172 { 173 struct batcher *b = container_of(_ws, struct batcher, commit_work); 174 blk_status_t r; 175 unsigned long flags; 176 struct list_head work_items; 177 struct work_struct *ws, *tmp; 178 struct continuation *k; 179 struct bio *bio; 180 struct bio_list bios; 181 182 INIT_LIST_HEAD(&work_items); 183 bio_list_init(&bios); 184 185 /* 186 * We have to grab these before the commit_op to avoid a race 187 * condition. 188 */ 189 spin_lock_irqsave(&b->lock, flags); 190 list_splice_init(&b->work_items, &work_items); 191 bio_list_merge(&bios, &b->bios); 192 bio_list_init(&b->bios); 193 b->commit_scheduled = false; 194 spin_unlock_irqrestore(&b->lock, flags); 195 196 r = b->commit_op(b->commit_context); 197 198 list_for_each_entry_safe(ws, tmp, &work_items, entry) { 199 k = container_of(ws, struct continuation, ws); 200 k->input = r; 201 INIT_LIST_HEAD(&ws->entry); /* to avoid a WARN_ON */ 202 queue_work(b->wq, ws); 203 } 204 205 while ((bio = bio_list_pop(&bios))) { 206 if (r) { 207 bio->bi_status = r; 208 bio_endio(bio); 209 } else 210 b->issue_op(bio, b->issue_context); 211 } 212 } 213 214 static void batcher_init(struct batcher *b, 215 blk_status_t (*commit_op)(void *), 216 void *commit_context, 217 void (*issue_op)(struct bio *bio, void *), 218 void *issue_context, 219 struct workqueue_struct *wq) 220 { 221 b->commit_op = commit_op; 222 b->commit_context = commit_context; 223 b->issue_op = issue_op; 224 b->issue_context = issue_context; 225 b->wq = wq; 226 227 spin_lock_init(&b->lock); 228 INIT_LIST_HEAD(&b->work_items); 229 bio_list_init(&b->bios); 230 INIT_WORK(&b->commit_work, __commit); 231 b->commit_scheduled = false; 232 } 233 234 static void async_commit(struct batcher *b) 235 { 236 queue_work(b->wq, &b->commit_work); 237 } 238 239 static void continue_after_commit(struct batcher *b, struct continuation *k) 240 { 241 unsigned long flags; 242 bool commit_scheduled; 243 244 spin_lock_irqsave(&b->lock, flags); 245 commit_scheduled = b->commit_scheduled; 246 list_add_tail(&k->ws.entry, &b->work_items); 247 spin_unlock_irqrestore(&b->lock, flags); 248 249 if (commit_scheduled) 250 async_commit(b); 251 } 252 253 /* 254 * Bios are errored if commit failed. 255 */ 256 static void issue_after_commit(struct batcher *b, struct bio *bio) 257 { 258 unsigned long flags; 259 bool commit_scheduled; 260 261 spin_lock_irqsave(&b->lock, flags); 262 commit_scheduled = b->commit_scheduled; 263 bio_list_add(&b->bios, bio); 264 spin_unlock_irqrestore(&b->lock, flags); 265 266 if (commit_scheduled) 267 async_commit(b); 268 } 269 270 /* 271 * Call this if some urgent work is waiting for the commit to complete. 272 */ 273 static void schedule_commit(struct batcher *b) 274 { 275 bool immediate; 276 unsigned long flags; 277 278 spin_lock_irqsave(&b->lock, flags); 279 immediate = !list_empty(&b->work_items) || !bio_list_empty(&b->bios); 280 b->commit_scheduled = true; 281 spin_unlock_irqrestore(&b->lock, flags); 282 283 if (immediate) 284 async_commit(b); 285 } 286 287 /* 288 * There are a couple of places where we let a bio run, but want to do some 289 * work before calling its endio function. We do this by temporarily 290 * changing the endio fn. 291 */ 292 struct dm_hook_info { 293 bio_end_io_t *bi_end_io; 294 }; 295 296 static void dm_hook_bio(struct dm_hook_info *h, struct bio *bio, 297 bio_end_io_t *bi_end_io, void *bi_private) 298 { 299 h->bi_end_io = bio->bi_end_io; 300 301 bio->bi_end_io = bi_end_io; 302 bio->bi_private = bi_private; 303 } 304 305 static void dm_unhook_bio(struct dm_hook_info *h, struct bio *bio) 306 { 307 bio->bi_end_io = h->bi_end_io; 308 } 309 310 /*----------------------------------------------------------------*/ 311 312 #define MIGRATION_POOL_SIZE 128 313 #define COMMIT_PERIOD HZ 314 #define MIGRATION_COUNT_WINDOW 10 315 316 /* 317 * The block size of the device holding cache data must be 318 * between 32KB and 1GB. 319 */ 320 #define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (32 * 1024 >> SECTOR_SHIFT) 321 #define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT) 322 323 enum cache_metadata_mode { 324 CM_WRITE, /* metadata may be changed */ 325 CM_READ_ONLY, /* metadata may not be changed */ 326 CM_FAIL 327 }; 328 329 enum cache_io_mode { 330 /* 331 * Data is written to cached blocks only. These blocks are marked 332 * dirty. If you lose the cache device you will lose data. 333 * Potential performance increase for both reads and writes. 334 */ 335 CM_IO_WRITEBACK, 336 337 /* 338 * Data is written to both cache and origin. Blocks are never 339 * dirty. Potential performance benfit for reads only. 340 */ 341 CM_IO_WRITETHROUGH, 342 343 /* 344 * A degraded mode useful for various cache coherency situations 345 * (eg, rolling back snapshots). Reads and writes always go to the 346 * origin. If a write goes to a cached oblock, then the cache 347 * block is invalidated. 348 */ 349 CM_IO_PASSTHROUGH 350 }; 351 352 struct cache_features { 353 enum cache_metadata_mode mode; 354 enum cache_io_mode io_mode; 355 unsigned metadata_version; 356 }; 357 358 struct cache_stats { 359 atomic_t read_hit; 360 atomic_t read_miss; 361 atomic_t write_hit; 362 atomic_t write_miss; 363 atomic_t demotion; 364 atomic_t promotion; 365 atomic_t writeback; 366 atomic_t copies_avoided; 367 atomic_t cache_cell_clash; 368 atomic_t commit_count; 369 atomic_t discard_count; 370 }; 371 372 struct cache { 373 struct dm_target *ti; 374 struct dm_target_callbacks callbacks; 375 376 struct dm_cache_metadata *cmd; 377 378 /* 379 * Metadata is written to this device. 380 */ 381 struct dm_dev *metadata_dev; 382 383 /* 384 * The slower of the two data devices. Typically a spindle. 385 */ 386 struct dm_dev *origin_dev; 387 388 /* 389 * The faster of the two data devices. Typically an SSD. 390 */ 391 struct dm_dev *cache_dev; 392 393 /* 394 * Size of the origin device in _complete_ blocks and native sectors. 395 */ 396 dm_oblock_t origin_blocks; 397 sector_t origin_sectors; 398 399 /* 400 * Size of the cache device in blocks. 401 */ 402 dm_cblock_t cache_size; 403 404 /* 405 * Fields for converting from sectors to blocks. 406 */ 407 sector_t sectors_per_block; 408 int sectors_per_block_shift; 409 410 spinlock_t lock; 411 struct bio_list deferred_bios; 412 sector_t migration_threshold; 413 wait_queue_head_t migration_wait; 414 atomic_t nr_allocated_migrations; 415 416 /* 417 * The number of in flight migrations that are performing 418 * background io. eg, promotion, writeback. 419 */ 420 atomic_t nr_io_migrations; 421 422 struct rw_semaphore quiesce_lock; 423 424 /* 425 * cache_size entries, dirty if set 426 */ 427 atomic_t nr_dirty; 428 unsigned long *dirty_bitset; 429 430 /* 431 * origin_blocks entries, discarded if set. 432 */ 433 dm_dblock_t discard_nr_blocks; 434 unsigned long *discard_bitset; 435 uint32_t discard_block_size; /* a power of 2 times sectors per block */ 436 437 /* 438 * Rather than reconstructing the table line for the status we just 439 * save it and regurgitate. 440 */ 441 unsigned nr_ctr_args; 442 const char **ctr_args; 443 444 struct dm_kcopyd_client *copier; 445 struct workqueue_struct *wq; 446 struct work_struct deferred_bio_worker; 447 struct work_struct migration_worker; 448 struct delayed_work waker; 449 struct dm_bio_prison_v2 *prison; 450 struct bio_set *bs; 451 452 mempool_t *migration_pool; 453 454 struct dm_cache_policy *policy; 455 unsigned policy_nr_args; 456 457 bool need_tick_bio:1; 458 bool sized:1; 459 bool invalidate:1; 460 bool commit_requested:1; 461 bool loaded_mappings:1; 462 bool loaded_discards:1; 463 464 /* 465 * Cache features such as write-through. 466 */ 467 struct cache_features features; 468 469 struct cache_stats stats; 470 471 /* 472 * Invalidation fields. 473 */ 474 spinlock_t invalidation_lock; 475 struct list_head invalidation_requests; 476 477 struct io_tracker tracker; 478 479 struct work_struct commit_ws; 480 struct batcher committer; 481 482 struct rw_semaphore background_work_lock; 483 }; 484 485 struct per_bio_data { 486 bool tick:1; 487 unsigned req_nr:2; 488 struct dm_bio_prison_cell_v2 *cell; 489 struct dm_hook_info hook_info; 490 sector_t len; 491 }; 492 493 struct dm_cache_migration { 494 struct continuation k; 495 struct cache *cache; 496 497 struct policy_work *op; 498 struct bio *overwrite_bio; 499 struct dm_bio_prison_cell_v2 *cell; 500 501 dm_cblock_t invalidate_cblock; 502 dm_oblock_t invalidate_oblock; 503 }; 504 505 /*----------------------------------------------------------------*/ 506 507 static bool writethrough_mode(struct cache *cache) 508 { 509 return cache->features.io_mode == CM_IO_WRITETHROUGH; 510 } 511 512 static bool writeback_mode(struct cache *cache) 513 { 514 return cache->features.io_mode == CM_IO_WRITEBACK; 515 } 516 517 static inline bool passthrough_mode(struct cache *cache) 518 { 519 return unlikely(cache->features.io_mode == CM_IO_PASSTHROUGH); 520 } 521 522 /*----------------------------------------------------------------*/ 523 524 static void wake_deferred_bio_worker(struct cache *cache) 525 { 526 queue_work(cache->wq, &cache->deferred_bio_worker); 527 } 528 529 static void wake_migration_worker(struct cache *cache) 530 { 531 if (passthrough_mode(cache)) 532 return; 533 534 queue_work(cache->wq, &cache->migration_worker); 535 } 536 537 /*----------------------------------------------------------------*/ 538 539 static struct dm_bio_prison_cell_v2 *alloc_prison_cell(struct cache *cache) 540 { 541 return dm_bio_prison_alloc_cell_v2(cache->prison, GFP_NOWAIT); 542 } 543 544 static void free_prison_cell(struct cache *cache, struct dm_bio_prison_cell_v2 *cell) 545 { 546 dm_bio_prison_free_cell_v2(cache->prison, cell); 547 } 548 549 static struct dm_cache_migration *alloc_migration(struct cache *cache) 550 { 551 struct dm_cache_migration *mg; 552 553 mg = mempool_alloc(cache->migration_pool, GFP_NOWAIT); 554 if (!mg) 555 return NULL; 556 557 memset(mg, 0, sizeof(*mg)); 558 559 mg->cache = cache; 560 atomic_inc(&cache->nr_allocated_migrations); 561 562 return mg; 563 } 564 565 static void free_migration(struct dm_cache_migration *mg) 566 { 567 struct cache *cache = mg->cache; 568 569 if (atomic_dec_and_test(&cache->nr_allocated_migrations)) 570 wake_up(&cache->migration_wait); 571 572 mempool_free(mg, cache->migration_pool); 573 } 574 575 /*----------------------------------------------------------------*/ 576 577 static inline dm_oblock_t oblock_succ(dm_oblock_t b) 578 { 579 return to_oblock(from_oblock(b) + 1ull); 580 } 581 582 static void build_key(dm_oblock_t begin, dm_oblock_t end, struct dm_cell_key_v2 *key) 583 { 584 key->virtual = 0; 585 key->dev = 0; 586 key->block_begin = from_oblock(begin); 587 key->block_end = from_oblock(end); 588 } 589 590 /* 591 * We have two lock levels. Level 0, which is used to prevent WRITEs, and 592 * level 1 which prevents *both* READs and WRITEs. 593 */ 594 #define WRITE_LOCK_LEVEL 0 595 #define READ_WRITE_LOCK_LEVEL 1 596 597 static unsigned lock_level(struct bio *bio) 598 { 599 return bio_data_dir(bio) == WRITE ? 600 WRITE_LOCK_LEVEL : 601 READ_WRITE_LOCK_LEVEL; 602 } 603 604 /*---------------------------------------------------------------- 605 * Per bio data 606 *--------------------------------------------------------------*/ 607 608 static struct per_bio_data *get_per_bio_data(struct bio *bio) 609 { 610 struct per_bio_data *pb = dm_per_bio_data(bio, sizeof(struct per_bio_data)); 611 BUG_ON(!pb); 612 return pb; 613 } 614 615 static struct per_bio_data *init_per_bio_data(struct bio *bio) 616 { 617 struct per_bio_data *pb = get_per_bio_data(bio); 618 619 pb->tick = false; 620 pb->req_nr = dm_bio_get_target_bio_nr(bio); 621 pb->cell = NULL; 622 pb->len = 0; 623 624 return pb; 625 } 626 627 /*----------------------------------------------------------------*/ 628 629 static void defer_bio(struct cache *cache, struct bio *bio) 630 { 631 unsigned long flags; 632 633 spin_lock_irqsave(&cache->lock, flags); 634 bio_list_add(&cache->deferred_bios, bio); 635 spin_unlock_irqrestore(&cache->lock, flags); 636 637 wake_deferred_bio_worker(cache); 638 } 639 640 static void defer_bios(struct cache *cache, struct bio_list *bios) 641 { 642 unsigned long flags; 643 644 spin_lock_irqsave(&cache->lock, flags); 645 bio_list_merge(&cache->deferred_bios, bios); 646 bio_list_init(bios); 647 spin_unlock_irqrestore(&cache->lock, flags); 648 649 wake_deferred_bio_worker(cache); 650 } 651 652 /*----------------------------------------------------------------*/ 653 654 static bool bio_detain_shared(struct cache *cache, dm_oblock_t oblock, struct bio *bio) 655 { 656 bool r; 657 struct per_bio_data *pb; 658 struct dm_cell_key_v2 key; 659 dm_oblock_t end = to_oblock(from_oblock(oblock) + 1ULL); 660 struct dm_bio_prison_cell_v2 *cell_prealloc, *cell; 661 662 cell_prealloc = alloc_prison_cell(cache); /* FIXME: allow wait if calling from worker */ 663 if (!cell_prealloc) { 664 defer_bio(cache, bio); 665 return false; 666 } 667 668 build_key(oblock, end, &key); 669 r = dm_cell_get_v2(cache->prison, &key, lock_level(bio), bio, cell_prealloc, &cell); 670 if (!r) { 671 /* 672 * Failed to get the lock. 673 */ 674 free_prison_cell(cache, cell_prealloc); 675 return r; 676 } 677 678 if (cell != cell_prealloc) 679 free_prison_cell(cache, cell_prealloc); 680 681 pb = get_per_bio_data(bio); 682 pb->cell = cell; 683 684 return r; 685 } 686 687 /*----------------------------------------------------------------*/ 688 689 static bool is_dirty(struct cache *cache, dm_cblock_t b) 690 { 691 return test_bit(from_cblock(b), cache->dirty_bitset); 692 } 693 694 static void set_dirty(struct cache *cache, dm_cblock_t cblock) 695 { 696 if (!test_and_set_bit(from_cblock(cblock), cache->dirty_bitset)) { 697 atomic_inc(&cache->nr_dirty); 698 policy_set_dirty(cache->policy, cblock); 699 } 700 } 701 702 /* 703 * These two are called when setting after migrations to force the policy 704 * and dirty bitset to be in sync. 705 */ 706 static void force_set_dirty(struct cache *cache, dm_cblock_t cblock) 707 { 708 if (!test_and_set_bit(from_cblock(cblock), cache->dirty_bitset)) 709 atomic_inc(&cache->nr_dirty); 710 policy_set_dirty(cache->policy, cblock); 711 } 712 713 static void force_clear_dirty(struct cache *cache, dm_cblock_t cblock) 714 { 715 if (test_and_clear_bit(from_cblock(cblock), cache->dirty_bitset)) { 716 if (atomic_dec_return(&cache->nr_dirty) == 0) 717 dm_table_event(cache->ti->table); 718 } 719 720 policy_clear_dirty(cache->policy, cblock); 721 } 722 723 /*----------------------------------------------------------------*/ 724 725 static bool block_size_is_power_of_two(struct cache *cache) 726 { 727 return cache->sectors_per_block_shift >= 0; 728 } 729 730 /* gcc on ARM generates spurious references to __udivdi3 and __umoddi3 */ 731 #if defined(CONFIG_ARM) && __GNUC__ == 4 && __GNUC_MINOR__ <= 6 732 __always_inline 733 #endif 734 static dm_block_t block_div(dm_block_t b, uint32_t n) 735 { 736 do_div(b, n); 737 738 return b; 739 } 740 741 static dm_block_t oblocks_per_dblock(struct cache *cache) 742 { 743 dm_block_t oblocks = cache->discard_block_size; 744 745 if (block_size_is_power_of_two(cache)) 746 oblocks >>= cache->sectors_per_block_shift; 747 else 748 oblocks = block_div(oblocks, cache->sectors_per_block); 749 750 return oblocks; 751 } 752 753 static dm_dblock_t oblock_to_dblock(struct cache *cache, dm_oblock_t oblock) 754 { 755 return to_dblock(block_div(from_oblock(oblock), 756 oblocks_per_dblock(cache))); 757 } 758 759 static void set_discard(struct cache *cache, dm_dblock_t b) 760 { 761 unsigned long flags; 762 763 BUG_ON(from_dblock(b) >= from_dblock(cache->discard_nr_blocks)); 764 atomic_inc(&cache->stats.discard_count); 765 766 spin_lock_irqsave(&cache->lock, flags); 767 set_bit(from_dblock(b), cache->discard_bitset); 768 spin_unlock_irqrestore(&cache->lock, flags); 769 } 770 771 static void clear_discard(struct cache *cache, dm_dblock_t b) 772 { 773 unsigned long flags; 774 775 spin_lock_irqsave(&cache->lock, flags); 776 clear_bit(from_dblock(b), cache->discard_bitset); 777 spin_unlock_irqrestore(&cache->lock, flags); 778 } 779 780 static bool is_discarded(struct cache *cache, dm_dblock_t b) 781 { 782 int r; 783 unsigned long flags; 784 785 spin_lock_irqsave(&cache->lock, flags); 786 r = test_bit(from_dblock(b), cache->discard_bitset); 787 spin_unlock_irqrestore(&cache->lock, flags); 788 789 return r; 790 } 791 792 static bool is_discarded_oblock(struct cache *cache, dm_oblock_t b) 793 { 794 int r; 795 unsigned long flags; 796 797 spin_lock_irqsave(&cache->lock, flags); 798 r = test_bit(from_dblock(oblock_to_dblock(cache, b)), 799 cache->discard_bitset); 800 spin_unlock_irqrestore(&cache->lock, flags); 801 802 return r; 803 } 804 805 /*---------------------------------------------------------------- 806 * Remapping 807 *--------------------------------------------------------------*/ 808 static void remap_to_origin(struct cache *cache, struct bio *bio) 809 { 810 bio_set_dev(bio, cache->origin_dev->bdev); 811 } 812 813 static void remap_to_cache(struct cache *cache, struct bio *bio, 814 dm_cblock_t cblock) 815 { 816 sector_t bi_sector = bio->bi_iter.bi_sector; 817 sector_t block = from_cblock(cblock); 818 819 bio_set_dev(bio, cache->cache_dev->bdev); 820 if (!block_size_is_power_of_two(cache)) 821 bio->bi_iter.bi_sector = 822 (block * cache->sectors_per_block) + 823 sector_div(bi_sector, cache->sectors_per_block); 824 else 825 bio->bi_iter.bi_sector = 826 (block << cache->sectors_per_block_shift) | 827 (bi_sector & (cache->sectors_per_block - 1)); 828 } 829 830 static void check_if_tick_bio_needed(struct cache *cache, struct bio *bio) 831 { 832 unsigned long flags; 833 struct per_bio_data *pb; 834 835 spin_lock_irqsave(&cache->lock, flags); 836 if (cache->need_tick_bio && !op_is_flush(bio->bi_opf) && 837 bio_op(bio) != REQ_OP_DISCARD) { 838 pb = get_per_bio_data(bio); 839 pb->tick = true; 840 cache->need_tick_bio = false; 841 } 842 spin_unlock_irqrestore(&cache->lock, flags); 843 } 844 845 static void __remap_to_origin_clear_discard(struct cache *cache, struct bio *bio, 846 dm_oblock_t oblock, bool bio_has_pbd) 847 { 848 if (bio_has_pbd) 849 check_if_tick_bio_needed(cache, bio); 850 remap_to_origin(cache, bio); 851 if (bio_data_dir(bio) == WRITE) 852 clear_discard(cache, oblock_to_dblock(cache, oblock)); 853 } 854 855 static void remap_to_origin_clear_discard(struct cache *cache, struct bio *bio, 856 dm_oblock_t oblock) 857 { 858 // FIXME: check_if_tick_bio_needed() is called way too much through this interface 859 __remap_to_origin_clear_discard(cache, bio, oblock, true); 860 } 861 862 static void remap_to_cache_dirty(struct cache *cache, struct bio *bio, 863 dm_oblock_t oblock, dm_cblock_t cblock) 864 { 865 check_if_tick_bio_needed(cache, bio); 866 remap_to_cache(cache, bio, cblock); 867 if (bio_data_dir(bio) == WRITE) { 868 set_dirty(cache, cblock); 869 clear_discard(cache, oblock_to_dblock(cache, oblock)); 870 } 871 } 872 873 static dm_oblock_t get_bio_block(struct cache *cache, struct bio *bio) 874 { 875 sector_t block_nr = bio->bi_iter.bi_sector; 876 877 if (!block_size_is_power_of_two(cache)) 878 (void) sector_div(block_nr, cache->sectors_per_block); 879 else 880 block_nr >>= cache->sectors_per_block_shift; 881 882 return to_oblock(block_nr); 883 } 884 885 static bool accountable_bio(struct cache *cache, struct bio *bio) 886 { 887 return bio_op(bio) != REQ_OP_DISCARD; 888 } 889 890 static void accounted_begin(struct cache *cache, struct bio *bio) 891 { 892 struct per_bio_data *pb; 893 894 if (accountable_bio(cache, bio)) { 895 pb = get_per_bio_data(bio); 896 pb->len = bio_sectors(bio); 897 iot_io_begin(&cache->tracker, pb->len); 898 } 899 } 900 901 static void accounted_complete(struct cache *cache, struct bio *bio) 902 { 903 struct per_bio_data *pb = get_per_bio_data(bio); 904 905 iot_io_end(&cache->tracker, pb->len); 906 } 907 908 static void accounted_request(struct cache *cache, struct bio *bio) 909 { 910 accounted_begin(cache, bio); 911 generic_make_request(bio); 912 } 913 914 static void issue_op(struct bio *bio, void *context) 915 { 916 struct cache *cache = context; 917 accounted_request(cache, bio); 918 } 919 920 /* 921 * When running in writethrough mode we need to send writes to clean blocks 922 * to both the cache and origin devices. Clone the bio and send them in parallel. 923 */ 924 static void remap_to_origin_and_cache(struct cache *cache, struct bio *bio, 925 dm_oblock_t oblock, dm_cblock_t cblock) 926 { 927 struct bio *origin_bio = bio_clone_fast(bio, GFP_NOIO, cache->bs); 928 929 BUG_ON(!origin_bio); 930 931 bio_chain(origin_bio, bio); 932 /* 933 * Passing false to __remap_to_origin_clear_discard() skips 934 * all code that might use per_bio_data (since clone doesn't have it) 935 */ 936 __remap_to_origin_clear_discard(cache, origin_bio, oblock, false); 937 submit_bio(origin_bio); 938 939 remap_to_cache(cache, bio, cblock); 940 } 941 942 /*---------------------------------------------------------------- 943 * Failure modes 944 *--------------------------------------------------------------*/ 945 static enum cache_metadata_mode get_cache_mode(struct cache *cache) 946 { 947 return cache->features.mode; 948 } 949 950 static const char *cache_device_name(struct cache *cache) 951 { 952 return dm_device_name(dm_table_get_md(cache->ti->table)); 953 } 954 955 static void notify_mode_switch(struct cache *cache, enum cache_metadata_mode mode) 956 { 957 const char *descs[] = { 958 "write", 959 "read-only", 960 "fail" 961 }; 962 963 dm_table_event(cache->ti->table); 964 DMINFO("%s: switching cache to %s mode", 965 cache_device_name(cache), descs[(int)mode]); 966 } 967 968 static void set_cache_mode(struct cache *cache, enum cache_metadata_mode new_mode) 969 { 970 bool needs_check; 971 enum cache_metadata_mode old_mode = get_cache_mode(cache); 972 973 if (dm_cache_metadata_needs_check(cache->cmd, &needs_check)) { 974 DMERR("%s: unable to read needs_check flag, setting failure mode.", 975 cache_device_name(cache)); 976 new_mode = CM_FAIL; 977 } 978 979 if (new_mode == CM_WRITE && needs_check) { 980 DMERR("%s: unable to switch cache to write mode until repaired.", 981 cache_device_name(cache)); 982 if (old_mode != new_mode) 983 new_mode = old_mode; 984 else 985 new_mode = CM_READ_ONLY; 986 } 987 988 /* Never move out of fail mode */ 989 if (old_mode == CM_FAIL) 990 new_mode = CM_FAIL; 991 992 switch (new_mode) { 993 case CM_FAIL: 994 case CM_READ_ONLY: 995 dm_cache_metadata_set_read_only(cache->cmd); 996 break; 997 998 case CM_WRITE: 999 dm_cache_metadata_set_read_write(cache->cmd); 1000 break; 1001 } 1002 1003 cache->features.mode = new_mode; 1004 1005 if (new_mode != old_mode) 1006 notify_mode_switch(cache, new_mode); 1007 } 1008 1009 static void abort_transaction(struct cache *cache) 1010 { 1011 const char *dev_name = cache_device_name(cache); 1012 1013 if (get_cache_mode(cache) >= CM_READ_ONLY) 1014 return; 1015 1016 if (dm_cache_metadata_set_needs_check(cache->cmd)) { 1017 DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name); 1018 set_cache_mode(cache, CM_FAIL); 1019 } 1020 1021 DMERR_LIMIT("%s: aborting current metadata transaction", dev_name); 1022 if (dm_cache_metadata_abort(cache->cmd)) { 1023 DMERR("%s: failed to abort metadata transaction", dev_name); 1024 set_cache_mode(cache, CM_FAIL); 1025 } 1026 } 1027 1028 static void metadata_operation_failed(struct cache *cache, const char *op, int r) 1029 { 1030 DMERR_LIMIT("%s: metadata operation '%s' failed: error = %d", 1031 cache_device_name(cache), op, r); 1032 abort_transaction(cache); 1033 set_cache_mode(cache, CM_READ_ONLY); 1034 } 1035 1036 /*----------------------------------------------------------------*/ 1037 1038 static void load_stats(struct cache *cache) 1039 { 1040 struct dm_cache_statistics stats; 1041 1042 dm_cache_metadata_get_stats(cache->cmd, &stats); 1043 atomic_set(&cache->stats.read_hit, stats.read_hits); 1044 atomic_set(&cache->stats.read_miss, stats.read_misses); 1045 atomic_set(&cache->stats.write_hit, stats.write_hits); 1046 atomic_set(&cache->stats.write_miss, stats.write_misses); 1047 } 1048 1049 static void save_stats(struct cache *cache) 1050 { 1051 struct dm_cache_statistics stats; 1052 1053 if (get_cache_mode(cache) >= CM_READ_ONLY) 1054 return; 1055 1056 stats.read_hits = atomic_read(&cache->stats.read_hit); 1057 stats.read_misses = atomic_read(&cache->stats.read_miss); 1058 stats.write_hits = atomic_read(&cache->stats.write_hit); 1059 stats.write_misses = atomic_read(&cache->stats.write_miss); 1060 1061 dm_cache_metadata_set_stats(cache->cmd, &stats); 1062 } 1063 1064 static void update_stats(struct cache_stats *stats, enum policy_operation op) 1065 { 1066 switch (op) { 1067 case POLICY_PROMOTE: 1068 atomic_inc(&stats->promotion); 1069 break; 1070 1071 case POLICY_DEMOTE: 1072 atomic_inc(&stats->demotion); 1073 break; 1074 1075 case POLICY_WRITEBACK: 1076 atomic_inc(&stats->writeback); 1077 break; 1078 } 1079 } 1080 1081 /*---------------------------------------------------------------- 1082 * Migration processing 1083 * 1084 * Migration covers moving data from the origin device to the cache, or 1085 * vice versa. 1086 *--------------------------------------------------------------*/ 1087 1088 static void inc_io_migrations(struct cache *cache) 1089 { 1090 atomic_inc(&cache->nr_io_migrations); 1091 } 1092 1093 static void dec_io_migrations(struct cache *cache) 1094 { 1095 atomic_dec(&cache->nr_io_migrations); 1096 } 1097 1098 static bool discard_or_flush(struct bio *bio) 1099 { 1100 return bio_op(bio) == REQ_OP_DISCARD || op_is_flush(bio->bi_opf); 1101 } 1102 1103 static void calc_discard_block_range(struct cache *cache, struct bio *bio, 1104 dm_dblock_t *b, dm_dblock_t *e) 1105 { 1106 sector_t sb = bio->bi_iter.bi_sector; 1107 sector_t se = bio_end_sector(bio); 1108 1109 *b = to_dblock(dm_sector_div_up(sb, cache->discard_block_size)); 1110 1111 if (se - sb < cache->discard_block_size) 1112 *e = *b; 1113 else 1114 *e = to_dblock(block_div(se, cache->discard_block_size)); 1115 } 1116 1117 /*----------------------------------------------------------------*/ 1118 1119 static void prevent_background_work(struct cache *cache) 1120 { 1121 lockdep_off(); 1122 down_write(&cache->background_work_lock); 1123 lockdep_on(); 1124 } 1125 1126 static void allow_background_work(struct cache *cache) 1127 { 1128 lockdep_off(); 1129 up_write(&cache->background_work_lock); 1130 lockdep_on(); 1131 } 1132 1133 static bool background_work_begin(struct cache *cache) 1134 { 1135 bool r; 1136 1137 lockdep_off(); 1138 r = down_read_trylock(&cache->background_work_lock); 1139 lockdep_on(); 1140 1141 return r; 1142 } 1143 1144 static void background_work_end(struct cache *cache) 1145 { 1146 lockdep_off(); 1147 up_read(&cache->background_work_lock); 1148 lockdep_on(); 1149 } 1150 1151 /*----------------------------------------------------------------*/ 1152 1153 static bool bio_writes_complete_block(struct cache *cache, struct bio *bio) 1154 { 1155 return (bio_data_dir(bio) == WRITE) && 1156 (bio->bi_iter.bi_size == (cache->sectors_per_block << SECTOR_SHIFT)); 1157 } 1158 1159 static bool optimisable_bio(struct cache *cache, struct bio *bio, dm_oblock_t block) 1160 { 1161 return writeback_mode(cache) && 1162 (is_discarded_oblock(cache, block) || bio_writes_complete_block(cache, bio)); 1163 } 1164 1165 static void quiesce(struct dm_cache_migration *mg, 1166 void (*continuation)(struct work_struct *)) 1167 { 1168 init_continuation(&mg->k, continuation); 1169 dm_cell_quiesce_v2(mg->cache->prison, mg->cell, &mg->k.ws); 1170 } 1171 1172 static struct dm_cache_migration *ws_to_mg(struct work_struct *ws) 1173 { 1174 struct continuation *k = container_of(ws, struct continuation, ws); 1175 return container_of(k, struct dm_cache_migration, k); 1176 } 1177 1178 static void copy_complete(int read_err, unsigned long write_err, void *context) 1179 { 1180 struct dm_cache_migration *mg = container_of(context, struct dm_cache_migration, k); 1181 1182 if (read_err || write_err) 1183 mg->k.input = BLK_STS_IOERR; 1184 1185 queue_continuation(mg->cache->wq, &mg->k); 1186 } 1187 1188 static int copy(struct dm_cache_migration *mg, bool promote) 1189 { 1190 int r; 1191 struct dm_io_region o_region, c_region; 1192 struct cache *cache = mg->cache; 1193 1194 o_region.bdev = cache->origin_dev->bdev; 1195 o_region.sector = from_oblock(mg->op->oblock) * cache->sectors_per_block; 1196 o_region.count = cache->sectors_per_block; 1197 1198 c_region.bdev = cache->cache_dev->bdev; 1199 c_region.sector = from_cblock(mg->op->cblock) * cache->sectors_per_block; 1200 c_region.count = cache->sectors_per_block; 1201 1202 if (promote) 1203 r = dm_kcopyd_copy(cache->copier, &o_region, 1, &c_region, 0, copy_complete, &mg->k); 1204 else 1205 r = dm_kcopyd_copy(cache->copier, &c_region, 1, &o_region, 0, copy_complete, &mg->k); 1206 1207 return r; 1208 } 1209 1210 static void bio_drop_shared_lock(struct cache *cache, struct bio *bio) 1211 { 1212 struct per_bio_data *pb = get_per_bio_data(bio); 1213 1214 if (pb->cell && dm_cell_put_v2(cache->prison, pb->cell)) 1215 free_prison_cell(cache, pb->cell); 1216 pb->cell = NULL; 1217 } 1218 1219 static void overwrite_endio(struct bio *bio) 1220 { 1221 struct dm_cache_migration *mg = bio->bi_private; 1222 struct cache *cache = mg->cache; 1223 struct per_bio_data *pb = get_per_bio_data(bio); 1224 1225 dm_unhook_bio(&pb->hook_info, bio); 1226 1227 if (bio->bi_status) 1228 mg->k.input = bio->bi_status; 1229 1230 queue_continuation(cache->wq, &mg->k); 1231 } 1232 1233 static void overwrite(struct dm_cache_migration *mg, 1234 void (*continuation)(struct work_struct *)) 1235 { 1236 struct bio *bio = mg->overwrite_bio; 1237 struct per_bio_data *pb = get_per_bio_data(bio); 1238 1239 dm_hook_bio(&pb->hook_info, bio, overwrite_endio, mg); 1240 1241 /* 1242 * The overwrite bio is part of the copy operation, as such it does 1243 * not set/clear discard or dirty flags. 1244 */ 1245 if (mg->op->op == POLICY_PROMOTE) 1246 remap_to_cache(mg->cache, bio, mg->op->cblock); 1247 else 1248 remap_to_origin(mg->cache, bio); 1249 1250 init_continuation(&mg->k, continuation); 1251 accounted_request(mg->cache, bio); 1252 } 1253 1254 /* 1255 * Migration steps: 1256 * 1257 * 1) exclusive lock preventing WRITEs 1258 * 2) quiesce 1259 * 3) copy or issue overwrite bio 1260 * 4) upgrade to exclusive lock preventing READs and WRITEs 1261 * 5) quiesce 1262 * 6) update metadata and commit 1263 * 7) unlock 1264 */ 1265 static void mg_complete(struct dm_cache_migration *mg, bool success) 1266 { 1267 struct bio_list bios; 1268 struct cache *cache = mg->cache; 1269 struct policy_work *op = mg->op; 1270 dm_cblock_t cblock = op->cblock; 1271 1272 if (success) 1273 update_stats(&cache->stats, op->op); 1274 1275 switch (op->op) { 1276 case POLICY_PROMOTE: 1277 clear_discard(cache, oblock_to_dblock(cache, op->oblock)); 1278 policy_complete_background_work(cache->policy, op, success); 1279 1280 if (mg->overwrite_bio) { 1281 if (success) 1282 force_set_dirty(cache, cblock); 1283 else if (mg->k.input) 1284 mg->overwrite_bio->bi_status = mg->k.input; 1285 else 1286 mg->overwrite_bio->bi_status = BLK_STS_IOERR; 1287 bio_endio(mg->overwrite_bio); 1288 } else { 1289 if (success) 1290 force_clear_dirty(cache, cblock); 1291 dec_io_migrations(cache); 1292 } 1293 break; 1294 1295 case POLICY_DEMOTE: 1296 /* 1297 * We clear dirty here to update the nr_dirty counter. 1298 */ 1299 if (success) 1300 force_clear_dirty(cache, cblock); 1301 policy_complete_background_work(cache->policy, op, success); 1302 dec_io_migrations(cache); 1303 break; 1304 1305 case POLICY_WRITEBACK: 1306 if (success) 1307 force_clear_dirty(cache, cblock); 1308 policy_complete_background_work(cache->policy, op, success); 1309 dec_io_migrations(cache); 1310 break; 1311 } 1312 1313 bio_list_init(&bios); 1314 if (mg->cell) { 1315 if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios)) 1316 free_prison_cell(cache, mg->cell); 1317 } 1318 1319 free_migration(mg); 1320 defer_bios(cache, &bios); 1321 wake_migration_worker(cache); 1322 1323 background_work_end(cache); 1324 } 1325 1326 static void mg_success(struct work_struct *ws) 1327 { 1328 struct dm_cache_migration *mg = ws_to_mg(ws); 1329 mg_complete(mg, mg->k.input == 0); 1330 } 1331 1332 static void mg_update_metadata(struct work_struct *ws) 1333 { 1334 int r; 1335 struct dm_cache_migration *mg = ws_to_mg(ws); 1336 struct cache *cache = mg->cache; 1337 struct policy_work *op = mg->op; 1338 1339 switch (op->op) { 1340 case POLICY_PROMOTE: 1341 r = dm_cache_insert_mapping(cache->cmd, op->cblock, op->oblock); 1342 if (r) { 1343 DMERR_LIMIT("%s: migration failed; couldn't insert mapping", 1344 cache_device_name(cache)); 1345 metadata_operation_failed(cache, "dm_cache_insert_mapping", r); 1346 1347 mg_complete(mg, false); 1348 return; 1349 } 1350 mg_complete(mg, true); 1351 break; 1352 1353 case POLICY_DEMOTE: 1354 r = dm_cache_remove_mapping(cache->cmd, op->cblock); 1355 if (r) { 1356 DMERR_LIMIT("%s: migration failed; couldn't update on disk metadata", 1357 cache_device_name(cache)); 1358 metadata_operation_failed(cache, "dm_cache_remove_mapping", r); 1359 1360 mg_complete(mg, false); 1361 return; 1362 } 1363 1364 /* 1365 * It would be nice if we only had to commit when a REQ_FLUSH 1366 * comes through. But there's one scenario that we have to 1367 * look out for: 1368 * 1369 * - vblock x in a cache block 1370 * - domotion occurs 1371 * - cache block gets reallocated and over written 1372 * - crash 1373 * 1374 * When we recover, because there was no commit the cache will 1375 * rollback to having the data for vblock x in the cache block. 1376 * But the cache block has since been overwritten, so it'll end 1377 * up pointing to data that was never in 'x' during the history 1378 * of the device. 1379 * 1380 * To avoid this issue we require a commit as part of the 1381 * demotion operation. 1382 */ 1383 init_continuation(&mg->k, mg_success); 1384 continue_after_commit(&cache->committer, &mg->k); 1385 schedule_commit(&cache->committer); 1386 break; 1387 1388 case POLICY_WRITEBACK: 1389 mg_complete(mg, true); 1390 break; 1391 } 1392 } 1393 1394 static void mg_update_metadata_after_copy(struct work_struct *ws) 1395 { 1396 struct dm_cache_migration *mg = ws_to_mg(ws); 1397 1398 /* 1399 * Did the copy succeed? 1400 */ 1401 if (mg->k.input) 1402 mg_complete(mg, false); 1403 else 1404 mg_update_metadata(ws); 1405 } 1406 1407 static void mg_upgrade_lock(struct work_struct *ws) 1408 { 1409 int r; 1410 struct dm_cache_migration *mg = ws_to_mg(ws); 1411 1412 /* 1413 * Did the copy succeed? 1414 */ 1415 if (mg->k.input) 1416 mg_complete(mg, false); 1417 1418 else { 1419 /* 1420 * Now we want the lock to prevent both reads and writes. 1421 */ 1422 r = dm_cell_lock_promote_v2(mg->cache->prison, mg->cell, 1423 READ_WRITE_LOCK_LEVEL); 1424 if (r < 0) 1425 mg_complete(mg, false); 1426 1427 else if (r) 1428 quiesce(mg, mg_update_metadata); 1429 1430 else 1431 mg_update_metadata(ws); 1432 } 1433 } 1434 1435 static void mg_full_copy(struct work_struct *ws) 1436 { 1437 struct dm_cache_migration *mg = ws_to_mg(ws); 1438 struct cache *cache = mg->cache; 1439 struct policy_work *op = mg->op; 1440 bool is_policy_promote = (op->op == POLICY_PROMOTE); 1441 1442 if ((!is_policy_promote && !is_dirty(cache, op->cblock)) || 1443 is_discarded_oblock(cache, op->oblock)) { 1444 mg_upgrade_lock(ws); 1445 return; 1446 } 1447 1448 init_continuation(&mg->k, mg_upgrade_lock); 1449 1450 if (copy(mg, is_policy_promote)) { 1451 DMERR_LIMIT("%s: migration copy failed", cache_device_name(cache)); 1452 mg->k.input = BLK_STS_IOERR; 1453 mg_complete(mg, false); 1454 } 1455 } 1456 1457 static void mg_copy(struct work_struct *ws) 1458 { 1459 struct dm_cache_migration *mg = ws_to_mg(ws); 1460 1461 if (mg->overwrite_bio) { 1462 /* 1463 * No exclusive lock was held when we last checked if the bio 1464 * was optimisable. So we have to check again in case things 1465 * have changed (eg, the block may no longer be discarded). 1466 */ 1467 if (!optimisable_bio(mg->cache, mg->overwrite_bio, mg->op->oblock)) { 1468 /* 1469 * Fallback to a real full copy after doing some tidying up. 1470 */ 1471 bool rb = bio_detain_shared(mg->cache, mg->op->oblock, mg->overwrite_bio); 1472 BUG_ON(rb); /* An exclussive lock must _not_ be held for this block */ 1473 mg->overwrite_bio = NULL; 1474 inc_io_migrations(mg->cache); 1475 mg_full_copy(ws); 1476 return; 1477 } 1478 1479 /* 1480 * It's safe to do this here, even though it's new data 1481 * because all IO has been locked out of the block. 1482 * 1483 * mg_lock_writes() already took READ_WRITE_LOCK_LEVEL 1484 * so _not_ using mg_upgrade_lock() as continutation. 1485 */ 1486 overwrite(mg, mg_update_metadata_after_copy); 1487 1488 } else 1489 mg_full_copy(ws); 1490 } 1491 1492 static int mg_lock_writes(struct dm_cache_migration *mg) 1493 { 1494 int r; 1495 struct dm_cell_key_v2 key; 1496 struct cache *cache = mg->cache; 1497 struct dm_bio_prison_cell_v2 *prealloc; 1498 1499 prealloc = alloc_prison_cell(cache); 1500 if (!prealloc) { 1501 DMERR_LIMIT("%s: alloc_prison_cell failed", cache_device_name(cache)); 1502 mg_complete(mg, false); 1503 return -ENOMEM; 1504 } 1505 1506 /* 1507 * Prevent writes to the block, but allow reads to continue. 1508 * Unless we're using an overwrite bio, in which case we lock 1509 * everything. 1510 */ 1511 build_key(mg->op->oblock, oblock_succ(mg->op->oblock), &key); 1512 r = dm_cell_lock_v2(cache->prison, &key, 1513 mg->overwrite_bio ? READ_WRITE_LOCK_LEVEL : WRITE_LOCK_LEVEL, 1514 prealloc, &mg->cell); 1515 if (r < 0) { 1516 free_prison_cell(cache, prealloc); 1517 mg_complete(mg, false); 1518 return r; 1519 } 1520 1521 if (mg->cell != prealloc) 1522 free_prison_cell(cache, prealloc); 1523 1524 if (r == 0) 1525 mg_copy(&mg->k.ws); 1526 else 1527 quiesce(mg, mg_copy); 1528 1529 return 0; 1530 } 1531 1532 static int mg_start(struct cache *cache, struct policy_work *op, struct bio *bio) 1533 { 1534 struct dm_cache_migration *mg; 1535 1536 if (!background_work_begin(cache)) { 1537 policy_complete_background_work(cache->policy, op, false); 1538 return -EPERM; 1539 } 1540 1541 mg = alloc_migration(cache); 1542 if (!mg) { 1543 policy_complete_background_work(cache->policy, op, false); 1544 background_work_end(cache); 1545 return -ENOMEM; 1546 } 1547 1548 mg->op = op; 1549 mg->overwrite_bio = bio; 1550 1551 if (!bio) 1552 inc_io_migrations(cache); 1553 1554 return mg_lock_writes(mg); 1555 } 1556 1557 /*---------------------------------------------------------------- 1558 * invalidation processing 1559 *--------------------------------------------------------------*/ 1560 1561 static void invalidate_complete(struct dm_cache_migration *mg, bool success) 1562 { 1563 struct bio_list bios; 1564 struct cache *cache = mg->cache; 1565 1566 bio_list_init(&bios); 1567 if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios)) 1568 free_prison_cell(cache, mg->cell); 1569 1570 if (!success && mg->overwrite_bio) 1571 bio_io_error(mg->overwrite_bio); 1572 1573 free_migration(mg); 1574 defer_bios(cache, &bios); 1575 1576 background_work_end(cache); 1577 } 1578 1579 static void invalidate_completed(struct work_struct *ws) 1580 { 1581 struct dm_cache_migration *mg = ws_to_mg(ws); 1582 invalidate_complete(mg, !mg->k.input); 1583 } 1584 1585 static int invalidate_cblock(struct cache *cache, dm_cblock_t cblock) 1586 { 1587 int r = policy_invalidate_mapping(cache->policy, cblock); 1588 if (!r) { 1589 r = dm_cache_remove_mapping(cache->cmd, cblock); 1590 if (r) { 1591 DMERR_LIMIT("%s: invalidation failed; couldn't update on disk metadata", 1592 cache_device_name(cache)); 1593 metadata_operation_failed(cache, "dm_cache_remove_mapping", r); 1594 } 1595 1596 } else if (r == -ENODATA) { 1597 /* 1598 * Harmless, already unmapped. 1599 */ 1600 r = 0; 1601 1602 } else 1603 DMERR("%s: policy_invalidate_mapping failed", cache_device_name(cache)); 1604 1605 return r; 1606 } 1607 1608 static void invalidate_remove(struct work_struct *ws) 1609 { 1610 int r; 1611 struct dm_cache_migration *mg = ws_to_mg(ws); 1612 struct cache *cache = mg->cache; 1613 1614 r = invalidate_cblock(cache, mg->invalidate_cblock); 1615 if (r) { 1616 invalidate_complete(mg, false); 1617 return; 1618 } 1619 1620 init_continuation(&mg->k, invalidate_completed); 1621 continue_after_commit(&cache->committer, &mg->k); 1622 remap_to_origin_clear_discard(cache, mg->overwrite_bio, mg->invalidate_oblock); 1623 mg->overwrite_bio = NULL; 1624 schedule_commit(&cache->committer); 1625 } 1626 1627 static int invalidate_lock(struct dm_cache_migration *mg) 1628 { 1629 int r; 1630 struct dm_cell_key_v2 key; 1631 struct cache *cache = mg->cache; 1632 struct dm_bio_prison_cell_v2 *prealloc; 1633 1634 prealloc = alloc_prison_cell(cache); 1635 if (!prealloc) { 1636 invalidate_complete(mg, false); 1637 return -ENOMEM; 1638 } 1639 1640 build_key(mg->invalidate_oblock, oblock_succ(mg->invalidate_oblock), &key); 1641 r = dm_cell_lock_v2(cache->prison, &key, 1642 READ_WRITE_LOCK_LEVEL, prealloc, &mg->cell); 1643 if (r < 0) { 1644 free_prison_cell(cache, prealloc); 1645 invalidate_complete(mg, false); 1646 return r; 1647 } 1648 1649 if (mg->cell != prealloc) 1650 free_prison_cell(cache, prealloc); 1651 1652 if (r) 1653 quiesce(mg, invalidate_remove); 1654 1655 else { 1656 /* 1657 * We can't call invalidate_remove() directly here because we 1658 * might still be in request context. 1659 */ 1660 init_continuation(&mg->k, invalidate_remove); 1661 queue_work(cache->wq, &mg->k.ws); 1662 } 1663 1664 return 0; 1665 } 1666 1667 static int invalidate_start(struct cache *cache, dm_cblock_t cblock, 1668 dm_oblock_t oblock, struct bio *bio) 1669 { 1670 struct dm_cache_migration *mg; 1671 1672 if (!background_work_begin(cache)) 1673 return -EPERM; 1674 1675 mg = alloc_migration(cache); 1676 if (!mg) { 1677 background_work_end(cache); 1678 return -ENOMEM; 1679 } 1680 1681 mg->overwrite_bio = bio; 1682 mg->invalidate_cblock = cblock; 1683 mg->invalidate_oblock = oblock; 1684 1685 return invalidate_lock(mg); 1686 } 1687 1688 /*---------------------------------------------------------------- 1689 * bio processing 1690 *--------------------------------------------------------------*/ 1691 1692 enum busy { 1693 IDLE, 1694 BUSY 1695 }; 1696 1697 static enum busy spare_migration_bandwidth(struct cache *cache) 1698 { 1699 bool idle = iot_idle_for(&cache->tracker, HZ); 1700 sector_t current_volume = (atomic_read(&cache->nr_io_migrations) + 1) * 1701 cache->sectors_per_block; 1702 1703 if (idle && current_volume <= cache->migration_threshold) 1704 return IDLE; 1705 else 1706 return BUSY; 1707 } 1708 1709 static void inc_hit_counter(struct cache *cache, struct bio *bio) 1710 { 1711 atomic_inc(bio_data_dir(bio) == READ ? 1712 &cache->stats.read_hit : &cache->stats.write_hit); 1713 } 1714 1715 static void inc_miss_counter(struct cache *cache, struct bio *bio) 1716 { 1717 atomic_inc(bio_data_dir(bio) == READ ? 1718 &cache->stats.read_miss : &cache->stats.write_miss); 1719 } 1720 1721 /*----------------------------------------------------------------*/ 1722 1723 static int map_bio(struct cache *cache, struct bio *bio, dm_oblock_t block, 1724 bool *commit_needed) 1725 { 1726 int r, data_dir; 1727 bool rb, background_queued; 1728 dm_cblock_t cblock; 1729 1730 *commit_needed = false; 1731 1732 rb = bio_detain_shared(cache, block, bio); 1733 if (!rb) { 1734 /* 1735 * An exclusive lock is held for this block, so we have to 1736 * wait. We set the commit_needed flag so the current 1737 * transaction will be committed asap, allowing this lock 1738 * to be dropped. 1739 */ 1740 *commit_needed = true; 1741 return DM_MAPIO_SUBMITTED; 1742 } 1743 1744 data_dir = bio_data_dir(bio); 1745 1746 if (optimisable_bio(cache, bio, block)) { 1747 struct policy_work *op = NULL; 1748 1749 r = policy_lookup_with_work(cache->policy, block, &cblock, data_dir, true, &op); 1750 if (unlikely(r && r != -ENOENT)) { 1751 DMERR_LIMIT("%s: policy_lookup_with_work() failed with r = %d", 1752 cache_device_name(cache), r); 1753 bio_io_error(bio); 1754 return DM_MAPIO_SUBMITTED; 1755 } 1756 1757 if (r == -ENOENT && op) { 1758 bio_drop_shared_lock(cache, bio); 1759 BUG_ON(op->op != POLICY_PROMOTE); 1760 mg_start(cache, op, bio); 1761 return DM_MAPIO_SUBMITTED; 1762 } 1763 } else { 1764 r = policy_lookup(cache->policy, block, &cblock, data_dir, false, &background_queued); 1765 if (unlikely(r && r != -ENOENT)) { 1766 DMERR_LIMIT("%s: policy_lookup() failed with r = %d", 1767 cache_device_name(cache), r); 1768 bio_io_error(bio); 1769 return DM_MAPIO_SUBMITTED; 1770 } 1771 1772 if (background_queued) 1773 wake_migration_worker(cache); 1774 } 1775 1776 if (r == -ENOENT) { 1777 struct per_bio_data *pb = get_per_bio_data(bio); 1778 1779 /* 1780 * Miss. 1781 */ 1782 inc_miss_counter(cache, bio); 1783 if (pb->req_nr == 0) { 1784 accounted_begin(cache, bio); 1785 remap_to_origin_clear_discard(cache, bio, block); 1786 } else { 1787 /* 1788 * This is a duplicate writethrough io that is no 1789 * longer needed because the block has been demoted. 1790 */ 1791 bio_endio(bio); 1792 return DM_MAPIO_SUBMITTED; 1793 } 1794 } else { 1795 /* 1796 * Hit. 1797 */ 1798 inc_hit_counter(cache, bio); 1799 1800 /* 1801 * Passthrough always maps to the origin, invalidating any 1802 * cache blocks that are written to. 1803 */ 1804 if (passthrough_mode(cache)) { 1805 if (bio_data_dir(bio) == WRITE) { 1806 bio_drop_shared_lock(cache, bio); 1807 atomic_inc(&cache->stats.demotion); 1808 invalidate_start(cache, cblock, block, bio); 1809 } else 1810 remap_to_origin_clear_discard(cache, bio, block); 1811 } else { 1812 if (bio_data_dir(bio) == WRITE && writethrough_mode(cache) && 1813 !is_dirty(cache, cblock)) { 1814 remap_to_origin_and_cache(cache, bio, block, cblock); 1815 accounted_begin(cache, bio); 1816 } else 1817 remap_to_cache_dirty(cache, bio, block, cblock); 1818 } 1819 } 1820 1821 /* 1822 * dm core turns FUA requests into a separate payload and FLUSH req. 1823 */ 1824 if (bio->bi_opf & REQ_FUA) { 1825 /* 1826 * issue_after_commit will call accounted_begin a second time. So 1827 * we call accounted_complete() to avoid double accounting. 1828 */ 1829 accounted_complete(cache, bio); 1830 issue_after_commit(&cache->committer, bio); 1831 *commit_needed = true; 1832 return DM_MAPIO_SUBMITTED; 1833 } 1834 1835 return DM_MAPIO_REMAPPED; 1836 } 1837 1838 static bool process_bio(struct cache *cache, struct bio *bio) 1839 { 1840 bool commit_needed; 1841 1842 if (map_bio(cache, bio, get_bio_block(cache, bio), &commit_needed) == DM_MAPIO_REMAPPED) 1843 generic_make_request(bio); 1844 1845 return commit_needed; 1846 } 1847 1848 /* 1849 * A non-zero return indicates read_only or fail_io mode. 1850 */ 1851 static int commit(struct cache *cache, bool clean_shutdown) 1852 { 1853 int r; 1854 1855 if (get_cache_mode(cache) >= CM_READ_ONLY) 1856 return -EINVAL; 1857 1858 atomic_inc(&cache->stats.commit_count); 1859 r = dm_cache_commit(cache->cmd, clean_shutdown); 1860 if (r) 1861 metadata_operation_failed(cache, "dm_cache_commit", r); 1862 1863 return r; 1864 } 1865 1866 /* 1867 * Used by the batcher. 1868 */ 1869 static blk_status_t commit_op(void *context) 1870 { 1871 struct cache *cache = context; 1872 1873 if (dm_cache_changed_this_transaction(cache->cmd)) 1874 return errno_to_blk_status(commit(cache, false)); 1875 1876 return 0; 1877 } 1878 1879 /*----------------------------------------------------------------*/ 1880 1881 static bool process_flush_bio(struct cache *cache, struct bio *bio) 1882 { 1883 struct per_bio_data *pb = get_per_bio_data(bio); 1884 1885 if (!pb->req_nr) 1886 remap_to_origin(cache, bio); 1887 else 1888 remap_to_cache(cache, bio, 0); 1889 1890 issue_after_commit(&cache->committer, bio); 1891 return true; 1892 } 1893 1894 static bool process_discard_bio(struct cache *cache, struct bio *bio) 1895 { 1896 dm_dblock_t b, e; 1897 1898 // FIXME: do we need to lock the region? Or can we just assume the 1899 // user wont be so foolish as to issue discard concurrently with 1900 // other IO? 1901 calc_discard_block_range(cache, bio, &b, &e); 1902 while (b != e) { 1903 set_discard(cache, b); 1904 b = to_dblock(from_dblock(b) + 1); 1905 } 1906 1907 bio_endio(bio); 1908 1909 return false; 1910 } 1911 1912 static void process_deferred_bios(struct work_struct *ws) 1913 { 1914 struct cache *cache = container_of(ws, struct cache, deferred_bio_worker); 1915 1916 unsigned long flags; 1917 bool commit_needed = false; 1918 struct bio_list bios; 1919 struct bio *bio; 1920 1921 bio_list_init(&bios); 1922 1923 spin_lock_irqsave(&cache->lock, flags); 1924 bio_list_merge(&bios, &cache->deferred_bios); 1925 bio_list_init(&cache->deferred_bios); 1926 spin_unlock_irqrestore(&cache->lock, flags); 1927 1928 while ((bio = bio_list_pop(&bios))) { 1929 if (bio->bi_opf & REQ_PREFLUSH) 1930 commit_needed = process_flush_bio(cache, bio) || commit_needed; 1931 1932 else if (bio_op(bio) == REQ_OP_DISCARD) 1933 commit_needed = process_discard_bio(cache, bio) || commit_needed; 1934 1935 else 1936 commit_needed = process_bio(cache, bio) || commit_needed; 1937 } 1938 1939 if (commit_needed) 1940 schedule_commit(&cache->committer); 1941 } 1942 1943 /*---------------------------------------------------------------- 1944 * Main worker loop 1945 *--------------------------------------------------------------*/ 1946 1947 static void requeue_deferred_bios(struct cache *cache) 1948 { 1949 struct bio *bio; 1950 struct bio_list bios; 1951 1952 bio_list_init(&bios); 1953 bio_list_merge(&bios, &cache->deferred_bios); 1954 bio_list_init(&cache->deferred_bios); 1955 1956 while ((bio = bio_list_pop(&bios))) { 1957 bio->bi_status = BLK_STS_DM_REQUEUE; 1958 bio_endio(bio); 1959 } 1960 } 1961 1962 /* 1963 * We want to commit periodically so that not too much 1964 * unwritten metadata builds up. 1965 */ 1966 static void do_waker(struct work_struct *ws) 1967 { 1968 struct cache *cache = container_of(to_delayed_work(ws), struct cache, waker); 1969 1970 policy_tick(cache->policy, true); 1971 wake_migration_worker(cache); 1972 schedule_commit(&cache->committer); 1973 queue_delayed_work(cache->wq, &cache->waker, COMMIT_PERIOD); 1974 } 1975 1976 static void check_migrations(struct work_struct *ws) 1977 { 1978 int r; 1979 struct policy_work *op; 1980 struct cache *cache = container_of(ws, struct cache, migration_worker); 1981 enum busy b; 1982 1983 for (;;) { 1984 b = spare_migration_bandwidth(cache); 1985 1986 r = policy_get_background_work(cache->policy, b == IDLE, &op); 1987 if (r == -ENODATA) 1988 break; 1989 1990 if (r) { 1991 DMERR_LIMIT("%s: policy_background_work failed", 1992 cache_device_name(cache)); 1993 break; 1994 } 1995 1996 r = mg_start(cache, op, NULL); 1997 if (r) 1998 break; 1999 } 2000 } 2001 2002 /*---------------------------------------------------------------- 2003 * Target methods 2004 *--------------------------------------------------------------*/ 2005 2006 /* 2007 * This function gets called on the error paths of the constructor, so we 2008 * have to cope with a partially initialised struct. 2009 */ 2010 static void destroy(struct cache *cache) 2011 { 2012 unsigned i; 2013 2014 mempool_destroy(cache->migration_pool); 2015 2016 if (cache->prison) 2017 dm_bio_prison_destroy_v2(cache->prison); 2018 2019 if (cache->wq) 2020 destroy_workqueue(cache->wq); 2021 2022 if (cache->dirty_bitset) 2023 free_bitset(cache->dirty_bitset); 2024 2025 if (cache->discard_bitset) 2026 free_bitset(cache->discard_bitset); 2027 2028 if (cache->copier) 2029 dm_kcopyd_client_destroy(cache->copier); 2030 2031 if (cache->cmd) 2032 dm_cache_metadata_close(cache->cmd); 2033 2034 if (cache->metadata_dev) 2035 dm_put_device(cache->ti, cache->metadata_dev); 2036 2037 if (cache->origin_dev) 2038 dm_put_device(cache->ti, cache->origin_dev); 2039 2040 if (cache->cache_dev) 2041 dm_put_device(cache->ti, cache->cache_dev); 2042 2043 if (cache->policy) 2044 dm_cache_policy_destroy(cache->policy); 2045 2046 for (i = 0; i < cache->nr_ctr_args ; i++) 2047 kfree(cache->ctr_args[i]); 2048 kfree(cache->ctr_args); 2049 2050 if (cache->bs) 2051 bioset_free(cache->bs); 2052 2053 kfree(cache); 2054 } 2055 2056 static void cache_dtr(struct dm_target *ti) 2057 { 2058 struct cache *cache = ti->private; 2059 2060 destroy(cache); 2061 } 2062 2063 static sector_t get_dev_size(struct dm_dev *dev) 2064 { 2065 return i_size_read(dev->bdev->bd_inode) >> SECTOR_SHIFT; 2066 } 2067 2068 /*----------------------------------------------------------------*/ 2069 2070 /* 2071 * Construct a cache device mapping. 2072 * 2073 * cache <metadata dev> <cache dev> <origin dev> <block size> 2074 * <#feature args> [<feature arg>]* 2075 * <policy> <#policy args> [<policy arg>]* 2076 * 2077 * metadata dev : fast device holding the persistent metadata 2078 * cache dev : fast device holding cached data blocks 2079 * origin dev : slow device holding original data blocks 2080 * block size : cache unit size in sectors 2081 * 2082 * #feature args : number of feature arguments passed 2083 * feature args : writethrough. (The default is writeback.) 2084 * 2085 * policy : the replacement policy to use 2086 * #policy args : an even number of policy arguments corresponding 2087 * to key/value pairs passed to the policy 2088 * policy args : key/value pairs passed to the policy 2089 * E.g. 'sequential_threshold 1024' 2090 * See cache-policies.txt for details. 2091 * 2092 * Optional feature arguments are: 2093 * writethrough : write through caching that prohibits cache block 2094 * content from being different from origin block content. 2095 * Without this argument, the default behaviour is to write 2096 * back cache block contents later for performance reasons, 2097 * so they may differ from the corresponding origin blocks. 2098 */ 2099 struct cache_args { 2100 struct dm_target *ti; 2101 2102 struct dm_dev *metadata_dev; 2103 2104 struct dm_dev *cache_dev; 2105 sector_t cache_sectors; 2106 2107 struct dm_dev *origin_dev; 2108 sector_t origin_sectors; 2109 2110 uint32_t block_size; 2111 2112 const char *policy_name; 2113 int policy_argc; 2114 const char **policy_argv; 2115 2116 struct cache_features features; 2117 }; 2118 2119 static void destroy_cache_args(struct cache_args *ca) 2120 { 2121 if (ca->metadata_dev) 2122 dm_put_device(ca->ti, ca->metadata_dev); 2123 2124 if (ca->cache_dev) 2125 dm_put_device(ca->ti, ca->cache_dev); 2126 2127 if (ca->origin_dev) 2128 dm_put_device(ca->ti, ca->origin_dev); 2129 2130 kfree(ca); 2131 } 2132 2133 static bool at_least_one_arg(struct dm_arg_set *as, char **error) 2134 { 2135 if (!as->argc) { 2136 *error = "Insufficient args"; 2137 return false; 2138 } 2139 2140 return true; 2141 } 2142 2143 static int parse_metadata_dev(struct cache_args *ca, struct dm_arg_set *as, 2144 char **error) 2145 { 2146 int r; 2147 sector_t metadata_dev_size; 2148 char b[BDEVNAME_SIZE]; 2149 2150 if (!at_least_one_arg(as, error)) 2151 return -EINVAL; 2152 2153 r = dm_get_device(ca->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE, 2154 &ca->metadata_dev); 2155 if (r) { 2156 *error = "Error opening metadata device"; 2157 return r; 2158 } 2159 2160 metadata_dev_size = get_dev_size(ca->metadata_dev); 2161 if (metadata_dev_size > DM_CACHE_METADATA_MAX_SECTORS_WARNING) 2162 DMWARN("Metadata device %s is larger than %u sectors: excess space will not be used.", 2163 bdevname(ca->metadata_dev->bdev, b), THIN_METADATA_MAX_SECTORS); 2164 2165 return 0; 2166 } 2167 2168 static int parse_cache_dev(struct cache_args *ca, struct dm_arg_set *as, 2169 char **error) 2170 { 2171 int r; 2172 2173 if (!at_least_one_arg(as, error)) 2174 return -EINVAL; 2175 2176 r = dm_get_device(ca->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE, 2177 &ca->cache_dev); 2178 if (r) { 2179 *error = "Error opening cache device"; 2180 return r; 2181 } 2182 ca->cache_sectors = get_dev_size(ca->cache_dev); 2183 2184 return 0; 2185 } 2186 2187 static int parse_origin_dev(struct cache_args *ca, struct dm_arg_set *as, 2188 char **error) 2189 { 2190 int r; 2191 2192 if (!at_least_one_arg(as, error)) 2193 return -EINVAL; 2194 2195 r = dm_get_device(ca->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE, 2196 &ca->origin_dev); 2197 if (r) { 2198 *error = "Error opening origin device"; 2199 return r; 2200 } 2201 2202 ca->origin_sectors = get_dev_size(ca->origin_dev); 2203 if (ca->ti->len > ca->origin_sectors) { 2204 *error = "Device size larger than cached device"; 2205 return -EINVAL; 2206 } 2207 2208 return 0; 2209 } 2210 2211 static int parse_block_size(struct cache_args *ca, struct dm_arg_set *as, 2212 char **error) 2213 { 2214 unsigned long block_size; 2215 2216 if (!at_least_one_arg(as, error)) 2217 return -EINVAL; 2218 2219 if (kstrtoul(dm_shift_arg(as), 10, &block_size) || !block_size || 2220 block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS || 2221 block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS || 2222 block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) { 2223 *error = "Invalid data block size"; 2224 return -EINVAL; 2225 } 2226 2227 if (block_size > ca->cache_sectors) { 2228 *error = "Data block size is larger than the cache device"; 2229 return -EINVAL; 2230 } 2231 2232 ca->block_size = block_size; 2233 2234 return 0; 2235 } 2236 2237 static void init_features(struct cache_features *cf) 2238 { 2239 cf->mode = CM_WRITE; 2240 cf->io_mode = CM_IO_WRITEBACK; 2241 cf->metadata_version = 1; 2242 } 2243 2244 static int parse_features(struct cache_args *ca, struct dm_arg_set *as, 2245 char **error) 2246 { 2247 static const struct dm_arg _args[] = { 2248 {0, 2, "Invalid number of cache feature arguments"}, 2249 }; 2250 2251 int r; 2252 unsigned argc; 2253 const char *arg; 2254 struct cache_features *cf = &ca->features; 2255 2256 init_features(cf); 2257 2258 r = dm_read_arg_group(_args, as, &argc, error); 2259 if (r) 2260 return -EINVAL; 2261 2262 while (argc--) { 2263 arg = dm_shift_arg(as); 2264 2265 if (!strcasecmp(arg, "writeback")) 2266 cf->io_mode = CM_IO_WRITEBACK; 2267 2268 else if (!strcasecmp(arg, "writethrough")) 2269 cf->io_mode = CM_IO_WRITETHROUGH; 2270 2271 else if (!strcasecmp(arg, "passthrough")) 2272 cf->io_mode = CM_IO_PASSTHROUGH; 2273 2274 else if (!strcasecmp(arg, "metadata2")) 2275 cf->metadata_version = 2; 2276 2277 else { 2278 *error = "Unrecognised cache feature requested"; 2279 return -EINVAL; 2280 } 2281 } 2282 2283 return 0; 2284 } 2285 2286 static int parse_policy(struct cache_args *ca, struct dm_arg_set *as, 2287 char **error) 2288 { 2289 static const struct dm_arg _args[] = { 2290 {0, 1024, "Invalid number of policy arguments"}, 2291 }; 2292 2293 int r; 2294 2295 if (!at_least_one_arg(as, error)) 2296 return -EINVAL; 2297 2298 ca->policy_name = dm_shift_arg(as); 2299 2300 r = dm_read_arg_group(_args, as, &ca->policy_argc, error); 2301 if (r) 2302 return -EINVAL; 2303 2304 ca->policy_argv = (const char **)as->argv; 2305 dm_consume_args(as, ca->policy_argc); 2306 2307 return 0; 2308 } 2309 2310 static int parse_cache_args(struct cache_args *ca, int argc, char **argv, 2311 char **error) 2312 { 2313 int r; 2314 struct dm_arg_set as; 2315 2316 as.argc = argc; 2317 as.argv = argv; 2318 2319 r = parse_metadata_dev(ca, &as, error); 2320 if (r) 2321 return r; 2322 2323 r = parse_cache_dev(ca, &as, error); 2324 if (r) 2325 return r; 2326 2327 r = parse_origin_dev(ca, &as, error); 2328 if (r) 2329 return r; 2330 2331 r = parse_block_size(ca, &as, error); 2332 if (r) 2333 return r; 2334 2335 r = parse_features(ca, &as, error); 2336 if (r) 2337 return r; 2338 2339 r = parse_policy(ca, &as, error); 2340 if (r) 2341 return r; 2342 2343 return 0; 2344 } 2345 2346 /*----------------------------------------------------------------*/ 2347 2348 static struct kmem_cache *migration_cache; 2349 2350 #define NOT_CORE_OPTION 1 2351 2352 static int process_config_option(struct cache *cache, const char *key, const char *value) 2353 { 2354 unsigned long tmp; 2355 2356 if (!strcasecmp(key, "migration_threshold")) { 2357 if (kstrtoul(value, 10, &tmp)) 2358 return -EINVAL; 2359 2360 cache->migration_threshold = tmp; 2361 return 0; 2362 } 2363 2364 return NOT_CORE_OPTION; 2365 } 2366 2367 static int set_config_value(struct cache *cache, const char *key, const char *value) 2368 { 2369 int r = process_config_option(cache, key, value); 2370 2371 if (r == NOT_CORE_OPTION) 2372 r = policy_set_config_value(cache->policy, key, value); 2373 2374 if (r) 2375 DMWARN("bad config value for %s: %s", key, value); 2376 2377 return r; 2378 } 2379 2380 static int set_config_values(struct cache *cache, int argc, const char **argv) 2381 { 2382 int r = 0; 2383 2384 if (argc & 1) { 2385 DMWARN("Odd number of policy arguments given but they should be <key> <value> pairs."); 2386 return -EINVAL; 2387 } 2388 2389 while (argc) { 2390 r = set_config_value(cache, argv[0], argv[1]); 2391 if (r) 2392 break; 2393 2394 argc -= 2; 2395 argv += 2; 2396 } 2397 2398 return r; 2399 } 2400 2401 static int create_cache_policy(struct cache *cache, struct cache_args *ca, 2402 char **error) 2403 { 2404 struct dm_cache_policy *p = dm_cache_policy_create(ca->policy_name, 2405 cache->cache_size, 2406 cache->origin_sectors, 2407 cache->sectors_per_block); 2408 if (IS_ERR(p)) { 2409 *error = "Error creating cache's policy"; 2410 return PTR_ERR(p); 2411 } 2412 cache->policy = p; 2413 BUG_ON(!cache->policy); 2414 2415 return 0; 2416 } 2417 2418 /* 2419 * We want the discard block size to be at least the size of the cache 2420 * block size and have no more than 2^14 discard blocks across the origin. 2421 */ 2422 #define MAX_DISCARD_BLOCKS (1 << 14) 2423 2424 static bool too_many_discard_blocks(sector_t discard_block_size, 2425 sector_t origin_size) 2426 { 2427 (void) sector_div(origin_size, discard_block_size); 2428 2429 return origin_size > MAX_DISCARD_BLOCKS; 2430 } 2431 2432 static sector_t calculate_discard_block_size(sector_t cache_block_size, 2433 sector_t origin_size) 2434 { 2435 sector_t discard_block_size = cache_block_size; 2436 2437 if (origin_size) 2438 while (too_many_discard_blocks(discard_block_size, origin_size)) 2439 discard_block_size *= 2; 2440 2441 return discard_block_size; 2442 } 2443 2444 static void set_cache_size(struct cache *cache, dm_cblock_t size) 2445 { 2446 dm_block_t nr_blocks = from_cblock(size); 2447 2448 if (nr_blocks > (1 << 20) && cache->cache_size != size) 2449 DMWARN_LIMIT("You have created a cache device with a lot of individual cache blocks (%llu)\n" 2450 "All these mappings can consume a lot of kernel memory, and take some time to read/write.\n" 2451 "Please consider increasing the cache block size to reduce the overall cache block count.", 2452 (unsigned long long) nr_blocks); 2453 2454 cache->cache_size = size; 2455 } 2456 2457 static int is_congested(struct dm_dev *dev, int bdi_bits) 2458 { 2459 struct request_queue *q = bdev_get_queue(dev->bdev); 2460 return bdi_congested(q->backing_dev_info, bdi_bits); 2461 } 2462 2463 static int cache_is_congested(struct dm_target_callbacks *cb, int bdi_bits) 2464 { 2465 struct cache *cache = container_of(cb, struct cache, callbacks); 2466 2467 return is_congested(cache->origin_dev, bdi_bits) || 2468 is_congested(cache->cache_dev, bdi_bits); 2469 } 2470 2471 #define DEFAULT_MIGRATION_THRESHOLD 2048 2472 2473 static int cache_create(struct cache_args *ca, struct cache **result) 2474 { 2475 int r = 0; 2476 char **error = &ca->ti->error; 2477 struct cache *cache; 2478 struct dm_target *ti = ca->ti; 2479 dm_block_t origin_blocks; 2480 struct dm_cache_metadata *cmd; 2481 bool may_format = ca->features.mode == CM_WRITE; 2482 2483 cache = kzalloc(sizeof(*cache), GFP_KERNEL); 2484 if (!cache) 2485 return -ENOMEM; 2486 2487 cache->ti = ca->ti; 2488 ti->private = cache; 2489 ti->num_flush_bios = 2; 2490 ti->flush_supported = true; 2491 2492 ti->num_discard_bios = 1; 2493 ti->discards_supported = true; 2494 ti->split_discard_bios = false; 2495 2496 ti->per_io_data_size = sizeof(struct per_bio_data); 2497 2498 cache->features = ca->features; 2499 if (writethrough_mode(cache)) { 2500 /* Create bioset for writethrough bios issued to origin */ 2501 cache->bs = bioset_create(BIO_POOL_SIZE, 0, 0); 2502 if (!cache->bs) 2503 goto bad; 2504 } 2505 2506 cache->callbacks.congested_fn = cache_is_congested; 2507 dm_table_add_target_callbacks(ti->table, &cache->callbacks); 2508 2509 cache->metadata_dev = ca->metadata_dev; 2510 cache->origin_dev = ca->origin_dev; 2511 cache->cache_dev = ca->cache_dev; 2512 2513 ca->metadata_dev = ca->origin_dev = ca->cache_dev = NULL; 2514 2515 origin_blocks = cache->origin_sectors = ca->origin_sectors; 2516 origin_blocks = block_div(origin_blocks, ca->block_size); 2517 cache->origin_blocks = to_oblock(origin_blocks); 2518 2519 cache->sectors_per_block = ca->block_size; 2520 if (dm_set_target_max_io_len(ti, cache->sectors_per_block)) { 2521 r = -EINVAL; 2522 goto bad; 2523 } 2524 2525 if (ca->block_size & (ca->block_size - 1)) { 2526 dm_block_t cache_size = ca->cache_sectors; 2527 2528 cache->sectors_per_block_shift = -1; 2529 cache_size = block_div(cache_size, ca->block_size); 2530 set_cache_size(cache, to_cblock(cache_size)); 2531 } else { 2532 cache->sectors_per_block_shift = __ffs(ca->block_size); 2533 set_cache_size(cache, to_cblock(ca->cache_sectors >> cache->sectors_per_block_shift)); 2534 } 2535 2536 r = create_cache_policy(cache, ca, error); 2537 if (r) 2538 goto bad; 2539 2540 cache->policy_nr_args = ca->policy_argc; 2541 cache->migration_threshold = DEFAULT_MIGRATION_THRESHOLD; 2542 2543 r = set_config_values(cache, ca->policy_argc, ca->policy_argv); 2544 if (r) { 2545 *error = "Error setting cache policy's config values"; 2546 goto bad; 2547 } 2548 2549 cmd = dm_cache_metadata_open(cache->metadata_dev->bdev, 2550 ca->block_size, may_format, 2551 dm_cache_policy_get_hint_size(cache->policy), 2552 ca->features.metadata_version); 2553 if (IS_ERR(cmd)) { 2554 *error = "Error creating metadata object"; 2555 r = PTR_ERR(cmd); 2556 goto bad; 2557 } 2558 cache->cmd = cmd; 2559 set_cache_mode(cache, CM_WRITE); 2560 if (get_cache_mode(cache) != CM_WRITE) { 2561 *error = "Unable to get write access to metadata, please check/repair metadata."; 2562 r = -EINVAL; 2563 goto bad; 2564 } 2565 2566 if (passthrough_mode(cache)) { 2567 bool all_clean; 2568 2569 r = dm_cache_metadata_all_clean(cache->cmd, &all_clean); 2570 if (r) { 2571 *error = "dm_cache_metadata_all_clean() failed"; 2572 goto bad; 2573 } 2574 2575 if (!all_clean) { 2576 *error = "Cannot enter passthrough mode unless all blocks are clean"; 2577 r = -EINVAL; 2578 goto bad; 2579 } 2580 2581 policy_allow_migrations(cache->policy, false); 2582 } 2583 2584 spin_lock_init(&cache->lock); 2585 bio_list_init(&cache->deferred_bios); 2586 atomic_set(&cache->nr_allocated_migrations, 0); 2587 atomic_set(&cache->nr_io_migrations, 0); 2588 init_waitqueue_head(&cache->migration_wait); 2589 2590 r = -ENOMEM; 2591 atomic_set(&cache->nr_dirty, 0); 2592 cache->dirty_bitset = alloc_bitset(from_cblock(cache->cache_size)); 2593 if (!cache->dirty_bitset) { 2594 *error = "could not allocate dirty bitset"; 2595 goto bad; 2596 } 2597 clear_bitset(cache->dirty_bitset, from_cblock(cache->cache_size)); 2598 2599 cache->discard_block_size = 2600 calculate_discard_block_size(cache->sectors_per_block, 2601 cache->origin_sectors); 2602 cache->discard_nr_blocks = to_dblock(dm_sector_div_up(cache->origin_sectors, 2603 cache->discard_block_size)); 2604 cache->discard_bitset = alloc_bitset(from_dblock(cache->discard_nr_blocks)); 2605 if (!cache->discard_bitset) { 2606 *error = "could not allocate discard bitset"; 2607 goto bad; 2608 } 2609 clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks)); 2610 2611 cache->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle); 2612 if (IS_ERR(cache->copier)) { 2613 *error = "could not create kcopyd client"; 2614 r = PTR_ERR(cache->copier); 2615 goto bad; 2616 } 2617 2618 cache->wq = alloc_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM, 0); 2619 if (!cache->wq) { 2620 *error = "could not create workqueue for metadata object"; 2621 goto bad; 2622 } 2623 INIT_WORK(&cache->deferred_bio_worker, process_deferred_bios); 2624 INIT_WORK(&cache->migration_worker, check_migrations); 2625 INIT_DELAYED_WORK(&cache->waker, do_waker); 2626 2627 cache->prison = dm_bio_prison_create_v2(cache->wq); 2628 if (!cache->prison) { 2629 *error = "could not create bio prison"; 2630 goto bad; 2631 } 2632 2633 cache->migration_pool = mempool_create_slab_pool(MIGRATION_POOL_SIZE, 2634 migration_cache); 2635 if (!cache->migration_pool) { 2636 *error = "Error creating cache's migration mempool"; 2637 goto bad; 2638 } 2639 2640 cache->need_tick_bio = true; 2641 cache->sized = false; 2642 cache->invalidate = false; 2643 cache->commit_requested = false; 2644 cache->loaded_mappings = false; 2645 cache->loaded_discards = false; 2646 2647 load_stats(cache); 2648 2649 atomic_set(&cache->stats.demotion, 0); 2650 atomic_set(&cache->stats.promotion, 0); 2651 atomic_set(&cache->stats.copies_avoided, 0); 2652 atomic_set(&cache->stats.cache_cell_clash, 0); 2653 atomic_set(&cache->stats.commit_count, 0); 2654 atomic_set(&cache->stats.discard_count, 0); 2655 2656 spin_lock_init(&cache->invalidation_lock); 2657 INIT_LIST_HEAD(&cache->invalidation_requests); 2658 2659 batcher_init(&cache->committer, commit_op, cache, 2660 issue_op, cache, cache->wq); 2661 iot_init(&cache->tracker); 2662 2663 init_rwsem(&cache->background_work_lock); 2664 prevent_background_work(cache); 2665 2666 *result = cache; 2667 return 0; 2668 bad: 2669 destroy(cache); 2670 return r; 2671 } 2672 2673 static int copy_ctr_args(struct cache *cache, int argc, const char **argv) 2674 { 2675 unsigned i; 2676 const char **copy; 2677 2678 copy = kcalloc(argc, sizeof(*copy), GFP_KERNEL); 2679 if (!copy) 2680 return -ENOMEM; 2681 for (i = 0; i < argc; i++) { 2682 copy[i] = kstrdup(argv[i], GFP_KERNEL); 2683 if (!copy[i]) { 2684 while (i--) 2685 kfree(copy[i]); 2686 kfree(copy); 2687 return -ENOMEM; 2688 } 2689 } 2690 2691 cache->nr_ctr_args = argc; 2692 cache->ctr_args = copy; 2693 2694 return 0; 2695 } 2696 2697 static int cache_ctr(struct dm_target *ti, unsigned argc, char **argv) 2698 { 2699 int r = -EINVAL; 2700 struct cache_args *ca; 2701 struct cache *cache = NULL; 2702 2703 ca = kzalloc(sizeof(*ca), GFP_KERNEL); 2704 if (!ca) { 2705 ti->error = "Error allocating memory for cache"; 2706 return -ENOMEM; 2707 } 2708 ca->ti = ti; 2709 2710 r = parse_cache_args(ca, argc, argv, &ti->error); 2711 if (r) 2712 goto out; 2713 2714 r = cache_create(ca, &cache); 2715 if (r) 2716 goto out; 2717 2718 r = copy_ctr_args(cache, argc - 3, (const char **)argv + 3); 2719 if (r) { 2720 destroy(cache); 2721 goto out; 2722 } 2723 2724 ti->private = cache; 2725 out: 2726 destroy_cache_args(ca); 2727 return r; 2728 } 2729 2730 /*----------------------------------------------------------------*/ 2731 2732 static int cache_map(struct dm_target *ti, struct bio *bio) 2733 { 2734 struct cache *cache = ti->private; 2735 2736 int r; 2737 bool commit_needed; 2738 dm_oblock_t block = get_bio_block(cache, bio); 2739 2740 init_per_bio_data(bio); 2741 if (unlikely(from_oblock(block) >= from_oblock(cache->origin_blocks))) { 2742 /* 2743 * This can only occur if the io goes to a partial block at 2744 * the end of the origin device. We don't cache these. 2745 * Just remap to the origin and carry on. 2746 */ 2747 remap_to_origin(cache, bio); 2748 accounted_begin(cache, bio); 2749 return DM_MAPIO_REMAPPED; 2750 } 2751 2752 if (discard_or_flush(bio)) { 2753 defer_bio(cache, bio); 2754 return DM_MAPIO_SUBMITTED; 2755 } 2756 2757 r = map_bio(cache, bio, block, &commit_needed); 2758 if (commit_needed) 2759 schedule_commit(&cache->committer); 2760 2761 return r; 2762 } 2763 2764 static int cache_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *error) 2765 { 2766 struct cache *cache = ti->private; 2767 unsigned long flags; 2768 struct per_bio_data *pb = get_per_bio_data(bio); 2769 2770 if (pb->tick) { 2771 policy_tick(cache->policy, false); 2772 2773 spin_lock_irqsave(&cache->lock, flags); 2774 cache->need_tick_bio = true; 2775 spin_unlock_irqrestore(&cache->lock, flags); 2776 } 2777 2778 bio_drop_shared_lock(cache, bio); 2779 accounted_complete(cache, bio); 2780 2781 return DM_ENDIO_DONE; 2782 } 2783 2784 static int write_dirty_bitset(struct cache *cache) 2785 { 2786 int r; 2787 2788 if (get_cache_mode(cache) >= CM_READ_ONLY) 2789 return -EINVAL; 2790 2791 r = dm_cache_set_dirty_bits(cache->cmd, from_cblock(cache->cache_size), cache->dirty_bitset); 2792 if (r) 2793 metadata_operation_failed(cache, "dm_cache_set_dirty_bits", r); 2794 2795 return r; 2796 } 2797 2798 static int write_discard_bitset(struct cache *cache) 2799 { 2800 unsigned i, r; 2801 2802 if (get_cache_mode(cache) >= CM_READ_ONLY) 2803 return -EINVAL; 2804 2805 r = dm_cache_discard_bitset_resize(cache->cmd, cache->discard_block_size, 2806 cache->discard_nr_blocks); 2807 if (r) { 2808 DMERR("%s: could not resize on-disk discard bitset", cache_device_name(cache)); 2809 metadata_operation_failed(cache, "dm_cache_discard_bitset_resize", r); 2810 return r; 2811 } 2812 2813 for (i = 0; i < from_dblock(cache->discard_nr_blocks); i++) { 2814 r = dm_cache_set_discard(cache->cmd, to_dblock(i), 2815 is_discarded(cache, to_dblock(i))); 2816 if (r) { 2817 metadata_operation_failed(cache, "dm_cache_set_discard", r); 2818 return r; 2819 } 2820 } 2821 2822 return 0; 2823 } 2824 2825 static int write_hints(struct cache *cache) 2826 { 2827 int r; 2828 2829 if (get_cache_mode(cache) >= CM_READ_ONLY) 2830 return -EINVAL; 2831 2832 r = dm_cache_write_hints(cache->cmd, cache->policy); 2833 if (r) { 2834 metadata_operation_failed(cache, "dm_cache_write_hints", r); 2835 return r; 2836 } 2837 2838 return 0; 2839 } 2840 2841 /* 2842 * returns true on success 2843 */ 2844 static bool sync_metadata(struct cache *cache) 2845 { 2846 int r1, r2, r3, r4; 2847 2848 r1 = write_dirty_bitset(cache); 2849 if (r1) 2850 DMERR("%s: could not write dirty bitset", cache_device_name(cache)); 2851 2852 r2 = write_discard_bitset(cache); 2853 if (r2) 2854 DMERR("%s: could not write discard bitset", cache_device_name(cache)); 2855 2856 save_stats(cache); 2857 2858 r3 = write_hints(cache); 2859 if (r3) 2860 DMERR("%s: could not write hints", cache_device_name(cache)); 2861 2862 /* 2863 * If writing the above metadata failed, we still commit, but don't 2864 * set the clean shutdown flag. This will effectively force every 2865 * dirty bit to be set on reload. 2866 */ 2867 r4 = commit(cache, !r1 && !r2 && !r3); 2868 if (r4) 2869 DMERR("%s: could not write cache metadata", cache_device_name(cache)); 2870 2871 return !r1 && !r2 && !r3 && !r4; 2872 } 2873 2874 static void cache_postsuspend(struct dm_target *ti) 2875 { 2876 struct cache *cache = ti->private; 2877 2878 prevent_background_work(cache); 2879 BUG_ON(atomic_read(&cache->nr_io_migrations)); 2880 2881 cancel_delayed_work(&cache->waker); 2882 flush_workqueue(cache->wq); 2883 WARN_ON(cache->tracker.in_flight); 2884 2885 /* 2886 * If it's a flush suspend there won't be any deferred bios, so this 2887 * call is harmless. 2888 */ 2889 requeue_deferred_bios(cache); 2890 2891 if (get_cache_mode(cache) == CM_WRITE) 2892 (void) sync_metadata(cache); 2893 } 2894 2895 static int load_mapping(void *context, dm_oblock_t oblock, dm_cblock_t cblock, 2896 bool dirty, uint32_t hint, bool hint_valid) 2897 { 2898 int r; 2899 struct cache *cache = context; 2900 2901 if (dirty) { 2902 set_bit(from_cblock(cblock), cache->dirty_bitset); 2903 atomic_inc(&cache->nr_dirty); 2904 } else 2905 clear_bit(from_cblock(cblock), cache->dirty_bitset); 2906 2907 r = policy_load_mapping(cache->policy, oblock, cblock, dirty, hint, hint_valid); 2908 if (r) 2909 return r; 2910 2911 return 0; 2912 } 2913 2914 /* 2915 * The discard block size in the on disk metadata is not 2916 * neccessarily the same as we're currently using. So we have to 2917 * be careful to only set the discarded attribute if we know it 2918 * covers a complete block of the new size. 2919 */ 2920 struct discard_load_info { 2921 struct cache *cache; 2922 2923 /* 2924 * These blocks are sized using the on disk dblock size, rather 2925 * than the current one. 2926 */ 2927 dm_block_t block_size; 2928 dm_block_t discard_begin, discard_end; 2929 }; 2930 2931 static void discard_load_info_init(struct cache *cache, 2932 struct discard_load_info *li) 2933 { 2934 li->cache = cache; 2935 li->discard_begin = li->discard_end = 0; 2936 } 2937 2938 static void set_discard_range(struct discard_load_info *li) 2939 { 2940 sector_t b, e; 2941 2942 if (li->discard_begin == li->discard_end) 2943 return; 2944 2945 /* 2946 * Convert to sectors. 2947 */ 2948 b = li->discard_begin * li->block_size; 2949 e = li->discard_end * li->block_size; 2950 2951 /* 2952 * Then convert back to the current dblock size. 2953 */ 2954 b = dm_sector_div_up(b, li->cache->discard_block_size); 2955 sector_div(e, li->cache->discard_block_size); 2956 2957 /* 2958 * The origin may have shrunk, so we need to check we're still in 2959 * bounds. 2960 */ 2961 if (e > from_dblock(li->cache->discard_nr_blocks)) 2962 e = from_dblock(li->cache->discard_nr_blocks); 2963 2964 for (; b < e; b++) 2965 set_discard(li->cache, to_dblock(b)); 2966 } 2967 2968 static int load_discard(void *context, sector_t discard_block_size, 2969 dm_dblock_t dblock, bool discard) 2970 { 2971 struct discard_load_info *li = context; 2972 2973 li->block_size = discard_block_size; 2974 2975 if (discard) { 2976 if (from_dblock(dblock) == li->discard_end) 2977 /* 2978 * We're already in a discard range, just extend it. 2979 */ 2980 li->discard_end = li->discard_end + 1ULL; 2981 2982 else { 2983 /* 2984 * Emit the old range and start a new one. 2985 */ 2986 set_discard_range(li); 2987 li->discard_begin = from_dblock(dblock); 2988 li->discard_end = li->discard_begin + 1ULL; 2989 } 2990 } else { 2991 set_discard_range(li); 2992 li->discard_begin = li->discard_end = 0; 2993 } 2994 2995 return 0; 2996 } 2997 2998 static dm_cblock_t get_cache_dev_size(struct cache *cache) 2999 { 3000 sector_t size = get_dev_size(cache->cache_dev); 3001 (void) sector_div(size, cache->sectors_per_block); 3002 return to_cblock(size); 3003 } 3004 3005 static bool can_resize(struct cache *cache, dm_cblock_t new_size) 3006 { 3007 if (from_cblock(new_size) > from_cblock(cache->cache_size)) 3008 return true; 3009 3010 /* 3011 * We can't drop a dirty block when shrinking the cache. 3012 */ 3013 while (from_cblock(new_size) < from_cblock(cache->cache_size)) { 3014 new_size = to_cblock(from_cblock(new_size) + 1); 3015 if (is_dirty(cache, new_size)) { 3016 DMERR("%s: unable to shrink cache; cache block %llu is dirty", 3017 cache_device_name(cache), 3018 (unsigned long long) from_cblock(new_size)); 3019 return false; 3020 } 3021 } 3022 3023 return true; 3024 } 3025 3026 static int resize_cache_dev(struct cache *cache, dm_cblock_t new_size) 3027 { 3028 int r; 3029 3030 r = dm_cache_resize(cache->cmd, new_size); 3031 if (r) { 3032 DMERR("%s: could not resize cache metadata", cache_device_name(cache)); 3033 metadata_operation_failed(cache, "dm_cache_resize", r); 3034 return r; 3035 } 3036 3037 set_cache_size(cache, new_size); 3038 3039 return 0; 3040 } 3041 3042 static int cache_preresume(struct dm_target *ti) 3043 { 3044 int r = 0; 3045 struct cache *cache = ti->private; 3046 dm_cblock_t csize = get_cache_dev_size(cache); 3047 3048 /* 3049 * Check to see if the cache has resized. 3050 */ 3051 if (!cache->sized) { 3052 r = resize_cache_dev(cache, csize); 3053 if (r) 3054 return r; 3055 3056 cache->sized = true; 3057 3058 } else if (csize != cache->cache_size) { 3059 if (!can_resize(cache, csize)) 3060 return -EINVAL; 3061 3062 r = resize_cache_dev(cache, csize); 3063 if (r) 3064 return r; 3065 } 3066 3067 if (!cache->loaded_mappings) { 3068 r = dm_cache_load_mappings(cache->cmd, cache->policy, 3069 load_mapping, cache); 3070 if (r) { 3071 DMERR("%s: could not load cache mappings", cache_device_name(cache)); 3072 metadata_operation_failed(cache, "dm_cache_load_mappings", r); 3073 return r; 3074 } 3075 3076 cache->loaded_mappings = true; 3077 } 3078 3079 if (!cache->loaded_discards) { 3080 struct discard_load_info li; 3081 3082 /* 3083 * The discard bitset could have been resized, or the 3084 * discard block size changed. To be safe we start by 3085 * setting every dblock to not discarded. 3086 */ 3087 clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks)); 3088 3089 discard_load_info_init(cache, &li); 3090 r = dm_cache_load_discards(cache->cmd, load_discard, &li); 3091 if (r) { 3092 DMERR("%s: could not load origin discards", cache_device_name(cache)); 3093 metadata_operation_failed(cache, "dm_cache_load_discards", r); 3094 return r; 3095 } 3096 set_discard_range(&li); 3097 3098 cache->loaded_discards = true; 3099 } 3100 3101 return r; 3102 } 3103 3104 static void cache_resume(struct dm_target *ti) 3105 { 3106 struct cache *cache = ti->private; 3107 3108 cache->need_tick_bio = true; 3109 allow_background_work(cache); 3110 do_waker(&cache->waker.work); 3111 } 3112 3113 /* 3114 * Status format: 3115 * 3116 * <metadata block size> <#used metadata blocks>/<#total metadata blocks> 3117 * <cache block size> <#used cache blocks>/<#total cache blocks> 3118 * <#read hits> <#read misses> <#write hits> <#write misses> 3119 * <#demotions> <#promotions> <#dirty> 3120 * <#features> <features>* 3121 * <#core args> <core args> 3122 * <policy name> <#policy args> <policy args>* <cache metadata mode> <needs_check> 3123 */ 3124 static void cache_status(struct dm_target *ti, status_type_t type, 3125 unsigned status_flags, char *result, unsigned maxlen) 3126 { 3127 int r = 0; 3128 unsigned i; 3129 ssize_t sz = 0; 3130 dm_block_t nr_free_blocks_metadata = 0; 3131 dm_block_t nr_blocks_metadata = 0; 3132 char buf[BDEVNAME_SIZE]; 3133 struct cache *cache = ti->private; 3134 dm_cblock_t residency; 3135 bool needs_check; 3136 3137 switch (type) { 3138 case STATUSTYPE_INFO: 3139 if (get_cache_mode(cache) == CM_FAIL) { 3140 DMEMIT("Fail"); 3141 break; 3142 } 3143 3144 /* Commit to ensure statistics aren't out-of-date */ 3145 if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti)) 3146 (void) commit(cache, false); 3147 3148 r = dm_cache_get_free_metadata_block_count(cache->cmd, &nr_free_blocks_metadata); 3149 if (r) { 3150 DMERR("%s: dm_cache_get_free_metadata_block_count returned %d", 3151 cache_device_name(cache), r); 3152 goto err; 3153 } 3154 3155 r = dm_cache_get_metadata_dev_size(cache->cmd, &nr_blocks_metadata); 3156 if (r) { 3157 DMERR("%s: dm_cache_get_metadata_dev_size returned %d", 3158 cache_device_name(cache), r); 3159 goto err; 3160 } 3161 3162 residency = policy_residency(cache->policy); 3163 3164 DMEMIT("%u %llu/%llu %llu %llu/%llu %u %u %u %u %u %u %lu ", 3165 (unsigned)DM_CACHE_METADATA_BLOCK_SIZE, 3166 (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata), 3167 (unsigned long long)nr_blocks_metadata, 3168 (unsigned long long)cache->sectors_per_block, 3169 (unsigned long long) from_cblock(residency), 3170 (unsigned long long) from_cblock(cache->cache_size), 3171 (unsigned) atomic_read(&cache->stats.read_hit), 3172 (unsigned) atomic_read(&cache->stats.read_miss), 3173 (unsigned) atomic_read(&cache->stats.write_hit), 3174 (unsigned) atomic_read(&cache->stats.write_miss), 3175 (unsigned) atomic_read(&cache->stats.demotion), 3176 (unsigned) atomic_read(&cache->stats.promotion), 3177 (unsigned long) atomic_read(&cache->nr_dirty)); 3178 3179 if (cache->features.metadata_version == 2) 3180 DMEMIT("2 metadata2 "); 3181 else 3182 DMEMIT("1 "); 3183 3184 if (writethrough_mode(cache)) 3185 DMEMIT("writethrough "); 3186 3187 else if (passthrough_mode(cache)) 3188 DMEMIT("passthrough "); 3189 3190 else if (writeback_mode(cache)) 3191 DMEMIT("writeback "); 3192 3193 else { 3194 DMERR("%s: internal error: unknown io mode: %d", 3195 cache_device_name(cache), (int) cache->features.io_mode); 3196 goto err; 3197 } 3198 3199 DMEMIT("2 migration_threshold %llu ", (unsigned long long) cache->migration_threshold); 3200 3201 DMEMIT("%s ", dm_cache_policy_get_name(cache->policy)); 3202 if (sz < maxlen) { 3203 r = policy_emit_config_values(cache->policy, result, maxlen, &sz); 3204 if (r) 3205 DMERR("%s: policy_emit_config_values returned %d", 3206 cache_device_name(cache), r); 3207 } 3208 3209 if (get_cache_mode(cache) == CM_READ_ONLY) 3210 DMEMIT("ro "); 3211 else 3212 DMEMIT("rw "); 3213 3214 r = dm_cache_metadata_needs_check(cache->cmd, &needs_check); 3215 3216 if (r || needs_check) 3217 DMEMIT("needs_check "); 3218 else 3219 DMEMIT("- "); 3220 3221 break; 3222 3223 case STATUSTYPE_TABLE: 3224 format_dev_t(buf, cache->metadata_dev->bdev->bd_dev); 3225 DMEMIT("%s ", buf); 3226 format_dev_t(buf, cache->cache_dev->bdev->bd_dev); 3227 DMEMIT("%s ", buf); 3228 format_dev_t(buf, cache->origin_dev->bdev->bd_dev); 3229 DMEMIT("%s", buf); 3230 3231 for (i = 0; i < cache->nr_ctr_args - 1; i++) 3232 DMEMIT(" %s", cache->ctr_args[i]); 3233 if (cache->nr_ctr_args) 3234 DMEMIT(" %s", cache->ctr_args[cache->nr_ctr_args - 1]); 3235 } 3236 3237 return; 3238 3239 err: 3240 DMEMIT("Error"); 3241 } 3242 3243 /* 3244 * Defines a range of cblocks, begin to (end - 1) are in the range. end is 3245 * the one-past-the-end value. 3246 */ 3247 struct cblock_range { 3248 dm_cblock_t begin; 3249 dm_cblock_t end; 3250 }; 3251 3252 /* 3253 * A cache block range can take two forms: 3254 * 3255 * i) A single cblock, eg. '3456' 3256 * ii) A begin and end cblock with a dash between, eg. 123-234 3257 */ 3258 static int parse_cblock_range(struct cache *cache, const char *str, 3259 struct cblock_range *result) 3260 { 3261 char dummy; 3262 uint64_t b, e; 3263 int r; 3264 3265 /* 3266 * Try and parse form (ii) first. 3267 */ 3268 r = sscanf(str, "%llu-%llu%c", &b, &e, &dummy); 3269 if (r < 0) 3270 return r; 3271 3272 if (r == 2) { 3273 result->begin = to_cblock(b); 3274 result->end = to_cblock(e); 3275 return 0; 3276 } 3277 3278 /* 3279 * That didn't work, try form (i). 3280 */ 3281 r = sscanf(str, "%llu%c", &b, &dummy); 3282 if (r < 0) 3283 return r; 3284 3285 if (r == 1) { 3286 result->begin = to_cblock(b); 3287 result->end = to_cblock(from_cblock(result->begin) + 1u); 3288 return 0; 3289 } 3290 3291 DMERR("%s: invalid cblock range '%s'", cache_device_name(cache), str); 3292 return -EINVAL; 3293 } 3294 3295 static int validate_cblock_range(struct cache *cache, struct cblock_range *range) 3296 { 3297 uint64_t b = from_cblock(range->begin); 3298 uint64_t e = from_cblock(range->end); 3299 uint64_t n = from_cblock(cache->cache_size); 3300 3301 if (b >= n) { 3302 DMERR("%s: begin cblock out of range: %llu >= %llu", 3303 cache_device_name(cache), b, n); 3304 return -EINVAL; 3305 } 3306 3307 if (e > n) { 3308 DMERR("%s: end cblock out of range: %llu > %llu", 3309 cache_device_name(cache), e, n); 3310 return -EINVAL; 3311 } 3312 3313 if (b >= e) { 3314 DMERR("%s: invalid cblock range: %llu >= %llu", 3315 cache_device_name(cache), b, e); 3316 return -EINVAL; 3317 } 3318 3319 return 0; 3320 } 3321 3322 static inline dm_cblock_t cblock_succ(dm_cblock_t b) 3323 { 3324 return to_cblock(from_cblock(b) + 1); 3325 } 3326 3327 static int request_invalidation(struct cache *cache, struct cblock_range *range) 3328 { 3329 int r = 0; 3330 3331 /* 3332 * We don't need to do any locking here because we know we're in 3333 * passthrough mode. There's is potential for a race between an 3334 * invalidation triggered by an io and an invalidation message. This 3335 * is harmless, we must not worry if the policy call fails. 3336 */ 3337 while (range->begin != range->end) { 3338 r = invalidate_cblock(cache, range->begin); 3339 if (r) 3340 return r; 3341 3342 range->begin = cblock_succ(range->begin); 3343 } 3344 3345 cache->commit_requested = true; 3346 return r; 3347 } 3348 3349 static int process_invalidate_cblocks_message(struct cache *cache, unsigned count, 3350 const char **cblock_ranges) 3351 { 3352 int r = 0; 3353 unsigned i; 3354 struct cblock_range range; 3355 3356 if (!passthrough_mode(cache)) { 3357 DMERR("%s: cache has to be in passthrough mode for invalidation", 3358 cache_device_name(cache)); 3359 return -EPERM; 3360 } 3361 3362 for (i = 0; i < count; i++) { 3363 r = parse_cblock_range(cache, cblock_ranges[i], &range); 3364 if (r) 3365 break; 3366 3367 r = validate_cblock_range(cache, &range); 3368 if (r) 3369 break; 3370 3371 /* 3372 * Pass begin and end origin blocks to the worker and wake it. 3373 */ 3374 r = request_invalidation(cache, &range); 3375 if (r) 3376 break; 3377 } 3378 3379 return r; 3380 } 3381 3382 /* 3383 * Supports 3384 * "<key> <value>" 3385 * and 3386 * "invalidate_cblocks [(<begin>)|(<begin>-<end>)]* 3387 * 3388 * The key migration_threshold is supported by the cache target core. 3389 */ 3390 static int cache_message(struct dm_target *ti, unsigned argc, char **argv) 3391 { 3392 struct cache *cache = ti->private; 3393 3394 if (!argc) 3395 return -EINVAL; 3396 3397 if (get_cache_mode(cache) >= CM_READ_ONLY) { 3398 DMERR("%s: unable to service cache target messages in READ_ONLY or FAIL mode", 3399 cache_device_name(cache)); 3400 return -EOPNOTSUPP; 3401 } 3402 3403 if (!strcasecmp(argv[0], "invalidate_cblocks")) 3404 return process_invalidate_cblocks_message(cache, argc - 1, (const char **) argv + 1); 3405 3406 if (argc != 2) 3407 return -EINVAL; 3408 3409 return set_config_value(cache, argv[0], argv[1]); 3410 } 3411 3412 static int cache_iterate_devices(struct dm_target *ti, 3413 iterate_devices_callout_fn fn, void *data) 3414 { 3415 int r = 0; 3416 struct cache *cache = ti->private; 3417 3418 r = fn(ti, cache->cache_dev, 0, get_dev_size(cache->cache_dev), data); 3419 if (!r) 3420 r = fn(ti, cache->origin_dev, 0, ti->len, data); 3421 3422 return r; 3423 } 3424 3425 static void set_discard_limits(struct cache *cache, struct queue_limits *limits) 3426 { 3427 /* 3428 * FIXME: these limits may be incompatible with the cache device 3429 */ 3430 limits->max_discard_sectors = min_t(sector_t, cache->discard_block_size * 1024, 3431 cache->origin_sectors); 3432 limits->discard_granularity = cache->discard_block_size << SECTOR_SHIFT; 3433 } 3434 3435 static void cache_io_hints(struct dm_target *ti, struct queue_limits *limits) 3436 { 3437 struct cache *cache = ti->private; 3438 uint64_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT; 3439 3440 /* 3441 * If the system-determined stacked limits are compatible with the 3442 * cache's blocksize (io_opt is a factor) do not override them. 3443 */ 3444 if (io_opt_sectors < cache->sectors_per_block || 3445 do_div(io_opt_sectors, cache->sectors_per_block)) { 3446 blk_limits_io_min(limits, cache->sectors_per_block << SECTOR_SHIFT); 3447 blk_limits_io_opt(limits, cache->sectors_per_block << SECTOR_SHIFT); 3448 } 3449 set_discard_limits(cache, limits); 3450 } 3451 3452 /*----------------------------------------------------------------*/ 3453 3454 static struct target_type cache_target = { 3455 .name = "cache", 3456 .version = {2, 0, 0}, 3457 .module = THIS_MODULE, 3458 .ctr = cache_ctr, 3459 .dtr = cache_dtr, 3460 .map = cache_map, 3461 .end_io = cache_end_io, 3462 .postsuspend = cache_postsuspend, 3463 .preresume = cache_preresume, 3464 .resume = cache_resume, 3465 .status = cache_status, 3466 .message = cache_message, 3467 .iterate_devices = cache_iterate_devices, 3468 .io_hints = cache_io_hints, 3469 }; 3470 3471 static int __init dm_cache_init(void) 3472 { 3473 int r; 3474 3475 migration_cache = KMEM_CACHE(dm_cache_migration, 0); 3476 if (!migration_cache) { 3477 dm_unregister_target(&cache_target); 3478 return -ENOMEM; 3479 } 3480 3481 r = dm_register_target(&cache_target); 3482 if (r) { 3483 DMERR("cache target registration failed: %d", r); 3484 return r; 3485 } 3486 3487 return 0; 3488 } 3489 3490 static void __exit dm_cache_exit(void) 3491 { 3492 dm_unregister_target(&cache_target); 3493 kmem_cache_destroy(migration_cache); 3494 } 3495 3496 module_init(dm_cache_init); 3497 module_exit(dm_cache_exit); 3498 3499 MODULE_DESCRIPTION(DM_NAME " cache target"); 3500 MODULE_AUTHOR("Joe Thornber <ejt@redhat.com>"); 3501 MODULE_LICENSE("GPL"); 3502