1 /* 2 * Copyright (C) 2012 Red Hat. All rights reserved. 3 * 4 * This file is released under the GPL. 5 */ 6 7 #include "dm.h" 8 #include "dm-bio-prison.h" 9 #include "dm-bio-record.h" 10 #include "dm-cache-metadata.h" 11 12 #include <linux/dm-io.h> 13 #include <linux/dm-kcopyd.h> 14 #include <linux/jiffies.h> 15 #include <linux/init.h> 16 #include <linux/mempool.h> 17 #include <linux/module.h> 18 #include <linux/slab.h> 19 #include <linux/vmalloc.h> 20 21 #define DM_MSG_PREFIX "cache" 22 23 DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(cache_copy_throttle, 24 "A percentage of time allocated for copying to and/or from cache"); 25 26 /*----------------------------------------------------------------*/ 27 28 #define IOT_RESOLUTION 4 29 30 struct io_tracker { 31 spinlock_t lock; 32 33 /* 34 * Sectors of in-flight IO. 35 */ 36 sector_t in_flight; 37 38 /* 39 * The time, in jiffies, when this device became idle (if it is 40 * indeed idle). 41 */ 42 unsigned long idle_time; 43 unsigned long last_update_time; 44 }; 45 46 static void iot_init(struct io_tracker *iot) 47 { 48 spin_lock_init(&iot->lock); 49 iot->in_flight = 0ul; 50 iot->idle_time = 0ul; 51 iot->last_update_time = jiffies; 52 } 53 54 static bool __iot_idle_for(struct io_tracker *iot, unsigned long jifs) 55 { 56 if (iot->in_flight) 57 return false; 58 59 return time_after(jiffies, iot->idle_time + jifs); 60 } 61 62 static bool iot_idle_for(struct io_tracker *iot, unsigned long jifs) 63 { 64 bool r; 65 unsigned long flags; 66 67 spin_lock_irqsave(&iot->lock, flags); 68 r = __iot_idle_for(iot, jifs); 69 spin_unlock_irqrestore(&iot->lock, flags); 70 71 return r; 72 } 73 74 static void iot_io_begin(struct io_tracker *iot, sector_t len) 75 { 76 unsigned long flags; 77 78 spin_lock_irqsave(&iot->lock, flags); 79 iot->in_flight += len; 80 spin_unlock_irqrestore(&iot->lock, flags); 81 } 82 83 static void __iot_io_end(struct io_tracker *iot, sector_t len) 84 { 85 iot->in_flight -= len; 86 if (!iot->in_flight) 87 iot->idle_time = jiffies; 88 } 89 90 static void iot_io_end(struct io_tracker *iot, sector_t len) 91 { 92 unsigned long flags; 93 94 spin_lock_irqsave(&iot->lock, flags); 95 __iot_io_end(iot, len); 96 spin_unlock_irqrestore(&iot->lock, flags); 97 } 98 99 /*----------------------------------------------------------------*/ 100 101 /* 102 * Glossary: 103 * 104 * oblock: index of an origin block 105 * cblock: index of a cache block 106 * promotion: movement of a block from origin to cache 107 * demotion: movement of a block from cache to origin 108 * migration: movement of a block between the origin and cache device, 109 * either direction 110 */ 111 112 /*----------------------------------------------------------------*/ 113 114 /* 115 * There are a couple of places where we let a bio run, but want to do some 116 * work before calling its endio function. We do this by temporarily 117 * changing the endio fn. 118 */ 119 struct dm_hook_info { 120 bio_end_io_t *bi_end_io; 121 void *bi_private; 122 }; 123 124 static void dm_hook_bio(struct dm_hook_info *h, struct bio *bio, 125 bio_end_io_t *bi_end_io, void *bi_private) 126 { 127 h->bi_end_io = bio->bi_end_io; 128 h->bi_private = bio->bi_private; 129 130 bio->bi_end_io = bi_end_io; 131 bio->bi_private = bi_private; 132 } 133 134 static void dm_unhook_bio(struct dm_hook_info *h, struct bio *bio) 135 { 136 bio->bi_end_io = h->bi_end_io; 137 bio->bi_private = h->bi_private; 138 } 139 140 /*----------------------------------------------------------------*/ 141 142 #define MIGRATION_POOL_SIZE 128 143 #define COMMIT_PERIOD HZ 144 #define MIGRATION_COUNT_WINDOW 10 145 146 /* 147 * The block size of the device holding cache data must be 148 * between 32KB and 1GB. 149 */ 150 #define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (32 * 1024 >> SECTOR_SHIFT) 151 #define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT) 152 153 enum cache_metadata_mode { 154 CM_WRITE, /* metadata may be changed */ 155 CM_READ_ONLY, /* metadata may not be changed */ 156 CM_FAIL 157 }; 158 159 enum cache_io_mode { 160 /* 161 * Data is written to cached blocks only. These blocks are marked 162 * dirty. If you lose the cache device you will lose data. 163 * Potential performance increase for both reads and writes. 164 */ 165 CM_IO_WRITEBACK, 166 167 /* 168 * Data is written to both cache and origin. Blocks are never 169 * dirty. Potential performance benfit for reads only. 170 */ 171 CM_IO_WRITETHROUGH, 172 173 /* 174 * A degraded mode useful for various cache coherency situations 175 * (eg, rolling back snapshots). Reads and writes always go to the 176 * origin. If a write goes to a cached oblock, then the cache 177 * block is invalidated. 178 */ 179 CM_IO_PASSTHROUGH 180 }; 181 182 struct cache_features { 183 enum cache_metadata_mode mode; 184 enum cache_io_mode io_mode; 185 }; 186 187 struct cache_stats { 188 atomic_t read_hit; 189 atomic_t read_miss; 190 atomic_t write_hit; 191 atomic_t write_miss; 192 atomic_t demotion; 193 atomic_t promotion; 194 atomic_t copies_avoided; 195 atomic_t cache_cell_clash; 196 atomic_t commit_count; 197 atomic_t discard_count; 198 }; 199 200 /* 201 * Defines a range of cblocks, begin to (end - 1) are in the range. end is 202 * the one-past-the-end value. 203 */ 204 struct cblock_range { 205 dm_cblock_t begin; 206 dm_cblock_t end; 207 }; 208 209 struct invalidation_request { 210 struct list_head list; 211 struct cblock_range *cblocks; 212 213 atomic_t complete; 214 int err; 215 216 wait_queue_head_t result_wait; 217 }; 218 219 struct cache { 220 struct dm_target *ti; 221 struct dm_target_callbacks callbacks; 222 223 struct dm_cache_metadata *cmd; 224 225 /* 226 * Metadata is written to this device. 227 */ 228 struct dm_dev *metadata_dev; 229 230 /* 231 * The slower of the two data devices. Typically a spindle. 232 */ 233 struct dm_dev *origin_dev; 234 235 /* 236 * The faster of the two data devices. Typically an SSD. 237 */ 238 struct dm_dev *cache_dev; 239 240 /* 241 * Size of the origin device in _complete_ blocks and native sectors. 242 */ 243 dm_oblock_t origin_blocks; 244 sector_t origin_sectors; 245 246 /* 247 * Size of the cache device in blocks. 248 */ 249 dm_cblock_t cache_size; 250 251 /* 252 * Fields for converting from sectors to blocks. 253 */ 254 uint32_t sectors_per_block; 255 int sectors_per_block_shift; 256 257 spinlock_t lock; 258 struct list_head deferred_cells; 259 struct bio_list deferred_bios; 260 struct bio_list deferred_flush_bios; 261 struct bio_list deferred_writethrough_bios; 262 struct list_head quiesced_migrations; 263 struct list_head completed_migrations; 264 struct list_head need_commit_migrations; 265 sector_t migration_threshold; 266 wait_queue_head_t migration_wait; 267 atomic_t nr_allocated_migrations; 268 269 /* 270 * The number of in flight migrations that are performing 271 * background io. eg, promotion, writeback. 272 */ 273 atomic_t nr_io_migrations; 274 275 wait_queue_head_t quiescing_wait; 276 atomic_t quiescing; 277 atomic_t quiescing_ack; 278 279 /* 280 * cache_size entries, dirty if set 281 */ 282 atomic_t nr_dirty; 283 unsigned long *dirty_bitset; 284 285 /* 286 * origin_blocks entries, discarded if set. 287 */ 288 dm_dblock_t discard_nr_blocks; 289 unsigned long *discard_bitset; 290 uint32_t discard_block_size; /* a power of 2 times sectors per block */ 291 292 /* 293 * Rather than reconstructing the table line for the status we just 294 * save it and regurgitate. 295 */ 296 unsigned nr_ctr_args; 297 const char **ctr_args; 298 299 struct dm_kcopyd_client *copier; 300 struct workqueue_struct *wq; 301 struct work_struct worker; 302 303 struct delayed_work waker; 304 unsigned long last_commit_jiffies; 305 306 struct dm_bio_prison *prison; 307 struct dm_deferred_set *all_io_ds; 308 309 mempool_t *migration_pool; 310 311 struct dm_cache_policy *policy; 312 unsigned policy_nr_args; 313 314 bool need_tick_bio:1; 315 bool sized:1; 316 bool invalidate:1; 317 bool commit_requested:1; 318 bool loaded_mappings:1; 319 bool loaded_discards:1; 320 321 /* 322 * Cache features such as write-through. 323 */ 324 struct cache_features features; 325 326 struct cache_stats stats; 327 328 /* 329 * Invalidation fields. 330 */ 331 spinlock_t invalidation_lock; 332 struct list_head invalidation_requests; 333 334 struct io_tracker origin_tracker; 335 }; 336 337 struct per_bio_data { 338 bool tick:1; 339 unsigned req_nr:2; 340 struct dm_deferred_entry *all_io_entry; 341 struct dm_hook_info hook_info; 342 sector_t len; 343 344 /* 345 * writethrough fields. These MUST remain at the end of this 346 * structure and the 'cache' member must be the first as it 347 * is used to determine the offset of the writethrough fields. 348 */ 349 struct cache *cache; 350 dm_cblock_t cblock; 351 struct dm_bio_details bio_details; 352 }; 353 354 struct dm_cache_migration { 355 struct list_head list; 356 struct cache *cache; 357 358 unsigned long start_jiffies; 359 dm_oblock_t old_oblock; 360 dm_oblock_t new_oblock; 361 dm_cblock_t cblock; 362 363 bool err:1; 364 bool discard:1; 365 bool writeback:1; 366 bool demote:1; 367 bool promote:1; 368 bool requeue_holder:1; 369 bool invalidate:1; 370 371 struct dm_bio_prison_cell *old_ocell; 372 struct dm_bio_prison_cell *new_ocell; 373 }; 374 375 /* 376 * Processing a bio in the worker thread may require these memory 377 * allocations. We prealloc to avoid deadlocks (the same worker thread 378 * frees them back to the mempool). 379 */ 380 struct prealloc { 381 struct dm_cache_migration *mg; 382 struct dm_bio_prison_cell *cell1; 383 struct dm_bio_prison_cell *cell2; 384 }; 385 386 static enum cache_metadata_mode get_cache_mode(struct cache *cache); 387 388 static void wake_worker(struct cache *cache) 389 { 390 queue_work(cache->wq, &cache->worker); 391 } 392 393 /*----------------------------------------------------------------*/ 394 395 static struct dm_bio_prison_cell *alloc_prison_cell(struct cache *cache) 396 { 397 /* FIXME: change to use a local slab. */ 398 return dm_bio_prison_alloc_cell(cache->prison, GFP_NOWAIT); 399 } 400 401 static void free_prison_cell(struct cache *cache, struct dm_bio_prison_cell *cell) 402 { 403 dm_bio_prison_free_cell(cache->prison, cell); 404 } 405 406 static struct dm_cache_migration *alloc_migration(struct cache *cache) 407 { 408 struct dm_cache_migration *mg; 409 410 mg = mempool_alloc(cache->migration_pool, GFP_NOWAIT); 411 if (mg) { 412 mg->cache = cache; 413 atomic_inc(&mg->cache->nr_allocated_migrations); 414 } 415 416 return mg; 417 } 418 419 static void free_migration(struct dm_cache_migration *mg) 420 { 421 struct cache *cache = mg->cache; 422 423 if (atomic_dec_and_test(&cache->nr_allocated_migrations)) 424 wake_up(&cache->migration_wait); 425 426 mempool_free(mg, cache->migration_pool); 427 } 428 429 static int prealloc_data_structs(struct cache *cache, struct prealloc *p) 430 { 431 if (!p->mg) { 432 p->mg = alloc_migration(cache); 433 if (!p->mg) 434 return -ENOMEM; 435 } 436 437 if (!p->cell1) { 438 p->cell1 = alloc_prison_cell(cache); 439 if (!p->cell1) 440 return -ENOMEM; 441 } 442 443 if (!p->cell2) { 444 p->cell2 = alloc_prison_cell(cache); 445 if (!p->cell2) 446 return -ENOMEM; 447 } 448 449 return 0; 450 } 451 452 static void prealloc_free_structs(struct cache *cache, struct prealloc *p) 453 { 454 if (p->cell2) 455 free_prison_cell(cache, p->cell2); 456 457 if (p->cell1) 458 free_prison_cell(cache, p->cell1); 459 460 if (p->mg) 461 free_migration(p->mg); 462 } 463 464 static struct dm_cache_migration *prealloc_get_migration(struct prealloc *p) 465 { 466 struct dm_cache_migration *mg = p->mg; 467 468 BUG_ON(!mg); 469 p->mg = NULL; 470 471 return mg; 472 } 473 474 /* 475 * You must have a cell within the prealloc struct to return. If not this 476 * function will BUG() rather than returning NULL. 477 */ 478 static struct dm_bio_prison_cell *prealloc_get_cell(struct prealloc *p) 479 { 480 struct dm_bio_prison_cell *r = NULL; 481 482 if (p->cell1) { 483 r = p->cell1; 484 p->cell1 = NULL; 485 486 } else if (p->cell2) { 487 r = p->cell2; 488 p->cell2 = NULL; 489 } else 490 BUG(); 491 492 return r; 493 } 494 495 /* 496 * You can't have more than two cells in a prealloc struct. BUG() will be 497 * called if you try and overfill. 498 */ 499 static void prealloc_put_cell(struct prealloc *p, struct dm_bio_prison_cell *cell) 500 { 501 if (!p->cell2) 502 p->cell2 = cell; 503 504 else if (!p->cell1) 505 p->cell1 = cell; 506 507 else 508 BUG(); 509 } 510 511 /*----------------------------------------------------------------*/ 512 513 static void build_key(dm_oblock_t begin, dm_oblock_t end, struct dm_cell_key *key) 514 { 515 key->virtual = 0; 516 key->dev = 0; 517 key->block_begin = from_oblock(begin); 518 key->block_end = from_oblock(end); 519 } 520 521 /* 522 * The caller hands in a preallocated cell, and a free function for it. 523 * The cell will be freed if there's an error, or if it wasn't used because 524 * a cell with that key already exists. 525 */ 526 typedef void (*cell_free_fn)(void *context, struct dm_bio_prison_cell *cell); 527 528 static int bio_detain_range(struct cache *cache, dm_oblock_t oblock_begin, dm_oblock_t oblock_end, 529 struct bio *bio, struct dm_bio_prison_cell *cell_prealloc, 530 cell_free_fn free_fn, void *free_context, 531 struct dm_bio_prison_cell **cell_result) 532 { 533 int r; 534 struct dm_cell_key key; 535 536 build_key(oblock_begin, oblock_end, &key); 537 r = dm_bio_detain(cache->prison, &key, bio, cell_prealloc, cell_result); 538 if (r) 539 free_fn(free_context, cell_prealloc); 540 541 return r; 542 } 543 544 static int bio_detain(struct cache *cache, dm_oblock_t oblock, 545 struct bio *bio, struct dm_bio_prison_cell *cell_prealloc, 546 cell_free_fn free_fn, void *free_context, 547 struct dm_bio_prison_cell **cell_result) 548 { 549 dm_oblock_t end = to_oblock(from_oblock(oblock) + 1ULL); 550 return bio_detain_range(cache, oblock, end, bio, 551 cell_prealloc, free_fn, free_context, cell_result); 552 } 553 554 static int get_cell(struct cache *cache, 555 dm_oblock_t oblock, 556 struct prealloc *structs, 557 struct dm_bio_prison_cell **cell_result) 558 { 559 int r; 560 struct dm_cell_key key; 561 struct dm_bio_prison_cell *cell_prealloc; 562 563 cell_prealloc = prealloc_get_cell(structs); 564 565 build_key(oblock, to_oblock(from_oblock(oblock) + 1ULL), &key); 566 r = dm_get_cell(cache->prison, &key, cell_prealloc, cell_result); 567 if (r) 568 prealloc_put_cell(structs, cell_prealloc); 569 570 return r; 571 } 572 573 /*----------------------------------------------------------------*/ 574 575 static bool is_dirty(struct cache *cache, dm_cblock_t b) 576 { 577 return test_bit(from_cblock(b), cache->dirty_bitset); 578 } 579 580 static void set_dirty(struct cache *cache, dm_oblock_t oblock, dm_cblock_t cblock) 581 { 582 if (!test_and_set_bit(from_cblock(cblock), cache->dirty_bitset)) { 583 atomic_inc(&cache->nr_dirty); 584 policy_set_dirty(cache->policy, oblock); 585 } 586 } 587 588 static void clear_dirty(struct cache *cache, dm_oblock_t oblock, dm_cblock_t cblock) 589 { 590 if (test_and_clear_bit(from_cblock(cblock), cache->dirty_bitset)) { 591 policy_clear_dirty(cache->policy, oblock); 592 if (atomic_dec_return(&cache->nr_dirty) == 0) 593 dm_table_event(cache->ti->table); 594 } 595 } 596 597 /*----------------------------------------------------------------*/ 598 599 static bool block_size_is_power_of_two(struct cache *cache) 600 { 601 return cache->sectors_per_block_shift >= 0; 602 } 603 604 /* gcc on ARM generates spurious references to __udivdi3 and __umoddi3 */ 605 #if defined(CONFIG_ARM) && __GNUC__ == 4 && __GNUC_MINOR__ <= 6 606 __always_inline 607 #endif 608 static dm_block_t block_div(dm_block_t b, uint32_t n) 609 { 610 do_div(b, n); 611 612 return b; 613 } 614 615 static dm_block_t oblocks_per_dblock(struct cache *cache) 616 { 617 dm_block_t oblocks = cache->discard_block_size; 618 619 if (block_size_is_power_of_two(cache)) 620 oblocks >>= cache->sectors_per_block_shift; 621 else 622 oblocks = block_div(oblocks, cache->sectors_per_block); 623 624 return oblocks; 625 } 626 627 static dm_dblock_t oblock_to_dblock(struct cache *cache, dm_oblock_t oblock) 628 { 629 return to_dblock(block_div(from_oblock(oblock), 630 oblocks_per_dblock(cache))); 631 } 632 633 static dm_oblock_t dblock_to_oblock(struct cache *cache, dm_dblock_t dblock) 634 { 635 return to_oblock(from_dblock(dblock) * oblocks_per_dblock(cache)); 636 } 637 638 static void set_discard(struct cache *cache, dm_dblock_t b) 639 { 640 unsigned long flags; 641 642 BUG_ON(from_dblock(b) >= from_dblock(cache->discard_nr_blocks)); 643 atomic_inc(&cache->stats.discard_count); 644 645 spin_lock_irqsave(&cache->lock, flags); 646 set_bit(from_dblock(b), cache->discard_bitset); 647 spin_unlock_irqrestore(&cache->lock, flags); 648 } 649 650 static void clear_discard(struct cache *cache, dm_dblock_t b) 651 { 652 unsigned long flags; 653 654 spin_lock_irqsave(&cache->lock, flags); 655 clear_bit(from_dblock(b), cache->discard_bitset); 656 spin_unlock_irqrestore(&cache->lock, flags); 657 } 658 659 static bool is_discarded(struct cache *cache, dm_dblock_t b) 660 { 661 int r; 662 unsigned long flags; 663 664 spin_lock_irqsave(&cache->lock, flags); 665 r = test_bit(from_dblock(b), cache->discard_bitset); 666 spin_unlock_irqrestore(&cache->lock, flags); 667 668 return r; 669 } 670 671 static bool is_discarded_oblock(struct cache *cache, dm_oblock_t b) 672 { 673 int r; 674 unsigned long flags; 675 676 spin_lock_irqsave(&cache->lock, flags); 677 r = test_bit(from_dblock(oblock_to_dblock(cache, b)), 678 cache->discard_bitset); 679 spin_unlock_irqrestore(&cache->lock, flags); 680 681 return r; 682 } 683 684 /*----------------------------------------------------------------*/ 685 686 static void load_stats(struct cache *cache) 687 { 688 struct dm_cache_statistics stats; 689 690 dm_cache_metadata_get_stats(cache->cmd, &stats); 691 atomic_set(&cache->stats.read_hit, stats.read_hits); 692 atomic_set(&cache->stats.read_miss, stats.read_misses); 693 atomic_set(&cache->stats.write_hit, stats.write_hits); 694 atomic_set(&cache->stats.write_miss, stats.write_misses); 695 } 696 697 static void save_stats(struct cache *cache) 698 { 699 struct dm_cache_statistics stats; 700 701 if (get_cache_mode(cache) >= CM_READ_ONLY) 702 return; 703 704 stats.read_hits = atomic_read(&cache->stats.read_hit); 705 stats.read_misses = atomic_read(&cache->stats.read_miss); 706 stats.write_hits = atomic_read(&cache->stats.write_hit); 707 stats.write_misses = atomic_read(&cache->stats.write_miss); 708 709 dm_cache_metadata_set_stats(cache->cmd, &stats); 710 } 711 712 /*---------------------------------------------------------------- 713 * Per bio data 714 *--------------------------------------------------------------*/ 715 716 /* 717 * If using writeback, leave out struct per_bio_data's writethrough fields. 718 */ 719 #define PB_DATA_SIZE_WB (offsetof(struct per_bio_data, cache)) 720 #define PB_DATA_SIZE_WT (sizeof(struct per_bio_data)) 721 722 static bool writethrough_mode(struct cache_features *f) 723 { 724 return f->io_mode == CM_IO_WRITETHROUGH; 725 } 726 727 static bool writeback_mode(struct cache_features *f) 728 { 729 return f->io_mode == CM_IO_WRITEBACK; 730 } 731 732 static bool passthrough_mode(struct cache_features *f) 733 { 734 return f->io_mode == CM_IO_PASSTHROUGH; 735 } 736 737 static size_t get_per_bio_data_size(struct cache *cache) 738 { 739 return writethrough_mode(&cache->features) ? PB_DATA_SIZE_WT : PB_DATA_SIZE_WB; 740 } 741 742 static struct per_bio_data *get_per_bio_data(struct bio *bio, size_t data_size) 743 { 744 struct per_bio_data *pb = dm_per_bio_data(bio, data_size); 745 BUG_ON(!pb); 746 return pb; 747 } 748 749 static struct per_bio_data *init_per_bio_data(struct bio *bio, size_t data_size) 750 { 751 struct per_bio_data *pb = get_per_bio_data(bio, data_size); 752 753 pb->tick = false; 754 pb->req_nr = dm_bio_get_target_bio_nr(bio); 755 pb->all_io_entry = NULL; 756 pb->len = 0; 757 758 return pb; 759 } 760 761 /*---------------------------------------------------------------- 762 * Remapping 763 *--------------------------------------------------------------*/ 764 static void remap_to_origin(struct cache *cache, struct bio *bio) 765 { 766 bio->bi_bdev = cache->origin_dev->bdev; 767 } 768 769 static void remap_to_cache(struct cache *cache, struct bio *bio, 770 dm_cblock_t cblock) 771 { 772 sector_t bi_sector = bio->bi_iter.bi_sector; 773 sector_t block = from_cblock(cblock); 774 775 bio->bi_bdev = cache->cache_dev->bdev; 776 if (!block_size_is_power_of_two(cache)) 777 bio->bi_iter.bi_sector = 778 (block * cache->sectors_per_block) + 779 sector_div(bi_sector, cache->sectors_per_block); 780 else 781 bio->bi_iter.bi_sector = 782 (block << cache->sectors_per_block_shift) | 783 (bi_sector & (cache->sectors_per_block - 1)); 784 } 785 786 static void check_if_tick_bio_needed(struct cache *cache, struct bio *bio) 787 { 788 unsigned long flags; 789 size_t pb_data_size = get_per_bio_data_size(cache); 790 struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size); 791 792 spin_lock_irqsave(&cache->lock, flags); 793 if (cache->need_tick_bio && 794 !(bio->bi_rw & (REQ_FUA | REQ_FLUSH | REQ_DISCARD))) { 795 pb->tick = true; 796 cache->need_tick_bio = false; 797 } 798 spin_unlock_irqrestore(&cache->lock, flags); 799 } 800 801 static void remap_to_origin_clear_discard(struct cache *cache, struct bio *bio, 802 dm_oblock_t oblock) 803 { 804 check_if_tick_bio_needed(cache, bio); 805 remap_to_origin(cache, bio); 806 if (bio_data_dir(bio) == WRITE) 807 clear_discard(cache, oblock_to_dblock(cache, oblock)); 808 } 809 810 static void remap_to_cache_dirty(struct cache *cache, struct bio *bio, 811 dm_oblock_t oblock, dm_cblock_t cblock) 812 { 813 check_if_tick_bio_needed(cache, bio); 814 remap_to_cache(cache, bio, cblock); 815 if (bio_data_dir(bio) == WRITE) { 816 set_dirty(cache, oblock, cblock); 817 clear_discard(cache, oblock_to_dblock(cache, oblock)); 818 } 819 } 820 821 static dm_oblock_t get_bio_block(struct cache *cache, struct bio *bio) 822 { 823 sector_t block_nr = bio->bi_iter.bi_sector; 824 825 if (!block_size_is_power_of_two(cache)) 826 (void) sector_div(block_nr, cache->sectors_per_block); 827 else 828 block_nr >>= cache->sectors_per_block_shift; 829 830 return to_oblock(block_nr); 831 } 832 833 static int bio_triggers_commit(struct cache *cache, struct bio *bio) 834 { 835 return bio->bi_rw & (REQ_FLUSH | REQ_FUA); 836 } 837 838 /* 839 * You must increment the deferred set whilst the prison cell is held. To 840 * encourage this, we ask for 'cell' to be passed in. 841 */ 842 static void inc_ds(struct cache *cache, struct bio *bio, 843 struct dm_bio_prison_cell *cell) 844 { 845 size_t pb_data_size = get_per_bio_data_size(cache); 846 struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size); 847 848 BUG_ON(!cell); 849 BUG_ON(pb->all_io_entry); 850 851 pb->all_io_entry = dm_deferred_entry_inc(cache->all_io_ds); 852 } 853 854 static bool accountable_bio(struct cache *cache, struct bio *bio) 855 { 856 return ((bio->bi_bdev == cache->origin_dev->bdev) && 857 !(bio->bi_rw & REQ_DISCARD)); 858 } 859 860 static void accounted_begin(struct cache *cache, struct bio *bio) 861 { 862 size_t pb_data_size = get_per_bio_data_size(cache); 863 struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size); 864 865 if (accountable_bio(cache, bio)) { 866 pb->len = bio_sectors(bio); 867 iot_io_begin(&cache->origin_tracker, pb->len); 868 } 869 } 870 871 static void accounted_complete(struct cache *cache, struct bio *bio) 872 { 873 size_t pb_data_size = get_per_bio_data_size(cache); 874 struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size); 875 876 iot_io_end(&cache->origin_tracker, pb->len); 877 } 878 879 static void accounted_request(struct cache *cache, struct bio *bio) 880 { 881 accounted_begin(cache, bio); 882 generic_make_request(bio); 883 } 884 885 static void issue(struct cache *cache, struct bio *bio) 886 { 887 unsigned long flags; 888 889 if (!bio_triggers_commit(cache, bio)) { 890 accounted_request(cache, bio); 891 return; 892 } 893 894 /* 895 * Batch together any bios that trigger commits and then issue a 896 * single commit for them in do_worker(). 897 */ 898 spin_lock_irqsave(&cache->lock, flags); 899 cache->commit_requested = true; 900 bio_list_add(&cache->deferred_flush_bios, bio); 901 spin_unlock_irqrestore(&cache->lock, flags); 902 } 903 904 static void inc_and_issue(struct cache *cache, struct bio *bio, struct dm_bio_prison_cell *cell) 905 { 906 inc_ds(cache, bio, cell); 907 issue(cache, bio); 908 } 909 910 static void defer_writethrough_bio(struct cache *cache, struct bio *bio) 911 { 912 unsigned long flags; 913 914 spin_lock_irqsave(&cache->lock, flags); 915 bio_list_add(&cache->deferred_writethrough_bios, bio); 916 spin_unlock_irqrestore(&cache->lock, flags); 917 918 wake_worker(cache); 919 } 920 921 static void writethrough_endio(struct bio *bio) 922 { 923 struct per_bio_data *pb = get_per_bio_data(bio, PB_DATA_SIZE_WT); 924 925 dm_unhook_bio(&pb->hook_info, bio); 926 927 if (bio->bi_error) { 928 bio_endio(bio); 929 return; 930 } 931 932 dm_bio_restore(&pb->bio_details, bio); 933 remap_to_cache(pb->cache, bio, pb->cblock); 934 935 /* 936 * We can't issue this bio directly, since we're in interrupt 937 * context. So it gets put on a bio list for processing by the 938 * worker thread. 939 */ 940 defer_writethrough_bio(pb->cache, bio); 941 } 942 943 /* 944 * When running in writethrough mode we need to send writes to clean blocks 945 * to both the cache and origin devices. In future we'd like to clone the 946 * bio and send them in parallel, but for now we're doing them in 947 * series as this is easier. 948 */ 949 static void remap_to_origin_then_cache(struct cache *cache, struct bio *bio, 950 dm_oblock_t oblock, dm_cblock_t cblock) 951 { 952 struct per_bio_data *pb = get_per_bio_data(bio, PB_DATA_SIZE_WT); 953 954 pb->cache = cache; 955 pb->cblock = cblock; 956 dm_hook_bio(&pb->hook_info, bio, writethrough_endio, NULL); 957 dm_bio_record(&pb->bio_details, bio); 958 959 remap_to_origin_clear_discard(pb->cache, bio, oblock); 960 } 961 962 /*---------------------------------------------------------------- 963 * Failure modes 964 *--------------------------------------------------------------*/ 965 static enum cache_metadata_mode get_cache_mode(struct cache *cache) 966 { 967 return cache->features.mode; 968 } 969 970 static const char *cache_device_name(struct cache *cache) 971 { 972 return dm_device_name(dm_table_get_md(cache->ti->table)); 973 } 974 975 static void notify_mode_switch(struct cache *cache, enum cache_metadata_mode mode) 976 { 977 const char *descs[] = { 978 "write", 979 "read-only", 980 "fail" 981 }; 982 983 dm_table_event(cache->ti->table); 984 DMINFO("%s: switching cache to %s mode", 985 cache_device_name(cache), descs[(int)mode]); 986 } 987 988 static void set_cache_mode(struct cache *cache, enum cache_metadata_mode new_mode) 989 { 990 bool needs_check = dm_cache_metadata_needs_check(cache->cmd); 991 enum cache_metadata_mode old_mode = get_cache_mode(cache); 992 993 if (new_mode == CM_WRITE && needs_check) { 994 DMERR("%s: unable to switch cache to write mode until repaired.", 995 cache_device_name(cache)); 996 if (old_mode != new_mode) 997 new_mode = old_mode; 998 else 999 new_mode = CM_READ_ONLY; 1000 } 1001 1002 /* Never move out of fail mode */ 1003 if (old_mode == CM_FAIL) 1004 new_mode = CM_FAIL; 1005 1006 switch (new_mode) { 1007 case CM_FAIL: 1008 case CM_READ_ONLY: 1009 dm_cache_metadata_set_read_only(cache->cmd); 1010 break; 1011 1012 case CM_WRITE: 1013 dm_cache_metadata_set_read_write(cache->cmd); 1014 break; 1015 } 1016 1017 cache->features.mode = new_mode; 1018 1019 if (new_mode != old_mode) 1020 notify_mode_switch(cache, new_mode); 1021 } 1022 1023 static void abort_transaction(struct cache *cache) 1024 { 1025 const char *dev_name = cache_device_name(cache); 1026 1027 if (get_cache_mode(cache) >= CM_READ_ONLY) 1028 return; 1029 1030 if (dm_cache_metadata_set_needs_check(cache->cmd)) { 1031 DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name); 1032 set_cache_mode(cache, CM_FAIL); 1033 } 1034 1035 DMERR_LIMIT("%s: aborting current metadata transaction", dev_name); 1036 if (dm_cache_metadata_abort(cache->cmd)) { 1037 DMERR("%s: failed to abort metadata transaction", dev_name); 1038 set_cache_mode(cache, CM_FAIL); 1039 } 1040 } 1041 1042 static void metadata_operation_failed(struct cache *cache, const char *op, int r) 1043 { 1044 DMERR_LIMIT("%s: metadata operation '%s' failed: error = %d", 1045 cache_device_name(cache), op, r); 1046 abort_transaction(cache); 1047 set_cache_mode(cache, CM_READ_ONLY); 1048 } 1049 1050 /*---------------------------------------------------------------- 1051 * Migration processing 1052 * 1053 * Migration covers moving data from the origin device to the cache, or 1054 * vice versa. 1055 *--------------------------------------------------------------*/ 1056 static void inc_io_migrations(struct cache *cache) 1057 { 1058 atomic_inc(&cache->nr_io_migrations); 1059 } 1060 1061 static void dec_io_migrations(struct cache *cache) 1062 { 1063 atomic_dec(&cache->nr_io_migrations); 1064 } 1065 1066 static bool discard_or_flush(struct bio *bio) 1067 { 1068 return bio->bi_rw & (REQ_FLUSH | REQ_FUA | REQ_DISCARD); 1069 } 1070 1071 static void __cell_defer(struct cache *cache, struct dm_bio_prison_cell *cell) 1072 { 1073 if (discard_or_flush(cell->holder)) { 1074 /* 1075 * We have to handle these bios individually. 1076 */ 1077 dm_cell_release(cache->prison, cell, &cache->deferred_bios); 1078 free_prison_cell(cache, cell); 1079 } else 1080 list_add_tail(&cell->user_list, &cache->deferred_cells); 1081 } 1082 1083 static void cell_defer(struct cache *cache, struct dm_bio_prison_cell *cell, bool holder) 1084 { 1085 unsigned long flags; 1086 1087 if (!holder && dm_cell_promote_or_release(cache->prison, cell)) { 1088 /* 1089 * There was no prisoner to promote to holder, the 1090 * cell has been released. 1091 */ 1092 free_prison_cell(cache, cell); 1093 return; 1094 } 1095 1096 spin_lock_irqsave(&cache->lock, flags); 1097 __cell_defer(cache, cell); 1098 spin_unlock_irqrestore(&cache->lock, flags); 1099 1100 wake_worker(cache); 1101 } 1102 1103 static void cell_error_with_code(struct cache *cache, struct dm_bio_prison_cell *cell, int err) 1104 { 1105 dm_cell_error(cache->prison, cell, err); 1106 free_prison_cell(cache, cell); 1107 } 1108 1109 static void cell_requeue(struct cache *cache, struct dm_bio_prison_cell *cell) 1110 { 1111 cell_error_with_code(cache, cell, DM_ENDIO_REQUEUE); 1112 } 1113 1114 static void free_io_migration(struct dm_cache_migration *mg) 1115 { 1116 struct cache *cache = mg->cache; 1117 1118 dec_io_migrations(cache); 1119 free_migration(mg); 1120 wake_worker(cache); 1121 } 1122 1123 static void migration_failure(struct dm_cache_migration *mg) 1124 { 1125 struct cache *cache = mg->cache; 1126 const char *dev_name = cache_device_name(cache); 1127 1128 if (mg->writeback) { 1129 DMERR_LIMIT("%s: writeback failed; couldn't copy block", dev_name); 1130 set_dirty(cache, mg->old_oblock, mg->cblock); 1131 cell_defer(cache, mg->old_ocell, false); 1132 1133 } else if (mg->demote) { 1134 DMERR_LIMIT("%s: demotion failed; couldn't copy block", dev_name); 1135 policy_force_mapping(cache->policy, mg->new_oblock, mg->old_oblock); 1136 1137 cell_defer(cache, mg->old_ocell, mg->promote ? false : true); 1138 if (mg->promote) 1139 cell_defer(cache, mg->new_ocell, true); 1140 } else { 1141 DMERR_LIMIT("%s: promotion failed; couldn't copy block", dev_name); 1142 policy_remove_mapping(cache->policy, mg->new_oblock); 1143 cell_defer(cache, mg->new_ocell, true); 1144 } 1145 1146 free_io_migration(mg); 1147 } 1148 1149 static void migration_success_pre_commit(struct dm_cache_migration *mg) 1150 { 1151 int r; 1152 unsigned long flags; 1153 struct cache *cache = mg->cache; 1154 1155 if (mg->writeback) { 1156 clear_dirty(cache, mg->old_oblock, mg->cblock); 1157 cell_defer(cache, mg->old_ocell, false); 1158 free_io_migration(mg); 1159 return; 1160 1161 } else if (mg->demote) { 1162 r = dm_cache_remove_mapping(cache->cmd, mg->cblock); 1163 if (r) { 1164 DMERR_LIMIT("%s: demotion failed; couldn't update on disk metadata", 1165 cache_device_name(cache)); 1166 metadata_operation_failed(cache, "dm_cache_remove_mapping", r); 1167 policy_force_mapping(cache->policy, mg->new_oblock, 1168 mg->old_oblock); 1169 if (mg->promote) 1170 cell_defer(cache, mg->new_ocell, true); 1171 free_io_migration(mg); 1172 return; 1173 } 1174 } else { 1175 r = dm_cache_insert_mapping(cache->cmd, mg->cblock, mg->new_oblock); 1176 if (r) { 1177 DMERR_LIMIT("%s: promotion failed; couldn't update on disk metadata", 1178 cache_device_name(cache)); 1179 metadata_operation_failed(cache, "dm_cache_insert_mapping", r); 1180 policy_remove_mapping(cache->policy, mg->new_oblock); 1181 free_io_migration(mg); 1182 return; 1183 } 1184 } 1185 1186 spin_lock_irqsave(&cache->lock, flags); 1187 list_add_tail(&mg->list, &cache->need_commit_migrations); 1188 cache->commit_requested = true; 1189 spin_unlock_irqrestore(&cache->lock, flags); 1190 } 1191 1192 static void migration_success_post_commit(struct dm_cache_migration *mg) 1193 { 1194 unsigned long flags; 1195 struct cache *cache = mg->cache; 1196 1197 if (mg->writeback) { 1198 DMWARN_LIMIT("%s: writeback unexpectedly triggered commit", 1199 cache_device_name(cache)); 1200 return; 1201 1202 } else if (mg->demote) { 1203 cell_defer(cache, mg->old_ocell, mg->promote ? false : true); 1204 1205 if (mg->promote) { 1206 mg->demote = false; 1207 1208 spin_lock_irqsave(&cache->lock, flags); 1209 list_add_tail(&mg->list, &cache->quiesced_migrations); 1210 spin_unlock_irqrestore(&cache->lock, flags); 1211 1212 } else { 1213 if (mg->invalidate) 1214 policy_remove_mapping(cache->policy, mg->old_oblock); 1215 free_io_migration(mg); 1216 } 1217 1218 } else { 1219 if (mg->requeue_holder) { 1220 clear_dirty(cache, mg->new_oblock, mg->cblock); 1221 cell_defer(cache, mg->new_ocell, true); 1222 } else { 1223 /* 1224 * The block was promoted via an overwrite, so it's dirty. 1225 */ 1226 set_dirty(cache, mg->new_oblock, mg->cblock); 1227 bio_endio(mg->new_ocell->holder); 1228 cell_defer(cache, mg->new_ocell, false); 1229 } 1230 free_io_migration(mg); 1231 } 1232 } 1233 1234 static void copy_complete(int read_err, unsigned long write_err, void *context) 1235 { 1236 unsigned long flags; 1237 struct dm_cache_migration *mg = (struct dm_cache_migration *) context; 1238 struct cache *cache = mg->cache; 1239 1240 if (read_err || write_err) 1241 mg->err = true; 1242 1243 spin_lock_irqsave(&cache->lock, flags); 1244 list_add_tail(&mg->list, &cache->completed_migrations); 1245 spin_unlock_irqrestore(&cache->lock, flags); 1246 1247 wake_worker(cache); 1248 } 1249 1250 static void issue_copy(struct dm_cache_migration *mg) 1251 { 1252 int r; 1253 struct dm_io_region o_region, c_region; 1254 struct cache *cache = mg->cache; 1255 sector_t cblock = from_cblock(mg->cblock); 1256 1257 o_region.bdev = cache->origin_dev->bdev; 1258 o_region.count = cache->sectors_per_block; 1259 1260 c_region.bdev = cache->cache_dev->bdev; 1261 c_region.sector = cblock * cache->sectors_per_block; 1262 c_region.count = cache->sectors_per_block; 1263 1264 if (mg->writeback || mg->demote) { 1265 /* demote */ 1266 o_region.sector = from_oblock(mg->old_oblock) * cache->sectors_per_block; 1267 r = dm_kcopyd_copy(cache->copier, &c_region, 1, &o_region, 0, copy_complete, mg); 1268 } else { 1269 /* promote */ 1270 o_region.sector = from_oblock(mg->new_oblock) * cache->sectors_per_block; 1271 r = dm_kcopyd_copy(cache->copier, &o_region, 1, &c_region, 0, copy_complete, mg); 1272 } 1273 1274 if (r < 0) { 1275 DMERR_LIMIT("%s: issuing migration failed", cache_device_name(cache)); 1276 migration_failure(mg); 1277 } 1278 } 1279 1280 static void overwrite_endio(struct bio *bio) 1281 { 1282 struct dm_cache_migration *mg = bio->bi_private; 1283 struct cache *cache = mg->cache; 1284 size_t pb_data_size = get_per_bio_data_size(cache); 1285 struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size); 1286 unsigned long flags; 1287 1288 dm_unhook_bio(&pb->hook_info, bio); 1289 1290 if (bio->bi_error) 1291 mg->err = true; 1292 1293 mg->requeue_holder = false; 1294 1295 spin_lock_irqsave(&cache->lock, flags); 1296 list_add_tail(&mg->list, &cache->completed_migrations); 1297 spin_unlock_irqrestore(&cache->lock, flags); 1298 1299 wake_worker(cache); 1300 } 1301 1302 static void issue_overwrite(struct dm_cache_migration *mg, struct bio *bio) 1303 { 1304 size_t pb_data_size = get_per_bio_data_size(mg->cache); 1305 struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size); 1306 1307 dm_hook_bio(&pb->hook_info, bio, overwrite_endio, mg); 1308 remap_to_cache_dirty(mg->cache, bio, mg->new_oblock, mg->cblock); 1309 1310 /* 1311 * No need to inc_ds() here, since the cell will be held for the 1312 * duration of the io. 1313 */ 1314 accounted_request(mg->cache, bio); 1315 } 1316 1317 static bool bio_writes_complete_block(struct cache *cache, struct bio *bio) 1318 { 1319 return (bio_data_dir(bio) == WRITE) && 1320 (bio->bi_iter.bi_size == (cache->sectors_per_block << SECTOR_SHIFT)); 1321 } 1322 1323 static void avoid_copy(struct dm_cache_migration *mg) 1324 { 1325 atomic_inc(&mg->cache->stats.copies_avoided); 1326 migration_success_pre_commit(mg); 1327 } 1328 1329 static void calc_discard_block_range(struct cache *cache, struct bio *bio, 1330 dm_dblock_t *b, dm_dblock_t *e) 1331 { 1332 sector_t sb = bio->bi_iter.bi_sector; 1333 sector_t se = bio_end_sector(bio); 1334 1335 *b = to_dblock(dm_sector_div_up(sb, cache->discard_block_size)); 1336 1337 if (se - sb < cache->discard_block_size) 1338 *e = *b; 1339 else 1340 *e = to_dblock(block_div(se, cache->discard_block_size)); 1341 } 1342 1343 static void issue_discard(struct dm_cache_migration *mg) 1344 { 1345 dm_dblock_t b, e; 1346 struct bio *bio = mg->new_ocell->holder; 1347 struct cache *cache = mg->cache; 1348 1349 calc_discard_block_range(cache, bio, &b, &e); 1350 while (b != e) { 1351 set_discard(cache, b); 1352 b = to_dblock(from_dblock(b) + 1); 1353 } 1354 1355 bio_endio(bio); 1356 cell_defer(cache, mg->new_ocell, false); 1357 free_migration(mg); 1358 wake_worker(cache); 1359 } 1360 1361 static void issue_copy_or_discard(struct dm_cache_migration *mg) 1362 { 1363 bool avoid; 1364 struct cache *cache = mg->cache; 1365 1366 if (mg->discard) { 1367 issue_discard(mg); 1368 return; 1369 } 1370 1371 if (mg->writeback || mg->demote) 1372 avoid = !is_dirty(cache, mg->cblock) || 1373 is_discarded_oblock(cache, mg->old_oblock); 1374 else { 1375 struct bio *bio = mg->new_ocell->holder; 1376 1377 avoid = is_discarded_oblock(cache, mg->new_oblock); 1378 1379 if (writeback_mode(&cache->features) && 1380 !avoid && bio_writes_complete_block(cache, bio)) { 1381 issue_overwrite(mg, bio); 1382 return; 1383 } 1384 } 1385 1386 avoid ? avoid_copy(mg) : issue_copy(mg); 1387 } 1388 1389 static void complete_migration(struct dm_cache_migration *mg) 1390 { 1391 if (mg->err) 1392 migration_failure(mg); 1393 else 1394 migration_success_pre_commit(mg); 1395 } 1396 1397 static void process_migrations(struct cache *cache, struct list_head *head, 1398 void (*fn)(struct dm_cache_migration *)) 1399 { 1400 unsigned long flags; 1401 struct list_head list; 1402 struct dm_cache_migration *mg, *tmp; 1403 1404 INIT_LIST_HEAD(&list); 1405 spin_lock_irqsave(&cache->lock, flags); 1406 list_splice_init(head, &list); 1407 spin_unlock_irqrestore(&cache->lock, flags); 1408 1409 list_for_each_entry_safe(mg, tmp, &list, list) 1410 fn(mg); 1411 } 1412 1413 static void __queue_quiesced_migration(struct dm_cache_migration *mg) 1414 { 1415 list_add_tail(&mg->list, &mg->cache->quiesced_migrations); 1416 } 1417 1418 static void queue_quiesced_migration(struct dm_cache_migration *mg) 1419 { 1420 unsigned long flags; 1421 struct cache *cache = mg->cache; 1422 1423 spin_lock_irqsave(&cache->lock, flags); 1424 __queue_quiesced_migration(mg); 1425 spin_unlock_irqrestore(&cache->lock, flags); 1426 1427 wake_worker(cache); 1428 } 1429 1430 static void queue_quiesced_migrations(struct cache *cache, struct list_head *work) 1431 { 1432 unsigned long flags; 1433 struct dm_cache_migration *mg, *tmp; 1434 1435 spin_lock_irqsave(&cache->lock, flags); 1436 list_for_each_entry_safe(mg, tmp, work, list) 1437 __queue_quiesced_migration(mg); 1438 spin_unlock_irqrestore(&cache->lock, flags); 1439 1440 wake_worker(cache); 1441 } 1442 1443 static void check_for_quiesced_migrations(struct cache *cache, 1444 struct per_bio_data *pb) 1445 { 1446 struct list_head work; 1447 1448 if (!pb->all_io_entry) 1449 return; 1450 1451 INIT_LIST_HEAD(&work); 1452 dm_deferred_entry_dec(pb->all_io_entry, &work); 1453 1454 if (!list_empty(&work)) 1455 queue_quiesced_migrations(cache, &work); 1456 } 1457 1458 static void quiesce_migration(struct dm_cache_migration *mg) 1459 { 1460 if (!dm_deferred_set_add_work(mg->cache->all_io_ds, &mg->list)) 1461 queue_quiesced_migration(mg); 1462 } 1463 1464 static void promote(struct cache *cache, struct prealloc *structs, 1465 dm_oblock_t oblock, dm_cblock_t cblock, 1466 struct dm_bio_prison_cell *cell) 1467 { 1468 struct dm_cache_migration *mg = prealloc_get_migration(structs); 1469 1470 mg->err = false; 1471 mg->discard = false; 1472 mg->writeback = false; 1473 mg->demote = false; 1474 mg->promote = true; 1475 mg->requeue_holder = true; 1476 mg->invalidate = false; 1477 mg->cache = cache; 1478 mg->new_oblock = oblock; 1479 mg->cblock = cblock; 1480 mg->old_ocell = NULL; 1481 mg->new_ocell = cell; 1482 mg->start_jiffies = jiffies; 1483 1484 inc_io_migrations(cache); 1485 quiesce_migration(mg); 1486 } 1487 1488 static void writeback(struct cache *cache, struct prealloc *structs, 1489 dm_oblock_t oblock, dm_cblock_t cblock, 1490 struct dm_bio_prison_cell *cell) 1491 { 1492 struct dm_cache_migration *mg = prealloc_get_migration(structs); 1493 1494 mg->err = false; 1495 mg->discard = false; 1496 mg->writeback = true; 1497 mg->demote = false; 1498 mg->promote = false; 1499 mg->requeue_holder = true; 1500 mg->invalidate = false; 1501 mg->cache = cache; 1502 mg->old_oblock = oblock; 1503 mg->cblock = cblock; 1504 mg->old_ocell = cell; 1505 mg->new_ocell = NULL; 1506 mg->start_jiffies = jiffies; 1507 1508 inc_io_migrations(cache); 1509 quiesce_migration(mg); 1510 } 1511 1512 static void demote_then_promote(struct cache *cache, struct prealloc *structs, 1513 dm_oblock_t old_oblock, dm_oblock_t new_oblock, 1514 dm_cblock_t cblock, 1515 struct dm_bio_prison_cell *old_ocell, 1516 struct dm_bio_prison_cell *new_ocell) 1517 { 1518 struct dm_cache_migration *mg = prealloc_get_migration(structs); 1519 1520 mg->err = false; 1521 mg->discard = false; 1522 mg->writeback = false; 1523 mg->demote = true; 1524 mg->promote = true; 1525 mg->requeue_holder = true; 1526 mg->invalidate = false; 1527 mg->cache = cache; 1528 mg->old_oblock = old_oblock; 1529 mg->new_oblock = new_oblock; 1530 mg->cblock = cblock; 1531 mg->old_ocell = old_ocell; 1532 mg->new_ocell = new_ocell; 1533 mg->start_jiffies = jiffies; 1534 1535 inc_io_migrations(cache); 1536 quiesce_migration(mg); 1537 } 1538 1539 /* 1540 * Invalidate a cache entry. No writeback occurs; any changes in the cache 1541 * block are thrown away. 1542 */ 1543 static void invalidate(struct cache *cache, struct prealloc *structs, 1544 dm_oblock_t oblock, dm_cblock_t cblock, 1545 struct dm_bio_prison_cell *cell) 1546 { 1547 struct dm_cache_migration *mg = prealloc_get_migration(structs); 1548 1549 mg->err = false; 1550 mg->discard = false; 1551 mg->writeback = false; 1552 mg->demote = true; 1553 mg->promote = false; 1554 mg->requeue_holder = true; 1555 mg->invalidate = true; 1556 mg->cache = cache; 1557 mg->old_oblock = oblock; 1558 mg->cblock = cblock; 1559 mg->old_ocell = cell; 1560 mg->new_ocell = NULL; 1561 mg->start_jiffies = jiffies; 1562 1563 inc_io_migrations(cache); 1564 quiesce_migration(mg); 1565 } 1566 1567 static void discard(struct cache *cache, struct prealloc *structs, 1568 struct dm_bio_prison_cell *cell) 1569 { 1570 struct dm_cache_migration *mg = prealloc_get_migration(structs); 1571 1572 mg->err = false; 1573 mg->discard = true; 1574 mg->writeback = false; 1575 mg->demote = false; 1576 mg->promote = false; 1577 mg->requeue_holder = false; 1578 mg->invalidate = false; 1579 mg->cache = cache; 1580 mg->old_ocell = NULL; 1581 mg->new_ocell = cell; 1582 mg->start_jiffies = jiffies; 1583 1584 quiesce_migration(mg); 1585 } 1586 1587 /*---------------------------------------------------------------- 1588 * bio processing 1589 *--------------------------------------------------------------*/ 1590 static void defer_bio(struct cache *cache, struct bio *bio) 1591 { 1592 unsigned long flags; 1593 1594 spin_lock_irqsave(&cache->lock, flags); 1595 bio_list_add(&cache->deferred_bios, bio); 1596 spin_unlock_irqrestore(&cache->lock, flags); 1597 1598 wake_worker(cache); 1599 } 1600 1601 static void process_flush_bio(struct cache *cache, struct bio *bio) 1602 { 1603 size_t pb_data_size = get_per_bio_data_size(cache); 1604 struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size); 1605 1606 BUG_ON(bio->bi_iter.bi_size); 1607 if (!pb->req_nr) 1608 remap_to_origin(cache, bio); 1609 else 1610 remap_to_cache(cache, bio, 0); 1611 1612 /* 1613 * REQ_FLUSH is not directed at any particular block so we don't 1614 * need to inc_ds(). REQ_FUA's are split into a write + REQ_FLUSH 1615 * by dm-core. 1616 */ 1617 issue(cache, bio); 1618 } 1619 1620 static void process_discard_bio(struct cache *cache, struct prealloc *structs, 1621 struct bio *bio) 1622 { 1623 int r; 1624 dm_dblock_t b, e; 1625 struct dm_bio_prison_cell *cell_prealloc, *new_ocell; 1626 1627 calc_discard_block_range(cache, bio, &b, &e); 1628 if (b == e) { 1629 bio_endio(bio); 1630 return; 1631 } 1632 1633 cell_prealloc = prealloc_get_cell(structs); 1634 r = bio_detain_range(cache, dblock_to_oblock(cache, b), dblock_to_oblock(cache, e), bio, cell_prealloc, 1635 (cell_free_fn) prealloc_put_cell, 1636 structs, &new_ocell); 1637 if (r > 0) 1638 return; 1639 1640 discard(cache, structs, new_ocell); 1641 } 1642 1643 static bool spare_migration_bandwidth(struct cache *cache) 1644 { 1645 sector_t current_volume = (atomic_read(&cache->nr_io_migrations) + 1) * 1646 cache->sectors_per_block; 1647 return current_volume < cache->migration_threshold; 1648 } 1649 1650 static void inc_hit_counter(struct cache *cache, struct bio *bio) 1651 { 1652 atomic_inc(bio_data_dir(bio) == READ ? 1653 &cache->stats.read_hit : &cache->stats.write_hit); 1654 } 1655 1656 static void inc_miss_counter(struct cache *cache, struct bio *bio) 1657 { 1658 atomic_inc(bio_data_dir(bio) == READ ? 1659 &cache->stats.read_miss : &cache->stats.write_miss); 1660 } 1661 1662 /*----------------------------------------------------------------*/ 1663 1664 struct inc_detail { 1665 struct cache *cache; 1666 struct bio_list bios_for_issue; 1667 struct bio_list unhandled_bios; 1668 bool any_writes; 1669 }; 1670 1671 static void inc_fn(void *context, struct dm_bio_prison_cell *cell) 1672 { 1673 struct bio *bio; 1674 struct inc_detail *detail = context; 1675 struct cache *cache = detail->cache; 1676 1677 inc_ds(cache, cell->holder, cell); 1678 if (bio_data_dir(cell->holder) == WRITE) 1679 detail->any_writes = true; 1680 1681 while ((bio = bio_list_pop(&cell->bios))) { 1682 if (discard_or_flush(bio)) { 1683 bio_list_add(&detail->unhandled_bios, bio); 1684 continue; 1685 } 1686 1687 if (bio_data_dir(bio) == WRITE) 1688 detail->any_writes = true; 1689 1690 bio_list_add(&detail->bios_for_issue, bio); 1691 inc_ds(cache, bio, cell); 1692 } 1693 } 1694 1695 // FIXME: refactor these two 1696 static void remap_cell_to_origin_clear_discard(struct cache *cache, 1697 struct dm_bio_prison_cell *cell, 1698 dm_oblock_t oblock, bool issue_holder) 1699 { 1700 struct bio *bio; 1701 unsigned long flags; 1702 struct inc_detail detail; 1703 1704 detail.cache = cache; 1705 bio_list_init(&detail.bios_for_issue); 1706 bio_list_init(&detail.unhandled_bios); 1707 detail.any_writes = false; 1708 1709 spin_lock_irqsave(&cache->lock, flags); 1710 dm_cell_visit_release(cache->prison, inc_fn, &detail, cell); 1711 bio_list_merge(&cache->deferred_bios, &detail.unhandled_bios); 1712 spin_unlock_irqrestore(&cache->lock, flags); 1713 1714 remap_to_origin(cache, cell->holder); 1715 if (issue_holder) 1716 issue(cache, cell->holder); 1717 else 1718 accounted_begin(cache, cell->holder); 1719 1720 if (detail.any_writes) 1721 clear_discard(cache, oblock_to_dblock(cache, oblock)); 1722 1723 while ((bio = bio_list_pop(&detail.bios_for_issue))) { 1724 remap_to_origin(cache, bio); 1725 issue(cache, bio); 1726 } 1727 1728 free_prison_cell(cache, cell); 1729 } 1730 1731 static void remap_cell_to_cache_dirty(struct cache *cache, struct dm_bio_prison_cell *cell, 1732 dm_oblock_t oblock, dm_cblock_t cblock, bool issue_holder) 1733 { 1734 struct bio *bio; 1735 unsigned long flags; 1736 struct inc_detail detail; 1737 1738 detail.cache = cache; 1739 bio_list_init(&detail.bios_for_issue); 1740 bio_list_init(&detail.unhandled_bios); 1741 detail.any_writes = false; 1742 1743 spin_lock_irqsave(&cache->lock, flags); 1744 dm_cell_visit_release(cache->prison, inc_fn, &detail, cell); 1745 bio_list_merge(&cache->deferred_bios, &detail.unhandled_bios); 1746 spin_unlock_irqrestore(&cache->lock, flags); 1747 1748 remap_to_cache(cache, cell->holder, cblock); 1749 if (issue_holder) 1750 issue(cache, cell->holder); 1751 else 1752 accounted_begin(cache, cell->holder); 1753 1754 if (detail.any_writes) { 1755 set_dirty(cache, oblock, cblock); 1756 clear_discard(cache, oblock_to_dblock(cache, oblock)); 1757 } 1758 1759 while ((bio = bio_list_pop(&detail.bios_for_issue))) { 1760 remap_to_cache(cache, bio, cblock); 1761 issue(cache, bio); 1762 } 1763 1764 free_prison_cell(cache, cell); 1765 } 1766 1767 /*----------------------------------------------------------------*/ 1768 1769 struct old_oblock_lock { 1770 struct policy_locker locker; 1771 struct cache *cache; 1772 struct prealloc *structs; 1773 struct dm_bio_prison_cell *cell; 1774 }; 1775 1776 static int null_locker(struct policy_locker *locker, dm_oblock_t b) 1777 { 1778 /* This should never be called */ 1779 BUG(); 1780 return 0; 1781 } 1782 1783 static int cell_locker(struct policy_locker *locker, dm_oblock_t b) 1784 { 1785 struct old_oblock_lock *l = container_of(locker, struct old_oblock_lock, locker); 1786 struct dm_bio_prison_cell *cell_prealloc = prealloc_get_cell(l->structs); 1787 1788 return bio_detain(l->cache, b, NULL, cell_prealloc, 1789 (cell_free_fn) prealloc_put_cell, 1790 l->structs, &l->cell); 1791 } 1792 1793 static void process_cell(struct cache *cache, struct prealloc *structs, 1794 struct dm_bio_prison_cell *new_ocell) 1795 { 1796 int r; 1797 bool release_cell = true; 1798 struct bio *bio = new_ocell->holder; 1799 dm_oblock_t block = get_bio_block(cache, bio); 1800 struct policy_result lookup_result; 1801 bool passthrough = passthrough_mode(&cache->features); 1802 bool fast_promotion, can_migrate; 1803 struct old_oblock_lock ool; 1804 1805 fast_promotion = is_discarded_oblock(cache, block) || bio_writes_complete_block(cache, bio); 1806 can_migrate = !passthrough && (fast_promotion || spare_migration_bandwidth(cache)); 1807 1808 ool.locker.fn = cell_locker; 1809 ool.cache = cache; 1810 ool.structs = structs; 1811 ool.cell = NULL; 1812 r = policy_map(cache->policy, block, true, can_migrate, fast_promotion, 1813 bio, &ool.locker, &lookup_result); 1814 1815 if (r == -EWOULDBLOCK) 1816 /* migration has been denied */ 1817 lookup_result.op = POLICY_MISS; 1818 1819 switch (lookup_result.op) { 1820 case POLICY_HIT: 1821 if (passthrough) { 1822 inc_miss_counter(cache, bio); 1823 1824 /* 1825 * Passthrough always maps to the origin, 1826 * invalidating any cache blocks that are written 1827 * to. 1828 */ 1829 1830 if (bio_data_dir(bio) == WRITE) { 1831 atomic_inc(&cache->stats.demotion); 1832 invalidate(cache, structs, block, lookup_result.cblock, new_ocell); 1833 release_cell = false; 1834 1835 } else { 1836 /* FIXME: factor out issue_origin() */ 1837 remap_to_origin_clear_discard(cache, bio, block); 1838 inc_and_issue(cache, bio, new_ocell); 1839 } 1840 } else { 1841 inc_hit_counter(cache, bio); 1842 1843 if (bio_data_dir(bio) == WRITE && 1844 writethrough_mode(&cache->features) && 1845 !is_dirty(cache, lookup_result.cblock)) { 1846 remap_to_origin_then_cache(cache, bio, block, lookup_result.cblock); 1847 inc_and_issue(cache, bio, new_ocell); 1848 1849 } else { 1850 remap_cell_to_cache_dirty(cache, new_ocell, block, lookup_result.cblock, true); 1851 release_cell = false; 1852 } 1853 } 1854 1855 break; 1856 1857 case POLICY_MISS: 1858 inc_miss_counter(cache, bio); 1859 remap_cell_to_origin_clear_discard(cache, new_ocell, block, true); 1860 release_cell = false; 1861 break; 1862 1863 case POLICY_NEW: 1864 atomic_inc(&cache->stats.promotion); 1865 promote(cache, structs, block, lookup_result.cblock, new_ocell); 1866 release_cell = false; 1867 break; 1868 1869 case POLICY_REPLACE: 1870 atomic_inc(&cache->stats.demotion); 1871 atomic_inc(&cache->stats.promotion); 1872 demote_then_promote(cache, structs, lookup_result.old_oblock, 1873 block, lookup_result.cblock, 1874 ool.cell, new_ocell); 1875 release_cell = false; 1876 break; 1877 1878 default: 1879 DMERR_LIMIT("%s: %s: erroring bio, unknown policy op: %u", 1880 cache_device_name(cache), __func__, 1881 (unsigned) lookup_result.op); 1882 bio_io_error(bio); 1883 } 1884 1885 if (release_cell) 1886 cell_defer(cache, new_ocell, false); 1887 } 1888 1889 static void process_bio(struct cache *cache, struct prealloc *structs, 1890 struct bio *bio) 1891 { 1892 int r; 1893 dm_oblock_t block = get_bio_block(cache, bio); 1894 struct dm_bio_prison_cell *cell_prealloc, *new_ocell; 1895 1896 /* 1897 * Check to see if that block is currently migrating. 1898 */ 1899 cell_prealloc = prealloc_get_cell(structs); 1900 r = bio_detain(cache, block, bio, cell_prealloc, 1901 (cell_free_fn) prealloc_put_cell, 1902 structs, &new_ocell); 1903 if (r > 0) 1904 return; 1905 1906 process_cell(cache, structs, new_ocell); 1907 } 1908 1909 static int need_commit_due_to_time(struct cache *cache) 1910 { 1911 return jiffies < cache->last_commit_jiffies || 1912 jiffies > cache->last_commit_jiffies + COMMIT_PERIOD; 1913 } 1914 1915 /* 1916 * A non-zero return indicates read_only or fail_io mode. 1917 */ 1918 static int commit(struct cache *cache, bool clean_shutdown) 1919 { 1920 int r; 1921 1922 if (get_cache_mode(cache) >= CM_READ_ONLY) 1923 return -EINVAL; 1924 1925 atomic_inc(&cache->stats.commit_count); 1926 r = dm_cache_commit(cache->cmd, clean_shutdown); 1927 if (r) 1928 metadata_operation_failed(cache, "dm_cache_commit", r); 1929 1930 return r; 1931 } 1932 1933 static int commit_if_needed(struct cache *cache) 1934 { 1935 int r = 0; 1936 1937 if ((cache->commit_requested || need_commit_due_to_time(cache)) && 1938 dm_cache_changed_this_transaction(cache->cmd)) { 1939 r = commit(cache, false); 1940 cache->commit_requested = false; 1941 cache->last_commit_jiffies = jiffies; 1942 } 1943 1944 return r; 1945 } 1946 1947 static void process_deferred_bios(struct cache *cache) 1948 { 1949 bool prealloc_used = false; 1950 unsigned long flags; 1951 struct bio_list bios; 1952 struct bio *bio; 1953 struct prealloc structs; 1954 1955 memset(&structs, 0, sizeof(structs)); 1956 bio_list_init(&bios); 1957 1958 spin_lock_irqsave(&cache->lock, flags); 1959 bio_list_merge(&bios, &cache->deferred_bios); 1960 bio_list_init(&cache->deferred_bios); 1961 spin_unlock_irqrestore(&cache->lock, flags); 1962 1963 while (!bio_list_empty(&bios)) { 1964 /* 1965 * If we've got no free migration structs, and processing 1966 * this bio might require one, we pause until there are some 1967 * prepared mappings to process. 1968 */ 1969 prealloc_used = true; 1970 if (prealloc_data_structs(cache, &structs)) { 1971 spin_lock_irqsave(&cache->lock, flags); 1972 bio_list_merge(&cache->deferred_bios, &bios); 1973 spin_unlock_irqrestore(&cache->lock, flags); 1974 break; 1975 } 1976 1977 bio = bio_list_pop(&bios); 1978 1979 if (bio->bi_rw & REQ_FLUSH) 1980 process_flush_bio(cache, bio); 1981 else if (bio->bi_rw & REQ_DISCARD) 1982 process_discard_bio(cache, &structs, bio); 1983 else 1984 process_bio(cache, &structs, bio); 1985 } 1986 1987 if (prealloc_used) 1988 prealloc_free_structs(cache, &structs); 1989 } 1990 1991 static void process_deferred_cells(struct cache *cache) 1992 { 1993 bool prealloc_used = false; 1994 unsigned long flags; 1995 struct dm_bio_prison_cell *cell, *tmp; 1996 struct list_head cells; 1997 struct prealloc structs; 1998 1999 memset(&structs, 0, sizeof(structs)); 2000 2001 INIT_LIST_HEAD(&cells); 2002 2003 spin_lock_irqsave(&cache->lock, flags); 2004 list_splice_init(&cache->deferred_cells, &cells); 2005 spin_unlock_irqrestore(&cache->lock, flags); 2006 2007 list_for_each_entry_safe(cell, tmp, &cells, user_list) { 2008 /* 2009 * If we've got no free migration structs, and processing 2010 * this bio might require one, we pause until there are some 2011 * prepared mappings to process. 2012 */ 2013 prealloc_used = true; 2014 if (prealloc_data_structs(cache, &structs)) { 2015 spin_lock_irqsave(&cache->lock, flags); 2016 list_splice(&cells, &cache->deferred_cells); 2017 spin_unlock_irqrestore(&cache->lock, flags); 2018 break; 2019 } 2020 2021 process_cell(cache, &structs, cell); 2022 } 2023 2024 if (prealloc_used) 2025 prealloc_free_structs(cache, &structs); 2026 } 2027 2028 static void process_deferred_flush_bios(struct cache *cache, bool submit_bios) 2029 { 2030 unsigned long flags; 2031 struct bio_list bios; 2032 struct bio *bio; 2033 2034 bio_list_init(&bios); 2035 2036 spin_lock_irqsave(&cache->lock, flags); 2037 bio_list_merge(&bios, &cache->deferred_flush_bios); 2038 bio_list_init(&cache->deferred_flush_bios); 2039 spin_unlock_irqrestore(&cache->lock, flags); 2040 2041 /* 2042 * These bios have already been through inc_ds() 2043 */ 2044 while ((bio = bio_list_pop(&bios))) 2045 submit_bios ? accounted_request(cache, bio) : bio_io_error(bio); 2046 } 2047 2048 static void process_deferred_writethrough_bios(struct cache *cache) 2049 { 2050 unsigned long flags; 2051 struct bio_list bios; 2052 struct bio *bio; 2053 2054 bio_list_init(&bios); 2055 2056 spin_lock_irqsave(&cache->lock, flags); 2057 bio_list_merge(&bios, &cache->deferred_writethrough_bios); 2058 bio_list_init(&cache->deferred_writethrough_bios); 2059 spin_unlock_irqrestore(&cache->lock, flags); 2060 2061 /* 2062 * These bios have already been through inc_ds() 2063 */ 2064 while ((bio = bio_list_pop(&bios))) 2065 accounted_request(cache, bio); 2066 } 2067 2068 static void writeback_some_dirty_blocks(struct cache *cache) 2069 { 2070 bool prealloc_used = false; 2071 dm_oblock_t oblock; 2072 dm_cblock_t cblock; 2073 struct prealloc structs; 2074 struct dm_bio_prison_cell *old_ocell; 2075 bool busy = !iot_idle_for(&cache->origin_tracker, HZ); 2076 2077 memset(&structs, 0, sizeof(structs)); 2078 2079 while (spare_migration_bandwidth(cache)) { 2080 if (policy_writeback_work(cache->policy, &oblock, &cblock, busy)) 2081 break; /* no work to do */ 2082 2083 prealloc_used = true; 2084 if (prealloc_data_structs(cache, &structs) || 2085 get_cell(cache, oblock, &structs, &old_ocell)) { 2086 policy_set_dirty(cache->policy, oblock); 2087 break; 2088 } 2089 2090 writeback(cache, &structs, oblock, cblock, old_ocell); 2091 } 2092 2093 if (prealloc_used) 2094 prealloc_free_structs(cache, &structs); 2095 } 2096 2097 /*---------------------------------------------------------------- 2098 * Invalidations. 2099 * Dropping something from the cache *without* writing back. 2100 *--------------------------------------------------------------*/ 2101 2102 static void process_invalidation_request(struct cache *cache, struct invalidation_request *req) 2103 { 2104 int r = 0; 2105 uint64_t begin = from_cblock(req->cblocks->begin); 2106 uint64_t end = from_cblock(req->cblocks->end); 2107 2108 while (begin != end) { 2109 r = policy_remove_cblock(cache->policy, to_cblock(begin)); 2110 if (!r) { 2111 r = dm_cache_remove_mapping(cache->cmd, to_cblock(begin)); 2112 if (r) { 2113 metadata_operation_failed(cache, "dm_cache_remove_mapping", r); 2114 break; 2115 } 2116 2117 } else if (r == -ENODATA) { 2118 /* harmless, already unmapped */ 2119 r = 0; 2120 2121 } else { 2122 DMERR("%s: policy_remove_cblock failed", cache_device_name(cache)); 2123 break; 2124 } 2125 2126 begin++; 2127 } 2128 2129 cache->commit_requested = true; 2130 2131 req->err = r; 2132 atomic_set(&req->complete, 1); 2133 2134 wake_up(&req->result_wait); 2135 } 2136 2137 static void process_invalidation_requests(struct cache *cache) 2138 { 2139 struct list_head list; 2140 struct invalidation_request *req, *tmp; 2141 2142 INIT_LIST_HEAD(&list); 2143 spin_lock(&cache->invalidation_lock); 2144 list_splice_init(&cache->invalidation_requests, &list); 2145 spin_unlock(&cache->invalidation_lock); 2146 2147 list_for_each_entry_safe (req, tmp, &list, list) 2148 process_invalidation_request(cache, req); 2149 } 2150 2151 /*---------------------------------------------------------------- 2152 * Main worker loop 2153 *--------------------------------------------------------------*/ 2154 static bool is_quiescing(struct cache *cache) 2155 { 2156 return atomic_read(&cache->quiescing); 2157 } 2158 2159 static void ack_quiescing(struct cache *cache) 2160 { 2161 if (is_quiescing(cache)) { 2162 atomic_inc(&cache->quiescing_ack); 2163 wake_up(&cache->quiescing_wait); 2164 } 2165 } 2166 2167 static void wait_for_quiescing_ack(struct cache *cache) 2168 { 2169 wait_event(cache->quiescing_wait, atomic_read(&cache->quiescing_ack)); 2170 } 2171 2172 static void start_quiescing(struct cache *cache) 2173 { 2174 atomic_inc(&cache->quiescing); 2175 wait_for_quiescing_ack(cache); 2176 } 2177 2178 static void stop_quiescing(struct cache *cache) 2179 { 2180 atomic_set(&cache->quiescing, 0); 2181 atomic_set(&cache->quiescing_ack, 0); 2182 } 2183 2184 static void wait_for_migrations(struct cache *cache) 2185 { 2186 wait_event(cache->migration_wait, !atomic_read(&cache->nr_allocated_migrations)); 2187 } 2188 2189 static void stop_worker(struct cache *cache) 2190 { 2191 cancel_delayed_work(&cache->waker); 2192 flush_workqueue(cache->wq); 2193 } 2194 2195 static void requeue_deferred_cells(struct cache *cache) 2196 { 2197 unsigned long flags; 2198 struct list_head cells; 2199 struct dm_bio_prison_cell *cell, *tmp; 2200 2201 INIT_LIST_HEAD(&cells); 2202 spin_lock_irqsave(&cache->lock, flags); 2203 list_splice_init(&cache->deferred_cells, &cells); 2204 spin_unlock_irqrestore(&cache->lock, flags); 2205 2206 list_for_each_entry_safe(cell, tmp, &cells, user_list) 2207 cell_requeue(cache, cell); 2208 } 2209 2210 static void requeue_deferred_bios(struct cache *cache) 2211 { 2212 struct bio *bio; 2213 struct bio_list bios; 2214 2215 bio_list_init(&bios); 2216 bio_list_merge(&bios, &cache->deferred_bios); 2217 bio_list_init(&cache->deferred_bios); 2218 2219 while ((bio = bio_list_pop(&bios))) { 2220 bio->bi_error = DM_ENDIO_REQUEUE; 2221 bio_endio(bio); 2222 } 2223 } 2224 2225 static int more_work(struct cache *cache) 2226 { 2227 if (is_quiescing(cache)) 2228 return !list_empty(&cache->quiesced_migrations) || 2229 !list_empty(&cache->completed_migrations) || 2230 !list_empty(&cache->need_commit_migrations); 2231 else 2232 return !bio_list_empty(&cache->deferred_bios) || 2233 !list_empty(&cache->deferred_cells) || 2234 !bio_list_empty(&cache->deferred_flush_bios) || 2235 !bio_list_empty(&cache->deferred_writethrough_bios) || 2236 !list_empty(&cache->quiesced_migrations) || 2237 !list_empty(&cache->completed_migrations) || 2238 !list_empty(&cache->need_commit_migrations) || 2239 cache->invalidate; 2240 } 2241 2242 static void do_worker(struct work_struct *ws) 2243 { 2244 struct cache *cache = container_of(ws, struct cache, worker); 2245 2246 do { 2247 if (!is_quiescing(cache)) { 2248 writeback_some_dirty_blocks(cache); 2249 process_deferred_writethrough_bios(cache); 2250 process_deferred_bios(cache); 2251 process_deferred_cells(cache); 2252 process_invalidation_requests(cache); 2253 } 2254 2255 process_migrations(cache, &cache->quiesced_migrations, issue_copy_or_discard); 2256 process_migrations(cache, &cache->completed_migrations, complete_migration); 2257 2258 if (commit_if_needed(cache)) { 2259 process_deferred_flush_bios(cache, false); 2260 process_migrations(cache, &cache->need_commit_migrations, migration_failure); 2261 } else { 2262 process_deferred_flush_bios(cache, true); 2263 process_migrations(cache, &cache->need_commit_migrations, 2264 migration_success_post_commit); 2265 } 2266 2267 ack_quiescing(cache); 2268 2269 } while (more_work(cache)); 2270 } 2271 2272 /* 2273 * We want to commit periodically so that not too much 2274 * unwritten metadata builds up. 2275 */ 2276 static void do_waker(struct work_struct *ws) 2277 { 2278 struct cache *cache = container_of(to_delayed_work(ws), struct cache, waker); 2279 policy_tick(cache->policy, true); 2280 wake_worker(cache); 2281 queue_delayed_work(cache->wq, &cache->waker, COMMIT_PERIOD); 2282 } 2283 2284 /*----------------------------------------------------------------*/ 2285 2286 static int is_congested(struct dm_dev *dev, int bdi_bits) 2287 { 2288 struct request_queue *q = bdev_get_queue(dev->bdev); 2289 return bdi_congested(&q->backing_dev_info, bdi_bits); 2290 } 2291 2292 static int cache_is_congested(struct dm_target_callbacks *cb, int bdi_bits) 2293 { 2294 struct cache *cache = container_of(cb, struct cache, callbacks); 2295 2296 return is_congested(cache->origin_dev, bdi_bits) || 2297 is_congested(cache->cache_dev, bdi_bits); 2298 } 2299 2300 /*---------------------------------------------------------------- 2301 * Target methods 2302 *--------------------------------------------------------------*/ 2303 2304 /* 2305 * This function gets called on the error paths of the constructor, so we 2306 * have to cope with a partially initialised struct. 2307 */ 2308 static void destroy(struct cache *cache) 2309 { 2310 unsigned i; 2311 2312 mempool_destroy(cache->migration_pool); 2313 2314 if (cache->all_io_ds) 2315 dm_deferred_set_destroy(cache->all_io_ds); 2316 2317 if (cache->prison) 2318 dm_bio_prison_destroy(cache->prison); 2319 2320 if (cache->wq) 2321 destroy_workqueue(cache->wq); 2322 2323 if (cache->dirty_bitset) 2324 free_bitset(cache->dirty_bitset); 2325 2326 if (cache->discard_bitset) 2327 free_bitset(cache->discard_bitset); 2328 2329 if (cache->copier) 2330 dm_kcopyd_client_destroy(cache->copier); 2331 2332 if (cache->cmd) 2333 dm_cache_metadata_close(cache->cmd); 2334 2335 if (cache->metadata_dev) 2336 dm_put_device(cache->ti, cache->metadata_dev); 2337 2338 if (cache->origin_dev) 2339 dm_put_device(cache->ti, cache->origin_dev); 2340 2341 if (cache->cache_dev) 2342 dm_put_device(cache->ti, cache->cache_dev); 2343 2344 if (cache->policy) 2345 dm_cache_policy_destroy(cache->policy); 2346 2347 for (i = 0; i < cache->nr_ctr_args ; i++) 2348 kfree(cache->ctr_args[i]); 2349 kfree(cache->ctr_args); 2350 2351 kfree(cache); 2352 } 2353 2354 static void cache_dtr(struct dm_target *ti) 2355 { 2356 struct cache *cache = ti->private; 2357 2358 destroy(cache); 2359 } 2360 2361 static sector_t get_dev_size(struct dm_dev *dev) 2362 { 2363 return i_size_read(dev->bdev->bd_inode) >> SECTOR_SHIFT; 2364 } 2365 2366 /*----------------------------------------------------------------*/ 2367 2368 /* 2369 * Construct a cache device mapping. 2370 * 2371 * cache <metadata dev> <cache dev> <origin dev> <block size> 2372 * <#feature args> [<feature arg>]* 2373 * <policy> <#policy args> [<policy arg>]* 2374 * 2375 * metadata dev : fast device holding the persistent metadata 2376 * cache dev : fast device holding cached data blocks 2377 * origin dev : slow device holding original data blocks 2378 * block size : cache unit size in sectors 2379 * 2380 * #feature args : number of feature arguments passed 2381 * feature args : writethrough. (The default is writeback.) 2382 * 2383 * policy : the replacement policy to use 2384 * #policy args : an even number of policy arguments corresponding 2385 * to key/value pairs passed to the policy 2386 * policy args : key/value pairs passed to the policy 2387 * E.g. 'sequential_threshold 1024' 2388 * See cache-policies.txt for details. 2389 * 2390 * Optional feature arguments are: 2391 * writethrough : write through caching that prohibits cache block 2392 * content from being different from origin block content. 2393 * Without this argument, the default behaviour is to write 2394 * back cache block contents later for performance reasons, 2395 * so they may differ from the corresponding origin blocks. 2396 */ 2397 struct cache_args { 2398 struct dm_target *ti; 2399 2400 struct dm_dev *metadata_dev; 2401 2402 struct dm_dev *cache_dev; 2403 sector_t cache_sectors; 2404 2405 struct dm_dev *origin_dev; 2406 sector_t origin_sectors; 2407 2408 uint32_t block_size; 2409 2410 const char *policy_name; 2411 int policy_argc; 2412 const char **policy_argv; 2413 2414 struct cache_features features; 2415 }; 2416 2417 static void destroy_cache_args(struct cache_args *ca) 2418 { 2419 if (ca->metadata_dev) 2420 dm_put_device(ca->ti, ca->metadata_dev); 2421 2422 if (ca->cache_dev) 2423 dm_put_device(ca->ti, ca->cache_dev); 2424 2425 if (ca->origin_dev) 2426 dm_put_device(ca->ti, ca->origin_dev); 2427 2428 kfree(ca); 2429 } 2430 2431 static bool at_least_one_arg(struct dm_arg_set *as, char **error) 2432 { 2433 if (!as->argc) { 2434 *error = "Insufficient args"; 2435 return false; 2436 } 2437 2438 return true; 2439 } 2440 2441 static int parse_metadata_dev(struct cache_args *ca, struct dm_arg_set *as, 2442 char **error) 2443 { 2444 int r; 2445 sector_t metadata_dev_size; 2446 char b[BDEVNAME_SIZE]; 2447 2448 if (!at_least_one_arg(as, error)) 2449 return -EINVAL; 2450 2451 r = dm_get_device(ca->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE, 2452 &ca->metadata_dev); 2453 if (r) { 2454 *error = "Error opening metadata device"; 2455 return r; 2456 } 2457 2458 metadata_dev_size = get_dev_size(ca->metadata_dev); 2459 if (metadata_dev_size > DM_CACHE_METADATA_MAX_SECTORS_WARNING) 2460 DMWARN("Metadata device %s is larger than %u sectors: excess space will not be used.", 2461 bdevname(ca->metadata_dev->bdev, b), THIN_METADATA_MAX_SECTORS); 2462 2463 return 0; 2464 } 2465 2466 static int parse_cache_dev(struct cache_args *ca, struct dm_arg_set *as, 2467 char **error) 2468 { 2469 int r; 2470 2471 if (!at_least_one_arg(as, error)) 2472 return -EINVAL; 2473 2474 r = dm_get_device(ca->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE, 2475 &ca->cache_dev); 2476 if (r) { 2477 *error = "Error opening cache device"; 2478 return r; 2479 } 2480 ca->cache_sectors = get_dev_size(ca->cache_dev); 2481 2482 return 0; 2483 } 2484 2485 static int parse_origin_dev(struct cache_args *ca, struct dm_arg_set *as, 2486 char **error) 2487 { 2488 int r; 2489 2490 if (!at_least_one_arg(as, error)) 2491 return -EINVAL; 2492 2493 r = dm_get_device(ca->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE, 2494 &ca->origin_dev); 2495 if (r) { 2496 *error = "Error opening origin device"; 2497 return r; 2498 } 2499 2500 ca->origin_sectors = get_dev_size(ca->origin_dev); 2501 if (ca->ti->len > ca->origin_sectors) { 2502 *error = "Device size larger than cached device"; 2503 return -EINVAL; 2504 } 2505 2506 return 0; 2507 } 2508 2509 static int parse_block_size(struct cache_args *ca, struct dm_arg_set *as, 2510 char **error) 2511 { 2512 unsigned long block_size; 2513 2514 if (!at_least_one_arg(as, error)) 2515 return -EINVAL; 2516 2517 if (kstrtoul(dm_shift_arg(as), 10, &block_size) || !block_size || 2518 block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS || 2519 block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS || 2520 block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) { 2521 *error = "Invalid data block size"; 2522 return -EINVAL; 2523 } 2524 2525 if (block_size > ca->cache_sectors) { 2526 *error = "Data block size is larger than the cache device"; 2527 return -EINVAL; 2528 } 2529 2530 ca->block_size = block_size; 2531 2532 return 0; 2533 } 2534 2535 static void init_features(struct cache_features *cf) 2536 { 2537 cf->mode = CM_WRITE; 2538 cf->io_mode = CM_IO_WRITEBACK; 2539 } 2540 2541 static int parse_features(struct cache_args *ca, struct dm_arg_set *as, 2542 char **error) 2543 { 2544 static struct dm_arg _args[] = { 2545 {0, 1, "Invalid number of cache feature arguments"}, 2546 }; 2547 2548 int r; 2549 unsigned argc; 2550 const char *arg; 2551 struct cache_features *cf = &ca->features; 2552 2553 init_features(cf); 2554 2555 r = dm_read_arg_group(_args, as, &argc, error); 2556 if (r) 2557 return -EINVAL; 2558 2559 while (argc--) { 2560 arg = dm_shift_arg(as); 2561 2562 if (!strcasecmp(arg, "writeback")) 2563 cf->io_mode = CM_IO_WRITEBACK; 2564 2565 else if (!strcasecmp(arg, "writethrough")) 2566 cf->io_mode = CM_IO_WRITETHROUGH; 2567 2568 else if (!strcasecmp(arg, "passthrough")) 2569 cf->io_mode = CM_IO_PASSTHROUGH; 2570 2571 else { 2572 *error = "Unrecognised cache feature requested"; 2573 return -EINVAL; 2574 } 2575 } 2576 2577 return 0; 2578 } 2579 2580 static int parse_policy(struct cache_args *ca, struct dm_arg_set *as, 2581 char **error) 2582 { 2583 static struct dm_arg _args[] = { 2584 {0, 1024, "Invalid number of policy arguments"}, 2585 }; 2586 2587 int r; 2588 2589 if (!at_least_one_arg(as, error)) 2590 return -EINVAL; 2591 2592 ca->policy_name = dm_shift_arg(as); 2593 2594 r = dm_read_arg_group(_args, as, &ca->policy_argc, error); 2595 if (r) 2596 return -EINVAL; 2597 2598 ca->policy_argv = (const char **)as->argv; 2599 dm_consume_args(as, ca->policy_argc); 2600 2601 return 0; 2602 } 2603 2604 static int parse_cache_args(struct cache_args *ca, int argc, char **argv, 2605 char **error) 2606 { 2607 int r; 2608 struct dm_arg_set as; 2609 2610 as.argc = argc; 2611 as.argv = argv; 2612 2613 r = parse_metadata_dev(ca, &as, error); 2614 if (r) 2615 return r; 2616 2617 r = parse_cache_dev(ca, &as, error); 2618 if (r) 2619 return r; 2620 2621 r = parse_origin_dev(ca, &as, error); 2622 if (r) 2623 return r; 2624 2625 r = parse_block_size(ca, &as, error); 2626 if (r) 2627 return r; 2628 2629 r = parse_features(ca, &as, error); 2630 if (r) 2631 return r; 2632 2633 r = parse_policy(ca, &as, error); 2634 if (r) 2635 return r; 2636 2637 return 0; 2638 } 2639 2640 /*----------------------------------------------------------------*/ 2641 2642 static struct kmem_cache *migration_cache; 2643 2644 #define NOT_CORE_OPTION 1 2645 2646 static int process_config_option(struct cache *cache, const char *key, const char *value) 2647 { 2648 unsigned long tmp; 2649 2650 if (!strcasecmp(key, "migration_threshold")) { 2651 if (kstrtoul(value, 10, &tmp)) 2652 return -EINVAL; 2653 2654 cache->migration_threshold = tmp; 2655 return 0; 2656 } 2657 2658 return NOT_CORE_OPTION; 2659 } 2660 2661 static int set_config_value(struct cache *cache, const char *key, const char *value) 2662 { 2663 int r = process_config_option(cache, key, value); 2664 2665 if (r == NOT_CORE_OPTION) 2666 r = policy_set_config_value(cache->policy, key, value); 2667 2668 if (r) 2669 DMWARN("bad config value for %s: %s", key, value); 2670 2671 return r; 2672 } 2673 2674 static int set_config_values(struct cache *cache, int argc, const char **argv) 2675 { 2676 int r = 0; 2677 2678 if (argc & 1) { 2679 DMWARN("Odd number of policy arguments given but they should be <key> <value> pairs."); 2680 return -EINVAL; 2681 } 2682 2683 while (argc) { 2684 r = set_config_value(cache, argv[0], argv[1]); 2685 if (r) 2686 break; 2687 2688 argc -= 2; 2689 argv += 2; 2690 } 2691 2692 return r; 2693 } 2694 2695 static int create_cache_policy(struct cache *cache, struct cache_args *ca, 2696 char **error) 2697 { 2698 struct dm_cache_policy *p = dm_cache_policy_create(ca->policy_name, 2699 cache->cache_size, 2700 cache->origin_sectors, 2701 cache->sectors_per_block); 2702 if (IS_ERR(p)) { 2703 *error = "Error creating cache's policy"; 2704 return PTR_ERR(p); 2705 } 2706 cache->policy = p; 2707 2708 return 0; 2709 } 2710 2711 /* 2712 * We want the discard block size to be at least the size of the cache 2713 * block size and have no more than 2^14 discard blocks across the origin. 2714 */ 2715 #define MAX_DISCARD_BLOCKS (1 << 14) 2716 2717 static bool too_many_discard_blocks(sector_t discard_block_size, 2718 sector_t origin_size) 2719 { 2720 (void) sector_div(origin_size, discard_block_size); 2721 2722 return origin_size > MAX_DISCARD_BLOCKS; 2723 } 2724 2725 static sector_t calculate_discard_block_size(sector_t cache_block_size, 2726 sector_t origin_size) 2727 { 2728 sector_t discard_block_size = cache_block_size; 2729 2730 if (origin_size) 2731 while (too_many_discard_blocks(discard_block_size, origin_size)) 2732 discard_block_size *= 2; 2733 2734 return discard_block_size; 2735 } 2736 2737 static void set_cache_size(struct cache *cache, dm_cblock_t size) 2738 { 2739 dm_block_t nr_blocks = from_cblock(size); 2740 2741 if (nr_blocks > (1 << 20) && cache->cache_size != size) 2742 DMWARN_LIMIT("You have created a cache device with a lot of individual cache blocks (%llu)\n" 2743 "All these mappings can consume a lot of kernel memory, and take some time to read/write.\n" 2744 "Please consider increasing the cache block size to reduce the overall cache block count.", 2745 (unsigned long long) nr_blocks); 2746 2747 cache->cache_size = size; 2748 } 2749 2750 #define DEFAULT_MIGRATION_THRESHOLD 2048 2751 2752 static int cache_create(struct cache_args *ca, struct cache **result) 2753 { 2754 int r = 0; 2755 char **error = &ca->ti->error; 2756 struct cache *cache; 2757 struct dm_target *ti = ca->ti; 2758 dm_block_t origin_blocks; 2759 struct dm_cache_metadata *cmd; 2760 bool may_format = ca->features.mode == CM_WRITE; 2761 2762 cache = kzalloc(sizeof(*cache), GFP_KERNEL); 2763 if (!cache) 2764 return -ENOMEM; 2765 2766 cache->ti = ca->ti; 2767 ti->private = cache; 2768 ti->num_flush_bios = 2; 2769 ti->flush_supported = true; 2770 2771 ti->num_discard_bios = 1; 2772 ti->discards_supported = true; 2773 ti->discard_zeroes_data_unsupported = true; 2774 ti->split_discard_bios = false; 2775 2776 cache->features = ca->features; 2777 ti->per_bio_data_size = get_per_bio_data_size(cache); 2778 2779 cache->callbacks.congested_fn = cache_is_congested; 2780 dm_table_add_target_callbacks(ti->table, &cache->callbacks); 2781 2782 cache->metadata_dev = ca->metadata_dev; 2783 cache->origin_dev = ca->origin_dev; 2784 cache->cache_dev = ca->cache_dev; 2785 2786 ca->metadata_dev = ca->origin_dev = ca->cache_dev = NULL; 2787 2788 /* FIXME: factor out this whole section */ 2789 origin_blocks = cache->origin_sectors = ca->origin_sectors; 2790 origin_blocks = block_div(origin_blocks, ca->block_size); 2791 cache->origin_blocks = to_oblock(origin_blocks); 2792 2793 cache->sectors_per_block = ca->block_size; 2794 if (dm_set_target_max_io_len(ti, cache->sectors_per_block)) { 2795 r = -EINVAL; 2796 goto bad; 2797 } 2798 2799 if (ca->block_size & (ca->block_size - 1)) { 2800 dm_block_t cache_size = ca->cache_sectors; 2801 2802 cache->sectors_per_block_shift = -1; 2803 cache_size = block_div(cache_size, ca->block_size); 2804 set_cache_size(cache, to_cblock(cache_size)); 2805 } else { 2806 cache->sectors_per_block_shift = __ffs(ca->block_size); 2807 set_cache_size(cache, to_cblock(ca->cache_sectors >> cache->sectors_per_block_shift)); 2808 } 2809 2810 r = create_cache_policy(cache, ca, error); 2811 if (r) 2812 goto bad; 2813 2814 cache->policy_nr_args = ca->policy_argc; 2815 cache->migration_threshold = DEFAULT_MIGRATION_THRESHOLD; 2816 2817 r = set_config_values(cache, ca->policy_argc, ca->policy_argv); 2818 if (r) { 2819 *error = "Error setting cache policy's config values"; 2820 goto bad; 2821 } 2822 2823 cmd = dm_cache_metadata_open(cache->metadata_dev->bdev, 2824 ca->block_size, may_format, 2825 dm_cache_policy_get_hint_size(cache->policy)); 2826 if (IS_ERR(cmd)) { 2827 *error = "Error creating metadata object"; 2828 r = PTR_ERR(cmd); 2829 goto bad; 2830 } 2831 cache->cmd = cmd; 2832 set_cache_mode(cache, CM_WRITE); 2833 if (get_cache_mode(cache) != CM_WRITE) { 2834 *error = "Unable to get write access to metadata, please check/repair metadata."; 2835 r = -EINVAL; 2836 goto bad; 2837 } 2838 2839 if (passthrough_mode(&cache->features)) { 2840 bool all_clean; 2841 2842 r = dm_cache_metadata_all_clean(cache->cmd, &all_clean); 2843 if (r) { 2844 *error = "dm_cache_metadata_all_clean() failed"; 2845 goto bad; 2846 } 2847 2848 if (!all_clean) { 2849 *error = "Cannot enter passthrough mode unless all blocks are clean"; 2850 r = -EINVAL; 2851 goto bad; 2852 } 2853 } 2854 2855 spin_lock_init(&cache->lock); 2856 INIT_LIST_HEAD(&cache->deferred_cells); 2857 bio_list_init(&cache->deferred_bios); 2858 bio_list_init(&cache->deferred_flush_bios); 2859 bio_list_init(&cache->deferred_writethrough_bios); 2860 INIT_LIST_HEAD(&cache->quiesced_migrations); 2861 INIT_LIST_HEAD(&cache->completed_migrations); 2862 INIT_LIST_HEAD(&cache->need_commit_migrations); 2863 atomic_set(&cache->nr_allocated_migrations, 0); 2864 atomic_set(&cache->nr_io_migrations, 0); 2865 init_waitqueue_head(&cache->migration_wait); 2866 2867 init_waitqueue_head(&cache->quiescing_wait); 2868 atomic_set(&cache->quiescing, 0); 2869 atomic_set(&cache->quiescing_ack, 0); 2870 2871 r = -ENOMEM; 2872 atomic_set(&cache->nr_dirty, 0); 2873 cache->dirty_bitset = alloc_bitset(from_cblock(cache->cache_size)); 2874 if (!cache->dirty_bitset) { 2875 *error = "could not allocate dirty bitset"; 2876 goto bad; 2877 } 2878 clear_bitset(cache->dirty_bitset, from_cblock(cache->cache_size)); 2879 2880 cache->discard_block_size = 2881 calculate_discard_block_size(cache->sectors_per_block, 2882 cache->origin_sectors); 2883 cache->discard_nr_blocks = to_dblock(dm_sector_div_up(cache->origin_sectors, 2884 cache->discard_block_size)); 2885 cache->discard_bitset = alloc_bitset(from_dblock(cache->discard_nr_blocks)); 2886 if (!cache->discard_bitset) { 2887 *error = "could not allocate discard bitset"; 2888 goto bad; 2889 } 2890 clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks)); 2891 2892 cache->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle); 2893 if (IS_ERR(cache->copier)) { 2894 *error = "could not create kcopyd client"; 2895 r = PTR_ERR(cache->copier); 2896 goto bad; 2897 } 2898 2899 cache->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM); 2900 if (!cache->wq) { 2901 *error = "could not create workqueue for metadata object"; 2902 goto bad; 2903 } 2904 INIT_WORK(&cache->worker, do_worker); 2905 INIT_DELAYED_WORK(&cache->waker, do_waker); 2906 cache->last_commit_jiffies = jiffies; 2907 2908 cache->prison = dm_bio_prison_create(); 2909 if (!cache->prison) { 2910 *error = "could not create bio prison"; 2911 goto bad; 2912 } 2913 2914 cache->all_io_ds = dm_deferred_set_create(); 2915 if (!cache->all_io_ds) { 2916 *error = "could not create all_io deferred set"; 2917 goto bad; 2918 } 2919 2920 cache->migration_pool = mempool_create_slab_pool(MIGRATION_POOL_SIZE, 2921 migration_cache); 2922 if (!cache->migration_pool) { 2923 *error = "Error creating cache's migration mempool"; 2924 goto bad; 2925 } 2926 2927 cache->need_tick_bio = true; 2928 cache->sized = false; 2929 cache->invalidate = false; 2930 cache->commit_requested = false; 2931 cache->loaded_mappings = false; 2932 cache->loaded_discards = false; 2933 2934 load_stats(cache); 2935 2936 atomic_set(&cache->stats.demotion, 0); 2937 atomic_set(&cache->stats.promotion, 0); 2938 atomic_set(&cache->stats.copies_avoided, 0); 2939 atomic_set(&cache->stats.cache_cell_clash, 0); 2940 atomic_set(&cache->stats.commit_count, 0); 2941 atomic_set(&cache->stats.discard_count, 0); 2942 2943 spin_lock_init(&cache->invalidation_lock); 2944 INIT_LIST_HEAD(&cache->invalidation_requests); 2945 2946 iot_init(&cache->origin_tracker); 2947 2948 *result = cache; 2949 return 0; 2950 2951 bad: 2952 destroy(cache); 2953 return r; 2954 } 2955 2956 static int copy_ctr_args(struct cache *cache, int argc, const char **argv) 2957 { 2958 unsigned i; 2959 const char **copy; 2960 2961 copy = kcalloc(argc, sizeof(*copy), GFP_KERNEL); 2962 if (!copy) 2963 return -ENOMEM; 2964 for (i = 0; i < argc; i++) { 2965 copy[i] = kstrdup(argv[i], GFP_KERNEL); 2966 if (!copy[i]) { 2967 while (i--) 2968 kfree(copy[i]); 2969 kfree(copy); 2970 return -ENOMEM; 2971 } 2972 } 2973 2974 cache->nr_ctr_args = argc; 2975 cache->ctr_args = copy; 2976 2977 return 0; 2978 } 2979 2980 static int cache_ctr(struct dm_target *ti, unsigned argc, char **argv) 2981 { 2982 int r = -EINVAL; 2983 struct cache_args *ca; 2984 struct cache *cache = NULL; 2985 2986 ca = kzalloc(sizeof(*ca), GFP_KERNEL); 2987 if (!ca) { 2988 ti->error = "Error allocating memory for cache"; 2989 return -ENOMEM; 2990 } 2991 ca->ti = ti; 2992 2993 r = parse_cache_args(ca, argc, argv, &ti->error); 2994 if (r) 2995 goto out; 2996 2997 r = cache_create(ca, &cache); 2998 if (r) 2999 goto out; 3000 3001 r = copy_ctr_args(cache, argc - 3, (const char **)argv + 3); 3002 if (r) { 3003 destroy(cache); 3004 goto out; 3005 } 3006 3007 ti->private = cache; 3008 3009 out: 3010 destroy_cache_args(ca); 3011 return r; 3012 } 3013 3014 /*----------------------------------------------------------------*/ 3015 3016 static int cache_map(struct dm_target *ti, struct bio *bio) 3017 { 3018 struct cache *cache = ti->private; 3019 3020 int r; 3021 struct dm_bio_prison_cell *cell = NULL; 3022 dm_oblock_t block = get_bio_block(cache, bio); 3023 size_t pb_data_size = get_per_bio_data_size(cache); 3024 bool can_migrate = false; 3025 bool fast_promotion; 3026 struct policy_result lookup_result; 3027 struct per_bio_data *pb = init_per_bio_data(bio, pb_data_size); 3028 struct old_oblock_lock ool; 3029 3030 ool.locker.fn = null_locker; 3031 3032 if (unlikely(from_oblock(block) >= from_oblock(cache->origin_blocks))) { 3033 /* 3034 * This can only occur if the io goes to a partial block at 3035 * the end of the origin device. We don't cache these. 3036 * Just remap to the origin and carry on. 3037 */ 3038 remap_to_origin(cache, bio); 3039 accounted_begin(cache, bio); 3040 return DM_MAPIO_REMAPPED; 3041 } 3042 3043 if (discard_or_flush(bio)) { 3044 defer_bio(cache, bio); 3045 return DM_MAPIO_SUBMITTED; 3046 } 3047 3048 /* 3049 * Check to see if that block is currently migrating. 3050 */ 3051 cell = alloc_prison_cell(cache); 3052 if (!cell) { 3053 defer_bio(cache, bio); 3054 return DM_MAPIO_SUBMITTED; 3055 } 3056 3057 r = bio_detain(cache, block, bio, cell, 3058 (cell_free_fn) free_prison_cell, 3059 cache, &cell); 3060 if (r) { 3061 if (r < 0) 3062 defer_bio(cache, bio); 3063 3064 return DM_MAPIO_SUBMITTED; 3065 } 3066 3067 fast_promotion = is_discarded_oblock(cache, block) || bio_writes_complete_block(cache, bio); 3068 3069 r = policy_map(cache->policy, block, false, can_migrate, fast_promotion, 3070 bio, &ool.locker, &lookup_result); 3071 if (r == -EWOULDBLOCK) { 3072 cell_defer(cache, cell, true); 3073 return DM_MAPIO_SUBMITTED; 3074 3075 } else if (r) { 3076 DMERR_LIMIT("%s: Unexpected return from cache replacement policy: %d", 3077 cache_device_name(cache), r); 3078 cell_defer(cache, cell, false); 3079 bio_io_error(bio); 3080 return DM_MAPIO_SUBMITTED; 3081 } 3082 3083 r = DM_MAPIO_REMAPPED; 3084 switch (lookup_result.op) { 3085 case POLICY_HIT: 3086 if (passthrough_mode(&cache->features)) { 3087 if (bio_data_dir(bio) == WRITE) { 3088 /* 3089 * We need to invalidate this block, so 3090 * defer for the worker thread. 3091 */ 3092 cell_defer(cache, cell, true); 3093 r = DM_MAPIO_SUBMITTED; 3094 3095 } else { 3096 inc_miss_counter(cache, bio); 3097 remap_to_origin_clear_discard(cache, bio, block); 3098 accounted_begin(cache, bio); 3099 inc_ds(cache, bio, cell); 3100 // FIXME: we want to remap hits or misses straight 3101 // away rather than passing over to the worker. 3102 cell_defer(cache, cell, false); 3103 } 3104 3105 } else { 3106 inc_hit_counter(cache, bio); 3107 if (bio_data_dir(bio) == WRITE && writethrough_mode(&cache->features) && 3108 !is_dirty(cache, lookup_result.cblock)) { 3109 remap_to_origin_then_cache(cache, bio, block, lookup_result.cblock); 3110 accounted_begin(cache, bio); 3111 inc_ds(cache, bio, cell); 3112 cell_defer(cache, cell, false); 3113 3114 } else 3115 remap_cell_to_cache_dirty(cache, cell, block, lookup_result.cblock, false); 3116 } 3117 break; 3118 3119 case POLICY_MISS: 3120 inc_miss_counter(cache, bio); 3121 if (pb->req_nr != 0) { 3122 /* 3123 * This is a duplicate writethrough io that is no 3124 * longer needed because the block has been demoted. 3125 */ 3126 bio_endio(bio); 3127 // FIXME: remap everything as a miss 3128 cell_defer(cache, cell, false); 3129 r = DM_MAPIO_SUBMITTED; 3130 3131 } else 3132 remap_cell_to_origin_clear_discard(cache, cell, block, false); 3133 break; 3134 3135 default: 3136 DMERR_LIMIT("%s: %s: erroring bio: unknown policy op: %u", 3137 cache_device_name(cache), __func__, 3138 (unsigned) lookup_result.op); 3139 cell_defer(cache, cell, false); 3140 bio_io_error(bio); 3141 r = DM_MAPIO_SUBMITTED; 3142 } 3143 3144 return r; 3145 } 3146 3147 static int cache_end_io(struct dm_target *ti, struct bio *bio, int error) 3148 { 3149 struct cache *cache = ti->private; 3150 unsigned long flags; 3151 size_t pb_data_size = get_per_bio_data_size(cache); 3152 struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size); 3153 3154 if (pb->tick) { 3155 policy_tick(cache->policy, false); 3156 3157 spin_lock_irqsave(&cache->lock, flags); 3158 cache->need_tick_bio = true; 3159 spin_unlock_irqrestore(&cache->lock, flags); 3160 } 3161 3162 check_for_quiesced_migrations(cache, pb); 3163 accounted_complete(cache, bio); 3164 3165 return 0; 3166 } 3167 3168 static int write_dirty_bitset(struct cache *cache) 3169 { 3170 unsigned i, r; 3171 3172 if (get_cache_mode(cache) >= CM_READ_ONLY) 3173 return -EINVAL; 3174 3175 for (i = 0; i < from_cblock(cache->cache_size); i++) { 3176 r = dm_cache_set_dirty(cache->cmd, to_cblock(i), 3177 is_dirty(cache, to_cblock(i))); 3178 if (r) { 3179 metadata_operation_failed(cache, "dm_cache_set_dirty", r); 3180 return r; 3181 } 3182 } 3183 3184 return 0; 3185 } 3186 3187 static int write_discard_bitset(struct cache *cache) 3188 { 3189 unsigned i, r; 3190 3191 if (get_cache_mode(cache) >= CM_READ_ONLY) 3192 return -EINVAL; 3193 3194 r = dm_cache_discard_bitset_resize(cache->cmd, cache->discard_block_size, 3195 cache->discard_nr_blocks); 3196 if (r) { 3197 DMERR("%s: could not resize on-disk discard bitset", cache_device_name(cache)); 3198 metadata_operation_failed(cache, "dm_cache_discard_bitset_resize", r); 3199 return r; 3200 } 3201 3202 for (i = 0; i < from_dblock(cache->discard_nr_blocks); i++) { 3203 r = dm_cache_set_discard(cache->cmd, to_dblock(i), 3204 is_discarded(cache, to_dblock(i))); 3205 if (r) { 3206 metadata_operation_failed(cache, "dm_cache_set_discard", r); 3207 return r; 3208 } 3209 } 3210 3211 return 0; 3212 } 3213 3214 static int write_hints(struct cache *cache) 3215 { 3216 int r; 3217 3218 if (get_cache_mode(cache) >= CM_READ_ONLY) 3219 return -EINVAL; 3220 3221 r = dm_cache_write_hints(cache->cmd, cache->policy); 3222 if (r) { 3223 metadata_operation_failed(cache, "dm_cache_write_hints", r); 3224 return r; 3225 } 3226 3227 return 0; 3228 } 3229 3230 /* 3231 * returns true on success 3232 */ 3233 static bool sync_metadata(struct cache *cache) 3234 { 3235 int r1, r2, r3, r4; 3236 3237 r1 = write_dirty_bitset(cache); 3238 if (r1) 3239 DMERR("%s: could not write dirty bitset", cache_device_name(cache)); 3240 3241 r2 = write_discard_bitset(cache); 3242 if (r2) 3243 DMERR("%s: could not write discard bitset", cache_device_name(cache)); 3244 3245 save_stats(cache); 3246 3247 r3 = write_hints(cache); 3248 if (r3) 3249 DMERR("%s: could not write hints", cache_device_name(cache)); 3250 3251 /* 3252 * If writing the above metadata failed, we still commit, but don't 3253 * set the clean shutdown flag. This will effectively force every 3254 * dirty bit to be set on reload. 3255 */ 3256 r4 = commit(cache, !r1 && !r2 && !r3); 3257 if (r4) 3258 DMERR("%s: could not write cache metadata", cache_device_name(cache)); 3259 3260 return !r1 && !r2 && !r3 && !r4; 3261 } 3262 3263 static void cache_postsuspend(struct dm_target *ti) 3264 { 3265 struct cache *cache = ti->private; 3266 3267 start_quiescing(cache); 3268 wait_for_migrations(cache); 3269 stop_worker(cache); 3270 requeue_deferred_bios(cache); 3271 requeue_deferred_cells(cache); 3272 stop_quiescing(cache); 3273 3274 if (get_cache_mode(cache) == CM_WRITE) 3275 (void) sync_metadata(cache); 3276 } 3277 3278 static int load_mapping(void *context, dm_oblock_t oblock, dm_cblock_t cblock, 3279 bool dirty, uint32_t hint, bool hint_valid) 3280 { 3281 int r; 3282 struct cache *cache = context; 3283 3284 r = policy_load_mapping(cache->policy, oblock, cblock, hint, hint_valid); 3285 if (r) 3286 return r; 3287 3288 if (dirty) 3289 set_dirty(cache, oblock, cblock); 3290 else 3291 clear_dirty(cache, oblock, cblock); 3292 3293 return 0; 3294 } 3295 3296 /* 3297 * The discard block size in the on disk metadata is not 3298 * neccessarily the same as we're currently using. So we have to 3299 * be careful to only set the discarded attribute if we know it 3300 * covers a complete block of the new size. 3301 */ 3302 struct discard_load_info { 3303 struct cache *cache; 3304 3305 /* 3306 * These blocks are sized using the on disk dblock size, rather 3307 * than the current one. 3308 */ 3309 dm_block_t block_size; 3310 dm_block_t discard_begin, discard_end; 3311 }; 3312 3313 static void discard_load_info_init(struct cache *cache, 3314 struct discard_load_info *li) 3315 { 3316 li->cache = cache; 3317 li->discard_begin = li->discard_end = 0; 3318 } 3319 3320 static void set_discard_range(struct discard_load_info *li) 3321 { 3322 sector_t b, e; 3323 3324 if (li->discard_begin == li->discard_end) 3325 return; 3326 3327 /* 3328 * Convert to sectors. 3329 */ 3330 b = li->discard_begin * li->block_size; 3331 e = li->discard_end * li->block_size; 3332 3333 /* 3334 * Then convert back to the current dblock size. 3335 */ 3336 b = dm_sector_div_up(b, li->cache->discard_block_size); 3337 sector_div(e, li->cache->discard_block_size); 3338 3339 /* 3340 * The origin may have shrunk, so we need to check we're still in 3341 * bounds. 3342 */ 3343 if (e > from_dblock(li->cache->discard_nr_blocks)) 3344 e = from_dblock(li->cache->discard_nr_blocks); 3345 3346 for (; b < e; b++) 3347 set_discard(li->cache, to_dblock(b)); 3348 } 3349 3350 static int load_discard(void *context, sector_t discard_block_size, 3351 dm_dblock_t dblock, bool discard) 3352 { 3353 struct discard_load_info *li = context; 3354 3355 li->block_size = discard_block_size; 3356 3357 if (discard) { 3358 if (from_dblock(dblock) == li->discard_end) 3359 /* 3360 * We're already in a discard range, just extend it. 3361 */ 3362 li->discard_end = li->discard_end + 1ULL; 3363 3364 else { 3365 /* 3366 * Emit the old range and start a new one. 3367 */ 3368 set_discard_range(li); 3369 li->discard_begin = from_dblock(dblock); 3370 li->discard_end = li->discard_begin + 1ULL; 3371 } 3372 } else { 3373 set_discard_range(li); 3374 li->discard_begin = li->discard_end = 0; 3375 } 3376 3377 return 0; 3378 } 3379 3380 static dm_cblock_t get_cache_dev_size(struct cache *cache) 3381 { 3382 sector_t size = get_dev_size(cache->cache_dev); 3383 (void) sector_div(size, cache->sectors_per_block); 3384 return to_cblock(size); 3385 } 3386 3387 static bool can_resize(struct cache *cache, dm_cblock_t new_size) 3388 { 3389 if (from_cblock(new_size) > from_cblock(cache->cache_size)) 3390 return true; 3391 3392 /* 3393 * We can't drop a dirty block when shrinking the cache. 3394 */ 3395 while (from_cblock(new_size) < from_cblock(cache->cache_size)) { 3396 new_size = to_cblock(from_cblock(new_size) + 1); 3397 if (is_dirty(cache, new_size)) { 3398 DMERR("%s: unable to shrink cache; cache block %llu is dirty", 3399 cache_device_name(cache), 3400 (unsigned long long) from_cblock(new_size)); 3401 return false; 3402 } 3403 } 3404 3405 return true; 3406 } 3407 3408 static int resize_cache_dev(struct cache *cache, dm_cblock_t new_size) 3409 { 3410 int r; 3411 3412 r = dm_cache_resize(cache->cmd, new_size); 3413 if (r) { 3414 DMERR("%s: could not resize cache metadata", cache_device_name(cache)); 3415 metadata_operation_failed(cache, "dm_cache_resize", r); 3416 return r; 3417 } 3418 3419 set_cache_size(cache, new_size); 3420 3421 return 0; 3422 } 3423 3424 static int cache_preresume(struct dm_target *ti) 3425 { 3426 int r = 0; 3427 struct cache *cache = ti->private; 3428 dm_cblock_t csize = get_cache_dev_size(cache); 3429 3430 /* 3431 * Check to see if the cache has resized. 3432 */ 3433 if (!cache->sized) { 3434 r = resize_cache_dev(cache, csize); 3435 if (r) 3436 return r; 3437 3438 cache->sized = true; 3439 3440 } else if (csize != cache->cache_size) { 3441 if (!can_resize(cache, csize)) 3442 return -EINVAL; 3443 3444 r = resize_cache_dev(cache, csize); 3445 if (r) 3446 return r; 3447 } 3448 3449 if (!cache->loaded_mappings) { 3450 r = dm_cache_load_mappings(cache->cmd, cache->policy, 3451 load_mapping, cache); 3452 if (r) { 3453 DMERR("%s: could not load cache mappings", cache_device_name(cache)); 3454 metadata_operation_failed(cache, "dm_cache_load_mappings", r); 3455 return r; 3456 } 3457 3458 cache->loaded_mappings = true; 3459 } 3460 3461 if (!cache->loaded_discards) { 3462 struct discard_load_info li; 3463 3464 /* 3465 * The discard bitset could have been resized, or the 3466 * discard block size changed. To be safe we start by 3467 * setting every dblock to not discarded. 3468 */ 3469 clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks)); 3470 3471 discard_load_info_init(cache, &li); 3472 r = dm_cache_load_discards(cache->cmd, load_discard, &li); 3473 if (r) { 3474 DMERR("%s: could not load origin discards", cache_device_name(cache)); 3475 metadata_operation_failed(cache, "dm_cache_load_discards", r); 3476 return r; 3477 } 3478 set_discard_range(&li); 3479 3480 cache->loaded_discards = true; 3481 } 3482 3483 return r; 3484 } 3485 3486 static void cache_resume(struct dm_target *ti) 3487 { 3488 struct cache *cache = ti->private; 3489 3490 cache->need_tick_bio = true; 3491 do_waker(&cache->waker.work); 3492 } 3493 3494 /* 3495 * Status format: 3496 * 3497 * <metadata block size> <#used metadata blocks>/<#total metadata blocks> 3498 * <cache block size> <#used cache blocks>/<#total cache blocks> 3499 * <#read hits> <#read misses> <#write hits> <#write misses> 3500 * <#demotions> <#promotions> <#dirty> 3501 * <#features> <features>* 3502 * <#core args> <core args> 3503 * <policy name> <#policy args> <policy args>* <cache metadata mode> <needs_check> 3504 */ 3505 static void cache_status(struct dm_target *ti, status_type_t type, 3506 unsigned status_flags, char *result, unsigned maxlen) 3507 { 3508 int r = 0; 3509 unsigned i; 3510 ssize_t sz = 0; 3511 dm_block_t nr_free_blocks_metadata = 0; 3512 dm_block_t nr_blocks_metadata = 0; 3513 char buf[BDEVNAME_SIZE]; 3514 struct cache *cache = ti->private; 3515 dm_cblock_t residency; 3516 3517 switch (type) { 3518 case STATUSTYPE_INFO: 3519 if (get_cache_mode(cache) == CM_FAIL) { 3520 DMEMIT("Fail"); 3521 break; 3522 } 3523 3524 /* Commit to ensure statistics aren't out-of-date */ 3525 if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti)) 3526 (void) commit(cache, false); 3527 3528 r = dm_cache_get_free_metadata_block_count(cache->cmd, &nr_free_blocks_metadata); 3529 if (r) { 3530 DMERR("%s: dm_cache_get_free_metadata_block_count returned %d", 3531 cache_device_name(cache), r); 3532 goto err; 3533 } 3534 3535 r = dm_cache_get_metadata_dev_size(cache->cmd, &nr_blocks_metadata); 3536 if (r) { 3537 DMERR("%s: dm_cache_get_metadata_dev_size returned %d", 3538 cache_device_name(cache), r); 3539 goto err; 3540 } 3541 3542 residency = policy_residency(cache->policy); 3543 3544 DMEMIT("%u %llu/%llu %u %llu/%llu %u %u %u %u %u %u %lu ", 3545 (unsigned)DM_CACHE_METADATA_BLOCK_SIZE, 3546 (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata), 3547 (unsigned long long)nr_blocks_metadata, 3548 cache->sectors_per_block, 3549 (unsigned long long) from_cblock(residency), 3550 (unsigned long long) from_cblock(cache->cache_size), 3551 (unsigned) atomic_read(&cache->stats.read_hit), 3552 (unsigned) atomic_read(&cache->stats.read_miss), 3553 (unsigned) atomic_read(&cache->stats.write_hit), 3554 (unsigned) atomic_read(&cache->stats.write_miss), 3555 (unsigned) atomic_read(&cache->stats.demotion), 3556 (unsigned) atomic_read(&cache->stats.promotion), 3557 (unsigned long) atomic_read(&cache->nr_dirty)); 3558 3559 if (writethrough_mode(&cache->features)) 3560 DMEMIT("1 writethrough "); 3561 3562 else if (passthrough_mode(&cache->features)) 3563 DMEMIT("1 passthrough "); 3564 3565 else if (writeback_mode(&cache->features)) 3566 DMEMIT("1 writeback "); 3567 3568 else { 3569 DMERR("%s: internal error: unknown io mode: %d", 3570 cache_device_name(cache), (int) cache->features.io_mode); 3571 goto err; 3572 } 3573 3574 DMEMIT("2 migration_threshold %llu ", (unsigned long long) cache->migration_threshold); 3575 3576 DMEMIT("%s ", dm_cache_policy_get_name(cache->policy)); 3577 if (sz < maxlen) { 3578 r = policy_emit_config_values(cache->policy, result, maxlen, &sz); 3579 if (r) 3580 DMERR("%s: policy_emit_config_values returned %d", 3581 cache_device_name(cache), r); 3582 } 3583 3584 if (get_cache_mode(cache) == CM_READ_ONLY) 3585 DMEMIT("ro "); 3586 else 3587 DMEMIT("rw "); 3588 3589 if (dm_cache_metadata_needs_check(cache->cmd)) 3590 DMEMIT("needs_check "); 3591 else 3592 DMEMIT("- "); 3593 3594 break; 3595 3596 case STATUSTYPE_TABLE: 3597 format_dev_t(buf, cache->metadata_dev->bdev->bd_dev); 3598 DMEMIT("%s ", buf); 3599 format_dev_t(buf, cache->cache_dev->bdev->bd_dev); 3600 DMEMIT("%s ", buf); 3601 format_dev_t(buf, cache->origin_dev->bdev->bd_dev); 3602 DMEMIT("%s", buf); 3603 3604 for (i = 0; i < cache->nr_ctr_args - 1; i++) 3605 DMEMIT(" %s", cache->ctr_args[i]); 3606 if (cache->nr_ctr_args) 3607 DMEMIT(" %s", cache->ctr_args[cache->nr_ctr_args - 1]); 3608 } 3609 3610 return; 3611 3612 err: 3613 DMEMIT("Error"); 3614 } 3615 3616 /* 3617 * A cache block range can take two forms: 3618 * 3619 * i) A single cblock, eg. '3456' 3620 * ii) A begin and end cblock with dots between, eg. 123-234 3621 */ 3622 static int parse_cblock_range(struct cache *cache, const char *str, 3623 struct cblock_range *result) 3624 { 3625 char dummy; 3626 uint64_t b, e; 3627 int r; 3628 3629 /* 3630 * Try and parse form (ii) first. 3631 */ 3632 r = sscanf(str, "%llu-%llu%c", &b, &e, &dummy); 3633 if (r < 0) 3634 return r; 3635 3636 if (r == 2) { 3637 result->begin = to_cblock(b); 3638 result->end = to_cblock(e); 3639 return 0; 3640 } 3641 3642 /* 3643 * That didn't work, try form (i). 3644 */ 3645 r = sscanf(str, "%llu%c", &b, &dummy); 3646 if (r < 0) 3647 return r; 3648 3649 if (r == 1) { 3650 result->begin = to_cblock(b); 3651 result->end = to_cblock(from_cblock(result->begin) + 1u); 3652 return 0; 3653 } 3654 3655 DMERR("%s: invalid cblock range '%s'", cache_device_name(cache), str); 3656 return -EINVAL; 3657 } 3658 3659 static int validate_cblock_range(struct cache *cache, struct cblock_range *range) 3660 { 3661 uint64_t b = from_cblock(range->begin); 3662 uint64_t e = from_cblock(range->end); 3663 uint64_t n = from_cblock(cache->cache_size); 3664 3665 if (b >= n) { 3666 DMERR("%s: begin cblock out of range: %llu >= %llu", 3667 cache_device_name(cache), b, n); 3668 return -EINVAL; 3669 } 3670 3671 if (e > n) { 3672 DMERR("%s: end cblock out of range: %llu > %llu", 3673 cache_device_name(cache), e, n); 3674 return -EINVAL; 3675 } 3676 3677 if (b >= e) { 3678 DMERR("%s: invalid cblock range: %llu >= %llu", 3679 cache_device_name(cache), b, e); 3680 return -EINVAL; 3681 } 3682 3683 return 0; 3684 } 3685 3686 static int request_invalidation(struct cache *cache, struct cblock_range *range) 3687 { 3688 struct invalidation_request req; 3689 3690 INIT_LIST_HEAD(&req.list); 3691 req.cblocks = range; 3692 atomic_set(&req.complete, 0); 3693 req.err = 0; 3694 init_waitqueue_head(&req.result_wait); 3695 3696 spin_lock(&cache->invalidation_lock); 3697 list_add(&req.list, &cache->invalidation_requests); 3698 spin_unlock(&cache->invalidation_lock); 3699 wake_worker(cache); 3700 3701 wait_event(req.result_wait, atomic_read(&req.complete)); 3702 return req.err; 3703 } 3704 3705 static int process_invalidate_cblocks_message(struct cache *cache, unsigned count, 3706 const char **cblock_ranges) 3707 { 3708 int r = 0; 3709 unsigned i; 3710 struct cblock_range range; 3711 3712 if (!passthrough_mode(&cache->features)) { 3713 DMERR("%s: cache has to be in passthrough mode for invalidation", 3714 cache_device_name(cache)); 3715 return -EPERM; 3716 } 3717 3718 for (i = 0; i < count; i++) { 3719 r = parse_cblock_range(cache, cblock_ranges[i], &range); 3720 if (r) 3721 break; 3722 3723 r = validate_cblock_range(cache, &range); 3724 if (r) 3725 break; 3726 3727 /* 3728 * Pass begin and end origin blocks to the worker and wake it. 3729 */ 3730 r = request_invalidation(cache, &range); 3731 if (r) 3732 break; 3733 } 3734 3735 return r; 3736 } 3737 3738 /* 3739 * Supports 3740 * "<key> <value>" 3741 * and 3742 * "invalidate_cblocks [(<begin>)|(<begin>-<end>)]* 3743 * 3744 * The key migration_threshold is supported by the cache target core. 3745 */ 3746 static int cache_message(struct dm_target *ti, unsigned argc, char **argv) 3747 { 3748 struct cache *cache = ti->private; 3749 3750 if (!argc) 3751 return -EINVAL; 3752 3753 if (get_cache_mode(cache) >= CM_READ_ONLY) { 3754 DMERR("%s: unable to service cache target messages in READ_ONLY or FAIL mode", 3755 cache_device_name(cache)); 3756 return -EOPNOTSUPP; 3757 } 3758 3759 if (!strcasecmp(argv[0], "invalidate_cblocks")) 3760 return process_invalidate_cblocks_message(cache, argc - 1, (const char **) argv + 1); 3761 3762 if (argc != 2) 3763 return -EINVAL; 3764 3765 return set_config_value(cache, argv[0], argv[1]); 3766 } 3767 3768 static int cache_iterate_devices(struct dm_target *ti, 3769 iterate_devices_callout_fn fn, void *data) 3770 { 3771 int r = 0; 3772 struct cache *cache = ti->private; 3773 3774 r = fn(ti, cache->cache_dev, 0, get_dev_size(cache->cache_dev), data); 3775 if (!r) 3776 r = fn(ti, cache->origin_dev, 0, ti->len, data); 3777 3778 return r; 3779 } 3780 3781 static void set_discard_limits(struct cache *cache, struct queue_limits *limits) 3782 { 3783 /* 3784 * FIXME: these limits may be incompatible with the cache device 3785 */ 3786 limits->max_discard_sectors = min_t(sector_t, cache->discard_block_size * 1024, 3787 cache->origin_sectors); 3788 limits->discard_granularity = cache->discard_block_size << SECTOR_SHIFT; 3789 } 3790 3791 static void cache_io_hints(struct dm_target *ti, struct queue_limits *limits) 3792 { 3793 struct cache *cache = ti->private; 3794 uint64_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT; 3795 3796 /* 3797 * If the system-determined stacked limits are compatible with the 3798 * cache's blocksize (io_opt is a factor) do not override them. 3799 */ 3800 if (io_opt_sectors < cache->sectors_per_block || 3801 do_div(io_opt_sectors, cache->sectors_per_block)) { 3802 blk_limits_io_min(limits, cache->sectors_per_block << SECTOR_SHIFT); 3803 blk_limits_io_opt(limits, cache->sectors_per_block << SECTOR_SHIFT); 3804 } 3805 set_discard_limits(cache, limits); 3806 } 3807 3808 /*----------------------------------------------------------------*/ 3809 3810 static struct target_type cache_target = { 3811 .name = "cache", 3812 .version = {1, 8, 0}, 3813 .module = THIS_MODULE, 3814 .ctr = cache_ctr, 3815 .dtr = cache_dtr, 3816 .map = cache_map, 3817 .end_io = cache_end_io, 3818 .postsuspend = cache_postsuspend, 3819 .preresume = cache_preresume, 3820 .resume = cache_resume, 3821 .status = cache_status, 3822 .message = cache_message, 3823 .iterate_devices = cache_iterate_devices, 3824 .io_hints = cache_io_hints, 3825 }; 3826 3827 static int __init dm_cache_init(void) 3828 { 3829 int r; 3830 3831 r = dm_register_target(&cache_target); 3832 if (r) { 3833 DMERR("cache target registration failed: %d", r); 3834 return r; 3835 } 3836 3837 migration_cache = KMEM_CACHE(dm_cache_migration, 0); 3838 if (!migration_cache) { 3839 dm_unregister_target(&cache_target); 3840 return -ENOMEM; 3841 } 3842 3843 return 0; 3844 } 3845 3846 static void __exit dm_cache_exit(void) 3847 { 3848 dm_unregister_target(&cache_target); 3849 kmem_cache_destroy(migration_cache); 3850 } 3851 3852 module_init(dm_cache_init); 3853 module_exit(dm_cache_exit); 3854 3855 MODULE_DESCRIPTION(DM_NAME " cache target"); 3856 MODULE_AUTHOR("Joe Thornber <ejt@redhat.com>"); 3857 MODULE_LICENSE("GPL"); 3858