1 /* 2 * Copyright (C) 2012 Red Hat, Inc. 3 * 4 * This file is released under the GPL. 5 */ 6 7 #include "dm-cache-metadata.h" 8 9 #include "persistent-data/dm-array.h" 10 #include "persistent-data/dm-bitset.h" 11 #include "persistent-data/dm-space-map.h" 12 #include "persistent-data/dm-space-map-disk.h" 13 #include "persistent-data/dm-transaction-manager.h" 14 15 #include <linux/device-mapper.h> 16 17 /*----------------------------------------------------------------*/ 18 19 #define DM_MSG_PREFIX "cache metadata" 20 21 #define CACHE_SUPERBLOCK_MAGIC 06142003 22 #define CACHE_SUPERBLOCK_LOCATION 0 23 24 /* 25 * defines a range of metadata versions that this module can handle. 26 */ 27 #define MIN_CACHE_VERSION 1 28 #define MAX_CACHE_VERSION 1 29 30 #define CACHE_METADATA_CACHE_SIZE 64 31 32 /* 33 * 3 for btree insert + 34 * 2 for btree lookup used within space map 35 */ 36 #define CACHE_MAX_CONCURRENT_LOCKS 5 37 #define SPACE_MAP_ROOT_SIZE 128 38 39 enum superblock_flag_bits { 40 /* for spotting crashes that would invalidate the dirty bitset */ 41 CLEAN_SHUTDOWN, 42 }; 43 44 /* 45 * Each mapping from cache block -> origin block carries a set of flags. 46 */ 47 enum mapping_bits { 48 /* 49 * A valid mapping. Because we're using an array we clear this 50 * flag for an non existant mapping. 51 */ 52 M_VALID = 1, 53 54 /* 55 * The data on the cache is different from that on the origin. 56 */ 57 M_DIRTY = 2 58 }; 59 60 struct cache_disk_superblock { 61 __le32 csum; 62 __le32 flags; 63 __le64 blocknr; 64 65 __u8 uuid[16]; 66 __le64 magic; 67 __le32 version; 68 69 __u8 policy_name[CACHE_POLICY_NAME_SIZE]; 70 __le32 policy_hint_size; 71 72 __u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE]; 73 __le64 mapping_root; 74 __le64 hint_root; 75 76 __le64 discard_root; 77 __le64 discard_block_size; 78 __le64 discard_nr_blocks; 79 80 __le32 data_block_size; 81 __le32 metadata_block_size; 82 __le32 cache_blocks; 83 84 __le32 compat_flags; 85 __le32 compat_ro_flags; 86 __le32 incompat_flags; 87 88 __le32 read_hits; 89 __le32 read_misses; 90 __le32 write_hits; 91 __le32 write_misses; 92 93 __le32 policy_version[CACHE_POLICY_VERSION_SIZE]; 94 } __packed; 95 96 struct dm_cache_metadata { 97 atomic_t ref_count; 98 struct list_head list; 99 100 struct block_device *bdev; 101 struct dm_block_manager *bm; 102 struct dm_space_map *metadata_sm; 103 struct dm_transaction_manager *tm; 104 105 struct dm_array_info info; 106 struct dm_array_info hint_info; 107 struct dm_disk_bitset discard_info; 108 109 struct rw_semaphore root_lock; 110 dm_block_t root; 111 dm_block_t hint_root; 112 dm_block_t discard_root; 113 114 sector_t discard_block_size; 115 dm_dblock_t discard_nr_blocks; 116 117 sector_t data_block_size; 118 dm_cblock_t cache_blocks; 119 bool changed:1; 120 bool clean_when_opened:1; 121 122 char policy_name[CACHE_POLICY_NAME_SIZE]; 123 unsigned policy_version[CACHE_POLICY_VERSION_SIZE]; 124 size_t policy_hint_size; 125 struct dm_cache_statistics stats; 126 127 /* 128 * Reading the space map root can fail, so we read it into this 129 * buffer before the superblock is locked and updated. 130 */ 131 __u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE]; 132 }; 133 134 /*------------------------------------------------------------------- 135 * superblock validator 136 *-----------------------------------------------------------------*/ 137 138 #define SUPERBLOCK_CSUM_XOR 9031977 139 140 static void sb_prepare_for_write(struct dm_block_validator *v, 141 struct dm_block *b, 142 size_t sb_block_size) 143 { 144 struct cache_disk_superblock *disk_super = dm_block_data(b); 145 146 disk_super->blocknr = cpu_to_le64(dm_block_location(b)); 147 disk_super->csum = cpu_to_le32(dm_bm_checksum(&disk_super->flags, 148 sb_block_size - sizeof(__le32), 149 SUPERBLOCK_CSUM_XOR)); 150 } 151 152 static int check_metadata_version(struct cache_disk_superblock *disk_super) 153 { 154 uint32_t metadata_version = le32_to_cpu(disk_super->version); 155 if (metadata_version < MIN_CACHE_VERSION || metadata_version > MAX_CACHE_VERSION) { 156 DMERR("Cache metadata version %u found, but only versions between %u and %u supported.", 157 metadata_version, MIN_CACHE_VERSION, MAX_CACHE_VERSION); 158 return -EINVAL; 159 } 160 161 return 0; 162 } 163 164 static int sb_check(struct dm_block_validator *v, 165 struct dm_block *b, 166 size_t sb_block_size) 167 { 168 struct cache_disk_superblock *disk_super = dm_block_data(b); 169 __le32 csum_le; 170 171 if (dm_block_location(b) != le64_to_cpu(disk_super->blocknr)) { 172 DMERR("sb_check failed: blocknr %llu: wanted %llu", 173 le64_to_cpu(disk_super->blocknr), 174 (unsigned long long)dm_block_location(b)); 175 return -ENOTBLK; 176 } 177 178 if (le64_to_cpu(disk_super->magic) != CACHE_SUPERBLOCK_MAGIC) { 179 DMERR("sb_check failed: magic %llu: wanted %llu", 180 le64_to_cpu(disk_super->magic), 181 (unsigned long long)CACHE_SUPERBLOCK_MAGIC); 182 return -EILSEQ; 183 } 184 185 csum_le = cpu_to_le32(dm_bm_checksum(&disk_super->flags, 186 sb_block_size - sizeof(__le32), 187 SUPERBLOCK_CSUM_XOR)); 188 if (csum_le != disk_super->csum) { 189 DMERR("sb_check failed: csum %u: wanted %u", 190 le32_to_cpu(csum_le), le32_to_cpu(disk_super->csum)); 191 return -EILSEQ; 192 } 193 194 return check_metadata_version(disk_super); 195 } 196 197 static struct dm_block_validator sb_validator = { 198 .name = "superblock", 199 .prepare_for_write = sb_prepare_for_write, 200 .check = sb_check 201 }; 202 203 /*----------------------------------------------------------------*/ 204 205 static int superblock_read_lock(struct dm_cache_metadata *cmd, 206 struct dm_block **sblock) 207 { 208 return dm_bm_read_lock(cmd->bm, CACHE_SUPERBLOCK_LOCATION, 209 &sb_validator, sblock); 210 } 211 212 static int superblock_lock_zero(struct dm_cache_metadata *cmd, 213 struct dm_block **sblock) 214 { 215 return dm_bm_write_lock_zero(cmd->bm, CACHE_SUPERBLOCK_LOCATION, 216 &sb_validator, sblock); 217 } 218 219 static int superblock_lock(struct dm_cache_metadata *cmd, 220 struct dm_block **sblock) 221 { 222 return dm_bm_write_lock(cmd->bm, CACHE_SUPERBLOCK_LOCATION, 223 &sb_validator, sblock); 224 } 225 226 /*----------------------------------------------------------------*/ 227 228 static int __superblock_all_zeroes(struct dm_block_manager *bm, bool *result) 229 { 230 int r; 231 unsigned i; 232 struct dm_block *b; 233 __le64 *data_le, zero = cpu_to_le64(0); 234 unsigned sb_block_size = dm_bm_block_size(bm) / sizeof(__le64); 235 236 /* 237 * We can't use a validator here - it may be all zeroes. 238 */ 239 r = dm_bm_read_lock(bm, CACHE_SUPERBLOCK_LOCATION, NULL, &b); 240 if (r) 241 return r; 242 243 data_le = dm_block_data(b); 244 *result = true; 245 for (i = 0; i < sb_block_size; i++) { 246 if (data_le[i] != zero) { 247 *result = false; 248 break; 249 } 250 } 251 252 return dm_bm_unlock(b); 253 } 254 255 static void __setup_mapping_info(struct dm_cache_metadata *cmd) 256 { 257 struct dm_btree_value_type vt; 258 259 vt.context = NULL; 260 vt.size = sizeof(__le64); 261 vt.inc = NULL; 262 vt.dec = NULL; 263 vt.equal = NULL; 264 dm_array_info_init(&cmd->info, cmd->tm, &vt); 265 266 if (cmd->policy_hint_size) { 267 vt.size = sizeof(__le32); 268 dm_array_info_init(&cmd->hint_info, cmd->tm, &vt); 269 } 270 } 271 272 static int __save_sm_root(struct dm_cache_metadata *cmd) 273 { 274 int r; 275 size_t metadata_len; 276 277 r = dm_sm_root_size(cmd->metadata_sm, &metadata_len); 278 if (r < 0) 279 return r; 280 281 return dm_sm_copy_root(cmd->metadata_sm, &cmd->metadata_space_map_root, 282 metadata_len); 283 } 284 285 static void __copy_sm_root(struct dm_cache_metadata *cmd, 286 struct cache_disk_superblock *disk_super) 287 { 288 memcpy(&disk_super->metadata_space_map_root, 289 &cmd->metadata_space_map_root, 290 sizeof(cmd->metadata_space_map_root)); 291 } 292 293 static int __write_initial_superblock(struct dm_cache_metadata *cmd) 294 { 295 int r; 296 struct dm_block *sblock; 297 struct cache_disk_superblock *disk_super; 298 sector_t bdev_size = i_size_read(cmd->bdev->bd_inode) >> SECTOR_SHIFT; 299 300 /* FIXME: see if we can lose the max sectors limit */ 301 if (bdev_size > DM_CACHE_METADATA_MAX_SECTORS) 302 bdev_size = DM_CACHE_METADATA_MAX_SECTORS; 303 304 r = dm_tm_pre_commit(cmd->tm); 305 if (r < 0) 306 return r; 307 308 /* 309 * dm_sm_copy_root() can fail. So we need to do it before we start 310 * updating the superblock. 311 */ 312 r = __save_sm_root(cmd); 313 if (r) 314 return r; 315 316 r = superblock_lock_zero(cmd, &sblock); 317 if (r) 318 return r; 319 320 disk_super = dm_block_data(sblock); 321 disk_super->flags = 0; 322 memset(disk_super->uuid, 0, sizeof(disk_super->uuid)); 323 disk_super->magic = cpu_to_le64(CACHE_SUPERBLOCK_MAGIC); 324 disk_super->version = cpu_to_le32(MAX_CACHE_VERSION); 325 memset(disk_super->policy_name, 0, sizeof(disk_super->policy_name)); 326 memset(disk_super->policy_version, 0, sizeof(disk_super->policy_version)); 327 disk_super->policy_hint_size = 0; 328 329 __copy_sm_root(cmd, disk_super); 330 331 disk_super->mapping_root = cpu_to_le64(cmd->root); 332 disk_super->hint_root = cpu_to_le64(cmd->hint_root); 333 disk_super->discard_root = cpu_to_le64(cmd->discard_root); 334 disk_super->discard_block_size = cpu_to_le64(cmd->discard_block_size); 335 disk_super->discard_nr_blocks = cpu_to_le64(from_dblock(cmd->discard_nr_blocks)); 336 disk_super->metadata_block_size = cpu_to_le32(DM_CACHE_METADATA_BLOCK_SIZE); 337 disk_super->data_block_size = cpu_to_le32(cmd->data_block_size); 338 disk_super->cache_blocks = cpu_to_le32(0); 339 340 disk_super->read_hits = cpu_to_le32(0); 341 disk_super->read_misses = cpu_to_le32(0); 342 disk_super->write_hits = cpu_to_le32(0); 343 disk_super->write_misses = cpu_to_le32(0); 344 345 return dm_tm_commit(cmd->tm, sblock); 346 } 347 348 static int __format_metadata(struct dm_cache_metadata *cmd) 349 { 350 int r; 351 352 r = dm_tm_create_with_sm(cmd->bm, CACHE_SUPERBLOCK_LOCATION, 353 &cmd->tm, &cmd->metadata_sm); 354 if (r < 0) { 355 DMERR("tm_create_with_sm failed"); 356 return r; 357 } 358 359 __setup_mapping_info(cmd); 360 361 r = dm_array_empty(&cmd->info, &cmd->root); 362 if (r < 0) 363 goto bad; 364 365 dm_disk_bitset_init(cmd->tm, &cmd->discard_info); 366 367 r = dm_bitset_empty(&cmd->discard_info, &cmd->discard_root); 368 if (r < 0) 369 goto bad; 370 371 cmd->discard_block_size = 0; 372 cmd->discard_nr_blocks = 0; 373 374 r = __write_initial_superblock(cmd); 375 if (r) 376 goto bad; 377 378 cmd->clean_when_opened = true; 379 return 0; 380 381 bad: 382 dm_tm_destroy(cmd->tm); 383 dm_sm_destroy(cmd->metadata_sm); 384 385 return r; 386 } 387 388 static int __check_incompat_features(struct cache_disk_superblock *disk_super, 389 struct dm_cache_metadata *cmd) 390 { 391 uint32_t features; 392 393 features = le32_to_cpu(disk_super->incompat_flags) & ~DM_CACHE_FEATURE_INCOMPAT_SUPP; 394 if (features) { 395 DMERR("could not access metadata due to unsupported optional features (%lx).", 396 (unsigned long)features); 397 return -EINVAL; 398 } 399 400 /* 401 * Check for read-only metadata to skip the following RDWR checks. 402 */ 403 if (get_disk_ro(cmd->bdev->bd_disk)) 404 return 0; 405 406 features = le32_to_cpu(disk_super->compat_ro_flags) & ~DM_CACHE_FEATURE_COMPAT_RO_SUPP; 407 if (features) { 408 DMERR("could not access metadata RDWR due to unsupported optional features (%lx).", 409 (unsigned long)features); 410 return -EINVAL; 411 } 412 413 return 0; 414 } 415 416 static int __open_metadata(struct dm_cache_metadata *cmd) 417 { 418 int r; 419 struct dm_block *sblock; 420 struct cache_disk_superblock *disk_super; 421 unsigned long sb_flags; 422 423 r = superblock_read_lock(cmd, &sblock); 424 if (r < 0) { 425 DMERR("couldn't read lock superblock"); 426 return r; 427 } 428 429 disk_super = dm_block_data(sblock); 430 431 /* Verify the data block size hasn't changed */ 432 if (le32_to_cpu(disk_super->data_block_size) != cmd->data_block_size) { 433 DMERR("changing the data block size (from %u to %llu) is not supported", 434 le32_to_cpu(disk_super->data_block_size), 435 (unsigned long long)cmd->data_block_size); 436 r = -EINVAL; 437 goto bad; 438 } 439 440 r = __check_incompat_features(disk_super, cmd); 441 if (r < 0) 442 goto bad; 443 444 r = dm_tm_open_with_sm(cmd->bm, CACHE_SUPERBLOCK_LOCATION, 445 disk_super->metadata_space_map_root, 446 sizeof(disk_super->metadata_space_map_root), 447 &cmd->tm, &cmd->metadata_sm); 448 if (r < 0) { 449 DMERR("tm_open_with_sm failed"); 450 goto bad; 451 } 452 453 __setup_mapping_info(cmd); 454 dm_disk_bitset_init(cmd->tm, &cmd->discard_info); 455 sb_flags = le32_to_cpu(disk_super->flags); 456 cmd->clean_when_opened = test_bit(CLEAN_SHUTDOWN, &sb_flags); 457 return dm_bm_unlock(sblock); 458 459 bad: 460 dm_bm_unlock(sblock); 461 return r; 462 } 463 464 static int __open_or_format_metadata(struct dm_cache_metadata *cmd, 465 bool format_device) 466 { 467 int r; 468 bool unformatted = false; 469 470 r = __superblock_all_zeroes(cmd->bm, &unformatted); 471 if (r) 472 return r; 473 474 if (unformatted) 475 return format_device ? __format_metadata(cmd) : -EPERM; 476 477 return __open_metadata(cmd); 478 } 479 480 static int __create_persistent_data_objects(struct dm_cache_metadata *cmd, 481 bool may_format_device) 482 { 483 int r; 484 cmd->bm = dm_block_manager_create(cmd->bdev, DM_CACHE_METADATA_BLOCK_SIZE << SECTOR_SHIFT, 485 CACHE_METADATA_CACHE_SIZE, 486 CACHE_MAX_CONCURRENT_LOCKS); 487 if (IS_ERR(cmd->bm)) { 488 DMERR("could not create block manager"); 489 return PTR_ERR(cmd->bm); 490 } 491 492 r = __open_or_format_metadata(cmd, may_format_device); 493 if (r) 494 dm_block_manager_destroy(cmd->bm); 495 496 return r; 497 } 498 499 static void __destroy_persistent_data_objects(struct dm_cache_metadata *cmd) 500 { 501 dm_sm_destroy(cmd->metadata_sm); 502 dm_tm_destroy(cmd->tm); 503 dm_block_manager_destroy(cmd->bm); 504 } 505 506 typedef unsigned long (*flags_mutator)(unsigned long); 507 508 static void update_flags(struct cache_disk_superblock *disk_super, 509 flags_mutator mutator) 510 { 511 uint32_t sb_flags = mutator(le32_to_cpu(disk_super->flags)); 512 disk_super->flags = cpu_to_le32(sb_flags); 513 } 514 515 static unsigned long set_clean_shutdown(unsigned long flags) 516 { 517 set_bit(CLEAN_SHUTDOWN, &flags); 518 return flags; 519 } 520 521 static unsigned long clear_clean_shutdown(unsigned long flags) 522 { 523 clear_bit(CLEAN_SHUTDOWN, &flags); 524 return flags; 525 } 526 527 static void read_superblock_fields(struct dm_cache_metadata *cmd, 528 struct cache_disk_superblock *disk_super) 529 { 530 cmd->root = le64_to_cpu(disk_super->mapping_root); 531 cmd->hint_root = le64_to_cpu(disk_super->hint_root); 532 cmd->discard_root = le64_to_cpu(disk_super->discard_root); 533 cmd->discard_block_size = le64_to_cpu(disk_super->discard_block_size); 534 cmd->discard_nr_blocks = to_dblock(le64_to_cpu(disk_super->discard_nr_blocks)); 535 cmd->data_block_size = le32_to_cpu(disk_super->data_block_size); 536 cmd->cache_blocks = to_cblock(le32_to_cpu(disk_super->cache_blocks)); 537 strncpy(cmd->policy_name, disk_super->policy_name, sizeof(cmd->policy_name)); 538 cmd->policy_version[0] = le32_to_cpu(disk_super->policy_version[0]); 539 cmd->policy_version[1] = le32_to_cpu(disk_super->policy_version[1]); 540 cmd->policy_version[2] = le32_to_cpu(disk_super->policy_version[2]); 541 cmd->policy_hint_size = le32_to_cpu(disk_super->policy_hint_size); 542 543 cmd->stats.read_hits = le32_to_cpu(disk_super->read_hits); 544 cmd->stats.read_misses = le32_to_cpu(disk_super->read_misses); 545 cmd->stats.write_hits = le32_to_cpu(disk_super->write_hits); 546 cmd->stats.write_misses = le32_to_cpu(disk_super->write_misses); 547 548 cmd->changed = false; 549 } 550 551 /* 552 * The mutator updates the superblock flags. 553 */ 554 static int __begin_transaction_flags(struct dm_cache_metadata *cmd, 555 flags_mutator mutator) 556 { 557 int r; 558 struct cache_disk_superblock *disk_super; 559 struct dm_block *sblock; 560 561 r = superblock_lock(cmd, &sblock); 562 if (r) 563 return r; 564 565 disk_super = dm_block_data(sblock); 566 update_flags(disk_super, mutator); 567 read_superblock_fields(cmd, disk_super); 568 dm_bm_unlock(sblock); 569 570 return dm_bm_flush(cmd->bm); 571 } 572 573 static int __begin_transaction(struct dm_cache_metadata *cmd) 574 { 575 int r; 576 struct cache_disk_superblock *disk_super; 577 struct dm_block *sblock; 578 579 /* 580 * We re-read the superblock every time. Shouldn't need to do this 581 * really. 582 */ 583 r = superblock_read_lock(cmd, &sblock); 584 if (r) 585 return r; 586 587 disk_super = dm_block_data(sblock); 588 read_superblock_fields(cmd, disk_super); 589 dm_bm_unlock(sblock); 590 591 return 0; 592 } 593 594 static int __commit_transaction(struct dm_cache_metadata *cmd, 595 flags_mutator mutator) 596 { 597 int r; 598 struct cache_disk_superblock *disk_super; 599 struct dm_block *sblock; 600 601 /* 602 * We need to know if the cache_disk_superblock exceeds a 512-byte sector. 603 */ 604 BUILD_BUG_ON(sizeof(struct cache_disk_superblock) > 512); 605 606 r = dm_bitset_flush(&cmd->discard_info, cmd->discard_root, 607 &cmd->discard_root); 608 if (r) 609 return r; 610 611 r = dm_tm_pre_commit(cmd->tm); 612 if (r < 0) 613 return r; 614 615 r = __save_sm_root(cmd); 616 if (r) 617 return r; 618 619 r = superblock_lock(cmd, &sblock); 620 if (r) 621 return r; 622 623 disk_super = dm_block_data(sblock); 624 625 if (mutator) 626 update_flags(disk_super, mutator); 627 628 disk_super->mapping_root = cpu_to_le64(cmd->root); 629 disk_super->hint_root = cpu_to_le64(cmd->hint_root); 630 disk_super->discard_root = cpu_to_le64(cmd->discard_root); 631 disk_super->discard_block_size = cpu_to_le64(cmd->discard_block_size); 632 disk_super->discard_nr_blocks = cpu_to_le64(from_dblock(cmd->discard_nr_blocks)); 633 disk_super->cache_blocks = cpu_to_le32(from_cblock(cmd->cache_blocks)); 634 strncpy(disk_super->policy_name, cmd->policy_name, sizeof(disk_super->policy_name)); 635 disk_super->policy_version[0] = cpu_to_le32(cmd->policy_version[0]); 636 disk_super->policy_version[1] = cpu_to_le32(cmd->policy_version[1]); 637 disk_super->policy_version[2] = cpu_to_le32(cmd->policy_version[2]); 638 639 disk_super->read_hits = cpu_to_le32(cmd->stats.read_hits); 640 disk_super->read_misses = cpu_to_le32(cmd->stats.read_misses); 641 disk_super->write_hits = cpu_to_le32(cmd->stats.write_hits); 642 disk_super->write_misses = cpu_to_le32(cmd->stats.write_misses); 643 __copy_sm_root(cmd, disk_super); 644 645 return dm_tm_commit(cmd->tm, sblock); 646 } 647 648 /*----------------------------------------------------------------*/ 649 650 /* 651 * The mappings are held in a dm-array that has 64-bit values stored in 652 * little-endian format. The index is the cblock, the high 48bits of the 653 * value are the oblock and the low 16 bit the flags. 654 */ 655 #define FLAGS_MASK ((1 << 16) - 1) 656 657 static __le64 pack_value(dm_oblock_t block, unsigned flags) 658 { 659 uint64_t value = from_oblock(block); 660 value <<= 16; 661 value = value | (flags & FLAGS_MASK); 662 return cpu_to_le64(value); 663 } 664 665 static void unpack_value(__le64 value_le, dm_oblock_t *block, unsigned *flags) 666 { 667 uint64_t value = le64_to_cpu(value_le); 668 uint64_t b = value >> 16; 669 *block = to_oblock(b); 670 *flags = value & FLAGS_MASK; 671 } 672 673 /*----------------------------------------------------------------*/ 674 675 static struct dm_cache_metadata *metadata_open(struct block_device *bdev, 676 sector_t data_block_size, 677 bool may_format_device, 678 size_t policy_hint_size) 679 { 680 int r; 681 struct dm_cache_metadata *cmd; 682 683 cmd = kzalloc(sizeof(*cmd), GFP_KERNEL); 684 if (!cmd) { 685 DMERR("could not allocate metadata struct"); 686 return NULL; 687 } 688 689 atomic_set(&cmd->ref_count, 1); 690 init_rwsem(&cmd->root_lock); 691 cmd->bdev = bdev; 692 cmd->data_block_size = data_block_size; 693 cmd->cache_blocks = 0; 694 cmd->policy_hint_size = policy_hint_size; 695 cmd->changed = true; 696 697 r = __create_persistent_data_objects(cmd, may_format_device); 698 if (r) { 699 kfree(cmd); 700 return ERR_PTR(r); 701 } 702 703 r = __begin_transaction_flags(cmd, clear_clean_shutdown); 704 if (r < 0) { 705 dm_cache_metadata_close(cmd); 706 return ERR_PTR(r); 707 } 708 709 return cmd; 710 } 711 712 /* 713 * We keep a little list of ref counted metadata objects to prevent two 714 * different target instances creating separate bufio instances. This is 715 * an issue if a table is reloaded before the suspend. 716 */ 717 static DEFINE_MUTEX(table_lock); 718 static LIST_HEAD(table); 719 720 static struct dm_cache_metadata *lookup(struct block_device *bdev) 721 { 722 struct dm_cache_metadata *cmd; 723 724 list_for_each_entry(cmd, &table, list) 725 if (cmd->bdev == bdev) { 726 atomic_inc(&cmd->ref_count); 727 return cmd; 728 } 729 730 return NULL; 731 } 732 733 static struct dm_cache_metadata *lookup_or_open(struct block_device *bdev, 734 sector_t data_block_size, 735 bool may_format_device, 736 size_t policy_hint_size) 737 { 738 struct dm_cache_metadata *cmd, *cmd2; 739 740 mutex_lock(&table_lock); 741 cmd = lookup(bdev); 742 mutex_unlock(&table_lock); 743 744 if (cmd) 745 return cmd; 746 747 cmd = metadata_open(bdev, data_block_size, may_format_device, policy_hint_size); 748 if (cmd) { 749 mutex_lock(&table_lock); 750 cmd2 = lookup(bdev); 751 if (cmd2) { 752 mutex_unlock(&table_lock); 753 __destroy_persistent_data_objects(cmd); 754 kfree(cmd); 755 return cmd2; 756 } 757 list_add(&cmd->list, &table); 758 mutex_unlock(&table_lock); 759 } 760 761 return cmd; 762 } 763 764 static bool same_params(struct dm_cache_metadata *cmd, sector_t data_block_size) 765 { 766 if (cmd->data_block_size != data_block_size) { 767 DMERR("data_block_size (%llu) different from that in metadata (%llu)\n", 768 (unsigned long long) data_block_size, 769 (unsigned long long) cmd->data_block_size); 770 return false; 771 } 772 773 return true; 774 } 775 776 struct dm_cache_metadata *dm_cache_metadata_open(struct block_device *bdev, 777 sector_t data_block_size, 778 bool may_format_device, 779 size_t policy_hint_size) 780 { 781 struct dm_cache_metadata *cmd = lookup_or_open(bdev, data_block_size, 782 may_format_device, policy_hint_size); 783 if (cmd && !same_params(cmd, data_block_size)) { 784 dm_cache_metadata_close(cmd); 785 return NULL; 786 } 787 788 return cmd; 789 } 790 791 void dm_cache_metadata_close(struct dm_cache_metadata *cmd) 792 { 793 if (atomic_dec_and_test(&cmd->ref_count)) { 794 mutex_lock(&table_lock); 795 list_del(&cmd->list); 796 mutex_unlock(&table_lock); 797 798 __destroy_persistent_data_objects(cmd); 799 kfree(cmd); 800 } 801 } 802 803 /* 804 * Checks that the given cache block is either unmapped or clean. 805 */ 806 static int block_unmapped_or_clean(struct dm_cache_metadata *cmd, dm_cblock_t b, 807 bool *result) 808 { 809 int r; 810 __le64 value; 811 dm_oblock_t ob; 812 unsigned flags; 813 814 r = dm_array_get_value(&cmd->info, cmd->root, from_cblock(b), &value); 815 if (r) { 816 DMERR("block_unmapped_or_clean failed"); 817 return r; 818 } 819 820 unpack_value(value, &ob, &flags); 821 *result = !((flags & M_VALID) && (flags & M_DIRTY)); 822 823 return 0; 824 } 825 826 static int blocks_are_unmapped_or_clean(struct dm_cache_metadata *cmd, 827 dm_cblock_t begin, dm_cblock_t end, 828 bool *result) 829 { 830 int r; 831 *result = true; 832 833 while (begin != end) { 834 r = block_unmapped_or_clean(cmd, begin, result); 835 if (r) 836 return r; 837 838 if (!*result) { 839 DMERR("cache block %llu is dirty", 840 (unsigned long long) from_cblock(begin)); 841 return 0; 842 } 843 844 begin = to_cblock(from_cblock(begin) + 1); 845 } 846 847 return 0; 848 } 849 850 int dm_cache_resize(struct dm_cache_metadata *cmd, dm_cblock_t new_cache_size) 851 { 852 int r; 853 bool clean; 854 __le64 null_mapping = pack_value(0, 0); 855 856 down_write(&cmd->root_lock); 857 __dm_bless_for_disk(&null_mapping); 858 859 if (from_cblock(new_cache_size) < from_cblock(cmd->cache_blocks)) { 860 r = blocks_are_unmapped_or_clean(cmd, new_cache_size, cmd->cache_blocks, &clean); 861 if (r) { 862 __dm_unbless_for_disk(&null_mapping); 863 goto out; 864 } 865 866 if (!clean) { 867 DMERR("unable to shrink cache due to dirty blocks"); 868 r = -EINVAL; 869 __dm_unbless_for_disk(&null_mapping); 870 goto out; 871 } 872 } 873 874 r = dm_array_resize(&cmd->info, cmd->root, from_cblock(cmd->cache_blocks), 875 from_cblock(new_cache_size), 876 &null_mapping, &cmd->root); 877 if (!r) 878 cmd->cache_blocks = new_cache_size; 879 cmd->changed = true; 880 881 out: 882 up_write(&cmd->root_lock); 883 884 return r; 885 } 886 887 int dm_cache_discard_bitset_resize(struct dm_cache_metadata *cmd, 888 sector_t discard_block_size, 889 dm_dblock_t new_nr_entries) 890 { 891 int r; 892 893 down_write(&cmd->root_lock); 894 r = dm_bitset_resize(&cmd->discard_info, 895 cmd->discard_root, 896 from_dblock(cmd->discard_nr_blocks), 897 from_dblock(new_nr_entries), 898 false, &cmd->discard_root); 899 if (!r) { 900 cmd->discard_block_size = discard_block_size; 901 cmd->discard_nr_blocks = new_nr_entries; 902 } 903 904 cmd->changed = true; 905 up_write(&cmd->root_lock); 906 907 return r; 908 } 909 910 static int __set_discard(struct dm_cache_metadata *cmd, dm_dblock_t b) 911 { 912 return dm_bitset_set_bit(&cmd->discard_info, cmd->discard_root, 913 from_dblock(b), &cmd->discard_root); 914 } 915 916 static int __clear_discard(struct dm_cache_metadata *cmd, dm_dblock_t b) 917 { 918 return dm_bitset_clear_bit(&cmd->discard_info, cmd->discard_root, 919 from_dblock(b), &cmd->discard_root); 920 } 921 922 static int __is_discarded(struct dm_cache_metadata *cmd, dm_dblock_t b, 923 bool *is_discarded) 924 { 925 return dm_bitset_test_bit(&cmd->discard_info, cmd->discard_root, 926 from_dblock(b), &cmd->discard_root, 927 is_discarded); 928 } 929 930 static int __discard(struct dm_cache_metadata *cmd, 931 dm_dblock_t dblock, bool discard) 932 { 933 int r; 934 935 r = (discard ? __set_discard : __clear_discard)(cmd, dblock); 936 if (r) 937 return r; 938 939 cmd->changed = true; 940 return 0; 941 } 942 943 int dm_cache_set_discard(struct dm_cache_metadata *cmd, 944 dm_dblock_t dblock, bool discard) 945 { 946 int r; 947 948 down_write(&cmd->root_lock); 949 r = __discard(cmd, dblock, discard); 950 up_write(&cmd->root_lock); 951 952 return r; 953 } 954 955 static int __load_discards(struct dm_cache_metadata *cmd, 956 load_discard_fn fn, void *context) 957 { 958 int r = 0; 959 dm_block_t b; 960 bool discard; 961 962 for (b = 0; b < from_dblock(cmd->discard_nr_blocks); b++) { 963 dm_dblock_t dblock = to_dblock(b); 964 965 if (cmd->clean_when_opened) { 966 r = __is_discarded(cmd, dblock, &discard); 967 if (r) 968 return r; 969 } else 970 discard = false; 971 972 r = fn(context, cmd->discard_block_size, dblock, discard); 973 if (r) 974 break; 975 } 976 977 return r; 978 } 979 980 int dm_cache_load_discards(struct dm_cache_metadata *cmd, 981 load_discard_fn fn, void *context) 982 { 983 int r; 984 985 down_read(&cmd->root_lock); 986 r = __load_discards(cmd, fn, context); 987 up_read(&cmd->root_lock); 988 989 return r; 990 } 991 992 dm_cblock_t dm_cache_size(struct dm_cache_metadata *cmd) 993 { 994 dm_cblock_t r; 995 996 down_read(&cmd->root_lock); 997 r = cmd->cache_blocks; 998 up_read(&cmd->root_lock); 999 1000 return r; 1001 } 1002 1003 static int __remove(struct dm_cache_metadata *cmd, dm_cblock_t cblock) 1004 { 1005 int r; 1006 __le64 value = pack_value(0, 0); 1007 1008 __dm_bless_for_disk(&value); 1009 r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock), 1010 &value, &cmd->root); 1011 if (r) 1012 return r; 1013 1014 cmd->changed = true; 1015 return 0; 1016 } 1017 1018 int dm_cache_remove_mapping(struct dm_cache_metadata *cmd, dm_cblock_t cblock) 1019 { 1020 int r; 1021 1022 down_write(&cmd->root_lock); 1023 r = __remove(cmd, cblock); 1024 up_write(&cmd->root_lock); 1025 1026 return r; 1027 } 1028 1029 static int __insert(struct dm_cache_metadata *cmd, 1030 dm_cblock_t cblock, dm_oblock_t oblock) 1031 { 1032 int r; 1033 __le64 value = pack_value(oblock, M_VALID); 1034 __dm_bless_for_disk(&value); 1035 1036 r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock), 1037 &value, &cmd->root); 1038 if (r) 1039 return r; 1040 1041 cmd->changed = true; 1042 return 0; 1043 } 1044 1045 int dm_cache_insert_mapping(struct dm_cache_metadata *cmd, 1046 dm_cblock_t cblock, dm_oblock_t oblock) 1047 { 1048 int r; 1049 1050 down_write(&cmd->root_lock); 1051 r = __insert(cmd, cblock, oblock); 1052 up_write(&cmd->root_lock); 1053 1054 return r; 1055 } 1056 1057 struct thunk { 1058 load_mapping_fn fn; 1059 void *context; 1060 1061 struct dm_cache_metadata *cmd; 1062 bool respect_dirty_flags; 1063 bool hints_valid; 1064 }; 1065 1066 static bool policy_unchanged(struct dm_cache_metadata *cmd, 1067 struct dm_cache_policy *policy) 1068 { 1069 const char *policy_name = dm_cache_policy_get_name(policy); 1070 const unsigned *policy_version = dm_cache_policy_get_version(policy); 1071 size_t policy_hint_size = dm_cache_policy_get_hint_size(policy); 1072 1073 /* 1074 * Ensure policy names match. 1075 */ 1076 if (strncmp(cmd->policy_name, policy_name, sizeof(cmd->policy_name))) 1077 return false; 1078 1079 /* 1080 * Ensure policy major versions match. 1081 */ 1082 if (cmd->policy_version[0] != policy_version[0]) 1083 return false; 1084 1085 /* 1086 * Ensure policy hint sizes match. 1087 */ 1088 if (cmd->policy_hint_size != policy_hint_size) 1089 return false; 1090 1091 return true; 1092 } 1093 1094 static bool hints_array_initialized(struct dm_cache_metadata *cmd) 1095 { 1096 return cmd->hint_root && cmd->policy_hint_size; 1097 } 1098 1099 static bool hints_array_available(struct dm_cache_metadata *cmd, 1100 struct dm_cache_policy *policy) 1101 { 1102 return cmd->clean_when_opened && policy_unchanged(cmd, policy) && 1103 hints_array_initialized(cmd); 1104 } 1105 1106 static int __load_mapping(void *context, uint64_t cblock, void *leaf) 1107 { 1108 int r = 0; 1109 bool dirty; 1110 __le64 value; 1111 __le32 hint_value = 0; 1112 dm_oblock_t oblock; 1113 unsigned flags; 1114 struct thunk *thunk = context; 1115 struct dm_cache_metadata *cmd = thunk->cmd; 1116 1117 memcpy(&value, leaf, sizeof(value)); 1118 unpack_value(value, &oblock, &flags); 1119 1120 if (flags & M_VALID) { 1121 if (thunk->hints_valid) { 1122 r = dm_array_get_value(&cmd->hint_info, cmd->hint_root, 1123 cblock, &hint_value); 1124 if (r && r != -ENODATA) 1125 return r; 1126 } 1127 1128 dirty = thunk->respect_dirty_flags ? (flags & M_DIRTY) : true; 1129 r = thunk->fn(thunk->context, oblock, to_cblock(cblock), 1130 dirty, le32_to_cpu(hint_value), thunk->hints_valid); 1131 } 1132 1133 return r; 1134 } 1135 1136 static int __load_mappings(struct dm_cache_metadata *cmd, 1137 struct dm_cache_policy *policy, 1138 load_mapping_fn fn, void *context) 1139 { 1140 struct thunk thunk; 1141 1142 thunk.fn = fn; 1143 thunk.context = context; 1144 1145 thunk.cmd = cmd; 1146 thunk.respect_dirty_flags = cmd->clean_when_opened; 1147 thunk.hints_valid = hints_array_available(cmd, policy); 1148 1149 return dm_array_walk(&cmd->info, cmd->root, __load_mapping, &thunk); 1150 } 1151 1152 int dm_cache_load_mappings(struct dm_cache_metadata *cmd, 1153 struct dm_cache_policy *policy, 1154 load_mapping_fn fn, void *context) 1155 { 1156 int r; 1157 1158 down_read(&cmd->root_lock); 1159 r = __load_mappings(cmd, policy, fn, context); 1160 up_read(&cmd->root_lock); 1161 1162 return r; 1163 } 1164 1165 static int __dump_mapping(void *context, uint64_t cblock, void *leaf) 1166 { 1167 int r = 0; 1168 __le64 value; 1169 dm_oblock_t oblock; 1170 unsigned flags; 1171 1172 memcpy(&value, leaf, sizeof(value)); 1173 unpack_value(value, &oblock, &flags); 1174 1175 return r; 1176 } 1177 1178 static int __dump_mappings(struct dm_cache_metadata *cmd) 1179 { 1180 return dm_array_walk(&cmd->info, cmd->root, __dump_mapping, NULL); 1181 } 1182 1183 void dm_cache_dump(struct dm_cache_metadata *cmd) 1184 { 1185 down_read(&cmd->root_lock); 1186 __dump_mappings(cmd); 1187 up_read(&cmd->root_lock); 1188 } 1189 1190 int dm_cache_changed_this_transaction(struct dm_cache_metadata *cmd) 1191 { 1192 int r; 1193 1194 down_read(&cmd->root_lock); 1195 r = cmd->changed; 1196 up_read(&cmd->root_lock); 1197 1198 return r; 1199 } 1200 1201 static int __dirty(struct dm_cache_metadata *cmd, dm_cblock_t cblock, bool dirty) 1202 { 1203 int r; 1204 unsigned flags; 1205 dm_oblock_t oblock; 1206 __le64 value; 1207 1208 r = dm_array_get_value(&cmd->info, cmd->root, from_cblock(cblock), &value); 1209 if (r) 1210 return r; 1211 1212 unpack_value(value, &oblock, &flags); 1213 1214 if (((flags & M_DIRTY) && dirty) || (!(flags & M_DIRTY) && !dirty)) 1215 /* nothing to be done */ 1216 return 0; 1217 1218 value = pack_value(oblock, (flags & ~M_DIRTY) | (dirty ? M_DIRTY : 0)); 1219 __dm_bless_for_disk(&value); 1220 1221 r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock), 1222 &value, &cmd->root); 1223 if (r) 1224 return r; 1225 1226 cmd->changed = true; 1227 return 0; 1228 1229 } 1230 1231 int dm_cache_set_dirty(struct dm_cache_metadata *cmd, 1232 dm_cblock_t cblock, bool dirty) 1233 { 1234 int r; 1235 1236 down_write(&cmd->root_lock); 1237 r = __dirty(cmd, cblock, dirty); 1238 up_write(&cmd->root_lock); 1239 1240 return r; 1241 } 1242 1243 void dm_cache_metadata_get_stats(struct dm_cache_metadata *cmd, 1244 struct dm_cache_statistics *stats) 1245 { 1246 down_read(&cmd->root_lock); 1247 *stats = cmd->stats; 1248 up_read(&cmd->root_lock); 1249 } 1250 1251 void dm_cache_metadata_set_stats(struct dm_cache_metadata *cmd, 1252 struct dm_cache_statistics *stats) 1253 { 1254 down_write(&cmd->root_lock); 1255 cmd->stats = *stats; 1256 up_write(&cmd->root_lock); 1257 } 1258 1259 int dm_cache_commit(struct dm_cache_metadata *cmd, bool clean_shutdown) 1260 { 1261 int r; 1262 flags_mutator mutator = (clean_shutdown ? set_clean_shutdown : 1263 clear_clean_shutdown); 1264 1265 down_write(&cmd->root_lock); 1266 r = __commit_transaction(cmd, mutator); 1267 if (r) 1268 goto out; 1269 1270 r = __begin_transaction(cmd); 1271 1272 out: 1273 up_write(&cmd->root_lock); 1274 return r; 1275 } 1276 1277 int dm_cache_get_free_metadata_block_count(struct dm_cache_metadata *cmd, 1278 dm_block_t *result) 1279 { 1280 int r = -EINVAL; 1281 1282 down_read(&cmd->root_lock); 1283 r = dm_sm_get_nr_free(cmd->metadata_sm, result); 1284 up_read(&cmd->root_lock); 1285 1286 return r; 1287 } 1288 1289 int dm_cache_get_metadata_dev_size(struct dm_cache_metadata *cmd, 1290 dm_block_t *result) 1291 { 1292 int r = -EINVAL; 1293 1294 down_read(&cmd->root_lock); 1295 r = dm_sm_get_nr_blocks(cmd->metadata_sm, result); 1296 up_read(&cmd->root_lock); 1297 1298 return r; 1299 } 1300 1301 /*----------------------------------------------------------------*/ 1302 1303 static int begin_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy) 1304 { 1305 int r; 1306 __le32 value; 1307 size_t hint_size; 1308 const char *policy_name = dm_cache_policy_get_name(policy); 1309 const unsigned *policy_version = dm_cache_policy_get_version(policy); 1310 1311 if (!policy_name[0] || 1312 (strlen(policy_name) > sizeof(cmd->policy_name) - 1)) 1313 return -EINVAL; 1314 1315 if (!policy_unchanged(cmd, policy)) { 1316 strncpy(cmd->policy_name, policy_name, sizeof(cmd->policy_name)); 1317 memcpy(cmd->policy_version, policy_version, sizeof(cmd->policy_version)); 1318 1319 hint_size = dm_cache_policy_get_hint_size(policy); 1320 if (!hint_size) 1321 return 0; /* short-circuit hints initialization */ 1322 cmd->policy_hint_size = hint_size; 1323 1324 if (cmd->hint_root) { 1325 r = dm_array_del(&cmd->hint_info, cmd->hint_root); 1326 if (r) 1327 return r; 1328 } 1329 1330 r = dm_array_empty(&cmd->hint_info, &cmd->hint_root); 1331 if (r) 1332 return r; 1333 1334 value = cpu_to_le32(0); 1335 __dm_bless_for_disk(&value); 1336 r = dm_array_resize(&cmd->hint_info, cmd->hint_root, 0, 1337 from_cblock(cmd->cache_blocks), 1338 &value, &cmd->hint_root); 1339 if (r) 1340 return r; 1341 } 1342 1343 return 0; 1344 } 1345 1346 static int save_hint(void *context, dm_cblock_t cblock, dm_oblock_t oblock, uint32_t hint) 1347 { 1348 struct dm_cache_metadata *cmd = context; 1349 __le32 value = cpu_to_le32(hint); 1350 int r; 1351 1352 __dm_bless_for_disk(&value); 1353 1354 r = dm_array_set_value(&cmd->hint_info, cmd->hint_root, 1355 from_cblock(cblock), &value, &cmd->hint_root); 1356 cmd->changed = true; 1357 1358 return r; 1359 } 1360 1361 static int write_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy) 1362 { 1363 int r; 1364 1365 r = begin_hints(cmd, policy); 1366 if (r) { 1367 DMERR("begin_hints failed"); 1368 return r; 1369 } 1370 1371 return policy_walk_mappings(policy, save_hint, cmd); 1372 } 1373 1374 int dm_cache_write_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy) 1375 { 1376 int r; 1377 1378 down_write(&cmd->root_lock); 1379 r = write_hints(cmd, policy); 1380 up_write(&cmd->root_lock); 1381 1382 return r; 1383 } 1384 1385 int dm_cache_metadata_all_clean(struct dm_cache_metadata *cmd, bool *result) 1386 { 1387 return blocks_are_unmapped_or_clean(cmd, 0, cmd->cache_blocks, result); 1388 } 1389