1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2012 Red Hat, Inc.
4  *
5  * This file is released under the GPL.
6  */
7 
8 #include "dm-cache-metadata.h"
9 
10 #include "persistent-data/dm-array.h"
11 #include "persistent-data/dm-bitset.h"
12 #include "persistent-data/dm-space-map.h"
13 #include "persistent-data/dm-space-map-disk.h"
14 #include "persistent-data/dm-transaction-manager.h"
15 
16 #include <linux/device-mapper.h>
17 #include <linux/refcount.h>
18 
19 /*----------------------------------------------------------------*/
20 
21 #define DM_MSG_PREFIX   "cache metadata"
22 
23 #define CACHE_SUPERBLOCK_MAGIC 06142003
24 #define CACHE_SUPERBLOCK_LOCATION 0
25 
26 /*
27  * defines a range of metadata versions that this module can handle.
28  */
29 #define MIN_CACHE_VERSION 1
30 #define MAX_CACHE_VERSION 2
31 
32 /*
33  *  3 for btree insert +
34  *  2 for btree lookup used within space map
35  */
36 #define CACHE_MAX_CONCURRENT_LOCKS 5
37 #define SPACE_MAP_ROOT_SIZE 128
38 
39 enum superblock_flag_bits {
40 	/* for spotting crashes that would invalidate the dirty bitset */
41 	CLEAN_SHUTDOWN,
42 	/* metadata must be checked using the tools */
43 	NEEDS_CHECK,
44 };
45 
46 /*
47  * Each mapping from cache block -> origin block carries a set of flags.
48  */
49 enum mapping_bits {
50 	/*
51 	 * A valid mapping.  Because we're using an array we clear this
52 	 * flag for an non existant mapping.
53 	 */
54 	M_VALID = 1,
55 
56 	/*
57 	 * The data on the cache is different from that on the origin.
58 	 * This flag is only used by metadata format 1.
59 	 */
60 	M_DIRTY = 2
61 };
62 
63 struct cache_disk_superblock {
64 	__le32 csum;
65 	__le32 flags;
66 	__le64 blocknr;
67 
68 	__u8 uuid[16];
69 	__le64 magic;
70 	__le32 version;
71 
72 	__u8 policy_name[CACHE_POLICY_NAME_SIZE];
73 	__le32 policy_hint_size;
74 
75 	__u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE];
76 	__le64 mapping_root;
77 	__le64 hint_root;
78 
79 	__le64 discard_root;
80 	__le64 discard_block_size;
81 	__le64 discard_nr_blocks;
82 
83 	__le32 data_block_size;
84 	__le32 metadata_block_size;
85 	__le32 cache_blocks;
86 
87 	__le32 compat_flags;
88 	__le32 compat_ro_flags;
89 	__le32 incompat_flags;
90 
91 	__le32 read_hits;
92 	__le32 read_misses;
93 	__le32 write_hits;
94 	__le32 write_misses;
95 
96 	__le32 policy_version[CACHE_POLICY_VERSION_SIZE];
97 
98 	/*
99 	 * Metadata format 2 fields.
100 	 */
101 	__le64 dirty_root;
102 } __packed;
103 
104 struct dm_cache_metadata {
105 	refcount_t ref_count;
106 	struct list_head list;
107 
108 	unsigned int version;
109 	struct block_device *bdev;
110 	struct dm_block_manager *bm;
111 	struct dm_space_map *metadata_sm;
112 	struct dm_transaction_manager *tm;
113 
114 	struct dm_array_info info;
115 	struct dm_array_info hint_info;
116 	struct dm_disk_bitset discard_info;
117 
118 	struct rw_semaphore root_lock;
119 	unsigned long flags;
120 	dm_block_t root;
121 	dm_block_t hint_root;
122 	dm_block_t discard_root;
123 
124 	sector_t discard_block_size;
125 	dm_dblock_t discard_nr_blocks;
126 
127 	sector_t data_block_size;
128 	dm_cblock_t cache_blocks;
129 	bool changed:1;
130 	bool clean_when_opened:1;
131 
132 	char policy_name[CACHE_POLICY_NAME_SIZE];
133 	unsigned int policy_version[CACHE_POLICY_VERSION_SIZE];
134 	size_t policy_hint_size;
135 	struct dm_cache_statistics stats;
136 
137 	/*
138 	 * Reading the space map root can fail, so we read it into this
139 	 * buffer before the superblock is locked and updated.
140 	 */
141 	__u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE];
142 
143 	/*
144 	 * Set if a transaction has to be aborted but the attempt to roll
145 	 * back to the previous (good) transaction failed.  The only
146 	 * metadata operation permissible in this state is the closing of
147 	 * the device.
148 	 */
149 	bool fail_io:1;
150 
151 	/*
152 	 * Metadata format 2 fields.
153 	 */
154 	dm_block_t dirty_root;
155 	struct dm_disk_bitset dirty_info;
156 
157 	/*
158 	 * These structures are used when loading metadata.  They're too
159 	 * big to put on the stack.
160 	 */
161 	struct dm_array_cursor mapping_cursor;
162 	struct dm_array_cursor hint_cursor;
163 	struct dm_bitset_cursor dirty_cursor;
164 };
165 
166 /*
167  *-----------------------------------------------------------------
168  * superblock validator
169  *-----------------------------------------------------------------
170  */
171 #define SUPERBLOCK_CSUM_XOR 9031977
172 
173 static void sb_prepare_for_write(struct dm_block_validator *v,
174 				 struct dm_block *b,
175 				 size_t sb_block_size)
176 {
177 	struct cache_disk_superblock *disk_super = dm_block_data(b);
178 
179 	disk_super->blocknr = cpu_to_le64(dm_block_location(b));
180 	disk_super->csum = cpu_to_le32(dm_bm_checksum(&disk_super->flags,
181 						      sb_block_size - sizeof(__le32),
182 						      SUPERBLOCK_CSUM_XOR));
183 }
184 
185 static int check_metadata_version(struct cache_disk_superblock *disk_super)
186 {
187 	uint32_t metadata_version = le32_to_cpu(disk_super->version);
188 
189 	if (metadata_version < MIN_CACHE_VERSION || metadata_version > MAX_CACHE_VERSION) {
190 		DMERR("Cache metadata version %u found, but only versions between %u and %u supported.",
191 		      metadata_version, MIN_CACHE_VERSION, MAX_CACHE_VERSION);
192 		return -EINVAL;
193 	}
194 
195 	return 0;
196 }
197 
198 static int sb_check(struct dm_block_validator *v,
199 		    struct dm_block *b,
200 		    size_t sb_block_size)
201 {
202 	struct cache_disk_superblock *disk_super = dm_block_data(b);
203 	__le32 csum_le;
204 
205 	if (dm_block_location(b) != le64_to_cpu(disk_super->blocknr)) {
206 		DMERR("%s failed: blocknr %llu: wanted %llu",
207 		      __func__, le64_to_cpu(disk_super->blocknr),
208 		      (unsigned long long)dm_block_location(b));
209 		return -ENOTBLK;
210 	}
211 
212 	if (le64_to_cpu(disk_super->magic) != CACHE_SUPERBLOCK_MAGIC) {
213 		DMERR("%s failed: magic %llu: wanted %llu",
214 		      __func__, le64_to_cpu(disk_super->magic),
215 		      (unsigned long long)CACHE_SUPERBLOCK_MAGIC);
216 		return -EILSEQ;
217 	}
218 
219 	csum_le = cpu_to_le32(dm_bm_checksum(&disk_super->flags,
220 					     sb_block_size - sizeof(__le32),
221 					     SUPERBLOCK_CSUM_XOR));
222 	if (csum_le != disk_super->csum) {
223 		DMERR("%s failed: csum %u: wanted %u",
224 		      __func__, le32_to_cpu(csum_le), le32_to_cpu(disk_super->csum));
225 		return -EILSEQ;
226 	}
227 
228 	return check_metadata_version(disk_super);
229 }
230 
231 static struct dm_block_validator sb_validator = {
232 	.name = "superblock",
233 	.prepare_for_write = sb_prepare_for_write,
234 	.check = sb_check
235 };
236 
237 /*----------------------------------------------------------------*/
238 
239 static int superblock_read_lock(struct dm_cache_metadata *cmd,
240 				struct dm_block **sblock)
241 {
242 	return dm_bm_read_lock(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
243 			       &sb_validator, sblock);
244 }
245 
246 static int superblock_lock_zero(struct dm_cache_metadata *cmd,
247 				struct dm_block **sblock)
248 {
249 	return dm_bm_write_lock_zero(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
250 				     &sb_validator, sblock);
251 }
252 
253 static int superblock_lock(struct dm_cache_metadata *cmd,
254 			   struct dm_block **sblock)
255 {
256 	return dm_bm_write_lock(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
257 				&sb_validator, sblock);
258 }
259 
260 /*----------------------------------------------------------------*/
261 
262 static int __superblock_all_zeroes(struct dm_block_manager *bm, bool *result)
263 {
264 	int r;
265 	unsigned int i;
266 	struct dm_block *b;
267 	__le64 *data_le, zero = cpu_to_le64(0);
268 	unsigned int sb_block_size = dm_bm_block_size(bm) / sizeof(__le64);
269 
270 	/*
271 	 * We can't use a validator here - it may be all zeroes.
272 	 */
273 	r = dm_bm_read_lock(bm, CACHE_SUPERBLOCK_LOCATION, NULL, &b);
274 	if (r)
275 		return r;
276 
277 	data_le = dm_block_data(b);
278 	*result = true;
279 	for (i = 0; i < sb_block_size; i++) {
280 		if (data_le[i] != zero) {
281 			*result = false;
282 			break;
283 		}
284 	}
285 
286 	dm_bm_unlock(b);
287 
288 	return 0;
289 }
290 
291 static void __setup_mapping_info(struct dm_cache_metadata *cmd)
292 {
293 	struct dm_btree_value_type vt;
294 
295 	vt.context = NULL;
296 	vt.size = sizeof(__le64);
297 	vt.inc = NULL;
298 	vt.dec = NULL;
299 	vt.equal = NULL;
300 	dm_array_info_init(&cmd->info, cmd->tm, &vt);
301 
302 	if (cmd->policy_hint_size) {
303 		vt.size = sizeof(__le32);
304 		dm_array_info_init(&cmd->hint_info, cmd->tm, &vt);
305 	}
306 }
307 
308 static int __save_sm_root(struct dm_cache_metadata *cmd)
309 {
310 	int r;
311 	size_t metadata_len;
312 
313 	r = dm_sm_root_size(cmd->metadata_sm, &metadata_len);
314 	if (r < 0)
315 		return r;
316 
317 	return dm_sm_copy_root(cmd->metadata_sm, &cmd->metadata_space_map_root,
318 			       metadata_len);
319 }
320 
321 static void __copy_sm_root(struct dm_cache_metadata *cmd,
322 			   struct cache_disk_superblock *disk_super)
323 {
324 	memcpy(&disk_super->metadata_space_map_root,
325 	       &cmd->metadata_space_map_root,
326 	       sizeof(cmd->metadata_space_map_root));
327 }
328 
329 static bool separate_dirty_bits(struct dm_cache_metadata *cmd)
330 {
331 	return cmd->version >= 2;
332 }
333 
334 static int __write_initial_superblock(struct dm_cache_metadata *cmd)
335 {
336 	int r;
337 	struct dm_block *sblock;
338 	struct cache_disk_superblock *disk_super;
339 	sector_t bdev_size = bdev_nr_sectors(cmd->bdev);
340 
341 	/* FIXME: see if we can lose the max sectors limit */
342 	if (bdev_size > DM_CACHE_METADATA_MAX_SECTORS)
343 		bdev_size = DM_CACHE_METADATA_MAX_SECTORS;
344 
345 	r = dm_tm_pre_commit(cmd->tm);
346 	if (r < 0)
347 		return r;
348 
349 	/*
350 	 * dm_sm_copy_root() can fail.  So we need to do it before we start
351 	 * updating the superblock.
352 	 */
353 	r = __save_sm_root(cmd);
354 	if (r)
355 		return r;
356 
357 	r = superblock_lock_zero(cmd, &sblock);
358 	if (r)
359 		return r;
360 
361 	disk_super = dm_block_data(sblock);
362 	disk_super->flags = 0;
363 	memset(disk_super->uuid, 0, sizeof(disk_super->uuid));
364 	disk_super->magic = cpu_to_le64(CACHE_SUPERBLOCK_MAGIC);
365 	disk_super->version = cpu_to_le32(cmd->version);
366 	memset(disk_super->policy_name, 0, sizeof(disk_super->policy_name));
367 	memset(disk_super->policy_version, 0, sizeof(disk_super->policy_version));
368 	disk_super->policy_hint_size = cpu_to_le32(0);
369 
370 	__copy_sm_root(cmd, disk_super);
371 
372 	disk_super->mapping_root = cpu_to_le64(cmd->root);
373 	disk_super->hint_root = cpu_to_le64(cmd->hint_root);
374 	disk_super->discard_root = cpu_to_le64(cmd->discard_root);
375 	disk_super->discard_block_size = cpu_to_le64(cmd->discard_block_size);
376 	disk_super->discard_nr_blocks = cpu_to_le64(from_dblock(cmd->discard_nr_blocks));
377 	disk_super->metadata_block_size = cpu_to_le32(DM_CACHE_METADATA_BLOCK_SIZE);
378 	disk_super->data_block_size = cpu_to_le32(cmd->data_block_size);
379 	disk_super->cache_blocks = cpu_to_le32(0);
380 
381 	disk_super->read_hits = cpu_to_le32(0);
382 	disk_super->read_misses = cpu_to_le32(0);
383 	disk_super->write_hits = cpu_to_le32(0);
384 	disk_super->write_misses = cpu_to_le32(0);
385 
386 	if (separate_dirty_bits(cmd))
387 		disk_super->dirty_root = cpu_to_le64(cmd->dirty_root);
388 
389 	return dm_tm_commit(cmd->tm, sblock);
390 }
391 
392 static int __format_metadata(struct dm_cache_metadata *cmd)
393 {
394 	int r;
395 
396 	r = dm_tm_create_with_sm(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
397 				 &cmd->tm, &cmd->metadata_sm);
398 	if (r < 0) {
399 		DMERR("tm_create_with_sm failed");
400 		return r;
401 	}
402 
403 	__setup_mapping_info(cmd);
404 
405 	r = dm_array_empty(&cmd->info, &cmd->root);
406 	if (r < 0)
407 		goto bad;
408 
409 	if (separate_dirty_bits(cmd)) {
410 		dm_disk_bitset_init(cmd->tm, &cmd->dirty_info);
411 		r = dm_bitset_empty(&cmd->dirty_info, &cmd->dirty_root);
412 		if (r < 0)
413 			goto bad;
414 	}
415 
416 	dm_disk_bitset_init(cmd->tm, &cmd->discard_info);
417 	r = dm_bitset_empty(&cmd->discard_info, &cmd->discard_root);
418 	if (r < 0)
419 		goto bad;
420 
421 	cmd->discard_block_size = 0;
422 	cmd->discard_nr_blocks = 0;
423 
424 	r = __write_initial_superblock(cmd);
425 	if (r)
426 		goto bad;
427 
428 	cmd->clean_when_opened = true;
429 	return 0;
430 
431 bad:
432 	dm_tm_destroy(cmd->tm);
433 	dm_sm_destroy(cmd->metadata_sm);
434 
435 	return r;
436 }
437 
438 static int __check_incompat_features(struct cache_disk_superblock *disk_super,
439 				     struct dm_cache_metadata *cmd)
440 {
441 	uint32_t incompat_flags, features;
442 
443 	incompat_flags = le32_to_cpu(disk_super->incompat_flags);
444 	features = incompat_flags & ~DM_CACHE_FEATURE_INCOMPAT_SUPP;
445 	if (features) {
446 		DMERR("could not access metadata due to unsupported optional features (%lx).",
447 		      (unsigned long)features);
448 		return -EINVAL;
449 	}
450 
451 	/*
452 	 * Check for read-only metadata to skip the following RDWR checks.
453 	 */
454 	if (bdev_read_only(cmd->bdev))
455 		return 0;
456 
457 	features = le32_to_cpu(disk_super->compat_ro_flags) & ~DM_CACHE_FEATURE_COMPAT_RO_SUPP;
458 	if (features) {
459 		DMERR("could not access metadata RDWR due to unsupported optional features (%lx).",
460 		      (unsigned long)features);
461 		return -EINVAL;
462 	}
463 
464 	return 0;
465 }
466 
467 static int __open_metadata(struct dm_cache_metadata *cmd)
468 {
469 	int r;
470 	struct dm_block *sblock;
471 	struct cache_disk_superblock *disk_super;
472 	unsigned long sb_flags;
473 
474 	r = superblock_read_lock(cmd, &sblock);
475 	if (r < 0) {
476 		DMERR("couldn't read lock superblock");
477 		return r;
478 	}
479 
480 	disk_super = dm_block_data(sblock);
481 
482 	/* Verify the data block size hasn't changed */
483 	if (le32_to_cpu(disk_super->data_block_size) != cmd->data_block_size) {
484 		DMERR("changing the data block size (from %u to %llu) is not supported",
485 		      le32_to_cpu(disk_super->data_block_size),
486 		      (unsigned long long)cmd->data_block_size);
487 		r = -EINVAL;
488 		goto bad;
489 	}
490 
491 	r = __check_incompat_features(disk_super, cmd);
492 	if (r < 0)
493 		goto bad;
494 
495 	r = dm_tm_open_with_sm(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
496 			       disk_super->metadata_space_map_root,
497 			       sizeof(disk_super->metadata_space_map_root),
498 			       &cmd->tm, &cmd->metadata_sm);
499 	if (r < 0) {
500 		DMERR("tm_open_with_sm failed");
501 		goto bad;
502 	}
503 
504 	__setup_mapping_info(cmd);
505 	dm_disk_bitset_init(cmd->tm, &cmd->dirty_info);
506 	dm_disk_bitset_init(cmd->tm, &cmd->discard_info);
507 	sb_flags = le32_to_cpu(disk_super->flags);
508 	cmd->clean_when_opened = test_bit(CLEAN_SHUTDOWN, &sb_flags);
509 	dm_bm_unlock(sblock);
510 
511 	return 0;
512 
513 bad:
514 	dm_bm_unlock(sblock);
515 	return r;
516 }
517 
518 static int __open_or_format_metadata(struct dm_cache_metadata *cmd,
519 				     bool format_device)
520 {
521 	int r;
522 	bool unformatted = false;
523 
524 	r = __superblock_all_zeroes(cmd->bm, &unformatted);
525 	if (r)
526 		return r;
527 
528 	if (unformatted)
529 		return format_device ? __format_metadata(cmd) : -EPERM;
530 
531 	return __open_metadata(cmd);
532 }
533 
534 static int __create_persistent_data_objects(struct dm_cache_metadata *cmd,
535 					    bool may_format_device)
536 {
537 	int r;
538 
539 	cmd->bm = dm_block_manager_create(cmd->bdev, DM_CACHE_METADATA_BLOCK_SIZE << SECTOR_SHIFT,
540 					  CACHE_MAX_CONCURRENT_LOCKS);
541 	if (IS_ERR(cmd->bm)) {
542 		DMERR("could not create block manager");
543 		r = PTR_ERR(cmd->bm);
544 		cmd->bm = NULL;
545 		return r;
546 	}
547 
548 	r = __open_or_format_metadata(cmd, may_format_device);
549 	if (r) {
550 		dm_block_manager_destroy(cmd->bm);
551 		cmd->bm = NULL;
552 	}
553 
554 	return r;
555 }
556 
557 static void __destroy_persistent_data_objects(struct dm_cache_metadata *cmd,
558 					      bool destroy_bm)
559 {
560 	dm_sm_destroy(cmd->metadata_sm);
561 	dm_tm_destroy(cmd->tm);
562 	if (destroy_bm)
563 		dm_block_manager_destroy(cmd->bm);
564 }
565 
566 typedef unsigned long (*flags_mutator)(unsigned long);
567 
568 static void update_flags(struct cache_disk_superblock *disk_super,
569 			 flags_mutator mutator)
570 {
571 	uint32_t sb_flags = mutator(le32_to_cpu(disk_super->flags));
572 
573 	disk_super->flags = cpu_to_le32(sb_flags);
574 }
575 
576 static unsigned long set_clean_shutdown(unsigned long flags)
577 {
578 	set_bit(CLEAN_SHUTDOWN, &flags);
579 	return flags;
580 }
581 
582 static unsigned long clear_clean_shutdown(unsigned long flags)
583 {
584 	clear_bit(CLEAN_SHUTDOWN, &flags);
585 	return flags;
586 }
587 
588 static void read_superblock_fields(struct dm_cache_metadata *cmd,
589 				   struct cache_disk_superblock *disk_super)
590 {
591 	cmd->version = le32_to_cpu(disk_super->version);
592 	cmd->flags = le32_to_cpu(disk_super->flags);
593 	cmd->root = le64_to_cpu(disk_super->mapping_root);
594 	cmd->hint_root = le64_to_cpu(disk_super->hint_root);
595 	cmd->discard_root = le64_to_cpu(disk_super->discard_root);
596 	cmd->discard_block_size = le64_to_cpu(disk_super->discard_block_size);
597 	cmd->discard_nr_blocks = to_dblock(le64_to_cpu(disk_super->discard_nr_blocks));
598 	cmd->data_block_size = le32_to_cpu(disk_super->data_block_size);
599 	cmd->cache_blocks = to_cblock(le32_to_cpu(disk_super->cache_blocks));
600 	strncpy(cmd->policy_name, disk_super->policy_name, sizeof(cmd->policy_name));
601 	cmd->policy_version[0] = le32_to_cpu(disk_super->policy_version[0]);
602 	cmd->policy_version[1] = le32_to_cpu(disk_super->policy_version[1]);
603 	cmd->policy_version[2] = le32_to_cpu(disk_super->policy_version[2]);
604 	cmd->policy_hint_size = le32_to_cpu(disk_super->policy_hint_size);
605 
606 	cmd->stats.read_hits = le32_to_cpu(disk_super->read_hits);
607 	cmd->stats.read_misses = le32_to_cpu(disk_super->read_misses);
608 	cmd->stats.write_hits = le32_to_cpu(disk_super->write_hits);
609 	cmd->stats.write_misses = le32_to_cpu(disk_super->write_misses);
610 
611 	if (separate_dirty_bits(cmd))
612 		cmd->dirty_root = le64_to_cpu(disk_super->dirty_root);
613 
614 	cmd->changed = false;
615 }
616 
617 /*
618  * The mutator updates the superblock flags.
619  */
620 static int __begin_transaction_flags(struct dm_cache_metadata *cmd,
621 				     flags_mutator mutator)
622 {
623 	int r;
624 	struct cache_disk_superblock *disk_super;
625 	struct dm_block *sblock;
626 
627 	r = superblock_lock(cmd, &sblock);
628 	if (r)
629 		return r;
630 
631 	disk_super = dm_block_data(sblock);
632 	update_flags(disk_super, mutator);
633 	read_superblock_fields(cmd, disk_super);
634 	dm_bm_unlock(sblock);
635 
636 	return dm_bm_flush(cmd->bm);
637 }
638 
639 static int __begin_transaction(struct dm_cache_metadata *cmd)
640 {
641 	int r;
642 	struct cache_disk_superblock *disk_super;
643 	struct dm_block *sblock;
644 
645 	/*
646 	 * We re-read the superblock every time.  Shouldn't need to do this
647 	 * really.
648 	 */
649 	r = superblock_read_lock(cmd, &sblock);
650 	if (r)
651 		return r;
652 
653 	disk_super = dm_block_data(sblock);
654 	read_superblock_fields(cmd, disk_super);
655 	dm_bm_unlock(sblock);
656 
657 	return 0;
658 }
659 
660 static int __commit_transaction(struct dm_cache_metadata *cmd,
661 				flags_mutator mutator)
662 {
663 	int r;
664 	struct cache_disk_superblock *disk_super;
665 	struct dm_block *sblock;
666 
667 	/*
668 	 * We need to know if the cache_disk_superblock exceeds a 512-byte sector.
669 	 */
670 	BUILD_BUG_ON(sizeof(struct cache_disk_superblock) > 512);
671 
672 	if (separate_dirty_bits(cmd)) {
673 		r = dm_bitset_flush(&cmd->dirty_info, cmd->dirty_root,
674 				    &cmd->dirty_root);
675 		if (r)
676 			return r;
677 	}
678 
679 	r = dm_bitset_flush(&cmd->discard_info, cmd->discard_root,
680 			    &cmd->discard_root);
681 	if (r)
682 		return r;
683 
684 	r = dm_tm_pre_commit(cmd->tm);
685 	if (r < 0)
686 		return r;
687 
688 	r = __save_sm_root(cmd);
689 	if (r)
690 		return r;
691 
692 	r = superblock_lock(cmd, &sblock);
693 	if (r)
694 		return r;
695 
696 	disk_super = dm_block_data(sblock);
697 
698 	disk_super->flags = cpu_to_le32(cmd->flags);
699 	if (mutator)
700 		update_flags(disk_super, mutator);
701 
702 	disk_super->mapping_root = cpu_to_le64(cmd->root);
703 	if (separate_dirty_bits(cmd))
704 		disk_super->dirty_root = cpu_to_le64(cmd->dirty_root);
705 	disk_super->hint_root = cpu_to_le64(cmd->hint_root);
706 	disk_super->discard_root = cpu_to_le64(cmd->discard_root);
707 	disk_super->discard_block_size = cpu_to_le64(cmd->discard_block_size);
708 	disk_super->discard_nr_blocks = cpu_to_le64(from_dblock(cmd->discard_nr_blocks));
709 	disk_super->cache_blocks = cpu_to_le32(from_cblock(cmd->cache_blocks));
710 	strncpy(disk_super->policy_name, cmd->policy_name, sizeof(disk_super->policy_name));
711 	disk_super->policy_version[0] = cpu_to_le32(cmd->policy_version[0]);
712 	disk_super->policy_version[1] = cpu_to_le32(cmd->policy_version[1]);
713 	disk_super->policy_version[2] = cpu_to_le32(cmd->policy_version[2]);
714 	disk_super->policy_hint_size = cpu_to_le32(cmd->policy_hint_size);
715 
716 	disk_super->read_hits = cpu_to_le32(cmd->stats.read_hits);
717 	disk_super->read_misses = cpu_to_le32(cmd->stats.read_misses);
718 	disk_super->write_hits = cpu_to_le32(cmd->stats.write_hits);
719 	disk_super->write_misses = cpu_to_le32(cmd->stats.write_misses);
720 	__copy_sm_root(cmd, disk_super);
721 
722 	return dm_tm_commit(cmd->tm, sblock);
723 }
724 
725 /*----------------------------------------------------------------*/
726 
727 /*
728  * The mappings are held in a dm-array that has 64-bit values stored in
729  * little-endian format.  The index is the cblock, the high 48bits of the
730  * value are the oblock and the low 16 bit the flags.
731  */
732 #define FLAGS_MASK ((1 << 16) - 1)
733 
734 static __le64 pack_value(dm_oblock_t block, unsigned int flags)
735 {
736 	uint64_t value = from_oblock(block);
737 
738 	value <<= 16;
739 	value = value | (flags & FLAGS_MASK);
740 	return cpu_to_le64(value);
741 }
742 
743 static void unpack_value(__le64 value_le, dm_oblock_t *block, unsigned int *flags)
744 {
745 	uint64_t value = le64_to_cpu(value_le);
746 	uint64_t b = value >> 16;
747 
748 	*block = to_oblock(b);
749 	*flags = value & FLAGS_MASK;
750 }
751 
752 /*----------------------------------------------------------------*/
753 
754 static struct dm_cache_metadata *metadata_open(struct block_device *bdev,
755 					       sector_t data_block_size,
756 					       bool may_format_device,
757 					       size_t policy_hint_size,
758 					       unsigned int metadata_version)
759 {
760 	int r;
761 	struct dm_cache_metadata *cmd;
762 
763 	cmd = kzalloc(sizeof(*cmd), GFP_KERNEL);
764 	if (!cmd) {
765 		DMERR("could not allocate metadata struct");
766 		return ERR_PTR(-ENOMEM);
767 	}
768 
769 	cmd->version = metadata_version;
770 	refcount_set(&cmd->ref_count, 1);
771 	init_rwsem(&cmd->root_lock);
772 	cmd->bdev = bdev;
773 	cmd->data_block_size = data_block_size;
774 	cmd->cache_blocks = 0;
775 	cmd->policy_hint_size = policy_hint_size;
776 	cmd->changed = true;
777 	cmd->fail_io = false;
778 
779 	r = __create_persistent_data_objects(cmd, may_format_device);
780 	if (r) {
781 		kfree(cmd);
782 		return ERR_PTR(r);
783 	}
784 
785 	r = __begin_transaction_flags(cmd, clear_clean_shutdown);
786 	if (r < 0) {
787 		dm_cache_metadata_close(cmd);
788 		return ERR_PTR(r);
789 	}
790 
791 	return cmd;
792 }
793 
794 /*
795  * We keep a little list of ref counted metadata objects to prevent two
796  * different target instances creating separate bufio instances.  This is
797  * an issue if a table is reloaded before the suspend.
798  */
799 static DEFINE_MUTEX(table_lock);
800 static LIST_HEAD(table);
801 
802 static struct dm_cache_metadata *lookup(struct block_device *bdev)
803 {
804 	struct dm_cache_metadata *cmd;
805 
806 	list_for_each_entry(cmd, &table, list)
807 		if (cmd->bdev == bdev) {
808 			refcount_inc(&cmd->ref_count);
809 			return cmd;
810 		}
811 
812 	return NULL;
813 }
814 
815 static struct dm_cache_metadata *lookup_or_open(struct block_device *bdev,
816 						sector_t data_block_size,
817 						bool may_format_device,
818 						size_t policy_hint_size,
819 						unsigned int metadata_version)
820 {
821 	struct dm_cache_metadata *cmd, *cmd2;
822 
823 	mutex_lock(&table_lock);
824 	cmd = lookup(bdev);
825 	mutex_unlock(&table_lock);
826 
827 	if (cmd)
828 		return cmd;
829 
830 	cmd = metadata_open(bdev, data_block_size, may_format_device,
831 			    policy_hint_size, metadata_version);
832 	if (!IS_ERR(cmd)) {
833 		mutex_lock(&table_lock);
834 		cmd2 = lookup(bdev);
835 		if (cmd2) {
836 			mutex_unlock(&table_lock);
837 			__destroy_persistent_data_objects(cmd, true);
838 			kfree(cmd);
839 			return cmd2;
840 		}
841 		list_add(&cmd->list, &table);
842 		mutex_unlock(&table_lock);
843 	}
844 
845 	return cmd;
846 }
847 
848 static bool same_params(struct dm_cache_metadata *cmd, sector_t data_block_size)
849 {
850 	if (cmd->data_block_size != data_block_size) {
851 		DMERR("data_block_size (%llu) different from that in metadata (%llu)",
852 		      (unsigned long long) data_block_size,
853 		      (unsigned long long) cmd->data_block_size);
854 		return false;
855 	}
856 
857 	return true;
858 }
859 
860 struct dm_cache_metadata *dm_cache_metadata_open(struct block_device *bdev,
861 						 sector_t data_block_size,
862 						 bool may_format_device,
863 						 size_t policy_hint_size,
864 						 unsigned int metadata_version)
865 {
866 	struct dm_cache_metadata *cmd = lookup_or_open(bdev, data_block_size, may_format_device,
867 						       policy_hint_size, metadata_version);
868 
869 	if (!IS_ERR(cmd) && !same_params(cmd, data_block_size)) {
870 		dm_cache_metadata_close(cmd);
871 		return ERR_PTR(-EINVAL);
872 	}
873 
874 	return cmd;
875 }
876 
877 void dm_cache_metadata_close(struct dm_cache_metadata *cmd)
878 {
879 	if (refcount_dec_and_test(&cmd->ref_count)) {
880 		mutex_lock(&table_lock);
881 		list_del(&cmd->list);
882 		mutex_unlock(&table_lock);
883 
884 		if (!cmd->fail_io)
885 			__destroy_persistent_data_objects(cmd, true);
886 		kfree(cmd);
887 	}
888 }
889 
890 /*
891  * Checks that the given cache block is either unmapped or clean.
892  */
893 static int block_clean_combined_dirty(struct dm_cache_metadata *cmd, dm_cblock_t b,
894 				      bool *result)
895 {
896 	int r;
897 	__le64 value;
898 	dm_oblock_t ob;
899 	unsigned int flags;
900 
901 	r = dm_array_get_value(&cmd->info, cmd->root, from_cblock(b), &value);
902 	if (r)
903 		return r;
904 
905 	unpack_value(value, &ob, &flags);
906 	*result = !((flags & M_VALID) && (flags & M_DIRTY));
907 
908 	return 0;
909 }
910 
911 static int blocks_are_clean_combined_dirty(struct dm_cache_metadata *cmd,
912 					   dm_cblock_t begin, dm_cblock_t end,
913 					   bool *result)
914 {
915 	int r;
916 	*result = true;
917 
918 	while (begin != end) {
919 		r = block_clean_combined_dirty(cmd, begin, result);
920 		if (r) {
921 			DMERR("block_clean_combined_dirty failed");
922 			return r;
923 		}
924 
925 		if (!*result) {
926 			DMERR("cache block %llu is dirty",
927 			      (unsigned long long) from_cblock(begin));
928 			return 0;
929 		}
930 
931 		begin = to_cblock(from_cblock(begin) + 1);
932 	}
933 
934 	return 0;
935 }
936 
937 static int blocks_are_clean_separate_dirty(struct dm_cache_metadata *cmd,
938 					   dm_cblock_t begin, dm_cblock_t end,
939 					   bool *result)
940 {
941 	int r;
942 	bool dirty_flag;
943 	*result = true;
944 
945 	if (from_cblock(cmd->cache_blocks) == 0)
946 		/* Nothing to do */
947 		return 0;
948 
949 	r = dm_bitset_cursor_begin(&cmd->dirty_info, cmd->dirty_root,
950 				   from_cblock(cmd->cache_blocks), &cmd->dirty_cursor);
951 	if (r) {
952 		DMERR("%s: dm_bitset_cursor_begin for dirty failed", __func__);
953 		return r;
954 	}
955 
956 	r = dm_bitset_cursor_skip(&cmd->dirty_cursor, from_cblock(begin));
957 	if (r) {
958 		DMERR("%s: dm_bitset_cursor_skip for dirty failed", __func__);
959 		dm_bitset_cursor_end(&cmd->dirty_cursor);
960 		return r;
961 	}
962 
963 	while (begin != end) {
964 		/*
965 		 * We assume that unmapped blocks have their dirty bit
966 		 * cleared.
967 		 */
968 		dirty_flag = dm_bitset_cursor_get_value(&cmd->dirty_cursor);
969 		if (dirty_flag) {
970 			DMERR("%s: cache block %llu is dirty", __func__,
971 			      (unsigned long long) from_cblock(begin));
972 			dm_bitset_cursor_end(&cmd->dirty_cursor);
973 			*result = false;
974 			return 0;
975 		}
976 
977 		begin = to_cblock(from_cblock(begin) + 1);
978 		if (begin == end)
979 			break;
980 
981 		r = dm_bitset_cursor_next(&cmd->dirty_cursor);
982 		if (r) {
983 			DMERR("%s: dm_bitset_cursor_next for dirty failed", __func__);
984 			dm_bitset_cursor_end(&cmd->dirty_cursor);
985 			return r;
986 		}
987 	}
988 
989 	dm_bitset_cursor_end(&cmd->dirty_cursor);
990 
991 	return 0;
992 }
993 
994 static int blocks_are_unmapped_or_clean(struct dm_cache_metadata *cmd,
995 					dm_cblock_t begin, dm_cblock_t end,
996 					bool *result)
997 {
998 	if (separate_dirty_bits(cmd))
999 		return blocks_are_clean_separate_dirty(cmd, begin, end, result);
1000 	else
1001 		return blocks_are_clean_combined_dirty(cmd, begin, end, result);
1002 }
1003 
1004 static bool cmd_write_lock(struct dm_cache_metadata *cmd)
1005 {
1006 	down_write(&cmd->root_lock);
1007 	if (cmd->fail_io || dm_bm_is_read_only(cmd->bm)) {
1008 		up_write(&cmd->root_lock);
1009 		return false;
1010 	}
1011 	return true;
1012 }
1013 
1014 #define WRITE_LOCK(cmd)				\
1015 	do {					\
1016 		if (!cmd_write_lock((cmd)))	\
1017 			return -EINVAL;		\
1018 	} while (0)
1019 
1020 #define WRITE_LOCK_VOID(cmd)			\
1021 	do {					\
1022 		if (!cmd_write_lock((cmd)))	\
1023 			return;			\
1024 	} while (0)
1025 
1026 #define WRITE_UNLOCK(cmd) \
1027 	up_write(&(cmd)->root_lock)
1028 
1029 static bool cmd_read_lock(struct dm_cache_metadata *cmd)
1030 {
1031 	down_read(&cmd->root_lock);
1032 	if (cmd->fail_io) {
1033 		up_read(&cmd->root_lock);
1034 		return false;
1035 	}
1036 	return true;
1037 }
1038 
1039 #define READ_LOCK(cmd)				\
1040 	do {					\
1041 		if (!cmd_read_lock((cmd)))	\
1042 			return -EINVAL;		\
1043 	} while (0)
1044 
1045 #define READ_LOCK_VOID(cmd)			\
1046 	do {					\
1047 		if (!cmd_read_lock((cmd)))	\
1048 			return;			\
1049 	} while (0)
1050 
1051 #define READ_UNLOCK(cmd) \
1052 	up_read(&(cmd)->root_lock)
1053 
1054 int dm_cache_resize(struct dm_cache_metadata *cmd, dm_cblock_t new_cache_size)
1055 {
1056 	int r;
1057 	bool clean;
1058 	__le64 null_mapping = pack_value(0, 0);
1059 
1060 	WRITE_LOCK(cmd);
1061 	__dm_bless_for_disk(&null_mapping);
1062 
1063 	if (from_cblock(new_cache_size) < from_cblock(cmd->cache_blocks)) {
1064 		r = blocks_are_unmapped_or_clean(cmd, new_cache_size, cmd->cache_blocks, &clean);
1065 		if (r) {
1066 			__dm_unbless_for_disk(&null_mapping);
1067 			goto out;
1068 		}
1069 
1070 		if (!clean) {
1071 			DMERR("unable to shrink cache due to dirty blocks");
1072 			r = -EINVAL;
1073 			__dm_unbless_for_disk(&null_mapping);
1074 			goto out;
1075 		}
1076 	}
1077 
1078 	r = dm_array_resize(&cmd->info, cmd->root, from_cblock(cmd->cache_blocks),
1079 			    from_cblock(new_cache_size),
1080 			    &null_mapping, &cmd->root);
1081 	if (r)
1082 		goto out;
1083 
1084 	if (separate_dirty_bits(cmd)) {
1085 		r = dm_bitset_resize(&cmd->dirty_info, cmd->dirty_root,
1086 				     from_cblock(cmd->cache_blocks), from_cblock(new_cache_size),
1087 				     false, &cmd->dirty_root);
1088 		if (r)
1089 			goto out;
1090 	}
1091 
1092 	cmd->cache_blocks = new_cache_size;
1093 	cmd->changed = true;
1094 
1095 out:
1096 	WRITE_UNLOCK(cmd);
1097 
1098 	return r;
1099 }
1100 
1101 int dm_cache_discard_bitset_resize(struct dm_cache_metadata *cmd,
1102 				   sector_t discard_block_size,
1103 				   dm_dblock_t new_nr_entries)
1104 {
1105 	int r;
1106 
1107 	WRITE_LOCK(cmd);
1108 	r = dm_bitset_resize(&cmd->discard_info,
1109 			     cmd->discard_root,
1110 			     from_dblock(cmd->discard_nr_blocks),
1111 			     from_dblock(new_nr_entries),
1112 			     false, &cmd->discard_root);
1113 	if (!r) {
1114 		cmd->discard_block_size = discard_block_size;
1115 		cmd->discard_nr_blocks = new_nr_entries;
1116 	}
1117 
1118 	cmd->changed = true;
1119 	WRITE_UNLOCK(cmd);
1120 
1121 	return r;
1122 }
1123 
1124 static int __set_discard(struct dm_cache_metadata *cmd, dm_dblock_t b)
1125 {
1126 	return dm_bitset_set_bit(&cmd->discard_info, cmd->discard_root,
1127 				 from_dblock(b), &cmd->discard_root);
1128 }
1129 
1130 static int __clear_discard(struct dm_cache_metadata *cmd, dm_dblock_t b)
1131 {
1132 	return dm_bitset_clear_bit(&cmd->discard_info, cmd->discard_root,
1133 				   from_dblock(b), &cmd->discard_root);
1134 }
1135 
1136 static int __discard(struct dm_cache_metadata *cmd,
1137 		     dm_dblock_t dblock, bool discard)
1138 {
1139 	int r;
1140 
1141 	r = (discard ? __set_discard : __clear_discard)(cmd, dblock);
1142 	if (r)
1143 		return r;
1144 
1145 	cmd->changed = true;
1146 	return 0;
1147 }
1148 
1149 int dm_cache_set_discard(struct dm_cache_metadata *cmd,
1150 			 dm_dblock_t dblock, bool discard)
1151 {
1152 	int r;
1153 
1154 	WRITE_LOCK(cmd);
1155 	r = __discard(cmd, dblock, discard);
1156 	WRITE_UNLOCK(cmd);
1157 
1158 	return r;
1159 }
1160 
1161 static int __load_discards(struct dm_cache_metadata *cmd,
1162 			   load_discard_fn fn, void *context)
1163 {
1164 	int r = 0;
1165 	uint32_t b;
1166 	struct dm_bitset_cursor c;
1167 
1168 	if (from_dblock(cmd->discard_nr_blocks) == 0)
1169 		/* nothing to do */
1170 		return 0;
1171 
1172 	if (cmd->clean_when_opened) {
1173 		r = dm_bitset_flush(&cmd->discard_info, cmd->discard_root, &cmd->discard_root);
1174 		if (r)
1175 			return r;
1176 
1177 		r = dm_bitset_cursor_begin(&cmd->discard_info, cmd->discard_root,
1178 					   from_dblock(cmd->discard_nr_blocks), &c);
1179 		if (r)
1180 			return r;
1181 
1182 		for (b = 0; ; b++) {
1183 			r = fn(context, cmd->discard_block_size, to_dblock(b),
1184 			       dm_bitset_cursor_get_value(&c));
1185 			if (r)
1186 				break;
1187 
1188 			if (b >= (from_dblock(cmd->discard_nr_blocks) - 1))
1189 				break;
1190 
1191 			r = dm_bitset_cursor_next(&c);
1192 			if (r)
1193 				break;
1194 		}
1195 
1196 		dm_bitset_cursor_end(&c);
1197 
1198 	} else {
1199 		for (b = 0; b < from_dblock(cmd->discard_nr_blocks); b++) {
1200 			r = fn(context, cmd->discard_block_size, to_dblock(b), false);
1201 			if (r)
1202 				return r;
1203 		}
1204 	}
1205 
1206 	return r;
1207 }
1208 
1209 int dm_cache_load_discards(struct dm_cache_metadata *cmd,
1210 			   load_discard_fn fn, void *context)
1211 {
1212 	int r;
1213 
1214 	READ_LOCK(cmd);
1215 	r = __load_discards(cmd, fn, context);
1216 	READ_UNLOCK(cmd);
1217 
1218 	return r;
1219 }
1220 
1221 int dm_cache_size(struct dm_cache_metadata *cmd, dm_cblock_t *result)
1222 {
1223 	READ_LOCK(cmd);
1224 	*result = cmd->cache_blocks;
1225 	READ_UNLOCK(cmd);
1226 
1227 	return 0;
1228 }
1229 
1230 static int __remove(struct dm_cache_metadata *cmd, dm_cblock_t cblock)
1231 {
1232 	int r;
1233 	__le64 value = pack_value(0, 0);
1234 
1235 	__dm_bless_for_disk(&value);
1236 	r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),
1237 			       &value, &cmd->root);
1238 	if (r)
1239 		return r;
1240 
1241 	cmd->changed = true;
1242 	return 0;
1243 }
1244 
1245 int dm_cache_remove_mapping(struct dm_cache_metadata *cmd, dm_cblock_t cblock)
1246 {
1247 	int r;
1248 
1249 	WRITE_LOCK(cmd);
1250 	r = __remove(cmd, cblock);
1251 	WRITE_UNLOCK(cmd);
1252 
1253 	return r;
1254 }
1255 
1256 static int __insert(struct dm_cache_metadata *cmd,
1257 		    dm_cblock_t cblock, dm_oblock_t oblock)
1258 {
1259 	int r;
1260 	__le64 value = pack_value(oblock, M_VALID);
1261 
1262 	__dm_bless_for_disk(&value);
1263 
1264 	r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),
1265 			       &value, &cmd->root);
1266 	if (r)
1267 		return r;
1268 
1269 	cmd->changed = true;
1270 	return 0;
1271 }
1272 
1273 int dm_cache_insert_mapping(struct dm_cache_metadata *cmd,
1274 			    dm_cblock_t cblock, dm_oblock_t oblock)
1275 {
1276 	int r;
1277 
1278 	WRITE_LOCK(cmd);
1279 	r = __insert(cmd, cblock, oblock);
1280 	WRITE_UNLOCK(cmd);
1281 
1282 	return r;
1283 }
1284 
1285 struct thunk {
1286 	load_mapping_fn fn;
1287 	void *context;
1288 
1289 	struct dm_cache_metadata *cmd;
1290 	bool respect_dirty_flags;
1291 	bool hints_valid;
1292 };
1293 
1294 static bool policy_unchanged(struct dm_cache_metadata *cmd,
1295 			     struct dm_cache_policy *policy)
1296 {
1297 	const char *policy_name = dm_cache_policy_get_name(policy);
1298 	const unsigned int *policy_version = dm_cache_policy_get_version(policy);
1299 	size_t policy_hint_size = dm_cache_policy_get_hint_size(policy);
1300 
1301 	/*
1302 	 * Ensure policy names match.
1303 	 */
1304 	if (strncmp(cmd->policy_name, policy_name, sizeof(cmd->policy_name)))
1305 		return false;
1306 
1307 	/*
1308 	 * Ensure policy major versions match.
1309 	 */
1310 	if (cmd->policy_version[0] != policy_version[0])
1311 		return false;
1312 
1313 	/*
1314 	 * Ensure policy hint sizes match.
1315 	 */
1316 	if (cmd->policy_hint_size != policy_hint_size)
1317 		return false;
1318 
1319 	return true;
1320 }
1321 
1322 static bool hints_array_initialized(struct dm_cache_metadata *cmd)
1323 {
1324 	return cmd->hint_root && cmd->policy_hint_size;
1325 }
1326 
1327 static bool hints_array_available(struct dm_cache_metadata *cmd,
1328 				  struct dm_cache_policy *policy)
1329 {
1330 	return cmd->clean_when_opened && policy_unchanged(cmd, policy) &&
1331 		hints_array_initialized(cmd);
1332 }
1333 
1334 static int __load_mapping_v1(struct dm_cache_metadata *cmd,
1335 			     uint64_t cb, bool hints_valid,
1336 			     struct dm_array_cursor *mapping_cursor,
1337 			     struct dm_array_cursor *hint_cursor,
1338 			     load_mapping_fn fn, void *context)
1339 {
1340 	int r = 0;
1341 
1342 	__le64 mapping;
1343 	__le32 hint = 0;
1344 
1345 	__le64 *mapping_value_le;
1346 	__le32 *hint_value_le;
1347 
1348 	dm_oblock_t oblock;
1349 	unsigned int flags;
1350 	bool dirty = true;
1351 
1352 	dm_array_cursor_get_value(mapping_cursor, (void **) &mapping_value_le);
1353 	memcpy(&mapping, mapping_value_le, sizeof(mapping));
1354 	unpack_value(mapping, &oblock, &flags);
1355 
1356 	if (flags & M_VALID) {
1357 		if (hints_valid) {
1358 			dm_array_cursor_get_value(hint_cursor, (void **) &hint_value_le);
1359 			memcpy(&hint, hint_value_le, sizeof(hint));
1360 		}
1361 		if (cmd->clean_when_opened)
1362 			dirty = flags & M_DIRTY;
1363 
1364 		r = fn(context, oblock, to_cblock(cb), dirty,
1365 		       le32_to_cpu(hint), hints_valid);
1366 		if (r) {
1367 			DMERR("policy couldn't load cache block %llu",
1368 			      (unsigned long long) from_cblock(to_cblock(cb)));
1369 		}
1370 	}
1371 
1372 	return r;
1373 }
1374 
1375 static int __load_mapping_v2(struct dm_cache_metadata *cmd,
1376 			     uint64_t cb, bool hints_valid,
1377 			     struct dm_array_cursor *mapping_cursor,
1378 			     struct dm_array_cursor *hint_cursor,
1379 			     struct dm_bitset_cursor *dirty_cursor,
1380 			     load_mapping_fn fn, void *context)
1381 {
1382 	int r = 0;
1383 
1384 	__le64 mapping;
1385 	__le32 hint = 0;
1386 
1387 	__le64 *mapping_value_le;
1388 	__le32 *hint_value_le;
1389 
1390 	dm_oblock_t oblock;
1391 	unsigned int flags;
1392 	bool dirty = true;
1393 
1394 	dm_array_cursor_get_value(mapping_cursor, (void **) &mapping_value_le);
1395 	memcpy(&mapping, mapping_value_le, sizeof(mapping));
1396 	unpack_value(mapping, &oblock, &flags);
1397 
1398 	if (flags & M_VALID) {
1399 		if (hints_valid) {
1400 			dm_array_cursor_get_value(hint_cursor, (void **) &hint_value_le);
1401 			memcpy(&hint, hint_value_le, sizeof(hint));
1402 		}
1403 		if (cmd->clean_when_opened)
1404 			dirty = dm_bitset_cursor_get_value(dirty_cursor);
1405 
1406 		r = fn(context, oblock, to_cblock(cb), dirty,
1407 		       le32_to_cpu(hint), hints_valid);
1408 		if (r) {
1409 			DMERR("policy couldn't load cache block %llu",
1410 			      (unsigned long long) from_cblock(to_cblock(cb)));
1411 		}
1412 	}
1413 
1414 	return r;
1415 }
1416 
1417 static int __load_mappings(struct dm_cache_metadata *cmd,
1418 			   struct dm_cache_policy *policy,
1419 			   load_mapping_fn fn, void *context)
1420 {
1421 	int r;
1422 	uint64_t cb;
1423 
1424 	bool hints_valid = hints_array_available(cmd, policy);
1425 
1426 	if (from_cblock(cmd->cache_blocks) == 0)
1427 		/* Nothing to do */
1428 		return 0;
1429 
1430 	r = dm_array_cursor_begin(&cmd->info, cmd->root, &cmd->mapping_cursor);
1431 	if (r)
1432 		return r;
1433 
1434 	if (hints_valid) {
1435 		r = dm_array_cursor_begin(&cmd->hint_info, cmd->hint_root, &cmd->hint_cursor);
1436 		if (r) {
1437 			dm_array_cursor_end(&cmd->mapping_cursor);
1438 			return r;
1439 		}
1440 	}
1441 
1442 	if (separate_dirty_bits(cmd)) {
1443 		r = dm_bitset_cursor_begin(&cmd->dirty_info, cmd->dirty_root,
1444 					   from_cblock(cmd->cache_blocks),
1445 					   &cmd->dirty_cursor);
1446 		if (r) {
1447 			dm_array_cursor_end(&cmd->hint_cursor);
1448 			dm_array_cursor_end(&cmd->mapping_cursor);
1449 			return r;
1450 		}
1451 	}
1452 
1453 	for (cb = 0; ; cb++) {
1454 		if (separate_dirty_bits(cmd))
1455 			r = __load_mapping_v2(cmd, cb, hints_valid,
1456 					      &cmd->mapping_cursor,
1457 					      &cmd->hint_cursor,
1458 					      &cmd->dirty_cursor,
1459 					      fn, context);
1460 		else
1461 			r = __load_mapping_v1(cmd, cb, hints_valid,
1462 					      &cmd->mapping_cursor, &cmd->hint_cursor,
1463 					      fn, context);
1464 		if (r)
1465 			goto out;
1466 
1467 		/*
1468 		 * We need to break out before we move the cursors.
1469 		 */
1470 		if (cb >= (from_cblock(cmd->cache_blocks) - 1))
1471 			break;
1472 
1473 		r = dm_array_cursor_next(&cmd->mapping_cursor);
1474 		if (r) {
1475 			DMERR("dm_array_cursor_next for mapping failed");
1476 			goto out;
1477 		}
1478 
1479 		if (hints_valid) {
1480 			r = dm_array_cursor_next(&cmd->hint_cursor);
1481 			if (r) {
1482 				dm_array_cursor_end(&cmd->hint_cursor);
1483 				hints_valid = false;
1484 			}
1485 		}
1486 
1487 		if (separate_dirty_bits(cmd)) {
1488 			r = dm_bitset_cursor_next(&cmd->dirty_cursor);
1489 			if (r) {
1490 				DMERR("dm_bitset_cursor_next for dirty failed");
1491 				goto out;
1492 			}
1493 		}
1494 	}
1495 out:
1496 	dm_array_cursor_end(&cmd->mapping_cursor);
1497 	if (hints_valid)
1498 		dm_array_cursor_end(&cmd->hint_cursor);
1499 
1500 	if (separate_dirty_bits(cmd))
1501 		dm_bitset_cursor_end(&cmd->dirty_cursor);
1502 
1503 	return r;
1504 }
1505 
1506 int dm_cache_load_mappings(struct dm_cache_metadata *cmd,
1507 			   struct dm_cache_policy *policy,
1508 			   load_mapping_fn fn, void *context)
1509 {
1510 	int r;
1511 
1512 	READ_LOCK(cmd);
1513 	r = __load_mappings(cmd, policy, fn, context);
1514 	READ_UNLOCK(cmd);
1515 
1516 	return r;
1517 }
1518 
1519 static int __dump_mapping(void *context, uint64_t cblock, void *leaf)
1520 {
1521 	__le64 value;
1522 	dm_oblock_t oblock;
1523 	unsigned int flags;
1524 
1525 	memcpy(&value, leaf, sizeof(value));
1526 	unpack_value(value, &oblock, &flags);
1527 
1528 	return 0;
1529 }
1530 
1531 static int __dump_mappings(struct dm_cache_metadata *cmd)
1532 {
1533 	return dm_array_walk(&cmd->info, cmd->root, __dump_mapping, NULL);
1534 }
1535 
1536 void dm_cache_dump(struct dm_cache_metadata *cmd)
1537 {
1538 	READ_LOCK_VOID(cmd);
1539 	__dump_mappings(cmd);
1540 	READ_UNLOCK(cmd);
1541 }
1542 
1543 int dm_cache_changed_this_transaction(struct dm_cache_metadata *cmd)
1544 {
1545 	int r;
1546 
1547 	READ_LOCK(cmd);
1548 	r = cmd->changed;
1549 	READ_UNLOCK(cmd);
1550 
1551 	return r;
1552 }
1553 
1554 static int __dirty(struct dm_cache_metadata *cmd, dm_cblock_t cblock, bool dirty)
1555 {
1556 	int r;
1557 	unsigned int flags;
1558 	dm_oblock_t oblock;
1559 	__le64 value;
1560 
1561 	r = dm_array_get_value(&cmd->info, cmd->root, from_cblock(cblock), &value);
1562 	if (r)
1563 		return r;
1564 
1565 	unpack_value(value, &oblock, &flags);
1566 
1567 	if (((flags & M_DIRTY) && dirty) || (!(flags & M_DIRTY) && !dirty))
1568 		/* nothing to be done */
1569 		return 0;
1570 
1571 	value = pack_value(oblock, (flags & ~M_DIRTY) | (dirty ? M_DIRTY : 0));
1572 	__dm_bless_for_disk(&value);
1573 
1574 	r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),
1575 			       &value, &cmd->root);
1576 	if (r)
1577 		return r;
1578 
1579 	cmd->changed = true;
1580 	return 0;
1581 
1582 }
1583 
1584 static int __set_dirty_bits_v1(struct dm_cache_metadata *cmd, unsigned int nr_bits, unsigned long *bits)
1585 {
1586 	int r;
1587 	unsigned int i;
1588 
1589 	for (i = 0; i < nr_bits; i++) {
1590 		r = __dirty(cmd, to_cblock(i), test_bit(i, bits));
1591 		if (r)
1592 			return r;
1593 	}
1594 
1595 	return 0;
1596 }
1597 
1598 static int is_dirty_callback(uint32_t index, bool *value, void *context)
1599 {
1600 	unsigned long *bits = context;
1601 	*value = test_bit(index, bits);
1602 	return 0;
1603 }
1604 
1605 static int __set_dirty_bits_v2(struct dm_cache_metadata *cmd, unsigned int nr_bits, unsigned long *bits)
1606 {
1607 	int r = 0;
1608 
1609 	/* nr_bits is really just a sanity check */
1610 	if (nr_bits != from_cblock(cmd->cache_blocks)) {
1611 		DMERR("dirty bitset is wrong size");
1612 		return -EINVAL;
1613 	}
1614 
1615 	r = dm_bitset_del(&cmd->dirty_info, cmd->dirty_root);
1616 	if (r)
1617 		return r;
1618 
1619 	cmd->changed = true;
1620 	return dm_bitset_new(&cmd->dirty_info, &cmd->dirty_root, nr_bits, is_dirty_callback, bits);
1621 }
1622 
1623 int dm_cache_set_dirty_bits(struct dm_cache_metadata *cmd,
1624 			    unsigned int nr_bits,
1625 			    unsigned long *bits)
1626 {
1627 	int r;
1628 
1629 	WRITE_LOCK(cmd);
1630 	if (separate_dirty_bits(cmd))
1631 		r = __set_dirty_bits_v2(cmd, nr_bits, bits);
1632 	else
1633 		r = __set_dirty_bits_v1(cmd, nr_bits, bits);
1634 	WRITE_UNLOCK(cmd);
1635 
1636 	return r;
1637 }
1638 
1639 void dm_cache_metadata_get_stats(struct dm_cache_metadata *cmd,
1640 				 struct dm_cache_statistics *stats)
1641 {
1642 	READ_LOCK_VOID(cmd);
1643 	*stats = cmd->stats;
1644 	READ_UNLOCK(cmd);
1645 }
1646 
1647 void dm_cache_metadata_set_stats(struct dm_cache_metadata *cmd,
1648 				 struct dm_cache_statistics *stats)
1649 {
1650 	WRITE_LOCK_VOID(cmd);
1651 	cmd->stats = *stats;
1652 	WRITE_UNLOCK(cmd);
1653 }
1654 
1655 int dm_cache_commit(struct dm_cache_metadata *cmd, bool clean_shutdown)
1656 {
1657 	int r = -EINVAL;
1658 	flags_mutator mutator = (clean_shutdown ? set_clean_shutdown :
1659 				 clear_clean_shutdown);
1660 
1661 	WRITE_LOCK(cmd);
1662 	if (cmd->fail_io)
1663 		goto out;
1664 
1665 	r = __commit_transaction(cmd, mutator);
1666 	if (r)
1667 		goto out;
1668 
1669 	r = __begin_transaction(cmd);
1670 out:
1671 	WRITE_UNLOCK(cmd);
1672 	return r;
1673 }
1674 
1675 int dm_cache_get_free_metadata_block_count(struct dm_cache_metadata *cmd,
1676 					   dm_block_t *result)
1677 {
1678 	int r = -EINVAL;
1679 
1680 	READ_LOCK(cmd);
1681 	if (!cmd->fail_io)
1682 		r = dm_sm_get_nr_free(cmd->metadata_sm, result);
1683 	READ_UNLOCK(cmd);
1684 
1685 	return r;
1686 }
1687 
1688 int dm_cache_get_metadata_dev_size(struct dm_cache_metadata *cmd,
1689 				   dm_block_t *result)
1690 {
1691 	int r = -EINVAL;
1692 
1693 	READ_LOCK(cmd);
1694 	if (!cmd->fail_io)
1695 		r = dm_sm_get_nr_blocks(cmd->metadata_sm, result);
1696 	READ_UNLOCK(cmd);
1697 
1698 	return r;
1699 }
1700 
1701 /*----------------------------------------------------------------*/
1702 
1703 static int get_hint(uint32_t index, void *value_le, void *context)
1704 {
1705 	uint32_t value;
1706 	struct dm_cache_policy *policy = context;
1707 
1708 	value = policy_get_hint(policy, to_cblock(index));
1709 	*((__le32 *) value_le) = cpu_to_le32(value);
1710 
1711 	return 0;
1712 }
1713 
1714 /*
1715  * It's quicker to always delete the hint array, and recreate with
1716  * dm_array_new().
1717  */
1718 static int write_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy)
1719 {
1720 	int r;
1721 	size_t hint_size;
1722 	const char *policy_name = dm_cache_policy_get_name(policy);
1723 	const unsigned int *policy_version = dm_cache_policy_get_version(policy);
1724 
1725 	if (!policy_name[0] ||
1726 	    (strlen(policy_name) > sizeof(cmd->policy_name) - 1))
1727 		return -EINVAL;
1728 
1729 	strncpy(cmd->policy_name, policy_name, sizeof(cmd->policy_name));
1730 	memcpy(cmd->policy_version, policy_version, sizeof(cmd->policy_version));
1731 
1732 	hint_size = dm_cache_policy_get_hint_size(policy);
1733 	if (!hint_size)
1734 		return 0; /* short-circuit hints initialization */
1735 	cmd->policy_hint_size = hint_size;
1736 
1737 	if (cmd->hint_root) {
1738 		r = dm_array_del(&cmd->hint_info, cmd->hint_root);
1739 		if (r)
1740 			return r;
1741 	}
1742 
1743 	return dm_array_new(&cmd->hint_info, &cmd->hint_root,
1744 			    from_cblock(cmd->cache_blocks),
1745 			    get_hint, policy);
1746 }
1747 
1748 int dm_cache_write_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy)
1749 {
1750 	int r;
1751 
1752 	WRITE_LOCK(cmd);
1753 	r = write_hints(cmd, policy);
1754 	WRITE_UNLOCK(cmd);
1755 
1756 	return r;
1757 }
1758 
1759 int dm_cache_metadata_all_clean(struct dm_cache_metadata *cmd, bool *result)
1760 {
1761 	int r;
1762 
1763 	READ_LOCK(cmd);
1764 	r = blocks_are_unmapped_or_clean(cmd, 0, cmd->cache_blocks, result);
1765 	READ_UNLOCK(cmd);
1766 
1767 	return r;
1768 }
1769 
1770 void dm_cache_metadata_set_read_only(struct dm_cache_metadata *cmd)
1771 {
1772 	WRITE_LOCK_VOID(cmd);
1773 	dm_bm_set_read_only(cmd->bm);
1774 	WRITE_UNLOCK(cmd);
1775 }
1776 
1777 void dm_cache_metadata_set_read_write(struct dm_cache_metadata *cmd)
1778 {
1779 	WRITE_LOCK_VOID(cmd);
1780 	dm_bm_set_read_write(cmd->bm);
1781 	WRITE_UNLOCK(cmd);
1782 }
1783 
1784 int dm_cache_metadata_set_needs_check(struct dm_cache_metadata *cmd)
1785 {
1786 	int r;
1787 	struct dm_block *sblock;
1788 	struct cache_disk_superblock *disk_super;
1789 
1790 	WRITE_LOCK(cmd);
1791 	set_bit(NEEDS_CHECK, &cmd->flags);
1792 
1793 	r = superblock_lock(cmd, &sblock);
1794 	if (r) {
1795 		DMERR("couldn't read superblock");
1796 		goto out;
1797 	}
1798 
1799 	disk_super = dm_block_data(sblock);
1800 	disk_super->flags = cpu_to_le32(cmd->flags);
1801 
1802 	dm_bm_unlock(sblock);
1803 
1804 out:
1805 	WRITE_UNLOCK(cmd);
1806 	return r;
1807 }
1808 
1809 int dm_cache_metadata_needs_check(struct dm_cache_metadata *cmd, bool *result)
1810 {
1811 	READ_LOCK(cmd);
1812 	*result = !!test_bit(NEEDS_CHECK, &cmd->flags);
1813 	READ_UNLOCK(cmd);
1814 
1815 	return 0;
1816 }
1817 
1818 int dm_cache_metadata_abort(struct dm_cache_metadata *cmd)
1819 {
1820 	int r = -EINVAL;
1821 	struct dm_block_manager *old_bm = NULL, *new_bm = NULL;
1822 
1823 	/* fail_io is double-checked with cmd->root_lock held below */
1824 	if (unlikely(cmd->fail_io))
1825 		return r;
1826 
1827 	/*
1828 	 * Replacement block manager (new_bm) is created and old_bm destroyed outside of
1829 	 * cmd root_lock to avoid ABBA deadlock that would result (due to life-cycle of
1830 	 * shrinker associated with the block manager's bufio client vs cmd root_lock).
1831 	 * - must take shrinker_rwsem without holding cmd->root_lock
1832 	 */
1833 	new_bm = dm_block_manager_create(cmd->bdev, DM_CACHE_METADATA_BLOCK_SIZE << SECTOR_SHIFT,
1834 					 CACHE_MAX_CONCURRENT_LOCKS);
1835 
1836 	WRITE_LOCK(cmd);
1837 	if (cmd->fail_io) {
1838 		WRITE_UNLOCK(cmd);
1839 		goto out;
1840 	}
1841 
1842 	__destroy_persistent_data_objects(cmd, false);
1843 	old_bm = cmd->bm;
1844 	if (IS_ERR(new_bm)) {
1845 		DMERR("could not create block manager during abort");
1846 		cmd->bm = NULL;
1847 		r = PTR_ERR(new_bm);
1848 		goto out_unlock;
1849 	}
1850 
1851 	cmd->bm = new_bm;
1852 	r = __open_or_format_metadata(cmd, false);
1853 	if (r) {
1854 		cmd->bm = NULL;
1855 		goto out_unlock;
1856 	}
1857 	new_bm = NULL;
1858 out_unlock:
1859 	if (r)
1860 		cmd->fail_io = true;
1861 	WRITE_UNLOCK(cmd);
1862 	dm_block_manager_destroy(old_bm);
1863 out:
1864 	if (new_bm && !IS_ERR(new_bm))
1865 		dm_block_manager_destroy(new_bm);
1866 
1867 	return r;
1868 }
1869