1 /*
2  * Copyright (C) 2012 Red Hat, Inc.
3  *
4  * This file is released under the GPL.
5  */
6 
7 #include "dm-cache-metadata.h"
8 
9 #include "persistent-data/dm-array.h"
10 #include "persistent-data/dm-bitset.h"
11 #include "persistent-data/dm-space-map.h"
12 #include "persistent-data/dm-space-map-disk.h"
13 #include "persistent-data/dm-transaction-manager.h"
14 
15 #include <linux/device-mapper.h>
16 
17 /*----------------------------------------------------------------*/
18 
19 #define DM_MSG_PREFIX   "cache metadata"
20 
21 #define CACHE_SUPERBLOCK_MAGIC 06142003
22 #define CACHE_SUPERBLOCK_LOCATION 0
23 
24 /*
25  * defines a range of metadata versions that this module can handle.
26  */
27 #define MIN_CACHE_VERSION 1
28 #define MAX_CACHE_VERSION 1
29 
30 #define CACHE_METADATA_CACHE_SIZE 64
31 
32 /*
33  *  3 for btree insert +
34  *  2 for btree lookup used within space map
35  */
36 #define CACHE_MAX_CONCURRENT_LOCKS 5
37 #define SPACE_MAP_ROOT_SIZE 128
38 
39 enum superblock_flag_bits {
40 	/* for spotting crashes that would invalidate the dirty bitset */
41 	CLEAN_SHUTDOWN,
42 	/* metadata must be checked using the tools */
43 	NEEDS_CHECK,
44 };
45 
46 /*
47  * Each mapping from cache block -> origin block carries a set of flags.
48  */
49 enum mapping_bits {
50 	/*
51 	 * A valid mapping.  Because we're using an array we clear this
52 	 * flag for an non existant mapping.
53 	 */
54 	M_VALID = 1,
55 
56 	/*
57 	 * The data on the cache is different from that on the origin.
58 	 */
59 	M_DIRTY = 2
60 };
61 
62 struct cache_disk_superblock {
63 	__le32 csum;
64 	__le32 flags;
65 	__le64 blocknr;
66 
67 	__u8 uuid[16];
68 	__le64 magic;
69 	__le32 version;
70 
71 	__u8 policy_name[CACHE_POLICY_NAME_SIZE];
72 	__le32 policy_hint_size;
73 
74 	__u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE];
75 	__le64 mapping_root;
76 	__le64 hint_root;
77 
78 	__le64 discard_root;
79 	__le64 discard_block_size;
80 	__le64 discard_nr_blocks;
81 
82 	__le32 data_block_size;
83 	__le32 metadata_block_size;
84 	__le32 cache_blocks;
85 
86 	__le32 compat_flags;
87 	__le32 compat_ro_flags;
88 	__le32 incompat_flags;
89 
90 	__le32 read_hits;
91 	__le32 read_misses;
92 	__le32 write_hits;
93 	__le32 write_misses;
94 
95 	__le32 policy_version[CACHE_POLICY_VERSION_SIZE];
96 } __packed;
97 
98 struct dm_cache_metadata {
99 	atomic_t ref_count;
100 	struct list_head list;
101 
102 	struct block_device *bdev;
103 	struct dm_block_manager *bm;
104 	struct dm_space_map *metadata_sm;
105 	struct dm_transaction_manager *tm;
106 
107 	struct dm_array_info info;
108 	struct dm_array_info hint_info;
109 	struct dm_disk_bitset discard_info;
110 
111 	struct rw_semaphore root_lock;
112 	unsigned long flags;
113 	dm_block_t root;
114 	dm_block_t hint_root;
115 	dm_block_t discard_root;
116 
117 	sector_t discard_block_size;
118 	dm_dblock_t discard_nr_blocks;
119 
120 	sector_t data_block_size;
121 	dm_cblock_t cache_blocks;
122 	bool changed:1;
123 	bool clean_when_opened:1;
124 
125 	char policy_name[CACHE_POLICY_NAME_SIZE];
126 	unsigned policy_version[CACHE_POLICY_VERSION_SIZE];
127 	size_t policy_hint_size;
128 	struct dm_cache_statistics stats;
129 
130 	/*
131 	 * Reading the space map root can fail, so we read it into this
132 	 * buffer before the superblock is locked and updated.
133 	 */
134 	__u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE];
135 
136 	/*
137 	 * Set if a transaction has to be aborted but the attempt to roll
138 	 * back to the previous (good) transaction failed.  The only
139 	 * metadata operation permissible in this state is the closing of
140 	 * the device.
141 	 */
142 	bool fail_io:1;
143 };
144 
145 /*-------------------------------------------------------------------
146  * superblock validator
147  *-----------------------------------------------------------------*/
148 
149 #define SUPERBLOCK_CSUM_XOR 9031977
150 
151 static void sb_prepare_for_write(struct dm_block_validator *v,
152 				 struct dm_block *b,
153 				 size_t sb_block_size)
154 {
155 	struct cache_disk_superblock *disk_super = dm_block_data(b);
156 
157 	disk_super->blocknr = cpu_to_le64(dm_block_location(b));
158 	disk_super->csum = cpu_to_le32(dm_bm_checksum(&disk_super->flags,
159 						      sb_block_size - sizeof(__le32),
160 						      SUPERBLOCK_CSUM_XOR));
161 }
162 
163 static int check_metadata_version(struct cache_disk_superblock *disk_super)
164 {
165 	uint32_t metadata_version = le32_to_cpu(disk_super->version);
166 	if (metadata_version < MIN_CACHE_VERSION || metadata_version > MAX_CACHE_VERSION) {
167 		DMERR("Cache metadata version %u found, but only versions between %u and %u supported.",
168 		      metadata_version, MIN_CACHE_VERSION, MAX_CACHE_VERSION);
169 		return -EINVAL;
170 	}
171 
172 	return 0;
173 }
174 
175 static int sb_check(struct dm_block_validator *v,
176 		    struct dm_block *b,
177 		    size_t sb_block_size)
178 {
179 	struct cache_disk_superblock *disk_super = dm_block_data(b);
180 	__le32 csum_le;
181 
182 	if (dm_block_location(b) != le64_to_cpu(disk_super->blocknr)) {
183 		DMERR("sb_check failed: blocknr %llu: wanted %llu",
184 		      le64_to_cpu(disk_super->blocknr),
185 		      (unsigned long long)dm_block_location(b));
186 		return -ENOTBLK;
187 	}
188 
189 	if (le64_to_cpu(disk_super->magic) != CACHE_SUPERBLOCK_MAGIC) {
190 		DMERR("sb_check failed: magic %llu: wanted %llu",
191 		      le64_to_cpu(disk_super->magic),
192 		      (unsigned long long)CACHE_SUPERBLOCK_MAGIC);
193 		return -EILSEQ;
194 	}
195 
196 	csum_le = cpu_to_le32(dm_bm_checksum(&disk_super->flags,
197 					     sb_block_size - sizeof(__le32),
198 					     SUPERBLOCK_CSUM_XOR));
199 	if (csum_le != disk_super->csum) {
200 		DMERR("sb_check failed: csum %u: wanted %u",
201 		      le32_to_cpu(csum_le), le32_to_cpu(disk_super->csum));
202 		return -EILSEQ;
203 	}
204 
205 	return check_metadata_version(disk_super);
206 }
207 
208 static struct dm_block_validator sb_validator = {
209 	.name = "superblock",
210 	.prepare_for_write = sb_prepare_for_write,
211 	.check = sb_check
212 };
213 
214 /*----------------------------------------------------------------*/
215 
216 static int superblock_read_lock(struct dm_cache_metadata *cmd,
217 				struct dm_block **sblock)
218 {
219 	return dm_bm_read_lock(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
220 			       &sb_validator, sblock);
221 }
222 
223 static int superblock_lock_zero(struct dm_cache_metadata *cmd,
224 				struct dm_block **sblock)
225 {
226 	return dm_bm_write_lock_zero(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
227 				     &sb_validator, sblock);
228 }
229 
230 static int superblock_lock(struct dm_cache_metadata *cmd,
231 			   struct dm_block **sblock)
232 {
233 	return dm_bm_write_lock(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
234 				&sb_validator, sblock);
235 }
236 
237 /*----------------------------------------------------------------*/
238 
239 static int __superblock_all_zeroes(struct dm_block_manager *bm, bool *result)
240 {
241 	int r;
242 	unsigned i;
243 	struct dm_block *b;
244 	__le64 *data_le, zero = cpu_to_le64(0);
245 	unsigned sb_block_size = dm_bm_block_size(bm) / sizeof(__le64);
246 
247 	/*
248 	 * We can't use a validator here - it may be all zeroes.
249 	 */
250 	r = dm_bm_read_lock(bm, CACHE_SUPERBLOCK_LOCATION, NULL, &b);
251 	if (r)
252 		return r;
253 
254 	data_le = dm_block_data(b);
255 	*result = true;
256 	for (i = 0; i < sb_block_size; i++) {
257 		if (data_le[i] != zero) {
258 			*result = false;
259 			break;
260 		}
261 	}
262 
263 	dm_bm_unlock(b);
264 
265 	return 0;
266 }
267 
268 static void __setup_mapping_info(struct dm_cache_metadata *cmd)
269 {
270 	struct dm_btree_value_type vt;
271 
272 	vt.context = NULL;
273 	vt.size = sizeof(__le64);
274 	vt.inc = NULL;
275 	vt.dec = NULL;
276 	vt.equal = NULL;
277 	dm_array_info_init(&cmd->info, cmd->tm, &vt);
278 
279 	if (cmd->policy_hint_size) {
280 		vt.size = sizeof(__le32);
281 		dm_array_info_init(&cmd->hint_info, cmd->tm, &vt);
282 	}
283 }
284 
285 static int __save_sm_root(struct dm_cache_metadata *cmd)
286 {
287 	int r;
288 	size_t metadata_len;
289 
290 	r = dm_sm_root_size(cmd->metadata_sm, &metadata_len);
291 	if (r < 0)
292 		return r;
293 
294 	return dm_sm_copy_root(cmd->metadata_sm, &cmd->metadata_space_map_root,
295 			       metadata_len);
296 }
297 
298 static void __copy_sm_root(struct dm_cache_metadata *cmd,
299 			   struct cache_disk_superblock *disk_super)
300 {
301 	memcpy(&disk_super->metadata_space_map_root,
302 	       &cmd->metadata_space_map_root,
303 	       sizeof(cmd->metadata_space_map_root));
304 }
305 
306 static int __write_initial_superblock(struct dm_cache_metadata *cmd)
307 {
308 	int r;
309 	struct dm_block *sblock;
310 	struct cache_disk_superblock *disk_super;
311 	sector_t bdev_size = i_size_read(cmd->bdev->bd_inode) >> SECTOR_SHIFT;
312 
313 	/* FIXME: see if we can lose the max sectors limit */
314 	if (bdev_size > DM_CACHE_METADATA_MAX_SECTORS)
315 		bdev_size = DM_CACHE_METADATA_MAX_SECTORS;
316 
317 	r = dm_tm_pre_commit(cmd->tm);
318 	if (r < 0)
319 		return r;
320 
321 	/*
322 	 * dm_sm_copy_root() can fail.  So we need to do it before we start
323 	 * updating the superblock.
324 	 */
325 	r = __save_sm_root(cmd);
326 	if (r)
327 		return r;
328 
329 	r = superblock_lock_zero(cmd, &sblock);
330 	if (r)
331 		return r;
332 
333 	disk_super = dm_block_data(sblock);
334 	disk_super->flags = 0;
335 	memset(disk_super->uuid, 0, sizeof(disk_super->uuid));
336 	disk_super->magic = cpu_to_le64(CACHE_SUPERBLOCK_MAGIC);
337 	disk_super->version = cpu_to_le32(MAX_CACHE_VERSION);
338 	memset(disk_super->policy_name, 0, sizeof(disk_super->policy_name));
339 	memset(disk_super->policy_version, 0, sizeof(disk_super->policy_version));
340 	disk_super->policy_hint_size = 0;
341 
342 	__copy_sm_root(cmd, disk_super);
343 
344 	disk_super->mapping_root = cpu_to_le64(cmd->root);
345 	disk_super->hint_root = cpu_to_le64(cmd->hint_root);
346 	disk_super->discard_root = cpu_to_le64(cmd->discard_root);
347 	disk_super->discard_block_size = cpu_to_le64(cmd->discard_block_size);
348 	disk_super->discard_nr_blocks = cpu_to_le64(from_dblock(cmd->discard_nr_blocks));
349 	disk_super->metadata_block_size = cpu_to_le32(DM_CACHE_METADATA_BLOCK_SIZE);
350 	disk_super->data_block_size = cpu_to_le32(cmd->data_block_size);
351 	disk_super->cache_blocks = cpu_to_le32(0);
352 
353 	disk_super->read_hits = cpu_to_le32(0);
354 	disk_super->read_misses = cpu_to_le32(0);
355 	disk_super->write_hits = cpu_to_le32(0);
356 	disk_super->write_misses = cpu_to_le32(0);
357 
358 	return dm_tm_commit(cmd->tm, sblock);
359 }
360 
361 static int __format_metadata(struct dm_cache_metadata *cmd)
362 {
363 	int r;
364 
365 	r = dm_tm_create_with_sm(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
366 				 &cmd->tm, &cmd->metadata_sm);
367 	if (r < 0) {
368 		DMERR("tm_create_with_sm failed");
369 		return r;
370 	}
371 
372 	__setup_mapping_info(cmd);
373 
374 	r = dm_array_empty(&cmd->info, &cmd->root);
375 	if (r < 0)
376 		goto bad;
377 
378 	dm_disk_bitset_init(cmd->tm, &cmd->discard_info);
379 
380 	r = dm_bitset_empty(&cmd->discard_info, &cmd->discard_root);
381 	if (r < 0)
382 		goto bad;
383 
384 	cmd->discard_block_size = 0;
385 	cmd->discard_nr_blocks = 0;
386 
387 	r = __write_initial_superblock(cmd);
388 	if (r)
389 		goto bad;
390 
391 	cmd->clean_when_opened = true;
392 	return 0;
393 
394 bad:
395 	dm_tm_destroy(cmd->tm);
396 	dm_sm_destroy(cmd->metadata_sm);
397 
398 	return r;
399 }
400 
401 static int __check_incompat_features(struct cache_disk_superblock *disk_super,
402 				     struct dm_cache_metadata *cmd)
403 {
404 	uint32_t features;
405 
406 	features = le32_to_cpu(disk_super->incompat_flags) & ~DM_CACHE_FEATURE_INCOMPAT_SUPP;
407 	if (features) {
408 		DMERR("could not access metadata due to unsupported optional features (%lx).",
409 		      (unsigned long)features);
410 		return -EINVAL;
411 	}
412 
413 	/*
414 	 * Check for read-only metadata to skip the following RDWR checks.
415 	 */
416 	if (get_disk_ro(cmd->bdev->bd_disk))
417 		return 0;
418 
419 	features = le32_to_cpu(disk_super->compat_ro_flags) & ~DM_CACHE_FEATURE_COMPAT_RO_SUPP;
420 	if (features) {
421 		DMERR("could not access metadata RDWR due to unsupported optional features (%lx).",
422 		      (unsigned long)features);
423 		return -EINVAL;
424 	}
425 
426 	return 0;
427 }
428 
429 static int __open_metadata(struct dm_cache_metadata *cmd)
430 {
431 	int r;
432 	struct dm_block *sblock;
433 	struct cache_disk_superblock *disk_super;
434 	unsigned long sb_flags;
435 
436 	r = superblock_read_lock(cmd, &sblock);
437 	if (r < 0) {
438 		DMERR("couldn't read lock superblock");
439 		return r;
440 	}
441 
442 	disk_super = dm_block_data(sblock);
443 
444 	/* Verify the data block size hasn't changed */
445 	if (le32_to_cpu(disk_super->data_block_size) != cmd->data_block_size) {
446 		DMERR("changing the data block size (from %u to %llu) is not supported",
447 		      le32_to_cpu(disk_super->data_block_size),
448 		      (unsigned long long)cmd->data_block_size);
449 		r = -EINVAL;
450 		goto bad;
451 	}
452 
453 	r = __check_incompat_features(disk_super, cmd);
454 	if (r < 0)
455 		goto bad;
456 
457 	r = dm_tm_open_with_sm(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
458 			       disk_super->metadata_space_map_root,
459 			       sizeof(disk_super->metadata_space_map_root),
460 			       &cmd->tm, &cmd->metadata_sm);
461 	if (r < 0) {
462 		DMERR("tm_open_with_sm failed");
463 		goto bad;
464 	}
465 
466 	__setup_mapping_info(cmd);
467 	dm_disk_bitset_init(cmd->tm, &cmd->discard_info);
468 	sb_flags = le32_to_cpu(disk_super->flags);
469 	cmd->clean_when_opened = test_bit(CLEAN_SHUTDOWN, &sb_flags);
470 	dm_bm_unlock(sblock);
471 
472 	return 0;
473 
474 bad:
475 	dm_bm_unlock(sblock);
476 	return r;
477 }
478 
479 static int __open_or_format_metadata(struct dm_cache_metadata *cmd,
480 				     bool format_device)
481 {
482 	int r;
483 	bool unformatted = false;
484 
485 	r = __superblock_all_zeroes(cmd->bm, &unformatted);
486 	if (r)
487 		return r;
488 
489 	if (unformatted)
490 		return format_device ? __format_metadata(cmd) : -EPERM;
491 
492 	return __open_metadata(cmd);
493 }
494 
495 static int __create_persistent_data_objects(struct dm_cache_metadata *cmd,
496 					    bool may_format_device)
497 {
498 	int r;
499 	cmd->bm = dm_block_manager_create(cmd->bdev, DM_CACHE_METADATA_BLOCK_SIZE << SECTOR_SHIFT,
500 					  CACHE_METADATA_CACHE_SIZE,
501 					  CACHE_MAX_CONCURRENT_LOCKS);
502 	if (IS_ERR(cmd->bm)) {
503 		DMERR("could not create block manager");
504 		return PTR_ERR(cmd->bm);
505 	}
506 
507 	r = __open_or_format_metadata(cmd, may_format_device);
508 	if (r)
509 		dm_block_manager_destroy(cmd->bm);
510 
511 	return r;
512 }
513 
514 static void __destroy_persistent_data_objects(struct dm_cache_metadata *cmd)
515 {
516 	dm_sm_destroy(cmd->metadata_sm);
517 	dm_tm_destroy(cmd->tm);
518 	dm_block_manager_destroy(cmd->bm);
519 }
520 
521 typedef unsigned long (*flags_mutator)(unsigned long);
522 
523 static void update_flags(struct cache_disk_superblock *disk_super,
524 			 flags_mutator mutator)
525 {
526 	uint32_t sb_flags = mutator(le32_to_cpu(disk_super->flags));
527 	disk_super->flags = cpu_to_le32(sb_flags);
528 }
529 
530 static unsigned long set_clean_shutdown(unsigned long flags)
531 {
532 	set_bit(CLEAN_SHUTDOWN, &flags);
533 	return flags;
534 }
535 
536 static unsigned long clear_clean_shutdown(unsigned long flags)
537 {
538 	clear_bit(CLEAN_SHUTDOWN, &flags);
539 	return flags;
540 }
541 
542 static void read_superblock_fields(struct dm_cache_metadata *cmd,
543 				   struct cache_disk_superblock *disk_super)
544 {
545 	cmd->flags = le32_to_cpu(disk_super->flags);
546 	cmd->root = le64_to_cpu(disk_super->mapping_root);
547 	cmd->hint_root = le64_to_cpu(disk_super->hint_root);
548 	cmd->discard_root = le64_to_cpu(disk_super->discard_root);
549 	cmd->discard_block_size = le64_to_cpu(disk_super->discard_block_size);
550 	cmd->discard_nr_blocks = to_dblock(le64_to_cpu(disk_super->discard_nr_blocks));
551 	cmd->data_block_size = le32_to_cpu(disk_super->data_block_size);
552 	cmd->cache_blocks = to_cblock(le32_to_cpu(disk_super->cache_blocks));
553 	strncpy(cmd->policy_name, disk_super->policy_name, sizeof(cmd->policy_name));
554 	cmd->policy_version[0] = le32_to_cpu(disk_super->policy_version[0]);
555 	cmd->policy_version[1] = le32_to_cpu(disk_super->policy_version[1]);
556 	cmd->policy_version[2] = le32_to_cpu(disk_super->policy_version[2]);
557 	cmd->policy_hint_size = le32_to_cpu(disk_super->policy_hint_size);
558 
559 	cmd->stats.read_hits = le32_to_cpu(disk_super->read_hits);
560 	cmd->stats.read_misses = le32_to_cpu(disk_super->read_misses);
561 	cmd->stats.write_hits = le32_to_cpu(disk_super->write_hits);
562 	cmd->stats.write_misses = le32_to_cpu(disk_super->write_misses);
563 
564 	cmd->changed = false;
565 }
566 
567 /*
568  * The mutator updates the superblock flags.
569  */
570 static int __begin_transaction_flags(struct dm_cache_metadata *cmd,
571 				     flags_mutator mutator)
572 {
573 	int r;
574 	struct cache_disk_superblock *disk_super;
575 	struct dm_block *sblock;
576 
577 	r = superblock_lock(cmd, &sblock);
578 	if (r)
579 		return r;
580 
581 	disk_super = dm_block_data(sblock);
582 	update_flags(disk_super, mutator);
583 	read_superblock_fields(cmd, disk_super);
584 	dm_bm_unlock(sblock);
585 
586 	return dm_bm_flush(cmd->bm);
587 }
588 
589 static int __begin_transaction(struct dm_cache_metadata *cmd)
590 {
591 	int r;
592 	struct cache_disk_superblock *disk_super;
593 	struct dm_block *sblock;
594 
595 	/*
596 	 * We re-read the superblock every time.  Shouldn't need to do this
597 	 * really.
598 	 */
599 	r = superblock_read_lock(cmd, &sblock);
600 	if (r)
601 		return r;
602 
603 	disk_super = dm_block_data(sblock);
604 	read_superblock_fields(cmd, disk_super);
605 	dm_bm_unlock(sblock);
606 
607 	return 0;
608 }
609 
610 static int __commit_transaction(struct dm_cache_metadata *cmd,
611 				flags_mutator mutator)
612 {
613 	int r;
614 	struct cache_disk_superblock *disk_super;
615 	struct dm_block *sblock;
616 
617 	/*
618 	 * We need to know if the cache_disk_superblock exceeds a 512-byte sector.
619 	 */
620 	BUILD_BUG_ON(sizeof(struct cache_disk_superblock) > 512);
621 
622 	r = dm_bitset_flush(&cmd->discard_info, cmd->discard_root,
623 			    &cmd->discard_root);
624 	if (r)
625 		return r;
626 
627 	r = dm_tm_pre_commit(cmd->tm);
628 	if (r < 0)
629 		return r;
630 
631 	r = __save_sm_root(cmd);
632 	if (r)
633 		return r;
634 
635 	r = superblock_lock(cmd, &sblock);
636 	if (r)
637 		return r;
638 
639 	disk_super = dm_block_data(sblock);
640 
641 	disk_super->flags = cpu_to_le32(cmd->flags);
642 	if (mutator)
643 		update_flags(disk_super, mutator);
644 
645 	disk_super->mapping_root = cpu_to_le64(cmd->root);
646 	disk_super->hint_root = cpu_to_le64(cmd->hint_root);
647 	disk_super->discard_root = cpu_to_le64(cmd->discard_root);
648 	disk_super->discard_block_size = cpu_to_le64(cmd->discard_block_size);
649 	disk_super->discard_nr_blocks = cpu_to_le64(from_dblock(cmd->discard_nr_blocks));
650 	disk_super->cache_blocks = cpu_to_le32(from_cblock(cmd->cache_blocks));
651 	strncpy(disk_super->policy_name, cmd->policy_name, sizeof(disk_super->policy_name));
652 	disk_super->policy_version[0] = cpu_to_le32(cmd->policy_version[0]);
653 	disk_super->policy_version[1] = cpu_to_le32(cmd->policy_version[1]);
654 	disk_super->policy_version[2] = cpu_to_le32(cmd->policy_version[2]);
655 
656 	disk_super->read_hits = cpu_to_le32(cmd->stats.read_hits);
657 	disk_super->read_misses = cpu_to_le32(cmd->stats.read_misses);
658 	disk_super->write_hits = cpu_to_le32(cmd->stats.write_hits);
659 	disk_super->write_misses = cpu_to_le32(cmd->stats.write_misses);
660 	__copy_sm_root(cmd, disk_super);
661 
662 	return dm_tm_commit(cmd->tm, sblock);
663 }
664 
665 /*----------------------------------------------------------------*/
666 
667 /*
668  * The mappings are held in a dm-array that has 64-bit values stored in
669  * little-endian format.  The index is the cblock, the high 48bits of the
670  * value are the oblock and the low 16 bit the flags.
671  */
672 #define FLAGS_MASK ((1 << 16) - 1)
673 
674 static __le64 pack_value(dm_oblock_t block, unsigned flags)
675 {
676 	uint64_t value = from_oblock(block);
677 	value <<= 16;
678 	value = value | (flags & FLAGS_MASK);
679 	return cpu_to_le64(value);
680 }
681 
682 static void unpack_value(__le64 value_le, dm_oblock_t *block, unsigned *flags)
683 {
684 	uint64_t value = le64_to_cpu(value_le);
685 	uint64_t b = value >> 16;
686 	*block = to_oblock(b);
687 	*flags = value & FLAGS_MASK;
688 }
689 
690 /*----------------------------------------------------------------*/
691 
692 static struct dm_cache_metadata *metadata_open(struct block_device *bdev,
693 					       sector_t data_block_size,
694 					       bool may_format_device,
695 					       size_t policy_hint_size)
696 {
697 	int r;
698 	struct dm_cache_metadata *cmd;
699 
700 	cmd = kzalloc(sizeof(*cmd), GFP_KERNEL);
701 	if (!cmd) {
702 		DMERR("could not allocate metadata struct");
703 		return ERR_PTR(-ENOMEM);
704 	}
705 
706 	atomic_set(&cmd->ref_count, 1);
707 	init_rwsem(&cmd->root_lock);
708 	cmd->bdev = bdev;
709 	cmd->data_block_size = data_block_size;
710 	cmd->cache_blocks = 0;
711 	cmd->policy_hint_size = policy_hint_size;
712 	cmd->changed = true;
713 	cmd->fail_io = false;
714 
715 	r = __create_persistent_data_objects(cmd, may_format_device);
716 	if (r) {
717 		kfree(cmd);
718 		return ERR_PTR(r);
719 	}
720 
721 	r = __begin_transaction_flags(cmd, clear_clean_shutdown);
722 	if (r < 0) {
723 		dm_cache_metadata_close(cmd);
724 		return ERR_PTR(r);
725 	}
726 
727 	return cmd;
728 }
729 
730 /*
731  * We keep a little list of ref counted metadata objects to prevent two
732  * different target instances creating separate bufio instances.  This is
733  * an issue if a table is reloaded before the suspend.
734  */
735 static DEFINE_MUTEX(table_lock);
736 static LIST_HEAD(table);
737 
738 static struct dm_cache_metadata *lookup(struct block_device *bdev)
739 {
740 	struct dm_cache_metadata *cmd;
741 
742 	list_for_each_entry(cmd, &table, list)
743 		if (cmd->bdev == bdev) {
744 			atomic_inc(&cmd->ref_count);
745 			return cmd;
746 		}
747 
748 	return NULL;
749 }
750 
751 static struct dm_cache_metadata *lookup_or_open(struct block_device *bdev,
752 						sector_t data_block_size,
753 						bool may_format_device,
754 						size_t policy_hint_size)
755 {
756 	struct dm_cache_metadata *cmd, *cmd2;
757 
758 	mutex_lock(&table_lock);
759 	cmd = lookup(bdev);
760 	mutex_unlock(&table_lock);
761 
762 	if (cmd)
763 		return cmd;
764 
765 	cmd = metadata_open(bdev, data_block_size, may_format_device, policy_hint_size);
766 	if (!IS_ERR(cmd)) {
767 		mutex_lock(&table_lock);
768 		cmd2 = lookup(bdev);
769 		if (cmd2) {
770 			mutex_unlock(&table_lock);
771 			__destroy_persistent_data_objects(cmd);
772 			kfree(cmd);
773 			return cmd2;
774 		}
775 		list_add(&cmd->list, &table);
776 		mutex_unlock(&table_lock);
777 	}
778 
779 	return cmd;
780 }
781 
782 static bool same_params(struct dm_cache_metadata *cmd, sector_t data_block_size)
783 {
784 	if (cmd->data_block_size != data_block_size) {
785 		DMERR("data_block_size (%llu) different from that in metadata (%llu)\n",
786 		      (unsigned long long) data_block_size,
787 		      (unsigned long long) cmd->data_block_size);
788 		return false;
789 	}
790 
791 	return true;
792 }
793 
794 struct dm_cache_metadata *dm_cache_metadata_open(struct block_device *bdev,
795 						 sector_t data_block_size,
796 						 bool may_format_device,
797 						 size_t policy_hint_size)
798 {
799 	struct dm_cache_metadata *cmd = lookup_or_open(bdev, data_block_size,
800 						       may_format_device, policy_hint_size);
801 
802 	if (!IS_ERR(cmd) && !same_params(cmd, data_block_size)) {
803 		dm_cache_metadata_close(cmd);
804 		return ERR_PTR(-EINVAL);
805 	}
806 
807 	return cmd;
808 }
809 
810 void dm_cache_metadata_close(struct dm_cache_metadata *cmd)
811 {
812 	if (atomic_dec_and_test(&cmd->ref_count)) {
813 		mutex_lock(&table_lock);
814 		list_del(&cmd->list);
815 		mutex_unlock(&table_lock);
816 
817 		if (!cmd->fail_io)
818 			__destroy_persistent_data_objects(cmd);
819 		kfree(cmd);
820 	}
821 }
822 
823 /*
824  * Checks that the given cache block is either unmapped or clean.
825  */
826 static int block_unmapped_or_clean(struct dm_cache_metadata *cmd, dm_cblock_t b,
827 				   bool *result)
828 {
829 	int r;
830 	__le64 value;
831 	dm_oblock_t ob;
832 	unsigned flags;
833 
834 	r = dm_array_get_value(&cmd->info, cmd->root, from_cblock(b), &value);
835 	if (r) {
836 		DMERR("block_unmapped_or_clean failed");
837 		return r;
838 	}
839 
840 	unpack_value(value, &ob, &flags);
841 	*result = !((flags & M_VALID) && (flags & M_DIRTY));
842 
843 	return 0;
844 }
845 
846 static int blocks_are_unmapped_or_clean(struct dm_cache_metadata *cmd,
847 					dm_cblock_t begin, dm_cblock_t end,
848 					bool *result)
849 {
850 	int r;
851 	*result = true;
852 
853 	while (begin != end) {
854 		r = block_unmapped_or_clean(cmd, begin, result);
855 		if (r)
856 			return r;
857 
858 		if (!*result) {
859 			DMERR("cache block %llu is dirty",
860 			      (unsigned long long) from_cblock(begin));
861 			return 0;
862 		}
863 
864 		begin = to_cblock(from_cblock(begin) + 1);
865 	}
866 
867 	return 0;
868 }
869 
870 #define WRITE_LOCK(cmd) \
871 	if (cmd->fail_io || dm_bm_is_read_only(cmd->bm)) \
872 		return -EINVAL; \
873 	down_write(&cmd->root_lock)
874 
875 #define WRITE_LOCK_VOID(cmd) \
876 	if (cmd->fail_io || dm_bm_is_read_only(cmd->bm)) \
877 		return; \
878 	down_write(&cmd->root_lock)
879 
880 #define WRITE_UNLOCK(cmd) \
881 	up_write(&cmd->root_lock)
882 
883 int dm_cache_resize(struct dm_cache_metadata *cmd, dm_cblock_t new_cache_size)
884 {
885 	int r;
886 	bool clean;
887 	__le64 null_mapping = pack_value(0, 0);
888 
889 	WRITE_LOCK(cmd);
890 	__dm_bless_for_disk(&null_mapping);
891 
892 	if (from_cblock(new_cache_size) < from_cblock(cmd->cache_blocks)) {
893 		r = blocks_are_unmapped_or_clean(cmd, new_cache_size, cmd->cache_blocks, &clean);
894 		if (r) {
895 			__dm_unbless_for_disk(&null_mapping);
896 			goto out;
897 		}
898 
899 		if (!clean) {
900 			DMERR("unable to shrink cache due to dirty blocks");
901 			r = -EINVAL;
902 			__dm_unbless_for_disk(&null_mapping);
903 			goto out;
904 		}
905 	}
906 
907 	r = dm_array_resize(&cmd->info, cmd->root, from_cblock(cmd->cache_blocks),
908 			    from_cblock(new_cache_size),
909 			    &null_mapping, &cmd->root);
910 	if (!r)
911 		cmd->cache_blocks = new_cache_size;
912 	cmd->changed = true;
913 
914 out:
915 	WRITE_UNLOCK(cmd);
916 
917 	return r;
918 }
919 
920 int dm_cache_discard_bitset_resize(struct dm_cache_metadata *cmd,
921 				   sector_t discard_block_size,
922 				   dm_dblock_t new_nr_entries)
923 {
924 	int r;
925 
926 	WRITE_LOCK(cmd);
927 	r = dm_bitset_resize(&cmd->discard_info,
928 			     cmd->discard_root,
929 			     from_dblock(cmd->discard_nr_blocks),
930 			     from_dblock(new_nr_entries),
931 			     false, &cmd->discard_root);
932 	if (!r) {
933 		cmd->discard_block_size = discard_block_size;
934 		cmd->discard_nr_blocks = new_nr_entries;
935 	}
936 
937 	cmd->changed = true;
938 	WRITE_UNLOCK(cmd);
939 
940 	return r;
941 }
942 
943 static int __set_discard(struct dm_cache_metadata *cmd, dm_dblock_t b)
944 {
945 	return dm_bitset_set_bit(&cmd->discard_info, cmd->discard_root,
946 				 from_dblock(b), &cmd->discard_root);
947 }
948 
949 static int __clear_discard(struct dm_cache_metadata *cmd, dm_dblock_t b)
950 {
951 	return dm_bitset_clear_bit(&cmd->discard_info, cmd->discard_root,
952 				   from_dblock(b), &cmd->discard_root);
953 }
954 
955 static int __is_discarded(struct dm_cache_metadata *cmd, dm_dblock_t b,
956 			  bool *is_discarded)
957 {
958 	return dm_bitset_test_bit(&cmd->discard_info, cmd->discard_root,
959 				  from_dblock(b), &cmd->discard_root,
960 				  is_discarded);
961 }
962 
963 static int __discard(struct dm_cache_metadata *cmd,
964 		     dm_dblock_t dblock, bool discard)
965 {
966 	int r;
967 
968 	r = (discard ? __set_discard : __clear_discard)(cmd, dblock);
969 	if (r)
970 		return r;
971 
972 	cmd->changed = true;
973 	return 0;
974 }
975 
976 int dm_cache_set_discard(struct dm_cache_metadata *cmd,
977 			 dm_dblock_t dblock, bool discard)
978 {
979 	int r;
980 
981 	WRITE_LOCK(cmd);
982 	r = __discard(cmd, dblock, discard);
983 	WRITE_UNLOCK(cmd);
984 
985 	return r;
986 }
987 
988 static int __load_discards(struct dm_cache_metadata *cmd,
989 			   load_discard_fn fn, void *context)
990 {
991 	int r = 0;
992 	dm_block_t b;
993 	bool discard;
994 
995 	for (b = 0; b < from_dblock(cmd->discard_nr_blocks); b++) {
996 		dm_dblock_t dblock = to_dblock(b);
997 
998 		if (cmd->clean_when_opened) {
999 			r = __is_discarded(cmd, dblock, &discard);
1000 			if (r)
1001 				return r;
1002 		} else
1003 			discard = false;
1004 
1005 		r = fn(context, cmd->discard_block_size, dblock, discard);
1006 		if (r)
1007 			break;
1008 	}
1009 
1010 	return r;
1011 }
1012 
1013 int dm_cache_load_discards(struct dm_cache_metadata *cmd,
1014 			   load_discard_fn fn, void *context)
1015 {
1016 	int r;
1017 
1018 	down_read(&cmd->root_lock);
1019 	r = __load_discards(cmd, fn, context);
1020 	up_read(&cmd->root_lock);
1021 
1022 	return r;
1023 }
1024 
1025 dm_cblock_t dm_cache_size(struct dm_cache_metadata *cmd)
1026 {
1027 	dm_cblock_t r;
1028 
1029 	down_read(&cmd->root_lock);
1030 	r = cmd->cache_blocks;
1031 	up_read(&cmd->root_lock);
1032 
1033 	return r;
1034 }
1035 
1036 static int __remove(struct dm_cache_metadata *cmd, dm_cblock_t cblock)
1037 {
1038 	int r;
1039 	__le64 value = pack_value(0, 0);
1040 
1041 	__dm_bless_for_disk(&value);
1042 	r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),
1043 			       &value, &cmd->root);
1044 	if (r)
1045 		return r;
1046 
1047 	cmd->changed = true;
1048 	return 0;
1049 }
1050 
1051 int dm_cache_remove_mapping(struct dm_cache_metadata *cmd, dm_cblock_t cblock)
1052 {
1053 	int r;
1054 
1055 	WRITE_LOCK(cmd);
1056 	r = __remove(cmd, cblock);
1057 	WRITE_UNLOCK(cmd);
1058 
1059 	return r;
1060 }
1061 
1062 static int __insert(struct dm_cache_metadata *cmd,
1063 		    dm_cblock_t cblock, dm_oblock_t oblock)
1064 {
1065 	int r;
1066 	__le64 value = pack_value(oblock, M_VALID);
1067 	__dm_bless_for_disk(&value);
1068 
1069 	r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),
1070 			       &value, &cmd->root);
1071 	if (r)
1072 		return r;
1073 
1074 	cmd->changed = true;
1075 	return 0;
1076 }
1077 
1078 int dm_cache_insert_mapping(struct dm_cache_metadata *cmd,
1079 			    dm_cblock_t cblock, dm_oblock_t oblock)
1080 {
1081 	int r;
1082 
1083 	WRITE_LOCK(cmd);
1084 	r = __insert(cmd, cblock, oblock);
1085 	WRITE_UNLOCK(cmd);
1086 
1087 	return r;
1088 }
1089 
1090 struct thunk {
1091 	load_mapping_fn fn;
1092 	void *context;
1093 
1094 	struct dm_cache_metadata *cmd;
1095 	bool respect_dirty_flags;
1096 	bool hints_valid;
1097 };
1098 
1099 static bool policy_unchanged(struct dm_cache_metadata *cmd,
1100 			     struct dm_cache_policy *policy)
1101 {
1102 	const char *policy_name = dm_cache_policy_get_name(policy);
1103 	const unsigned *policy_version = dm_cache_policy_get_version(policy);
1104 	size_t policy_hint_size = dm_cache_policy_get_hint_size(policy);
1105 
1106 	/*
1107 	 * Ensure policy names match.
1108 	 */
1109 	if (strncmp(cmd->policy_name, policy_name, sizeof(cmd->policy_name)))
1110 		return false;
1111 
1112 	/*
1113 	 * Ensure policy major versions match.
1114 	 */
1115 	if (cmd->policy_version[0] != policy_version[0])
1116 		return false;
1117 
1118 	/*
1119 	 * Ensure policy hint sizes match.
1120 	 */
1121 	if (cmd->policy_hint_size != policy_hint_size)
1122 		return false;
1123 
1124 	return true;
1125 }
1126 
1127 static bool hints_array_initialized(struct dm_cache_metadata *cmd)
1128 {
1129 	return cmd->hint_root && cmd->policy_hint_size;
1130 }
1131 
1132 static bool hints_array_available(struct dm_cache_metadata *cmd,
1133 				  struct dm_cache_policy *policy)
1134 {
1135 	return cmd->clean_when_opened && policy_unchanged(cmd, policy) &&
1136 		hints_array_initialized(cmd);
1137 }
1138 
1139 static int __load_mapping(void *context, uint64_t cblock, void *leaf)
1140 {
1141 	int r = 0;
1142 	bool dirty;
1143 	__le64 value;
1144 	__le32 hint_value = 0;
1145 	dm_oblock_t oblock;
1146 	unsigned flags;
1147 	struct thunk *thunk = context;
1148 	struct dm_cache_metadata *cmd = thunk->cmd;
1149 
1150 	memcpy(&value, leaf, sizeof(value));
1151 	unpack_value(value, &oblock, &flags);
1152 
1153 	if (flags & M_VALID) {
1154 		if (thunk->hints_valid) {
1155 			r = dm_array_get_value(&cmd->hint_info, cmd->hint_root,
1156 					       cblock, &hint_value);
1157 			if (r && r != -ENODATA)
1158 				return r;
1159 		}
1160 
1161 		dirty = thunk->respect_dirty_flags ? (flags & M_DIRTY) : true;
1162 		r = thunk->fn(thunk->context, oblock, to_cblock(cblock),
1163 			      dirty, le32_to_cpu(hint_value), thunk->hints_valid);
1164 	}
1165 
1166 	return r;
1167 }
1168 
1169 static int __load_mappings(struct dm_cache_metadata *cmd,
1170 			   struct dm_cache_policy *policy,
1171 			   load_mapping_fn fn, void *context)
1172 {
1173 	struct thunk thunk;
1174 
1175 	thunk.fn = fn;
1176 	thunk.context = context;
1177 
1178 	thunk.cmd = cmd;
1179 	thunk.respect_dirty_flags = cmd->clean_when_opened;
1180 	thunk.hints_valid = hints_array_available(cmd, policy);
1181 
1182 	return dm_array_walk(&cmd->info, cmd->root, __load_mapping, &thunk);
1183 }
1184 
1185 int dm_cache_load_mappings(struct dm_cache_metadata *cmd,
1186 			   struct dm_cache_policy *policy,
1187 			   load_mapping_fn fn, void *context)
1188 {
1189 	int r;
1190 
1191 	down_read(&cmd->root_lock);
1192 	r = __load_mappings(cmd, policy, fn, context);
1193 	up_read(&cmd->root_lock);
1194 
1195 	return r;
1196 }
1197 
1198 static int __dump_mapping(void *context, uint64_t cblock, void *leaf)
1199 {
1200 	int r = 0;
1201 	__le64 value;
1202 	dm_oblock_t oblock;
1203 	unsigned flags;
1204 
1205 	memcpy(&value, leaf, sizeof(value));
1206 	unpack_value(value, &oblock, &flags);
1207 
1208 	return r;
1209 }
1210 
1211 static int __dump_mappings(struct dm_cache_metadata *cmd)
1212 {
1213 	return dm_array_walk(&cmd->info, cmd->root, __dump_mapping, NULL);
1214 }
1215 
1216 void dm_cache_dump(struct dm_cache_metadata *cmd)
1217 {
1218 	down_read(&cmd->root_lock);
1219 	__dump_mappings(cmd);
1220 	up_read(&cmd->root_lock);
1221 }
1222 
1223 int dm_cache_changed_this_transaction(struct dm_cache_metadata *cmd)
1224 {
1225 	int r;
1226 
1227 	down_read(&cmd->root_lock);
1228 	r = cmd->changed;
1229 	up_read(&cmd->root_lock);
1230 
1231 	return r;
1232 }
1233 
1234 static int __dirty(struct dm_cache_metadata *cmd, dm_cblock_t cblock, bool dirty)
1235 {
1236 	int r;
1237 	unsigned flags;
1238 	dm_oblock_t oblock;
1239 	__le64 value;
1240 
1241 	r = dm_array_get_value(&cmd->info, cmd->root, from_cblock(cblock), &value);
1242 	if (r)
1243 		return r;
1244 
1245 	unpack_value(value, &oblock, &flags);
1246 
1247 	if (((flags & M_DIRTY) && dirty) || (!(flags & M_DIRTY) && !dirty))
1248 		/* nothing to be done */
1249 		return 0;
1250 
1251 	value = pack_value(oblock, (flags & ~M_DIRTY) | (dirty ? M_DIRTY : 0));
1252 	__dm_bless_for_disk(&value);
1253 
1254 	r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),
1255 			       &value, &cmd->root);
1256 	if (r)
1257 		return r;
1258 
1259 	cmd->changed = true;
1260 	return 0;
1261 
1262 }
1263 
1264 int dm_cache_set_dirty(struct dm_cache_metadata *cmd,
1265 		       dm_cblock_t cblock, bool dirty)
1266 {
1267 	int r;
1268 
1269 	WRITE_LOCK(cmd);
1270 	r = __dirty(cmd, cblock, dirty);
1271 	WRITE_UNLOCK(cmd);
1272 
1273 	return r;
1274 }
1275 
1276 void dm_cache_metadata_get_stats(struct dm_cache_metadata *cmd,
1277 				 struct dm_cache_statistics *stats)
1278 {
1279 	down_read(&cmd->root_lock);
1280 	*stats = cmd->stats;
1281 	up_read(&cmd->root_lock);
1282 }
1283 
1284 void dm_cache_metadata_set_stats(struct dm_cache_metadata *cmd,
1285 				 struct dm_cache_statistics *stats)
1286 {
1287 	WRITE_LOCK_VOID(cmd);
1288 	cmd->stats = *stats;
1289 	WRITE_UNLOCK(cmd);
1290 }
1291 
1292 int dm_cache_commit(struct dm_cache_metadata *cmd, bool clean_shutdown)
1293 {
1294 	int r;
1295 	flags_mutator mutator = (clean_shutdown ? set_clean_shutdown :
1296 				 clear_clean_shutdown);
1297 
1298 	WRITE_LOCK(cmd);
1299 	r = __commit_transaction(cmd, mutator);
1300 	if (r)
1301 		goto out;
1302 
1303 	r = __begin_transaction(cmd);
1304 
1305 out:
1306 	WRITE_UNLOCK(cmd);
1307 	return r;
1308 }
1309 
1310 int dm_cache_get_free_metadata_block_count(struct dm_cache_metadata *cmd,
1311 					   dm_block_t *result)
1312 {
1313 	int r = -EINVAL;
1314 
1315 	down_read(&cmd->root_lock);
1316 	r = dm_sm_get_nr_free(cmd->metadata_sm, result);
1317 	up_read(&cmd->root_lock);
1318 
1319 	return r;
1320 }
1321 
1322 int dm_cache_get_metadata_dev_size(struct dm_cache_metadata *cmd,
1323 				   dm_block_t *result)
1324 {
1325 	int r = -EINVAL;
1326 
1327 	down_read(&cmd->root_lock);
1328 	r = dm_sm_get_nr_blocks(cmd->metadata_sm, result);
1329 	up_read(&cmd->root_lock);
1330 
1331 	return r;
1332 }
1333 
1334 /*----------------------------------------------------------------*/
1335 
1336 static int begin_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy)
1337 {
1338 	int r;
1339 	__le32 value;
1340 	size_t hint_size;
1341 	const char *policy_name = dm_cache_policy_get_name(policy);
1342 	const unsigned *policy_version = dm_cache_policy_get_version(policy);
1343 
1344 	if (!policy_name[0] ||
1345 	    (strlen(policy_name) > sizeof(cmd->policy_name) - 1))
1346 		return -EINVAL;
1347 
1348 	if (!policy_unchanged(cmd, policy)) {
1349 		strncpy(cmd->policy_name, policy_name, sizeof(cmd->policy_name));
1350 		memcpy(cmd->policy_version, policy_version, sizeof(cmd->policy_version));
1351 
1352 		hint_size = dm_cache_policy_get_hint_size(policy);
1353 		if (!hint_size)
1354 			return 0; /* short-circuit hints initialization */
1355 		cmd->policy_hint_size = hint_size;
1356 
1357 		if (cmd->hint_root) {
1358 			r = dm_array_del(&cmd->hint_info, cmd->hint_root);
1359 			if (r)
1360 				return r;
1361 		}
1362 
1363 		r = dm_array_empty(&cmd->hint_info, &cmd->hint_root);
1364 		if (r)
1365 			return r;
1366 
1367 		value = cpu_to_le32(0);
1368 		__dm_bless_for_disk(&value);
1369 		r = dm_array_resize(&cmd->hint_info, cmd->hint_root, 0,
1370 				    from_cblock(cmd->cache_blocks),
1371 				    &value, &cmd->hint_root);
1372 		if (r)
1373 			return r;
1374 	}
1375 
1376 	return 0;
1377 }
1378 
1379 static int save_hint(void *context, dm_cblock_t cblock, dm_oblock_t oblock, uint32_t hint)
1380 {
1381 	struct dm_cache_metadata *cmd = context;
1382 	__le32 value = cpu_to_le32(hint);
1383 	int r;
1384 
1385 	__dm_bless_for_disk(&value);
1386 
1387 	r = dm_array_set_value(&cmd->hint_info, cmd->hint_root,
1388 			       from_cblock(cblock), &value, &cmd->hint_root);
1389 	cmd->changed = true;
1390 
1391 	return r;
1392 }
1393 
1394 static int write_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy)
1395 {
1396 	int r;
1397 
1398 	r = begin_hints(cmd, policy);
1399 	if (r) {
1400 		DMERR("begin_hints failed");
1401 		return r;
1402 	}
1403 
1404 	return policy_walk_mappings(policy, save_hint, cmd);
1405 }
1406 
1407 int dm_cache_write_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy)
1408 {
1409 	int r;
1410 
1411 	WRITE_LOCK(cmd);
1412 	r = write_hints(cmd, policy);
1413 	WRITE_UNLOCK(cmd);
1414 
1415 	return r;
1416 }
1417 
1418 int dm_cache_metadata_all_clean(struct dm_cache_metadata *cmd, bool *result)
1419 {
1420 	return blocks_are_unmapped_or_clean(cmd, 0, cmd->cache_blocks, result);
1421 }
1422 
1423 void dm_cache_metadata_set_read_only(struct dm_cache_metadata *cmd)
1424 {
1425 	WRITE_LOCK_VOID(cmd);
1426 	dm_bm_set_read_only(cmd->bm);
1427 	WRITE_UNLOCK(cmd);
1428 }
1429 
1430 void dm_cache_metadata_set_read_write(struct dm_cache_metadata *cmd)
1431 {
1432 	WRITE_LOCK_VOID(cmd);
1433 	dm_bm_set_read_write(cmd->bm);
1434 	WRITE_UNLOCK(cmd);
1435 }
1436 
1437 int dm_cache_metadata_set_needs_check(struct dm_cache_metadata *cmd)
1438 {
1439 	int r;
1440 	struct dm_block *sblock;
1441 	struct cache_disk_superblock *disk_super;
1442 
1443 	/*
1444 	 * We ignore fail_io for this function.
1445 	 */
1446 	down_write(&cmd->root_lock);
1447 	set_bit(NEEDS_CHECK, &cmd->flags);
1448 
1449 	r = superblock_lock(cmd, &sblock);
1450 	if (r) {
1451 		DMERR("couldn't read superblock");
1452 		goto out;
1453 	}
1454 
1455 	disk_super = dm_block_data(sblock);
1456 	disk_super->flags = cpu_to_le32(cmd->flags);
1457 
1458 	dm_bm_unlock(sblock);
1459 
1460 out:
1461 	up_write(&cmd->root_lock);
1462 	return r;
1463 }
1464 
1465 bool dm_cache_metadata_needs_check(struct dm_cache_metadata *cmd)
1466 {
1467 	bool needs_check;
1468 
1469 	down_read(&cmd->root_lock);
1470 	needs_check = !!test_bit(NEEDS_CHECK, &cmd->flags);
1471 	up_read(&cmd->root_lock);
1472 
1473 	return needs_check;
1474 }
1475 
1476 int dm_cache_metadata_abort(struct dm_cache_metadata *cmd)
1477 {
1478 	int r;
1479 
1480 	WRITE_LOCK(cmd);
1481 	__destroy_persistent_data_objects(cmd);
1482 	r = __create_persistent_data_objects(cmd, false);
1483 	if (r)
1484 		cmd->fail_io = true;
1485 	WRITE_UNLOCK(cmd);
1486 
1487 	return r;
1488 }
1489