xref: /openbmc/linux/drivers/md/dm-cache-target.c (revision ddc141e5)
1 /*
2  * Copyright (C) 2012 Red Hat. All rights reserved.
3  *
4  * This file is released under the GPL.
5  */
6 
7 #include "dm.h"
8 #include "dm-bio-prison-v2.h"
9 #include "dm-bio-record.h"
10 #include "dm-cache-metadata.h"
11 
12 #include <linux/dm-io.h>
13 #include <linux/dm-kcopyd.h>
14 #include <linux/jiffies.h>
15 #include <linux/init.h>
16 #include <linux/mempool.h>
17 #include <linux/module.h>
18 #include <linux/rwsem.h>
19 #include <linux/slab.h>
20 #include <linux/vmalloc.h>
21 
22 #define DM_MSG_PREFIX "cache"
23 
24 DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(cache_copy_throttle,
25 	"A percentage of time allocated for copying to and/or from cache");
26 
27 /*----------------------------------------------------------------*/
28 
29 /*
30  * Glossary:
31  *
32  * oblock: index of an origin block
33  * cblock: index of a cache block
34  * promotion: movement of a block from origin to cache
35  * demotion: movement of a block from cache to origin
36  * migration: movement of a block between the origin and cache device,
37  *	      either direction
38  */
39 
40 /*----------------------------------------------------------------*/
41 
42 struct io_tracker {
43 	spinlock_t lock;
44 
45 	/*
46 	 * Sectors of in-flight IO.
47 	 */
48 	sector_t in_flight;
49 
50 	/*
51 	 * The time, in jiffies, when this device became idle (if it is
52 	 * indeed idle).
53 	 */
54 	unsigned long idle_time;
55 	unsigned long last_update_time;
56 };
57 
58 static void iot_init(struct io_tracker *iot)
59 {
60 	spin_lock_init(&iot->lock);
61 	iot->in_flight = 0ul;
62 	iot->idle_time = 0ul;
63 	iot->last_update_time = jiffies;
64 }
65 
66 static bool __iot_idle_for(struct io_tracker *iot, unsigned long jifs)
67 {
68 	if (iot->in_flight)
69 		return false;
70 
71 	return time_after(jiffies, iot->idle_time + jifs);
72 }
73 
74 static bool iot_idle_for(struct io_tracker *iot, unsigned long jifs)
75 {
76 	bool r;
77 	unsigned long flags;
78 
79 	spin_lock_irqsave(&iot->lock, flags);
80 	r = __iot_idle_for(iot, jifs);
81 	spin_unlock_irqrestore(&iot->lock, flags);
82 
83 	return r;
84 }
85 
86 static void iot_io_begin(struct io_tracker *iot, sector_t len)
87 {
88 	unsigned long flags;
89 
90 	spin_lock_irqsave(&iot->lock, flags);
91 	iot->in_flight += len;
92 	spin_unlock_irqrestore(&iot->lock, flags);
93 }
94 
95 static void __iot_io_end(struct io_tracker *iot, sector_t len)
96 {
97 	if (!len)
98 		return;
99 
100 	iot->in_flight -= len;
101 	if (!iot->in_flight)
102 		iot->idle_time = jiffies;
103 }
104 
105 static void iot_io_end(struct io_tracker *iot, sector_t len)
106 {
107 	unsigned long flags;
108 
109 	spin_lock_irqsave(&iot->lock, flags);
110 	__iot_io_end(iot, len);
111 	spin_unlock_irqrestore(&iot->lock, flags);
112 }
113 
114 /*----------------------------------------------------------------*/
115 
116 /*
117  * Represents a chunk of future work.  'input' allows continuations to pass
118  * values between themselves, typically error values.
119  */
120 struct continuation {
121 	struct work_struct ws;
122 	blk_status_t input;
123 };
124 
125 static inline void init_continuation(struct continuation *k,
126 				     void (*fn)(struct work_struct *))
127 {
128 	INIT_WORK(&k->ws, fn);
129 	k->input = 0;
130 }
131 
132 static inline void queue_continuation(struct workqueue_struct *wq,
133 				      struct continuation *k)
134 {
135 	queue_work(wq, &k->ws);
136 }
137 
138 /*----------------------------------------------------------------*/
139 
140 /*
141  * The batcher collects together pieces of work that need a particular
142  * operation to occur before they can proceed (typically a commit).
143  */
144 struct batcher {
145 	/*
146 	 * The operation that everyone is waiting for.
147 	 */
148 	blk_status_t (*commit_op)(void *context);
149 	void *commit_context;
150 
151 	/*
152 	 * This is how bios should be issued once the commit op is complete
153 	 * (accounted_request).
154 	 */
155 	void (*issue_op)(struct bio *bio, void *context);
156 	void *issue_context;
157 
158 	/*
159 	 * Queued work gets put on here after commit.
160 	 */
161 	struct workqueue_struct *wq;
162 
163 	spinlock_t lock;
164 	struct list_head work_items;
165 	struct bio_list bios;
166 	struct work_struct commit_work;
167 
168 	bool commit_scheduled;
169 };
170 
171 static void __commit(struct work_struct *_ws)
172 {
173 	struct batcher *b = container_of(_ws, struct batcher, commit_work);
174 	blk_status_t r;
175 	unsigned long flags;
176 	struct list_head work_items;
177 	struct work_struct *ws, *tmp;
178 	struct continuation *k;
179 	struct bio *bio;
180 	struct bio_list bios;
181 
182 	INIT_LIST_HEAD(&work_items);
183 	bio_list_init(&bios);
184 
185 	/*
186 	 * We have to grab these before the commit_op to avoid a race
187 	 * condition.
188 	 */
189 	spin_lock_irqsave(&b->lock, flags);
190 	list_splice_init(&b->work_items, &work_items);
191 	bio_list_merge(&bios, &b->bios);
192 	bio_list_init(&b->bios);
193 	b->commit_scheduled = false;
194 	spin_unlock_irqrestore(&b->lock, flags);
195 
196 	r = b->commit_op(b->commit_context);
197 
198 	list_for_each_entry_safe(ws, tmp, &work_items, entry) {
199 		k = container_of(ws, struct continuation, ws);
200 		k->input = r;
201 		INIT_LIST_HEAD(&ws->entry); /* to avoid a WARN_ON */
202 		queue_work(b->wq, ws);
203 	}
204 
205 	while ((bio = bio_list_pop(&bios))) {
206 		if (r) {
207 			bio->bi_status = r;
208 			bio_endio(bio);
209 		} else
210 			b->issue_op(bio, b->issue_context);
211 	}
212 }
213 
214 static void batcher_init(struct batcher *b,
215 			 blk_status_t (*commit_op)(void *),
216 			 void *commit_context,
217 			 void (*issue_op)(struct bio *bio, void *),
218 			 void *issue_context,
219 			 struct workqueue_struct *wq)
220 {
221 	b->commit_op = commit_op;
222 	b->commit_context = commit_context;
223 	b->issue_op = issue_op;
224 	b->issue_context = issue_context;
225 	b->wq = wq;
226 
227 	spin_lock_init(&b->lock);
228 	INIT_LIST_HEAD(&b->work_items);
229 	bio_list_init(&b->bios);
230 	INIT_WORK(&b->commit_work, __commit);
231 	b->commit_scheduled = false;
232 }
233 
234 static void async_commit(struct batcher *b)
235 {
236 	queue_work(b->wq, &b->commit_work);
237 }
238 
239 static void continue_after_commit(struct batcher *b, struct continuation *k)
240 {
241 	unsigned long flags;
242 	bool commit_scheduled;
243 
244 	spin_lock_irqsave(&b->lock, flags);
245 	commit_scheduled = b->commit_scheduled;
246 	list_add_tail(&k->ws.entry, &b->work_items);
247 	spin_unlock_irqrestore(&b->lock, flags);
248 
249 	if (commit_scheduled)
250 		async_commit(b);
251 }
252 
253 /*
254  * Bios are errored if commit failed.
255  */
256 static void issue_after_commit(struct batcher *b, struct bio *bio)
257 {
258        unsigned long flags;
259        bool commit_scheduled;
260 
261        spin_lock_irqsave(&b->lock, flags);
262        commit_scheduled = b->commit_scheduled;
263        bio_list_add(&b->bios, bio);
264        spin_unlock_irqrestore(&b->lock, flags);
265 
266        if (commit_scheduled)
267 	       async_commit(b);
268 }
269 
270 /*
271  * Call this if some urgent work is waiting for the commit to complete.
272  */
273 static void schedule_commit(struct batcher *b)
274 {
275 	bool immediate;
276 	unsigned long flags;
277 
278 	spin_lock_irqsave(&b->lock, flags);
279 	immediate = !list_empty(&b->work_items) || !bio_list_empty(&b->bios);
280 	b->commit_scheduled = true;
281 	spin_unlock_irqrestore(&b->lock, flags);
282 
283 	if (immediate)
284 		async_commit(b);
285 }
286 
287 /*
288  * There are a couple of places where we let a bio run, but want to do some
289  * work before calling its endio function.  We do this by temporarily
290  * changing the endio fn.
291  */
292 struct dm_hook_info {
293 	bio_end_io_t *bi_end_io;
294 };
295 
296 static void dm_hook_bio(struct dm_hook_info *h, struct bio *bio,
297 			bio_end_io_t *bi_end_io, void *bi_private)
298 {
299 	h->bi_end_io = bio->bi_end_io;
300 
301 	bio->bi_end_io = bi_end_io;
302 	bio->bi_private = bi_private;
303 }
304 
305 static void dm_unhook_bio(struct dm_hook_info *h, struct bio *bio)
306 {
307 	bio->bi_end_io = h->bi_end_io;
308 }
309 
310 /*----------------------------------------------------------------*/
311 
312 #define MIGRATION_POOL_SIZE 128
313 #define COMMIT_PERIOD HZ
314 #define MIGRATION_COUNT_WINDOW 10
315 
316 /*
317  * The block size of the device holding cache data must be
318  * between 32KB and 1GB.
319  */
320 #define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (32 * 1024 >> SECTOR_SHIFT)
321 #define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT)
322 
323 enum cache_metadata_mode {
324 	CM_WRITE,		/* metadata may be changed */
325 	CM_READ_ONLY,		/* metadata may not be changed */
326 	CM_FAIL
327 };
328 
329 enum cache_io_mode {
330 	/*
331 	 * Data is written to cached blocks only.  These blocks are marked
332 	 * dirty.  If you lose the cache device you will lose data.
333 	 * Potential performance increase for both reads and writes.
334 	 */
335 	CM_IO_WRITEBACK,
336 
337 	/*
338 	 * Data is written to both cache and origin.  Blocks are never
339 	 * dirty.  Potential performance benfit for reads only.
340 	 */
341 	CM_IO_WRITETHROUGH,
342 
343 	/*
344 	 * A degraded mode useful for various cache coherency situations
345 	 * (eg, rolling back snapshots).  Reads and writes always go to the
346 	 * origin.  If a write goes to a cached oblock, then the cache
347 	 * block is invalidated.
348 	 */
349 	CM_IO_PASSTHROUGH
350 };
351 
352 struct cache_features {
353 	enum cache_metadata_mode mode;
354 	enum cache_io_mode io_mode;
355 	unsigned metadata_version;
356 };
357 
358 struct cache_stats {
359 	atomic_t read_hit;
360 	atomic_t read_miss;
361 	atomic_t write_hit;
362 	atomic_t write_miss;
363 	atomic_t demotion;
364 	atomic_t promotion;
365 	atomic_t writeback;
366 	atomic_t copies_avoided;
367 	atomic_t cache_cell_clash;
368 	atomic_t commit_count;
369 	atomic_t discard_count;
370 };
371 
372 struct cache {
373 	struct dm_target *ti;
374 	struct dm_target_callbacks callbacks;
375 
376 	struct dm_cache_metadata *cmd;
377 
378 	/*
379 	 * Metadata is written to this device.
380 	 */
381 	struct dm_dev *metadata_dev;
382 
383 	/*
384 	 * The slower of the two data devices.  Typically a spindle.
385 	 */
386 	struct dm_dev *origin_dev;
387 
388 	/*
389 	 * The faster of the two data devices.  Typically an SSD.
390 	 */
391 	struct dm_dev *cache_dev;
392 
393 	/*
394 	 * Size of the origin device in _complete_ blocks and native sectors.
395 	 */
396 	dm_oblock_t origin_blocks;
397 	sector_t origin_sectors;
398 
399 	/*
400 	 * Size of the cache device in blocks.
401 	 */
402 	dm_cblock_t cache_size;
403 
404 	/*
405 	 * Fields for converting from sectors to blocks.
406 	 */
407 	sector_t sectors_per_block;
408 	int sectors_per_block_shift;
409 
410 	spinlock_t lock;
411 	struct bio_list deferred_bios;
412 	sector_t migration_threshold;
413 	wait_queue_head_t migration_wait;
414 	atomic_t nr_allocated_migrations;
415 
416 	/*
417 	 * The number of in flight migrations that are performing
418 	 * background io. eg, promotion, writeback.
419 	 */
420 	atomic_t nr_io_migrations;
421 
422 	struct rw_semaphore quiesce_lock;
423 
424 	/*
425 	 * cache_size entries, dirty if set
426 	 */
427 	atomic_t nr_dirty;
428 	unsigned long *dirty_bitset;
429 
430 	/*
431 	 * origin_blocks entries, discarded if set.
432 	 */
433 	dm_dblock_t discard_nr_blocks;
434 	unsigned long *discard_bitset;
435 	uint32_t discard_block_size; /* a power of 2 times sectors per block */
436 
437 	/*
438 	 * Rather than reconstructing the table line for the status we just
439 	 * save it and regurgitate.
440 	 */
441 	unsigned nr_ctr_args;
442 	const char **ctr_args;
443 
444 	struct dm_kcopyd_client *copier;
445 	struct workqueue_struct *wq;
446 	struct work_struct deferred_bio_worker;
447 	struct work_struct migration_worker;
448 	struct delayed_work waker;
449 	struct dm_bio_prison_v2 *prison;
450 	struct bio_set *bs;
451 
452 	mempool_t *migration_pool;
453 
454 	struct dm_cache_policy *policy;
455 	unsigned policy_nr_args;
456 
457 	bool need_tick_bio:1;
458 	bool sized:1;
459 	bool invalidate:1;
460 	bool commit_requested:1;
461 	bool loaded_mappings:1;
462 	bool loaded_discards:1;
463 
464 	/*
465 	 * Cache features such as write-through.
466 	 */
467 	struct cache_features features;
468 
469 	struct cache_stats stats;
470 
471 	/*
472 	 * Invalidation fields.
473 	 */
474 	spinlock_t invalidation_lock;
475 	struct list_head invalidation_requests;
476 
477 	struct io_tracker tracker;
478 
479 	struct work_struct commit_ws;
480 	struct batcher committer;
481 
482 	struct rw_semaphore background_work_lock;
483 };
484 
485 struct per_bio_data {
486 	bool tick:1;
487 	unsigned req_nr:2;
488 	struct dm_bio_prison_cell_v2 *cell;
489 	struct dm_hook_info hook_info;
490 	sector_t len;
491 };
492 
493 struct dm_cache_migration {
494 	struct continuation k;
495 	struct cache *cache;
496 
497 	struct policy_work *op;
498 	struct bio *overwrite_bio;
499 	struct dm_bio_prison_cell_v2 *cell;
500 
501 	dm_cblock_t invalidate_cblock;
502 	dm_oblock_t invalidate_oblock;
503 };
504 
505 /*----------------------------------------------------------------*/
506 
507 static bool writethrough_mode(struct cache *cache)
508 {
509 	return cache->features.io_mode == CM_IO_WRITETHROUGH;
510 }
511 
512 static bool writeback_mode(struct cache *cache)
513 {
514 	return cache->features.io_mode == CM_IO_WRITEBACK;
515 }
516 
517 static inline bool passthrough_mode(struct cache *cache)
518 {
519 	return unlikely(cache->features.io_mode == CM_IO_PASSTHROUGH);
520 }
521 
522 /*----------------------------------------------------------------*/
523 
524 static void wake_deferred_bio_worker(struct cache *cache)
525 {
526 	queue_work(cache->wq, &cache->deferred_bio_worker);
527 }
528 
529 static void wake_migration_worker(struct cache *cache)
530 {
531 	if (passthrough_mode(cache))
532 		return;
533 
534 	queue_work(cache->wq, &cache->migration_worker);
535 }
536 
537 /*----------------------------------------------------------------*/
538 
539 static struct dm_bio_prison_cell_v2 *alloc_prison_cell(struct cache *cache)
540 {
541 	return dm_bio_prison_alloc_cell_v2(cache->prison, GFP_NOWAIT);
542 }
543 
544 static void free_prison_cell(struct cache *cache, struct dm_bio_prison_cell_v2 *cell)
545 {
546 	dm_bio_prison_free_cell_v2(cache->prison, cell);
547 }
548 
549 static struct dm_cache_migration *alloc_migration(struct cache *cache)
550 {
551 	struct dm_cache_migration *mg;
552 
553 	mg = mempool_alloc(cache->migration_pool, GFP_NOWAIT);
554 	if (!mg)
555 		return NULL;
556 
557 	memset(mg, 0, sizeof(*mg));
558 
559 	mg->cache = cache;
560 	atomic_inc(&cache->nr_allocated_migrations);
561 
562 	return mg;
563 }
564 
565 static void free_migration(struct dm_cache_migration *mg)
566 {
567 	struct cache *cache = mg->cache;
568 
569 	if (atomic_dec_and_test(&cache->nr_allocated_migrations))
570 		wake_up(&cache->migration_wait);
571 
572 	mempool_free(mg, cache->migration_pool);
573 }
574 
575 /*----------------------------------------------------------------*/
576 
577 static inline dm_oblock_t oblock_succ(dm_oblock_t b)
578 {
579 	return to_oblock(from_oblock(b) + 1ull);
580 }
581 
582 static void build_key(dm_oblock_t begin, dm_oblock_t end, struct dm_cell_key_v2 *key)
583 {
584 	key->virtual = 0;
585 	key->dev = 0;
586 	key->block_begin = from_oblock(begin);
587 	key->block_end = from_oblock(end);
588 }
589 
590 /*
591  * We have two lock levels.  Level 0, which is used to prevent WRITEs, and
592  * level 1 which prevents *both* READs and WRITEs.
593  */
594 #define WRITE_LOCK_LEVEL 0
595 #define READ_WRITE_LOCK_LEVEL 1
596 
597 static unsigned lock_level(struct bio *bio)
598 {
599 	return bio_data_dir(bio) == WRITE ?
600 		WRITE_LOCK_LEVEL :
601 		READ_WRITE_LOCK_LEVEL;
602 }
603 
604 /*----------------------------------------------------------------
605  * Per bio data
606  *--------------------------------------------------------------*/
607 
608 static struct per_bio_data *get_per_bio_data(struct bio *bio)
609 {
610 	struct per_bio_data *pb = dm_per_bio_data(bio, sizeof(struct per_bio_data));
611 	BUG_ON(!pb);
612 	return pb;
613 }
614 
615 static struct per_bio_data *init_per_bio_data(struct bio *bio)
616 {
617 	struct per_bio_data *pb = get_per_bio_data(bio);
618 
619 	pb->tick = false;
620 	pb->req_nr = dm_bio_get_target_bio_nr(bio);
621 	pb->cell = NULL;
622 	pb->len = 0;
623 
624 	return pb;
625 }
626 
627 /*----------------------------------------------------------------*/
628 
629 static void defer_bio(struct cache *cache, struct bio *bio)
630 {
631 	unsigned long flags;
632 
633 	spin_lock_irqsave(&cache->lock, flags);
634 	bio_list_add(&cache->deferred_bios, bio);
635 	spin_unlock_irqrestore(&cache->lock, flags);
636 
637 	wake_deferred_bio_worker(cache);
638 }
639 
640 static void defer_bios(struct cache *cache, struct bio_list *bios)
641 {
642 	unsigned long flags;
643 
644 	spin_lock_irqsave(&cache->lock, flags);
645 	bio_list_merge(&cache->deferred_bios, bios);
646 	bio_list_init(bios);
647 	spin_unlock_irqrestore(&cache->lock, flags);
648 
649 	wake_deferred_bio_worker(cache);
650 }
651 
652 /*----------------------------------------------------------------*/
653 
654 static bool bio_detain_shared(struct cache *cache, dm_oblock_t oblock, struct bio *bio)
655 {
656 	bool r;
657 	struct per_bio_data *pb;
658 	struct dm_cell_key_v2 key;
659 	dm_oblock_t end = to_oblock(from_oblock(oblock) + 1ULL);
660 	struct dm_bio_prison_cell_v2 *cell_prealloc, *cell;
661 
662 	cell_prealloc = alloc_prison_cell(cache); /* FIXME: allow wait if calling from worker */
663 	if (!cell_prealloc) {
664 		defer_bio(cache, bio);
665 		return false;
666 	}
667 
668 	build_key(oblock, end, &key);
669 	r = dm_cell_get_v2(cache->prison, &key, lock_level(bio), bio, cell_prealloc, &cell);
670 	if (!r) {
671 		/*
672 		 * Failed to get the lock.
673 		 */
674 		free_prison_cell(cache, cell_prealloc);
675 		return r;
676 	}
677 
678 	if (cell != cell_prealloc)
679 		free_prison_cell(cache, cell_prealloc);
680 
681 	pb = get_per_bio_data(bio);
682 	pb->cell = cell;
683 
684 	return r;
685 }
686 
687 /*----------------------------------------------------------------*/
688 
689 static bool is_dirty(struct cache *cache, dm_cblock_t b)
690 {
691 	return test_bit(from_cblock(b), cache->dirty_bitset);
692 }
693 
694 static void set_dirty(struct cache *cache, dm_cblock_t cblock)
695 {
696 	if (!test_and_set_bit(from_cblock(cblock), cache->dirty_bitset)) {
697 		atomic_inc(&cache->nr_dirty);
698 		policy_set_dirty(cache->policy, cblock);
699 	}
700 }
701 
702 /*
703  * These two are called when setting after migrations to force the policy
704  * and dirty bitset to be in sync.
705  */
706 static void force_set_dirty(struct cache *cache, dm_cblock_t cblock)
707 {
708 	if (!test_and_set_bit(from_cblock(cblock), cache->dirty_bitset))
709 		atomic_inc(&cache->nr_dirty);
710 	policy_set_dirty(cache->policy, cblock);
711 }
712 
713 static void force_clear_dirty(struct cache *cache, dm_cblock_t cblock)
714 {
715 	if (test_and_clear_bit(from_cblock(cblock), cache->dirty_bitset)) {
716 		if (atomic_dec_return(&cache->nr_dirty) == 0)
717 			dm_table_event(cache->ti->table);
718 	}
719 
720 	policy_clear_dirty(cache->policy, cblock);
721 }
722 
723 /*----------------------------------------------------------------*/
724 
725 static bool block_size_is_power_of_two(struct cache *cache)
726 {
727 	return cache->sectors_per_block_shift >= 0;
728 }
729 
730 /* gcc on ARM generates spurious references to __udivdi3 and __umoddi3 */
731 #if defined(CONFIG_ARM) && __GNUC__ == 4 && __GNUC_MINOR__ <= 6
732 __always_inline
733 #endif
734 static dm_block_t block_div(dm_block_t b, uint32_t n)
735 {
736 	do_div(b, n);
737 
738 	return b;
739 }
740 
741 static dm_block_t oblocks_per_dblock(struct cache *cache)
742 {
743 	dm_block_t oblocks = cache->discard_block_size;
744 
745 	if (block_size_is_power_of_two(cache))
746 		oblocks >>= cache->sectors_per_block_shift;
747 	else
748 		oblocks = block_div(oblocks, cache->sectors_per_block);
749 
750 	return oblocks;
751 }
752 
753 static dm_dblock_t oblock_to_dblock(struct cache *cache, dm_oblock_t oblock)
754 {
755 	return to_dblock(block_div(from_oblock(oblock),
756 				   oblocks_per_dblock(cache)));
757 }
758 
759 static void set_discard(struct cache *cache, dm_dblock_t b)
760 {
761 	unsigned long flags;
762 
763 	BUG_ON(from_dblock(b) >= from_dblock(cache->discard_nr_blocks));
764 	atomic_inc(&cache->stats.discard_count);
765 
766 	spin_lock_irqsave(&cache->lock, flags);
767 	set_bit(from_dblock(b), cache->discard_bitset);
768 	spin_unlock_irqrestore(&cache->lock, flags);
769 }
770 
771 static void clear_discard(struct cache *cache, dm_dblock_t b)
772 {
773 	unsigned long flags;
774 
775 	spin_lock_irqsave(&cache->lock, flags);
776 	clear_bit(from_dblock(b), cache->discard_bitset);
777 	spin_unlock_irqrestore(&cache->lock, flags);
778 }
779 
780 static bool is_discarded(struct cache *cache, dm_dblock_t b)
781 {
782 	int r;
783 	unsigned long flags;
784 
785 	spin_lock_irqsave(&cache->lock, flags);
786 	r = test_bit(from_dblock(b), cache->discard_bitset);
787 	spin_unlock_irqrestore(&cache->lock, flags);
788 
789 	return r;
790 }
791 
792 static bool is_discarded_oblock(struct cache *cache, dm_oblock_t b)
793 {
794 	int r;
795 	unsigned long flags;
796 
797 	spin_lock_irqsave(&cache->lock, flags);
798 	r = test_bit(from_dblock(oblock_to_dblock(cache, b)),
799 		     cache->discard_bitset);
800 	spin_unlock_irqrestore(&cache->lock, flags);
801 
802 	return r;
803 }
804 
805 /*----------------------------------------------------------------
806  * Remapping
807  *--------------------------------------------------------------*/
808 static void remap_to_origin(struct cache *cache, struct bio *bio)
809 {
810 	bio_set_dev(bio, cache->origin_dev->bdev);
811 }
812 
813 static void remap_to_cache(struct cache *cache, struct bio *bio,
814 			   dm_cblock_t cblock)
815 {
816 	sector_t bi_sector = bio->bi_iter.bi_sector;
817 	sector_t block = from_cblock(cblock);
818 
819 	bio_set_dev(bio, cache->cache_dev->bdev);
820 	if (!block_size_is_power_of_two(cache))
821 		bio->bi_iter.bi_sector =
822 			(block * cache->sectors_per_block) +
823 			sector_div(bi_sector, cache->sectors_per_block);
824 	else
825 		bio->bi_iter.bi_sector =
826 			(block << cache->sectors_per_block_shift) |
827 			(bi_sector & (cache->sectors_per_block - 1));
828 }
829 
830 static void check_if_tick_bio_needed(struct cache *cache, struct bio *bio)
831 {
832 	unsigned long flags;
833 	struct per_bio_data *pb;
834 
835 	spin_lock_irqsave(&cache->lock, flags);
836 	if (cache->need_tick_bio && !op_is_flush(bio->bi_opf) &&
837 	    bio_op(bio) != REQ_OP_DISCARD) {
838 		pb = get_per_bio_data(bio);
839 		pb->tick = true;
840 		cache->need_tick_bio = false;
841 	}
842 	spin_unlock_irqrestore(&cache->lock, flags);
843 }
844 
845 static void __remap_to_origin_clear_discard(struct cache *cache, struct bio *bio,
846 					    dm_oblock_t oblock, bool bio_has_pbd)
847 {
848 	if (bio_has_pbd)
849 		check_if_tick_bio_needed(cache, bio);
850 	remap_to_origin(cache, bio);
851 	if (bio_data_dir(bio) == WRITE)
852 		clear_discard(cache, oblock_to_dblock(cache, oblock));
853 }
854 
855 static void remap_to_origin_clear_discard(struct cache *cache, struct bio *bio,
856 					  dm_oblock_t oblock)
857 {
858 	// FIXME: check_if_tick_bio_needed() is called way too much through this interface
859 	__remap_to_origin_clear_discard(cache, bio, oblock, true);
860 }
861 
862 static void remap_to_cache_dirty(struct cache *cache, struct bio *bio,
863 				 dm_oblock_t oblock, dm_cblock_t cblock)
864 {
865 	check_if_tick_bio_needed(cache, bio);
866 	remap_to_cache(cache, bio, cblock);
867 	if (bio_data_dir(bio) == WRITE) {
868 		set_dirty(cache, cblock);
869 		clear_discard(cache, oblock_to_dblock(cache, oblock));
870 	}
871 }
872 
873 static dm_oblock_t get_bio_block(struct cache *cache, struct bio *bio)
874 {
875 	sector_t block_nr = bio->bi_iter.bi_sector;
876 
877 	if (!block_size_is_power_of_two(cache))
878 		(void) sector_div(block_nr, cache->sectors_per_block);
879 	else
880 		block_nr >>= cache->sectors_per_block_shift;
881 
882 	return to_oblock(block_nr);
883 }
884 
885 static bool accountable_bio(struct cache *cache, struct bio *bio)
886 {
887 	return bio_op(bio) != REQ_OP_DISCARD;
888 }
889 
890 static void accounted_begin(struct cache *cache, struct bio *bio)
891 {
892 	struct per_bio_data *pb;
893 
894 	if (accountable_bio(cache, bio)) {
895 		pb = get_per_bio_data(bio);
896 		pb->len = bio_sectors(bio);
897 		iot_io_begin(&cache->tracker, pb->len);
898 	}
899 }
900 
901 static void accounted_complete(struct cache *cache, struct bio *bio)
902 {
903 	struct per_bio_data *pb = get_per_bio_data(bio);
904 
905 	iot_io_end(&cache->tracker, pb->len);
906 }
907 
908 static void accounted_request(struct cache *cache, struct bio *bio)
909 {
910 	accounted_begin(cache, bio);
911 	generic_make_request(bio);
912 }
913 
914 static void issue_op(struct bio *bio, void *context)
915 {
916 	struct cache *cache = context;
917 	accounted_request(cache, bio);
918 }
919 
920 /*
921  * When running in writethrough mode we need to send writes to clean blocks
922  * to both the cache and origin devices.  Clone the bio and send them in parallel.
923  */
924 static void remap_to_origin_and_cache(struct cache *cache, struct bio *bio,
925 				      dm_oblock_t oblock, dm_cblock_t cblock)
926 {
927 	struct bio *origin_bio = bio_clone_fast(bio, GFP_NOIO, cache->bs);
928 
929 	BUG_ON(!origin_bio);
930 
931 	bio_chain(origin_bio, bio);
932 	/*
933 	 * Passing false to __remap_to_origin_clear_discard() skips
934 	 * all code that might use per_bio_data (since clone doesn't have it)
935 	 */
936 	__remap_to_origin_clear_discard(cache, origin_bio, oblock, false);
937 	submit_bio(origin_bio);
938 
939 	remap_to_cache(cache, bio, cblock);
940 }
941 
942 /*----------------------------------------------------------------
943  * Failure modes
944  *--------------------------------------------------------------*/
945 static enum cache_metadata_mode get_cache_mode(struct cache *cache)
946 {
947 	return cache->features.mode;
948 }
949 
950 static const char *cache_device_name(struct cache *cache)
951 {
952 	return dm_device_name(dm_table_get_md(cache->ti->table));
953 }
954 
955 static void notify_mode_switch(struct cache *cache, enum cache_metadata_mode mode)
956 {
957 	const char *descs[] = {
958 		"write",
959 		"read-only",
960 		"fail"
961 	};
962 
963 	dm_table_event(cache->ti->table);
964 	DMINFO("%s: switching cache to %s mode",
965 	       cache_device_name(cache), descs[(int)mode]);
966 }
967 
968 static void set_cache_mode(struct cache *cache, enum cache_metadata_mode new_mode)
969 {
970 	bool needs_check;
971 	enum cache_metadata_mode old_mode = get_cache_mode(cache);
972 
973 	if (dm_cache_metadata_needs_check(cache->cmd, &needs_check)) {
974 		DMERR("%s: unable to read needs_check flag, setting failure mode.",
975 		      cache_device_name(cache));
976 		new_mode = CM_FAIL;
977 	}
978 
979 	if (new_mode == CM_WRITE && needs_check) {
980 		DMERR("%s: unable to switch cache to write mode until repaired.",
981 		      cache_device_name(cache));
982 		if (old_mode != new_mode)
983 			new_mode = old_mode;
984 		else
985 			new_mode = CM_READ_ONLY;
986 	}
987 
988 	/* Never move out of fail mode */
989 	if (old_mode == CM_FAIL)
990 		new_mode = CM_FAIL;
991 
992 	switch (new_mode) {
993 	case CM_FAIL:
994 	case CM_READ_ONLY:
995 		dm_cache_metadata_set_read_only(cache->cmd);
996 		break;
997 
998 	case CM_WRITE:
999 		dm_cache_metadata_set_read_write(cache->cmd);
1000 		break;
1001 	}
1002 
1003 	cache->features.mode = new_mode;
1004 
1005 	if (new_mode != old_mode)
1006 		notify_mode_switch(cache, new_mode);
1007 }
1008 
1009 static void abort_transaction(struct cache *cache)
1010 {
1011 	const char *dev_name = cache_device_name(cache);
1012 
1013 	if (get_cache_mode(cache) >= CM_READ_ONLY)
1014 		return;
1015 
1016 	if (dm_cache_metadata_set_needs_check(cache->cmd)) {
1017 		DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name);
1018 		set_cache_mode(cache, CM_FAIL);
1019 	}
1020 
1021 	DMERR_LIMIT("%s: aborting current metadata transaction", dev_name);
1022 	if (dm_cache_metadata_abort(cache->cmd)) {
1023 		DMERR("%s: failed to abort metadata transaction", dev_name);
1024 		set_cache_mode(cache, CM_FAIL);
1025 	}
1026 }
1027 
1028 static void metadata_operation_failed(struct cache *cache, const char *op, int r)
1029 {
1030 	DMERR_LIMIT("%s: metadata operation '%s' failed: error = %d",
1031 		    cache_device_name(cache), op, r);
1032 	abort_transaction(cache);
1033 	set_cache_mode(cache, CM_READ_ONLY);
1034 }
1035 
1036 /*----------------------------------------------------------------*/
1037 
1038 static void load_stats(struct cache *cache)
1039 {
1040 	struct dm_cache_statistics stats;
1041 
1042 	dm_cache_metadata_get_stats(cache->cmd, &stats);
1043 	atomic_set(&cache->stats.read_hit, stats.read_hits);
1044 	atomic_set(&cache->stats.read_miss, stats.read_misses);
1045 	atomic_set(&cache->stats.write_hit, stats.write_hits);
1046 	atomic_set(&cache->stats.write_miss, stats.write_misses);
1047 }
1048 
1049 static void save_stats(struct cache *cache)
1050 {
1051 	struct dm_cache_statistics stats;
1052 
1053 	if (get_cache_mode(cache) >= CM_READ_ONLY)
1054 		return;
1055 
1056 	stats.read_hits = atomic_read(&cache->stats.read_hit);
1057 	stats.read_misses = atomic_read(&cache->stats.read_miss);
1058 	stats.write_hits = atomic_read(&cache->stats.write_hit);
1059 	stats.write_misses = atomic_read(&cache->stats.write_miss);
1060 
1061 	dm_cache_metadata_set_stats(cache->cmd, &stats);
1062 }
1063 
1064 static void update_stats(struct cache_stats *stats, enum policy_operation op)
1065 {
1066 	switch (op) {
1067 	case POLICY_PROMOTE:
1068 		atomic_inc(&stats->promotion);
1069 		break;
1070 
1071 	case POLICY_DEMOTE:
1072 		atomic_inc(&stats->demotion);
1073 		break;
1074 
1075 	case POLICY_WRITEBACK:
1076 		atomic_inc(&stats->writeback);
1077 		break;
1078 	}
1079 }
1080 
1081 /*----------------------------------------------------------------
1082  * Migration processing
1083  *
1084  * Migration covers moving data from the origin device to the cache, or
1085  * vice versa.
1086  *--------------------------------------------------------------*/
1087 
1088 static void inc_io_migrations(struct cache *cache)
1089 {
1090 	atomic_inc(&cache->nr_io_migrations);
1091 }
1092 
1093 static void dec_io_migrations(struct cache *cache)
1094 {
1095 	atomic_dec(&cache->nr_io_migrations);
1096 }
1097 
1098 static bool discard_or_flush(struct bio *bio)
1099 {
1100 	return bio_op(bio) == REQ_OP_DISCARD || op_is_flush(bio->bi_opf);
1101 }
1102 
1103 static void calc_discard_block_range(struct cache *cache, struct bio *bio,
1104 				     dm_dblock_t *b, dm_dblock_t *e)
1105 {
1106 	sector_t sb = bio->bi_iter.bi_sector;
1107 	sector_t se = bio_end_sector(bio);
1108 
1109 	*b = to_dblock(dm_sector_div_up(sb, cache->discard_block_size));
1110 
1111 	if (se - sb < cache->discard_block_size)
1112 		*e = *b;
1113 	else
1114 		*e = to_dblock(block_div(se, cache->discard_block_size));
1115 }
1116 
1117 /*----------------------------------------------------------------*/
1118 
1119 static void prevent_background_work(struct cache *cache)
1120 {
1121 	lockdep_off();
1122 	down_write(&cache->background_work_lock);
1123 	lockdep_on();
1124 }
1125 
1126 static void allow_background_work(struct cache *cache)
1127 {
1128 	lockdep_off();
1129 	up_write(&cache->background_work_lock);
1130 	lockdep_on();
1131 }
1132 
1133 static bool background_work_begin(struct cache *cache)
1134 {
1135 	bool r;
1136 
1137 	lockdep_off();
1138 	r = down_read_trylock(&cache->background_work_lock);
1139 	lockdep_on();
1140 
1141 	return r;
1142 }
1143 
1144 static void background_work_end(struct cache *cache)
1145 {
1146 	lockdep_off();
1147 	up_read(&cache->background_work_lock);
1148 	lockdep_on();
1149 }
1150 
1151 /*----------------------------------------------------------------*/
1152 
1153 static bool bio_writes_complete_block(struct cache *cache, struct bio *bio)
1154 {
1155 	return (bio_data_dir(bio) == WRITE) &&
1156 		(bio->bi_iter.bi_size == (cache->sectors_per_block << SECTOR_SHIFT));
1157 }
1158 
1159 static bool optimisable_bio(struct cache *cache, struct bio *bio, dm_oblock_t block)
1160 {
1161 	return writeback_mode(cache) &&
1162 		(is_discarded_oblock(cache, block) || bio_writes_complete_block(cache, bio));
1163 }
1164 
1165 static void quiesce(struct dm_cache_migration *mg,
1166 		    void (*continuation)(struct work_struct *))
1167 {
1168 	init_continuation(&mg->k, continuation);
1169 	dm_cell_quiesce_v2(mg->cache->prison, mg->cell, &mg->k.ws);
1170 }
1171 
1172 static struct dm_cache_migration *ws_to_mg(struct work_struct *ws)
1173 {
1174 	struct continuation *k = container_of(ws, struct continuation, ws);
1175 	return container_of(k, struct dm_cache_migration, k);
1176 }
1177 
1178 static void copy_complete(int read_err, unsigned long write_err, void *context)
1179 {
1180 	struct dm_cache_migration *mg = container_of(context, struct dm_cache_migration, k);
1181 
1182 	if (read_err || write_err)
1183 		mg->k.input = BLK_STS_IOERR;
1184 
1185 	queue_continuation(mg->cache->wq, &mg->k);
1186 }
1187 
1188 static int copy(struct dm_cache_migration *mg, bool promote)
1189 {
1190 	int r;
1191 	struct dm_io_region o_region, c_region;
1192 	struct cache *cache = mg->cache;
1193 
1194 	o_region.bdev = cache->origin_dev->bdev;
1195 	o_region.sector = from_oblock(mg->op->oblock) * cache->sectors_per_block;
1196 	o_region.count = cache->sectors_per_block;
1197 
1198 	c_region.bdev = cache->cache_dev->bdev;
1199 	c_region.sector = from_cblock(mg->op->cblock) * cache->sectors_per_block;
1200 	c_region.count = cache->sectors_per_block;
1201 
1202 	if (promote)
1203 		r = dm_kcopyd_copy(cache->copier, &o_region, 1, &c_region, 0, copy_complete, &mg->k);
1204 	else
1205 		r = dm_kcopyd_copy(cache->copier, &c_region, 1, &o_region, 0, copy_complete, &mg->k);
1206 
1207 	return r;
1208 }
1209 
1210 static void bio_drop_shared_lock(struct cache *cache, struct bio *bio)
1211 {
1212 	struct per_bio_data *pb = get_per_bio_data(bio);
1213 
1214 	if (pb->cell && dm_cell_put_v2(cache->prison, pb->cell))
1215 		free_prison_cell(cache, pb->cell);
1216 	pb->cell = NULL;
1217 }
1218 
1219 static void overwrite_endio(struct bio *bio)
1220 {
1221 	struct dm_cache_migration *mg = bio->bi_private;
1222 	struct cache *cache = mg->cache;
1223 	struct per_bio_data *pb = get_per_bio_data(bio);
1224 
1225 	dm_unhook_bio(&pb->hook_info, bio);
1226 
1227 	if (bio->bi_status)
1228 		mg->k.input = bio->bi_status;
1229 
1230 	queue_continuation(cache->wq, &mg->k);
1231 }
1232 
1233 static void overwrite(struct dm_cache_migration *mg,
1234 		      void (*continuation)(struct work_struct *))
1235 {
1236 	struct bio *bio = mg->overwrite_bio;
1237 	struct per_bio_data *pb = get_per_bio_data(bio);
1238 
1239 	dm_hook_bio(&pb->hook_info, bio, overwrite_endio, mg);
1240 
1241 	/*
1242 	 * The overwrite bio is part of the copy operation, as such it does
1243 	 * not set/clear discard or dirty flags.
1244 	 */
1245 	if (mg->op->op == POLICY_PROMOTE)
1246 		remap_to_cache(mg->cache, bio, mg->op->cblock);
1247 	else
1248 		remap_to_origin(mg->cache, bio);
1249 
1250 	init_continuation(&mg->k, continuation);
1251 	accounted_request(mg->cache, bio);
1252 }
1253 
1254 /*
1255  * Migration steps:
1256  *
1257  * 1) exclusive lock preventing WRITEs
1258  * 2) quiesce
1259  * 3) copy or issue overwrite bio
1260  * 4) upgrade to exclusive lock preventing READs and WRITEs
1261  * 5) quiesce
1262  * 6) update metadata and commit
1263  * 7) unlock
1264  */
1265 static void mg_complete(struct dm_cache_migration *mg, bool success)
1266 {
1267 	struct bio_list bios;
1268 	struct cache *cache = mg->cache;
1269 	struct policy_work *op = mg->op;
1270 	dm_cblock_t cblock = op->cblock;
1271 
1272 	if (success)
1273 		update_stats(&cache->stats, op->op);
1274 
1275 	switch (op->op) {
1276 	case POLICY_PROMOTE:
1277 		clear_discard(cache, oblock_to_dblock(cache, op->oblock));
1278 		policy_complete_background_work(cache->policy, op, success);
1279 
1280 		if (mg->overwrite_bio) {
1281 			if (success)
1282 				force_set_dirty(cache, cblock);
1283 			else if (mg->k.input)
1284 				mg->overwrite_bio->bi_status = mg->k.input;
1285 			else
1286 				mg->overwrite_bio->bi_status = BLK_STS_IOERR;
1287 			bio_endio(mg->overwrite_bio);
1288 		} else {
1289 			if (success)
1290 				force_clear_dirty(cache, cblock);
1291 			dec_io_migrations(cache);
1292 		}
1293 		break;
1294 
1295 	case POLICY_DEMOTE:
1296 		/*
1297 		 * We clear dirty here to update the nr_dirty counter.
1298 		 */
1299 		if (success)
1300 			force_clear_dirty(cache, cblock);
1301 		policy_complete_background_work(cache->policy, op, success);
1302 		dec_io_migrations(cache);
1303 		break;
1304 
1305 	case POLICY_WRITEBACK:
1306 		if (success)
1307 			force_clear_dirty(cache, cblock);
1308 		policy_complete_background_work(cache->policy, op, success);
1309 		dec_io_migrations(cache);
1310 		break;
1311 	}
1312 
1313 	bio_list_init(&bios);
1314 	if (mg->cell) {
1315 		if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios))
1316 			free_prison_cell(cache, mg->cell);
1317 	}
1318 
1319 	free_migration(mg);
1320 	defer_bios(cache, &bios);
1321 	wake_migration_worker(cache);
1322 
1323 	background_work_end(cache);
1324 }
1325 
1326 static void mg_success(struct work_struct *ws)
1327 {
1328 	struct dm_cache_migration *mg = ws_to_mg(ws);
1329 	mg_complete(mg, mg->k.input == 0);
1330 }
1331 
1332 static void mg_update_metadata(struct work_struct *ws)
1333 {
1334 	int r;
1335 	struct dm_cache_migration *mg = ws_to_mg(ws);
1336 	struct cache *cache = mg->cache;
1337 	struct policy_work *op = mg->op;
1338 
1339 	switch (op->op) {
1340 	case POLICY_PROMOTE:
1341 		r = dm_cache_insert_mapping(cache->cmd, op->cblock, op->oblock);
1342 		if (r) {
1343 			DMERR_LIMIT("%s: migration failed; couldn't insert mapping",
1344 				    cache_device_name(cache));
1345 			metadata_operation_failed(cache, "dm_cache_insert_mapping", r);
1346 
1347 			mg_complete(mg, false);
1348 			return;
1349 		}
1350 		mg_complete(mg, true);
1351 		break;
1352 
1353 	case POLICY_DEMOTE:
1354 		r = dm_cache_remove_mapping(cache->cmd, op->cblock);
1355 		if (r) {
1356 			DMERR_LIMIT("%s: migration failed; couldn't update on disk metadata",
1357 				    cache_device_name(cache));
1358 			metadata_operation_failed(cache, "dm_cache_remove_mapping", r);
1359 
1360 			mg_complete(mg, false);
1361 			return;
1362 		}
1363 
1364 		/*
1365 		 * It would be nice if we only had to commit when a REQ_FLUSH
1366 		 * comes through.  But there's one scenario that we have to
1367 		 * look out for:
1368 		 *
1369 		 * - vblock x in a cache block
1370 		 * - domotion occurs
1371 		 * - cache block gets reallocated and over written
1372 		 * - crash
1373 		 *
1374 		 * When we recover, because there was no commit the cache will
1375 		 * rollback to having the data for vblock x in the cache block.
1376 		 * But the cache block has since been overwritten, so it'll end
1377 		 * up pointing to data that was never in 'x' during the history
1378 		 * of the device.
1379 		 *
1380 		 * To avoid this issue we require a commit as part of the
1381 		 * demotion operation.
1382 		 */
1383 		init_continuation(&mg->k, mg_success);
1384 		continue_after_commit(&cache->committer, &mg->k);
1385 		schedule_commit(&cache->committer);
1386 		break;
1387 
1388 	case POLICY_WRITEBACK:
1389 		mg_complete(mg, true);
1390 		break;
1391 	}
1392 }
1393 
1394 static void mg_update_metadata_after_copy(struct work_struct *ws)
1395 {
1396 	struct dm_cache_migration *mg = ws_to_mg(ws);
1397 
1398 	/*
1399 	 * Did the copy succeed?
1400 	 */
1401 	if (mg->k.input)
1402 		mg_complete(mg, false);
1403 	else
1404 		mg_update_metadata(ws);
1405 }
1406 
1407 static void mg_upgrade_lock(struct work_struct *ws)
1408 {
1409 	int r;
1410 	struct dm_cache_migration *mg = ws_to_mg(ws);
1411 
1412 	/*
1413 	 * Did the copy succeed?
1414 	 */
1415 	if (mg->k.input)
1416 		mg_complete(mg, false);
1417 
1418 	else {
1419 		/*
1420 		 * Now we want the lock to prevent both reads and writes.
1421 		 */
1422 		r = dm_cell_lock_promote_v2(mg->cache->prison, mg->cell,
1423 					    READ_WRITE_LOCK_LEVEL);
1424 		if (r < 0)
1425 			mg_complete(mg, false);
1426 
1427 		else if (r)
1428 			quiesce(mg, mg_update_metadata);
1429 
1430 		else
1431 			mg_update_metadata(ws);
1432 	}
1433 }
1434 
1435 static void mg_full_copy(struct work_struct *ws)
1436 {
1437 	struct dm_cache_migration *mg = ws_to_mg(ws);
1438 	struct cache *cache = mg->cache;
1439 	struct policy_work *op = mg->op;
1440 	bool is_policy_promote = (op->op == POLICY_PROMOTE);
1441 
1442 	if ((!is_policy_promote && !is_dirty(cache, op->cblock)) ||
1443 	    is_discarded_oblock(cache, op->oblock)) {
1444 		mg_upgrade_lock(ws);
1445 		return;
1446 	}
1447 
1448 	init_continuation(&mg->k, mg_upgrade_lock);
1449 
1450 	if (copy(mg, is_policy_promote)) {
1451 		DMERR_LIMIT("%s: migration copy failed", cache_device_name(cache));
1452 		mg->k.input = BLK_STS_IOERR;
1453 		mg_complete(mg, false);
1454 	}
1455 }
1456 
1457 static void mg_copy(struct work_struct *ws)
1458 {
1459 	struct dm_cache_migration *mg = ws_to_mg(ws);
1460 
1461 	if (mg->overwrite_bio) {
1462 		/*
1463 		 * No exclusive lock was held when we last checked if the bio
1464 		 * was optimisable.  So we have to check again in case things
1465 		 * have changed (eg, the block may no longer be discarded).
1466 		 */
1467 		if (!optimisable_bio(mg->cache, mg->overwrite_bio, mg->op->oblock)) {
1468 			/*
1469 			 * Fallback to a real full copy after doing some tidying up.
1470 			 */
1471 			bool rb = bio_detain_shared(mg->cache, mg->op->oblock, mg->overwrite_bio);
1472 			BUG_ON(rb); /* An exclussive lock must _not_ be held for this block */
1473 			mg->overwrite_bio = NULL;
1474 			inc_io_migrations(mg->cache);
1475 			mg_full_copy(ws);
1476 			return;
1477 		}
1478 
1479 		/*
1480 		 * It's safe to do this here, even though it's new data
1481 		 * because all IO has been locked out of the block.
1482 		 *
1483 		 * mg_lock_writes() already took READ_WRITE_LOCK_LEVEL
1484 		 * so _not_ using mg_upgrade_lock() as continutation.
1485 		 */
1486 		overwrite(mg, mg_update_metadata_after_copy);
1487 
1488 	} else
1489 		mg_full_copy(ws);
1490 }
1491 
1492 static int mg_lock_writes(struct dm_cache_migration *mg)
1493 {
1494 	int r;
1495 	struct dm_cell_key_v2 key;
1496 	struct cache *cache = mg->cache;
1497 	struct dm_bio_prison_cell_v2 *prealloc;
1498 
1499 	prealloc = alloc_prison_cell(cache);
1500 	if (!prealloc) {
1501 		DMERR_LIMIT("%s: alloc_prison_cell failed", cache_device_name(cache));
1502 		mg_complete(mg, false);
1503 		return -ENOMEM;
1504 	}
1505 
1506 	/*
1507 	 * Prevent writes to the block, but allow reads to continue.
1508 	 * Unless we're using an overwrite bio, in which case we lock
1509 	 * everything.
1510 	 */
1511 	build_key(mg->op->oblock, oblock_succ(mg->op->oblock), &key);
1512 	r = dm_cell_lock_v2(cache->prison, &key,
1513 			    mg->overwrite_bio ?  READ_WRITE_LOCK_LEVEL : WRITE_LOCK_LEVEL,
1514 			    prealloc, &mg->cell);
1515 	if (r < 0) {
1516 		free_prison_cell(cache, prealloc);
1517 		mg_complete(mg, false);
1518 		return r;
1519 	}
1520 
1521 	if (mg->cell != prealloc)
1522 		free_prison_cell(cache, prealloc);
1523 
1524 	if (r == 0)
1525 		mg_copy(&mg->k.ws);
1526 	else
1527 		quiesce(mg, mg_copy);
1528 
1529 	return 0;
1530 }
1531 
1532 static int mg_start(struct cache *cache, struct policy_work *op, struct bio *bio)
1533 {
1534 	struct dm_cache_migration *mg;
1535 
1536 	if (!background_work_begin(cache)) {
1537 		policy_complete_background_work(cache->policy, op, false);
1538 		return -EPERM;
1539 	}
1540 
1541 	mg = alloc_migration(cache);
1542 	if (!mg) {
1543 		policy_complete_background_work(cache->policy, op, false);
1544 		background_work_end(cache);
1545 		return -ENOMEM;
1546 	}
1547 
1548 	mg->op = op;
1549 	mg->overwrite_bio = bio;
1550 
1551 	if (!bio)
1552 		inc_io_migrations(cache);
1553 
1554 	return mg_lock_writes(mg);
1555 }
1556 
1557 /*----------------------------------------------------------------
1558  * invalidation processing
1559  *--------------------------------------------------------------*/
1560 
1561 static void invalidate_complete(struct dm_cache_migration *mg, bool success)
1562 {
1563 	struct bio_list bios;
1564 	struct cache *cache = mg->cache;
1565 
1566 	bio_list_init(&bios);
1567 	if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios))
1568 		free_prison_cell(cache, mg->cell);
1569 
1570 	if (!success && mg->overwrite_bio)
1571 		bio_io_error(mg->overwrite_bio);
1572 
1573 	free_migration(mg);
1574 	defer_bios(cache, &bios);
1575 
1576 	background_work_end(cache);
1577 }
1578 
1579 static void invalidate_completed(struct work_struct *ws)
1580 {
1581 	struct dm_cache_migration *mg = ws_to_mg(ws);
1582 	invalidate_complete(mg, !mg->k.input);
1583 }
1584 
1585 static int invalidate_cblock(struct cache *cache, dm_cblock_t cblock)
1586 {
1587 	int r = policy_invalidate_mapping(cache->policy, cblock);
1588 	if (!r) {
1589 		r = dm_cache_remove_mapping(cache->cmd, cblock);
1590 		if (r) {
1591 			DMERR_LIMIT("%s: invalidation failed; couldn't update on disk metadata",
1592 				    cache_device_name(cache));
1593 			metadata_operation_failed(cache, "dm_cache_remove_mapping", r);
1594 		}
1595 
1596 	} else if (r == -ENODATA) {
1597 		/*
1598 		 * Harmless, already unmapped.
1599 		 */
1600 		r = 0;
1601 
1602 	} else
1603 		DMERR("%s: policy_invalidate_mapping failed", cache_device_name(cache));
1604 
1605 	return r;
1606 }
1607 
1608 static void invalidate_remove(struct work_struct *ws)
1609 {
1610 	int r;
1611 	struct dm_cache_migration *mg = ws_to_mg(ws);
1612 	struct cache *cache = mg->cache;
1613 
1614 	r = invalidate_cblock(cache, mg->invalidate_cblock);
1615 	if (r) {
1616 		invalidate_complete(mg, false);
1617 		return;
1618 	}
1619 
1620 	init_continuation(&mg->k, invalidate_completed);
1621 	continue_after_commit(&cache->committer, &mg->k);
1622 	remap_to_origin_clear_discard(cache, mg->overwrite_bio, mg->invalidate_oblock);
1623 	mg->overwrite_bio = NULL;
1624 	schedule_commit(&cache->committer);
1625 }
1626 
1627 static int invalidate_lock(struct dm_cache_migration *mg)
1628 {
1629 	int r;
1630 	struct dm_cell_key_v2 key;
1631 	struct cache *cache = mg->cache;
1632 	struct dm_bio_prison_cell_v2 *prealloc;
1633 
1634 	prealloc = alloc_prison_cell(cache);
1635 	if (!prealloc) {
1636 		invalidate_complete(mg, false);
1637 		return -ENOMEM;
1638 	}
1639 
1640 	build_key(mg->invalidate_oblock, oblock_succ(mg->invalidate_oblock), &key);
1641 	r = dm_cell_lock_v2(cache->prison, &key,
1642 			    READ_WRITE_LOCK_LEVEL, prealloc, &mg->cell);
1643 	if (r < 0) {
1644 		free_prison_cell(cache, prealloc);
1645 		invalidate_complete(mg, false);
1646 		return r;
1647 	}
1648 
1649 	if (mg->cell != prealloc)
1650 		free_prison_cell(cache, prealloc);
1651 
1652 	if (r)
1653 		quiesce(mg, invalidate_remove);
1654 
1655 	else {
1656 		/*
1657 		 * We can't call invalidate_remove() directly here because we
1658 		 * might still be in request context.
1659 		 */
1660 		init_continuation(&mg->k, invalidate_remove);
1661 		queue_work(cache->wq, &mg->k.ws);
1662 	}
1663 
1664 	return 0;
1665 }
1666 
1667 static int invalidate_start(struct cache *cache, dm_cblock_t cblock,
1668 			    dm_oblock_t oblock, struct bio *bio)
1669 {
1670 	struct dm_cache_migration *mg;
1671 
1672 	if (!background_work_begin(cache))
1673 		return -EPERM;
1674 
1675 	mg = alloc_migration(cache);
1676 	if (!mg) {
1677 		background_work_end(cache);
1678 		return -ENOMEM;
1679 	}
1680 
1681 	mg->overwrite_bio = bio;
1682 	mg->invalidate_cblock = cblock;
1683 	mg->invalidate_oblock = oblock;
1684 
1685 	return invalidate_lock(mg);
1686 }
1687 
1688 /*----------------------------------------------------------------
1689  * bio processing
1690  *--------------------------------------------------------------*/
1691 
1692 enum busy {
1693 	IDLE,
1694 	BUSY
1695 };
1696 
1697 static enum busy spare_migration_bandwidth(struct cache *cache)
1698 {
1699 	bool idle = iot_idle_for(&cache->tracker, HZ);
1700 	sector_t current_volume = (atomic_read(&cache->nr_io_migrations) + 1) *
1701 		cache->sectors_per_block;
1702 
1703 	if (idle && current_volume <= cache->migration_threshold)
1704 		return IDLE;
1705 	else
1706 		return BUSY;
1707 }
1708 
1709 static void inc_hit_counter(struct cache *cache, struct bio *bio)
1710 {
1711 	atomic_inc(bio_data_dir(bio) == READ ?
1712 		   &cache->stats.read_hit : &cache->stats.write_hit);
1713 }
1714 
1715 static void inc_miss_counter(struct cache *cache, struct bio *bio)
1716 {
1717 	atomic_inc(bio_data_dir(bio) == READ ?
1718 		   &cache->stats.read_miss : &cache->stats.write_miss);
1719 }
1720 
1721 /*----------------------------------------------------------------*/
1722 
1723 static int map_bio(struct cache *cache, struct bio *bio, dm_oblock_t block,
1724 		   bool *commit_needed)
1725 {
1726 	int r, data_dir;
1727 	bool rb, background_queued;
1728 	dm_cblock_t cblock;
1729 
1730 	*commit_needed = false;
1731 
1732 	rb = bio_detain_shared(cache, block, bio);
1733 	if (!rb) {
1734 		/*
1735 		 * An exclusive lock is held for this block, so we have to
1736 		 * wait.  We set the commit_needed flag so the current
1737 		 * transaction will be committed asap, allowing this lock
1738 		 * to be dropped.
1739 		 */
1740 		*commit_needed = true;
1741 		return DM_MAPIO_SUBMITTED;
1742 	}
1743 
1744 	data_dir = bio_data_dir(bio);
1745 
1746 	if (optimisable_bio(cache, bio, block)) {
1747 		struct policy_work *op = NULL;
1748 
1749 		r = policy_lookup_with_work(cache->policy, block, &cblock, data_dir, true, &op);
1750 		if (unlikely(r && r != -ENOENT)) {
1751 			DMERR_LIMIT("%s: policy_lookup_with_work() failed with r = %d",
1752 				    cache_device_name(cache), r);
1753 			bio_io_error(bio);
1754 			return DM_MAPIO_SUBMITTED;
1755 		}
1756 
1757 		if (r == -ENOENT && op) {
1758 			bio_drop_shared_lock(cache, bio);
1759 			BUG_ON(op->op != POLICY_PROMOTE);
1760 			mg_start(cache, op, bio);
1761 			return DM_MAPIO_SUBMITTED;
1762 		}
1763 	} else {
1764 		r = policy_lookup(cache->policy, block, &cblock, data_dir, false, &background_queued);
1765 		if (unlikely(r && r != -ENOENT)) {
1766 			DMERR_LIMIT("%s: policy_lookup() failed with r = %d",
1767 				    cache_device_name(cache), r);
1768 			bio_io_error(bio);
1769 			return DM_MAPIO_SUBMITTED;
1770 		}
1771 
1772 		if (background_queued)
1773 			wake_migration_worker(cache);
1774 	}
1775 
1776 	if (r == -ENOENT) {
1777 		struct per_bio_data *pb = get_per_bio_data(bio);
1778 
1779 		/*
1780 		 * Miss.
1781 		 */
1782 		inc_miss_counter(cache, bio);
1783 		if (pb->req_nr == 0) {
1784 			accounted_begin(cache, bio);
1785 			remap_to_origin_clear_discard(cache, bio, block);
1786 		} else {
1787 			/*
1788 			 * This is a duplicate writethrough io that is no
1789 			 * longer needed because the block has been demoted.
1790 			 */
1791 			bio_endio(bio);
1792 			return DM_MAPIO_SUBMITTED;
1793 		}
1794 	} else {
1795 		/*
1796 		 * Hit.
1797 		 */
1798 		inc_hit_counter(cache, bio);
1799 
1800 		/*
1801 		 * Passthrough always maps to the origin, invalidating any
1802 		 * cache blocks that are written to.
1803 		 */
1804 		if (passthrough_mode(cache)) {
1805 			if (bio_data_dir(bio) == WRITE) {
1806 				bio_drop_shared_lock(cache, bio);
1807 				atomic_inc(&cache->stats.demotion);
1808 				invalidate_start(cache, cblock, block, bio);
1809 			} else
1810 				remap_to_origin_clear_discard(cache, bio, block);
1811 		} else {
1812 			if (bio_data_dir(bio) == WRITE && writethrough_mode(cache) &&
1813 			    !is_dirty(cache, cblock)) {
1814 				remap_to_origin_and_cache(cache, bio, block, cblock);
1815 				accounted_begin(cache, bio);
1816 			} else
1817 				remap_to_cache_dirty(cache, bio, block, cblock);
1818 		}
1819 	}
1820 
1821 	/*
1822 	 * dm core turns FUA requests into a separate payload and FLUSH req.
1823 	 */
1824 	if (bio->bi_opf & REQ_FUA) {
1825 		/*
1826 		 * issue_after_commit will call accounted_begin a second time.  So
1827 		 * we call accounted_complete() to avoid double accounting.
1828 		 */
1829 		accounted_complete(cache, bio);
1830 		issue_after_commit(&cache->committer, bio);
1831 		*commit_needed = true;
1832 		return DM_MAPIO_SUBMITTED;
1833 	}
1834 
1835 	return DM_MAPIO_REMAPPED;
1836 }
1837 
1838 static bool process_bio(struct cache *cache, struct bio *bio)
1839 {
1840 	bool commit_needed;
1841 
1842 	if (map_bio(cache, bio, get_bio_block(cache, bio), &commit_needed) == DM_MAPIO_REMAPPED)
1843 		generic_make_request(bio);
1844 
1845 	return commit_needed;
1846 }
1847 
1848 /*
1849  * A non-zero return indicates read_only or fail_io mode.
1850  */
1851 static int commit(struct cache *cache, bool clean_shutdown)
1852 {
1853 	int r;
1854 
1855 	if (get_cache_mode(cache) >= CM_READ_ONLY)
1856 		return -EINVAL;
1857 
1858 	atomic_inc(&cache->stats.commit_count);
1859 	r = dm_cache_commit(cache->cmd, clean_shutdown);
1860 	if (r)
1861 		metadata_operation_failed(cache, "dm_cache_commit", r);
1862 
1863 	return r;
1864 }
1865 
1866 /*
1867  * Used by the batcher.
1868  */
1869 static blk_status_t commit_op(void *context)
1870 {
1871 	struct cache *cache = context;
1872 
1873 	if (dm_cache_changed_this_transaction(cache->cmd))
1874 		return errno_to_blk_status(commit(cache, false));
1875 
1876 	return 0;
1877 }
1878 
1879 /*----------------------------------------------------------------*/
1880 
1881 static bool process_flush_bio(struct cache *cache, struct bio *bio)
1882 {
1883 	struct per_bio_data *pb = get_per_bio_data(bio);
1884 
1885 	if (!pb->req_nr)
1886 		remap_to_origin(cache, bio);
1887 	else
1888 		remap_to_cache(cache, bio, 0);
1889 
1890 	issue_after_commit(&cache->committer, bio);
1891 	return true;
1892 }
1893 
1894 static bool process_discard_bio(struct cache *cache, struct bio *bio)
1895 {
1896 	dm_dblock_t b, e;
1897 
1898 	// FIXME: do we need to lock the region?  Or can we just assume the
1899 	// user wont be so foolish as to issue discard concurrently with
1900 	// other IO?
1901 	calc_discard_block_range(cache, bio, &b, &e);
1902 	while (b != e) {
1903 		set_discard(cache, b);
1904 		b = to_dblock(from_dblock(b) + 1);
1905 	}
1906 
1907 	bio_endio(bio);
1908 
1909 	return false;
1910 }
1911 
1912 static void process_deferred_bios(struct work_struct *ws)
1913 {
1914 	struct cache *cache = container_of(ws, struct cache, deferred_bio_worker);
1915 
1916 	unsigned long flags;
1917 	bool commit_needed = false;
1918 	struct bio_list bios;
1919 	struct bio *bio;
1920 
1921 	bio_list_init(&bios);
1922 
1923 	spin_lock_irqsave(&cache->lock, flags);
1924 	bio_list_merge(&bios, &cache->deferred_bios);
1925 	bio_list_init(&cache->deferred_bios);
1926 	spin_unlock_irqrestore(&cache->lock, flags);
1927 
1928 	while ((bio = bio_list_pop(&bios))) {
1929 		if (bio->bi_opf & REQ_PREFLUSH)
1930 			commit_needed = process_flush_bio(cache, bio) || commit_needed;
1931 
1932 		else if (bio_op(bio) == REQ_OP_DISCARD)
1933 			commit_needed = process_discard_bio(cache, bio) || commit_needed;
1934 
1935 		else
1936 			commit_needed = process_bio(cache, bio) || commit_needed;
1937 	}
1938 
1939 	if (commit_needed)
1940 		schedule_commit(&cache->committer);
1941 }
1942 
1943 /*----------------------------------------------------------------
1944  * Main worker loop
1945  *--------------------------------------------------------------*/
1946 
1947 static void requeue_deferred_bios(struct cache *cache)
1948 {
1949 	struct bio *bio;
1950 	struct bio_list bios;
1951 
1952 	bio_list_init(&bios);
1953 	bio_list_merge(&bios, &cache->deferred_bios);
1954 	bio_list_init(&cache->deferred_bios);
1955 
1956 	while ((bio = bio_list_pop(&bios))) {
1957 		bio->bi_status = BLK_STS_DM_REQUEUE;
1958 		bio_endio(bio);
1959 	}
1960 }
1961 
1962 /*
1963  * We want to commit periodically so that not too much
1964  * unwritten metadata builds up.
1965  */
1966 static void do_waker(struct work_struct *ws)
1967 {
1968 	struct cache *cache = container_of(to_delayed_work(ws), struct cache, waker);
1969 
1970 	policy_tick(cache->policy, true);
1971 	wake_migration_worker(cache);
1972 	schedule_commit(&cache->committer);
1973 	queue_delayed_work(cache->wq, &cache->waker, COMMIT_PERIOD);
1974 }
1975 
1976 static void check_migrations(struct work_struct *ws)
1977 {
1978 	int r;
1979 	struct policy_work *op;
1980 	struct cache *cache = container_of(ws, struct cache, migration_worker);
1981 	enum busy b;
1982 
1983 	for (;;) {
1984 		b = spare_migration_bandwidth(cache);
1985 
1986 		r = policy_get_background_work(cache->policy, b == IDLE, &op);
1987 		if (r == -ENODATA)
1988 			break;
1989 
1990 		if (r) {
1991 			DMERR_LIMIT("%s: policy_background_work failed",
1992 				    cache_device_name(cache));
1993 			break;
1994 		}
1995 
1996 		r = mg_start(cache, op, NULL);
1997 		if (r)
1998 			break;
1999 	}
2000 }
2001 
2002 /*----------------------------------------------------------------
2003  * Target methods
2004  *--------------------------------------------------------------*/
2005 
2006 /*
2007  * This function gets called on the error paths of the constructor, so we
2008  * have to cope with a partially initialised struct.
2009  */
2010 static void destroy(struct cache *cache)
2011 {
2012 	unsigned i;
2013 
2014 	mempool_destroy(cache->migration_pool);
2015 
2016 	if (cache->prison)
2017 		dm_bio_prison_destroy_v2(cache->prison);
2018 
2019 	if (cache->wq)
2020 		destroy_workqueue(cache->wq);
2021 
2022 	if (cache->dirty_bitset)
2023 		free_bitset(cache->dirty_bitset);
2024 
2025 	if (cache->discard_bitset)
2026 		free_bitset(cache->discard_bitset);
2027 
2028 	if (cache->copier)
2029 		dm_kcopyd_client_destroy(cache->copier);
2030 
2031 	if (cache->cmd)
2032 		dm_cache_metadata_close(cache->cmd);
2033 
2034 	if (cache->metadata_dev)
2035 		dm_put_device(cache->ti, cache->metadata_dev);
2036 
2037 	if (cache->origin_dev)
2038 		dm_put_device(cache->ti, cache->origin_dev);
2039 
2040 	if (cache->cache_dev)
2041 		dm_put_device(cache->ti, cache->cache_dev);
2042 
2043 	if (cache->policy)
2044 		dm_cache_policy_destroy(cache->policy);
2045 
2046 	for (i = 0; i < cache->nr_ctr_args ; i++)
2047 		kfree(cache->ctr_args[i]);
2048 	kfree(cache->ctr_args);
2049 
2050 	if (cache->bs)
2051 		bioset_free(cache->bs);
2052 
2053 	kfree(cache);
2054 }
2055 
2056 static void cache_dtr(struct dm_target *ti)
2057 {
2058 	struct cache *cache = ti->private;
2059 
2060 	destroy(cache);
2061 }
2062 
2063 static sector_t get_dev_size(struct dm_dev *dev)
2064 {
2065 	return i_size_read(dev->bdev->bd_inode) >> SECTOR_SHIFT;
2066 }
2067 
2068 /*----------------------------------------------------------------*/
2069 
2070 /*
2071  * Construct a cache device mapping.
2072  *
2073  * cache <metadata dev> <cache dev> <origin dev> <block size>
2074  *       <#feature args> [<feature arg>]*
2075  *       <policy> <#policy args> [<policy arg>]*
2076  *
2077  * metadata dev    : fast device holding the persistent metadata
2078  * cache dev	   : fast device holding cached data blocks
2079  * origin dev	   : slow device holding original data blocks
2080  * block size	   : cache unit size in sectors
2081  *
2082  * #feature args   : number of feature arguments passed
2083  * feature args    : writethrough.  (The default is writeback.)
2084  *
2085  * policy	   : the replacement policy to use
2086  * #policy args    : an even number of policy arguments corresponding
2087  *		     to key/value pairs passed to the policy
2088  * policy args	   : key/value pairs passed to the policy
2089  *		     E.g. 'sequential_threshold 1024'
2090  *		     See cache-policies.txt for details.
2091  *
2092  * Optional feature arguments are:
2093  *   writethrough  : write through caching that prohibits cache block
2094  *		     content from being different from origin block content.
2095  *		     Without this argument, the default behaviour is to write
2096  *		     back cache block contents later for performance reasons,
2097  *		     so they may differ from the corresponding origin blocks.
2098  */
2099 struct cache_args {
2100 	struct dm_target *ti;
2101 
2102 	struct dm_dev *metadata_dev;
2103 
2104 	struct dm_dev *cache_dev;
2105 	sector_t cache_sectors;
2106 
2107 	struct dm_dev *origin_dev;
2108 	sector_t origin_sectors;
2109 
2110 	uint32_t block_size;
2111 
2112 	const char *policy_name;
2113 	int policy_argc;
2114 	const char **policy_argv;
2115 
2116 	struct cache_features features;
2117 };
2118 
2119 static void destroy_cache_args(struct cache_args *ca)
2120 {
2121 	if (ca->metadata_dev)
2122 		dm_put_device(ca->ti, ca->metadata_dev);
2123 
2124 	if (ca->cache_dev)
2125 		dm_put_device(ca->ti, ca->cache_dev);
2126 
2127 	if (ca->origin_dev)
2128 		dm_put_device(ca->ti, ca->origin_dev);
2129 
2130 	kfree(ca);
2131 }
2132 
2133 static bool at_least_one_arg(struct dm_arg_set *as, char **error)
2134 {
2135 	if (!as->argc) {
2136 		*error = "Insufficient args";
2137 		return false;
2138 	}
2139 
2140 	return true;
2141 }
2142 
2143 static int parse_metadata_dev(struct cache_args *ca, struct dm_arg_set *as,
2144 			      char **error)
2145 {
2146 	int r;
2147 	sector_t metadata_dev_size;
2148 	char b[BDEVNAME_SIZE];
2149 
2150 	if (!at_least_one_arg(as, error))
2151 		return -EINVAL;
2152 
2153 	r = dm_get_device(ca->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
2154 			  &ca->metadata_dev);
2155 	if (r) {
2156 		*error = "Error opening metadata device";
2157 		return r;
2158 	}
2159 
2160 	metadata_dev_size = get_dev_size(ca->metadata_dev);
2161 	if (metadata_dev_size > DM_CACHE_METADATA_MAX_SECTORS_WARNING)
2162 		DMWARN("Metadata device %s is larger than %u sectors: excess space will not be used.",
2163 		       bdevname(ca->metadata_dev->bdev, b), THIN_METADATA_MAX_SECTORS);
2164 
2165 	return 0;
2166 }
2167 
2168 static int parse_cache_dev(struct cache_args *ca, struct dm_arg_set *as,
2169 			   char **error)
2170 {
2171 	int r;
2172 
2173 	if (!at_least_one_arg(as, error))
2174 		return -EINVAL;
2175 
2176 	r = dm_get_device(ca->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
2177 			  &ca->cache_dev);
2178 	if (r) {
2179 		*error = "Error opening cache device";
2180 		return r;
2181 	}
2182 	ca->cache_sectors = get_dev_size(ca->cache_dev);
2183 
2184 	return 0;
2185 }
2186 
2187 static int parse_origin_dev(struct cache_args *ca, struct dm_arg_set *as,
2188 			    char **error)
2189 {
2190 	int r;
2191 
2192 	if (!at_least_one_arg(as, error))
2193 		return -EINVAL;
2194 
2195 	r = dm_get_device(ca->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
2196 			  &ca->origin_dev);
2197 	if (r) {
2198 		*error = "Error opening origin device";
2199 		return r;
2200 	}
2201 
2202 	ca->origin_sectors = get_dev_size(ca->origin_dev);
2203 	if (ca->ti->len > ca->origin_sectors) {
2204 		*error = "Device size larger than cached device";
2205 		return -EINVAL;
2206 	}
2207 
2208 	return 0;
2209 }
2210 
2211 static int parse_block_size(struct cache_args *ca, struct dm_arg_set *as,
2212 			    char **error)
2213 {
2214 	unsigned long block_size;
2215 
2216 	if (!at_least_one_arg(as, error))
2217 		return -EINVAL;
2218 
2219 	if (kstrtoul(dm_shift_arg(as), 10, &block_size) || !block_size ||
2220 	    block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS ||
2221 	    block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS ||
2222 	    block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) {
2223 		*error = "Invalid data block size";
2224 		return -EINVAL;
2225 	}
2226 
2227 	if (block_size > ca->cache_sectors) {
2228 		*error = "Data block size is larger than the cache device";
2229 		return -EINVAL;
2230 	}
2231 
2232 	ca->block_size = block_size;
2233 
2234 	return 0;
2235 }
2236 
2237 static void init_features(struct cache_features *cf)
2238 {
2239 	cf->mode = CM_WRITE;
2240 	cf->io_mode = CM_IO_WRITEBACK;
2241 	cf->metadata_version = 1;
2242 }
2243 
2244 static int parse_features(struct cache_args *ca, struct dm_arg_set *as,
2245 			  char **error)
2246 {
2247 	static const struct dm_arg _args[] = {
2248 		{0, 2, "Invalid number of cache feature arguments"},
2249 	};
2250 
2251 	int r;
2252 	unsigned argc;
2253 	const char *arg;
2254 	struct cache_features *cf = &ca->features;
2255 
2256 	init_features(cf);
2257 
2258 	r = dm_read_arg_group(_args, as, &argc, error);
2259 	if (r)
2260 		return -EINVAL;
2261 
2262 	while (argc--) {
2263 		arg = dm_shift_arg(as);
2264 
2265 		if (!strcasecmp(arg, "writeback"))
2266 			cf->io_mode = CM_IO_WRITEBACK;
2267 
2268 		else if (!strcasecmp(arg, "writethrough"))
2269 			cf->io_mode = CM_IO_WRITETHROUGH;
2270 
2271 		else if (!strcasecmp(arg, "passthrough"))
2272 			cf->io_mode = CM_IO_PASSTHROUGH;
2273 
2274 		else if (!strcasecmp(arg, "metadata2"))
2275 			cf->metadata_version = 2;
2276 
2277 		else {
2278 			*error = "Unrecognised cache feature requested";
2279 			return -EINVAL;
2280 		}
2281 	}
2282 
2283 	return 0;
2284 }
2285 
2286 static int parse_policy(struct cache_args *ca, struct dm_arg_set *as,
2287 			char **error)
2288 {
2289 	static const struct dm_arg _args[] = {
2290 		{0, 1024, "Invalid number of policy arguments"},
2291 	};
2292 
2293 	int r;
2294 
2295 	if (!at_least_one_arg(as, error))
2296 		return -EINVAL;
2297 
2298 	ca->policy_name = dm_shift_arg(as);
2299 
2300 	r = dm_read_arg_group(_args, as, &ca->policy_argc, error);
2301 	if (r)
2302 		return -EINVAL;
2303 
2304 	ca->policy_argv = (const char **)as->argv;
2305 	dm_consume_args(as, ca->policy_argc);
2306 
2307 	return 0;
2308 }
2309 
2310 static int parse_cache_args(struct cache_args *ca, int argc, char **argv,
2311 			    char **error)
2312 {
2313 	int r;
2314 	struct dm_arg_set as;
2315 
2316 	as.argc = argc;
2317 	as.argv = argv;
2318 
2319 	r = parse_metadata_dev(ca, &as, error);
2320 	if (r)
2321 		return r;
2322 
2323 	r = parse_cache_dev(ca, &as, error);
2324 	if (r)
2325 		return r;
2326 
2327 	r = parse_origin_dev(ca, &as, error);
2328 	if (r)
2329 		return r;
2330 
2331 	r = parse_block_size(ca, &as, error);
2332 	if (r)
2333 		return r;
2334 
2335 	r = parse_features(ca, &as, error);
2336 	if (r)
2337 		return r;
2338 
2339 	r = parse_policy(ca, &as, error);
2340 	if (r)
2341 		return r;
2342 
2343 	return 0;
2344 }
2345 
2346 /*----------------------------------------------------------------*/
2347 
2348 static struct kmem_cache *migration_cache;
2349 
2350 #define NOT_CORE_OPTION 1
2351 
2352 static int process_config_option(struct cache *cache, const char *key, const char *value)
2353 {
2354 	unsigned long tmp;
2355 
2356 	if (!strcasecmp(key, "migration_threshold")) {
2357 		if (kstrtoul(value, 10, &tmp))
2358 			return -EINVAL;
2359 
2360 		cache->migration_threshold = tmp;
2361 		return 0;
2362 	}
2363 
2364 	return NOT_CORE_OPTION;
2365 }
2366 
2367 static int set_config_value(struct cache *cache, const char *key, const char *value)
2368 {
2369 	int r = process_config_option(cache, key, value);
2370 
2371 	if (r == NOT_CORE_OPTION)
2372 		r = policy_set_config_value(cache->policy, key, value);
2373 
2374 	if (r)
2375 		DMWARN("bad config value for %s: %s", key, value);
2376 
2377 	return r;
2378 }
2379 
2380 static int set_config_values(struct cache *cache, int argc, const char **argv)
2381 {
2382 	int r = 0;
2383 
2384 	if (argc & 1) {
2385 		DMWARN("Odd number of policy arguments given but they should be <key> <value> pairs.");
2386 		return -EINVAL;
2387 	}
2388 
2389 	while (argc) {
2390 		r = set_config_value(cache, argv[0], argv[1]);
2391 		if (r)
2392 			break;
2393 
2394 		argc -= 2;
2395 		argv += 2;
2396 	}
2397 
2398 	return r;
2399 }
2400 
2401 static int create_cache_policy(struct cache *cache, struct cache_args *ca,
2402 			       char **error)
2403 {
2404 	struct dm_cache_policy *p = dm_cache_policy_create(ca->policy_name,
2405 							   cache->cache_size,
2406 							   cache->origin_sectors,
2407 							   cache->sectors_per_block);
2408 	if (IS_ERR(p)) {
2409 		*error = "Error creating cache's policy";
2410 		return PTR_ERR(p);
2411 	}
2412 	cache->policy = p;
2413 	BUG_ON(!cache->policy);
2414 
2415 	return 0;
2416 }
2417 
2418 /*
2419  * We want the discard block size to be at least the size of the cache
2420  * block size and have no more than 2^14 discard blocks across the origin.
2421  */
2422 #define MAX_DISCARD_BLOCKS (1 << 14)
2423 
2424 static bool too_many_discard_blocks(sector_t discard_block_size,
2425 				    sector_t origin_size)
2426 {
2427 	(void) sector_div(origin_size, discard_block_size);
2428 
2429 	return origin_size > MAX_DISCARD_BLOCKS;
2430 }
2431 
2432 static sector_t calculate_discard_block_size(sector_t cache_block_size,
2433 					     sector_t origin_size)
2434 {
2435 	sector_t discard_block_size = cache_block_size;
2436 
2437 	if (origin_size)
2438 		while (too_many_discard_blocks(discard_block_size, origin_size))
2439 			discard_block_size *= 2;
2440 
2441 	return discard_block_size;
2442 }
2443 
2444 static void set_cache_size(struct cache *cache, dm_cblock_t size)
2445 {
2446 	dm_block_t nr_blocks = from_cblock(size);
2447 
2448 	if (nr_blocks > (1 << 20) && cache->cache_size != size)
2449 		DMWARN_LIMIT("You have created a cache device with a lot of individual cache blocks (%llu)\n"
2450 			     "All these mappings can consume a lot of kernel memory, and take some time to read/write.\n"
2451 			     "Please consider increasing the cache block size to reduce the overall cache block count.",
2452 			     (unsigned long long) nr_blocks);
2453 
2454 	cache->cache_size = size;
2455 }
2456 
2457 static int is_congested(struct dm_dev *dev, int bdi_bits)
2458 {
2459 	struct request_queue *q = bdev_get_queue(dev->bdev);
2460 	return bdi_congested(q->backing_dev_info, bdi_bits);
2461 }
2462 
2463 static int cache_is_congested(struct dm_target_callbacks *cb, int bdi_bits)
2464 {
2465 	struct cache *cache = container_of(cb, struct cache, callbacks);
2466 
2467 	return is_congested(cache->origin_dev, bdi_bits) ||
2468 		is_congested(cache->cache_dev, bdi_bits);
2469 }
2470 
2471 #define DEFAULT_MIGRATION_THRESHOLD 2048
2472 
2473 static int cache_create(struct cache_args *ca, struct cache **result)
2474 {
2475 	int r = 0;
2476 	char **error = &ca->ti->error;
2477 	struct cache *cache;
2478 	struct dm_target *ti = ca->ti;
2479 	dm_block_t origin_blocks;
2480 	struct dm_cache_metadata *cmd;
2481 	bool may_format = ca->features.mode == CM_WRITE;
2482 
2483 	cache = kzalloc(sizeof(*cache), GFP_KERNEL);
2484 	if (!cache)
2485 		return -ENOMEM;
2486 
2487 	cache->ti = ca->ti;
2488 	ti->private = cache;
2489 	ti->num_flush_bios = 2;
2490 	ti->flush_supported = true;
2491 
2492 	ti->num_discard_bios = 1;
2493 	ti->discards_supported = true;
2494 	ti->split_discard_bios = false;
2495 
2496 	ti->per_io_data_size = sizeof(struct per_bio_data);
2497 
2498 	cache->features = ca->features;
2499 	if (writethrough_mode(cache)) {
2500 		/* Create bioset for writethrough bios issued to origin */
2501 		cache->bs = bioset_create(BIO_POOL_SIZE, 0, 0);
2502 		if (!cache->bs)
2503 			goto bad;
2504 	}
2505 
2506 	cache->callbacks.congested_fn = cache_is_congested;
2507 	dm_table_add_target_callbacks(ti->table, &cache->callbacks);
2508 
2509 	cache->metadata_dev = ca->metadata_dev;
2510 	cache->origin_dev = ca->origin_dev;
2511 	cache->cache_dev = ca->cache_dev;
2512 
2513 	ca->metadata_dev = ca->origin_dev = ca->cache_dev = NULL;
2514 
2515 	origin_blocks = cache->origin_sectors = ca->origin_sectors;
2516 	origin_blocks = block_div(origin_blocks, ca->block_size);
2517 	cache->origin_blocks = to_oblock(origin_blocks);
2518 
2519 	cache->sectors_per_block = ca->block_size;
2520 	if (dm_set_target_max_io_len(ti, cache->sectors_per_block)) {
2521 		r = -EINVAL;
2522 		goto bad;
2523 	}
2524 
2525 	if (ca->block_size & (ca->block_size - 1)) {
2526 		dm_block_t cache_size = ca->cache_sectors;
2527 
2528 		cache->sectors_per_block_shift = -1;
2529 		cache_size = block_div(cache_size, ca->block_size);
2530 		set_cache_size(cache, to_cblock(cache_size));
2531 	} else {
2532 		cache->sectors_per_block_shift = __ffs(ca->block_size);
2533 		set_cache_size(cache, to_cblock(ca->cache_sectors >> cache->sectors_per_block_shift));
2534 	}
2535 
2536 	r = create_cache_policy(cache, ca, error);
2537 	if (r)
2538 		goto bad;
2539 
2540 	cache->policy_nr_args = ca->policy_argc;
2541 	cache->migration_threshold = DEFAULT_MIGRATION_THRESHOLD;
2542 
2543 	r = set_config_values(cache, ca->policy_argc, ca->policy_argv);
2544 	if (r) {
2545 		*error = "Error setting cache policy's config values";
2546 		goto bad;
2547 	}
2548 
2549 	cmd = dm_cache_metadata_open(cache->metadata_dev->bdev,
2550 				     ca->block_size, may_format,
2551 				     dm_cache_policy_get_hint_size(cache->policy),
2552 				     ca->features.metadata_version);
2553 	if (IS_ERR(cmd)) {
2554 		*error = "Error creating metadata object";
2555 		r = PTR_ERR(cmd);
2556 		goto bad;
2557 	}
2558 	cache->cmd = cmd;
2559 	set_cache_mode(cache, CM_WRITE);
2560 	if (get_cache_mode(cache) != CM_WRITE) {
2561 		*error = "Unable to get write access to metadata, please check/repair metadata.";
2562 		r = -EINVAL;
2563 		goto bad;
2564 	}
2565 
2566 	if (passthrough_mode(cache)) {
2567 		bool all_clean;
2568 
2569 		r = dm_cache_metadata_all_clean(cache->cmd, &all_clean);
2570 		if (r) {
2571 			*error = "dm_cache_metadata_all_clean() failed";
2572 			goto bad;
2573 		}
2574 
2575 		if (!all_clean) {
2576 			*error = "Cannot enter passthrough mode unless all blocks are clean";
2577 			r = -EINVAL;
2578 			goto bad;
2579 		}
2580 
2581 		policy_allow_migrations(cache->policy, false);
2582 	}
2583 
2584 	spin_lock_init(&cache->lock);
2585 	bio_list_init(&cache->deferred_bios);
2586 	atomic_set(&cache->nr_allocated_migrations, 0);
2587 	atomic_set(&cache->nr_io_migrations, 0);
2588 	init_waitqueue_head(&cache->migration_wait);
2589 
2590 	r = -ENOMEM;
2591 	atomic_set(&cache->nr_dirty, 0);
2592 	cache->dirty_bitset = alloc_bitset(from_cblock(cache->cache_size));
2593 	if (!cache->dirty_bitset) {
2594 		*error = "could not allocate dirty bitset";
2595 		goto bad;
2596 	}
2597 	clear_bitset(cache->dirty_bitset, from_cblock(cache->cache_size));
2598 
2599 	cache->discard_block_size =
2600 		calculate_discard_block_size(cache->sectors_per_block,
2601 					     cache->origin_sectors);
2602 	cache->discard_nr_blocks = to_dblock(dm_sector_div_up(cache->origin_sectors,
2603 							      cache->discard_block_size));
2604 	cache->discard_bitset = alloc_bitset(from_dblock(cache->discard_nr_blocks));
2605 	if (!cache->discard_bitset) {
2606 		*error = "could not allocate discard bitset";
2607 		goto bad;
2608 	}
2609 	clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks));
2610 
2611 	cache->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle);
2612 	if (IS_ERR(cache->copier)) {
2613 		*error = "could not create kcopyd client";
2614 		r = PTR_ERR(cache->copier);
2615 		goto bad;
2616 	}
2617 
2618 	cache->wq = alloc_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM, 0);
2619 	if (!cache->wq) {
2620 		*error = "could not create workqueue for metadata object";
2621 		goto bad;
2622 	}
2623 	INIT_WORK(&cache->deferred_bio_worker, process_deferred_bios);
2624 	INIT_WORK(&cache->migration_worker, check_migrations);
2625 	INIT_DELAYED_WORK(&cache->waker, do_waker);
2626 
2627 	cache->prison = dm_bio_prison_create_v2(cache->wq);
2628 	if (!cache->prison) {
2629 		*error = "could not create bio prison";
2630 		goto bad;
2631 	}
2632 
2633 	cache->migration_pool = mempool_create_slab_pool(MIGRATION_POOL_SIZE,
2634 							 migration_cache);
2635 	if (!cache->migration_pool) {
2636 		*error = "Error creating cache's migration mempool";
2637 		goto bad;
2638 	}
2639 
2640 	cache->need_tick_bio = true;
2641 	cache->sized = false;
2642 	cache->invalidate = false;
2643 	cache->commit_requested = false;
2644 	cache->loaded_mappings = false;
2645 	cache->loaded_discards = false;
2646 
2647 	load_stats(cache);
2648 
2649 	atomic_set(&cache->stats.demotion, 0);
2650 	atomic_set(&cache->stats.promotion, 0);
2651 	atomic_set(&cache->stats.copies_avoided, 0);
2652 	atomic_set(&cache->stats.cache_cell_clash, 0);
2653 	atomic_set(&cache->stats.commit_count, 0);
2654 	atomic_set(&cache->stats.discard_count, 0);
2655 
2656 	spin_lock_init(&cache->invalidation_lock);
2657 	INIT_LIST_HEAD(&cache->invalidation_requests);
2658 
2659 	batcher_init(&cache->committer, commit_op, cache,
2660 		     issue_op, cache, cache->wq);
2661 	iot_init(&cache->tracker);
2662 
2663 	init_rwsem(&cache->background_work_lock);
2664 	prevent_background_work(cache);
2665 
2666 	*result = cache;
2667 	return 0;
2668 bad:
2669 	destroy(cache);
2670 	return r;
2671 }
2672 
2673 static int copy_ctr_args(struct cache *cache, int argc, const char **argv)
2674 {
2675 	unsigned i;
2676 	const char **copy;
2677 
2678 	copy = kcalloc(argc, sizeof(*copy), GFP_KERNEL);
2679 	if (!copy)
2680 		return -ENOMEM;
2681 	for (i = 0; i < argc; i++) {
2682 		copy[i] = kstrdup(argv[i], GFP_KERNEL);
2683 		if (!copy[i]) {
2684 			while (i--)
2685 				kfree(copy[i]);
2686 			kfree(copy);
2687 			return -ENOMEM;
2688 		}
2689 	}
2690 
2691 	cache->nr_ctr_args = argc;
2692 	cache->ctr_args = copy;
2693 
2694 	return 0;
2695 }
2696 
2697 static int cache_ctr(struct dm_target *ti, unsigned argc, char **argv)
2698 {
2699 	int r = -EINVAL;
2700 	struct cache_args *ca;
2701 	struct cache *cache = NULL;
2702 
2703 	ca = kzalloc(sizeof(*ca), GFP_KERNEL);
2704 	if (!ca) {
2705 		ti->error = "Error allocating memory for cache";
2706 		return -ENOMEM;
2707 	}
2708 	ca->ti = ti;
2709 
2710 	r = parse_cache_args(ca, argc, argv, &ti->error);
2711 	if (r)
2712 		goto out;
2713 
2714 	r = cache_create(ca, &cache);
2715 	if (r)
2716 		goto out;
2717 
2718 	r = copy_ctr_args(cache, argc - 3, (const char **)argv + 3);
2719 	if (r) {
2720 		destroy(cache);
2721 		goto out;
2722 	}
2723 
2724 	ti->private = cache;
2725 out:
2726 	destroy_cache_args(ca);
2727 	return r;
2728 }
2729 
2730 /*----------------------------------------------------------------*/
2731 
2732 static int cache_map(struct dm_target *ti, struct bio *bio)
2733 {
2734 	struct cache *cache = ti->private;
2735 
2736 	int r;
2737 	bool commit_needed;
2738 	dm_oblock_t block = get_bio_block(cache, bio);
2739 
2740 	init_per_bio_data(bio);
2741 	if (unlikely(from_oblock(block) >= from_oblock(cache->origin_blocks))) {
2742 		/*
2743 		 * This can only occur if the io goes to a partial block at
2744 		 * the end of the origin device.  We don't cache these.
2745 		 * Just remap to the origin and carry on.
2746 		 */
2747 		remap_to_origin(cache, bio);
2748 		accounted_begin(cache, bio);
2749 		return DM_MAPIO_REMAPPED;
2750 	}
2751 
2752 	if (discard_or_flush(bio)) {
2753 		defer_bio(cache, bio);
2754 		return DM_MAPIO_SUBMITTED;
2755 	}
2756 
2757 	r = map_bio(cache, bio, block, &commit_needed);
2758 	if (commit_needed)
2759 		schedule_commit(&cache->committer);
2760 
2761 	return r;
2762 }
2763 
2764 static int cache_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *error)
2765 {
2766 	struct cache *cache = ti->private;
2767 	unsigned long flags;
2768 	struct per_bio_data *pb = get_per_bio_data(bio);
2769 
2770 	if (pb->tick) {
2771 		policy_tick(cache->policy, false);
2772 
2773 		spin_lock_irqsave(&cache->lock, flags);
2774 		cache->need_tick_bio = true;
2775 		spin_unlock_irqrestore(&cache->lock, flags);
2776 	}
2777 
2778 	bio_drop_shared_lock(cache, bio);
2779 	accounted_complete(cache, bio);
2780 
2781 	return DM_ENDIO_DONE;
2782 }
2783 
2784 static int write_dirty_bitset(struct cache *cache)
2785 {
2786 	int r;
2787 
2788 	if (get_cache_mode(cache) >= CM_READ_ONLY)
2789 		return -EINVAL;
2790 
2791 	r = dm_cache_set_dirty_bits(cache->cmd, from_cblock(cache->cache_size), cache->dirty_bitset);
2792 	if (r)
2793 		metadata_operation_failed(cache, "dm_cache_set_dirty_bits", r);
2794 
2795 	return r;
2796 }
2797 
2798 static int write_discard_bitset(struct cache *cache)
2799 {
2800 	unsigned i, r;
2801 
2802 	if (get_cache_mode(cache) >= CM_READ_ONLY)
2803 		return -EINVAL;
2804 
2805 	r = dm_cache_discard_bitset_resize(cache->cmd, cache->discard_block_size,
2806 					   cache->discard_nr_blocks);
2807 	if (r) {
2808 		DMERR("%s: could not resize on-disk discard bitset", cache_device_name(cache));
2809 		metadata_operation_failed(cache, "dm_cache_discard_bitset_resize", r);
2810 		return r;
2811 	}
2812 
2813 	for (i = 0; i < from_dblock(cache->discard_nr_blocks); i++) {
2814 		r = dm_cache_set_discard(cache->cmd, to_dblock(i),
2815 					 is_discarded(cache, to_dblock(i)));
2816 		if (r) {
2817 			metadata_operation_failed(cache, "dm_cache_set_discard", r);
2818 			return r;
2819 		}
2820 	}
2821 
2822 	return 0;
2823 }
2824 
2825 static int write_hints(struct cache *cache)
2826 {
2827 	int r;
2828 
2829 	if (get_cache_mode(cache) >= CM_READ_ONLY)
2830 		return -EINVAL;
2831 
2832 	r = dm_cache_write_hints(cache->cmd, cache->policy);
2833 	if (r) {
2834 		metadata_operation_failed(cache, "dm_cache_write_hints", r);
2835 		return r;
2836 	}
2837 
2838 	return 0;
2839 }
2840 
2841 /*
2842  * returns true on success
2843  */
2844 static bool sync_metadata(struct cache *cache)
2845 {
2846 	int r1, r2, r3, r4;
2847 
2848 	r1 = write_dirty_bitset(cache);
2849 	if (r1)
2850 		DMERR("%s: could not write dirty bitset", cache_device_name(cache));
2851 
2852 	r2 = write_discard_bitset(cache);
2853 	if (r2)
2854 		DMERR("%s: could not write discard bitset", cache_device_name(cache));
2855 
2856 	save_stats(cache);
2857 
2858 	r3 = write_hints(cache);
2859 	if (r3)
2860 		DMERR("%s: could not write hints", cache_device_name(cache));
2861 
2862 	/*
2863 	 * If writing the above metadata failed, we still commit, but don't
2864 	 * set the clean shutdown flag.  This will effectively force every
2865 	 * dirty bit to be set on reload.
2866 	 */
2867 	r4 = commit(cache, !r1 && !r2 && !r3);
2868 	if (r4)
2869 		DMERR("%s: could not write cache metadata", cache_device_name(cache));
2870 
2871 	return !r1 && !r2 && !r3 && !r4;
2872 }
2873 
2874 static void cache_postsuspend(struct dm_target *ti)
2875 {
2876 	struct cache *cache = ti->private;
2877 
2878 	prevent_background_work(cache);
2879 	BUG_ON(atomic_read(&cache->nr_io_migrations));
2880 
2881 	cancel_delayed_work(&cache->waker);
2882 	flush_workqueue(cache->wq);
2883 	WARN_ON(cache->tracker.in_flight);
2884 
2885 	/*
2886 	 * If it's a flush suspend there won't be any deferred bios, so this
2887 	 * call is harmless.
2888 	 */
2889 	requeue_deferred_bios(cache);
2890 
2891 	if (get_cache_mode(cache) == CM_WRITE)
2892 		(void) sync_metadata(cache);
2893 }
2894 
2895 static int load_mapping(void *context, dm_oblock_t oblock, dm_cblock_t cblock,
2896 			bool dirty, uint32_t hint, bool hint_valid)
2897 {
2898 	int r;
2899 	struct cache *cache = context;
2900 
2901 	if (dirty) {
2902 		set_bit(from_cblock(cblock), cache->dirty_bitset);
2903 		atomic_inc(&cache->nr_dirty);
2904 	} else
2905 		clear_bit(from_cblock(cblock), cache->dirty_bitset);
2906 
2907 	r = policy_load_mapping(cache->policy, oblock, cblock, dirty, hint, hint_valid);
2908 	if (r)
2909 		return r;
2910 
2911 	return 0;
2912 }
2913 
2914 /*
2915  * The discard block size in the on disk metadata is not
2916  * neccessarily the same as we're currently using.  So we have to
2917  * be careful to only set the discarded attribute if we know it
2918  * covers a complete block of the new size.
2919  */
2920 struct discard_load_info {
2921 	struct cache *cache;
2922 
2923 	/*
2924 	 * These blocks are sized using the on disk dblock size, rather
2925 	 * than the current one.
2926 	 */
2927 	dm_block_t block_size;
2928 	dm_block_t discard_begin, discard_end;
2929 };
2930 
2931 static void discard_load_info_init(struct cache *cache,
2932 				   struct discard_load_info *li)
2933 {
2934 	li->cache = cache;
2935 	li->discard_begin = li->discard_end = 0;
2936 }
2937 
2938 static void set_discard_range(struct discard_load_info *li)
2939 {
2940 	sector_t b, e;
2941 
2942 	if (li->discard_begin == li->discard_end)
2943 		return;
2944 
2945 	/*
2946 	 * Convert to sectors.
2947 	 */
2948 	b = li->discard_begin * li->block_size;
2949 	e = li->discard_end * li->block_size;
2950 
2951 	/*
2952 	 * Then convert back to the current dblock size.
2953 	 */
2954 	b = dm_sector_div_up(b, li->cache->discard_block_size);
2955 	sector_div(e, li->cache->discard_block_size);
2956 
2957 	/*
2958 	 * The origin may have shrunk, so we need to check we're still in
2959 	 * bounds.
2960 	 */
2961 	if (e > from_dblock(li->cache->discard_nr_blocks))
2962 		e = from_dblock(li->cache->discard_nr_blocks);
2963 
2964 	for (; b < e; b++)
2965 		set_discard(li->cache, to_dblock(b));
2966 }
2967 
2968 static int load_discard(void *context, sector_t discard_block_size,
2969 			dm_dblock_t dblock, bool discard)
2970 {
2971 	struct discard_load_info *li = context;
2972 
2973 	li->block_size = discard_block_size;
2974 
2975 	if (discard) {
2976 		if (from_dblock(dblock) == li->discard_end)
2977 			/*
2978 			 * We're already in a discard range, just extend it.
2979 			 */
2980 			li->discard_end = li->discard_end + 1ULL;
2981 
2982 		else {
2983 			/*
2984 			 * Emit the old range and start a new one.
2985 			 */
2986 			set_discard_range(li);
2987 			li->discard_begin = from_dblock(dblock);
2988 			li->discard_end = li->discard_begin + 1ULL;
2989 		}
2990 	} else {
2991 		set_discard_range(li);
2992 		li->discard_begin = li->discard_end = 0;
2993 	}
2994 
2995 	return 0;
2996 }
2997 
2998 static dm_cblock_t get_cache_dev_size(struct cache *cache)
2999 {
3000 	sector_t size = get_dev_size(cache->cache_dev);
3001 	(void) sector_div(size, cache->sectors_per_block);
3002 	return to_cblock(size);
3003 }
3004 
3005 static bool can_resize(struct cache *cache, dm_cblock_t new_size)
3006 {
3007 	if (from_cblock(new_size) > from_cblock(cache->cache_size))
3008 		return true;
3009 
3010 	/*
3011 	 * We can't drop a dirty block when shrinking the cache.
3012 	 */
3013 	while (from_cblock(new_size) < from_cblock(cache->cache_size)) {
3014 		new_size = to_cblock(from_cblock(new_size) + 1);
3015 		if (is_dirty(cache, new_size)) {
3016 			DMERR("%s: unable to shrink cache; cache block %llu is dirty",
3017 			      cache_device_name(cache),
3018 			      (unsigned long long) from_cblock(new_size));
3019 			return false;
3020 		}
3021 	}
3022 
3023 	return true;
3024 }
3025 
3026 static int resize_cache_dev(struct cache *cache, dm_cblock_t new_size)
3027 {
3028 	int r;
3029 
3030 	r = dm_cache_resize(cache->cmd, new_size);
3031 	if (r) {
3032 		DMERR("%s: could not resize cache metadata", cache_device_name(cache));
3033 		metadata_operation_failed(cache, "dm_cache_resize", r);
3034 		return r;
3035 	}
3036 
3037 	set_cache_size(cache, new_size);
3038 
3039 	return 0;
3040 }
3041 
3042 static int cache_preresume(struct dm_target *ti)
3043 {
3044 	int r = 0;
3045 	struct cache *cache = ti->private;
3046 	dm_cblock_t csize = get_cache_dev_size(cache);
3047 
3048 	/*
3049 	 * Check to see if the cache has resized.
3050 	 */
3051 	if (!cache->sized) {
3052 		r = resize_cache_dev(cache, csize);
3053 		if (r)
3054 			return r;
3055 
3056 		cache->sized = true;
3057 
3058 	} else if (csize != cache->cache_size) {
3059 		if (!can_resize(cache, csize))
3060 			return -EINVAL;
3061 
3062 		r = resize_cache_dev(cache, csize);
3063 		if (r)
3064 			return r;
3065 	}
3066 
3067 	if (!cache->loaded_mappings) {
3068 		r = dm_cache_load_mappings(cache->cmd, cache->policy,
3069 					   load_mapping, cache);
3070 		if (r) {
3071 			DMERR("%s: could not load cache mappings", cache_device_name(cache));
3072 			metadata_operation_failed(cache, "dm_cache_load_mappings", r);
3073 			return r;
3074 		}
3075 
3076 		cache->loaded_mappings = true;
3077 	}
3078 
3079 	if (!cache->loaded_discards) {
3080 		struct discard_load_info li;
3081 
3082 		/*
3083 		 * The discard bitset could have been resized, or the
3084 		 * discard block size changed.  To be safe we start by
3085 		 * setting every dblock to not discarded.
3086 		 */
3087 		clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks));
3088 
3089 		discard_load_info_init(cache, &li);
3090 		r = dm_cache_load_discards(cache->cmd, load_discard, &li);
3091 		if (r) {
3092 			DMERR("%s: could not load origin discards", cache_device_name(cache));
3093 			metadata_operation_failed(cache, "dm_cache_load_discards", r);
3094 			return r;
3095 		}
3096 		set_discard_range(&li);
3097 
3098 		cache->loaded_discards = true;
3099 	}
3100 
3101 	return r;
3102 }
3103 
3104 static void cache_resume(struct dm_target *ti)
3105 {
3106 	struct cache *cache = ti->private;
3107 
3108 	cache->need_tick_bio = true;
3109 	allow_background_work(cache);
3110 	do_waker(&cache->waker.work);
3111 }
3112 
3113 /*
3114  * Status format:
3115  *
3116  * <metadata block size> <#used metadata blocks>/<#total metadata blocks>
3117  * <cache block size> <#used cache blocks>/<#total cache blocks>
3118  * <#read hits> <#read misses> <#write hits> <#write misses>
3119  * <#demotions> <#promotions> <#dirty>
3120  * <#features> <features>*
3121  * <#core args> <core args>
3122  * <policy name> <#policy args> <policy args>* <cache metadata mode> <needs_check>
3123  */
3124 static void cache_status(struct dm_target *ti, status_type_t type,
3125 			 unsigned status_flags, char *result, unsigned maxlen)
3126 {
3127 	int r = 0;
3128 	unsigned i;
3129 	ssize_t sz = 0;
3130 	dm_block_t nr_free_blocks_metadata = 0;
3131 	dm_block_t nr_blocks_metadata = 0;
3132 	char buf[BDEVNAME_SIZE];
3133 	struct cache *cache = ti->private;
3134 	dm_cblock_t residency;
3135 	bool needs_check;
3136 
3137 	switch (type) {
3138 	case STATUSTYPE_INFO:
3139 		if (get_cache_mode(cache) == CM_FAIL) {
3140 			DMEMIT("Fail");
3141 			break;
3142 		}
3143 
3144 		/* Commit to ensure statistics aren't out-of-date */
3145 		if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti))
3146 			(void) commit(cache, false);
3147 
3148 		r = dm_cache_get_free_metadata_block_count(cache->cmd, &nr_free_blocks_metadata);
3149 		if (r) {
3150 			DMERR("%s: dm_cache_get_free_metadata_block_count returned %d",
3151 			      cache_device_name(cache), r);
3152 			goto err;
3153 		}
3154 
3155 		r = dm_cache_get_metadata_dev_size(cache->cmd, &nr_blocks_metadata);
3156 		if (r) {
3157 			DMERR("%s: dm_cache_get_metadata_dev_size returned %d",
3158 			      cache_device_name(cache), r);
3159 			goto err;
3160 		}
3161 
3162 		residency = policy_residency(cache->policy);
3163 
3164 		DMEMIT("%u %llu/%llu %llu %llu/%llu %u %u %u %u %u %u %lu ",
3165 		       (unsigned)DM_CACHE_METADATA_BLOCK_SIZE,
3166 		       (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata),
3167 		       (unsigned long long)nr_blocks_metadata,
3168 		       (unsigned long long)cache->sectors_per_block,
3169 		       (unsigned long long) from_cblock(residency),
3170 		       (unsigned long long) from_cblock(cache->cache_size),
3171 		       (unsigned) atomic_read(&cache->stats.read_hit),
3172 		       (unsigned) atomic_read(&cache->stats.read_miss),
3173 		       (unsigned) atomic_read(&cache->stats.write_hit),
3174 		       (unsigned) atomic_read(&cache->stats.write_miss),
3175 		       (unsigned) atomic_read(&cache->stats.demotion),
3176 		       (unsigned) atomic_read(&cache->stats.promotion),
3177 		       (unsigned long) atomic_read(&cache->nr_dirty));
3178 
3179 		if (cache->features.metadata_version == 2)
3180 			DMEMIT("2 metadata2 ");
3181 		else
3182 			DMEMIT("1 ");
3183 
3184 		if (writethrough_mode(cache))
3185 			DMEMIT("writethrough ");
3186 
3187 		else if (passthrough_mode(cache))
3188 			DMEMIT("passthrough ");
3189 
3190 		else if (writeback_mode(cache))
3191 			DMEMIT("writeback ");
3192 
3193 		else {
3194 			DMERR("%s: internal error: unknown io mode: %d",
3195 			      cache_device_name(cache), (int) cache->features.io_mode);
3196 			goto err;
3197 		}
3198 
3199 		DMEMIT("2 migration_threshold %llu ", (unsigned long long) cache->migration_threshold);
3200 
3201 		DMEMIT("%s ", dm_cache_policy_get_name(cache->policy));
3202 		if (sz < maxlen) {
3203 			r = policy_emit_config_values(cache->policy, result, maxlen, &sz);
3204 			if (r)
3205 				DMERR("%s: policy_emit_config_values returned %d",
3206 				      cache_device_name(cache), r);
3207 		}
3208 
3209 		if (get_cache_mode(cache) == CM_READ_ONLY)
3210 			DMEMIT("ro ");
3211 		else
3212 			DMEMIT("rw ");
3213 
3214 		r = dm_cache_metadata_needs_check(cache->cmd, &needs_check);
3215 
3216 		if (r || needs_check)
3217 			DMEMIT("needs_check ");
3218 		else
3219 			DMEMIT("- ");
3220 
3221 		break;
3222 
3223 	case STATUSTYPE_TABLE:
3224 		format_dev_t(buf, cache->metadata_dev->bdev->bd_dev);
3225 		DMEMIT("%s ", buf);
3226 		format_dev_t(buf, cache->cache_dev->bdev->bd_dev);
3227 		DMEMIT("%s ", buf);
3228 		format_dev_t(buf, cache->origin_dev->bdev->bd_dev);
3229 		DMEMIT("%s", buf);
3230 
3231 		for (i = 0; i < cache->nr_ctr_args - 1; i++)
3232 			DMEMIT(" %s", cache->ctr_args[i]);
3233 		if (cache->nr_ctr_args)
3234 			DMEMIT(" %s", cache->ctr_args[cache->nr_ctr_args - 1]);
3235 	}
3236 
3237 	return;
3238 
3239 err:
3240 	DMEMIT("Error");
3241 }
3242 
3243 /*
3244  * Defines a range of cblocks, begin to (end - 1) are in the range.  end is
3245  * the one-past-the-end value.
3246  */
3247 struct cblock_range {
3248 	dm_cblock_t begin;
3249 	dm_cblock_t end;
3250 };
3251 
3252 /*
3253  * A cache block range can take two forms:
3254  *
3255  * i) A single cblock, eg. '3456'
3256  * ii) A begin and end cblock with a dash between, eg. 123-234
3257  */
3258 static int parse_cblock_range(struct cache *cache, const char *str,
3259 			      struct cblock_range *result)
3260 {
3261 	char dummy;
3262 	uint64_t b, e;
3263 	int r;
3264 
3265 	/*
3266 	 * Try and parse form (ii) first.
3267 	 */
3268 	r = sscanf(str, "%llu-%llu%c", &b, &e, &dummy);
3269 	if (r < 0)
3270 		return r;
3271 
3272 	if (r == 2) {
3273 		result->begin = to_cblock(b);
3274 		result->end = to_cblock(e);
3275 		return 0;
3276 	}
3277 
3278 	/*
3279 	 * That didn't work, try form (i).
3280 	 */
3281 	r = sscanf(str, "%llu%c", &b, &dummy);
3282 	if (r < 0)
3283 		return r;
3284 
3285 	if (r == 1) {
3286 		result->begin = to_cblock(b);
3287 		result->end = to_cblock(from_cblock(result->begin) + 1u);
3288 		return 0;
3289 	}
3290 
3291 	DMERR("%s: invalid cblock range '%s'", cache_device_name(cache), str);
3292 	return -EINVAL;
3293 }
3294 
3295 static int validate_cblock_range(struct cache *cache, struct cblock_range *range)
3296 {
3297 	uint64_t b = from_cblock(range->begin);
3298 	uint64_t e = from_cblock(range->end);
3299 	uint64_t n = from_cblock(cache->cache_size);
3300 
3301 	if (b >= n) {
3302 		DMERR("%s: begin cblock out of range: %llu >= %llu",
3303 		      cache_device_name(cache), b, n);
3304 		return -EINVAL;
3305 	}
3306 
3307 	if (e > n) {
3308 		DMERR("%s: end cblock out of range: %llu > %llu",
3309 		      cache_device_name(cache), e, n);
3310 		return -EINVAL;
3311 	}
3312 
3313 	if (b >= e) {
3314 		DMERR("%s: invalid cblock range: %llu >= %llu",
3315 		      cache_device_name(cache), b, e);
3316 		return -EINVAL;
3317 	}
3318 
3319 	return 0;
3320 }
3321 
3322 static inline dm_cblock_t cblock_succ(dm_cblock_t b)
3323 {
3324 	return to_cblock(from_cblock(b) + 1);
3325 }
3326 
3327 static int request_invalidation(struct cache *cache, struct cblock_range *range)
3328 {
3329 	int r = 0;
3330 
3331 	/*
3332 	 * We don't need to do any locking here because we know we're in
3333 	 * passthrough mode.  There's is potential for a race between an
3334 	 * invalidation triggered by an io and an invalidation message.  This
3335 	 * is harmless, we must not worry if the policy call fails.
3336 	 */
3337 	while (range->begin != range->end) {
3338 		r = invalidate_cblock(cache, range->begin);
3339 		if (r)
3340 			return r;
3341 
3342 		range->begin = cblock_succ(range->begin);
3343 	}
3344 
3345 	cache->commit_requested = true;
3346 	return r;
3347 }
3348 
3349 static int process_invalidate_cblocks_message(struct cache *cache, unsigned count,
3350 					      const char **cblock_ranges)
3351 {
3352 	int r = 0;
3353 	unsigned i;
3354 	struct cblock_range range;
3355 
3356 	if (!passthrough_mode(cache)) {
3357 		DMERR("%s: cache has to be in passthrough mode for invalidation",
3358 		      cache_device_name(cache));
3359 		return -EPERM;
3360 	}
3361 
3362 	for (i = 0; i < count; i++) {
3363 		r = parse_cblock_range(cache, cblock_ranges[i], &range);
3364 		if (r)
3365 			break;
3366 
3367 		r = validate_cblock_range(cache, &range);
3368 		if (r)
3369 			break;
3370 
3371 		/*
3372 		 * Pass begin and end origin blocks to the worker and wake it.
3373 		 */
3374 		r = request_invalidation(cache, &range);
3375 		if (r)
3376 			break;
3377 	}
3378 
3379 	return r;
3380 }
3381 
3382 /*
3383  * Supports
3384  *	"<key> <value>"
3385  * and
3386  *     "invalidate_cblocks [(<begin>)|(<begin>-<end>)]*
3387  *
3388  * The key migration_threshold is supported by the cache target core.
3389  */
3390 static int cache_message(struct dm_target *ti, unsigned argc, char **argv)
3391 {
3392 	struct cache *cache = ti->private;
3393 
3394 	if (!argc)
3395 		return -EINVAL;
3396 
3397 	if (get_cache_mode(cache) >= CM_READ_ONLY) {
3398 		DMERR("%s: unable to service cache target messages in READ_ONLY or FAIL mode",
3399 		      cache_device_name(cache));
3400 		return -EOPNOTSUPP;
3401 	}
3402 
3403 	if (!strcasecmp(argv[0], "invalidate_cblocks"))
3404 		return process_invalidate_cblocks_message(cache, argc - 1, (const char **) argv + 1);
3405 
3406 	if (argc != 2)
3407 		return -EINVAL;
3408 
3409 	return set_config_value(cache, argv[0], argv[1]);
3410 }
3411 
3412 static int cache_iterate_devices(struct dm_target *ti,
3413 				 iterate_devices_callout_fn fn, void *data)
3414 {
3415 	int r = 0;
3416 	struct cache *cache = ti->private;
3417 
3418 	r = fn(ti, cache->cache_dev, 0, get_dev_size(cache->cache_dev), data);
3419 	if (!r)
3420 		r = fn(ti, cache->origin_dev, 0, ti->len, data);
3421 
3422 	return r;
3423 }
3424 
3425 static void set_discard_limits(struct cache *cache, struct queue_limits *limits)
3426 {
3427 	/*
3428 	 * FIXME: these limits may be incompatible with the cache device
3429 	 */
3430 	limits->max_discard_sectors = min_t(sector_t, cache->discard_block_size * 1024,
3431 					    cache->origin_sectors);
3432 	limits->discard_granularity = cache->discard_block_size << SECTOR_SHIFT;
3433 }
3434 
3435 static void cache_io_hints(struct dm_target *ti, struct queue_limits *limits)
3436 {
3437 	struct cache *cache = ti->private;
3438 	uint64_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
3439 
3440 	/*
3441 	 * If the system-determined stacked limits are compatible with the
3442 	 * cache's blocksize (io_opt is a factor) do not override them.
3443 	 */
3444 	if (io_opt_sectors < cache->sectors_per_block ||
3445 	    do_div(io_opt_sectors, cache->sectors_per_block)) {
3446 		blk_limits_io_min(limits, cache->sectors_per_block << SECTOR_SHIFT);
3447 		blk_limits_io_opt(limits, cache->sectors_per_block << SECTOR_SHIFT);
3448 	}
3449 	set_discard_limits(cache, limits);
3450 }
3451 
3452 /*----------------------------------------------------------------*/
3453 
3454 static struct target_type cache_target = {
3455 	.name = "cache",
3456 	.version = {2, 0, 0},
3457 	.module = THIS_MODULE,
3458 	.ctr = cache_ctr,
3459 	.dtr = cache_dtr,
3460 	.map = cache_map,
3461 	.end_io = cache_end_io,
3462 	.postsuspend = cache_postsuspend,
3463 	.preresume = cache_preresume,
3464 	.resume = cache_resume,
3465 	.status = cache_status,
3466 	.message = cache_message,
3467 	.iterate_devices = cache_iterate_devices,
3468 	.io_hints = cache_io_hints,
3469 };
3470 
3471 static int __init dm_cache_init(void)
3472 {
3473 	int r;
3474 
3475 	migration_cache = KMEM_CACHE(dm_cache_migration, 0);
3476 	if (!migration_cache) {
3477 		dm_unregister_target(&cache_target);
3478 		return -ENOMEM;
3479 	}
3480 
3481 	r = dm_register_target(&cache_target);
3482 	if (r) {
3483 		DMERR("cache target registration failed: %d", r);
3484 		return r;
3485 	}
3486 
3487 	return 0;
3488 }
3489 
3490 static void __exit dm_cache_exit(void)
3491 {
3492 	dm_unregister_target(&cache_target);
3493 	kmem_cache_destroy(migration_cache);
3494 }
3495 
3496 module_init(dm_cache_init);
3497 module_exit(dm_cache_exit);
3498 
3499 MODULE_DESCRIPTION(DM_NAME " cache target");
3500 MODULE_AUTHOR("Joe Thornber <ejt@redhat.com>");
3501 MODULE_LICENSE("GPL");
3502