Lines Matching +full:i +full:- +full:cache +full:- +full:block +full:- +full:size
1 // SPDX-License-Identifier: GPL-2.0-only
9 #include "dm-bio-prison-v2.h"
10 #include "dm-bio-record.h"
11 #include "dm-cache-metadata.h"
12 #include "dm-io-tracker.h"
13 #include "dm-cache-background-tracker.h"
15 #include <linux/dm-io.h>
16 #include <linux/dm-kcopyd.h>
25 #define DM_MSG_PREFIX "cache"
28 "A percentage of time allocated for copying to and/or from cache");
30 /*----------------------------------------------------------------*/
35 * oblock: index of an origin block
36 * cblock: index of a cache block
37 * promotion: movement of a block from origin to cache
38 * demotion: movement of a block from cache to origin
39 * migration: movement of a block between the origin and cache device,
43 /*----------------------------------------------------------------*/
57 INIT_WORK(&k->ws, fn); in init_continuation()
58 k->input = 0; in init_continuation()
64 queue_work(wq, &k->ws); in queue_continuation()
67 /*----------------------------------------------------------------*/
117 spin_lock_irq(&b->lock); in __commit()
118 list_splice_init(&b->work_items, &work_items); in __commit()
119 bio_list_merge(&bios, &b->bios); in __commit()
120 bio_list_init(&b->bios); in __commit()
121 b->commit_scheduled = false; in __commit()
122 spin_unlock_irq(&b->lock); in __commit()
124 r = b->commit_op(b->commit_context); in __commit()
128 k->input = r; in __commit()
129 INIT_LIST_HEAD(&ws->entry); /* to avoid a WARN_ON */ in __commit()
130 queue_work(b->wq, ws); in __commit()
135 bio->bi_status = r; in __commit()
138 b->issue_op(bio, b->issue_context); in __commit()
149 b->commit_op = commit_op; in batcher_init()
150 b->commit_context = commit_context; in batcher_init()
151 b->issue_op = issue_op; in batcher_init()
152 b->issue_context = issue_context; in batcher_init()
153 b->wq = wq; in batcher_init()
155 spin_lock_init(&b->lock); in batcher_init()
156 INIT_LIST_HEAD(&b->work_items); in batcher_init()
157 bio_list_init(&b->bios); in batcher_init()
158 INIT_WORK(&b->commit_work, __commit); in batcher_init()
159 b->commit_scheduled = false; in batcher_init()
164 queue_work(b->wq, &b->commit_work); in async_commit()
171 spin_lock_irq(&b->lock); in continue_after_commit()
172 commit_scheduled = b->commit_scheduled; in continue_after_commit()
173 list_add_tail(&k->ws.entry, &b->work_items); in continue_after_commit()
174 spin_unlock_irq(&b->lock); in continue_after_commit()
187 spin_lock_irq(&b->lock); in issue_after_commit()
188 commit_scheduled = b->commit_scheduled; in issue_after_commit()
189 bio_list_add(&b->bios, bio); in issue_after_commit()
190 spin_unlock_irq(&b->lock); in issue_after_commit()
203 spin_lock_irq(&b->lock); in schedule_commit()
204 immediate = !list_empty(&b->work_items) || !bio_list_empty(&b->bios); in schedule_commit()
205 b->commit_scheduled = true; in schedule_commit()
206 spin_unlock_irq(&b->lock); in schedule_commit()
224 h->bi_end_io = bio->bi_end_io; in dm_hook_bio()
226 bio->bi_end_io = bi_end_io; in dm_hook_bio()
227 bio->bi_private = bi_private; in dm_hook_bio()
232 bio->bi_end_io = h->bi_end_io; in dm_unhook_bio()
235 /*----------------------------------------------------------------*/
242 * The block size of the device holding cache data must be
257 * dirty. If you lose the cache device you will lose data.
263 * Data is written to both cache and origin. Blocks are never
269 * A degraded mode useful for various cache coherency situations
271 * origin. If a write goes to a cached oblock, then the cache
272 * block is invalidated.
298 struct cache { struct
326 * Size of the origin device in _complete_ blocks and native sectors.
332 * Size of the cache device in blocks. argument
361 uint32_t discard_block_size; /* a power of 2 times sectors per block */
387 * Cache features such as write-through.
422 struct cache *cache; argument
432 /*----------------------------------------------------------------*/
434 static bool writethrough_mode(struct cache *cache) in writethrough_mode() argument
436 return cache->features.io_mode == CM_IO_WRITETHROUGH; in writethrough_mode()
439 static bool writeback_mode(struct cache *cache) in writeback_mode() argument
441 return cache->features.io_mode == CM_IO_WRITEBACK; in writeback_mode()
444 static inline bool passthrough_mode(struct cache *cache) in passthrough_mode() argument
446 return unlikely(cache->features.io_mode == CM_IO_PASSTHROUGH); in passthrough_mode()
449 /*----------------------------------------------------------------*/
451 static void wake_deferred_bio_worker(struct cache *cache) in wake_deferred_bio_worker() argument
453 queue_work(cache->wq, &cache->deferred_bio_worker); in wake_deferred_bio_worker()
456 static void wake_migration_worker(struct cache *cache) in wake_migration_worker() argument
458 if (passthrough_mode(cache)) in wake_migration_worker()
461 queue_work(cache->wq, &cache->migration_worker); in wake_migration_worker()
464 /*----------------------------------------------------------------*/
466 static struct dm_bio_prison_cell_v2 *alloc_prison_cell(struct cache *cache) in alloc_prison_cell() argument
468 return dm_bio_prison_alloc_cell_v2(cache->prison, GFP_NOIO); in alloc_prison_cell()
471 static void free_prison_cell(struct cache *cache, struct dm_bio_prison_cell_v2 *cell) in free_prison_cell() argument
473 dm_bio_prison_free_cell_v2(cache->prison, cell); in free_prison_cell()
476 static struct dm_cache_migration *alloc_migration(struct cache *cache) in alloc_migration() argument
480 mg = mempool_alloc(&cache->migration_pool, GFP_NOIO); in alloc_migration()
484 mg->cache = cache; in alloc_migration()
485 atomic_inc(&cache->nr_allocated_migrations); in alloc_migration()
492 struct cache *cache = mg->cache; in free_migration() local
494 if (atomic_dec_and_test(&cache->nr_allocated_migrations)) in free_migration()
495 wake_up(&cache->migration_wait); in free_migration()
497 mempool_free(mg, &cache->migration_pool); in free_migration()
500 /*----------------------------------------------------------------*/
509 key->virtual = 0; in build_key()
510 key->dev = 0; in build_key()
511 key->block_begin = from_oblock(begin); in build_key()
512 key->block_end = from_oblock(end); in build_key()
530 *--------------------------------------------------------------
532 *--------------------------------------------------------------
547 pb->tick = false; in init_per_bio_data()
548 pb->req_nr = dm_bio_get_target_bio_nr(bio); in init_per_bio_data()
549 pb->cell = NULL; in init_per_bio_data()
550 pb->len = 0; in init_per_bio_data()
555 /*----------------------------------------------------------------*/
557 static void defer_bio(struct cache *cache, struct bio *bio) in defer_bio() argument
559 spin_lock_irq(&cache->lock); in defer_bio()
560 bio_list_add(&cache->deferred_bios, bio); in defer_bio()
561 spin_unlock_irq(&cache->lock); in defer_bio()
563 wake_deferred_bio_worker(cache); in defer_bio()
566 static void defer_bios(struct cache *cache, struct bio_list *bios) in defer_bios() argument
568 spin_lock_irq(&cache->lock); in defer_bios()
569 bio_list_merge(&cache->deferred_bios, bios); in defer_bios()
571 spin_unlock_irq(&cache->lock); in defer_bios()
573 wake_deferred_bio_worker(cache); in defer_bios()
576 /*----------------------------------------------------------------*/
578 static bool bio_detain_shared(struct cache *cache, dm_oblock_t oblock, struct bio *bio) in bio_detain_shared() argument
586 cell_prealloc = alloc_prison_cell(cache); /* FIXME: allow wait if calling from worker */ in bio_detain_shared()
589 r = dm_cell_get_v2(cache->prison, &key, lock_level(bio), bio, cell_prealloc, &cell); in bio_detain_shared()
594 free_prison_cell(cache, cell_prealloc); in bio_detain_shared()
599 free_prison_cell(cache, cell_prealloc); in bio_detain_shared()
602 pb->cell = cell; in bio_detain_shared()
607 /*----------------------------------------------------------------*/
609 static bool is_dirty(struct cache *cache, dm_cblock_t b) in is_dirty() argument
611 return test_bit(from_cblock(b), cache->dirty_bitset); in is_dirty()
614 static void set_dirty(struct cache *cache, dm_cblock_t cblock) in set_dirty() argument
616 if (!test_and_set_bit(from_cblock(cblock), cache->dirty_bitset)) { in set_dirty()
617 atomic_inc(&cache->nr_dirty); in set_dirty()
618 policy_set_dirty(cache->policy, cblock); in set_dirty()
626 static void force_set_dirty(struct cache *cache, dm_cblock_t cblock) in force_set_dirty() argument
628 if (!test_and_set_bit(from_cblock(cblock), cache->dirty_bitset)) in force_set_dirty()
629 atomic_inc(&cache->nr_dirty); in force_set_dirty()
630 policy_set_dirty(cache->policy, cblock); in force_set_dirty()
633 static void force_clear_dirty(struct cache *cache, dm_cblock_t cblock) in force_clear_dirty() argument
635 if (test_and_clear_bit(from_cblock(cblock), cache->dirty_bitset)) { in force_clear_dirty()
636 if (atomic_dec_return(&cache->nr_dirty) == 0) in force_clear_dirty()
637 dm_table_event(cache->ti->table); in force_clear_dirty()
640 policy_clear_dirty(cache->policy, cblock); in force_clear_dirty()
643 /*----------------------------------------------------------------*/
645 static bool block_size_is_power_of_two(struct cache *cache) in block_size_is_power_of_two() argument
647 return cache->sectors_per_block_shift >= 0; in block_size_is_power_of_two()
657 static dm_block_t oblocks_per_dblock(struct cache *cache) in oblocks_per_dblock() argument
659 dm_block_t oblocks = cache->discard_block_size; in oblocks_per_dblock()
661 if (block_size_is_power_of_two(cache)) in oblocks_per_dblock()
662 oblocks >>= cache->sectors_per_block_shift; in oblocks_per_dblock()
664 oblocks = block_div(oblocks, cache->sectors_per_block); in oblocks_per_dblock()
669 static dm_dblock_t oblock_to_dblock(struct cache *cache, dm_oblock_t oblock) in oblock_to_dblock() argument
672 oblocks_per_dblock(cache))); in oblock_to_dblock()
675 static void set_discard(struct cache *cache, dm_dblock_t b) in set_discard() argument
677 BUG_ON(from_dblock(b) >= from_dblock(cache->discard_nr_blocks)); in set_discard()
678 atomic_inc(&cache->stats.discard_count); in set_discard()
680 spin_lock_irq(&cache->lock); in set_discard()
681 set_bit(from_dblock(b), cache->discard_bitset); in set_discard()
682 spin_unlock_irq(&cache->lock); in set_discard()
685 static void clear_discard(struct cache *cache, dm_dblock_t b) in clear_discard() argument
687 spin_lock_irq(&cache->lock); in clear_discard()
688 clear_bit(from_dblock(b), cache->discard_bitset); in clear_discard()
689 spin_unlock_irq(&cache->lock); in clear_discard()
692 static bool is_discarded(struct cache *cache, dm_dblock_t b) in is_discarded() argument
696 spin_lock_irq(&cache->lock); in is_discarded()
697 r = test_bit(from_dblock(b), cache->discard_bitset); in is_discarded()
698 spin_unlock_irq(&cache->lock); in is_discarded()
703 static bool is_discarded_oblock(struct cache *cache, dm_oblock_t b) in is_discarded_oblock() argument
707 spin_lock_irq(&cache->lock); in is_discarded_oblock()
708 r = test_bit(from_dblock(oblock_to_dblock(cache, b)), in is_discarded_oblock()
709 cache->discard_bitset); in is_discarded_oblock()
710 spin_unlock_irq(&cache->lock); in is_discarded_oblock()
716 * -------------------------------------------------------------
718 *--------------------------------------------------------------
720 static void remap_to_origin(struct cache *cache, struct bio *bio) in remap_to_origin() argument
722 bio_set_dev(bio, cache->origin_dev->bdev); in remap_to_origin()
725 static void remap_to_cache(struct cache *cache, struct bio *bio, in remap_to_cache() argument
728 sector_t bi_sector = bio->bi_iter.bi_sector; in remap_to_cache()
729 sector_t block = from_cblock(cblock); in remap_to_cache() local
731 bio_set_dev(bio, cache->cache_dev->bdev); in remap_to_cache()
732 if (!block_size_is_power_of_two(cache)) in remap_to_cache()
733 bio->bi_iter.bi_sector = in remap_to_cache()
734 (block * cache->sectors_per_block) + in remap_to_cache()
735 sector_div(bi_sector, cache->sectors_per_block); in remap_to_cache()
737 bio->bi_iter.bi_sector = in remap_to_cache()
738 (block << cache->sectors_per_block_shift) | in remap_to_cache()
739 (bi_sector & (cache->sectors_per_block - 1)); in remap_to_cache()
742 static void check_if_tick_bio_needed(struct cache *cache, struct bio *bio) in check_if_tick_bio_needed() argument
746 spin_lock_irq(&cache->lock); in check_if_tick_bio_needed()
747 if (cache->need_tick_bio && !op_is_flush(bio->bi_opf) && in check_if_tick_bio_needed()
750 pb->tick = true; in check_if_tick_bio_needed()
751 cache->need_tick_bio = false; in check_if_tick_bio_needed()
753 spin_unlock_irq(&cache->lock); in check_if_tick_bio_needed()
756 static void remap_to_origin_clear_discard(struct cache *cache, struct bio *bio, in remap_to_origin_clear_discard() argument
760 check_if_tick_bio_needed(cache, bio); in remap_to_origin_clear_discard()
761 remap_to_origin(cache, bio); in remap_to_origin_clear_discard()
763 clear_discard(cache, oblock_to_dblock(cache, oblock)); in remap_to_origin_clear_discard()
766 static void remap_to_cache_dirty(struct cache *cache, struct bio *bio, in remap_to_cache_dirty() argument
769 check_if_tick_bio_needed(cache, bio); in remap_to_cache_dirty()
770 remap_to_cache(cache, bio, cblock); in remap_to_cache_dirty()
772 set_dirty(cache, cblock); in remap_to_cache_dirty()
773 clear_discard(cache, oblock_to_dblock(cache, oblock)); in remap_to_cache_dirty()
777 static dm_oblock_t get_bio_block(struct cache *cache, struct bio *bio) in get_bio_block() argument
779 sector_t block_nr = bio->bi_iter.bi_sector; in get_bio_block()
781 if (!block_size_is_power_of_two(cache)) in get_bio_block()
782 (void) sector_div(block_nr, cache->sectors_per_block); in get_bio_block()
784 block_nr >>= cache->sectors_per_block_shift; in get_bio_block()
789 static bool accountable_bio(struct cache *cache, struct bio *bio) in accountable_bio() argument
794 static void accounted_begin(struct cache *cache, struct bio *bio) in accounted_begin() argument
798 if (accountable_bio(cache, bio)) { in accounted_begin()
800 pb->len = bio_sectors(bio); in accounted_begin()
801 dm_iot_io_begin(&cache->tracker, pb->len); in accounted_begin()
805 static void accounted_complete(struct cache *cache, struct bio *bio) in accounted_complete() argument
809 dm_iot_io_end(&cache->tracker, pb->len); in accounted_complete()
812 static void accounted_request(struct cache *cache, struct bio *bio) in accounted_request() argument
814 accounted_begin(cache, bio); in accounted_request()
820 struct cache *cache = context; in issue_op() local
822 accounted_request(cache, bio); in issue_op()
827 * to both the cache and origin devices. Clone the bio and send them in parallel.
829 static void remap_to_origin_and_cache(struct cache *cache, struct bio *bio, in remap_to_origin_and_cache() argument
832 struct bio *origin_bio = bio_alloc_clone(cache->origin_dev->bdev, bio, in remap_to_origin_and_cache()
833 GFP_NOIO, &cache->bs); in remap_to_origin_and_cache()
840 clear_discard(cache, oblock_to_dblock(cache, oblock)); in remap_to_origin_and_cache()
843 remap_to_cache(cache, bio, cblock); in remap_to_origin_and_cache()
847 *--------------------------------------------------------------
849 *--------------------------------------------------------------
851 static enum cache_metadata_mode get_cache_mode(struct cache *cache) in get_cache_mode() argument
853 return cache->features.mode; in get_cache_mode()
856 static const char *cache_device_name(struct cache *cache) in cache_device_name() argument
858 return dm_table_device_name(cache->ti->table); in cache_device_name()
861 static void notify_mode_switch(struct cache *cache, enum cache_metadata_mode mode) in notify_mode_switch() argument
865 "read-only", in notify_mode_switch()
869 dm_table_event(cache->ti->table); in notify_mode_switch()
870 DMINFO("%s: switching cache to %s mode", in notify_mode_switch()
871 cache_device_name(cache), descs[(int)mode]); in notify_mode_switch()
874 static void set_cache_mode(struct cache *cache, enum cache_metadata_mode new_mode) in set_cache_mode() argument
877 enum cache_metadata_mode old_mode = get_cache_mode(cache); in set_cache_mode()
879 if (dm_cache_metadata_needs_check(cache->cmd, &needs_check)) { in set_cache_mode()
881 cache_device_name(cache)); in set_cache_mode()
886 DMERR("%s: unable to switch cache to write mode until repaired.", in set_cache_mode()
887 cache_device_name(cache)); in set_cache_mode()
901 dm_cache_metadata_set_read_only(cache->cmd); in set_cache_mode()
905 dm_cache_metadata_set_read_write(cache->cmd); in set_cache_mode()
909 cache->features.mode = new_mode; in set_cache_mode()
912 notify_mode_switch(cache, new_mode); in set_cache_mode()
915 static void abort_transaction(struct cache *cache) in abort_transaction() argument
917 const char *dev_name = cache_device_name(cache); in abort_transaction()
919 if (get_cache_mode(cache) >= CM_READ_ONLY) in abort_transaction()
923 if (dm_cache_metadata_abort(cache->cmd)) { in abort_transaction()
925 set_cache_mode(cache, CM_FAIL); in abort_transaction()
928 if (dm_cache_metadata_set_needs_check(cache->cmd)) { in abort_transaction()
930 set_cache_mode(cache, CM_FAIL); in abort_transaction()
934 static void metadata_operation_failed(struct cache *cache, const char *op, int r) in metadata_operation_failed() argument
937 cache_device_name(cache), op, r); in metadata_operation_failed()
938 abort_transaction(cache); in metadata_operation_failed()
939 set_cache_mode(cache, CM_READ_ONLY); in metadata_operation_failed()
942 /*----------------------------------------------------------------*/
944 static void load_stats(struct cache *cache) in load_stats() argument
948 dm_cache_metadata_get_stats(cache->cmd, &stats); in load_stats()
949 atomic_set(&cache->stats.read_hit, stats.read_hits); in load_stats()
950 atomic_set(&cache->stats.read_miss, stats.read_misses); in load_stats()
951 atomic_set(&cache->stats.write_hit, stats.write_hits); in load_stats()
952 atomic_set(&cache->stats.write_miss, stats.write_misses); in load_stats()
955 static void save_stats(struct cache *cache) in save_stats() argument
959 if (get_cache_mode(cache) >= CM_READ_ONLY) in save_stats()
962 stats.read_hits = atomic_read(&cache->stats.read_hit); in save_stats()
963 stats.read_misses = atomic_read(&cache->stats.read_miss); in save_stats()
964 stats.write_hits = atomic_read(&cache->stats.write_hit); in save_stats()
965 stats.write_misses = atomic_read(&cache->stats.write_miss); in save_stats()
967 dm_cache_metadata_set_stats(cache->cmd, &stats); in save_stats()
974 atomic_inc(&stats->promotion); in update_stats()
978 atomic_inc(&stats->demotion); in update_stats()
982 atomic_inc(&stats->writeback); in update_stats()
988 *---------------------------------------------------------------------
991 * Migration covers moving data from the origin device to the cache, or
993 *---------------------------------------------------------------------
995 static void inc_io_migrations(struct cache *cache) in inc_io_migrations() argument
997 atomic_inc(&cache->nr_io_migrations); in inc_io_migrations()
1000 static void dec_io_migrations(struct cache *cache) in dec_io_migrations() argument
1002 atomic_dec(&cache->nr_io_migrations); in dec_io_migrations()
1007 return bio_op(bio) == REQ_OP_DISCARD || op_is_flush(bio->bi_opf); in discard_or_flush()
1010 static void calc_discard_block_range(struct cache *cache, struct bio *bio, in calc_discard_block_range() argument
1013 sector_t sb = bio->bi_iter.bi_sector; in calc_discard_block_range()
1016 *b = to_dblock(dm_sector_div_up(sb, cache->discard_block_size)); in calc_discard_block_range()
1018 if (se - sb < cache->discard_block_size) in calc_discard_block_range()
1021 *e = to_dblock(block_div(se, cache->discard_block_size)); in calc_discard_block_range()
1024 /*----------------------------------------------------------------*/
1026 static void prevent_background_work(struct cache *cache) in prevent_background_work() argument
1029 down_write(&cache->background_work_lock); in prevent_background_work()
1033 static void allow_background_work(struct cache *cache) in allow_background_work() argument
1036 up_write(&cache->background_work_lock); in allow_background_work()
1040 static bool background_work_begin(struct cache *cache) in background_work_begin() argument
1045 r = down_read_trylock(&cache->background_work_lock); in background_work_begin()
1051 static void background_work_end(struct cache *cache) in background_work_end() argument
1054 up_read(&cache->background_work_lock); in background_work_end()
1058 /*----------------------------------------------------------------*/
1060 static bool bio_writes_complete_block(struct cache *cache, struct bio *bio) in bio_writes_complete_block() argument
1063 (bio->bi_iter.bi_size == (cache->sectors_per_block << SECTOR_SHIFT)); in bio_writes_complete_block()
1066 static bool optimisable_bio(struct cache *cache, struct bio *bio, dm_oblock_t block) in optimisable_bio() argument
1068 return writeback_mode(cache) && in optimisable_bio()
1069 (is_discarded_oblock(cache, block) || bio_writes_complete_block(cache, bio)); in optimisable_bio()
1075 init_continuation(&mg->k, continuation); in quiesce()
1076 dm_cell_quiesce_v2(mg->cache->prison, mg->cell, &mg->k.ws); in quiesce()
1091 mg->k.input = BLK_STS_IOERR; in copy_complete()
1093 queue_continuation(mg->cache->wq, &mg->k); in copy_complete()
1099 struct cache *cache = mg->cache; in copy() local
1101 o_region.bdev = cache->origin_dev->bdev; in copy()
1102 o_region.sector = from_oblock(mg->op->oblock) * cache->sectors_per_block; in copy()
1103 o_region.count = cache->sectors_per_block; in copy()
1105 c_region.bdev = cache->cache_dev->bdev; in copy()
1106 c_region.sector = from_cblock(mg->op->cblock) * cache->sectors_per_block; in copy()
1107 c_region.count = cache->sectors_per_block; in copy()
1110 dm_kcopyd_copy(cache->copier, &o_region, 1, &c_region, 0, copy_complete, &mg->k); in copy()
1112 dm_kcopyd_copy(cache->copier, &c_region, 1, &o_region, 0, copy_complete, &mg->k); in copy()
1115 static void bio_drop_shared_lock(struct cache *cache, struct bio *bio) in bio_drop_shared_lock() argument
1119 if (pb->cell && dm_cell_put_v2(cache->prison, pb->cell)) in bio_drop_shared_lock()
1120 free_prison_cell(cache, pb->cell); in bio_drop_shared_lock()
1121 pb->cell = NULL; in bio_drop_shared_lock()
1126 struct dm_cache_migration *mg = bio->bi_private; in overwrite_endio()
1127 struct cache *cache = mg->cache; in overwrite_endio() local
1130 dm_unhook_bio(&pb->hook_info, bio); in overwrite_endio()
1132 if (bio->bi_status) in overwrite_endio()
1133 mg->k.input = bio->bi_status; in overwrite_endio()
1135 queue_continuation(cache->wq, &mg->k); in overwrite_endio()
1141 struct bio *bio = mg->overwrite_bio; in overwrite()
1144 dm_hook_bio(&pb->hook_info, bio, overwrite_endio, mg); in overwrite()
1150 if (mg->op->op == POLICY_PROMOTE) in overwrite()
1151 remap_to_cache(mg->cache, bio, mg->op->cblock); in overwrite()
1153 remap_to_origin(mg->cache, bio); in overwrite()
1155 init_continuation(&mg->k, continuation); in overwrite()
1156 accounted_request(mg->cache, bio); in overwrite()
1173 struct cache *cache = mg->cache; in mg_complete() local
1174 struct policy_work *op = mg->op; in mg_complete()
1175 dm_cblock_t cblock = op->cblock; in mg_complete()
1178 update_stats(&cache->stats, op->op); in mg_complete()
1180 switch (op->op) { in mg_complete()
1182 clear_discard(cache, oblock_to_dblock(cache, op->oblock)); in mg_complete()
1183 policy_complete_background_work(cache->policy, op, success); in mg_complete()
1185 if (mg->overwrite_bio) { in mg_complete()
1187 force_set_dirty(cache, cblock); in mg_complete()
1188 else if (mg->k.input) in mg_complete()
1189 mg->overwrite_bio->bi_status = mg->k.input; in mg_complete()
1191 mg->overwrite_bio->bi_status = BLK_STS_IOERR; in mg_complete()
1192 bio_endio(mg->overwrite_bio); in mg_complete()
1195 force_clear_dirty(cache, cblock); in mg_complete()
1196 dec_io_migrations(cache); in mg_complete()
1205 force_clear_dirty(cache, cblock); in mg_complete()
1206 policy_complete_background_work(cache->policy, op, success); in mg_complete()
1207 dec_io_migrations(cache); in mg_complete()
1212 force_clear_dirty(cache, cblock); in mg_complete()
1213 policy_complete_background_work(cache->policy, op, success); in mg_complete()
1214 dec_io_migrations(cache); in mg_complete()
1219 if (mg->cell) { in mg_complete()
1220 if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios)) in mg_complete()
1221 free_prison_cell(cache, mg->cell); in mg_complete()
1225 defer_bios(cache, &bios); in mg_complete()
1226 wake_migration_worker(cache); in mg_complete()
1228 background_work_end(cache); in mg_complete()
1235 mg_complete(mg, mg->k.input == 0); in mg_success()
1242 struct cache *cache = mg->cache; in mg_update_metadata() local
1243 struct policy_work *op = mg->op; in mg_update_metadata()
1245 switch (op->op) { in mg_update_metadata()
1247 r = dm_cache_insert_mapping(cache->cmd, op->cblock, op->oblock); in mg_update_metadata()
1250 cache_device_name(cache)); in mg_update_metadata()
1251 metadata_operation_failed(cache, "dm_cache_insert_mapping", r); in mg_update_metadata()
1260 r = dm_cache_remove_mapping(cache->cmd, op->cblock); in mg_update_metadata()
1263 cache_device_name(cache)); in mg_update_metadata()
1264 metadata_operation_failed(cache, "dm_cache_remove_mapping", r); in mg_update_metadata()
1275 * - vblock x in a cache block in mg_update_metadata()
1276 * - domotion occurs in mg_update_metadata()
1277 * - cache block gets reallocated and over written in mg_update_metadata()
1278 * - crash in mg_update_metadata()
1280 * When we recover, because there was no commit the cache will in mg_update_metadata()
1281 * rollback to having the data for vblock x in the cache block. in mg_update_metadata()
1282 * But the cache block has since been overwritten, so it'll end in mg_update_metadata()
1289 init_continuation(&mg->k, mg_success); in mg_update_metadata()
1290 continue_after_commit(&cache->committer, &mg->k); in mg_update_metadata()
1291 schedule_commit(&cache->committer); in mg_update_metadata()
1307 if (mg->k.input) in mg_update_metadata_after_copy()
1321 if (mg->k.input) in mg_upgrade_lock()
1328 r = dm_cell_lock_promote_v2(mg->cache->prison, mg->cell, in mg_upgrade_lock()
1344 struct cache *cache = mg->cache; in mg_full_copy() local
1345 struct policy_work *op = mg->op; in mg_full_copy()
1346 bool is_policy_promote = (op->op == POLICY_PROMOTE); in mg_full_copy()
1348 if ((!is_policy_promote && !is_dirty(cache, op->cblock)) || in mg_full_copy()
1349 is_discarded_oblock(cache, op->oblock)) { in mg_full_copy()
1354 init_continuation(&mg->k, mg_upgrade_lock); in mg_full_copy()
1362 if (mg->overwrite_bio) { in mg_copy()
1366 * have changed (eg, the block may no longer be discarded). in mg_copy()
1368 if (!optimisable_bio(mg->cache, mg->overwrite_bio, mg->op->oblock)) { in mg_copy()
1372 bool rb = bio_detain_shared(mg->cache, mg->op->oblock, mg->overwrite_bio); in mg_copy()
1374 BUG_ON(rb); /* An exclussive lock must _not_ be held for this block */ in mg_copy()
1375 mg->overwrite_bio = NULL; in mg_copy()
1376 inc_io_migrations(mg->cache); in mg_copy()
1383 * because all IO has been locked out of the block. in mg_copy()
1398 struct cache *cache = mg->cache; in mg_lock_writes() local
1401 prealloc = alloc_prison_cell(cache); in mg_lock_writes()
1404 * Prevent writes to the block, but allow reads to continue. in mg_lock_writes()
1408 build_key(mg->op->oblock, oblock_succ(mg->op->oblock), &key); in mg_lock_writes()
1409 r = dm_cell_lock_v2(cache->prison, &key, in mg_lock_writes()
1410 mg->overwrite_bio ? READ_WRITE_LOCK_LEVEL : WRITE_LOCK_LEVEL, in mg_lock_writes()
1411 prealloc, &mg->cell); in mg_lock_writes()
1413 free_prison_cell(cache, prealloc); in mg_lock_writes()
1418 if (mg->cell != prealloc) in mg_lock_writes()
1419 free_prison_cell(cache, prealloc); in mg_lock_writes()
1422 mg_copy(&mg->k.ws); in mg_lock_writes()
1429 static int mg_start(struct cache *cache, struct policy_work *op, struct bio *bio) in mg_start() argument
1433 if (!background_work_begin(cache)) { in mg_start()
1434 policy_complete_background_work(cache->policy, op, false); in mg_start()
1435 return -EPERM; in mg_start()
1438 mg = alloc_migration(cache); in mg_start()
1440 mg->op = op; in mg_start()
1441 mg->overwrite_bio = bio; in mg_start()
1444 inc_io_migrations(cache); in mg_start()
1450 *--------------------------------------------------------------
1452 *--------------------------------------------------------------
1458 struct cache *cache = mg->cache; in invalidate_complete() local
1461 if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios)) in invalidate_complete()
1462 free_prison_cell(cache, mg->cell); in invalidate_complete()
1464 if (!success && mg->overwrite_bio) in invalidate_complete()
1465 bio_io_error(mg->overwrite_bio); in invalidate_complete()
1468 defer_bios(cache, &bios); in invalidate_complete()
1470 background_work_end(cache); in invalidate_complete()
1477 invalidate_complete(mg, !mg->k.input); in invalidate_completed()
1480 static int invalidate_cblock(struct cache *cache, dm_cblock_t cblock) in invalidate_cblock() argument
1484 r = policy_invalidate_mapping(cache->policy, cblock); in invalidate_cblock()
1486 r = dm_cache_remove_mapping(cache->cmd, cblock); in invalidate_cblock()
1489 cache_device_name(cache)); in invalidate_cblock()
1490 metadata_operation_failed(cache, "dm_cache_remove_mapping", r); in invalidate_cblock()
1493 } else if (r == -ENODATA) { in invalidate_cblock()
1500 DMERR("%s: policy_invalidate_mapping failed", cache_device_name(cache)); in invalidate_cblock()
1509 struct cache *cache = mg->cache; in invalidate_remove() local
1511 r = invalidate_cblock(cache, mg->invalidate_cblock); in invalidate_remove()
1517 init_continuation(&mg->k, invalidate_completed); in invalidate_remove()
1518 continue_after_commit(&cache->committer, &mg->k); in invalidate_remove()
1519 remap_to_origin_clear_discard(cache, mg->overwrite_bio, mg->invalidate_oblock); in invalidate_remove()
1520 mg->overwrite_bio = NULL; in invalidate_remove()
1521 schedule_commit(&cache->committer); in invalidate_remove()
1528 struct cache *cache = mg->cache; in invalidate_lock() local
1531 prealloc = alloc_prison_cell(cache); in invalidate_lock()
1533 build_key(mg->invalidate_oblock, oblock_succ(mg->invalidate_oblock), &key); in invalidate_lock()
1534 r = dm_cell_lock_v2(cache->prison, &key, in invalidate_lock()
1535 READ_WRITE_LOCK_LEVEL, prealloc, &mg->cell); in invalidate_lock()
1537 free_prison_cell(cache, prealloc); in invalidate_lock()
1542 if (mg->cell != prealloc) in invalidate_lock()
1543 free_prison_cell(cache, prealloc); in invalidate_lock()
1553 init_continuation(&mg->k, invalidate_remove); in invalidate_lock()
1554 queue_work(cache->wq, &mg->k.ws); in invalidate_lock()
1560 static int invalidate_start(struct cache *cache, dm_cblock_t cblock, in invalidate_start() argument
1565 if (!background_work_begin(cache)) in invalidate_start()
1566 return -EPERM; in invalidate_start()
1568 mg = alloc_migration(cache); in invalidate_start()
1570 mg->overwrite_bio = bio; in invalidate_start()
1571 mg->invalidate_cblock = cblock; in invalidate_start()
1572 mg->invalidate_oblock = oblock; in invalidate_start()
1578 *--------------------------------------------------------------
1580 *--------------------------------------------------------------
1588 static enum busy spare_migration_bandwidth(struct cache *cache) in spare_migration_bandwidth() argument
1590 bool idle = dm_iot_idle_for(&cache->tracker, HZ); in spare_migration_bandwidth()
1591 sector_t current_volume = (atomic_read(&cache->nr_io_migrations) + 1) * in spare_migration_bandwidth()
1592 cache->sectors_per_block; in spare_migration_bandwidth()
1594 if (idle && current_volume <= cache->migration_threshold) in spare_migration_bandwidth()
1600 static void inc_hit_counter(struct cache *cache, struct bio *bio) in inc_hit_counter() argument
1603 &cache->stats.read_hit : &cache->stats.write_hit); in inc_hit_counter()
1606 static void inc_miss_counter(struct cache *cache, struct bio *bio) in inc_miss_counter() argument
1609 &cache->stats.read_miss : &cache->stats.write_miss); in inc_miss_counter()
1612 /*----------------------------------------------------------------*/
1614 static int map_bio(struct cache *cache, struct bio *bio, dm_oblock_t block, in map_bio() argument
1623 rb = bio_detain_shared(cache, block, bio); in map_bio()
1626 * An exclusive lock is held for this block, so we have to in map_bio()
1637 if (optimisable_bio(cache, bio, block)) { in map_bio()
1640 r = policy_lookup_with_work(cache->policy, block, &cblock, data_dir, true, &op); in map_bio()
1641 if (unlikely(r && r != -ENOENT)) { in map_bio()
1643 cache_device_name(cache), r); in map_bio()
1648 if (r == -ENOENT && op) { in map_bio()
1649 bio_drop_shared_lock(cache, bio); in map_bio()
1650 BUG_ON(op->op != POLICY_PROMOTE); in map_bio()
1651 mg_start(cache, op, bio); in map_bio()
1655 r = policy_lookup(cache->policy, block, &cblock, data_dir, false, &background_queued); in map_bio()
1656 if (unlikely(r && r != -ENOENT)) { in map_bio()
1658 cache_device_name(cache), r); in map_bio()
1664 wake_migration_worker(cache); in map_bio()
1667 if (r == -ENOENT) { in map_bio()
1673 inc_miss_counter(cache, bio); in map_bio()
1674 if (pb->req_nr == 0) { in map_bio()
1675 accounted_begin(cache, bio); in map_bio()
1676 remap_to_origin_clear_discard(cache, bio, block); in map_bio()
1680 * longer needed because the block has been demoted. in map_bio()
1689 inc_hit_counter(cache, bio); in map_bio()
1693 * cache blocks that are written to. in map_bio()
1695 if (passthrough_mode(cache)) { in map_bio()
1697 bio_drop_shared_lock(cache, bio); in map_bio()
1698 atomic_inc(&cache->stats.demotion); in map_bio()
1699 invalidate_start(cache, cblock, block, bio); in map_bio()
1701 remap_to_origin_clear_discard(cache, bio, block); in map_bio()
1703 if (bio_data_dir(bio) == WRITE && writethrough_mode(cache) && in map_bio()
1704 !is_dirty(cache, cblock)) { in map_bio()
1705 remap_to_origin_and_cache(cache, bio, block, cblock); in map_bio()
1706 accounted_begin(cache, bio); in map_bio()
1708 remap_to_cache_dirty(cache, bio, block, cblock); in map_bio()
1715 if (bio->bi_opf & REQ_FUA) { in map_bio()
1720 accounted_complete(cache, bio); in map_bio()
1721 issue_after_commit(&cache->committer, bio); in map_bio()
1729 static bool process_bio(struct cache *cache, struct bio *bio) in process_bio() argument
1733 if (map_bio(cache, bio, get_bio_block(cache, bio), &commit_needed) == DM_MAPIO_REMAPPED) in process_bio()
1740 * A non-zero return indicates read_only or fail_io mode.
1742 static int commit(struct cache *cache, bool clean_shutdown) in commit() argument
1746 if (get_cache_mode(cache) >= CM_READ_ONLY) in commit()
1747 return -EINVAL; in commit()
1749 atomic_inc(&cache->stats.commit_count); in commit()
1750 r = dm_cache_commit(cache->cmd, clean_shutdown); in commit()
1752 metadata_operation_failed(cache, "dm_cache_commit", r); in commit()
1762 struct cache *cache = context; in commit_op() local
1764 if (dm_cache_changed_this_transaction(cache->cmd)) in commit_op()
1765 return errno_to_blk_status(commit(cache, false)); in commit_op()
1770 /*----------------------------------------------------------------*/
1772 static bool process_flush_bio(struct cache *cache, struct bio *bio) in process_flush_bio() argument
1776 if (!pb->req_nr) in process_flush_bio()
1777 remap_to_origin(cache, bio); in process_flush_bio()
1779 remap_to_cache(cache, bio, 0); in process_flush_bio()
1781 issue_after_commit(&cache->committer, bio); in process_flush_bio()
1785 static bool process_discard_bio(struct cache *cache, struct bio *bio) in process_discard_bio() argument
1794 calc_discard_block_range(cache, bio, &b, &e); in process_discard_bio()
1796 set_discard(cache, b); in process_discard_bio()
1800 if (cache->features.discard_passdown) { in process_discard_bio()
1801 remap_to_origin(cache, bio); in process_discard_bio()
1811 struct cache *cache = container_of(ws, struct cache, deferred_bio_worker); in process_deferred_bios() local
1819 spin_lock_irq(&cache->lock); in process_deferred_bios()
1820 bio_list_merge(&bios, &cache->deferred_bios); in process_deferred_bios()
1821 bio_list_init(&cache->deferred_bios); in process_deferred_bios()
1822 spin_unlock_irq(&cache->lock); in process_deferred_bios()
1825 if (bio->bi_opf & REQ_PREFLUSH) in process_deferred_bios()
1826 commit_needed = process_flush_bio(cache, bio) || commit_needed; in process_deferred_bios()
1829 commit_needed = process_discard_bio(cache, bio) || commit_needed; in process_deferred_bios()
1832 commit_needed = process_bio(cache, bio) || commit_needed; in process_deferred_bios()
1837 schedule_commit(&cache->committer); in process_deferred_bios()
1841 *--------------------------------------------------------------
1843 *--------------------------------------------------------------
1845 static void requeue_deferred_bios(struct cache *cache) in requeue_deferred_bios() argument
1851 bio_list_merge(&bios, &cache->deferred_bios); in requeue_deferred_bios()
1852 bio_list_init(&cache->deferred_bios); in requeue_deferred_bios()
1855 bio->bi_status = BLK_STS_DM_REQUEUE; in requeue_deferred_bios()
1867 struct cache *cache = container_of(to_delayed_work(ws), struct cache, waker); in do_waker() local
1869 policy_tick(cache->policy, true); in do_waker()
1870 wake_migration_worker(cache); in do_waker()
1871 schedule_commit(&cache->committer); in do_waker()
1872 queue_delayed_work(cache->wq, &cache->waker, COMMIT_PERIOD); in do_waker()
1879 struct cache *cache = container_of(ws, struct cache, migration_worker); in check_migrations() local
1883 b = spare_migration_bandwidth(cache); in check_migrations()
1885 r = policy_get_background_work(cache->policy, b == IDLE, &op); in check_migrations()
1886 if (r == -ENODATA) in check_migrations()
1891 cache_device_name(cache)); in check_migrations()
1895 r = mg_start(cache, op, NULL); in check_migrations()
1904 *--------------------------------------------------------------
1906 *--------------------------------------------------------------
1913 static void __destroy(struct cache *cache) in __destroy() argument
1915 mempool_exit(&cache->migration_pool); in __destroy()
1917 if (cache->prison) in __destroy()
1918 dm_bio_prison_destroy_v2(cache->prison); in __destroy()
1920 if (cache->wq) in __destroy()
1921 destroy_workqueue(cache->wq); in __destroy()
1923 if (cache->dirty_bitset) in __destroy()
1924 free_bitset(cache->dirty_bitset); in __destroy()
1926 if (cache->discard_bitset) in __destroy()
1927 free_bitset(cache->discard_bitset); in __destroy()
1929 if (cache->copier) in __destroy()
1930 dm_kcopyd_client_destroy(cache->copier); in __destroy()
1932 if (cache->cmd) in __destroy()
1933 dm_cache_metadata_close(cache->cmd); in __destroy()
1935 if (cache->metadata_dev) in __destroy()
1936 dm_put_device(cache->ti, cache->metadata_dev); in __destroy()
1938 if (cache->origin_dev) in __destroy()
1939 dm_put_device(cache->ti, cache->origin_dev); in __destroy()
1941 if (cache->cache_dev) in __destroy()
1942 dm_put_device(cache->ti, cache->cache_dev); in __destroy()
1944 if (cache->policy) in __destroy()
1945 dm_cache_policy_destroy(cache->policy); in __destroy()
1947 bioset_exit(&cache->bs); in __destroy()
1949 kfree(cache); in __destroy()
1952 static void destroy(struct cache *cache) in destroy() argument
1954 unsigned int i; in destroy() local
1956 cancel_delayed_work_sync(&cache->waker); in destroy()
1958 for (i = 0; i < cache->nr_ctr_args ; i++) in destroy()
1959 kfree(cache->ctr_args[i]); in destroy()
1960 kfree(cache->ctr_args); in destroy()
1962 __destroy(cache); in destroy()
1967 struct cache *cache = ti->private; in cache_dtr() local
1969 destroy(cache); in cache_dtr()
1974 return bdev_nr_sectors(dev->bdev); in get_dev_size()
1977 /*----------------------------------------------------------------*/
1980 * Construct a cache device mapping.
1982 * cache <metadata dev> <cache dev> <origin dev> <block size>
1987 * cache dev : fast device holding cached data blocks
1989 * block size : cache unit size in sectors
1999 * See cache-policies.txt for details.
2002 * writethrough : write through caching that prohibits cache block
2003 * content from being different from origin block content.
2005 * back cache block contents later for performance reasons,
2029 if (ca->metadata_dev) in destroy_cache_args()
2030 dm_put_device(ca->ti, ca->metadata_dev); in destroy_cache_args()
2032 if (ca->cache_dev) in destroy_cache_args()
2033 dm_put_device(ca->ti, ca->cache_dev); in destroy_cache_args()
2035 if (ca->origin_dev) in destroy_cache_args()
2036 dm_put_device(ca->ti, ca->origin_dev); in destroy_cache_args()
2043 if (!as->argc) { in at_least_one_arg()
2058 return -EINVAL; in parse_metadata_dev()
2060 r = dm_get_device(ca->ti, dm_shift_arg(as), in parse_metadata_dev()
2061 BLK_OPEN_READ | BLK_OPEN_WRITE, &ca->metadata_dev); in parse_metadata_dev()
2067 metadata_dev_size = get_dev_size(ca->metadata_dev); in parse_metadata_dev()
2070 ca->metadata_dev->bdev, THIN_METADATA_MAX_SECTORS); in parse_metadata_dev()
2081 return -EINVAL; in parse_cache_dev()
2083 r = dm_get_device(ca->ti, dm_shift_arg(as), in parse_cache_dev()
2084 BLK_OPEN_READ | BLK_OPEN_WRITE, &ca->cache_dev); in parse_cache_dev()
2086 *error = "Error opening cache device"; in parse_cache_dev()
2089 ca->cache_sectors = get_dev_size(ca->cache_dev); in parse_cache_dev()
2101 return -EINVAL; in parse_origin_dev()
2103 r = dm_get_device(ca->ti, dm_shift_arg(as), in parse_origin_dev()
2104 BLK_OPEN_READ | BLK_OPEN_WRITE, &ca->origin_dev); in parse_origin_dev()
2110 origin_sectors = get_dev_size(ca->origin_dev); in parse_origin_dev()
2111 if (ca->ti->len > origin_sectors) { in parse_origin_dev()
2112 *error = "Device size larger than cached device"; in parse_origin_dev()
2113 return -EINVAL; in parse_origin_dev()
2125 return -EINVAL; in parse_block_size()
2130 block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) { in parse_block_size()
2131 *error = "Invalid data block size"; in parse_block_size()
2132 return -EINVAL; in parse_block_size()
2135 if (block_size > ca->cache_sectors) { in parse_block_size()
2136 *error = "Data block size is larger than the cache device"; in parse_block_size()
2137 return -EINVAL; in parse_block_size()
2140 ca->block_size = block_size; in parse_block_size()
2147 cf->mode = CM_WRITE; in init_features()
2148 cf->io_mode = CM_IO_WRITEBACK; in init_features()
2149 cf->metadata_version = 1; in init_features()
2150 cf->discard_passdown = true; in init_features()
2157 {0, 3, "Invalid number of cache feature arguments"}, in parse_features()
2163 struct cache_features *cf = &ca->features; in parse_features()
2169 return -EINVAL; in parse_features()
2171 while (argc--) { in parse_features()
2175 cf->io_mode = CM_IO_WRITEBACK; in parse_features()
2180 cf->io_mode = CM_IO_WRITETHROUGH; in parse_features()
2185 cf->io_mode = CM_IO_PASSTHROUGH; in parse_features()
2190 cf->metadata_version = 2; in parse_features()
2193 cf->discard_passdown = false; in parse_features()
2196 *error = "Unrecognised cache feature requested"; in parse_features()
2197 return -EINVAL; in parse_features()
2202 *error = "Duplicate cache io_mode features requested"; in parse_features()
2203 return -EINVAL; in parse_features()
2219 return -EINVAL; in parse_policy()
2221 ca->policy_name = dm_shift_arg(as); in parse_policy()
2223 r = dm_read_arg_group(_args, as, &ca->policy_argc, error); in parse_policy()
2225 return -EINVAL; in parse_policy()
2227 ca->policy_argv = (const char **)as->argv; in parse_policy()
2228 dm_consume_args(as, ca->policy_argc); in parse_policy()
2269 /*----------------------------------------------------------------*/
2275 static int process_config_option(struct cache *cache, const char *key, const char *value) in process_config_option() argument
2281 return -EINVAL; in process_config_option()
2283 cache->migration_threshold = tmp; in process_config_option()
2290 static int set_config_value(struct cache *cache, const char *key, const char *value) in set_config_value() argument
2292 int r = process_config_option(cache, key, value); in set_config_value()
2295 r = policy_set_config_value(cache->policy, key, value); in set_config_value()
2303 static int set_config_values(struct cache *cache, int argc, const char **argv) in set_config_values() argument
2309 return -EINVAL; in set_config_values()
2313 r = set_config_value(cache, argv[0], argv[1]); in set_config_values()
2317 argc -= 2; in set_config_values()
2324 static int create_cache_policy(struct cache *cache, struct cache_args *ca, in create_cache_policy() argument
2327 struct dm_cache_policy *p = dm_cache_policy_create(ca->policy_name, in create_cache_policy()
2328 cache->cache_size, in create_cache_policy()
2329 cache->origin_sectors, in create_cache_policy()
2330 cache->sectors_per_block); in create_cache_policy()
2332 *error = "Error creating cache's policy"; in create_cache_policy()
2335 cache->policy = p; in create_cache_policy()
2336 BUG_ON(!cache->policy); in create_cache_policy()
2342 * We want the discard block size to be at least the size of the cache
2343 * block size and have no more than 2^14 discard blocks across the origin.
2367 static void set_cache_size(struct cache *cache, dm_cblock_t size) in set_cache_size() argument
2369 dm_block_t nr_blocks = from_cblock(size); in set_cache_size()
2371 if (nr_blocks > (1 << 20) && cache->cache_size != size) in set_cache_size()
2372 DMWARN_LIMIT("You have created a cache device with a lot of individual cache blocks (%llu)\n" in set_cache_size()
2374 "Please consider increasing the cache block size to reduce the overall cache block count.", in set_cache_size()
2377 cache->cache_size = size; in set_cache_size()
2382 static int cache_create(struct cache_args *ca, struct cache **result) in cache_create()
2385 char **error = &ca->ti->error; in cache_create()
2386 struct cache *cache; in cache_create() local
2387 struct dm_target *ti = ca->ti; in cache_create()
2390 bool may_format = ca->features.mode == CM_WRITE; in cache_create()
2392 cache = kzalloc(sizeof(*cache), GFP_KERNEL); in cache_create()
2393 if (!cache) in cache_create()
2394 return -ENOMEM; in cache_create()
2396 cache->ti = ca->ti; in cache_create()
2397 ti->private = cache; in cache_create()
2398 ti->accounts_remapped_io = true; in cache_create()
2399 ti->num_flush_bios = 2; in cache_create()
2400 ti->flush_supported = true; in cache_create()
2402 ti->num_discard_bios = 1; in cache_create()
2403 ti->discards_supported = true; in cache_create()
2405 ti->per_io_data_size = sizeof(struct per_bio_data); in cache_create()
2407 cache->features = ca->features; in cache_create()
2408 if (writethrough_mode(cache)) { in cache_create()
2410 r = bioset_init(&cache->bs, BIO_POOL_SIZE, 0, 0); in cache_create()
2415 cache->metadata_dev = ca->metadata_dev; in cache_create()
2416 cache->origin_dev = ca->origin_dev; in cache_create()
2417 cache->cache_dev = ca->cache_dev; in cache_create()
2419 ca->metadata_dev = ca->origin_dev = ca->cache_dev = NULL; in cache_create()
2421 origin_blocks = cache->origin_sectors = ti->len; in cache_create()
2422 origin_blocks = block_div(origin_blocks, ca->block_size); in cache_create()
2423 cache->origin_blocks = to_oblock(origin_blocks); in cache_create()
2425 cache->sectors_per_block = ca->block_size; in cache_create()
2426 if (dm_set_target_max_io_len(ti, cache->sectors_per_block)) { in cache_create()
2427 r = -EINVAL; in cache_create()
2431 if (ca->block_size & (ca->block_size - 1)) { in cache_create()
2432 dm_block_t cache_size = ca->cache_sectors; in cache_create()
2434 cache->sectors_per_block_shift = -1; in cache_create()
2435 cache_size = block_div(cache_size, ca->block_size); in cache_create()
2436 set_cache_size(cache, to_cblock(cache_size)); in cache_create()
2438 cache->sectors_per_block_shift = __ffs(ca->block_size); in cache_create()
2439 set_cache_size(cache, to_cblock(ca->cache_sectors >> cache->sectors_per_block_shift)); in cache_create()
2442 r = create_cache_policy(cache, ca, error); in cache_create()
2446 cache->policy_nr_args = ca->policy_argc; in cache_create()
2447 cache->migration_threshold = DEFAULT_MIGRATION_THRESHOLD; in cache_create()
2449 r = set_config_values(cache, ca->policy_argc, ca->policy_argv); in cache_create()
2451 *error = "Error setting cache policy's config values"; in cache_create()
2455 cmd = dm_cache_metadata_open(cache->metadata_dev->bdev, in cache_create()
2456 ca->block_size, may_format, in cache_create()
2457 dm_cache_policy_get_hint_size(cache->policy), in cache_create()
2458 ca->features.metadata_version); in cache_create()
2464 cache->cmd = cmd; in cache_create()
2465 set_cache_mode(cache, CM_WRITE); in cache_create()
2466 if (get_cache_mode(cache) != CM_WRITE) { in cache_create()
2468 r = -EINVAL; in cache_create()
2472 if (passthrough_mode(cache)) { in cache_create()
2475 r = dm_cache_metadata_all_clean(cache->cmd, &all_clean); in cache_create()
2483 r = -EINVAL; in cache_create()
2487 policy_allow_migrations(cache->policy, false); in cache_create()
2490 spin_lock_init(&cache->lock); in cache_create()
2491 bio_list_init(&cache->deferred_bios); in cache_create()
2492 atomic_set(&cache->nr_allocated_migrations, 0); in cache_create()
2493 atomic_set(&cache->nr_io_migrations, 0); in cache_create()
2494 init_waitqueue_head(&cache->migration_wait); in cache_create()
2496 r = -ENOMEM; in cache_create()
2497 atomic_set(&cache->nr_dirty, 0); in cache_create()
2498 cache->dirty_bitset = alloc_bitset(from_cblock(cache->cache_size)); in cache_create()
2499 if (!cache->dirty_bitset) { in cache_create()
2503 clear_bitset(cache->dirty_bitset, from_cblock(cache->cache_size)); in cache_create()
2505 cache->discard_block_size = in cache_create()
2506 calculate_discard_block_size(cache->sectors_per_block, in cache_create()
2507 cache->origin_sectors); in cache_create()
2508 cache->discard_nr_blocks = to_dblock(dm_sector_div_up(cache->origin_sectors, in cache_create()
2509 cache->discard_block_size)); in cache_create()
2510 cache->discard_bitset = alloc_bitset(from_dblock(cache->discard_nr_blocks)); in cache_create()
2511 if (!cache->discard_bitset) { in cache_create()
2515 clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks)); in cache_create()
2517 cache->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle); in cache_create()
2518 if (IS_ERR(cache->copier)) { in cache_create()
2520 r = PTR_ERR(cache->copier); in cache_create()
2524 cache->wq = alloc_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM, 0); in cache_create()
2525 if (!cache->wq) { in cache_create()
2529 INIT_WORK(&cache->deferred_bio_worker, process_deferred_bios); in cache_create()
2530 INIT_WORK(&cache->migration_worker, check_migrations); in cache_create()
2531 INIT_DELAYED_WORK(&cache->waker, do_waker); in cache_create()
2533 cache->prison = dm_bio_prison_create_v2(cache->wq); in cache_create()
2534 if (!cache->prison) { in cache_create()
2539 r = mempool_init_slab_pool(&cache->migration_pool, MIGRATION_POOL_SIZE, in cache_create()
2542 *error = "Error creating cache's migration mempool"; in cache_create()
2546 cache->need_tick_bio = true; in cache_create()
2547 cache->sized = false; in cache_create()
2548 cache->invalidate = false; in cache_create()
2549 cache->commit_requested = false; in cache_create()
2550 cache->loaded_mappings = false; in cache_create()
2551 cache->loaded_discards = false; in cache_create()
2553 load_stats(cache); in cache_create()
2555 atomic_set(&cache->stats.demotion, 0); in cache_create()
2556 atomic_set(&cache->stats.promotion, 0); in cache_create()
2557 atomic_set(&cache->stats.copies_avoided, 0); in cache_create()
2558 atomic_set(&cache->stats.cache_cell_clash, 0); in cache_create()
2559 atomic_set(&cache->stats.commit_count, 0); in cache_create()
2560 atomic_set(&cache->stats.discard_count, 0); in cache_create()
2562 spin_lock_init(&cache->invalidation_lock); in cache_create()
2563 INIT_LIST_HEAD(&cache->invalidation_requests); in cache_create()
2565 batcher_init(&cache->committer, commit_op, cache, in cache_create()
2566 issue_op, cache, cache->wq); in cache_create()
2567 dm_iot_init(&cache->tracker); in cache_create()
2569 init_rwsem(&cache->background_work_lock); in cache_create()
2570 prevent_background_work(cache); in cache_create()
2572 *result = cache; in cache_create()
2575 __destroy(cache); in cache_create()
2579 static int copy_ctr_args(struct cache *cache, int argc, const char **argv) in copy_ctr_args() argument
2581 unsigned int i; in copy_ctr_args() local
2586 return -ENOMEM; in copy_ctr_args()
2587 for (i = 0; i < argc; i++) { in copy_ctr_args()
2588 copy[i] = kstrdup(argv[i], GFP_KERNEL); in copy_ctr_args()
2589 if (!copy[i]) { in copy_ctr_args()
2590 while (i--) in copy_ctr_args()
2591 kfree(copy[i]); in copy_ctr_args()
2593 return -ENOMEM; in copy_ctr_args()
2597 cache->nr_ctr_args = argc; in copy_ctr_args()
2598 cache->ctr_args = copy; in copy_ctr_args()
2605 int r = -EINVAL; in cache_ctr()
2607 struct cache *cache = NULL; in cache_ctr() local
2611 ti->error = "Error allocating memory for cache"; in cache_ctr()
2612 return -ENOMEM; in cache_ctr()
2614 ca->ti = ti; in cache_ctr()
2616 r = parse_cache_args(ca, argc, argv, &ti->error); in cache_ctr()
2620 r = cache_create(ca, &cache); in cache_ctr()
2624 r = copy_ctr_args(cache, argc - 3, (const char **)argv + 3); in cache_ctr()
2626 __destroy(cache); in cache_ctr()
2630 ti->private = cache; in cache_ctr()
2636 /*----------------------------------------------------------------*/
2640 struct cache *cache = ti->private; in cache_map() local
2644 dm_oblock_t block = get_bio_block(cache, bio); in cache_map() local
2647 if (unlikely(from_oblock(block) >= from_oblock(cache->origin_blocks))) { in cache_map()
2649 * This can only occur if the io goes to a partial block at in cache_map()
2650 * the end of the origin device. We don't cache these. in cache_map()
2653 remap_to_origin(cache, bio); in cache_map()
2654 accounted_begin(cache, bio); in cache_map()
2659 defer_bio(cache, bio); in cache_map()
2663 r = map_bio(cache, bio, block, &commit_needed); in cache_map()
2665 schedule_commit(&cache->committer); in cache_map()
2672 struct cache *cache = ti->private; in cache_end_io() local
2676 if (pb->tick) { in cache_end_io()
2677 policy_tick(cache->policy, false); in cache_end_io()
2679 spin_lock_irqsave(&cache->lock, flags); in cache_end_io()
2680 cache->need_tick_bio = true; in cache_end_io()
2681 spin_unlock_irqrestore(&cache->lock, flags); in cache_end_io()
2684 bio_drop_shared_lock(cache, bio); in cache_end_io()
2685 accounted_complete(cache, bio); in cache_end_io()
2690 static int write_dirty_bitset(struct cache *cache) in write_dirty_bitset() argument
2694 if (get_cache_mode(cache) >= CM_READ_ONLY) in write_dirty_bitset()
2695 return -EINVAL; in write_dirty_bitset()
2697 r = dm_cache_set_dirty_bits(cache->cmd, from_cblock(cache->cache_size), cache->dirty_bitset); in write_dirty_bitset()
2699 metadata_operation_failed(cache, "dm_cache_set_dirty_bits", r); in write_dirty_bitset()
2704 static int write_discard_bitset(struct cache *cache) in write_discard_bitset() argument
2706 unsigned int i, r; in write_discard_bitset() local
2708 if (get_cache_mode(cache) >= CM_READ_ONLY) in write_discard_bitset()
2709 return -EINVAL; in write_discard_bitset()
2711 r = dm_cache_discard_bitset_resize(cache->cmd, cache->discard_block_size, in write_discard_bitset()
2712 cache->discard_nr_blocks); in write_discard_bitset()
2714 DMERR("%s: could not resize on-disk discard bitset", cache_device_name(cache)); in write_discard_bitset()
2715 metadata_operation_failed(cache, "dm_cache_discard_bitset_resize", r); in write_discard_bitset()
2719 for (i = 0; i < from_dblock(cache->discard_nr_blocks); i++) { in write_discard_bitset()
2720 r = dm_cache_set_discard(cache->cmd, to_dblock(i), in write_discard_bitset()
2721 is_discarded(cache, to_dblock(i))); in write_discard_bitset()
2723 metadata_operation_failed(cache, "dm_cache_set_discard", r); in write_discard_bitset()
2731 static int write_hints(struct cache *cache) in write_hints() argument
2735 if (get_cache_mode(cache) >= CM_READ_ONLY) in write_hints()
2736 return -EINVAL; in write_hints()
2738 r = dm_cache_write_hints(cache->cmd, cache->policy); in write_hints()
2740 metadata_operation_failed(cache, "dm_cache_write_hints", r); in write_hints()
2750 static bool sync_metadata(struct cache *cache) in sync_metadata() argument
2754 r1 = write_dirty_bitset(cache); in sync_metadata()
2756 DMERR("%s: could not write dirty bitset", cache_device_name(cache)); in sync_metadata()
2758 r2 = write_discard_bitset(cache); in sync_metadata()
2760 DMERR("%s: could not write discard bitset", cache_device_name(cache)); in sync_metadata()
2762 save_stats(cache); in sync_metadata()
2764 r3 = write_hints(cache); in sync_metadata()
2766 DMERR("%s: could not write hints", cache_device_name(cache)); in sync_metadata()
2773 r4 = commit(cache, !r1 && !r2 && !r3); in sync_metadata()
2775 DMERR("%s: could not write cache metadata", cache_device_name(cache)); in sync_metadata()
2782 struct cache *cache = ti->private; in cache_postsuspend() local
2784 prevent_background_work(cache); in cache_postsuspend()
2785 BUG_ON(atomic_read(&cache->nr_io_migrations)); in cache_postsuspend()
2787 cancel_delayed_work_sync(&cache->waker); in cache_postsuspend()
2788 drain_workqueue(cache->wq); in cache_postsuspend()
2789 WARN_ON(cache->tracker.in_flight); in cache_postsuspend()
2795 requeue_deferred_bios(cache); in cache_postsuspend()
2797 if (get_cache_mode(cache) == CM_WRITE) in cache_postsuspend()
2798 (void) sync_metadata(cache); in cache_postsuspend()
2804 struct cache *cache = context; in load_mapping() local
2807 set_bit(from_cblock(cblock), cache->dirty_bitset); in load_mapping()
2808 atomic_inc(&cache->nr_dirty); in load_mapping()
2810 clear_bit(from_cblock(cblock), cache->dirty_bitset); in load_mapping()
2812 return policy_load_mapping(cache->policy, oblock, cblock, dirty, hint, hint_valid); in load_mapping()
2816 * The discard block size in the on disk metadata is not
2819 * covers a complete block of the new size.
2822 struct cache *cache; member
2825 * These blocks are sized using the on disk dblock size, rather
2832 static void discard_load_info_init(struct cache *cache, in discard_load_info_init() argument
2835 li->cache = cache; in discard_load_info_init()
2836 li->discard_begin = li->discard_end = 0; in discard_load_info_init()
2843 if (li->discard_begin == li->discard_end) in set_discard_range()
2849 b = li->discard_begin * li->block_size; in set_discard_range()
2850 e = li->discard_end * li->block_size; in set_discard_range()
2853 * Then convert back to the current dblock size. in set_discard_range()
2855 b = dm_sector_div_up(b, li->cache->discard_block_size); in set_discard_range()
2856 sector_div(e, li->cache->discard_block_size); in set_discard_range()
2862 if (e > from_dblock(li->cache->discard_nr_blocks)) in set_discard_range()
2863 e = from_dblock(li->cache->discard_nr_blocks); in set_discard_range()
2866 set_discard(li->cache, to_dblock(b)); in set_discard_range()
2874 li->block_size = discard_block_size; in load_discard()
2877 if (from_dblock(dblock) == li->discard_end) in load_discard()
2881 li->discard_end = li->discard_end + 1ULL; in load_discard()
2888 li->discard_begin = from_dblock(dblock); in load_discard()
2889 li->discard_end = li->discard_begin + 1ULL; in load_discard()
2893 li->discard_begin = li->discard_end = 0; in load_discard()
2899 static dm_cblock_t get_cache_dev_size(struct cache *cache) in get_cache_dev_size() argument
2901 sector_t size = get_dev_size(cache->cache_dev); in get_cache_dev_size() local
2902 (void) sector_div(size, cache->sectors_per_block); in get_cache_dev_size()
2903 return to_cblock(size); in get_cache_dev_size()
2906 static bool can_resize(struct cache *cache, dm_cblock_t new_size) in can_resize() argument
2908 if (from_cblock(new_size) > from_cblock(cache->cache_size)) { in can_resize()
2909 DMERR("%s: unable to extend cache due to missing cache table reload", in can_resize()
2910 cache_device_name(cache)); in can_resize()
2915 * We can't drop a dirty block when shrinking the cache. in can_resize()
2917 if (cache->loaded_mappings) { in can_resize()
2918 new_size = to_cblock(find_next_bit(cache->dirty_bitset, in can_resize()
2919 from_cblock(cache->cache_size), in can_resize()
2921 if (new_size != cache->cache_size) { in can_resize()
2922 DMERR("%s: unable to shrink cache; cache block %llu is dirty", in can_resize()
2923 cache_device_name(cache), in can_resize()
2932 static int resize_cache_dev(struct cache *cache, dm_cblock_t new_size) in resize_cache_dev() argument
2936 r = dm_cache_resize(cache->cmd, new_size); in resize_cache_dev()
2938 DMERR("%s: could not resize cache metadata", cache_device_name(cache)); in resize_cache_dev()
2939 metadata_operation_failed(cache, "dm_cache_resize", r); in resize_cache_dev()
2943 set_cache_size(cache, new_size); in resize_cache_dev()
2951 struct cache *cache = ti->private; in cache_preresume() local
2952 dm_cblock_t csize = get_cache_dev_size(cache); in cache_preresume()
2955 * Check to see if the cache has resized. in cache_preresume()
2957 if (!cache->sized || csize != cache->cache_size) { in cache_preresume()
2958 if (!can_resize(cache, csize)) in cache_preresume()
2959 return -EINVAL; in cache_preresume()
2961 r = resize_cache_dev(cache, csize); in cache_preresume()
2965 cache->sized = true; in cache_preresume()
2968 if (!cache->loaded_mappings) { in cache_preresume()
2969 r = dm_cache_load_mappings(cache->cmd, cache->policy, in cache_preresume()
2970 load_mapping, cache); in cache_preresume()
2972 DMERR("%s: could not load cache mappings", cache_device_name(cache)); in cache_preresume()
2973 metadata_operation_failed(cache, "dm_cache_load_mappings", r); in cache_preresume()
2977 cache->loaded_mappings = true; in cache_preresume()
2980 if (!cache->loaded_discards) { in cache_preresume()
2985 * discard block size changed. To be safe we start by in cache_preresume()
2988 clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks)); in cache_preresume()
2990 discard_load_info_init(cache, &li); in cache_preresume()
2991 r = dm_cache_load_discards(cache->cmd, load_discard, &li); in cache_preresume()
2993 DMERR("%s: could not load origin discards", cache_device_name(cache)); in cache_preresume()
2994 metadata_operation_failed(cache, "dm_cache_load_discards", r); in cache_preresume()
2999 cache->loaded_discards = true; in cache_preresume()
3007 struct cache *cache = ti->private; in cache_resume() local
3009 cache->need_tick_bio = true; in cache_resume()
3010 allow_background_work(cache); in cache_resume()
3011 do_waker(&cache->waker.work); in cache_resume()
3014 static void emit_flags(struct cache *cache, char *result, in emit_flags() argument
3018 struct cache_features *cf = &cache->features; in emit_flags()
3019 unsigned int count = (cf->metadata_version == 2) + !cf->discard_passdown + 1; in emit_flags()
3023 if (cf->metadata_version == 2) in emit_flags()
3026 if (writethrough_mode(cache)) in emit_flags()
3029 else if (passthrough_mode(cache)) in emit_flags()
3032 else if (writeback_mode(cache)) in emit_flags()
3038 cache_device_name(cache), (int) cf->io_mode); in emit_flags()
3041 if (!cf->discard_passdown) in emit_flags()
3050 * <metadata block size> <#used metadata blocks>/<#total metadata blocks>
3051 * <cache block size> <#used cache blocks>/<#total cache blocks>
3056 * <policy name> <#policy args> <policy args>* <cache metadata mode> <needs_check>
3062 unsigned int i; in cache_status() local
3067 struct cache *cache = ti->private; in cache_status() local
3073 if (get_cache_mode(cache) == CM_FAIL) { in cache_status()
3078 /* Commit to ensure statistics aren't out-of-date */ in cache_status()
3080 (void) commit(cache, false); in cache_status()
3082 r = dm_cache_get_free_metadata_block_count(cache->cmd, &nr_free_blocks_metadata); in cache_status()
3085 cache_device_name(cache), r); in cache_status()
3089 r = dm_cache_get_metadata_dev_size(cache->cmd, &nr_blocks_metadata); in cache_status()
3092 cache_device_name(cache), r); in cache_status()
3096 residency = policy_residency(cache->policy); in cache_status()
3100 (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata), in cache_status()
3102 (unsigned long long)cache->sectors_per_block, in cache_status()
3104 (unsigned long long) from_cblock(cache->cache_size), in cache_status()
3105 (unsigned int) atomic_read(&cache->stats.read_hit), in cache_status()
3106 (unsigned int) atomic_read(&cache->stats.read_miss), in cache_status()
3107 (unsigned int) atomic_read(&cache->stats.write_hit), in cache_status()
3108 (unsigned int) atomic_read(&cache->stats.write_miss), in cache_status()
3109 (unsigned int) atomic_read(&cache->stats.demotion), in cache_status()
3110 (unsigned int) atomic_read(&cache->stats.promotion), in cache_status()
3111 (unsigned long) atomic_read(&cache->nr_dirty)); in cache_status()
3113 emit_flags(cache, result, maxlen, &sz); in cache_status()
3115 DMEMIT("2 migration_threshold %llu ", (unsigned long long) cache->migration_threshold); in cache_status()
3117 DMEMIT("%s ", dm_cache_policy_get_name(cache->policy)); in cache_status()
3119 r = policy_emit_config_values(cache->policy, result, maxlen, &sz); in cache_status()
3122 cache_device_name(cache), r); in cache_status()
3125 if (get_cache_mode(cache) == CM_READ_ONLY) in cache_status()
3130 r = dm_cache_metadata_needs_check(cache->cmd, &needs_check); in cache_status()
3135 DMEMIT("- "); in cache_status()
3140 format_dev_t(buf, cache->metadata_dev->bdev->bd_dev); in cache_status()
3142 format_dev_t(buf, cache->cache_dev->bdev->bd_dev); in cache_status()
3144 format_dev_t(buf, cache->origin_dev->bdev->bd_dev); in cache_status()
3147 for (i = 0; i < cache->nr_ctr_args - 1; i++) in cache_status()
3148 DMEMIT(" %s", cache->ctr_args[i]); in cache_status()
3149 if (cache->nr_ctr_args) in cache_status()
3150 DMEMIT(" %s", cache->ctr_args[cache->nr_ctr_args - 1]); in cache_status()
3154 DMEMIT_TARGET_NAME_VERSION(ti->type); in cache_status()
3155 if (get_cache_mode(cache) == CM_FAIL) in cache_status()
3157 else if (get_cache_mode(cache) == CM_READ_ONLY) in cache_status()
3162 format_dev_t(buf, cache->metadata_dev->bdev->bd_dev); in cache_status()
3164 format_dev_t(buf, cache->cache_dev->bdev->bd_dev); in cache_status()
3166 format_dev_t(buf, cache->origin_dev->bdev->bd_dev); in cache_status()
3168 DMEMIT(",writethrough=%c", writethrough_mode(cache) ? 'y' : 'n'); in cache_status()
3169 DMEMIT(",writeback=%c", writeback_mode(cache) ? 'y' : 'n'); in cache_status()
3170 DMEMIT(",passthrough=%c", passthrough_mode(cache) ? 'y' : 'n'); in cache_status()
3171 DMEMIT(",metadata2=%c", cache->features.metadata_version == 2 ? 'y' : 'n'); in cache_status()
3172 DMEMIT(",no_discard_passdown=%c", cache->features.discard_passdown ? 'n' : 'y'); in cache_status()
3184 * Defines a range of cblocks, begin to (end - 1) are in the range. end is
3185 * the one-past-the-end value.
3193 * A cache block range can take two forms:
3195 * i) A single cblock, eg. '3456'
3196 * ii) A begin and end cblock with a dash between, eg. 123-234
3198 static int parse_cblock_range(struct cache *cache, const char *str, in parse_cblock_range() argument
3208 r = sscanf(str, "%llu-%llu%c", &b, &e, &dummy); in parse_cblock_range()
3213 result->begin = to_cblock(b); in parse_cblock_range()
3214 result->end = to_cblock(e); in parse_cblock_range()
3219 * That didn't work, try form (i). in parse_cblock_range()
3226 result->begin = to_cblock(b); in parse_cblock_range()
3227 result->end = to_cblock(from_cblock(result->begin) + 1u); in parse_cblock_range()
3231 DMERR("%s: invalid cblock range '%s'", cache_device_name(cache), str); in parse_cblock_range()
3232 return -EINVAL; in parse_cblock_range()
3235 static int validate_cblock_range(struct cache *cache, struct cblock_range *range) in validate_cblock_range() argument
3237 uint64_t b = from_cblock(range->begin); in validate_cblock_range()
3238 uint64_t e = from_cblock(range->end); in validate_cblock_range()
3239 uint64_t n = from_cblock(cache->cache_size); in validate_cblock_range()
3243 cache_device_name(cache), b, n); in validate_cblock_range()
3244 return -EINVAL; in validate_cblock_range()
3249 cache_device_name(cache), e, n); in validate_cblock_range()
3250 return -EINVAL; in validate_cblock_range()
3255 cache_device_name(cache), b, e); in validate_cblock_range()
3256 return -EINVAL; in validate_cblock_range()
3267 static int request_invalidation(struct cache *cache, struct cblock_range *range) in request_invalidation() argument
3277 while (range->begin != range->end) { in request_invalidation()
3278 r = invalidate_cblock(cache, range->begin); in request_invalidation()
3282 range->begin = cblock_succ(range->begin); in request_invalidation()
3285 cache->commit_requested = true; in request_invalidation()
3289 static int process_invalidate_cblocks_message(struct cache *cache, unsigned int count, in process_invalidate_cblocks_message() argument
3293 unsigned int i; in process_invalidate_cblocks_message() local
3296 if (!passthrough_mode(cache)) { in process_invalidate_cblocks_message()
3297 DMERR("%s: cache has to be in passthrough mode for invalidation", in process_invalidate_cblocks_message()
3298 cache_device_name(cache)); in process_invalidate_cblocks_message()
3299 return -EPERM; in process_invalidate_cblocks_message()
3302 for (i = 0; i < count; i++) { in process_invalidate_cblocks_message()
3303 r = parse_cblock_range(cache, cblock_ranges[i], &range); in process_invalidate_cblocks_message()
3307 r = validate_cblock_range(cache, &range); in process_invalidate_cblocks_message()
3314 r = request_invalidation(cache, &range); in process_invalidate_cblocks_message()
3326 * "invalidate_cblocks [(<begin>)|(<begin>-<end>)]*
3328 * The key migration_threshold is supported by the cache target core.
3333 struct cache *cache = ti->private; in cache_message() local
3336 return -EINVAL; in cache_message()
3338 if (get_cache_mode(cache) >= CM_READ_ONLY) { in cache_message()
3339 DMERR("%s: unable to service cache target messages in READ_ONLY or FAIL mode", in cache_message()
3340 cache_device_name(cache)); in cache_message()
3341 return -EOPNOTSUPP; in cache_message()
3345 return process_invalidate_cblocks_message(cache, argc - 1, (const char **) argv + 1); in cache_message()
3348 return -EINVAL; in cache_message()
3350 return set_config_value(cache, argv[0], argv[1]); in cache_message()
3357 struct cache *cache = ti->private; in cache_iterate_devices() local
3359 r = fn(ti, cache->cache_dev, 0, get_dev_size(cache->cache_dev), data); in cache_iterate_devices()
3361 r = fn(ti, cache->origin_dev, 0, ti->len, data); in cache_iterate_devices()
3370 static void disable_passdown_if_not_supported(struct cache *cache) in disable_passdown_if_not_supported() argument
3372 struct block_device *origin_bdev = cache->origin_dev->bdev; in disable_passdown_if_not_supported()
3373 struct queue_limits *origin_limits = &bdev_get_queue(origin_bdev)->limits; in disable_passdown_if_not_supported()
3376 if (!cache->features.discard_passdown) in disable_passdown_if_not_supported()
3382 else if (origin_limits->max_discard_sectors < cache->sectors_per_block) in disable_passdown_if_not_supported()
3383 reason = "max discard sectors smaller than a block"; in disable_passdown_if_not_supported()
3388 cache->features.discard_passdown = false; in disable_passdown_if_not_supported()
3392 static void set_discard_limits(struct cache *cache, struct queue_limits *limits) in set_discard_limits() argument
3394 struct block_device *origin_bdev = cache->origin_dev->bdev; in set_discard_limits()
3395 struct queue_limits *origin_limits = &bdev_get_queue(origin_bdev)->limits; in set_discard_limits()
3397 if (!cache->features.discard_passdown) { in set_discard_limits()
3399 limits->max_discard_sectors = min_t(sector_t, cache->discard_block_size * 1024, in set_discard_limits()
3400 cache->origin_sectors); in set_discard_limits()
3401 limits->discard_granularity = cache->discard_block_size << SECTOR_SHIFT; in set_discard_limits()
3409 limits->max_discard_sectors = origin_limits->max_discard_sectors; in set_discard_limits()
3410 limits->max_hw_discard_sectors = origin_limits->max_hw_discard_sectors; in set_discard_limits()
3411 limits->discard_granularity = origin_limits->discard_granularity; in set_discard_limits()
3412 limits->discard_alignment = origin_limits->discard_alignment; in set_discard_limits()
3413 limits->discard_misaligned = origin_limits->discard_misaligned; in set_discard_limits()
3418 struct cache *cache = ti->private; in cache_io_hints() local
3419 uint64_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT; in cache_io_hints()
3422 * If the system-determined stacked limits are compatible with the in cache_io_hints()
3423 * cache's blocksize (io_opt is a factor) do not override them. in cache_io_hints()
3425 if (io_opt_sectors < cache->sectors_per_block || in cache_io_hints()
3426 do_div(io_opt_sectors, cache->sectors_per_block)) { in cache_io_hints()
3427 blk_limits_io_min(limits, cache->sectors_per_block << SECTOR_SHIFT); in cache_io_hints()
3428 blk_limits_io_opt(limits, cache->sectors_per_block << SECTOR_SHIFT); in cache_io_hints()
3431 disable_passdown_if_not_supported(cache); in cache_io_hints()
3432 set_discard_limits(cache, limits); in cache_io_hints()
3435 /*----------------------------------------------------------------*/
3438 .name = "cache",
3460 r = -ENOMEM; in dm_cache_init()
3467 r = -ENOMEM; in dm_cache_init()
3494 MODULE_DESCRIPTION(DM_NAME " cache target");