13bd94003SHeinz Mauelshagen // SPDX-License-Identifier: GPL-2.0-only
2c6b4fcbaSJoe Thornber /*
3c6b4fcbaSJoe Thornber * Copyright (C) 2012 Red Hat. All rights reserved.
4c6b4fcbaSJoe Thornber *
5c6b4fcbaSJoe Thornber * This file is released under the GPL.
6c6b4fcbaSJoe Thornber */
7c6b4fcbaSJoe Thornber
8c6b4fcbaSJoe Thornber #include "dm.h"
9b29d4986SJoe Thornber #include "dm-bio-prison-v2.h"
10b844fe69SDarrick J. Wong #include "dm-bio-record.h"
11c6b4fcbaSJoe Thornber #include "dm-cache-metadata.h"
12dc4fa29fSMike Snitzer #include "dm-io-tracker.h"
13c6b4fcbaSJoe Thornber
14c6b4fcbaSJoe Thornber #include <linux/dm-io.h>
15c6b4fcbaSJoe Thornber #include <linux/dm-kcopyd.h>
160f30af98SManuel Schölling #include <linux/jiffies.h>
17c6b4fcbaSJoe Thornber #include <linux/init.h>
18c6b4fcbaSJoe Thornber #include <linux/mempool.h>
19c6b4fcbaSJoe Thornber #include <linux/module.h>
20b29d4986SJoe Thornber #include <linux/rwsem.h>
21c6b4fcbaSJoe Thornber #include <linux/slab.h>
22c6b4fcbaSJoe Thornber #include <linux/vmalloc.h>
23c6b4fcbaSJoe Thornber
24c6b4fcbaSJoe Thornber #define DM_MSG_PREFIX "cache"
25c6b4fcbaSJoe Thornber
26c6b4fcbaSJoe Thornber DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(cache_copy_throttle,
27c6b4fcbaSJoe Thornber "A percentage of time allocated for copying to and/or from cache");
28c6b4fcbaSJoe Thornber
29c6b4fcbaSJoe Thornber /*----------------------------------------------------------------*/
30c6b4fcbaSJoe Thornber
31b29d4986SJoe Thornber /*
32b29d4986SJoe Thornber * Glossary:
33b29d4986SJoe Thornber *
34b29d4986SJoe Thornber * oblock: index of an origin block
35b29d4986SJoe Thornber * cblock: index of a cache block
36b29d4986SJoe Thornber * promotion: movement of a block from origin to cache
37b29d4986SJoe Thornber * demotion: movement of a block from cache to origin
38b29d4986SJoe Thornber * migration: movement of a block between the origin and cache device,
39b29d4986SJoe Thornber * either direction
40b29d4986SJoe Thornber */
41b29d4986SJoe Thornber
42b29d4986SJoe Thornber /*----------------------------------------------------------------*/
4377289d32SJoe Thornber
44c6b4fcbaSJoe Thornber /*
45b29d4986SJoe Thornber * Represents a chunk of future work. 'input' allows continuations to pass
46b29d4986SJoe Thornber * values between themselves, typically error values.
47c6b4fcbaSJoe Thornber */
48b29d4986SJoe Thornber struct continuation {
49b29d4986SJoe Thornber struct work_struct ws;
504e4cbee9SChristoph Hellwig blk_status_t input;
51b29d4986SJoe Thornber };
52b29d4986SJoe Thornber
init_continuation(struct continuation * k,void (* fn)(struct work_struct *))53b29d4986SJoe Thornber static inline void init_continuation(struct continuation *k,
54b29d4986SJoe Thornber void (*fn)(struct work_struct *))
55b29d4986SJoe Thornber {
56b29d4986SJoe Thornber INIT_WORK(&k->ws, fn);
57b29d4986SJoe Thornber k->input = 0;
58b29d4986SJoe Thornber }
59b29d4986SJoe Thornber
queue_continuation(struct workqueue_struct * wq,struct continuation * k)60b29d4986SJoe Thornber static inline void queue_continuation(struct workqueue_struct *wq,
61b29d4986SJoe Thornber struct continuation *k)
62b29d4986SJoe Thornber {
63b29d4986SJoe Thornber queue_work(wq, &k->ws);
64b29d4986SJoe Thornber }
65c6b4fcbaSJoe Thornber
66c6b4fcbaSJoe Thornber /*----------------------------------------------------------------*/
67c6b4fcbaSJoe Thornber
68c9d28d5dSJoe Thornber /*
69b29d4986SJoe Thornber * The batcher collects together pieces of work that need a particular
70b29d4986SJoe Thornber * operation to occur before they can proceed (typically a commit).
71b29d4986SJoe Thornber */
72b29d4986SJoe Thornber struct batcher {
73b29d4986SJoe Thornber /*
74b29d4986SJoe Thornber * The operation that everyone is waiting for.
75b29d4986SJoe Thornber */
764e4cbee9SChristoph Hellwig blk_status_t (*commit_op)(void *context);
77b29d4986SJoe Thornber void *commit_context;
78b29d4986SJoe Thornber
79b29d4986SJoe Thornber /*
80b29d4986SJoe Thornber * This is how bios should be issued once the commit op is complete
81b29d4986SJoe Thornber * (accounted_request).
82b29d4986SJoe Thornber */
83b29d4986SJoe Thornber void (*issue_op)(struct bio *bio, void *context);
84b29d4986SJoe Thornber void *issue_context;
85b29d4986SJoe Thornber
86b29d4986SJoe Thornber /*
87b29d4986SJoe Thornber * Queued work gets put on here after commit.
88b29d4986SJoe Thornber */
89b29d4986SJoe Thornber struct workqueue_struct *wq;
90b29d4986SJoe Thornber
91b29d4986SJoe Thornber spinlock_t lock;
92b29d4986SJoe Thornber struct list_head work_items;
93b29d4986SJoe Thornber struct bio_list bios;
94b29d4986SJoe Thornber struct work_struct commit_work;
95b29d4986SJoe Thornber
96b29d4986SJoe Thornber bool commit_scheduled;
97b29d4986SJoe Thornber };
98b29d4986SJoe Thornber
__commit(struct work_struct * _ws)99b29d4986SJoe Thornber static void __commit(struct work_struct *_ws)
100b29d4986SJoe Thornber {
101b29d4986SJoe Thornber struct batcher *b = container_of(_ws, struct batcher, commit_work);
1024e4cbee9SChristoph Hellwig blk_status_t r;
103b29d4986SJoe Thornber struct list_head work_items;
104b29d4986SJoe Thornber struct work_struct *ws, *tmp;
105b29d4986SJoe Thornber struct continuation *k;
106b29d4986SJoe Thornber struct bio *bio;
107b29d4986SJoe Thornber struct bio_list bios;
108b29d4986SJoe Thornber
109b29d4986SJoe Thornber INIT_LIST_HEAD(&work_items);
110b29d4986SJoe Thornber bio_list_init(&bios);
111b29d4986SJoe Thornber
112b29d4986SJoe Thornber /*
113b29d4986SJoe Thornber * We have to grab these before the commit_op to avoid a race
114b29d4986SJoe Thornber * condition.
115b29d4986SJoe Thornber */
11626b924b9SMikulas Patocka spin_lock_irq(&b->lock);
117b29d4986SJoe Thornber list_splice_init(&b->work_items, &work_items);
118b29d4986SJoe Thornber bio_list_merge(&bios, &b->bios);
119b29d4986SJoe Thornber bio_list_init(&b->bios);
120b29d4986SJoe Thornber b->commit_scheduled = false;
12126b924b9SMikulas Patocka spin_unlock_irq(&b->lock);
122b29d4986SJoe Thornber
123b29d4986SJoe Thornber r = b->commit_op(b->commit_context);
124b29d4986SJoe Thornber
125b29d4986SJoe Thornber list_for_each_entry_safe(ws, tmp, &work_items, entry) {
126b29d4986SJoe Thornber k = container_of(ws, struct continuation, ws);
127b29d4986SJoe Thornber k->input = r;
128b29d4986SJoe Thornber INIT_LIST_HEAD(&ws->entry); /* to avoid a WARN_ON */
129b29d4986SJoe Thornber queue_work(b->wq, ws);
130b29d4986SJoe Thornber }
131b29d4986SJoe Thornber
132b29d4986SJoe Thornber while ((bio = bio_list_pop(&bios))) {
133b29d4986SJoe Thornber if (r) {
1344e4cbee9SChristoph Hellwig bio->bi_status = r;
135b29d4986SJoe Thornber bio_endio(bio);
136b29d4986SJoe Thornber } else
137b29d4986SJoe Thornber b->issue_op(bio, b->issue_context);
138b29d4986SJoe Thornber }
139b29d4986SJoe Thornber }
140b29d4986SJoe Thornber
batcher_init(struct batcher * b,blk_status_t (* commit_op)(void *),void * commit_context,void (* issue_op)(struct bio * bio,void *),void * issue_context,struct workqueue_struct * wq)141b29d4986SJoe Thornber static void batcher_init(struct batcher *b,
1424e4cbee9SChristoph Hellwig blk_status_t (*commit_op)(void *),
143b29d4986SJoe Thornber void *commit_context,
144b29d4986SJoe Thornber void (*issue_op)(struct bio *bio, void *),
145b29d4986SJoe Thornber void *issue_context,
146b29d4986SJoe Thornber struct workqueue_struct *wq)
147b29d4986SJoe Thornber {
148b29d4986SJoe Thornber b->commit_op = commit_op;
149b29d4986SJoe Thornber b->commit_context = commit_context;
150b29d4986SJoe Thornber b->issue_op = issue_op;
151b29d4986SJoe Thornber b->issue_context = issue_context;
152b29d4986SJoe Thornber b->wq = wq;
153b29d4986SJoe Thornber
154b29d4986SJoe Thornber spin_lock_init(&b->lock);
155b29d4986SJoe Thornber INIT_LIST_HEAD(&b->work_items);
156b29d4986SJoe Thornber bio_list_init(&b->bios);
157b29d4986SJoe Thornber INIT_WORK(&b->commit_work, __commit);
158b29d4986SJoe Thornber b->commit_scheduled = false;
159b29d4986SJoe Thornber }
160b29d4986SJoe Thornber
async_commit(struct batcher * b)161b29d4986SJoe Thornber static void async_commit(struct batcher *b)
162b29d4986SJoe Thornber {
163b29d4986SJoe Thornber queue_work(b->wq, &b->commit_work);
164b29d4986SJoe Thornber }
165b29d4986SJoe Thornber
continue_after_commit(struct batcher * b,struct continuation * k)166b29d4986SJoe Thornber static void continue_after_commit(struct batcher *b, struct continuation *k)
167b29d4986SJoe Thornber {
168b29d4986SJoe Thornber bool commit_scheduled;
169b29d4986SJoe Thornber
17026b924b9SMikulas Patocka spin_lock_irq(&b->lock);
171b29d4986SJoe Thornber commit_scheduled = b->commit_scheduled;
172b29d4986SJoe Thornber list_add_tail(&k->ws.entry, &b->work_items);
17326b924b9SMikulas Patocka spin_unlock_irq(&b->lock);
174b29d4986SJoe Thornber
175b29d4986SJoe Thornber if (commit_scheduled)
176b29d4986SJoe Thornber async_commit(b);
177b29d4986SJoe Thornber }
178b29d4986SJoe Thornber
179b29d4986SJoe Thornber /*
180b29d4986SJoe Thornber * Bios are errored if commit failed.
181b29d4986SJoe Thornber */
issue_after_commit(struct batcher * b,struct bio * bio)182b29d4986SJoe Thornber static void issue_after_commit(struct batcher *b, struct bio *bio)
183b29d4986SJoe Thornber {
184b29d4986SJoe Thornber bool commit_scheduled;
185b29d4986SJoe Thornber
18626b924b9SMikulas Patocka spin_lock_irq(&b->lock);
187b29d4986SJoe Thornber commit_scheduled = b->commit_scheduled;
188b29d4986SJoe Thornber bio_list_add(&b->bios, bio);
18926b924b9SMikulas Patocka spin_unlock_irq(&b->lock);
190b29d4986SJoe Thornber
191b29d4986SJoe Thornber if (commit_scheduled)
192b29d4986SJoe Thornber async_commit(b);
193b29d4986SJoe Thornber }
194b29d4986SJoe Thornber
195b29d4986SJoe Thornber /*
196b29d4986SJoe Thornber * Call this if some urgent work is waiting for the commit to complete.
197b29d4986SJoe Thornber */
schedule_commit(struct batcher * b)198b29d4986SJoe Thornber static void schedule_commit(struct batcher *b)
199b29d4986SJoe Thornber {
200b29d4986SJoe Thornber bool immediate;
201b29d4986SJoe Thornber
20226b924b9SMikulas Patocka spin_lock_irq(&b->lock);
203b29d4986SJoe Thornber immediate = !list_empty(&b->work_items) || !bio_list_empty(&b->bios);
204b29d4986SJoe Thornber b->commit_scheduled = true;
20526b924b9SMikulas Patocka spin_unlock_irq(&b->lock);
206b29d4986SJoe Thornber
207b29d4986SJoe Thornber if (immediate)
208b29d4986SJoe Thornber async_commit(b);
209b29d4986SJoe Thornber }
210b29d4986SJoe Thornber
211b29d4986SJoe Thornber /*
212c9d28d5dSJoe Thornber * There are a couple of places where we let a bio run, but want to do some
213c9d28d5dSJoe Thornber * work before calling its endio function. We do this by temporarily
214c9d28d5dSJoe Thornber * changing the endio fn.
215c9d28d5dSJoe Thornber */
216c9d28d5dSJoe Thornber struct dm_hook_info {
217c9d28d5dSJoe Thornber bio_end_io_t *bi_end_io;
218c9d28d5dSJoe Thornber };
219c9d28d5dSJoe Thornber
dm_hook_bio(struct dm_hook_info * h,struct bio * bio,bio_end_io_t * bi_end_io,void * bi_private)220c9d28d5dSJoe Thornber static void dm_hook_bio(struct dm_hook_info *h, struct bio *bio,
221c9d28d5dSJoe Thornber bio_end_io_t *bi_end_io, void *bi_private)
222c9d28d5dSJoe Thornber {
223c9d28d5dSJoe Thornber h->bi_end_io = bio->bi_end_io;
224c9d28d5dSJoe Thornber
225c9d28d5dSJoe Thornber bio->bi_end_io = bi_end_io;
226c9d28d5dSJoe Thornber bio->bi_private = bi_private;
227c9d28d5dSJoe Thornber }
228c9d28d5dSJoe Thornber
dm_unhook_bio(struct dm_hook_info * h,struct bio * bio)229c9d28d5dSJoe Thornber static void dm_unhook_bio(struct dm_hook_info *h, struct bio *bio)
230c9d28d5dSJoe Thornber {
231c9d28d5dSJoe Thornber bio->bi_end_io = h->bi_end_io;
232c9d28d5dSJoe Thornber }
233c9d28d5dSJoe Thornber
234c9d28d5dSJoe Thornber /*----------------------------------------------------------------*/
235c9d28d5dSJoe Thornber
236c6b4fcbaSJoe Thornber #define MIGRATION_POOL_SIZE 128
237c6b4fcbaSJoe Thornber #define COMMIT_PERIOD HZ
238c6b4fcbaSJoe Thornber #define MIGRATION_COUNT_WINDOW 10
239c6b4fcbaSJoe Thornber
240c6b4fcbaSJoe Thornber /*
24105473044SMike Snitzer * The block size of the device holding cache data must be
24205473044SMike Snitzer * between 32KB and 1GB.
243c6b4fcbaSJoe Thornber */
244c6b4fcbaSJoe Thornber #define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (32 * 1024 >> SECTOR_SHIFT)
24505473044SMike Snitzer #define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT)
246c6b4fcbaSJoe Thornber
2472ee57d58SJoe Thornber enum cache_metadata_mode {
248c6b4fcbaSJoe Thornber CM_WRITE, /* metadata may be changed */
249c6b4fcbaSJoe Thornber CM_READ_ONLY, /* metadata may not be changed */
250028ae9f7SJoe Thornber CM_FAIL
251c6b4fcbaSJoe Thornber };
252c6b4fcbaSJoe Thornber
2532ee57d58SJoe Thornber enum cache_io_mode {
2542ee57d58SJoe Thornber /*
2552ee57d58SJoe Thornber * Data is written to cached blocks only. These blocks are marked
2562ee57d58SJoe Thornber * dirty. If you lose the cache device you will lose data.
2572ee57d58SJoe Thornber * Potential performance increase for both reads and writes.
2582ee57d58SJoe Thornber */
2592ee57d58SJoe Thornber CM_IO_WRITEBACK,
2602ee57d58SJoe Thornber
2612ee57d58SJoe Thornber /*
2622ee57d58SJoe Thornber * Data is written to both cache and origin. Blocks are never
2632ee57d58SJoe Thornber * dirty. Potential performance benfit for reads only.
2642ee57d58SJoe Thornber */
2652ee57d58SJoe Thornber CM_IO_WRITETHROUGH,
2662ee57d58SJoe Thornber
2672ee57d58SJoe Thornber /*
2682ee57d58SJoe Thornber * A degraded mode useful for various cache coherency situations
2692ee57d58SJoe Thornber * (eg, rolling back snapshots). Reads and writes always go to the
2702ee57d58SJoe Thornber * origin. If a write goes to a cached oblock, then the cache
2712ee57d58SJoe Thornber * block is invalidated.
2722ee57d58SJoe Thornber */
2732ee57d58SJoe Thornber CM_IO_PASSTHROUGH
2742ee57d58SJoe Thornber };
2752ee57d58SJoe Thornber
276c6b4fcbaSJoe Thornber struct cache_features {
2772ee57d58SJoe Thornber enum cache_metadata_mode mode;
2782ee57d58SJoe Thornber enum cache_io_mode io_mode;
27986a3238cSHeinz Mauelshagen unsigned int metadata_version;
280de7180ffSMike Snitzer bool discard_passdown:1;
281c6b4fcbaSJoe Thornber };
282c6b4fcbaSJoe Thornber
283c6b4fcbaSJoe Thornber struct cache_stats {
284c6b4fcbaSJoe Thornber atomic_t read_hit;
285c6b4fcbaSJoe Thornber atomic_t read_miss;
286c6b4fcbaSJoe Thornber atomic_t write_hit;
287c6b4fcbaSJoe Thornber atomic_t write_miss;
288c6b4fcbaSJoe Thornber atomic_t demotion;
289c6b4fcbaSJoe Thornber atomic_t promotion;
290b29d4986SJoe Thornber atomic_t writeback;
291c6b4fcbaSJoe Thornber atomic_t copies_avoided;
292c6b4fcbaSJoe Thornber atomic_t cache_cell_clash;
293c6b4fcbaSJoe Thornber atomic_t commit_count;
294c6b4fcbaSJoe Thornber atomic_t discard_count;
295c6b4fcbaSJoe Thornber };
296c6b4fcbaSJoe Thornber
297c6b4fcbaSJoe Thornber struct cache {
298c6b4fcbaSJoe Thornber struct dm_target *ti;
29972d711c8SMike Snitzer spinlock_t lock;
30072d711c8SMike Snitzer
30172d711c8SMike Snitzer /*
30272d711c8SMike Snitzer * Fields for converting from sectors to blocks.
30372d711c8SMike Snitzer */
30472d711c8SMike Snitzer int sectors_per_block_shift;
30572d711c8SMike Snitzer sector_t sectors_per_block;
306c6b4fcbaSJoe Thornber
307c9ec5d7cSMike Snitzer struct dm_cache_metadata *cmd;
308c9ec5d7cSMike Snitzer
309c6b4fcbaSJoe Thornber /*
310c6b4fcbaSJoe Thornber * Metadata is written to this device.
311c6b4fcbaSJoe Thornber */
312c6b4fcbaSJoe Thornber struct dm_dev *metadata_dev;
313c6b4fcbaSJoe Thornber
314c6b4fcbaSJoe Thornber /*
315c6b4fcbaSJoe Thornber * The slower of the two data devices. Typically a spindle.
316c6b4fcbaSJoe Thornber */
317c6b4fcbaSJoe Thornber struct dm_dev *origin_dev;
318c6b4fcbaSJoe Thornber
319c6b4fcbaSJoe Thornber /*
320c6b4fcbaSJoe Thornber * The faster of the two data devices. Typically an SSD.
321c6b4fcbaSJoe Thornber */
322c6b4fcbaSJoe Thornber struct dm_dev *cache_dev;
323c6b4fcbaSJoe Thornber
324c6b4fcbaSJoe Thornber /*
325c6b4fcbaSJoe Thornber * Size of the origin device in _complete_ blocks and native sectors.
326c6b4fcbaSJoe Thornber */
327c6b4fcbaSJoe Thornber dm_oblock_t origin_blocks;
328c6b4fcbaSJoe Thornber sector_t origin_sectors;
329c6b4fcbaSJoe Thornber
330c6b4fcbaSJoe Thornber /*
331c6b4fcbaSJoe Thornber * Size of the cache device in blocks.
332c6b4fcbaSJoe Thornber */
333c6b4fcbaSJoe Thornber dm_cblock_t cache_size;
334c6b4fcbaSJoe Thornber
335c6b4fcbaSJoe Thornber /*
33672d711c8SMike Snitzer * Invalidation fields.
337c6b4fcbaSJoe Thornber */
33872d711c8SMike Snitzer spinlock_t invalidation_lock;
33972d711c8SMike Snitzer struct list_head invalidation_requests;
340c6b4fcbaSJoe Thornber
341c6b4fcbaSJoe Thornber sector_t migration_threshold;
342c6b4fcbaSJoe Thornber wait_queue_head_t migration_wait;
343a59db676SJoe Thornber atomic_t nr_allocated_migrations;
344a59db676SJoe Thornber
345a59db676SJoe Thornber /*
346a59db676SJoe Thornber * The number of in flight migrations that are performing
347a59db676SJoe Thornber * background io. eg, promotion, writeback.
348a59db676SJoe Thornber */
349a59db676SJoe Thornber atomic_t nr_io_migrations;
350c6b4fcbaSJoe Thornber
35172d711c8SMike Snitzer struct bio_list deferred_bios;
35272d711c8SMike Snitzer
353b29d4986SJoe Thornber struct rw_semaphore quiesce_lock;
35466cb1910SJoe Thornber
355c6b4fcbaSJoe Thornber /*
356c6b4fcbaSJoe Thornber * origin_blocks entries, discarded if set.
357c6b4fcbaSJoe Thornber */
3581bad9bc4SJoe Thornber dm_dblock_t discard_nr_blocks;
359c6b4fcbaSJoe Thornber unsigned long *discard_bitset;
36008b18451SJoe Thornber uint32_t discard_block_size; /* a power of 2 times sectors per block */
361c9ec5d7cSMike Snitzer
362c9ec5d7cSMike Snitzer /*
363c9ec5d7cSMike Snitzer * Rather than reconstructing the table line for the status we just
364c9ec5d7cSMike Snitzer * save it and regurgitate.
365c9ec5d7cSMike Snitzer */
36686a3238cSHeinz Mauelshagen unsigned int nr_ctr_args;
367c9ec5d7cSMike Snitzer const char **ctr_args;
368c6b4fcbaSJoe Thornber
369c6b4fcbaSJoe Thornber struct dm_kcopyd_client *copier;
370b29d4986SJoe Thornber struct work_struct deferred_bio_worker;
371b29d4986SJoe Thornber struct work_struct migration_worker;
37272d711c8SMike Snitzer struct workqueue_struct *wq;
373c6b4fcbaSJoe Thornber struct delayed_work waker;
374b29d4986SJoe Thornber struct dm_bio_prison_v2 *prison;
375c6b4fcbaSJoe Thornber
37672d711c8SMike Snitzer /*
37772d711c8SMike Snitzer * cache_size entries, dirty if set
37872d711c8SMike Snitzer */
37972d711c8SMike Snitzer unsigned long *dirty_bitset;
38072d711c8SMike Snitzer atomic_t nr_dirty;
381c6b4fcbaSJoe Thornber
38286a3238cSHeinz Mauelshagen unsigned int policy_nr_args;
38372d711c8SMike Snitzer struct dm_cache_policy *policy;
384c6b4fcbaSJoe Thornber
385c6b4fcbaSJoe Thornber /*
386c9ec5d7cSMike Snitzer * Cache features such as write-through.
387c6b4fcbaSJoe Thornber */
388c9ec5d7cSMike Snitzer struct cache_features features;
389c9ec5d7cSMike Snitzer
390c9ec5d7cSMike Snitzer struct cache_stats stats;
39165790ff9SJoe Thornber
39272d711c8SMike Snitzer bool need_tick_bio:1;
39372d711c8SMike Snitzer bool sized:1;
39472d711c8SMike Snitzer bool invalidate:1;
39572d711c8SMike Snitzer bool commit_requested:1;
39672d711c8SMike Snitzer bool loaded_mappings:1;
39772d711c8SMike Snitzer bool loaded_discards:1;
39872d711c8SMike Snitzer
39972d711c8SMike Snitzer struct rw_semaphore background_work_lock;
40072d711c8SMike Snitzer
40172d711c8SMike Snitzer struct batcher committer;
40272d711c8SMike Snitzer struct work_struct commit_ws;
403066dbaa3SJoe Thornber
404dc4fa29fSMike Snitzer struct dm_io_tracker tracker;
405b29d4986SJoe Thornber
40672d711c8SMike Snitzer mempool_t migration_pool;
407b29d4986SJoe Thornber
40872d711c8SMike Snitzer struct bio_set bs;
409c6b4fcbaSJoe Thornber };
410c6b4fcbaSJoe Thornber
411c6b4fcbaSJoe Thornber struct per_bio_data {
412c6b4fcbaSJoe Thornber bool tick:1;
41386a3238cSHeinz Mauelshagen unsigned int req_nr:2;
414b29d4986SJoe Thornber struct dm_bio_prison_cell_v2 *cell;
415c6eda5e8SMike Snitzer struct dm_hook_info hook_info;
416066dbaa3SJoe Thornber sector_t len;
417c6b4fcbaSJoe Thornber };
418c6b4fcbaSJoe Thornber
419c6b4fcbaSJoe Thornber struct dm_cache_migration {
420b29d4986SJoe Thornber struct continuation k;
421c6b4fcbaSJoe Thornber struct cache *cache;
422c6b4fcbaSJoe Thornber
423b29d4986SJoe Thornber struct policy_work *op;
424b29d4986SJoe Thornber struct bio *overwrite_bio;
425b29d4986SJoe Thornber struct dm_bio_prison_cell_v2 *cell;
426c6b4fcbaSJoe Thornber
427b29d4986SJoe Thornber dm_cblock_t invalidate_cblock;
428b29d4986SJoe Thornber dm_oblock_t invalidate_oblock;
429c6b4fcbaSJoe Thornber };
430c6b4fcbaSJoe Thornber
431b29d4986SJoe Thornber /*----------------------------------------------------------------*/
432c6b4fcbaSJoe Thornber
writethrough_mode(struct cache * cache)4338e3c3827SMike Snitzer static bool writethrough_mode(struct cache *cache)
434c6b4fcbaSJoe Thornber {
4358e3c3827SMike Snitzer return cache->features.io_mode == CM_IO_WRITETHROUGH;
436b29d4986SJoe Thornber }
437b29d4986SJoe Thornber
writeback_mode(struct cache * cache)4388e3c3827SMike Snitzer static bool writeback_mode(struct cache *cache)
439b29d4986SJoe Thornber {
4408e3c3827SMike Snitzer return cache->features.io_mode == CM_IO_WRITEBACK;
441b29d4986SJoe Thornber }
442b29d4986SJoe Thornber
passthrough_mode(struct cache * cache)4438e3c3827SMike Snitzer static inline bool passthrough_mode(struct cache *cache)
444b29d4986SJoe Thornber {
4458e3c3827SMike Snitzer return unlikely(cache->features.io_mode == CM_IO_PASSTHROUGH);
446c6b4fcbaSJoe Thornber }
447c6b4fcbaSJoe Thornber
448c6b4fcbaSJoe Thornber /*----------------------------------------------------------------*/
449c6b4fcbaSJoe Thornber
wake_deferred_bio_worker(struct cache * cache)450b29d4986SJoe Thornber static void wake_deferred_bio_worker(struct cache *cache)
451c6b4fcbaSJoe Thornber {
452b29d4986SJoe Thornber queue_work(cache->wq, &cache->deferred_bio_worker);
453c6b4fcbaSJoe Thornber }
454c6b4fcbaSJoe Thornber
wake_migration_worker(struct cache * cache)455b29d4986SJoe Thornber static void wake_migration_worker(struct cache *cache)
456b29d4986SJoe Thornber {
4578e3c3827SMike Snitzer if (passthrough_mode(cache))
458b29d4986SJoe Thornber return;
459b29d4986SJoe Thornber
460b29d4986SJoe Thornber queue_work(cache->wq, &cache->migration_worker);
461b29d4986SJoe Thornber }
462b29d4986SJoe Thornber
463b29d4986SJoe Thornber /*----------------------------------------------------------------*/
464b29d4986SJoe Thornber
alloc_prison_cell(struct cache * cache)465b29d4986SJoe Thornber static struct dm_bio_prison_cell_v2 *alloc_prison_cell(struct cache *cache)
466b29d4986SJoe Thornber {
46713bd677aSMikulas Patocka return dm_bio_prison_alloc_cell_v2(cache->prison, GFP_NOIO);
468b29d4986SJoe Thornber }
469b29d4986SJoe Thornber
free_prison_cell(struct cache * cache,struct dm_bio_prison_cell_v2 * cell)470b29d4986SJoe Thornber static void free_prison_cell(struct cache *cache, struct dm_bio_prison_cell_v2 *cell)
471b29d4986SJoe Thornber {
472b29d4986SJoe Thornber dm_bio_prison_free_cell_v2(cache->prison, cell);
473c6b4fcbaSJoe Thornber }
474c6b4fcbaSJoe Thornber
alloc_migration(struct cache * cache)475a59db676SJoe Thornber static struct dm_cache_migration *alloc_migration(struct cache *cache)
476a59db676SJoe Thornber {
477a59db676SJoe Thornber struct dm_cache_migration *mg;
478a59db676SJoe Thornber
47913bd677aSMikulas Patocka mg = mempool_alloc(&cache->migration_pool, GFP_NOIO);
480ef7afb36SMike Snitzer
481ef7afb36SMike Snitzer memset(mg, 0, sizeof(*mg));
482ef7afb36SMike Snitzer
483a59db676SJoe Thornber mg->cache = cache;
484ef7afb36SMike Snitzer atomic_inc(&cache->nr_allocated_migrations);
485a59db676SJoe Thornber
486a59db676SJoe Thornber return mg;
487a59db676SJoe Thornber }
488a59db676SJoe Thornber
free_migration(struct dm_cache_migration * mg)489a59db676SJoe Thornber static void free_migration(struct dm_cache_migration *mg)
490a59db676SJoe Thornber {
49188bf5184SJoe Thornber struct cache *cache = mg->cache;
492a59db676SJoe Thornber
49388bf5184SJoe Thornber if (atomic_dec_and_test(&cache->nr_allocated_migrations))
49488bf5184SJoe Thornber wake_up(&cache->migration_wait);
49588bf5184SJoe Thornber
4966f1c819cSKent Overstreet mempool_free(mg, &cache->migration_pool);
497a59db676SJoe Thornber }
498a59db676SJoe Thornber
499c6b4fcbaSJoe Thornber /*----------------------------------------------------------------*/
500c6b4fcbaSJoe Thornber
oblock_succ(dm_oblock_t b)501b29d4986SJoe Thornber static inline dm_oblock_t oblock_succ(dm_oblock_t b)
502b29d4986SJoe Thornber {
503b29d4986SJoe Thornber return to_oblock(from_oblock(b) + 1ull);
504b29d4986SJoe Thornber }
505b29d4986SJoe Thornber
build_key(dm_oblock_t begin,dm_oblock_t end,struct dm_cell_key_v2 * key)506b29d4986SJoe Thornber static void build_key(dm_oblock_t begin, dm_oblock_t end, struct dm_cell_key_v2 *key)
507c6b4fcbaSJoe Thornber {
508c6b4fcbaSJoe Thornber key->virtual = 0;
509c6b4fcbaSJoe Thornber key->dev = 0;
5107ae34e77SJoe Thornber key->block_begin = from_oblock(begin);
5117ae34e77SJoe Thornber key->block_end = from_oblock(end);
512c6b4fcbaSJoe Thornber }
513c6b4fcbaSJoe Thornber
514c6b4fcbaSJoe Thornber /*
515b29d4986SJoe Thornber * We have two lock levels. Level 0, which is used to prevent WRITEs, and
516b29d4986SJoe Thornber * level 1 which prevents *both* READs and WRITEs.
517c6b4fcbaSJoe Thornber */
518b29d4986SJoe Thornber #define WRITE_LOCK_LEVEL 0
519b29d4986SJoe Thornber #define READ_WRITE_LOCK_LEVEL 1
520c6b4fcbaSJoe Thornber
lock_level(struct bio * bio)52186a3238cSHeinz Mauelshagen static unsigned int lock_level(struct bio *bio)
522c6b4fcbaSJoe Thornber {
523b29d4986SJoe Thornber return bio_data_dir(bio) == WRITE ?
524b29d4986SJoe Thornber WRITE_LOCK_LEVEL :
525b29d4986SJoe Thornber READ_WRITE_LOCK_LEVEL;
526b29d4986SJoe Thornber }
527c6b4fcbaSJoe Thornber
528a4a82ce3SHeinz Mauelshagen /*
529a4a82ce3SHeinz Mauelshagen *--------------------------------------------------------------
530b29d4986SJoe Thornber * Per bio data
531a4a82ce3SHeinz Mauelshagen *--------------------------------------------------------------
532a4a82ce3SHeinz Mauelshagen */
533c6b4fcbaSJoe Thornber
get_per_bio_data(struct bio * bio)534693b960eSMike Snitzer static struct per_bio_data *get_per_bio_data(struct bio *bio)
535b29d4986SJoe Thornber {
536693b960eSMike Snitzer struct per_bio_data *pb = dm_per_bio_data(bio, sizeof(struct per_bio_data));
5370ef0b471SHeinz Mauelshagen
538b29d4986SJoe Thornber BUG_ON(!pb);
539b29d4986SJoe Thornber return pb;
540b29d4986SJoe Thornber }
541b29d4986SJoe Thornber
init_per_bio_data(struct bio * bio)542693b960eSMike Snitzer static struct per_bio_data *init_per_bio_data(struct bio *bio)
543b29d4986SJoe Thornber {
544693b960eSMike Snitzer struct per_bio_data *pb = get_per_bio_data(bio);
545b29d4986SJoe Thornber
546b29d4986SJoe Thornber pb->tick = false;
547b29d4986SJoe Thornber pb->req_nr = dm_bio_get_target_bio_nr(bio);
548b29d4986SJoe Thornber pb->cell = NULL;
549b29d4986SJoe Thornber pb->len = 0;
550b29d4986SJoe Thornber
551b29d4986SJoe Thornber return pb;
552b29d4986SJoe Thornber }
553b29d4986SJoe Thornber
554b29d4986SJoe Thornber /*----------------------------------------------------------------*/
555b29d4986SJoe Thornber
defer_bio(struct cache * cache,struct bio * bio)556b29d4986SJoe Thornber static void defer_bio(struct cache *cache, struct bio *bio)
557b29d4986SJoe Thornber {
55826b924b9SMikulas Patocka spin_lock_irq(&cache->lock);
559b29d4986SJoe Thornber bio_list_add(&cache->deferred_bios, bio);
56026b924b9SMikulas Patocka spin_unlock_irq(&cache->lock);
561b29d4986SJoe Thornber
562b29d4986SJoe Thornber wake_deferred_bio_worker(cache);
563b29d4986SJoe Thornber }
564b29d4986SJoe Thornber
defer_bios(struct cache * cache,struct bio_list * bios)565b29d4986SJoe Thornber static void defer_bios(struct cache *cache, struct bio_list *bios)
566b29d4986SJoe Thornber {
56726b924b9SMikulas Patocka spin_lock_irq(&cache->lock);
568b29d4986SJoe Thornber bio_list_merge(&cache->deferred_bios, bios);
569b29d4986SJoe Thornber bio_list_init(bios);
57026b924b9SMikulas Patocka spin_unlock_irq(&cache->lock);
571b29d4986SJoe Thornber
572b29d4986SJoe Thornber wake_deferred_bio_worker(cache);
573b29d4986SJoe Thornber }
574b29d4986SJoe Thornber
575b29d4986SJoe Thornber /*----------------------------------------------------------------*/
576b29d4986SJoe Thornber
bio_detain_shared(struct cache * cache,dm_oblock_t oblock,struct bio * bio)577b29d4986SJoe Thornber static bool bio_detain_shared(struct cache *cache, dm_oblock_t oblock, struct bio *bio)
578b29d4986SJoe Thornber {
579b29d4986SJoe Thornber bool r;
580b29d4986SJoe Thornber struct per_bio_data *pb;
581b29d4986SJoe Thornber struct dm_cell_key_v2 key;
582b29d4986SJoe Thornber dm_oblock_t end = to_oblock(from_oblock(oblock) + 1ULL);
583b29d4986SJoe Thornber struct dm_bio_prison_cell_v2 *cell_prealloc, *cell;
584b29d4986SJoe Thornber
585b29d4986SJoe Thornber cell_prealloc = alloc_prison_cell(cache); /* FIXME: allow wait if calling from worker */
586b29d4986SJoe Thornber
587b29d4986SJoe Thornber build_key(oblock, end, &key);
588b29d4986SJoe Thornber r = dm_cell_get_v2(cache->prison, &key, lock_level(bio), bio, cell_prealloc, &cell);
589b29d4986SJoe Thornber if (!r) {
590b29d4986SJoe Thornber /*
591b29d4986SJoe Thornber * Failed to get the lock.
592b29d4986SJoe Thornber */
593b29d4986SJoe Thornber free_prison_cell(cache, cell_prealloc);
594c6b4fcbaSJoe Thornber return r;
595c6b4fcbaSJoe Thornber }
596c6b4fcbaSJoe Thornber
597b29d4986SJoe Thornber if (cell != cell_prealloc)
598b29d4986SJoe Thornber free_prison_cell(cache, cell_prealloc);
5997ae34e77SJoe Thornber
600693b960eSMike Snitzer pb = get_per_bio_data(bio);
601b29d4986SJoe Thornber pb->cell = cell;
602c6b4fcbaSJoe Thornber
603c6b4fcbaSJoe Thornber return r;
604c6b4fcbaSJoe Thornber }
605c6b4fcbaSJoe Thornber
606c6b4fcbaSJoe Thornber /*----------------------------------------------------------------*/
607c6b4fcbaSJoe Thornber
is_dirty(struct cache * cache,dm_cblock_t b)608c6b4fcbaSJoe Thornber static bool is_dirty(struct cache *cache, dm_cblock_t b)
609c6b4fcbaSJoe Thornber {
610c6b4fcbaSJoe Thornber return test_bit(from_cblock(b), cache->dirty_bitset);
611c6b4fcbaSJoe Thornber }
612c6b4fcbaSJoe Thornber
set_dirty(struct cache * cache,dm_cblock_t cblock)613b29d4986SJoe Thornber static void set_dirty(struct cache *cache, dm_cblock_t cblock)
614c6b4fcbaSJoe Thornber {
615c6b4fcbaSJoe Thornber if (!test_and_set_bit(from_cblock(cblock), cache->dirty_bitset)) {
61644fa816bSAnssi Hannula atomic_inc(&cache->nr_dirty);
617b29d4986SJoe Thornber policy_set_dirty(cache->policy, cblock);
618c6b4fcbaSJoe Thornber }
619c6b4fcbaSJoe Thornber }
620c6b4fcbaSJoe Thornber
621b29d4986SJoe Thornber /*
622b29d4986SJoe Thornber * These two are called when setting after migrations to force the policy
623b29d4986SJoe Thornber * and dirty bitset to be in sync.
624b29d4986SJoe Thornber */
force_set_dirty(struct cache * cache,dm_cblock_t cblock)625b29d4986SJoe Thornber static void force_set_dirty(struct cache *cache, dm_cblock_t cblock)
626b29d4986SJoe Thornber {
627b29d4986SJoe Thornber if (!test_and_set_bit(from_cblock(cblock), cache->dirty_bitset))
628b29d4986SJoe Thornber atomic_inc(&cache->nr_dirty);
629b29d4986SJoe Thornber policy_set_dirty(cache->policy, cblock);
630b29d4986SJoe Thornber }
631b29d4986SJoe Thornber
force_clear_dirty(struct cache * cache,dm_cblock_t cblock)632b29d4986SJoe Thornber static void force_clear_dirty(struct cache *cache, dm_cblock_t cblock)
633c6b4fcbaSJoe Thornber {
634c6b4fcbaSJoe Thornber if (test_and_clear_bit(from_cblock(cblock), cache->dirty_bitset)) {
63544fa816bSAnssi Hannula if (atomic_dec_return(&cache->nr_dirty) == 0)
636c6b4fcbaSJoe Thornber dm_table_event(cache->ti->table);
637c6b4fcbaSJoe Thornber }
638b29d4986SJoe Thornber
639b29d4986SJoe Thornber policy_clear_dirty(cache->policy, cblock);
640c6b4fcbaSJoe Thornber }
641c6b4fcbaSJoe Thornber
642c6b4fcbaSJoe Thornber /*----------------------------------------------------------------*/
643aeed1420SJoe Thornber
block_size_is_power_of_two(struct cache * cache)644c6b4fcbaSJoe Thornber static bool block_size_is_power_of_two(struct cache *cache)
645c6b4fcbaSJoe Thornber {
646c6b4fcbaSJoe Thornber return cache->sectors_per_block_shift >= 0;
647c6b4fcbaSJoe Thornber }
648c6b4fcbaSJoe Thornber
block_div(dm_block_t b,uint32_t n)649414dd67dSJoe Thornber static dm_block_t block_div(dm_block_t b, uint32_t n)
650414dd67dSJoe Thornber {
651414dd67dSJoe Thornber do_div(b, n);
652414dd67dSJoe Thornber
653414dd67dSJoe Thornber return b;
654414dd67dSJoe Thornber }
655414dd67dSJoe Thornber
oblocks_per_dblock(struct cache * cache)6567ae34e77SJoe Thornber static dm_block_t oblocks_per_dblock(struct cache *cache)
6577ae34e77SJoe Thornber {
6587ae34e77SJoe Thornber dm_block_t oblocks = cache->discard_block_size;
6597ae34e77SJoe Thornber
6607ae34e77SJoe Thornber if (block_size_is_power_of_two(cache))
6617ae34e77SJoe Thornber oblocks >>= cache->sectors_per_block_shift;
6627ae34e77SJoe Thornber else
6637ae34e77SJoe Thornber oblocks = block_div(oblocks, cache->sectors_per_block);
6647ae34e77SJoe Thornber
6657ae34e77SJoe Thornber return oblocks;
6667ae34e77SJoe Thornber }
6677ae34e77SJoe Thornber
oblock_to_dblock(struct cache * cache,dm_oblock_t oblock)6681bad9bc4SJoe Thornber static dm_dblock_t oblock_to_dblock(struct cache *cache, dm_oblock_t oblock)
6691bad9bc4SJoe Thornber {
6707ae34e77SJoe Thornber return to_dblock(block_div(from_oblock(oblock),
6717ae34e77SJoe Thornber oblocks_per_dblock(cache)));
6727ae34e77SJoe Thornber }
6731bad9bc4SJoe Thornber
set_discard(struct cache * cache,dm_dblock_t b)6741bad9bc4SJoe Thornber static void set_discard(struct cache *cache, dm_dblock_t b)
675c6b4fcbaSJoe Thornber {
6767ae34e77SJoe Thornber BUG_ON(from_dblock(b) >= from_dblock(cache->discard_nr_blocks));
677c6b4fcbaSJoe Thornber atomic_inc(&cache->stats.discard_count);
678c6b4fcbaSJoe Thornber
67926b924b9SMikulas Patocka spin_lock_irq(&cache->lock);
6801bad9bc4SJoe Thornber set_bit(from_dblock(b), cache->discard_bitset);
68126b924b9SMikulas Patocka spin_unlock_irq(&cache->lock);
682c6b4fcbaSJoe Thornber }
683c6b4fcbaSJoe Thornber
clear_discard(struct cache * cache,dm_dblock_t b)6841bad9bc4SJoe Thornber static void clear_discard(struct cache *cache, dm_dblock_t b)
685c6b4fcbaSJoe Thornber {
68626b924b9SMikulas Patocka spin_lock_irq(&cache->lock);
6871bad9bc4SJoe Thornber clear_bit(from_dblock(b), cache->discard_bitset);
68826b924b9SMikulas Patocka spin_unlock_irq(&cache->lock);
689c6b4fcbaSJoe Thornber }
690c6b4fcbaSJoe Thornber
is_discarded(struct cache * cache,dm_dblock_t b)6911bad9bc4SJoe Thornber static bool is_discarded(struct cache *cache, dm_dblock_t b)
692c6b4fcbaSJoe Thornber {
693c6b4fcbaSJoe Thornber int r;
6940ef0b471SHeinz Mauelshagen
69526b924b9SMikulas Patocka spin_lock_irq(&cache->lock);
6961bad9bc4SJoe Thornber r = test_bit(from_dblock(b), cache->discard_bitset);
69726b924b9SMikulas Patocka spin_unlock_irq(&cache->lock);
698c6b4fcbaSJoe Thornber
699c6b4fcbaSJoe Thornber return r;
700c6b4fcbaSJoe Thornber }
701c6b4fcbaSJoe Thornber
is_discarded_oblock(struct cache * cache,dm_oblock_t b)702c6b4fcbaSJoe Thornber static bool is_discarded_oblock(struct cache *cache, dm_oblock_t b)
703c6b4fcbaSJoe Thornber {
704c6b4fcbaSJoe Thornber int r;
7050ef0b471SHeinz Mauelshagen
70626b924b9SMikulas Patocka spin_lock_irq(&cache->lock);
7071bad9bc4SJoe Thornber r = test_bit(from_dblock(oblock_to_dblock(cache, b)),
7081bad9bc4SJoe Thornber cache->discard_bitset);
70926b924b9SMikulas Patocka spin_unlock_irq(&cache->lock);
710c6b4fcbaSJoe Thornber
711c6b4fcbaSJoe Thornber return r;
712c6b4fcbaSJoe Thornber }
713c6b4fcbaSJoe Thornber
714a4a82ce3SHeinz Mauelshagen /*
715a4a82ce3SHeinz Mauelshagen * -------------------------------------------------------------
716c6b4fcbaSJoe Thornber * Remapping
717a4a82ce3SHeinz Mauelshagen *--------------------------------------------------------------
718a4a82ce3SHeinz Mauelshagen */
remap_to_origin(struct cache * cache,struct bio * bio)719c6b4fcbaSJoe Thornber static void remap_to_origin(struct cache *cache, struct bio *bio)
720c6b4fcbaSJoe Thornber {
72174d46992SChristoph Hellwig bio_set_dev(bio, cache->origin_dev->bdev);
722c6b4fcbaSJoe Thornber }
723c6b4fcbaSJoe Thornber
remap_to_cache(struct cache * cache,struct bio * bio,dm_cblock_t cblock)724c6b4fcbaSJoe Thornber static void remap_to_cache(struct cache *cache, struct bio *bio,
725c6b4fcbaSJoe Thornber dm_cblock_t cblock)
726c6b4fcbaSJoe Thornber {
7274f024f37SKent Overstreet sector_t bi_sector = bio->bi_iter.bi_sector;
728e0d849faSHeinz Mauelshagen sector_t block = from_cblock(cblock);
729c6b4fcbaSJoe Thornber
73074d46992SChristoph Hellwig bio_set_dev(bio, cache->cache_dev->bdev);
731c6b4fcbaSJoe Thornber if (!block_size_is_power_of_two(cache))
7324f024f37SKent Overstreet bio->bi_iter.bi_sector =
733e0d849faSHeinz Mauelshagen (block * cache->sectors_per_block) +
734c6b4fcbaSJoe Thornber sector_div(bi_sector, cache->sectors_per_block);
735c6b4fcbaSJoe Thornber else
7364f024f37SKent Overstreet bio->bi_iter.bi_sector =
737e0d849faSHeinz Mauelshagen (block << cache->sectors_per_block_shift) |
738c6b4fcbaSJoe Thornber (bi_sector & (cache->sectors_per_block - 1));
739c6b4fcbaSJoe Thornber }
740c6b4fcbaSJoe Thornber
check_if_tick_bio_needed(struct cache * cache,struct bio * bio)741c6b4fcbaSJoe Thornber static void check_if_tick_bio_needed(struct cache *cache, struct bio *bio)
742c6b4fcbaSJoe Thornber {
743693b960eSMike Snitzer struct per_bio_data *pb;
744c6b4fcbaSJoe Thornber
74526b924b9SMikulas Patocka spin_lock_irq(&cache->lock);
746f73f44ebSChristoph Hellwig if (cache->need_tick_bio && !op_is_flush(bio->bi_opf) &&
747e6047149SMike Christie bio_op(bio) != REQ_OP_DISCARD) {
748693b960eSMike Snitzer pb = get_per_bio_data(bio);
749c6b4fcbaSJoe Thornber pb->tick = true;
750c6b4fcbaSJoe Thornber cache->need_tick_bio = false;
751c6b4fcbaSJoe Thornber }
75226b924b9SMikulas Patocka spin_unlock_irq(&cache->lock);
753c6b4fcbaSJoe Thornber }
754c6b4fcbaSJoe Thornber
remap_to_origin_clear_discard(struct cache * cache,struct bio * bio,dm_oblock_t oblock)7552df3bae9SMike Snitzer static void remap_to_origin_clear_discard(struct cache *cache, struct bio *bio,
7562df3bae9SMike Snitzer dm_oblock_t oblock)
7572df3bae9SMike Snitzer {
7582df3bae9SMike Snitzer // FIXME: check_if_tick_bio_needed() is called way too much through this interface
7593c4b455eSChristoph Hellwig check_if_tick_bio_needed(cache, bio);
7603c4b455eSChristoph Hellwig remap_to_origin(cache, bio);
7613c4b455eSChristoph Hellwig if (bio_data_dir(bio) == WRITE)
7623c4b455eSChristoph Hellwig clear_discard(cache, oblock_to_dblock(cache, oblock));
7632df3bae9SMike Snitzer }
7642df3bae9SMike Snitzer
remap_to_cache_dirty(struct cache * cache,struct bio * bio,dm_oblock_t oblock,dm_cblock_t cblock)765c6b4fcbaSJoe Thornber static void remap_to_cache_dirty(struct cache *cache, struct bio *bio,
766c6b4fcbaSJoe Thornber dm_oblock_t oblock, dm_cblock_t cblock)
767c6b4fcbaSJoe Thornber {
768f8e5f01aSJoe Thornber check_if_tick_bio_needed(cache, bio);
769c6b4fcbaSJoe Thornber remap_to_cache(cache, bio, cblock);
770c6b4fcbaSJoe Thornber if (bio_data_dir(bio) == WRITE) {
771b29d4986SJoe Thornber set_dirty(cache, cblock);
7721bad9bc4SJoe Thornber clear_discard(cache, oblock_to_dblock(cache, oblock));
773c6b4fcbaSJoe Thornber }
774c6b4fcbaSJoe Thornber }
775c6b4fcbaSJoe Thornber
get_bio_block(struct cache * cache,struct bio * bio)776c6b4fcbaSJoe Thornber static dm_oblock_t get_bio_block(struct cache *cache, struct bio *bio)
777c6b4fcbaSJoe Thornber {
7784f024f37SKent Overstreet sector_t block_nr = bio->bi_iter.bi_sector;
779c6b4fcbaSJoe Thornber
780c6b4fcbaSJoe Thornber if (!block_size_is_power_of_two(cache))
781c6b4fcbaSJoe Thornber (void) sector_div(block_nr, cache->sectors_per_block);
782c6b4fcbaSJoe Thornber else
783c6b4fcbaSJoe Thornber block_nr >>= cache->sectors_per_block_shift;
784c6b4fcbaSJoe Thornber
785c6b4fcbaSJoe Thornber return to_oblock(block_nr);
786c6b4fcbaSJoe Thornber }
787c6b4fcbaSJoe Thornber
accountable_bio(struct cache * cache,struct bio * bio)788066dbaa3SJoe Thornber static bool accountable_bio(struct cache *cache, struct bio *bio)
789066dbaa3SJoe Thornber {
790701e03e4SJoe Thornber return bio_op(bio) != REQ_OP_DISCARD;
791066dbaa3SJoe Thornber }
792066dbaa3SJoe Thornber
accounted_begin(struct cache * cache,struct bio * bio)793066dbaa3SJoe Thornber static void accounted_begin(struct cache *cache, struct bio *bio)
794066dbaa3SJoe Thornber {
795693b960eSMike Snitzer struct per_bio_data *pb;
796066dbaa3SJoe Thornber
797066dbaa3SJoe Thornber if (accountable_bio(cache, bio)) {
798693b960eSMike Snitzer pb = get_per_bio_data(bio);
799066dbaa3SJoe Thornber pb->len = bio_sectors(bio);
800dc4fa29fSMike Snitzer dm_iot_io_begin(&cache->tracker, pb->len);
801066dbaa3SJoe Thornber }
802066dbaa3SJoe Thornber }
803066dbaa3SJoe Thornber
accounted_complete(struct cache * cache,struct bio * bio)804066dbaa3SJoe Thornber static void accounted_complete(struct cache *cache, struct bio *bio)
805066dbaa3SJoe Thornber {
806693b960eSMike Snitzer struct per_bio_data *pb = get_per_bio_data(bio);
807066dbaa3SJoe Thornber
808dc4fa29fSMike Snitzer dm_iot_io_end(&cache->tracker, pb->len);
809066dbaa3SJoe Thornber }
810066dbaa3SJoe Thornber
accounted_request(struct cache * cache,struct bio * bio)811066dbaa3SJoe Thornber static void accounted_request(struct cache *cache, struct bio *bio)
812066dbaa3SJoe Thornber {
813066dbaa3SJoe Thornber accounted_begin(cache, bio);
81469596f55SMike Snitzer dm_submit_bio_remap(bio, NULL);
815066dbaa3SJoe Thornber }
816066dbaa3SJoe Thornber
issue_op(struct bio * bio,void * context)817b29d4986SJoe Thornber static void issue_op(struct bio *bio, void *context)
818c6b4fcbaSJoe Thornber {
819b29d4986SJoe Thornber struct cache *cache = context;
8200ef0b471SHeinz Mauelshagen
821066dbaa3SJoe Thornber accounted_request(cache, bio);
8228c081b52SJoe Thornber }
8238c081b52SJoe Thornber
824e2e74d61SJoe Thornber /*
825e2e74d61SJoe Thornber * When running in writethrough mode we need to send writes to clean blocks
8262df3bae9SMike Snitzer * to both the cache and origin devices. Clone the bio and send them in parallel.
827e2e74d61SJoe Thornber */
remap_to_origin_and_cache(struct cache * cache,struct bio * bio,dm_oblock_t oblock,dm_cblock_t cblock)8282df3bae9SMike Snitzer static void remap_to_origin_and_cache(struct cache *cache, struct bio *bio,
829e2e74d61SJoe Thornber dm_oblock_t oblock, dm_cblock_t cblock)
830e2e74d61SJoe Thornber {
831abfc426dSChristoph Hellwig struct bio *origin_bio = bio_alloc_clone(cache->origin_dev->bdev, bio,
832abfc426dSChristoph Hellwig GFP_NOIO, &cache->bs);
833e2e74d61SJoe Thornber
8342df3bae9SMike Snitzer BUG_ON(!origin_bio);
835e2e74d61SJoe Thornber
8362df3bae9SMike Snitzer bio_chain(origin_bio, bio);
8373c4b455eSChristoph Hellwig
8383c4b455eSChristoph Hellwig if (bio_data_dir(origin_bio) == WRITE)
8393c4b455eSChristoph Hellwig clear_discard(cache, oblock_to_dblock(cache, oblock));
8402df3bae9SMike Snitzer submit_bio(origin_bio);
8412df3bae9SMike Snitzer
8422df3bae9SMike Snitzer remap_to_cache(cache, bio, cblock);
843e2e74d61SJoe Thornber }
844e2e74d61SJoe Thornber
845a4a82ce3SHeinz Mauelshagen /*
846a4a82ce3SHeinz Mauelshagen *--------------------------------------------------------------
847028ae9f7SJoe Thornber * Failure modes
848a4a82ce3SHeinz Mauelshagen *--------------------------------------------------------------
849a4a82ce3SHeinz Mauelshagen */
get_cache_mode(struct cache * cache)850028ae9f7SJoe Thornber static enum cache_metadata_mode get_cache_mode(struct cache *cache)
851028ae9f7SJoe Thornber {
852028ae9f7SJoe Thornber return cache->features.mode;
853028ae9f7SJoe Thornber }
854028ae9f7SJoe Thornber
cache_device_name(struct cache * cache)855b61d9509SMike Snitzer static const char *cache_device_name(struct cache *cache)
856b61d9509SMike Snitzer {
857d4a512edSMike Snitzer return dm_table_device_name(cache->ti->table);
858b61d9509SMike Snitzer }
859b61d9509SMike Snitzer
notify_mode_switch(struct cache * cache,enum cache_metadata_mode mode)860028ae9f7SJoe Thornber static void notify_mode_switch(struct cache *cache, enum cache_metadata_mode mode)
861028ae9f7SJoe Thornber {
862774f13acSHeinz Mauelshagen static const char *descs[] = {
863028ae9f7SJoe Thornber "write",
864028ae9f7SJoe Thornber "read-only",
865028ae9f7SJoe Thornber "fail"
866028ae9f7SJoe Thornber };
867028ae9f7SJoe Thornber
868028ae9f7SJoe Thornber dm_table_event(cache->ti->table);
869b61d9509SMike Snitzer DMINFO("%s: switching cache to %s mode",
870b61d9509SMike Snitzer cache_device_name(cache), descs[(int)mode]);
871028ae9f7SJoe Thornber }
872028ae9f7SJoe Thornber
set_cache_mode(struct cache * cache,enum cache_metadata_mode new_mode)873028ae9f7SJoe Thornber static void set_cache_mode(struct cache *cache, enum cache_metadata_mode new_mode)
874028ae9f7SJoe Thornber {
875d14fcf3dSJoe Thornber bool needs_check;
876028ae9f7SJoe Thornber enum cache_metadata_mode old_mode = get_cache_mode(cache);
877028ae9f7SJoe Thornber
878d14fcf3dSJoe Thornber if (dm_cache_metadata_needs_check(cache->cmd, &needs_check)) {
87923cab26dSMike Snitzer DMERR("%s: unable to read needs_check flag, setting failure mode.",
88023cab26dSMike Snitzer cache_device_name(cache));
881d14fcf3dSJoe Thornber new_mode = CM_FAIL;
882d14fcf3dSJoe Thornber }
883d14fcf3dSJoe Thornber
884028ae9f7SJoe Thornber if (new_mode == CM_WRITE && needs_check) {
885b61d9509SMike Snitzer DMERR("%s: unable to switch cache to write mode until repaired.",
886b61d9509SMike Snitzer cache_device_name(cache));
887028ae9f7SJoe Thornber if (old_mode != new_mode)
888028ae9f7SJoe Thornber new_mode = old_mode;
889028ae9f7SJoe Thornber else
890028ae9f7SJoe Thornber new_mode = CM_READ_ONLY;
891028ae9f7SJoe Thornber }
892028ae9f7SJoe Thornber
893028ae9f7SJoe Thornber /* Never move out of fail mode */
894028ae9f7SJoe Thornber if (old_mode == CM_FAIL)
895028ae9f7SJoe Thornber new_mode = CM_FAIL;
896028ae9f7SJoe Thornber
897028ae9f7SJoe Thornber switch (new_mode) {
898028ae9f7SJoe Thornber case CM_FAIL:
899028ae9f7SJoe Thornber case CM_READ_ONLY:
900028ae9f7SJoe Thornber dm_cache_metadata_set_read_only(cache->cmd);
901028ae9f7SJoe Thornber break;
902028ae9f7SJoe Thornber
903028ae9f7SJoe Thornber case CM_WRITE:
904028ae9f7SJoe Thornber dm_cache_metadata_set_read_write(cache->cmd);
905028ae9f7SJoe Thornber break;
906028ae9f7SJoe Thornber }
907028ae9f7SJoe Thornber
908028ae9f7SJoe Thornber cache->features.mode = new_mode;
909028ae9f7SJoe Thornber
910028ae9f7SJoe Thornber if (new_mode != old_mode)
911028ae9f7SJoe Thornber notify_mode_switch(cache, new_mode);
912028ae9f7SJoe Thornber }
913028ae9f7SJoe Thornber
abort_transaction(struct cache * cache)914028ae9f7SJoe Thornber static void abort_transaction(struct cache *cache)
915028ae9f7SJoe Thornber {
916b61d9509SMike Snitzer const char *dev_name = cache_device_name(cache);
917b61d9509SMike Snitzer
918028ae9f7SJoe Thornber if (get_cache_mode(cache) >= CM_READ_ONLY)
919028ae9f7SJoe Thornber return;
920028ae9f7SJoe Thornber
921b61d9509SMike Snitzer DMERR_LIMIT("%s: aborting current metadata transaction", dev_name);
922028ae9f7SJoe Thornber if (dm_cache_metadata_abort(cache->cmd)) {
923b61d9509SMike Snitzer DMERR("%s: failed to abort metadata transaction", dev_name);
924028ae9f7SJoe Thornber set_cache_mode(cache, CM_FAIL);
925028ae9f7SJoe Thornber }
9266b997386SMike Snitzer
9276b997386SMike Snitzer if (dm_cache_metadata_set_needs_check(cache->cmd)) {
9286b997386SMike Snitzer DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name);
9296b997386SMike Snitzer set_cache_mode(cache, CM_FAIL);
9306b997386SMike Snitzer }
931028ae9f7SJoe Thornber }
932028ae9f7SJoe Thornber
metadata_operation_failed(struct cache * cache,const char * op,int r)933028ae9f7SJoe Thornber static void metadata_operation_failed(struct cache *cache, const char *op, int r)
934028ae9f7SJoe Thornber {
935b61d9509SMike Snitzer DMERR_LIMIT("%s: metadata operation '%s' failed: error = %d",
936b61d9509SMike Snitzer cache_device_name(cache), op, r);
937028ae9f7SJoe Thornber abort_transaction(cache);
938028ae9f7SJoe Thornber set_cache_mode(cache, CM_READ_ONLY);
939028ae9f7SJoe Thornber }
940028ae9f7SJoe Thornber
941b29d4986SJoe Thornber /*----------------------------------------------------------------*/
942b29d4986SJoe Thornber
load_stats(struct cache * cache)943b29d4986SJoe Thornber static void load_stats(struct cache *cache)
944b29d4986SJoe Thornber {
945b29d4986SJoe Thornber struct dm_cache_statistics stats;
946b29d4986SJoe Thornber
947b29d4986SJoe Thornber dm_cache_metadata_get_stats(cache->cmd, &stats);
948b29d4986SJoe Thornber atomic_set(&cache->stats.read_hit, stats.read_hits);
949b29d4986SJoe Thornber atomic_set(&cache->stats.read_miss, stats.read_misses);
950b29d4986SJoe Thornber atomic_set(&cache->stats.write_hit, stats.write_hits);
951b29d4986SJoe Thornber atomic_set(&cache->stats.write_miss, stats.write_misses);
952b29d4986SJoe Thornber }
953b29d4986SJoe Thornber
save_stats(struct cache * cache)954b29d4986SJoe Thornber static void save_stats(struct cache *cache)
955b29d4986SJoe Thornber {
956b29d4986SJoe Thornber struct dm_cache_statistics stats;
957b29d4986SJoe Thornber
958b29d4986SJoe Thornber if (get_cache_mode(cache) >= CM_READ_ONLY)
959b29d4986SJoe Thornber return;
960b29d4986SJoe Thornber
961b29d4986SJoe Thornber stats.read_hits = atomic_read(&cache->stats.read_hit);
962b29d4986SJoe Thornber stats.read_misses = atomic_read(&cache->stats.read_miss);
963b29d4986SJoe Thornber stats.write_hits = atomic_read(&cache->stats.write_hit);
964b29d4986SJoe Thornber stats.write_misses = atomic_read(&cache->stats.write_miss);
965b29d4986SJoe Thornber
966b29d4986SJoe Thornber dm_cache_metadata_set_stats(cache->cmd, &stats);
967b29d4986SJoe Thornber }
968b29d4986SJoe Thornber
update_stats(struct cache_stats * stats,enum policy_operation op)969b29d4986SJoe Thornber static void update_stats(struct cache_stats *stats, enum policy_operation op)
970b29d4986SJoe Thornber {
971b29d4986SJoe Thornber switch (op) {
972b29d4986SJoe Thornber case POLICY_PROMOTE:
973b29d4986SJoe Thornber atomic_inc(&stats->promotion);
974b29d4986SJoe Thornber break;
975b29d4986SJoe Thornber
976b29d4986SJoe Thornber case POLICY_DEMOTE:
977b29d4986SJoe Thornber atomic_inc(&stats->demotion);
978b29d4986SJoe Thornber break;
979b29d4986SJoe Thornber
980b29d4986SJoe Thornber case POLICY_WRITEBACK:
981b29d4986SJoe Thornber atomic_inc(&stats->writeback);
982b29d4986SJoe Thornber break;
983b29d4986SJoe Thornber }
984b29d4986SJoe Thornber }
985b29d4986SJoe Thornber
986a4a82ce3SHeinz Mauelshagen /*
987a4a82ce3SHeinz Mauelshagen *---------------------------------------------------------------------
988c6b4fcbaSJoe Thornber * Migration processing
989c6b4fcbaSJoe Thornber *
990c6b4fcbaSJoe Thornber * Migration covers moving data from the origin device to the cache, or
991c6b4fcbaSJoe Thornber * vice versa.
992a4a82ce3SHeinz Mauelshagen *---------------------------------------------------------------------
993a4a82ce3SHeinz Mauelshagen */
inc_io_migrations(struct cache * cache)994a59db676SJoe Thornber static void inc_io_migrations(struct cache *cache)
995c6b4fcbaSJoe Thornber {
996a59db676SJoe Thornber atomic_inc(&cache->nr_io_migrations);
997c6b4fcbaSJoe Thornber }
998c6b4fcbaSJoe Thornber
dec_io_migrations(struct cache * cache)999a59db676SJoe Thornber static void dec_io_migrations(struct cache *cache)
1000c6b4fcbaSJoe Thornber {
1001a59db676SJoe Thornber atomic_dec(&cache->nr_io_migrations);
1002c6b4fcbaSJoe Thornber }
1003c6b4fcbaSJoe Thornber
discard_or_flush(struct bio * bio)1004651f5fa2SJoe Thornber static bool discard_or_flush(struct bio *bio)
1005651f5fa2SJoe Thornber {
1006f73f44ebSChristoph Hellwig return bio_op(bio) == REQ_OP_DISCARD || op_is_flush(bio->bi_opf);
1007651f5fa2SJoe Thornber }
1008651f5fa2SJoe Thornber
calc_discard_block_range(struct cache * cache,struct bio * bio,dm_dblock_t * b,dm_dblock_t * e)10097ae34e77SJoe Thornber static void calc_discard_block_range(struct cache *cache, struct bio *bio,
10107ae34e77SJoe Thornber dm_dblock_t *b, dm_dblock_t *e)
10117ae34e77SJoe Thornber {
10127ae34e77SJoe Thornber sector_t sb = bio->bi_iter.bi_sector;
10137ae34e77SJoe Thornber sector_t se = bio_end_sector(bio);
10147ae34e77SJoe Thornber
10157ae34e77SJoe Thornber *b = to_dblock(dm_sector_div_up(sb, cache->discard_block_size));
10167ae34e77SJoe Thornber
10177ae34e77SJoe Thornber if (se - sb < cache->discard_block_size)
10187ae34e77SJoe Thornber *e = *b;
10197ae34e77SJoe Thornber else
10207ae34e77SJoe Thornber *e = to_dblock(block_div(se, cache->discard_block_size));
10217ae34e77SJoe Thornber }
10227ae34e77SJoe Thornber
1023b29d4986SJoe Thornber /*----------------------------------------------------------------*/
1024b29d4986SJoe Thornber
prevent_background_work(struct cache * cache)1025b29d4986SJoe Thornber static void prevent_background_work(struct cache *cache)
10267ae34e77SJoe Thornber {
1027b29d4986SJoe Thornber lockdep_off();
1028b29d4986SJoe Thornber down_write(&cache->background_work_lock);
1029b29d4986SJoe Thornber lockdep_on();
1030b29d4986SJoe Thornber }
1031b29d4986SJoe Thornber
allow_background_work(struct cache * cache)1032b29d4986SJoe Thornber static void allow_background_work(struct cache *cache)
1033b29d4986SJoe Thornber {
1034b29d4986SJoe Thornber lockdep_off();
1035b29d4986SJoe Thornber up_write(&cache->background_work_lock);
1036b29d4986SJoe Thornber lockdep_on();
1037b29d4986SJoe Thornber }
1038b29d4986SJoe Thornber
background_work_begin(struct cache * cache)1039b29d4986SJoe Thornber static bool background_work_begin(struct cache *cache)
1040b29d4986SJoe Thornber {
1041b29d4986SJoe Thornber bool r;
1042b29d4986SJoe Thornber
1043b29d4986SJoe Thornber lockdep_off();
1044b29d4986SJoe Thornber r = down_read_trylock(&cache->background_work_lock);
1045b29d4986SJoe Thornber lockdep_on();
1046b29d4986SJoe Thornber
1047b29d4986SJoe Thornber return r;
1048b29d4986SJoe Thornber }
1049b29d4986SJoe Thornber
background_work_end(struct cache * cache)1050b29d4986SJoe Thornber static void background_work_end(struct cache *cache)
1051b29d4986SJoe Thornber {
1052b29d4986SJoe Thornber lockdep_off();
1053b29d4986SJoe Thornber up_read(&cache->background_work_lock);
1054b29d4986SJoe Thornber lockdep_on();
1055b29d4986SJoe Thornber }
1056b29d4986SJoe Thornber
1057b29d4986SJoe Thornber /*----------------------------------------------------------------*/
1058b29d4986SJoe Thornber
bio_writes_complete_block(struct cache * cache,struct bio * bio)1059d1260e2aSJoe Thornber static bool bio_writes_complete_block(struct cache *cache, struct bio *bio)
1060d1260e2aSJoe Thornber {
1061d1260e2aSJoe Thornber return (bio_data_dir(bio) == WRITE) &&
1062d1260e2aSJoe Thornber (bio->bi_iter.bi_size == (cache->sectors_per_block << SECTOR_SHIFT));
1063d1260e2aSJoe Thornber }
1064d1260e2aSJoe Thornber
optimisable_bio(struct cache * cache,struct bio * bio,dm_oblock_t block)1065d1260e2aSJoe Thornber static bool optimisable_bio(struct cache *cache, struct bio *bio, dm_oblock_t block)
1066d1260e2aSJoe Thornber {
10678e3c3827SMike Snitzer return writeback_mode(cache) &&
1068d1260e2aSJoe Thornber (is_discarded_oblock(cache, block) || bio_writes_complete_block(cache, bio));
1069d1260e2aSJoe Thornber }
1070d1260e2aSJoe Thornber
quiesce(struct dm_cache_migration * mg,void (* continuation)(struct work_struct *))1071b29d4986SJoe Thornber static void quiesce(struct dm_cache_migration *mg,
1072b29d4986SJoe Thornber void (*continuation)(struct work_struct *))
1073b29d4986SJoe Thornber {
1074b29d4986SJoe Thornber init_continuation(&mg->k, continuation);
1075b29d4986SJoe Thornber dm_cell_quiesce_v2(mg->cache->prison, mg->cell, &mg->k.ws);
1076b29d4986SJoe Thornber }
1077b29d4986SJoe Thornber
ws_to_mg(struct work_struct * ws)1078b29d4986SJoe Thornber static struct dm_cache_migration *ws_to_mg(struct work_struct *ws)
1079b29d4986SJoe Thornber {
1080b29d4986SJoe Thornber struct continuation *k = container_of(ws, struct continuation, ws);
10810ef0b471SHeinz Mauelshagen
1082b29d4986SJoe Thornber return container_of(k, struct dm_cache_migration, k);
1083b29d4986SJoe Thornber }
1084b29d4986SJoe Thornber
copy_complete(int read_err,unsigned long write_err,void * context)1085b29d4986SJoe Thornber static void copy_complete(int read_err, unsigned long write_err, void *context)
1086b29d4986SJoe Thornber {
1087b29d4986SJoe Thornber struct dm_cache_migration *mg = container_of(context, struct dm_cache_migration, k);
1088b29d4986SJoe Thornber
1089b29d4986SJoe Thornber if (read_err || write_err)
10904e4cbee9SChristoph Hellwig mg->k.input = BLK_STS_IOERR;
1091b29d4986SJoe Thornber
1092b29d4986SJoe Thornber queue_continuation(mg->cache->wq, &mg->k);
1093b29d4986SJoe Thornber }
1094b29d4986SJoe Thornber
copy(struct dm_cache_migration * mg,bool promote)10957209049dSMike Snitzer static void copy(struct dm_cache_migration *mg, bool promote)
1096b29d4986SJoe Thornber {
1097b29d4986SJoe Thornber struct dm_io_region o_region, c_region;
1098cc7da0baSJoe Thornber struct cache *cache = mg->cache;
10997ae34e77SJoe Thornber
1100b29d4986SJoe Thornber o_region.bdev = cache->origin_dev->bdev;
1101b29d4986SJoe Thornber o_region.sector = from_oblock(mg->op->oblock) * cache->sectors_per_block;
1102b29d4986SJoe Thornber o_region.count = cache->sectors_per_block;
11037ae34e77SJoe Thornber
1104b29d4986SJoe Thornber c_region.bdev = cache->cache_dev->bdev;
1105b29d4986SJoe Thornber c_region.sector = from_cblock(mg->op->cblock) * cache->sectors_per_block;
1106b29d4986SJoe Thornber c_region.count = cache->sectors_per_block;
11077ae34e77SJoe Thornber
1108b29d4986SJoe Thornber if (promote)
11097209049dSMike Snitzer dm_kcopyd_copy(cache->copier, &o_region, 1, &c_region, 0, copy_complete, &mg->k);
1110c6b4fcbaSJoe Thornber else
11117209049dSMike Snitzer dm_kcopyd_copy(cache->copier, &c_region, 1, &o_region, 0, copy_complete, &mg->k);
1112c6b4fcbaSJoe Thornber }
1113c6b4fcbaSJoe Thornber
bio_drop_shared_lock(struct cache * cache,struct bio * bio)1114b29d4986SJoe Thornber static void bio_drop_shared_lock(struct cache *cache, struct bio *bio)
1115c6b4fcbaSJoe Thornber {
1116693b960eSMike Snitzer struct per_bio_data *pb = get_per_bio_data(bio);
1117c6b4fcbaSJoe Thornber
1118b29d4986SJoe Thornber if (pb->cell && dm_cell_put_v2(cache->prison, pb->cell))
1119b29d4986SJoe Thornber free_prison_cell(cache, pb->cell);
1120b29d4986SJoe Thornber pb->cell = NULL;
1121c6b4fcbaSJoe Thornber }
1122c6b4fcbaSJoe Thornber
overwrite_endio(struct bio * bio)1123b29d4986SJoe Thornber static void overwrite_endio(struct bio *bio)
1124c6b4fcbaSJoe Thornber {
1125b29d4986SJoe Thornber struct dm_cache_migration *mg = bio->bi_private;
1126c6b4fcbaSJoe Thornber struct cache *cache = mg->cache;
1127693b960eSMike Snitzer struct per_bio_data *pb = get_per_bio_data(bio);
1128c6b4fcbaSJoe Thornber
1129b29d4986SJoe Thornber dm_unhook_bio(&pb->hook_info, bio);
1130c6b4fcbaSJoe Thornber
11314e4cbee9SChristoph Hellwig if (bio->bi_status)
11324e4cbee9SChristoph Hellwig mg->k.input = bio->bi_status;
1133b29d4986SJoe Thornber
1134693b960eSMike Snitzer queue_continuation(cache->wq, &mg->k);
1135c6b4fcbaSJoe Thornber }
1136c6b4fcbaSJoe Thornber
overwrite(struct dm_cache_migration * mg,void (* continuation)(struct work_struct *))1137b29d4986SJoe Thornber static void overwrite(struct dm_cache_migration *mg,
1138b29d4986SJoe Thornber void (*continuation)(struct work_struct *))
1139c6b4fcbaSJoe Thornber {
1140b29d4986SJoe Thornber struct bio *bio = mg->overwrite_bio;
1141693b960eSMike Snitzer struct per_bio_data *pb = get_per_bio_data(bio);
1142c6b4fcbaSJoe Thornber
1143b29d4986SJoe Thornber dm_hook_bio(&pb->hook_info, bio, overwrite_endio, mg);
1144c6b4fcbaSJoe Thornber
1145b29d4986SJoe Thornber /*
1146b29d4986SJoe Thornber * The overwrite bio is part of the copy operation, as such it does
1147b29d4986SJoe Thornber * not set/clear discard or dirty flags.
1148b29d4986SJoe Thornber */
1149b29d4986SJoe Thornber if (mg->op->op == POLICY_PROMOTE)
1150b29d4986SJoe Thornber remap_to_cache(mg->cache, bio, mg->op->cblock);
1151b29d4986SJoe Thornber else
1152b29d4986SJoe Thornber remap_to_origin(mg->cache, bio);
1153c6b4fcbaSJoe Thornber
1154b29d4986SJoe Thornber init_continuation(&mg->k, continuation);
1155b29d4986SJoe Thornber accounted_request(mg->cache, bio);
1156c6b4fcbaSJoe Thornber }
1157c6b4fcbaSJoe Thornber
11582ee57d58SJoe Thornber /*
1159b29d4986SJoe Thornber * Migration steps:
1160b29d4986SJoe Thornber *
1161b29d4986SJoe Thornber * 1) exclusive lock preventing WRITEs
1162b29d4986SJoe Thornber * 2) quiesce
1163b29d4986SJoe Thornber * 3) copy or issue overwrite bio
1164b29d4986SJoe Thornber * 4) upgrade to exclusive lock preventing READs and WRITEs
1165b29d4986SJoe Thornber * 5) quiesce
1166b29d4986SJoe Thornber * 6) update metadata and commit
1167b29d4986SJoe Thornber * 7) unlock
11682ee57d58SJoe Thornber */
mg_complete(struct dm_cache_migration * mg,bool success)1169b29d4986SJoe Thornber static void mg_complete(struct dm_cache_migration *mg, bool success)
11702ee57d58SJoe Thornber {
1171b29d4986SJoe Thornber struct bio_list bios;
1172b29d4986SJoe Thornber struct cache *cache = mg->cache;
1173b29d4986SJoe Thornber struct policy_work *op = mg->op;
1174b29d4986SJoe Thornber dm_cblock_t cblock = op->cblock;
11752ee57d58SJoe Thornber
1176b29d4986SJoe Thornber if (success)
1177b29d4986SJoe Thornber update_stats(&cache->stats, op->op);
11782ee57d58SJoe Thornber
1179b29d4986SJoe Thornber switch (op->op) {
1180b29d4986SJoe Thornber case POLICY_PROMOTE:
1181b29d4986SJoe Thornber clear_discard(cache, oblock_to_dblock(cache, op->oblock));
1182b29d4986SJoe Thornber policy_complete_background_work(cache->policy, op, success);
1183b29d4986SJoe Thornber
1184b29d4986SJoe Thornber if (mg->overwrite_bio) {
1185b29d4986SJoe Thornber if (success)
1186b29d4986SJoe Thornber force_set_dirty(cache, cblock);
11874e4cbee9SChristoph Hellwig else if (mg->k.input)
11884e4cbee9SChristoph Hellwig mg->overwrite_bio->bi_status = mg->k.input;
1189b29d4986SJoe Thornber else
11904e4cbee9SChristoph Hellwig mg->overwrite_bio->bi_status = BLK_STS_IOERR;
1191b29d4986SJoe Thornber bio_endio(mg->overwrite_bio);
1192b29d4986SJoe Thornber } else {
1193b29d4986SJoe Thornber if (success)
1194b29d4986SJoe Thornber force_clear_dirty(cache, cblock);
1195b29d4986SJoe Thornber dec_io_migrations(cache);
1196b29d4986SJoe Thornber }
1197b29d4986SJoe Thornber break;
1198b29d4986SJoe Thornber
1199b29d4986SJoe Thornber case POLICY_DEMOTE:
1200b29d4986SJoe Thornber /*
1201b29d4986SJoe Thornber * We clear dirty here to update the nr_dirty counter.
1202b29d4986SJoe Thornber */
1203b29d4986SJoe Thornber if (success)
1204b29d4986SJoe Thornber force_clear_dirty(cache, cblock);
1205b29d4986SJoe Thornber policy_complete_background_work(cache->policy, op, success);
1206b29d4986SJoe Thornber dec_io_migrations(cache);
1207b29d4986SJoe Thornber break;
1208b29d4986SJoe Thornber
1209b29d4986SJoe Thornber case POLICY_WRITEBACK:
1210b29d4986SJoe Thornber if (success)
1211b29d4986SJoe Thornber force_clear_dirty(cache, cblock);
1212b29d4986SJoe Thornber policy_complete_background_work(cache->policy, op, success);
1213b29d4986SJoe Thornber dec_io_migrations(cache);
1214b29d4986SJoe Thornber break;
12152ee57d58SJoe Thornber }
12162ee57d58SJoe Thornber
1217b29d4986SJoe Thornber bio_list_init(&bios);
1218b29d4986SJoe Thornber if (mg->cell) {
1219b29d4986SJoe Thornber if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios))
1220b29d4986SJoe Thornber free_prison_cell(cache, mg->cell);
1221b29d4986SJoe Thornber }
1222b29d4986SJoe Thornber
1223b29d4986SJoe Thornber free_migration(mg);
1224b29d4986SJoe Thornber defer_bios(cache, &bios);
1225b29d4986SJoe Thornber wake_migration_worker(cache);
1226b29d4986SJoe Thornber
1227b29d4986SJoe Thornber background_work_end(cache);
1228b29d4986SJoe Thornber }
1229b29d4986SJoe Thornber
mg_success(struct work_struct * ws)1230b29d4986SJoe Thornber static void mg_success(struct work_struct *ws)
12317ae34e77SJoe Thornber {
1232b29d4986SJoe Thornber struct dm_cache_migration *mg = ws_to_mg(ws);
12330ef0b471SHeinz Mauelshagen
1234b29d4986SJoe Thornber mg_complete(mg, mg->k.input == 0);
1235b29d4986SJoe Thornber }
12367ae34e77SJoe Thornber
mg_update_metadata(struct work_struct * ws)1237b29d4986SJoe Thornber static void mg_update_metadata(struct work_struct *ws)
1238b29d4986SJoe Thornber {
1239b29d4986SJoe Thornber int r;
1240b29d4986SJoe Thornber struct dm_cache_migration *mg = ws_to_mg(ws);
1241b29d4986SJoe Thornber struct cache *cache = mg->cache;
1242b29d4986SJoe Thornber struct policy_work *op = mg->op;
1243b29d4986SJoe Thornber
1244b29d4986SJoe Thornber switch (op->op) {
1245b29d4986SJoe Thornber case POLICY_PROMOTE:
1246b29d4986SJoe Thornber r = dm_cache_insert_mapping(cache->cmd, op->cblock, op->oblock);
1247b29d4986SJoe Thornber if (r) {
1248b29d4986SJoe Thornber DMERR_LIMIT("%s: migration failed; couldn't insert mapping",
1249b29d4986SJoe Thornber cache_device_name(cache));
1250b29d4986SJoe Thornber metadata_operation_failed(cache, "dm_cache_insert_mapping", r);
1251b29d4986SJoe Thornber
1252b29d4986SJoe Thornber mg_complete(mg, false);
1253b29d4986SJoe Thornber return;
1254b29d4986SJoe Thornber }
1255b29d4986SJoe Thornber mg_complete(mg, true);
1256b29d4986SJoe Thornber break;
1257b29d4986SJoe Thornber
1258b29d4986SJoe Thornber case POLICY_DEMOTE:
1259b29d4986SJoe Thornber r = dm_cache_remove_mapping(cache->cmd, op->cblock);
1260b29d4986SJoe Thornber if (r) {
1261b29d4986SJoe Thornber DMERR_LIMIT("%s: migration failed; couldn't update on disk metadata",
1262b29d4986SJoe Thornber cache_device_name(cache));
1263b29d4986SJoe Thornber metadata_operation_failed(cache, "dm_cache_remove_mapping", r);
1264b29d4986SJoe Thornber
1265b29d4986SJoe Thornber mg_complete(mg, false);
1266b29d4986SJoe Thornber return;
1267b29d4986SJoe Thornber }
1268b29d4986SJoe Thornber
1269b29d4986SJoe Thornber /*
1270b29d4986SJoe Thornber * It would be nice if we only had to commit when a REQ_FLUSH
1271b29d4986SJoe Thornber * comes through. But there's one scenario that we have to
1272b29d4986SJoe Thornber * look out for:
1273b29d4986SJoe Thornber *
1274b29d4986SJoe Thornber * - vblock x in a cache block
1275b29d4986SJoe Thornber * - domotion occurs
1276b29d4986SJoe Thornber * - cache block gets reallocated and over written
1277b29d4986SJoe Thornber * - crash
1278b29d4986SJoe Thornber *
1279b29d4986SJoe Thornber * When we recover, because there was no commit the cache will
1280b29d4986SJoe Thornber * rollback to having the data for vblock x in the cache block.
1281b29d4986SJoe Thornber * But the cache block has since been overwritten, so it'll end
1282b29d4986SJoe Thornber * up pointing to data that was never in 'x' during the history
1283b29d4986SJoe Thornber * of the device.
1284b29d4986SJoe Thornber *
1285b29d4986SJoe Thornber * To avoid this issue we require a commit as part of the
1286b29d4986SJoe Thornber * demotion operation.
1287b29d4986SJoe Thornber */
1288b29d4986SJoe Thornber init_continuation(&mg->k, mg_success);
1289b29d4986SJoe Thornber continue_after_commit(&cache->committer, &mg->k);
1290b29d4986SJoe Thornber schedule_commit(&cache->committer);
1291b29d4986SJoe Thornber break;
1292b29d4986SJoe Thornber
1293b29d4986SJoe Thornber case POLICY_WRITEBACK:
1294b29d4986SJoe Thornber mg_complete(mg, true);
1295b29d4986SJoe Thornber break;
1296b29d4986SJoe Thornber }
1297b29d4986SJoe Thornber }
1298b29d4986SJoe Thornber
mg_update_metadata_after_copy(struct work_struct * ws)1299b29d4986SJoe Thornber static void mg_update_metadata_after_copy(struct work_struct *ws)
1300b29d4986SJoe Thornber {
1301b29d4986SJoe Thornber struct dm_cache_migration *mg = ws_to_mg(ws);
1302b29d4986SJoe Thornber
1303b29d4986SJoe Thornber /*
1304b29d4986SJoe Thornber * Did the copy succeed?
1305b29d4986SJoe Thornber */
1306b29d4986SJoe Thornber if (mg->k.input)
1307b29d4986SJoe Thornber mg_complete(mg, false);
1308b29d4986SJoe Thornber else
1309b29d4986SJoe Thornber mg_update_metadata(ws);
1310b29d4986SJoe Thornber }
1311b29d4986SJoe Thornber
mg_upgrade_lock(struct work_struct * ws)1312b29d4986SJoe Thornber static void mg_upgrade_lock(struct work_struct *ws)
1313b29d4986SJoe Thornber {
1314b29d4986SJoe Thornber int r;
1315b29d4986SJoe Thornber struct dm_cache_migration *mg = ws_to_mg(ws);
1316b29d4986SJoe Thornber
1317b29d4986SJoe Thornber /*
1318b29d4986SJoe Thornber * Did the copy succeed?
1319b29d4986SJoe Thornber */
1320b29d4986SJoe Thornber if (mg->k.input)
1321b29d4986SJoe Thornber mg_complete(mg, false);
1322b29d4986SJoe Thornber
1323b29d4986SJoe Thornber else {
1324b29d4986SJoe Thornber /*
1325b29d4986SJoe Thornber * Now we want the lock to prevent both reads and writes.
1326b29d4986SJoe Thornber */
1327b29d4986SJoe Thornber r = dm_cell_lock_promote_v2(mg->cache->prison, mg->cell,
1328b29d4986SJoe Thornber READ_WRITE_LOCK_LEVEL);
1329b29d4986SJoe Thornber if (r < 0)
1330b29d4986SJoe Thornber mg_complete(mg, false);
1331b29d4986SJoe Thornber
1332b29d4986SJoe Thornber else if (r)
1333b29d4986SJoe Thornber quiesce(mg, mg_update_metadata);
1334b29d4986SJoe Thornber
1335b29d4986SJoe Thornber else
1336b29d4986SJoe Thornber mg_update_metadata(ws);
1337b29d4986SJoe Thornber }
1338b29d4986SJoe Thornber }
1339b29d4986SJoe Thornber
mg_full_copy(struct work_struct * ws)1340d1260e2aSJoe Thornber static void mg_full_copy(struct work_struct *ws)
1341b29d4986SJoe Thornber {
1342b29d4986SJoe Thornber struct dm_cache_migration *mg = ws_to_mg(ws);
1343b29d4986SJoe Thornber struct cache *cache = mg->cache;
1344b29d4986SJoe Thornber struct policy_work *op = mg->op;
1345b29d4986SJoe Thornber bool is_policy_promote = (op->op == POLICY_PROMOTE);
1346b29d4986SJoe Thornber
1347b29d4986SJoe Thornber if ((!is_policy_promote && !is_dirty(cache, op->cblock)) ||
1348b29d4986SJoe Thornber is_discarded_oblock(cache, op->oblock)) {
1349b29d4986SJoe Thornber mg_upgrade_lock(ws);
1350b29d4986SJoe Thornber return;
1351b29d4986SJoe Thornber }
1352b29d4986SJoe Thornber
1353b29d4986SJoe Thornber init_continuation(&mg->k, mg_upgrade_lock);
13547209049dSMike Snitzer copy(mg, is_policy_promote);
1355b29d4986SJoe Thornber }
1356d1260e2aSJoe Thornber
mg_copy(struct work_struct * ws)1357d1260e2aSJoe Thornber static void mg_copy(struct work_struct *ws)
1358d1260e2aSJoe Thornber {
1359d1260e2aSJoe Thornber struct dm_cache_migration *mg = ws_to_mg(ws);
1360d1260e2aSJoe Thornber
1361d1260e2aSJoe Thornber if (mg->overwrite_bio) {
1362d1260e2aSJoe Thornber /*
1363d1260e2aSJoe Thornber * No exclusive lock was held when we last checked if the bio
1364d1260e2aSJoe Thornber * was optimisable. So we have to check again in case things
1365d1260e2aSJoe Thornber * have changed (eg, the block may no longer be discarded).
1366d1260e2aSJoe Thornber */
1367d1260e2aSJoe Thornber if (!optimisable_bio(mg->cache, mg->overwrite_bio, mg->op->oblock)) {
1368d1260e2aSJoe Thornber /*
1369d1260e2aSJoe Thornber * Fallback to a real full copy after doing some tidying up.
1370d1260e2aSJoe Thornber */
1371d1260e2aSJoe Thornber bool rb = bio_detain_shared(mg->cache, mg->op->oblock, mg->overwrite_bio);
13720ef0b471SHeinz Mauelshagen
1373d1260e2aSJoe Thornber BUG_ON(rb); /* An exclussive lock must _not_ be held for this block */
1374d1260e2aSJoe Thornber mg->overwrite_bio = NULL;
1375d1260e2aSJoe Thornber inc_io_migrations(mg->cache);
1376d1260e2aSJoe Thornber mg_full_copy(ws);
1377d1260e2aSJoe Thornber return;
1378d1260e2aSJoe Thornber }
1379d1260e2aSJoe Thornber
1380d1260e2aSJoe Thornber /*
1381d1260e2aSJoe Thornber * It's safe to do this here, even though it's new data
1382d1260e2aSJoe Thornber * because all IO has been locked out of the block.
1383d1260e2aSJoe Thornber *
1384d1260e2aSJoe Thornber * mg_lock_writes() already took READ_WRITE_LOCK_LEVEL
1385d1260e2aSJoe Thornber * so _not_ using mg_upgrade_lock() as continutation.
1386d1260e2aSJoe Thornber */
1387d1260e2aSJoe Thornber overwrite(mg, mg_update_metadata_after_copy);
1388d1260e2aSJoe Thornber
1389d1260e2aSJoe Thornber } else
1390d1260e2aSJoe Thornber mg_full_copy(ws);
1391b29d4986SJoe Thornber }
1392b29d4986SJoe Thornber
mg_lock_writes(struct dm_cache_migration * mg)1393b29d4986SJoe Thornber static int mg_lock_writes(struct dm_cache_migration *mg)
1394b29d4986SJoe Thornber {
1395b29d4986SJoe Thornber int r;
1396b29d4986SJoe Thornber struct dm_cell_key_v2 key;
1397b29d4986SJoe Thornber struct cache *cache = mg->cache;
1398b29d4986SJoe Thornber struct dm_bio_prison_cell_v2 *prealloc;
1399b29d4986SJoe Thornber
1400b29d4986SJoe Thornber prealloc = alloc_prison_cell(cache);
1401b29d4986SJoe Thornber
1402b29d4986SJoe Thornber /*
1403b29d4986SJoe Thornber * Prevent writes to the block, but allow reads to continue.
1404b29d4986SJoe Thornber * Unless we're using an overwrite bio, in which case we lock
1405b29d4986SJoe Thornber * everything.
1406b29d4986SJoe Thornber */
1407b29d4986SJoe Thornber build_key(mg->op->oblock, oblock_succ(mg->op->oblock), &key);
1408b29d4986SJoe Thornber r = dm_cell_lock_v2(cache->prison, &key,
1409b29d4986SJoe Thornber mg->overwrite_bio ? READ_WRITE_LOCK_LEVEL : WRITE_LOCK_LEVEL,
1410b29d4986SJoe Thornber prealloc, &mg->cell);
1411b29d4986SJoe Thornber if (r < 0) {
1412b29d4986SJoe Thornber free_prison_cell(cache, prealloc);
1413b29d4986SJoe Thornber mg_complete(mg, false);
1414b29d4986SJoe Thornber return r;
1415b29d4986SJoe Thornber }
1416b29d4986SJoe Thornber
1417b29d4986SJoe Thornber if (mg->cell != prealloc)
1418b29d4986SJoe Thornber free_prison_cell(cache, prealloc);
1419b29d4986SJoe Thornber
1420b29d4986SJoe Thornber if (r == 0)
1421b29d4986SJoe Thornber mg_copy(&mg->k.ws);
1422b29d4986SJoe Thornber else
1423b29d4986SJoe Thornber quiesce(mg, mg_copy);
1424b29d4986SJoe Thornber
1425b29d4986SJoe Thornber return 0;
1426b29d4986SJoe Thornber }
1427b29d4986SJoe Thornber
mg_start(struct cache * cache,struct policy_work * op,struct bio * bio)1428b29d4986SJoe Thornber static int mg_start(struct cache *cache, struct policy_work *op, struct bio *bio)
1429b29d4986SJoe Thornber {
1430b29d4986SJoe Thornber struct dm_cache_migration *mg;
1431b29d4986SJoe Thornber
1432b29d4986SJoe Thornber if (!background_work_begin(cache)) {
1433b29d4986SJoe Thornber policy_complete_background_work(cache->policy, op, false);
1434b29d4986SJoe Thornber return -EPERM;
1435b29d4986SJoe Thornber }
1436b29d4986SJoe Thornber
1437b29d4986SJoe Thornber mg = alloc_migration(cache);
1438b29d4986SJoe Thornber
1439b29d4986SJoe Thornber mg->op = op;
1440b29d4986SJoe Thornber mg->overwrite_bio = bio;
14417ae34e77SJoe Thornber
1442b29d4986SJoe Thornber if (!bio)
1443b29d4986SJoe Thornber inc_io_migrations(cache);
1444b29d4986SJoe Thornber
1445b29d4986SJoe Thornber return mg_lock_writes(mg);
1446b29d4986SJoe Thornber }
1447b29d4986SJoe Thornber
1448a4a82ce3SHeinz Mauelshagen /*
1449a4a82ce3SHeinz Mauelshagen *--------------------------------------------------------------
1450b29d4986SJoe Thornber * invalidation processing
1451a4a82ce3SHeinz Mauelshagen *--------------------------------------------------------------
1452a4a82ce3SHeinz Mauelshagen */
1453b29d4986SJoe Thornber
invalidate_complete(struct dm_cache_migration * mg,bool success)1454b29d4986SJoe Thornber static void invalidate_complete(struct dm_cache_migration *mg, bool success)
1455b29d4986SJoe Thornber {
1456b29d4986SJoe Thornber struct bio_list bios;
1457b29d4986SJoe Thornber struct cache *cache = mg->cache;
1458b29d4986SJoe Thornber
1459b29d4986SJoe Thornber bio_list_init(&bios);
1460b29d4986SJoe Thornber if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios))
1461b29d4986SJoe Thornber free_prison_cell(cache, mg->cell);
1462b29d4986SJoe Thornber
1463b29d4986SJoe Thornber if (!success && mg->overwrite_bio)
1464b29d4986SJoe Thornber bio_io_error(mg->overwrite_bio);
1465b29d4986SJoe Thornber
1466b29d4986SJoe Thornber free_migration(mg);
1467b29d4986SJoe Thornber defer_bios(cache, &bios);
1468b29d4986SJoe Thornber
1469b29d4986SJoe Thornber background_work_end(cache);
1470b29d4986SJoe Thornber }
1471b29d4986SJoe Thornber
invalidate_completed(struct work_struct * ws)1472b29d4986SJoe Thornber static void invalidate_completed(struct work_struct *ws)
1473b29d4986SJoe Thornber {
1474b29d4986SJoe Thornber struct dm_cache_migration *mg = ws_to_mg(ws);
14750ef0b471SHeinz Mauelshagen
1476b29d4986SJoe Thornber invalidate_complete(mg, !mg->k.input);
1477b29d4986SJoe Thornber }
1478b29d4986SJoe Thornber
invalidate_cblock(struct cache * cache,dm_cblock_t cblock)1479b29d4986SJoe Thornber static int invalidate_cblock(struct cache *cache, dm_cblock_t cblock)
1480b29d4986SJoe Thornber {
14810ef0b471SHeinz Mauelshagen int r;
14820ef0b471SHeinz Mauelshagen
14830ef0b471SHeinz Mauelshagen r = policy_invalidate_mapping(cache->policy, cblock);
1484b29d4986SJoe Thornber if (!r) {
1485b29d4986SJoe Thornber r = dm_cache_remove_mapping(cache->cmd, cblock);
1486b29d4986SJoe Thornber if (r) {
1487b29d4986SJoe Thornber DMERR_LIMIT("%s: invalidation failed; couldn't update on disk metadata",
1488b29d4986SJoe Thornber cache_device_name(cache));
1489b29d4986SJoe Thornber metadata_operation_failed(cache, "dm_cache_remove_mapping", r);
1490b29d4986SJoe Thornber }
1491b29d4986SJoe Thornber
1492b29d4986SJoe Thornber } else if (r == -ENODATA) {
1493b29d4986SJoe Thornber /*
1494b29d4986SJoe Thornber * Harmless, already unmapped.
1495b29d4986SJoe Thornber */
1496b29d4986SJoe Thornber r = 0;
1497b29d4986SJoe Thornber
1498b29d4986SJoe Thornber } else
1499b29d4986SJoe Thornber DMERR("%s: policy_invalidate_mapping failed", cache_device_name(cache));
1500b29d4986SJoe Thornber
1501b29d4986SJoe Thornber return r;
1502b29d4986SJoe Thornber }
1503b29d4986SJoe Thornber
invalidate_remove(struct work_struct * ws)1504b29d4986SJoe Thornber static void invalidate_remove(struct work_struct *ws)
1505b29d4986SJoe Thornber {
1506b29d4986SJoe Thornber int r;
1507b29d4986SJoe Thornber struct dm_cache_migration *mg = ws_to_mg(ws);
1508b29d4986SJoe Thornber struct cache *cache = mg->cache;
1509b29d4986SJoe Thornber
1510b29d4986SJoe Thornber r = invalidate_cblock(cache, mg->invalidate_cblock);
1511b29d4986SJoe Thornber if (r) {
1512b29d4986SJoe Thornber invalidate_complete(mg, false);
1513b29d4986SJoe Thornber return;
1514b29d4986SJoe Thornber }
1515b29d4986SJoe Thornber
1516b29d4986SJoe Thornber init_continuation(&mg->k, invalidate_completed);
1517b29d4986SJoe Thornber continue_after_commit(&cache->committer, &mg->k);
1518b29d4986SJoe Thornber remap_to_origin_clear_discard(cache, mg->overwrite_bio, mg->invalidate_oblock);
1519b29d4986SJoe Thornber mg->overwrite_bio = NULL;
1520b29d4986SJoe Thornber schedule_commit(&cache->committer);
1521b29d4986SJoe Thornber }
1522b29d4986SJoe Thornber
invalidate_lock(struct dm_cache_migration * mg)1523b29d4986SJoe Thornber static int invalidate_lock(struct dm_cache_migration *mg)
1524b29d4986SJoe Thornber {
1525b29d4986SJoe Thornber int r;
1526b29d4986SJoe Thornber struct dm_cell_key_v2 key;
1527b29d4986SJoe Thornber struct cache *cache = mg->cache;
1528b29d4986SJoe Thornber struct dm_bio_prison_cell_v2 *prealloc;
1529b29d4986SJoe Thornber
1530b29d4986SJoe Thornber prealloc = alloc_prison_cell(cache);
1531b29d4986SJoe Thornber
1532b29d4986SJoe Thornber build_key(mg->invalidate_oblock, oblock_succ(mg->invalidate_oblock), &key);
1533b29d4986SJoe Thornber r = dm_cell_lock_v2(cache->prison, &key,
1534b29d4986SJoe Thornber READ_WRITE_LOCK_LEVEL, prealloc, &mg->cell);
1535b29d4986SJoe Thornber if (r < 0) {
1536b29d4986SJoe Thornber free_prison_cell(cache, prealloc);
1537b29d4986SJoe Thornber invalidate_complete(mg, false);
1538b29d4986SJoe Thornber return r;
1539b29d4986SJoe Thornber }
1540b29d4986SJoe Thornber
1541b29d4986SJoe Thornber if (mg->cell != prealloc)
1542b29d4986SJoe Thornber free_prison_cell(cache, prealloc);
1543b29d4986SJoe Thornber
1544b29d4986SJoe Thornber if (r)
1545b29d4986SJoe Thornber quiesce(mg, invalidate_remove);
1546b29d4986SJoe Thornber
1547b29d4986SJoe Thornber else {
1548b29d4986SJoe Thornber /*
1549b29d4986SJoe Thornber * We can't call invalidate_remove() directly here because we
1550b29d4986SJoe Thornber * might still be in request context.
1551b29d4986SJoe Thornber */
1552b29d4986SJoe Thornber init_continuation(&mg->k, invalidate_remove);
1553b29d4986SJoe Thornber queue_work(cache->wq, &mg->k.ws);
1554b29d4986SJoe Thornber }
1555b29d4986SJoe Thornber
1556b29d4986SJoe Thornber return 0;
1557b29d4986SJoe Thornber }
1558b29d4986SJoe Thornber
invalidate_start(struct cache * cache,dm_cblock_t cblock,dm_oblock_t oblock,struct bio * bio)1559b29d4986SJoe Thornber static int invalidate_start(struct cache *cache, dm_cblock_t cblock,
1560b29d4986SJoe Thornber dm_oblock_t oblock, struct bio *bio)
1561b29d4986SJoe Thornber {
1562b29d4986SJoe Thornber struct dm_cache_migration *mg;
1563b29d4986SJoe Thornber
1564b29d4986SJoe Thornber if (!background_work_begin(cache))
1565b29d4986SJoe Thornber return -EPERM;
1566b29d4986SJoe Thornber
1567b29d4986SJoe Thornber mg = alloc_migration(cache);
1568b29d4986SJoe Thornber
1569b29d4986SJoe Thornber mg->overwrite_bio = bio;
1570b29d4986SJoe Thornber mg->invalidate_cblock = cblock;
1571b29d4986SJoe Thornber mg->invalidate_oblock = oblock;
1572b29d4986SJoe Thornber
1573b29d4986SJoe Thornber return invalidate_lock(mg);
15747ae34e77SJoe Thornber }
15757ae34e77SJoe Thornber
1576a4a82ce3SHeinz Mauelshagen /*
1577a4a82ce3SHeinz Mauelshagen *--------------------------------------------------------------
1578c6b4fcbaSJoe Thornber * bio processing
1579a4a82ce3SHeinz Mauelshagen *--------------------------------------------------------------
1580a4a82ce3SHeinz Mauelshagen */
1581b29d4986SJoe Thornber
1582b29d4986SJoe Thornber enum busy {
1583b29d4986SJoe Thornber IDLE,
1584b29d4986SJoe Thornber BUSY
1585b29d4986SJoe Thornber };
1586b29d4986SJoe Thornber
spare_migration_bandwidth(struct cache * cache)1587b29d4986SJoe Thornber static enum busy spare_migration_bandwidth(struct cache *cache)
1588c6b4fcbaSJoe Thornber {
1589dc4fa29fSMike Snitzer bool idle = dm_iot_idle_for(&cache->tracker, HZ);
1590a59db676SJoe Thornber sector_t current_volume = (atomic_read(&cache->nr_io_migrations) + 1) *
1591c6b4fcbaSJoe Thornber cache->sectors_per_block;
1592b29d4986SJoe Thornber
159349b7f768SJoe Thornber if (idle && current_volume <= cache->migration_threshold)
159449b7f768SJoe Thornber return IDLE;
1595b29d4986SJoe Thornber else
159649b7f768SJoe Thornber return BUSY;
1597c6b4fcbaSJoe Thornber }
1598c6b4fcbaSJoe Thornber
inc_hit_counter(struct cache * cache,struct bio * bio)1599c6b4fcbaSJoe Thornber static void inc_hit_counter(struct cache *cache, struct bio *bio)
1600c6b4fcbaSJoe Thornber {
1601c6b4fcbaSJoe Thornber atomic_inc(bio_data_dir(bio) == READ ?
1602c6b4fcbaSJoe Thornber &cache->stats.read_hit : &cache->stats.write_hit);
1603c6b4fcbaSJoe Thornber }
1604c6b4fcbaSJoe Thornber
inc_miss_counter(struct cache * cache,struct bio * bio)1605c6b4fcbaSJoe Thornber static void inc_miss_counter(struct cache *cache, struct bio *bio)
1606c6b4fcbaSJoe Thornber {
1607c6b4fcbaSJoe Thornber atomic_inc(bio_data_dir(bio) == READ ?
1608c6b4fcbaSJoe Thornber &cache->stats.read_miss : &cache->stats.write_miss);
1609c6b4fcbaSJoe Thornber }
1610c6b4fcbaSJoe Thornber
1611fb4100aeSJoe Thornber /*----------------------------------------------------------------*/
1612fb4100aeSJoe Thornber
map_bio(struct cache * cache,struct bio * bio,dm_oblock_t block,bool * commit_needed)1613b29d4986SJoe Thornber static int map_bio(struct cache *cache, struct bio *bio, dm_oblock_t block,
1614b29d4986SJoe Thornber bool *commit_needed)
1615651f5fa2SJoe Thornber {
1616b29d4986SJoe Thornber int r, data_dir;
1617b29d4986SJoe Thornber bool rb, background_queued;
1618b29d4986SJoe Thornber dm_cblock_t cblock;
1619651f5fa2SJoe Thornber
1620b29d4986SJoe Thornber *commit_needed = false;
1621651f5fa2SJoe Thornber
1622b29d4986SJoe Thornber rb = bio_detain_shared(cache, block, bio);
1623b29d4986SJoe Thornber if (!rb) {
16242ee57d58SJoe Thornber /*
1625b29d4986SJoe Thornber * An exclusive lock is held for this block, so we have to
1626b29d4986SJoe Thornber * wait. We set the commit_needed flag so the current
1627b29d4986SJoe Thornber * transaction will be committed asap, allowing this lock
1628b29d4986SJoe Thornber * to be dropped.
16292ee57d58SJoe Thornber */
1630b29d4986SJoe Thornber *commit_needed = true;
1631b29d4986SJoe Thornber return DM_MAPIO_SUBMITTED;
1632b29d4986SJoe Thornber }
16332ee57d58SJoe Thornber
1634b29d4986SJoe Thornber data_dir = bio_data_dir(bio);
16352ee57d58SJoe Thornber
1636b29d4986SJoe Thornber if (optimisable_bio(cache, bio, block)) {
1637b29d4986SJoe Thornber struct policy_work *op = NULL;
1638b29d4986SJoe Thornber
1639b29d4986SJoe Thornber r = policy_lookup_with_work(cache->policy, block, &cblock, data_dir, true, &op);
1640b29d4986SJoe Thornber if (unlikely(r && r != -ENOENT)) {
1641b29d4986SJoe Thornber DMERR_LIMIT("%s: policy_lookup_with_work() failed with r = %d",
1642b29d4986SJoe Thornber cache_device_name(cache), r);
1643b29d4986SJoe Thornber bio_io_error(bio);
1644b29d4986SJoe Thornber return DM_MAPIO_SUBMITTED;
1645b29d4986SJoe Thornber }
1646b29d4986SJoe Thornber
1647b29d4986SJoe Thornber if (r == -ENOENT && op) {
1648b29d4986SJoe Thornber bio_drop_shared_lock(cache, bio);
1649b29d4986SJoe Thornber BUG_ON(op->op != POLICY_PROMOTE);
1650b29d4986SJoe Thornber mg_start(cache, op, bio);
1651b29d4986SJoe Thornber return DM_MAPIO_SUBMITTED;
16522ee57d58SJoe Thornber }
16532ee57d58SJoe Thornber } else {
1654b29d4986SJoe Thornber r = policy_lookup(cache->policy, block, &cblock, data_dir, false, &background_queued);
1655b29d4986SJoe Thornber if (unlikely(r && r != -ENOENT)) {
1656b29d4986SJoe Thornber DMERR_LIMIT("%s: policy_lookup() failed with r = %d",
1657b29d4986SJoe Thornber cache_device_name(cache), r);
1658b29d4986SJoe Thornber bio_io_error(bio);
1659b29d4986SJoe Thornber return DM_MAPIO_SUBMITTED;
1660b29d4986SJoe Thornber }
1661b29d4986SJoe Thornber
1662b29d4986SJoe Thornber if (background_queued)
1663b29d4986SJoe Thornber wake_migration_worker(cache);
1664b29d4986SJoe Thornber }
1665b29d4986SJoe Thornber
1666b29d4986SJoe Thornber if (r == -ENOENT) {
1667693b960eSMike Snitzer struct per_bio_data *pb = get_per_bio_data(bio);
1668693b960eSMike Snitzer
1669b29d4986SJoe Thornber /*
1670b29d4986SJoe Thornber * Miss.
1671b29d4986SJoe Thornber */
1672b29d4986SJoe Thornber inc_miss_counter(cache, bio);
1673b29d4986SJoe Thornber if (pb->req_nr == 0) {
1674b29d4986SJoe Thornber accounted_begin(cache, bio);
1675b29d4986SJoe Thornber remap_to_origin_clear_discard(cache, bio, block);
1676b29d4986SJoe Thornber } else {
1677b29d4986SJoe Thornber /*
1678b29d4986SJoe Thornber * This is a duplicate writethrough io that is no
1679b29d4986SJoe Thornber * longer needed because the block has been demoted.
1680b29d4986SJoe Thornber */
1681b29d4986SJoe Thornber bio_endio(bio);
1682b29d4986SJoe Thornber return DM_MAPIO_SUBMITTED;
1683b29d4986SJoe Thornber }
1684b29d4986SJoe Thornber } else {
1685b29d4986SJoe Thornber /*
1686b29d4986SJoe Thornber * Hit.
1687b29d4986SJoe Thornber */
16882ee57d58SJoe Thornber inc_hit_counter(cache, bio);
16892ee57d58SJoe Thornber
1690b29d4986SJoe Thornber /*
1691b29d4986SJoe Thornber * Passthrough always maps to the origin, invalidating any
1692b29d4986SJoe Thornber * cache blocks that are written to.
1693b29d4986SJoe Thornber */
16948e3c3827SMike Snitzer if (passthrough_mode(cache)) {
1695b29d4986SJoe Thornber if (bio_data_dir(bio) == WRITE) {
1696b29d4986SJoe Thornber bio_drop_shared_lock(cache, bio);
1697b29d4986SJoe Thornber atomic_inc(&cache->stats.demotion);
1698b29d4986SJoe Thornber invalidate_start(cache, cblock, block, bio);
1699b29d4986SJoe Thornber } else
1700b29d4986SJoe Thornber remap_to_origin_clear_discard(cache, bio, block);
17018c081b52SJoe Thornber } else {
17028e3c3827SMike Snitzer if (bio_data_dir(bio) == WRITE && writethrough_mode(cache) &&
1703b29d4986SJoe Thornber !is_dirty(cache, cblock)) {
17042df3bae9SMike Snitzer remap_to_origin_and_cache(cache, bio, block, cblock);
1705b29d4986SJoe Thornber accounted_begin(cache, bio);
1706b29d4986SJoe Thornber } else
1707b29d4986SJoe Thornber remap_to_cache_dirty(cache, bio, block, cblock);
17088c081b52SJoe Thornber }
17092ee57d58SJoe Thornber }
17102ee57d58SJoe Thornber
1711651f5fa2SJoe Thornber /*
1712b29d4986SJoe Thornber * dm core turns FUA requests into a separate payload and FLUSH req.
1713651f5fa2SJoe Thornber */
1714b29d4986SJoe Thornber if (bio->bi_opf & REQ_FUA) {
1715b29d4986SJoe Thornber /*
1716b29d4986SJoe Thornber * issue_after_commit will call accounted_begin a second time. So
1717b29d4986SJoe Thornber * we call accounted_complete() to avoid double accounting.
1718b29d4986SJoe Thornber */
1719b29d4986SJoe Thornber accounted_complete(cache, bio);
1720b29d4986SJoe Thornber issue_after_commit(&cache->committer, bio);
1721b29d4986SJoe Thornber *commit_needed = true;
1722b29d4986SJoe Thornber return DM_MAPIO_SUBMITTED;
1723651f5fa2SJoe Thornber }
1724651f5fa2SJoe Thornber
1725b29d4986SJoe Thornber return DM_MAPIO_REMAPPED;
1726b29d4986SJoe Thornber }
1727b29d4986SJoe Thornber
process_bio(struct cache * cache,struct bio * bio)1728b29d4986SJoe Thornber static bool process_bio(struct cache *cache, struct bio *bio)
1729c6b4fcbaSJoe Thornber {
1730b29d4986SJoe Thornber bool commit_needed;
1731b29d4986SJoe Thornber
1732b29d4986SJoe Thornber if (map_bio(cache, bio, get_bio_block(cache, bio), &commit_needed) == DM_MAPIO_REMAPPED)
173369596f55SMike Snitzer dm_submit_bio_remap(bio, NULL);
1734b29d4986SJoe Thornber
1735b29d4986SJoe Thornber return commit_needed;
1736c6b4fcbaSJoe Thornber }
1737c6b4fcbaSJoe Thornber
1738028ae9f7SJoe Thornber /*
1739028ae9f7SJoe Thornber * A non-zero return indicates read_only or fail_io mode.
1740028ae9f7SJoe Thornber */
commit(struct cache * cache,bool clean_shutdown)1741028ae9f7SJoe Thornber static int commit(struct cache *cache, bool clean_shutdown)
1742028ae9f7SJoe Thornber {
1743028ae9f7SJoe Thornber int r;
1744028ae9f7SJoe Thornber
1745028ae9f7SJoe Thornber if (get_cache_mode(cache) >= CM_READ_ONLY)
1746028ae9f7SJoe Thornber return -EINVAL;
1747028ae9f7SJoe Thornber
1748028ae9f7SJoe Thornber atomic_inc(&cache->stats.commit_count);
1749028ae9f7SJoe Thornber r = dm_cache_commit(cache->cmd, clean_shutdown);
1750028ae9f7SJoe Thornber if (r)
1751028ae9f7SJoe Thornber metadata_operation_failed(cache, "dm_cache_commit", r);
1752028ae9f7SJoe Thornber
1753028ae9f7SJoe Thornber return r;
1754028ae9f7SJoe Thornber }
1755028ae9f7SJoe Thornber
1756b29d4986SJoe Thornber /*
1757b29d4986SJoe Thornber * Used by the batcher.
1758b29d4986SJoe Thornber */
commit_op(void * context)17594e4cbee9SChristoph Hellwig static blk_status_t commit_op(void *context)
1760c6b4fcbaSJoe Thornber {
1761b29d4986SJoe Thornber struct cache *cache = context;
1762ffcbcb67SHeinz Mauelshagen
1763b29d4986SJoe Thornber if (dm_cache_changed_this_transaction(cache->cmd))
17644e4cbee9SChristoph Hellwig return errno_to_blk_status(commit(cache, false));
1765b29d4986SJoe Thornber
1766b29d4986SJoe Thornber return 0;
1767c6b4fcbaSJoe Thornber }
1768c6b4fcbaSJoe Thornber
1769b29d4986SJoe Thornber /*----------------------------------------------------------------*/
1770b29d4986SJoe Thornber
process_flush_bio(struct cache * cache,struct bio * bio)1771b29d4986SJoe Thornber static bool process_flush_bio(struct cache *cache, struct bio *bio)
1772b29d4986SJoe Thornber {
1773693b960eSMike Snitzer struct per_bio_data *pb = get_per_bio_data(bio);
1774b29d4986SJoe Thornber
1775b29d4986SJoe Thornber if (!pb->req_nr)
1776b29d4986SJoe Thornber remap_to_origin(cache, bio);
1777b29d4986SJoe Thornber else
1778b29d4986SJoe Thornber remap_to_cache(cache, bio, 0);
1779b29d4986SJoe Thornber
1780b29d4986SJoe Thornber issue_after_commit(&cache->committer, bio);
1781b29d4986SJoe Thornber return true;
1782c6b4fcbaSJoe Thornber }
1783c6b4fcbaSJoe Thornber
process_discard_bio(struct cache * cache,struct bio * bio)1784b29d4986SJoe Thornber static bool process_discard_bio(struct cache *cache, struct bio *bio)
1785c6b4fcbaSJoe Thornber {
1786b29d4986SJoe Thornber dm_dblock_t b, e;
1787b29d4986SJoe Thornber
1788a4a82ce3SHeinz Mauelshagen /*
1789a4a82ce3SHeinz Mauelshagen * FIXME: do we need to lock the region? Or can we just assume the
1790a4a82ce3SHeinz Mauelshagen * user wont be so foolish as to issue discard concurrently with
1791a4a82ce3SHeinz Mauelshagen * other IO?
1792a4a82ce3SHeinz Mauelshagen */
1793b29d4986SJoe Thornber calc_discard_block_range(cache, bio, &b, &e);
1794b29d4986SJoe Thornber while (b != e) {
1795b29d4986SJoe Thornber set_discard(cache, b);
1796b29d4986SJoe Thornber b = to_dblock(from_dblock(b) + 1);
1797b29d4986SJoe Thornber }
1798b29d4986SJoe Thornber
1799de7180ffSMike Snitzer if (cache->features.discard_passdown) {
1800de7180ffSMike Snitzer remap_to_origin(cache, bio);
180169596f55SMike Snitzer dm_submit_bio_remap(bio, NULL);
1802de7180ffSMike Snitzer } else
1803b29d4986SJoe Thornber bio_endio(bio);
1804b29d4986SJoe Thornber
1805b29d4986SJoe Thornber return false;
1806b29d4986SJoe Thornber }
1807b29d4986SJoe Thornber
process_deferred_bios(struct work_struct * ws)1808b29d4986SJoe Thornber static void process_deferred_bios(struct work_struct *ws)
1809b29d4986SJoe Thornber {
1810b29d4986SJoe Thornber struct cache *cache = container_of(ws, struct cache, deferred_bio_worker);
1811b29d4986SJoe Thornber
1812b29d4986SJoe Thornber bool commit_needed = false;
1813c6b4fcbaSJoe Thornber struct bio_list bios;
1814c6b4fcbaSJoe Thornber struct bio *bio;
1815c6b4fcbaSJoe Thornber
1816c6b4fcbaSJoe Thornber bio_list_init(&bios);
1817c6b4fcbaSJoe Thornber
181826b924b9SMikulas Patocka spin_lock_irq(&cache->lock);
1819c6b4fcbaSJoe Thornber bio_list_merge(&bios, &cache->deferred_bios);
1820c6b4fcbaSJoe Thornber bio_list_init(&cache->deferred_bios);
182126b924b9SMikulas Patocka spin_unlock_irq(&cache->lock);
1822c6b4fcbaSJoe Thornber
1823b29d4986SJoe Thornber while ((bio = bio_list_pop(&bios))) {
18241eff9d32SJens Axboe if (bio->bi_opf & REQ_PREFLUSH)
1825b29d4986SJoe Thornber commit_needed = process_flush_bio(cache, bio) || commit_needed;
1826b29d4986SJoe Thornber
1827e6047149SMike Christie else if (bio_op(bio) == REQ_OP_DISCARD)
1828b29d4986SJoe Thornber commit_needed = process_discard_bio(cache, bio) || commit_needed;
1829b29d4986SJoe Thornber
1830c6b4fcbaSJoe Thornber else
1831b29d4986SJoe Thornber commit_needed = process_bio(cache, bio) || commit_needed;
183276227f6dSMike Snitzer cond_resched();
1833c6b4fcbaSJoe Thornber }
1834c6b4fcbaSJoe Thornber
1835b29d4986SJoe Thornber if (commit_needed)
1836b29d4986SJoe Thornber schedule_commit(&cache->committer);
1837c6b4fcbaSJoe Thornber }
1838c6b4fcbaSJoe Thornber
1839a4a82ce3SHeinz Mauelshagen /*
1840a4a82ce3SHeinz Mauelshagen *--------------------------------------------------------------
1841c6b4fcbaSJoe Thornber * Main worker loop
1842a4a82ce3SHeinz Mauelshagen *--------------------------------------------------------------
1843a4a82ce3SHeinz Mauelshagen */
requeue_deferred_bios(struct cache * cache)1844651f5fa2SJoe Thornber static void requeue_deferred_bios(struct cache *cache)
1845c6b4fcbaSJoe Thornber {
1846c6b4fcbaSJoe Thornber struct bio *bio;
1847c6b4fcbaSJoe Thornber struct bio_list bios;
1848c6b4fcbaSJoe Thornber
1849c6b4fcbaSJoe Thornber bio_list_init(&bios);
1850c6b4fcbaSJoe Thornber bio_list_merge(&bios, &cache->deferred_bios);
1851c6b4fcbaSJoe Thornber bio_list_init(&cache->deferred_bios);
1852c6b4fcbaSJoe Thornber
18534246a0b6SChristoph Hellwig while ((bio = bio_list_pop(&bios))) {
18544e4cbee9SChristoph Hellwig bio->bi_status = BLK_STS_DM_REQUEUE;
18554246a0b6SChristoph Hellwig bio_endio(bio);
185676227f6dSMike Snitzer cond_resched();
18574246a0b6SChristoph Hellwig }
1858c6b4fcbaSJoe Thornber }
1859c6b4fcbaSJoe Thornber
1860c6b4fcbaSJoe Thornber /*
1861c6b4fcbaSJoe Thornber * We want to commit periodically so that not too much
1862c6b4fcbaSJoe Thornber * unwritten metadata builds up.
1863c6b4fcbaSJoe Thornber */
do_waker(struct work_struct * ws)1864c6b4fcbaSJoe Thornber static void do_waker(struct work_struct *ws)
1865c6b4fcbaSJoe Thornber {
1866c6b4fcbaSJoe Thornber struct cache *cache = container_of(to_delayed_work(ws), struct cache, waker);
1867b29d4986SJoe Thornber
1868fba10109SJoe Thornber policy_tick(cache->policy, true);
1869b29d4986SJoe Thornber wake_migration_worker(cache);
1870b29d4986SJoe Thornber schedule_commit(&cache->committer);
1871c6b4fcbaSJoe Thornber queue_delayed_work(cache->wq, &cache->waker, COMMIT_PERIOD);
1872c6b4fcbaSJoe Thornber }
1873c6b4fcbaSJoe Thornber
check_migrations(struct work_struct * ws)1874b29d4986SJoe Thornber static void check_migrations(struct work_struct *ws)
1875c6b4fcbaSJoe Thornber {
1876b29d4986SJoe Thornber int r;
1877b29d4986SJoe Thornber struct policy_work *op;
1878b29d4986SJoe Thornber struct cache *cache = container_of(ws, struct cache, migration_worker);
1879b29d4986SJoe Thornber enum busy b;
1880b29d4986SJoe Thornber
1881b29d4986SJoe Thornber for (;;) {
1882b29d4986SJoe Thornber b = spare_migration_bandwidth(cache);
1883b29d4986SJoe Thornber
1884b29d4986SJoe Thornber r = policy_get_background_work(cache->policy, b == IDLE, &op);
1885b29d4986SJoe Thornber if (r == -ENODATA)
1886b29d4986SJoe Thornber break;
1887b29d4986SJoe Thornber
1888b29d4986SJoe Thornber if (r) {
1889b29d4986SJoe Thornber DMERR_LIMIT("%s: policy_background_work failed",
1890b29d4986SJoe Thornber cache_device_name(cache));
1891b29d4986SJoe Thornber break;
1892c6b4fcbaSJoe Thornber }
1893c6b4fcbaSJoe Thornber
1894b29d4986SJoe Thornber r = mg_start(cache, op, NULL);
1895b29d4986SJoe Thornber if (r)
1896b29d4986SJoe Thornber break;
189776227f6dSMike Snitzer
189876227f6dSMike Snitzer cond_resched();
1899b29d4986SJoe Thornber }
1900c6b4fcbaSJoe Thornber }
1901c6b4fcbaSJoe Thornber
1902a4a82ce3SHeinz Mauelshagen /*
1903a4a82ce3SHeinz Mauelshagen *--------------------------------------------------------------
1904c6b4fcbaSJoe Thornber * Target methods
1905a4a82ce3SHeinz Mauelshagen *--------------------------------------------------------------
1906a4a82ce3SHeinz Mauelshagen */
1907c6b4fcbaSJoe Thornber
1908c6b4fcbaSJoe Thornber /*
1909c6b4fcbaSJoe Thornber * This function gets called on the error paths of the constructor, so we
1910c6b4fcbaSJoe Thornber * have to cope with a partially initialised struct.
1911c6b4fcbaSJoe Thornber */
destroy(struct cache * cache)1912c6b4fcbaSJoe Thornber static void destroy(struct cache *cache)
1913c6b4fcbaSJoe Thornber {
191486a3238cSHeinz Mauelshagen unsigned int i;
1915c6b4fcbaSJoe Thornber
19166f1c819cSKent Overstreet mempool_exit(&cache->migration_pool);
1917c6b4fcbaSJoe Thornber
1918c6b4fcbaSJoe Thornber if (cache->prison)
1919b29d4986SJoe Thornber dm_bio_prison_destroy_v2(cache->prison);
1920c6b4fcbaSJoe Thornber
19216a459d8eSLuo Meng cancel_delayed_work_sync(&cache->waker);
1922c6b4fcbaSJoe Thornber if (cache->wq)
1923c6b4fcbaSJoe Thornber destroy_workqueue(cache->wq);
1924c6b4fcbaSJoe Thornber
1925c6b4fcbaSJoe Thornber if (cache->dirty_bitset)
1926c6b4fcbaSJoe Thornber free_bitset(cache->dirty_bitset);
1927c6b4fcbaSJoe Thornber
1928c6b4fcbaSJoe Thornber if (cache->discard_bitset)
1929c6b4fcbaSJoe Thornber free_bitset(cache->discard_bitset);
1930c6b4fcbaSJoe Thornber
1931c6b4fcbaSJoe Thornber if (cache->copier)
1932c6b4fcbaSJoe Thornber dm_kcopyd_client_destroy(cache->copier);
1933c6b4fcbaSJoe Thornber
1934c6b4fcbaSJoe Thornber if (cache->cmd)
1935c6b4fcbaSJoe Thornber dm_cache_metadata_close(cache->cmd);
1936c6b4fcbaSJoe Thornber
1937c6b4fcbaSJoe Thornber if (cache->metadata_dev)
1938c6b4fcbaSJoe Thornber dm_put_device(cache->ti, cache->metadata_dev);
1939c6b4fcbaSJoe Thornber
1940c6b4fcbaSJoe Thornber if (cache->origin_dev)
1941c6b4fcbaSJoe Thornber dm_put_device(cache->ti, cache->origin_dev);
1942c6b4fcbaSJoe Thornber
1943c6b4fcbaSJoe Thornber if (cache->cache_dev)
1944c6b4fcbaSJoe Thornber dm_put_device(cache->ti, cache->cache_dev);
1945c6b4fcbaSJoe Thornber
1946c6b4fcbaSJoe Thornber if (cache->policy)
1947c6b4fcbaSJoe Thornber dm_cache_policy_destroy(cache->policy);
1948c6b4fcbaSJoe Thornber
1949c6b4fcbaSJoe Thornber for (i = 0; i < cache->nr_ctr_args ; i++)
1950c6b4fcbaSJoe Thornber kfree(cache->ctr_args[i]);
1951c6b4fcbaSJoe Thornber kfree(cache->ctr_args);
1952c6b4fcbaSJoe Thornber
19536f1c819cSKent Overstreet bioset_exit(&cache->bs);
19542df3bae9SMike Snitzer
1955c6b4fcbaSJoe Thornber kfree(cache);
1956c6b4fcbaSJoe Thornber }
1957c6b4fcbaSJoe Thornber
cache_dtr(struct dm_target * ti)1958c6b4fcbaSJoe Thornber static void cache_dtr(struct dm_target *ti)
1959c6b4fcbaSJoe Thornber {
1960c6b4fcbaSJoe Thornber struct cache *cache = ti->private;
1961c6b4fcbaSJoe Thornber
1962c6b4fcbaSJoe Thornber destroy(cache);
1963c6b4fcbaSJoe Thornber }
1964c6b4fcbaSJoe Thornber
get_dev_size(struct dm_dev * dev)1965c6b4fcbaSJoe Thornber static sector_t get_dev_size(struct dm_dev *dev)
1966c6b4fcbaSJoe Thornber {
19676dcbb52cSChristoph Hellwig return bdev_nr_sectors(dev->bdev);
1968c6b4fcbaSJoe Thornber }
1969c6b4fcbaSJoe Thornber
1970c6b4fcbaSJoe Thornber /*----------------------------------------------------------------*/
1971c6b4fcbaSJoe Thornber
1972c6b4fcbaSJoe Thornber /*
1973c6b4fcbaSJoe Thornber * Construct a cache device mapping.
1974c6b4fcbaSJoe Thornber *
1975c6b4fcbaSJoe Thornber * cache <metadata dev> <cache dev> <origin dev> <block size>
1976c6b4fcbaSJoe Thornber * <#feature args> [<feature arg>]*
1977c6b4fcbaSJoe Thornber * <policy> <#policy args> [<policy arg>]*
1978c6b4fcbaSJoe Thornber *
1979c6b4fcbaSJoe Thornber * metadata dev : fast device holding the persistent metadata
1980c6b4fcbaSJoe Thornber * cache dev : fast device holding cached data blocks
1981c6b4fcbaSJoe Thornber * origin dev : slow device holding original data blocks
1982c6b4fcbaSJoe Thornber * block size : cache unit size in sectors
1983c6b4fcbaSJoe Thornber *
1984c6b4fcbaSJoe Thornber * #feature args : number of feature arguments passed
1985c6b4fcbaSJoe Thornber * feature args : writethrough. (The default is writeback.)
1986c6b4fcbaSJoe Thornber *
1987c6b4fcbaSJoe Thornber * policy : the replacement policy to use
1988c6b4fcbaSJoe Thornber * #policy args : an even number of policy arguments corresponding
1989c6b4fcbaSJoe Thornber * to key/value pairs passed to the policy
1990c6b4fcbaSJoe Thornber * policy args : key/value pairs passed to the policy
1991c6b4fcbaSJoe Thornber * E.g. 'sequential_threshold 1024'
1992c6b4fcbaSJoe Thornber * See cache-policies.txt for details.
1993c6b4fcbaSJoe Thornber *
1994c6b4fcbaSJoe Thornber * Optional feature arguments are:
1995c6b4fcbaSJoe Thornber * writethrough : write through caching that prohibits cache block
1996c6b4fcbaSJoe Thornber * content from being different from origin block content.
1997c6b4fcbaSJoe Thornber * Without this argument, the default behaviour is to write
1998c6b4fcbaSJoe Thornber * back cache block contents later for performance reasons,
1999c6b4fcbaSJoe Thornber * so they may differ from the corresponding origin blocks.
2000c6b4fcbaSJoe Thornber */
2001c6b4fcbaSJoe Thornber struct cache_args {
2002c6b4fcbaSJoe Thornber struct dm_target *ti;
2003c6b4fcbaSJoe Thornber
2004c6b4fcbaSJoe Thornber struct dm_dev *metadata_dev;
2005c6b4fcbaSJoe Thornber
2006c6b4fcbaSJoe Thornber struct dm_dev *cache_dev;
2007c6b4fcbaSJoe Thornber sector_t cache_sectors;
2008c6b4fcbaSJoe Thornber
2009c6b4fcbaSJoe Thornber struct dm_dev *origin_dev;
2010c6b4fcbaSJoe Thornber sector_t origin_sectors;
2011c6b4fcbaSJoe Thornber
2012c6b4fcbaSJoe Thornber uint32_t block_size;
2013c6b4fcbaSJoe Thornber
2014c6b4fcbaSJoe Thornber const char *policy_name;
2015c6b4fcbaSJoe Thornber int policy_argc;
2016c6b4fcbaSJoe Thornber const char **policy_argv;
2017c6b4fcbaSJoe Thornber
2018c6b4fcbaSJoe Thornber struct cache_features features;
2019c6b4fcbaSJoe Thornber };
2020c6b4fcbaSJoe Thornber
destroy_cache_args(struct cache_args * ca)2021c6b4fcbaSJoe Thornber static void destroy_cache_args(struct cache_args *ca)
2022c6b4fcbaSJoe Thornber {
2023c6b4fcbaSJoe Thornber if (ca->metadata_dev)
2024c6b4fcbaSJoe Thornber dm_put_device(ca->ti, ca->metadata_dev);
2025c6b4fcbaSJoe Thornber
2026c6b4fcbaSJoe Thornber if (ca->cache_dev)
2027c6b4fcbaSJoe Thornber dm_put_device(ca->ti, ca->cache_dev);
2028c6b4fcbaSJoe Thornber
2029c6b4fcbaSJoe Thornber if (ca->origin_dev)
2030c6b4fcbaSJoe Thornber dm_put_device(ca->ti, ca->origin_dev);
2031c6b4fcbaSJoe Thornber
2032c6b4fcbaSJoe Thornber kfree(ca);
2033c6b4fcbaSJoe Thornber }
2034c6b4fcbaSJoe Thornber
at_least_one_arg(struct dm_arg_set * as,char ** error)2035c6b4fcbaSJoe Thornber static bool at_least_one_arg(struct dm_arg_set *as, char **error)
2036c6b4fcbaSJoe Thornber {
2037c6b4fcbaSJoe Thornber if (!as->argc) {
2038c6b4fcbaSJoe Thornber *error = "Insufficient args";
2039c6b4fcbaSJoe Thornber return false;
2040c6b4fcbaSJoe Thornber }
2041c6b4fcbaSJoe Thornber
2042c6b4fcbaSJoe Thornber return true;
2043c6b4fcbaSJoe Thornber }
2044c6b4fcbaSJoe Thornber
parse_metadata_dev(struct cache_args * ca,struct dm_arg_set * as,char ** error)2045c6b4fcbaSJoe Thornber static int parse_metadata_dev(struct cache_args *ca, struct dm_arg_set *as,
2046c6b4fcbaSJoe Thornber char **error)
2047c6b4fcbaSJoe Thornber {
2048c6b4fcbaSJoe Thornber int r;
2049c6b4fcbaSJoe Thornber sector_t metadata_dev_size;
2050c6b4fcbaSJoe Thornber
2051c6b4fcbaSJoe Thornber if (!at_least_one_arg(as, error))
2052c6b4fcbaSJoe Thornber return -EINVAL;
2053c6b4fcbaSJoe Thornber
2054*05bdb996SChristoph Hellwig r = dm_get_device(ca->ti, dm_shift_arg(as),
2055*05bdb996SChristoph Hellwig BLK_OPEN_READ | BLK_OPEN_WRITE, &ca->metadata_dev);
2056c6b4fcbaSJoe Thornber if (r) {
2057c6b4fcbaSJoe Thornber *error = "Error opening metadata device";
2058c6b4fcbaSJoe Thornber return r;
2059c6b4fcbaSJoe Thornber }
2060c6b4fcbaSJoe Thornber
2061c6b4fcbaSJoe Thornber metadata_dev_size = get_dev_size(ca->metadata_dev);
2062c6b4fcbaSJoe Thornber if (metadata_dev_size > DM_CACHE_METADATA_MAX_SECTORS_WARNING)
2063385411ffSChristoph Hellwig DMWARN("Metadata device %pg is larger than %u sectors: excess space will not be used.",
2064385411ffSChristoph Hellwig ca->metadata_dev->bdev, THIN_METADATA_MAX_SECTORS);
2065c6b4fcbaSJoe Thornber
2066c6b4fcbaSJoe Thornber return 0;
2067c6b4fcbaSJoe Thornber }
2068c6b4fcbaSJoe Thornber
parse_cache_dev(struct cache_args * ca,struct dm_arg_set * as,char ** error)2069c6b4fcbaSJoe Thornber static int parse_cache_dev(struct cache_args *ca, struct dm_arg_set *as,
2070c6b4fcbaSJoe Thornber char **error)
2071c6b4fcbaSJoe Thornber {
2072c6b4fcbaSJoe Thornber int r;
2073c6b4fcbaSJoe Thornber
2074c6b4fcbaSJoe Thornber if (!at_least_one_arg(as, error))
2075c6b4fcbaSJoe Thornber return -EINVAL;
2076c6b4fcbaSJoe Thornber
2077*05bdb996SChristoph Hellwig r = dm_get_device(ca->ti, dm_shift_arg(as),
2078*05bdb996SChristoph Hellwig BLK_OPEN_READ | BLK_OPEN_WRITE, &ca->cache_dev);
2079c6b4fcbaSJoe Thornber if (r) {
2080c6b4fcbaSJoe Thornber *error = "Error opening cache device";
2081c6b4fcbaSJoe Thornber return r;
2082c6b4fcbaSJoe Thornber }
2083c6b4fcbaSJoe Thornber ca->cache_sectors = get_dev_size(ca->cache_dev);
2084c6b4fcbaSJoe Thornber
2085c6b4fcbaSJoe Thornber return 0;
2086c6b4fcbaSJoe Thornber }
2087c6b4fcbaSJoe Thornber
parse_origin_dev(struct cache_args * ca,struct dm_arg_set * as,char ** error)2088c6b4fcbaSJoe Thornber static int parse_origin_dev(struct cache_args *ca, struct dm_arg_set *as,
2089c6b4fcbaSJoe Thornber char **error)
2090c6b4fcbaSJoe Thornber {
2091c6b4fcbaSJoe Thornber int r;
2092c6b4fcbaSJoe Thornber
2093c6b4fcbaSJoe Thornber if (!at_least_one_arg(as, error))
2094c6b4fcbaSJoe Thornber return -EINVAL;
2095c6b4fcbaSJoe Thornber
2096*05bdb996SChristoph Hellwig r = dm_get_device(ca->ti, dm_shift_arg(as),
2097*05bdb996SChristoph Hellwig BLK_OPEN_READ | BLK_OPEN_WRITE, &ca->origin_dev);
2098c6b4fcbaSJoe Thornber if (r) {
2099c6b4fcbaSJoe Thornber *error = "Error opening origin device";
2100c6b4fcbaSJoe Thornber return r;
2101c6b4fcbaSJoe Thornber }
2102c6b4fcbaSJoe Thornber
2103c6b4fcbaSJoe Thornber ca->origin_sectors = get_dev_size(ca->origin_dev);
2104c6b4fcbaSJoe Thornber if (ca->ti->len > ca->origin_sectors) {
2105c6b4fcbaSJoe Thornber *error = "Device size larger than cached device";
2106c6b4fcbaSJoe Thornber return -EINVAL;
2107c6b4fcbaSJoe Thornber }
2108c6b4fcbaSJoe Thornber
2109c6b4fcbaSJoe Thornber return 0;
2110c6b4fcbaSJoe Thornber }
2111c6b4fcbaSJoe Thornber
parse_block_size(struct cache_args * ca,struct dm_arg_set * as,char ** error)2112c6b4fcbaSJoe Thornber static int parse_block_size(struct cache_args *ca, struct dm_arg_set *as,
2113c6b4fcbaSJoe Thornber char **error)
2114c6b4fcbaSJoe Thornber {
211505473044SMike Snitzer unsigned long block_size;
2116c6b4fcbaSJoe Thornber
2117c6b4fcbaSJoe Thornber if (!at_least_one_arg(as, error))
2118c6b4fcbaSJoe Thornber return -EINVAL;
2119c6b4fcbaSJoe Thornber
212005473044SMike Snitzer if (kstrtoul(dm_shift_arg(as), 10, &block_size) || !block_size ||
212105473044SMike Snitzer block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS ||
212205473044SMike Snitzer block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS ||
212305473044SMike Snitzer block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) {
2124c6b4fcbaSJoe Thornber *error = "Invalid data block size";
2125c6b4fcbaSJoe Thornber return -EINVAL;
2126c6b4fcbaSJoe Thornber }
2127c6b4fcbaSJoe Thornber
212805473044SMike Snitzer if (block_size > ca->cache_sectors) {
2129c6b4fcbaSJoe Thornber *error = "Data block size is larger than the cache device";
2130c6b4fcbaSJoe Thornber return -EINVAL;
2131c6b4fcbaSJoe Thornber }
2132c6b4fcbaSJoe Thornber
213305473044SMike Snitzer ca->block_size = block_size;
2134c6b4fcbaSJoe Thornber
2135c6b4fcbaSJoe Thornber return 0;
2136c6b4fcbaSJoe Thornber }
2137c6b4fcbaSJoe Thornber
init_features(struct cache_features * cf)2138c6b4fcbaSJoe Thornber static void init_features(struct cache_features *cf)
2139c6b4fcbaSJoe Thornber {
2140c6b4fcbaSJoe Thornber cf->mode = CM_WRITE;
21412ee57d58SJoe Thornber cf->io_mode = CM_IO_WRITEBACK;
2142629d0a8aSJoe Thornber cf->metadata_version = 1;
2143de7180ffSMike Snitzer cf->discard_passdown = true;
2144c6b4fcbaSJoe Thornber }
2145c6b4fcbaSJoe Thornber
parse_features(struct cache_args * ca,struct dm_arg_set * as,char ** error)2146c6b4fcbaSJoe Thornber static int parse_features(struct cache_args *ca, struct dm_arg_set *as,
2147c6b4fcbaSJoe Thornber char **error)
2148c6b4fcbaSJoe Thornber {
21495916a22bSEric Biggers static const struct dm_arg _args[] = {
2150de7180ffSMike Snitzer {0, 3, "Invalid number of cache feature arguments"},
2151c6b4fcbaSJoe Thornber };
2152c6b4fcbaSJoe Thornber
2153af9313c3SJohn Pittman int r, mode_ctr = 0;
215486a3238cSHeinz Mauelshagen unsigned int argc;
2155c6b4fcbaSJoe Thornber const char *arg;
2156c6b4fcbaSJoe Thornber struct cache_features *cf = &ca->features;
2157c6b4fcbaSJoe Thornber
2158c6b4fcbaSJoe Thornber init_features(cf);
2159c6b4fcbaSJoe Thornber
2160c6b4fcbaSJoe Thornber r = dm_read_arg_group(_args, as, &argc, error);
2161c6b4fcbaSJoe Thornber if (r)
2162c6b4fcbaSJoe Thornber return -EINVAL;
2163c6b4fcbaSJoe Thornber
2164c6b4fcbaSJoe Thornber while (argc--) {
2165c6b4fcbaSJoe Thornber arg = dm_shift_arg(as);
2166c6b4fcbaSJoe Thornber
2167af9313c3SJohn Pittman if (!strcasecmp(arg, "writeback")) {
21682ee57d58SJoe Thornber cf->io_mode = CM_IO_WRITEBACK;
2169af9313c3SJohn Pittman mode_ctr++;
2170af9313c3SJohn Pittman }
2171c6b4fcbaSJoe Thornber
2172af9313c3SJohn Pittman else if (!strcasecmp(arg, "writethrough")) {
21732ee57d58SJoe Thornber cf->io_mode = CM_IO_WRITETHROUGH;
2174af9313c3SJohn Pittman mode_ctr++;
2175af9313c3SJohn Pittman }
21762ee57d58SJoe Thornber
2177af9313c3SJohn Pittman else if (!strcasecmp(arg, "passthrough")) {
21782ee57d58SJoe Thornber cf->io_mode = CM_IO_PASSTHROUGH;
2179af9313c3SJohn Pittman mode_ctr++;
2180af9313c3SJohn Pittman }
2181c6b4fcbaSJoe Thornber
2182629d0a8aSJoe Thornber else if (!strcasecmp(arg, "metadata2"))
2183629d0a8aSJoe Thornber cf->metadata_version = 2;
2184629d0a8aSJoe Thornber
2185de7180ffSMike Snitzer else if (!strcasecmp(arg, "no_discard_passdown"))
2186de7180ffSMike Snitzer cf->discard_passdown = false;
2187de7180ffSMike Snitzer
2188c6b4fcbaSJoe Thornber else {
2189c6b4fcbaSJoe Thornber *error = "Unrecognised cache feature requested";
2190c6b4fcbaSJoe Thornber return -EINVAL;
2191c6b4fcbaSJoe Thornber }
2192c6b4fcbaSJoe Thornber }
2193c6b4fcbaSJoe Thornber
2194af9313c3SJohn Pittman if (mode_ctr > 1) {
2195af9313c3SJohn Pittman *error = "Duplicate cache io_mode features requested";
2196af9313c3SJohn Pittman return -EINVAL;
2197af9313c3SJohn Pittman }
2198af9313c3SJohn Pittman
2199c6b4fcbaSJoe Thornber return 0;
2200c6b4fcbaSJoe Thornber }
2201c6b4fcbaSJoe Thornber
parse_policy(struct cache_args * ca,struct dm_arg_set * as,char ** error)2202c6b4fcbaSJoe Thornber static int parse_policy(struct cache_args *ca, struct dm_arg_set *as,
2203c6b4fcbaSJoe Thornber char **error)
2204c6b4fcbaSJoe Thornber {
22055916a22bSEric Biggers static const struct dm_arg _args[] = {
2206c6b4fcbaSJoe Thornber {0, 1024, "Invalid number of policy arguments"},
2207c6b4fcbaSJoe Thornber };
2208c6b4fcbaSJoe Thornber
2209c6b4fcbaSJoe Thornber int r;
2210c6b4fcbaSJoe Thornber
2211c6b4fcbaSJoe Thornber if (!at_least_one_arg(as, error))
2212c6b4fcbaSJoe Thornber return -EINVAL;
2213c6b4fcbaSJoe Thornber
2214c6b4fcbaSJoe Thornber ca->policy_name = dm_shift_arg(as);
2215c6b4fcbaSJoe Thornber
2216c6b4fcbaSJoe Thornber r = dm_read_arg_group(_args, as, &ca->policy_argc, error);
2217c6b4fcbaSJoe Thornber if (r)
2218c6b4fcbaSJoe Thornber return -EINVAL;
2219c6b4fcbaSJoe Thornber
2220c6b4fcbaSJoe Thornber ca->policy_argv = (const char **)as->argv;
2221c6b4fcbaSJoe Thornber dm_consume_args(as, ca->policy_argc);
2222c6b4fcbaSJoe Thornber
2223c6b4fcbaSJoe Thornber return 0;
2224c6b4fcbaSJoe Thornber }
2225c6b4fcbaSJoe Thornber
parse_cache_args(struct cache_args * ca,int argc,char ** argv,char ** error)2226c6b4fcbaSJoe Thornber static int parse_cache_args(struct cache_args *ca, int argc, char **argv,
2227c6b4fcbaSJoe Thornber char **error)
2228c6b4fcbaSJoe Thornber {
2229c6b4fcbaSJoe Thornber int r;
2230c6b4fcbaSJoe Thornber struct dm_arg_set as;
2231c6b4fcbaSJoe Thornber
2232c6b4fcbaSJoe Thornber as.argc = argc;
2233c6b4fcbaSJoe Thornber as.argv = argv;
2234c6b4fcbaSJoe Thornber
2235c6b4fcbaSJoe Thornber r = parse_metadata_dev(ca, &as, error);
2236c6b4fcbaSJoe Thornber if (r)
2237c6b4fcbaSJoe Thornber return r;
2238c6b4fcbaSJoe Thornber
2239c6b4fcbaSJoe Thornber r = parse_cache_dev(ca, &as, error);
2240c6b4fcbaSJoe Thornber if (r)
2241c6b4fcbaSJoe Thornber return r;
2242c6b4fcbaSJoe Thornber
2243c6b4fcbaSJoe Thornber r = parse_origin_dev(ca, &as, error);
2244c6b4fcbaSJoe Thornber if (r)
2245c6b4fcbaSJoe Thornber return r;
2246c6b4fcbaSJoe Thornber
2247c6b4fcbaSJoe Thornber r = parse_block_size(ca, &as, error);
2248c6b4fcbaSJoe Thornber if (r)
2249c6b4fcbaSJoe Thornber return r;
2250c6b4fcbaSJoe Thornber
2251c6b4fcbaSJoe Thornber r = parse_features(ca, &as, error);
2252c6b4fcbaSJoe Thornber if (r)
2253c6b4fcbaSJoe Thornber return r;
2254c6b4fcbaSJoe Thornber
2255c6b4fcbaSJoe Thornber r = parse_policy(ca, &as, error);
2256c6b4fcbaSJoe Thornber if (r)
2257c6b4fcbaSJoe Thornber return r;
2258c6b4fcbaSJoe Thornber
2259c6b4fcbaSJoe Thornber return 0;
2260c6b4fcbaSJoe Thornber }
2261c6b4fcbaSJoe Thornber
2262c6b4fcbaSJoe Thornber /*----------------------------------------------------------------*/
2263c6b4fcbaSJoe Thornber
2264c6b4fcbaSJoe Thornber static struct kmem_cache *migration_cache;
2265c6b4fcbaSJoe Thornber
22662c73c471SAlasdair G Kergon #define NOT_CORE_OPTION 1
22672c73c471SAlasdair G Kergon
process_config_option(struct cache * cache,const char * key,const char * value)22682f14f4b5SJoe Thornber static int process_config_option(struct cache *cache, const char *key, const char *value)
22692c73c471SAlasdair G Kergon {
22702c73c471SAlasdair G Kergon unsigned long tmp;
22712c73c471SAlasdair G Kergon
22722f14f4b5SJoe Thornber if (!strcasecmp(key, "migration_threshold")) {
22732f14f4b5SJoe Thornber if (kstrtoul(value, 10, &tmp))
22742c73c471SAlasdair G Kergon return -EINVAL;
22752c73c471SAlasdair G Kergon
22762c73c471SAlasdair G Kergon cache->migration_threshold = tmp;
22772c73c471SAlasdair G Kergon return 0;
22782c73c471SAlasdair G Kergon }
22792c73c471SAlasdair G Kergon
22802c73c471SAlasdair G Kergon return NOT_CORE_OPTION;
22812c73c471SAlasdair G Kergon }
22822c73c471SAlasdair G Kergon
set_config_value(struct cache * cache,const char * key,const char * value)22832f14f4b5SJoe Thornber static int set_config_value(struct cache *cache, const char *key, const char *value)
22842f14f4b5SJoe Thornber {
22852f14f4b5SJoe Thornber int r = process_config_option(cache, key, value);
22862f14f4b5SJoe Thornber
22872f14f4b5SJoe Thornber if (r == NOT_CORE_OPTION)
22882f14f4b5SJoe Thornber r = policy_set_config_value(cache->policy, key, value);
22892f14f4b5SJoe Thornber
22902f14f4b5SJoe Thornber if (r)
22912f14f4b5SJoe Thornber DMWARN("bad config value for %s: %s", key, value);
22922f14f4b5SJoe Thornber
22932f14f4b5SJoe Thornber return r;
22942f14f4b5SJoe Thornber }
22952f14f4b5SJoe Thornber
set_config_values(struct cache * cache,int argc,const char ** argv)22962f14f4b5SJoe Thornber static int set_config_values(struct cache *cache, int argc, const char **argv)
2297c6b4fcbaSJoe Thornber {
2298c6b4fcbaSJoe Thornber int r = 0;
2299c6b4fcbaSJoe Thornber
2300c6b4fcbaSJoe Thornber if (argc & 1) {
2301c6b4fcbaSJoe Thornber DMWARN("Odd number of policy arguments given but they should be <key> <value> pairs.");
2302c6b4fcbaSJoe Thornber return -EINVAL;
2303c6b4fcbaSJoe Thornber }
2304c6b4fcbaSJoe Thornber
2305c6b4fcbaSJoe Thornber while (argc) {
23062f14f4b5SJoe Thornber r = set_config_value(cache, argv[0], argv[1]);
23072f14f4b5SJoe Thornber if (r)
23082f14f4b5SJoe Thornber break;
2309c6b4fcbaSJoe Thornber
2310c6b4fcbaSJoe Thornber argc -= 2;
2311c6b4fcbaSJoe Thornber argv += 2;
2312c6b4fcbaSJoe Thornber }
2313c6b4fcbaSJoe Thornber
2314c6b4fcbaSJoe Thornber return r;
2315c6b4fcbaSJoe Thornber }
2316c6b4fcbaSJoe Thornber
create_cache_policy(struct cache * cache,struct cache_args * ca,char ** error)2317c6b4fcbaSJoe Thornber static int create_cache_policy(struct cache *cache, struct cache_args *ca,
2318c6b4fcbaSJoe Thornber char **error)
2319c6b4fcbaSJoe Thornber {
23204cb3e1dbSMikulas Patocka struct dm_cache_policy *p = dm_cache_policy_create(ca->policy_name,
2321c6b4fcbaSJoe Thornber cache->cache_size,
2322c6b4fcbaSJoe Thornber cache->origin_sectors,
2323c6b4fcbaSJoe Thornber cache->sectors_per_block);
23244cb3e1dbSMikulas Patocka if (IS_ERR(p)) {
2325c6b4fcbaSJoe Thornber *error = "Error creating cache's policy";
23264cb3e1dbSMikulas Patocka return PTR_ERR(p);
2327c6b4fcbaSJoe Thornber }
23284cb3e1dbSMikulas Patocka cache->policy = p;
2329b29d4986SJoe Thornber BUG_ON(!cache->policy);
2330c6b4fcbaSJoe Thornber
23312f14f4b5SJoe Thornber return 0;
2332c6b4fcbaSJoe Thornber }
2333c6b4fcbaSJoe Thornber
233408b18451SJoe Thornber /*
23352bb812dfSJoe Thornber * We want the discard block size to be at least the size of the cache
23362bb812dfSJoe Thornber * block size and have no more than 2^14 discard blocks across the origin.
233708b18451SJoe Thornber */
233808b18451SJoe Thornber #define MAX_DISCARD_BLOCKS (1 << 14)
233908b18451SJoe Thornber
too_many_discard_blocks(sector_t discard_block_size,sector_t origin_size)234008b18451SJoe Thornber static bool too_many_discard_blocks(sector_t discard_block_size,
234108b18451SJoe Thornber sector_t origin_size)
234208b18451SJoe Thornber {
234308b18451SJoe Thornber (void) sector_div(origin_size, discard_block_size);
234408b18451SJoe Thornber
234508b18451SJoe Thornber return origin_size > MAX_DISCARD_BLOCKS;
234608b18451SJoe Thornber }
234708b18451SJoe Thornber
calculate_discard_block_size(sector_t cache_block_size,sector_t origin_size)234808b18451SJoe Thornber static sector_t calculate_discard_block_size(sector_t cache_block_size,
234908b18451SJoe Thornber sector_t origin_size)
235008b18451SJoe Thornber {
23512bb812dfSJoe Thornber sector_t discard_block_size = cache_block_size;
235208b18451SJoe Thornber
235308b18451SJoe Thornber if (origin_size)
235408b18451SJoe Thornber while (too_many_discard_blocks(discard_block_size, origin_size))
235508b18451SJoe Thornber discard_block_size *= 2;
235608b18451SJoe Thornber
235708b18451SJoe Thornber return discard_block_size;
235808b18451SJoe Thornber }
235908b18451SJoe Thornber
set_cache_size(struct cache * cache,dm_cblock_t size)2360d1d9220cSJoe Thornber static void set_cache_size(struct cache *cache, dm_cblock_t size)
2361d1d9220cSJoe Thornber {
2362d1d9220cSJoe Thornber dm_block_t nr_blocks = from_cblock(size);
2363d1d9220cSJoe Thornber
2364d1d9220cSJoe Thornber if (nr_blocks > (1 << 20) && cache->cache_size != size)
2365d1d9220cSJoe Thornber DMWARN_LIMIT("You have created a cache device with a lot of individual cache blocks (%llu)\n"
2366d1d9220cSJoe Thornber "All these mappings can consume a lot of kernel memory, and take some time to read/write.\n"
2367d1d9220cSJoe Thornber "Please consider increasing the cache block size to reduce the overall cache block count.",
2368d1d9220cSJoe Thornber (unsigned long long) nr_blocks);
2369d1d9220cSJoe Thornber
2370d1d9220cSJoe Thornber cache->cache_size = size;
2371d1d9220cSJoe Thornber }
2372d1d9220cSJoe Thornber
2373f8350dafSJoe Thornber #define DEFAULT_MIGRATION_THRESHOLD 2048
2374c6b4fcbaSJoe Thornber
cache_create(struct cache_args * ca,struct cache ** result)2375c6b4fcbaSJoe Thornber static int cache_create(struct cache_args *ca, struct cache **result)
2376c6b4fcbaSJoe Thornber {
2377c6b4fcbaSJoe Thornber int r = 0;
2378c6b4fcbaSJoe Thornber char **error = &ca->ti->error;
2379c6b4fcbaSJoe Thornber struct cache *cache;
2380c6b4fcbaSJoe Thornber struct dm_target *ti = ca->ti;
2381c6b4fcbaSJoe Thornber dm_block_t origin_blocks;
2382c6b4fcbaSJoe Thornber struct dm_cache_metadata *cmd;
2383c6b4fcbaSJoe Thornber bool may_format = ca->features.mode == CM_WRITE;
2384c6b4fcbaSJoe Thornber
2385c6b4fcbaSJoe Thornber cache = kzalloc(sizeof(*cache), GFP_KERNEL);
2386c6b4fcbaSJoe Thornber if (!cache)
2387c6b4fcbaSJoe Thornber return -ENOMEM;
2388c6b4fcbaSJoe Thornber
2389c6b4fcbaSJoe Thornber cache->ti = ca->ti;
2390c6b4fcbaSJoe Thornber ti->private = cache;
239169596f55SMike Snitzer ti->accounts_remapped_io = true;
2392c6b4fcbaSJoe Thornber ti->num_flush_bios = 2;
2393c6b4fcbaSJoe Thornber ti->flush_supported = true;
2394c6b4fcbaSJoe Thornber
2395c6b4fcbaSJoe Thornber ti->num_discard_bios = 1;
2396c6b4fcbaSJoe Thornber ti->discards_supported = true;
2397c6b4fcbaSJoe Thornber
2398693b960eSMike Snitzer ti->per_io_data_size = sizeof(struct per_bio_data);
2399c6b4fcbaSJoe Thornber
2400693b960eSMike Snitzer cache->features = ca->features;
24012df3bae9SMike Snitzer if (writethrough_mode(cache)) {
24022df3bae9SMike Snitzer /* Create bioset for writethrough bios issued to origin */
24036f1c819cSKent Overstreet r = bioset_init(&cache->bs, BIO_POOL_SIZE, 0, 0);
24046f1c819cSKent Overstreet if (r)
24052df3bae9SMike Snitzer goto bad;
24062df3bae9SMike Snitzer }
24072df3bae9SMike Snitzer
2408c6b4fcbaSJoe Thornber cache->metadata_dev = ca->metadata_dev;
2409c6b4fcbaSJoe Thornber cache->origin_dev = ca->origin_dev;
2410c6b4fcbaSJoe Thornber cache->cache_dev = ca->cache_dev;
2411c6b4fcbaSJoe Thornber
2412c6b4fcbaSJoe Thornber ca->metadata_dev = ca->origin_dev = ca->cache_dev = NULL;
2413c6b4fcbaSJoe Thornber
2414c6b4fcbaSJoe Thornber origin_blocks = cache->origin_sectors = ca->origin_sectors;
2415414dd67dSJoe Thornber origin_blocks = block_div(origin_blocks, ca->block_size);
2416c6b4fcbaSJoe Thornber cache->origin_blocks = to_oblock(origin_blocks);
2417c6b4fcbaSJoe Thornber
2418c6b4fcbaSJoe Thornber cache->sectors_per_block = ca->block_size;
2419c6b4fcbaSJoe Thornber if (dm_set_target_max_io_len(ti, cache->sectors_per_block)) {
2420c6b4fcbaSJoe Thornber r = -EINVAL;
2421c6b4fcbaSJoe Thornber goto bad;
2422c6b4fcbaSJoe Thornber }
2423c6b4fcbaSJoe Thornber
2424c6b4fcbaSJoe Thornber if (ca->block_size & (ca->block_size - 1)) {
2425c6b4fcbaSJoe Thornber dm_block_t cache_size = ca->cache_sectors;
2426c6b4fcbaSJoe Thornber
2427c6b4fcbaSJoe Thornber cache->sectors_per_block_shift = -1;
2428414dd67dSJoe Thornber cache_size = block_div(cache_size, ca->block_size);
2429d1d9220cSJoe Thornber set_cache_size(cache, to_cblock(cache_size));
2430c6b4fcbaSJoe Thornber } else {
2431c6b4fcbaSJoe Thornber cache->sectors_per_block_shift = __ffs(ca->block_size);
2432d1d9220cSJoe Thornber set_cache_size(cache, to_cblock(ca->cache_sectors >> cache->sectors_per_block_shift));
2433c6b4fcbaSJoe Thornber }
2434c6b4fcbaSJoe Thornber
2435c6b4fcbaSJoe Thornber r = create_cache_policy(cache, ca, error);
2436c6b4fcbaSJoe Thornber if (r)
2437c6b4fcbaSJoe Thornber goto bad;
24382f14f4b5SJoe Thornber
2439c6b4fcbaSJoe Thornber cache->policy_nr_args = ca->policy_argc;
24402f14f4b5SJoe Thornber cache->migration_threshold = DEFAULT_MIGRATION_THRESHOLD;
24412f14f4b5SJoe Thornber
24422f14f4b5SJoe Thornber r = set_config_values(cache, ca->policy_argc, ca->policy_argv);
24432f14f4b5SJoe Thornber if (r) {
24442f14f4b5SJoe Thornber *error = "Error setting cache policy's config values";
24452f14f4b5SJoe Thornber goto bad;
24462f14f4b5SJoe Thornber }
2447c6b4fcbaSJoe Thornber
2448c6b4fcbaSJoe Thornber cmd = dm_cache_metadata_open(cache->metadata_dev->bdev,
2449c6b4fcbaSJoe Thornber ca->block_size, may_format,
2450629d0a8aSJoe Thornber dm_cache_policy_get_hint_size(cache->policy),
2451629d0a8aSJoe Thornber ca->features.metadata_version);
2452c6b4fcbaSJoe Thornber if (IS_ERR(cmd)) {
2453c6b4fcbaSJoe Thornber *error = "Error creating metadata object";
2454c6b4fcbaSJoe Thornber r = PTR_ERR(cmd);
2455c6b4fcbaSJoe Thornber goto bad;
2456c6b4fcbaSJoe Thornber }
2457c6b4fcbaSJoe Thornber cache->cmd = cmd;
2458028ae9f7SJoe Thornber set_cache_mode(cache, CM_WRITE);
2459028ae9f7SJoe Thornber if (get_cache_mode(cache) != CM_WRITE) {
2460028ae9f7SJoe Thornber *error = "Unable to get write access to metadata, please check/repair metadata.";
2461028ae9f7SJoe Thornber r = -EINVAL;
2462028ae9f7SJoe Thornber goto bad;
2463028ae9f7SJoe Thornber }
2464c6b4fcbaSJoe Thornber
24658e3c3827SMike Snitzer if (passthrough_mode(cache)) {
24662ee57d58SJoe Thornber bool all_clean;
24672ee57d58SJoe Thornber
24682ee57d58SJoe Thornber r = dm_cache_metadata_all_clean(cache->cmd, &all_clean);
24692ee57d58SJoe Thornber if (r) {
24702ee57d58SJoe Thornber *error = "dm_cache_metadata_all_clean() failed";
24712ee57d58SJoe Thornber goto bad;
24722ee57d58SJoe Thornber }
24732ee57d58SJoe Thornber
24742ee57d58SJoe Thornber if (!all_clean) {
24752ee57d58SJoe Thornber *error = "Cannot enter passthrough mode unless all blocks are clean";
24762ee57d58SJoe Thornber r = -EINVAL;
24772ee57d58SJoe Thornber goto bad;
24782ee57d58SJoe Thornber }
2479b29d4986SJoe Thornber
2480b29d4986SJoe Thornber policy_allow_migrations(cache->policy, false);
24812ee57d58SJoe Thornber }
24822ee57d58SJoe Thornber
2483c6b4fcbaSJoe Thornber spin_lock_init(&cache->lock);
2484c6b4fcbaSJoe Thornber bio_list_init(&cache->deferred_bios);
2485a59db676SJoe Thornber atomic_set(&cache->nr_allocated_migrations, 0);
2486a59db676SJoe Thornber atomic_set(&cache->nr_io_migrations, 0);
2487c6b4fcbaSJoe Thornber init_waitqueue_head(&cache->migration_wait);
2488c6b4fcbaSJoe Thornber
2489fa4d683aSWei Yongjun r = -ENOMEM;
249044fa816bSAnssi Hannula atomic_set(&cache->nr_dirty, 0);
2491c6b4fcbaSJoe Thornber cache->dirty_bitset = alloc_bitset(from_cblock(cache->cache_size));
2492c6b4fcbaSJoe Thornber if (!cache->dirty_bitset) {
2493c6b4fcbaSJoe Thornber *error = "could not allocate dirty bitset";
2494c6b4fcbaSJoe Thornber goto bad;
2495c6b4fcbaSJoe Thornber }
2496c6b4fcbaSJoe Thornber clear_bitset(cache->dirty_bitset, from_cblock(cache->cache_size));
2497c6b4fcbaSJoe Thornber
249808b18451SJoe Thornber cache->discard_block_size =
249908b18451SJoe Thornber calculate_discard_block_size(cache->sectors_per_block,
250008b18451SJoe Thornber cache->origin_sectors);
25012572629aSJoe Thornber cache->discard_nr_blocks = to_dblock(dm_sector_div_up(cache->origin_sectors,
25022572629aSJoe Thornber cache->discard_block_size));
25031bad9bc4SJoe Thornber cache->discard_bitset = alloc_bitset(from_dblock(cache->discard_nr_blocks));
2504c6b4fcbaSJoe Thornber if (!cache->discard_bitset) {
2505c6b4fcbaSJoe Thornber *error = "could not allocate discard bitset";
2506c6b4fcbaSJoe Thornber goto bad;
2507c6b4fcbaSJoe Thornber }
25081bad9bc4SJoe Thornber clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks));
2509c6b4fcbaSJoe Thornber
2510c6b4fcbaSJoe Thornber cache->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle);
2511c6b4fcbaSJoe Thornber if (IS_ERR(cache->copier)) {
2512c6b4fcbaSJoe Thornber *error = "could not create kcopyd client";
2513c6b4fcbaSJoe Thornber r = PTR_ERR(cache->copier);
2514c6b4fcbaSJoe Thornber goto bad;
2515c6b4fcbaSJoe Thornber }
2516c6b4fcbaSJoe Thornber
2517b29d4986SJoe Thornber cache->wq = alloc_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM, 0);
2518c6b4fcbaSJoe Thornber if (!cache->wq) {
2519c6b4fcbaSJoe Thornber *error = "could not create workqueue for metadata object";
2520c6b4fcbaSJoe Thornber goto bad;
2521c6b4fcbaSJoe Thornber }
2522b29d4986SJoe Thornber INIT_WORK(&cache->deferred_bio_worker, process_deferred_bios);
2523b29d4986SJoe Thornber INIT_WORK(&cache->migration_worker, check_migrations);
2524c6b4fcbaSJoe Thornber INIT_DELAYED_WORK(&cache->waker, do_waker);
2525c6b4fcbaSJoe Thornber
2526b29d4986SJoe Thornber cache->prison = dm_bio_prison_create_v2(cache->wq);
2527c6b4fcbaSJoe Thornber if (!cache->prison) {
2528c6b4fcbaSJoe Thornber *error = "could not create bio prison";
2529c6b4fcbaSJoe Thornber goto bad;
2530c6b4fcbaSJoe Thornber }
2531c6b4fcbaSJoe Thornber
25326f1c819cSKent Overstreet r = mempool_init_slab_pool(&cache->migration_pool, MIGRATION_POOL_SIZE,
2533c6b4fcbaSJoe Thornber migration_cache);
25346f1c819cSKent Overstreet if (r) {
2535c6b4fcbaSJoe Thornber *error = "Error creating cache's migration mempool";
2536c6b4fcbaSJoe Thornber goto bad;
2537c6b4fcbaSJoe Thornber }
2538c6b4fcbaSJoe Thornber
2539c6b4fcbaSJoe Thornber cache->need_tick_bio = true;
2540c6b4fcbaSJoe Thornber cache->sized = false;
254165790ff9SJoe Thornber cache->invalidate = false;
2542c6b4fcbaSJoe Thornber cache->commit_requested = false;
2543c6b4fcbaSJoe Thornber cache->loaded_mappings = false;
2544c6b4fcbaSJoe Thornber cache->loaded_discards = false;
2545c6b4fcbaSJoe Thornber
2546c6b4fcbaSJoe Thornber load_stats(cache);
2547c6b4fcbaSJoe Thornber
2548c6b4fcbaSJoe Thornber atomic_set(&cache->stats.demotion, 0);
2549c6b4fcbaSJoe Thornber atomic_set(&cache->stats.promotion, 0);
2550c6b4fcbaSJoe Thornber atomic_set(&cache->stats.copies_avoided, 0);
2551c6b4fcbaSJoe Thornber atomic_set(&cache->stats.cache_cell_clash, 0);
2552c6b4fcbaSJoe Thornber atomic_set(&cache->stats.commit_count, 0);
2553c6b4fcbaSJoe Thornber atomic_set(&cache->stats.discard_count, 0);
2554c6b4fcbaSJoe Thornber
255565790ff9SJoe Thornber spin_lock_init(&cache->invalidation_lock);
255665790ff9SJoe Thornber INIT_LIST_HEAD(&cache->invalidation_requests);
255765790ff9SJoe Thornber
2558b29d4986SJoe Thornber batcher_init(&cache->committer, commit_op, cache,
2559b29d4986SJoe Thornber issue_op, cache, cache->wq);
2560dc4fa29fSMike Snitzer dm_iot_init(&cache->tracker);
2561066dbaa3SJoe Thornber
2562b29d4986SJoe Thornber init_rwsem(&cache->background_work_lock);
2563b29d4986SJoe Thornber prevent_background_work(cache);
2564b29d4986SJoe Thornber
2565c6b4fcbaSJoe Thornber *result = cache;
2566c6b4fcbaSJoe Thornber return 0;
2567c6b4fcbaSJoe Thornber bad:
2568c6b4fcbaSJoe Thornber destroy(cache);
2569c6b4fcbaSJoe Thornber return r;
2570c6b4fcbaSJoe Thornber }
2571c6b4fcbaSJoe Thornber
copy_ctr_args(struct cache * cache,int argc,const char ** argv)2572c6b4fcbaSJoe Thornber static int copy_ctr_args(struct cache *cache, int argc, const char **argv)
2573c6b4fcbaSJoe Thornber {
257486a3238cSHeinz Mauelshagen unsigned int i;
2575c6b4fcbaSJoe Thornber const char **copy;
2576c6b4fcbaSJoe Thornber
2577c6b4fcbaSJoe Thornber copy = kcalloc(argc, sizeof(*copy), GFP_KERNEL);
2578c6b4fcbaSJoe Thornber if (!copy)
2579c6b4fcbaSJoe Thornber return -ENOMEM;
2580c6b4fcbaSJoe Thornber for (i = 0; i < argc; i++) {
2581c6b4fcbaSJoe Thornber copy[i] = kstrdup(argv[i], GFP_KERNEL);
2582c6b4fcbaSJoe Thornber if (!copy[i]) {
2583c6b4fcbaSJoe Thornber while (i--)
2584c6b4fcbaSJoe Thornber kfree(copy[i]);
2585c6b4fcbaSJoe Thornber kfree(copy);
2586c6b4fcbaSJoe Thornber return -ENOMEM;
2587c6b4fcbaSJoe Thornber }
2588c6b4fcbaSJoe Thornber }
2589c6b4fcbaSJoe Thornber
2590c6b4fcbaSJoe Thornber cache->nr_ctr_args = argc;
2591c6b4fcbaSJoe Thornber cache->ctr_args = copy;
2592c6b4fcbaSJoe Thornber
2593c6b4fcbaSJoe Thornber return 0;
2594c6b4fcbaSJoe Thornber }
2595c6b4fcbaSJoe Thornber
cache_ctr(struct dm_target * ti,unsigned int argc,char ** argv)259686a3238cSHeinz Mauelshagen static int cache_ctr(struct dm_target *ti, unsigned int argc, char **argv)
2597c6b4fcbaSJoe Thornber {
2598c6b4fcbaSJoe Thornber int r = -EINVAL;
2599c6b4fcbaSJoe Thornber struct cache_args *ca;
2600c6b4fcbaSJoe Thornber struct cache *cache = NULL;
2601c6b4fcbaSJoe Thornber
2602c6b4fcbaSJoe Thornber ca = kzalloc(sizeof(*ca), GFP_KERNEL);
2603c6b4fcbaSJoe Thornber if (!ca) {
2604c6b4fcbaSJoe Thornber ti->error = "Error allocating memory for cache";
2605c6b4fcbaSJoe Thornber return -ENOMEM;
2606c6b4fcbaSJoe Thornber }
2607c6b4fcbaSJoe Thornber ca->ti = ti;
2608c6b4fcbaSJoe Thornber
2609c6b4fcbaSJoe Thornber r = parse_cache_args(ca, argc, argv, &ti->error);
2610c6b4fcbaSJoe Thornber if (r)
2611c6b4fcbaSJoe Thornber goto out;
2612c6b4fcbaSJoe Thornber
2613c6b4fcbaSJoe Thornber r = cache_create(ca, &cache);
2614617a0b89SHeinz Mauelshagen if (r)
2615617a0b89SHeinz Mauelshagen goto out;
2616c6b4fcbaSJoe Thornber
2617c6b4fcbaSJoe Thornber r = copy_ctr_args(cache, argc - 3, (const char **)argv + 3);
2618c6b4fcbaSJoe Thornber if (r) {
2619c6b4fcbaSJoe Thornber destroy(cache);
2620c6b4fcbaSJoe Thornber goto out;
2621c6b4fcbaSJoe Thornber }
2622c6b4fcbaSJoe Thornber
2623c6b4fcbaSJoe Thornber ti->private = cache;
2624c6b4fcbaSJoe Thornber out:
2625c6b4fcbaSJoe Thornber destroy_cache_args(ca);
2626c6b4fcbaSJoe Thornber return r;
2627c6b4fcbaSJoe Thornber }
2628c6b4fcbaSJoe Thornber
2629651f5fa2SJoe Thornber /*----------------------------------------------------------------*/
2630651f5fa2SJoe Thornber
cache_map(struct dm_target * ti,struct bio * bio)2631651f5fa2SJoe Thornber static int cache_map(struct dm_target *ti, struct bio *bio)
2632c6b4fcbaSJoe Thornber {
2633651f5fa2SJoe Thornber struct cache *cache = ti->private;
2634651f5fa2SJoe Thornber
2635c6b4fcbaSJoe Thornber int r;
2636b29d4986SJoe Thornber bool commit_needed;
2637c6b4fcbaSJoe Thornber dm_oblock_t block = get_bio_block(cache, bio);
2638fb4100aeSJoe Thornber
2639693b960eSMike Snitzer init_per_bio_data(bio);
2640e893fba9SHeinz Mauelshagen if (unlikely(from_oblock(block) >= from_oblock(cache->origin_blocks))) {
2641c6b4fcbaSJoe Thornber /*
2642c6b4fcbaSJoe Thornber * This can only occur if the io goes to a partial block at
2643c6b4fcbaSJoe Thornber * the end of the origin device. We don't cache these.
2644c6b4fcbaSJoe Thornber * Just remap to the origin and carry on.
2645c6b4fcbaSJoe Thornber */
2646e893fba9SHeinz Mauelshagen remap_to_origin(cache, bio);
2647651f5fa2SJoe Thornber accounted_begin(cache, bio);
2648c6b4fcbaSJoe Thornber return DM_MAPIO_REMAPPED;
2649c6b4fcbaSJoe Thornber }
2650c6b4fcbaSJoe Thornber
2651651f5fa2SJoe Thornber if (discard_or_flush(bio)) {
2652c6b4fcbaSJoe Thornber defer_bio(cache, bio);
2653c6b4fcbaSJoe Thornber return DM_MAPIO_SUBMITTED;
2654c6b4fcbaSJoe Thornber }
2655c6b4fcbaSJoe Thornber
2656b29d4986SJoe Thornber r = map_bio(cache, bio, block, &commit_needed);
2657b29d4986SJoe Thornber if (commit_needed)
2658b29d4986SJoe Thornber schedule_commit(&cache->committer);
2659c6b4fcbaSJoe Thornber
26602ee57d58SJoe Thornber return r;
2661c6b4fcbaSJoe Thornber }
2662c6b4fcbaSJoe Thornber
cache_end_io(struct dm_target * ti,struct bio * bio,blk_status_t * error)2663693b960eSMike Snitzer static int cache_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *error)
2664c6b4fcbaSJoe Thornber {
2665c6b4fcbaSJoe Thornber struct cache *cache = ti->private;
2666c6b4fcbaSJoe Thornber unsigned long flags;
2667693b960eSMike Snitzer struct per_bio_data *pb = get_per_bio_data(bio);
2668c6b4fcbaSJoe Thornber
2669c6b4fcbaSJoe Thornber if (pb->tick) {
2670fba10109SJoe Thornber policy_tick(cache->policy, false);
2671c6b4fcbaSJoe Thornber
2672c6b4fcbaSJoe Thornber spin_lock_irqsave(&cache->lock, flags);
2673c6b4fcbaSJoe Thornber cache->need_tick_bio = true;
2674c6b4fcbaSJoe Thornber spin_unlock_irqrestore(&cache->lock, flags);
2675c6b4fcbaSJoe Thornber }
2676c6b4fcbaSJoe Thornber
2677b29d4986SJoe Thornber bio_drop_shared_lock(cache, bio);
2678066dbaa3SJoe Thornber accounted_complete(cache, bio);
2679c6b4fcbaSJoe Thornber
26801be56909SChristoph Hellwig return DM_ENDIO_DONE;
2681c6b4fcbaSJoe Thornber }
2682c6b4fcbaSJoe Thornber
write_dirty_bitset(struct cache * cache)2683c6b4fcbaSJoe Thornber static int write_dirty_bitset(struct cache *cache)
2684c6b4fcbaSJoe Thornber {
2685629d0a8aSJoe Thornber int r;
2686c6b4fcbaSJoe Thornber
2687028ae9f7SJoe Thornber if (get_cache_mode(cache) >= CM_READ_ONLY)
2688028ae9f7SJoe Thornber return -EINVAL;
2689028ae9f7SJoe Thornber
2690629d0a8aSJoe Thornber r = dm_cache_set_dirty_bits(cache->cmd, from_cblock(cache->cache_size), cache->dirty_bitset);
2691629d0a8aSJoe Thornber if (r)
2692629d0a8aSJoe Thornber metadata_operation_failed(cache, "dm_cache_set_dirty_bits", r);
2693c6b4fcbaSJoe Thornber
2694629d0a8aSJoe Thornber return r;
2695c6b4fcbaSJoe Thornber }
2696c6b4fcbaSJoe Thornber
write_discard_bitset(struct cache * cache)2697c6b4fcbaSJoe Thornber static int write_discard_bitset(struct cache *cache)
2698c6b4fcbaSJoe Thornber {
269986a3238cSHeinz Mauelshagen unsigned int i, r;
2700c6b4fcbaSJoe Thornber
2701028ae9f7SJoe Thornber if (get_cache_mode(cache) >= CM_READ_ONLY)
2702028ae9f7SJoe Thornber return -EINVAL;
2703028ae9f7SJoe Thornber
27041bad9bc4SJoe Thornber r = dm_cache_discard_bitset_resize(cache->cmd, cache->discard_block_size,
27051bad9bc4SJoe Thornber cache->discard_nr_blocks);
2706c6b4fcbaSJoe Thornber if (r) {
2707b61d9509SMike Snitzer DMERR("%s: could not resize on-disk discard bitset", cache_device_name(cache));
2708028ae9f7SJoe Thornber metadata_operation_failed(cache, "dm_cache_discard_bitset_resize", r);
2709c6b4fcbaSJoe Thornber return r;
2710c6b4fcbaSJoe Thornber }
2711c6b4fcbaSJoe Thornber
27121bad9bc4SJoe Thornber for (i = 0; i < from_dblock(cache->discard_nr_blocks); i++) {
27131bad9bc4SJoe Thornber r = dm_cache_set_discard(cache->cmd, to_dblock(i),
27141bad9bc4SJoe Thornber is_discarded(cache, to_dblock(i)));
2715028ae9f7SJoe Thornber if (r) {
2716028ae9f7SJoe Thornber metadata_operation_failed(cache, "dm_cache_set_discard", r);
2717028ae9f7SJoe Thornber return r;
2718028ae9f7SJoe Thornber }
2719028ae9f7SJoe Thornber }
2720028ae9f7SJoe Thornber
2721028ae9f7SJoe Thornber return 0;
2722028ae9f7SJoe Thornber }
2723028ae9f7SJoe Thornber
write_hints(struct cache * cache)2724028ae9f7SJoe Thornber static int write_hints(struct cache *cache)
2725028ae9f7SJoe Thornber {
2726028ae9f7SJoe Thornber int r;
2727028ae9f7SJoe Thornber
2728028ae9f7SJoe Thornber if (get_cache_mode(cache) >= CM_READ_ONLY)
2729028ae9f7SJoe Thornber return -EINVAL;
2730028ae9f7SJoe Thornber
2731028ae9f7SJoe Thornber r = dm_cache_write_hints(cache->cmd, cache->policy);
2732028ae9f7SJoe Thornber if (r) {
2733028ae9f7SJoe Thornber metadata_operation_failed(cache, "dm_cache_write_hints", r);
2734c6b4fcbaSJoe Thornber return r;
2735c6b4fcbaSJoe Thornber }
2736c6b4fcbaSJoe Thornber
2737c6b4fcbaSJoe Thornber return 0;
2738c6b4fcbaSJoe Thornber }
2739c6b4fcbaSJoe Thornber
2740c6b4fcbaSJoe Thornber /*
2741c6b4fcbaSJoe Thornber * returns true on success
2742c6b4fcbaSJoe Thornber */
sync_metadata(struct cache * cache)2743c6b4fcbaSJoe Thornber static bool sync_metadata(struct cache *cache)
2744c6b4fcbaSJoe Thornber {
2745c6b4fcbaSJoe Thornber int r1, r2, r3, r4;
2746c6b4fcbaSJoe Thornber
2747c6b4fcbaSJoe Thornber r1 = write_dirty_bitset(cache);
2748c6b4fcbaSJoe Thornber if (r1)
2749b61d9509SMike Snitzer DMERR("%s: could not write dirty bitset", cache_device_name(cache));
2750c6b4fcbaSJoe Thornber
2751c6b4fcbaSJoe Thornber r2 = write_discard_bitset(cache);
2752c6b4fcbaSJoe Thornber if (r2)
2753b61d9509SMike Snitzer DMERR("%s: could not write discard bitset", cache_device_name(cache));
2754c6b4fcbaSJoe Thornber
2755c6b4fcbaSJoe Thornber save_stats(cache);
2756c6b4fcbaSJoe Thornber
2757028ae9f7SJoe Thornber r3 = write_hints(cache);
2758c6b4fcbaSJoe Thornber if (r3)
2759b61d9509SMike Snitzer DMERR("%s: could not write hints", cache_device_name(cache));
2760c6b4fcbaSJoe Thornber
2761c6b4fcbaSJoe Thornber /*
2762c6b4fcbaSJoe Thornber * If writing the above metadata failed, we still commit, but don't
2763c6b4fcbaSJoe Thornber * set the clean shutdown flag. This will effectively force every
2764c6b4fcbaSJoe Thornber * dirty bit to be set on reload.
2765c6b4fcbaSJoe Thornber */
2766028ae9f7SJoe Thornber r4 = commit(cache, !r1 && !r2 && !r3);
2767c6b4fcbaSJoe Thornber if (r4)
2768b61d9509SMike Snitzer DMERR("%s: could not write cache metadata", cache_device_name(cache));
2769c6b4fcbaSJoe Thornber
2770c6b4fcbaSJoe Thornber return !r1 && !r2 && !r3 && !r4;
2771c6b4fcbaSJoe Thornber }
2772c6b4fcbaSJoe Thornber
cache_postsuspend(struct dm_target * ti)2773c6b4fcbaSJoe Thornber static void cache_postsuspend(struct dm_target *ti)
2774c6b4fcbaSJoe Thornber {
2775c6b4fcbaSJoe Thornber struct cache *cache = ti->private;
2776c6b4fcbaSJoe Thornber
2777b29d4986SJoe Thornber prevent_background_work(cache);
2778b29d4986SJoe Thornber BUG_ON(atomic_read(&cache->nr_io_migrations));
2779b29d4986SJoe Thornber
27807cdf6a0aSMikulas Patocka cancel_delayed_work_sync(&cache->waker);
27817cdf6a0aSMikulas Patocka drain_workqueue(cache->wq);
2782701e03e4SJoe Thornber WARN_ON(cache->tracker.in_flight);
2783b29d4986SJoe Thornber
2784b29d4986SJoe Thornber /*
2785b29d4986SJoe Thornber * If it's a flush suspend there won't be any deferred bios, so this
2786b29d4986SJoe Thornber * call is harmless.
2787b29d4986SJoe Thornber */
2788651f5fa2SJoe Thornber requeue_deferred_bios(cache);
2789c6b4fcbaSJoe Thornber
2790028ae9f7SJoe Thornber if (get_cache_mode(cache) == CM_WRITE)
2791c6b4fcbaSJoe Thornber (void) sync_metadata(cache);
2792c6b4fcbaSJoe Thornber }
2793c6b4fcbaSJoe Thornber
load_mapping(void * context,dm_oblock_t oblock,dm_cblock_t cblock,bool dirty,uint32_t hint,bool hint_valid)2794c6b4fcbaSJoe Thornber static int load_mapping(void *context, dm_oblock_t oblock, dm_cblock_t cblock,
2795c6b4fcbaSJoe Thornber bool dirty, uint32_t hint, bool hint_valid)
2796c6b4fcbaSJoe Thornber {
2797c6b4fcbaSJoe Thornber struct cache *cache = context;
2798c6b4fcbaSJoe Thornber
2799449b668cSJoe Thornber if (dirty) {
2800449b668cSJoe Thornber set_bit(from_cblock(cblock), cache->dirty_bitset);
2801449b668cSJoe Thornber atomic_inc(&cache->nr_dirty);
2802449b668cSJoe Thornber } else
2803449b668cSJoe Thornber clear_bit(from_cblock(cblock), cache->dirty_bitset);
2804449b668cSJoe Thornber
2805b7770923SZheng Yongjun return policy_load_mapping(cache->policy, oblock, cblock, dirty, hint, hint_valid);
2806c6b4fcbaSJoe Thornber }
2807c6b4fcbaSJoe Thornber
28083e2e1c30SJoe Thornber /*
28093e2e1c30SJoe Thornber * The discard block size in the on disk metadata is not
28105c29e784SSteven Lung * necessarily the same as we're currently using. So we have to
28113e2e1c30SJoe Thornber * be careful to only set the discarded attribute if we know it
28123e2e1c30SJoe Thornber * covers a complete block of the new size.
28133e2e1c30SJoe Thornber */
28143e2e1c30SJoe Thornber struct discard_load_info {
28153e2e1c30SJoe Thornber struct cache *cache;
28163e2e1c30SJoe Thornber
28173e2e1c30SJoe Thornber /*
28183e2e1c30SJoe Thornber * These blocks are sized using the on disk dblock size, rather
28193e2e1c30SJoe Thornber * than the current one.
28203e2e1c30SJoe Thornber */
28213e2e1c30SJoe Thornber dm_block_t block_size;
28223e2e1c30SJoe Thornber dm_block_t discard_begin, discard_end;
28233e2e1c30SJoe Thornber };
28243e2e1c30SJoe Thornber
discard_load_info_init(struct cache * cache,struct discard_load_info * li)28253e2e1c30SJoe Thornber static void discard_load_info_init(struct cache *cache,
28263e2e1c30SJoe Thornber struct discard_load_info *li)
28273e2e1c30SJoe Thornber {
28283e2e1c30SJoe Thornber li->cache = cache;
28293e2e1c30SJoe Thornber li->discard_begin = li->discard_end = 0;
28303e2e1c30SJoe Thornber }
28313e2e1c30SJoe Thornber
set_discard_range(struct discard_load_info * li)28323e2e1c30SJoe Thornber static void set_discard_range(struct discard_load_info *li)
28333e2e1c30SJoe Thornber {
28343e2e1c30SJoe Thornber sector_t b, e;
28353e2e1c30SJoe Thornber
28363e2e1c30SJoe Thornber if (li->discard_begin == li->discard_end)
28373e2e1c30SJoe Thornber return;
28383e2e1c30SJoe Thornber
28393e2e1c30SJoe Thornber /*
28403e2e1c30SJoe Thornber * Convert to sectors.
28413e2e1c30SJoe Thornber */
28423e2e1c30SJoe Thornber b = li->discard_begin * li->block_size;
28433e2e1c30SJoe Thornber e = li->discard_end * li->block_size;
28443e2e1c30SJoe Thornber
28453e2e1c30SJoe Thornber /*
28463e2e1c30SJoe Thornber * Then convert back to the current dblock size.
28473e2e1c30SJoe Thornber */
28483e2e1c30SJoe Thornber b = dm_sector_div_up(b, li->cache->discard_block_size);
28493e2e1c30SJoe Thornber sector_div(e, li->cache->discard_block_size);
28503e2e1c30SJoe Thornber
28513e2e1c30SJoe Thornber /*
28523e2e1c30SJoe Thornber * The origin may have shrunk, so we need to check we're still in
28533e2e1c30SJoe Thornber * bounds.
28543e2e1c30SJoe Thornber */
28553e2e1c30SJoe Thornber if (e > from_dblock(li->cache->discard_nr_blocks))
28563e2e1c30SJoe Thornber e = from_dblock(li->cache->discard_nr_blocks);
28573e2e1c30SJoe Thornber
28583e2e1c30SJoe Thornber for (; b < e; b++)
28593e2e1c30SJoe Thornber set_discard(li->cache, to_dblock(b));
28603e2e1c30SJoe Thornber }
28613e2e1c30SJoe Thornber
load_discard(void * context,sector_t discard_block_size,dm_dblock_t dblock,bool discard)2862c6b4fcbaSJoe Thornber static int load_discard(void *context, sector_t discard_block_size,
28631bad9bc4SJoe Thornber dm_dblock_t dblock, bool discard)
2864c6b4fcbaSJoe Thornber {
28653e2e1c30SJoe Thornber struct discard_load_info *li = context;
2866c6b4fcbaSJoe Thornber
28673e2e1c30SJoe Thornber li->block_size = discard_block_size;
28681bad9bc4SJoe Thornber
28693e2e1c30SJoe Thornber if (discard) {
28703e2e1c30SJoe Thornber if (from_dblock(dblock) == li->discard_end)
28713e2e1c30SJoe Thornber /*
28723e2e1c30SJoe Thornber * We're already in a discard range, just extend it.
28733e2e1c30SJoe Thornber */
28743e2e1c30SJoe Thornber li->discard_end = li->discard_end + 1ULL;
28753e2e1c30SJoe Thornber
28763e2e1c30SJoe Thornber else {
28773e2e1c30SJoe Thornber /*
28783e2e1c30SJoe Thornber * Emit the old range and start a new one.
28793e2e1c30SJoe Thornber */
28803e2e1c30SJoe Thornber set_discard_range(li);
28813e2e1c30SJoe Thornber li->discard_begin = from_dblock(dblock);
28823e2e1c30SJoe Thornber li->discard_end = li->discard_begin + 1ULL;
28833e2e1c30SJoe Thornber }
28843e2e1c30SJoe Thornber } else {
28853e2e1c30SJoe Thornber set_discard_range(li);
28863e2e1c30SJoe Thornber li->discard_begin = li->discard_end = 0;
28873e2e1c30SJoe Thornber }
2888c6b4fcbaSJoe Thornber
2889c6b4fcbaSJoe Thornber return 0;
2890c6b4fcbaSJoe Thornber }
2891c6b4fcbaSJoe Thornber
get_cache_dev_size(struct cache * cache)2892f494a9c6SJoe Thornber static dm_cblock_t get_cache_dev_size(struct cache *cache)
2893c6b4fcbaSJoe Thornber {
2894f494a9c6SJoe Thornber sector_t size = get_dev_size(cache->cache_dev);
2895f494a9c6SJoe Thornber (void) sector_div(size, cache->sectors_per_block);
2896f494a9c6SJoe Thornber return to_cblock(size);
2897f494a9c6SJoe Thornber }
2898f494a9c6SJoe Thornber
can_resize(struct cache * cache,dm_cblock_t new_size)2899f494a9c6SJoe Thornber static bool can_resize(struct cache *cache, dm_cblock_t new_size)
2900f494a9c6SJoe Thornber {
29015d07384aSMike Snitzer if (from_cblock(new_size) > from_cblock(cache->cache_size)) {
29025d07384aSMike Snitzer if (cache->sized) {
29035d07384aSMike Snitzer DMERR("%s: unable to extend cache due to missing cache table reload",
29045d07384aSMike Snitzer cache_device_name(cache));
29055d07384aSMike Snitzer return false;
29065d07384aSMike Snitzer }
29075d07384aSMike Snitzer }
2908c6b4fcbaSJoe Thornber
2909c6b4fcbaSJoe Thornber /*
2910f494a9c6SJoe Thornber * We can't drop a dirty block when shrinking the cache.
2911c6b4fcbaSJoe Thornber */
2912f494a9c6SJoe Thornber while (from_cblock(new_size) < from_cblock(cache->cache_size)) {
2913f494a9c6SJoe Thornber new_size = to_cblock(from_cblock(new_size) + 1);
2914f494a9c6SJoe Thornber if (is_dirty(cache, new_size)) {
2915b61d9509SMike Snitzer DMERR("%s: unable to shrink cache; cache block %llu is dirty",
2916b61d9509SMike Snitzer cache_device_name(cache),
2917f494a9c6SJoe Thornber (unsigned long long) from_cblock(new_size));
2918f494a9c6SJoe Thornber return false;
2919f494a9c6SJoe Thornber }
2920f494a9c6SJoe Thornber }
2921f494a9c6SJoe Thornber
2922f494a9c6SJoe Thornber return true;
2923f494a9c6SJoe Thornber }
2924f494a9c6SJoe Thornber
resize_cache_dev(struct cache * cache,dm_cblock_t new_size)2925f494a9c6SJoe Thornber static int resize_cache_dev(struct cache *cache, dm_cblock_t new_size)
2926f494a9c6SJoe Thornber {
2927f494a9c6SJoe Thornber int r;
2928c6b4fcbaSJoe Thornber
292908844800SVincent Pelletier r = dm_cache_resize(cache->cmd, new_size);
2930c6b4fcbaSJoe Thornber if (r) {
2931b61d9509SMike Snitzer DMERR("%s: could not resize cache metadata", cache_device_name(cache));
2932028ae9f7SJoe Thornber metadata_operation_failed(cache, "dm_cache_resize", r);
2933c6b4fcbaSJoe Thornber return r;
2934c6b4fcbaSJoe Thornber }
2935c6b4fcbaSJoe Thornber
2936d1d9220cSJoe Thornber set_cache_size(cache, new_size);
2937f494a9c6SJoe Thornber
2938f494a9c6SJoe Thornber return 0;
2939f494a9c6SJoe Thornber }
2940f494a9c6SJoe Thornber
cache_preresume(struct dm_target * ti)2941f494a9c6SJoe Thornber static int cache_preresume(struct dm_target *ti)
2942f494a9c6SJoe Thornber {
2943f494a9c6SJoe Thornber int r = 0;
2944f494a9c6SJoe Thornber struct cache *cache = ti->private;
2945f494a9c6SJoe Thornber dm_cblock_t csize = get_cache_dev_size(cache);
2946f494a9c6SJoe Thornber
2947f494a9c6SJoe Thornber /*
2948f494a9c6SJoe Thornber * Check to see if the cache has resized.
2949f494a9c6SJoe Thornber */
2950f494a9c6SJoe Thornber if (!cache->sized) {
2951f494a9c6SJoe Thornber r = resize_cache_dev(cache, csize);
2952f494a9c6SJoe Thornber if (r)
2953f494a9c6SJoe Thornber return r;
2954f494a9c6SJoe Thornber
2955c6b4fcbaSJoe Thornber cache->sized = true;
2956f494a9c6SJoe Thornber
2957f494a9c6SJoe Thornber } else if (csize != cache->cache_size) {
2958f494a9c6SJoe Thornber if (!can_resize(cache, csize))
2959f494a9c6SJoe Thornber return -EINVAL;
2960f494a9c6SJoe Thornber
2961f494a9c6SJoe Thornber r = resize_cache_dev(cache, csize);
2962f494a9c6SJoe Thornber if (r)
2963f494a9c6SJoe Thornber return r;
2964c6b4fcbaSJoe Thornber }
2965c6b4fcbaSJoe Thornber
2966c6b4fcbaSJoe Thornber if (!cache->loaded_mappings) {
2967ea2dd8c1SMike Snitzer r = dm_cache_load_mappings(cache->cmd, cache->policy,
2968c6b4fcbaSJoe Thornber load_mapping, cache);
2969c6b4fcbaSJoe Thornber if (r) {
2970b61d9509SMike Snitzer DMERR("%s: could not load cache mappings", cache_device_name(cache));
2971028ae9f7SJoe Thornber metadata_operation_failed(cache, "dm_cache_load_mappings", r);
2972c6b4fcbaSJoe Thornber return r;
2973c6b4fcbaSJoe Thornber }
2974c6b4fcbaSJoe Thornber
2975c6b4fcbaSJoe Thornber cache->loaded_mappings = true;
2976c6b4fcbaSJoe Thornber }
2977c6b4fcbaSJoe Thornber
2978c6b4fcbaSJoe Thornber if (!cache->loaded_discards) {
29793e2e1c30SJoe Thornber struct discard_load_info li;
29803e2e1c30SJoe Thornber
29813e2e1c30SJoe Thornber /*
29823e2e1c30SJoe Thornber * The discard bitset could have been resized, or the
29833e2e1c30SJoe Thornber * discard block size changed. To be safe we start by
29843e2e1c30SJoe Thornber * setting every dblock to not discarded.
29853e2e1c30SJoe Thornber */
29863e2e1c30SJoe Thornber clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks));
29873e2e1c30SJoe Thornber
29883e2e1c30SJoe Thornber discard_load_info_init(cache, &li);
29893e2e1c30SJoe Thornber r = dm_cache_load_discards(cache->cmd, load_discard, &li);
2990c6b4fcbaSJoe Thornber if (r) {
2991b61d9509SMike Snitzer DMERR("%s: could not load origin discards", cache_device_name(cache));
2992028ae9f7SJoe Thornber metadata_operation_failed(cache, "dm_cache_load_discards", r);
2993c6b4fcbaSJoe Thornber return r;
2994c6b4fcbaSJoe Thornber }
29953e2e1c30SJoe Thornber set_discard_range(&li);
2996c6b4fcbaSJoe Thornber
2997c6b4fcbaSJoe Thornber cache->loaded_discards = true;
2998c6b4fcbaSJoe Thornber }
2999c6b4fcbaSJoe Thornber
3000c6b4fcbaSJoe Thornber return r;
3001c6b4fcbaSJoe Thornber }
3002c6b4fcbaSJoe Thornber
cache_resume(struct dm_target * ti)3003c6b4fcbaSJoe Thornber static void cache_resume(struct dm_target *ti)
3004c6b4fcbaSJoe Thornber {
3005c6b4fcbaSJoe Thornber struct cache *cache = ti->private;
3006c6b4fcbaSJoe Thornber
3007c6b4fcbaSJoe Thornber cache->need_tick_bio = true;
3008b29d4986SJoe Thornber allow_background_work(cache);
3009c6b4fcbaSJoe Thornber do_waker(&cache->waker.work);
3010c6b4fcbaSJoe Thornber }
3011c6b4fcbaSJoe Thornber
emit_flags(struct cache * cache,char * result,unsigned int maxlen,ssize_t * sz_ptr)3012de7180ffSMike Snitzer static void emit_flags(struct cache *cache, char *result,
301386a3238cSHeinz Mauelshagen unsigned int maxlen, ssize_t *sz_ptr)
3014de7180ffSMike Snitzer {
3015de7180ffSMike Snitzer ssize_t sz = *sz_ptr;
3016de7180ffSMike Snitzer struct cache_features *cf = &cache->features;
301786a3238cSHeinz Mauelshagen unsigned int count = (cf->metadata_version == 2) + !cf->discard_passdown + 1;
3018de7180ffSMike Snitzer
3019de7180ffSMike Snitzer DMEMIT("%u ", count);
3020de7180ffSMike Snitzer
3021de7180ffSMike Snitzer if (cf->metadata_version == 2)
3022de7180ffSMike Snitzer DMEMIT("metadata2 ");
3023de7180ffSMike Snitzer
3024de7180ffSMike Snitzer if (writethrough_mode(cache))
3025de7180ffSMike Snitzer DMEMIT("writethrough ");
3026de7180ffSMike Snitzer
3027de7180ffSMike Snitzer else if (passthrough_mode(cache))
3028de7180ffSMike Snitzer DMEMIT("passthrough ");
3029de7180ffSMike Snitzer
3030de7180ffSMike Snitzer else if (writeback_mode(cache))
3031de7180ffSMike Snitzer DMEMIT("writeback ");
3032de7180ffSMike Snitzer
3033de7180ffSMike Snitzer else {
3034de7180ffSMike Snitzer DMEMIT("unknown ");
3035de7180ffSMike Snitzer DMERR("%s: internal error: unknown io mode: %d",
3036de7180ffSMike Snitzer cache_device_name(cache), (int) cf->io_mode);
3037de7180ffSMike Snitzer }
3038de7180ffSMike Snitzer
3039de7180ffSMike Snitzer if (!cf->discard_passdown)
3040de7180ffSMike Snitzer DMEMIT("no_discard_passdown ");
3041de7180ffSMike Snitzer
3042de7180ffSMike Snitzer *sz_ptr = sz;
3043de7180ffSMike Snitzer }
3044de7180ffSMike Snitzer
3045c6b4fcbaSJoe Thornber /*
3046c6b4fcbaSJoe Thornber * Status format:
3047c6b4fcbaSJoe Thornber *
30486a388618SMike Snitzer * <metadata block size> <#used metadata blocks>/<#total metadata blocks>
30496a388618SMike Snitzer * <cache block size> <#used cache blocks>/<#total cache blocks>
3050c6b4fcbaSJoe Thornber * <#read hits> <#read misses> <#write hits> <#write misses>
30516a388618SMike Snitzer * <#demotions> <#promotions> <#dirty>
3052c6b4fcbaSJoe Thornber * <#features> <features>*
3053c6b4fcbaSJoe Thornber * <#core args> <core args>
3054255eac20SMike Snitzer * <policy name> <#policy args> <policy args>* <cache metadata mode> <needs_check>
3055c6b4fcbaSJoe Thornber */
cache_status(struct dm_target * ti,status_type_t type,unsigned int status_flags,char * result,unsigned int maxlen)3056c6b4fcbaSJoe Thornber static void cache_status(struct dm_target *ti, status_type_t type,
305786a3238cSHeinz Mauelshagen unsigned int status_flags, char *result, unsigned int maxlen)
3058c6b4fcbaSJoe Thornber {
3059c6b4fcbaSJoe Thornber int r = 0;
306086a3238cSHeinz Mauelshagen unsigned int i;
3061c6b4fcbaSJoe Thornber ssize_t sz = 0;
3062c6b4fcbaSJoe Thornber dm_block_t nr_free_blocks_metadata = 0;
3063c6b4fcbaSJoe Thornber dm_block_t nr_blocks_metadata = 0;
3064c6b4fcbaSJoe Thornber char buf[BDEVNAME_SIZE];
3065c6b4fcbaSJoe Thornber struct cache *cache = ti->private;
3066c6b4fcbaSJoe Thornber dm_cblock_t residency;
3067d14fcf3dSJoe Thornber bool needs_check;
3068c6b4fcbaSJoe Thornber
3069c6b4fcbaSJoe Thornber switch (type) {
3070c6b4fcbaSJoe Thornber case STATUSTYPE_INFO:
3071028ae9f7SJoe Thornber if (get_cache_mode(cache) == CM_FAIL) {
3072028ae9f7SJoe Thornber DMEMIT("Fail");
3073028ae9f7SJoe Thornber break;
3074c6b4fcbaSJoe Thornber }
3075c6b4fcbaSJoe Thornber
3076028ae9f7SJoe Thornber /* Commit to ensure statistics aren't out-of-date */
3077028ae9f7SJoe Thornber if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti))
3078028ae9f7SJoe Thornber (void) commit(cache, false);
3079028ae9f7SJoe Thornber
3080b61d9509SMike Snitzer r = dm_cache_get_free_metadata_block_count(cache->cmd, &nr_free_blocks_metadata);
3081c6b4fcbaSJoe Thornber if (r) {
3082b61d9509SMike Snitzer DMERR("%s: dm_cache_get_free_metadata_block_count returned %d",
3083b61d9509SMike Snitzer cache_device_name(cache), r);
3084c6b4fcbaSJoe Thornber goto err;
3085c6b4fcbaSJoe Thornber }
3086c6b4fcbaSJoe Thornber
3087c6b4fcbaSJoe Thornber r = dm_cache_get_metadata_dev_size(cache->cmd, &nr_blocks_metadata);
3088c6b4fcbaSJoe Thornber if (r) {
3089b61d9509SMike Snitzer DMERR("%s: dm_cache_get_metadata_dev_size returned %d",
3090b61d9509SMike Snitzer cache_device_name(cache), r);
3091c6b4fcbaSJoe Thornber goto err;
3092c6b4fcbaSJoe Thornber }
3093c6b4fcbaSJoe Thornber
3094c6b4fcbaSJoe Thornber residency = policy_residency(cache->policy);
3095c6b4fcbaSJoe Thornber
3096ca763d0aSJoe Thornber DMEMIT("%u %llu/%llu %llu %llu/%llu %u %u %u %u %u %u %lu ",
309786a3238cSHeinz Mauelshagen (unsigned int)DM_CACHE_METADATA_BLOCK_SIZE,
3098c6b4fcbaSJoe Thornber (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata),
3099c6b4fcbaSJoe Thornber (unsigned long long)nr_blocks_metadata,
3100ca763d0aSJoe Thornber (unsigned long long)cache->sectors_per_block,
31016a388618SMike Snitzer (unsigned long long) from_cblock(residency),
31026a388618SMike Snitzer (unsigned long long) from_cblock(cache->cache_size),
310386a3238cSHeinz Mauelshagen (unsigned int) atomic_read(&cache->stats.read_hit),
310486a3238cSHeinz Mauelshagen (unsigned int) atomic_read(&cache->stats.read_miss),
310586a3238cSHeinz Mauelshagen (unsigned int) atomic_read(&cache->stats.write_hit),
310686a3238cSHeinz Mauelshagen (unsigned int) atomic_read(&cache->stats.write_miss),
310786a3238cSHeinz Mauelshagen (unsigned int) atomic_read(&cache->stats.demotion),
310886a3238cSHeinz Mauelshagen (unsigned int) atomic_read(&cache->stats.promotion),
310944fa816bSAnssi Hannula (unsigned long) atomic_read(&cache->nr_dirty));
3110c6b4fcbaSJoe Thornber
3111de7180ffSMike Snitzer emit_flags(cache, result, maxlen, &sz);
3112c6b4fcbaSJoe Thornber
3113c6b4fcbaSJoe Thornber DMEMIT("2 migration_threshold %llu ", (unsigned long long) cache->migration_threshold);
31142e68c4e6SMike Snitzer
31152e68c4e6SMike Snitzer DMEMIT("%s ", dm_cache_policy_get_name(cache->policy));
3116c6b4fcbaSJoe Thornber if (sz < maxlen) {
3117028ae9f7SJoe Thornber r = policy_emit_config_values(cache->policy, result, maxlen, &sz);
3118c6b4fcbaSJoe Thornber if (r)
3119b61d9509SMike Snitzer DMERR("%s: policy_emit_config_values returned %d",
3120b61d9509SMike Snitzer cache_device_name(cache), r);
3121c6b4fcbaSJoe Thornber }
3122c6b4fcbaSJoe Thornber
3123028ae9f7SJoe Thornber if (get_cache_mode(cache) == CM_READ_ONLY)
3124028ae9f7SJoe Thornber DMEMIT("ro ");
3125028ae9f7SJoe Thornber else
3126028ae9f7SJoe Thornber DMEMIT("rw ");
3127028ae9f7SJoe Thornber
3128d14fcf3dSJoe Thornber r = dm_cache_metadata_needs_check(cache->cmd, &needs_check);
3129d14fcf3dSJoe Thornber
3130d14fcf3dSJoe Thornber if (r || needs_check)
3131255eac20SMike Snitzer DMEMIT("needs_check ");
3132255eac20SMike Snitzer else
3133255eac20SMike Snitzer DMEMIT("- ");
3134255eac20SMike Snitzer
3135c6b4fcbaSJoe Thornber break;
3136c6b4fcbaSJoe Thornber
3137c6b4fcbaSJoe Thornber case STATUSTYPE_TABLE:
3138c6b4fcbaSJoe Thornber format_dev_t(buf, cache->metadata_dev->bdev->bd_dev);
3139c6b4fcbaSJoe Thornber DMEMIT("%s ", buf);
3140c6b4fcbaSJoe Thornber format_dev_t(buf, cache->cache_dev->bdev->bd_dev);
3141c6b4fcbaSJoe Thornber DMEMIT("%s ", buf);
3142c6b4fcbaSJoe Thornber format_dev_t(buf, cache->origin_dev->bdev->bd_dev);
3143c6b4fcbaSJoe Thornber DMEMIT("%s", buf);
3144c6b4fcbaSJoe Thornber
3145c6b4fcbaSJoe Thornber for (i = 0; i < cache->nr_ctr_args - 1; i++)
3146c6b4fcbaSJoe Thornber DMEMIT(" %s", cache->ctr_args[i]);
3147c6b4fcbaSJoe Thornber if (cache->nr_ctr_args)
3148c6b4fcbaSJoe Thornber DMEMIT(" %s", cache->ctr_args[cache->nr_ctr_args - 1]);
31498ec45662STushar Sugandhi break;
31508ec45662STushar Sugandhi
31518ec45662STushar Sugandhi case STATUSTYPE_IMA:
31528ec45662STushar Sugandhi DMEMIT_TARGET_NAME_VERSION(ti->type);
31538ec45662STushar Sugandhi if (get_cache_mode(cache) == CM_FAIL)
31548ec45662STushar Sugandhi DMEMIT(",metadata_mode=fail");
31558ec45662STushar Sugandhi else if (get_cache_mode(cache) == CM_READ_ONLY)
31568ec45662STushar Sugandhi DMEMIT(",metadata_mode=ro");
31578ec45662STushar Sugandhi else
31588ec45662STushar Sugandhi DMEMIT(",metadata_mode=rw");
31598ec45662STushar Sugandhi
31608ec45662STushar Sugandhi format_dev_t(buf, cache->metadata_dev->bdev->bd_dev);
31618ec45662STushar Sugandhi DMEMIT(",cache_metadata_device=%s", buf);
31628ec45662STushar Sugandhi format_dev_t(buf, cache->cache_dev->bdev->bd_dev);
31638ec45662STushar Sugandhi DMEMIT(",cache_device=%s", buf);
31648ec45662STushar Sugandhi format_dev_t(buf, cache->origin_dev->bdev->bd_dev);
31658ec45662STushar Sugandhi DMEMIT(",cache_origin_device=%s", buf);
31668ec45662STushar Sugandhi DMEMIT(",writethrough=%c", writethrough_mode(cache) ? 'y' : 'n');
31678ec45662STushar Sugandhi DMEMIT(",writeback=%c", writeback_mode(cache) ? 'y' : 'n');
31688ec45662STushar Sugandhi DMEMIT(",passthrough=%c", passthrough_mode(cache) ? 'y' : 'n');
31698ec45662STushar Sugandhi DMEMIT(",metadata2=%c", cache->features.metadata_version == 2 ? 'y' : 'n');
31708ec45662STushar Sugandhi DMEMIT(",no_discard_passdown=%c", cache->features.discard_passdown ? 'n' : 'y');
31718ec45662STushar Sugandhi DMEMIT(";");
31728ec45662STushar Sugandhi break;
3173c6b4fcbaSJoe Thornber }
3174c6b4fcbaSJoe Thornber
3175c6b4fcbaSJoe Thornber return;
3176c6b4fcbaSJoe Thornber
3177c6b4fcbaSJoe Thornber err:
3178c6b4fcbaSJoe Thornber DMEMIT("Error");
3179c6b4fcbaSJoe Thornber }
3180c6b4fcbaSJoe Thornber
3181c6b4fcbaSJoe Thornber /*
3182b29d4986SJoe Thornber * Defines a range of cblocks, begin to (end - 1) are in the range. end is
3183b29d4986SJoe Thornber * the one-past-the-end value.
3184b29d4986SJoe Thornber */
3185b29d4986SJoe Thornber struct cblock_range {
3186b29d4986SJoe Thornber dm_cblock_t begin;
3187b29d4986SJoe Thornber dm_cblock_t end;
3188b29d4986SJoe Thornber };
3189b29d4986SJoe Thornber
3190b29d4986SJoe Thornber /*
319165790ff9SJoe Thornber * A cache block range can take two forms:
319265790ff9SJoe Thornber *
319365790ff9SJoe Thornber * i) A single cblock, eg. '3456'
3194b29d4986SJoe Thornber * ii) A begin and end cblock with a dash between, eg. 123-234
319565790ff9SJoe Thornber */
parse_cblock_range(struct cache * cache,const char * str,struct cblock_range * result)319665790ff9SJoe Thornber static int parse_cblock_range(struct cache *cache, const char *str,
319765790ff9SJoe Thornber struct cblock_range *result)
319865790ff9SJoe Thornber {
319965790ff9SJoe Thornber char dummy;
320065790ff9SJoe Thornber uint64_t b, e;
320165790ff9SJoe Thornber int r;
320265790ff9SJoe Thornber
320365790ff9SJoe Thornber /*
320465790ff9SJoe Thornber * Try and parse form (ii) first.
320565790ff9SJoe Thornber */
320665790ff9SJoe Thornber r = sscanf(str, "%llu-%llu%c", &b, &e, &dummy);
320765790ff9SJoe Thornber if (r < 0)
320865790ff9SJoe Thornber return r;
320965790ff9SJoe Thornber
321065790ff9SJoe Thornber if (r == 2) {
321165790ff9SJoe Thornber result->begin = to_cblock(b);
321265790ff9SJoe Thornber result->end = to_cblock(e);
321365790ff9SJoe Thornber return 0;
321465790ff9SJoe Thornber }
321565790ff9SJoe Thornber
321665790ff9SJoe Thornber /*
321765790ff9SJoe Thornber * That didn't work, try form (i).
321865790ff9SJoe Thornber */
321965790ff9SJoe Thornber r = sscanf(str, "%llu%c", &b, &dummy);
322065790ff9SJoe Thornber if (r < 0)
322165790ff9SJoe Thornber return r;
322265790ff9SJoe Thornber
322365790ff9SJoe Thornber if (r == 1) {
322465790ff9SJoe Thornber result->begin = to_cblock(b);
322565790ff9SJoe Thornber result->end = to_cblock(from_cblock(result->begin) + 1u);
322665790ff9SJoe Thornber return 0;
322765790ff9SJoe Thornber }
322865790ff9SJoe Thornber
3229b61d9509SMike Snitzer DMERR("%s: invalid cblock range '%s'", cache_device_name(cache), str);
323065790ff9SJoe Thornber return -EINVAL;
323165790ff9SJoe Thornber }
323265790ff9SJoe Thornber
validate_cblock_range(struct cache * cache,struct cblock_range * range)323365790ff9SJoe Thornber static int validate_cblock_range(struct cache *cache, struct cblock_range *range)
323465790ff9SJoe Thornber {
323565790ff9SJoe Thornber uint64_t b = from_cblock(range->begin);
323665790ff9SJoe Thornber uint64_t e = from_cblock(range->end);
323765790ff9SJoe Thornber uint64_t n = from_cblock(cache->cache_size);
323865790ff9SJoe Thornber
323965790ff9SJoe Thornber if (b >= n) {
3240b61d9509SMike Snitzer DMERR("%s: begin cblock out of range: %llu >= %llu",
3241b61d9509SMike Snitzer cache_device_name(cache), b, n);
324265790ff9SJoe Thornber return -EINVAL;
324365790ff9SJoe Thornber }
324465790ff9SJoe Thornber
324565790ff9SJoe Thornber if (e > n) {
3246b61d9509SMike Snitzer DMERR("%s: end cblock out of range: %llu > %llu",
3247b61d9509SMike Snitzer cache_device_name(cache), e, n);
324865790ff9SJoe Thornber return -EINVAL;
324965790ff9SJoe Thornber }
325065790ff9SJoe Thornber
325165790ff9SJoe Thornber if (b >= e) {
3252b61d9509SMike Snitzer DMERR("%s: invalid cblock range: %llu >= %llu",
3253b61d9509SMike Snitzer cache_device_name(cache), b, e);
325465790ff9SJoe Thornber return -EINVAL;
325565790ff9SJoe Thornber }
325665790ff9SJoe Thornber
325765790ff9SJoe Thornber return 0;
325865790ff9SJoe Thornber }
325965790ff9SJoe Thornber
cblock_succ(dm_cblock_t b)3260b29d4986SJoe Thornber static inline dm_cblock_t cblock_succ(dm_cblock_t b)
3261b29d4986SJoe Thornber {
3262b29d4986SJoe Thornber return to_cblock(from_cblock(b) + 1);
3263b29d4986SJoe Thornber }
3264b29d4986SJoe Thornber
request_invalidation(struct cache * cache,struct cblock_range * range)326565790ff9SJoe Thornber static int request_invalidation(struct cache *cache, struct cblock_range *range)
326665790ff9SJoe Thornber {
3267b29d4986SJoe Thornber int r = 0;
326865790ff9SJoe Thornber
3269b29d4986SJoe Thornber /*
3270b29d4986SJoe Thornber * We don't need to do any locking here because we know we're in
3271b29d4986SJoe Thornber * passthrough mode. There's is potential for a race between an
3272b29d4986SJoe Thornber * invalidation triggered by an io and an invalidation message. This
3273b29d4986SJoe Thornber * is harmless, we must not worry if the policy call fails.
3274b29d4986SJoe Thornber */
3275b29d4986SJoe Thornber while (range->begin != range->end) {
3276b29d4986SJoe Thornber r = invalidate_cblock(cache, range->begin);
3277b29d4986SJoe Thornber if (r)
3278b29d4986SJoe Thornber return r;
327965790ff9SJoe Thornber
3280b29d4986SJoe Thornber range->begin = cblock_succ(range->begin);
3281b29d4986SJoe Thornber }
328265790ff9SJoe Thornber
3283b29d4986SJoe Thornber cache->commit_requested = true;
3284b29d4986SJoe Thornber return r;
328565790ff9SJoe Thornber }
328665790ff9SJoe Thornber
process_invalidate_cblocks_message(struct cache * cache,unsigned int count,const char ** cblock_ranges)328786a3238cSHeinz Mauelshagen static int process_invalidate_cblocks_message(struct cache *cache, unsigned int count,
328865790ff9SJoe Thornber const char **cblock_ranges)
328965790ff9SJoe Thornber {
329065790ff9SJoe Thornber int r = 0;
329186a3238cSHeinz Mauelshagen unsigned int i;
329265790ff9SJoe Thornber struct cblock_range range;
329365790ff9SJoe Thornber
32948e3c3827SMike Snitzer if (!passthrough_mode(cache)) {
3295b61d9509SMike Snitzer DMERR("%s: cache has to be in passthrough mode for invalidation",
3296b61d9509SMike Snitzer cache_device_name(cache));
329765790ff9SJoe Thornber return -EPERM;
329865790ff9SJoe Thornber }
329965790ff9SJoe Thornber
330065790ff9SJoe Thornber for (i = 0; i < count; i++) {
330165790ff9SJoe Thornber r = parse_cblock_range(cache, cblock_ranges[i], &range);
330265790ff9SJoe Thornber if (r)
330365790ff9SJoe Thornber break;
330465790ff9SJoe Thornber
330565790ff9SJoe Thornber r = validate_cblock_range(cache, &range);
330665790ff9SJoe Thornber if (r)
330765790ff9SJoe Thornber break;
330865790ff9SJoe Thornber
330965790ff9SJoe Thornber /*
331065790ff9SJoe Thornber * Pass begin and end origin blocks to the worker and wake it.
331165790ff9SJoe Thornber */
331265790ff9SJoe Thornber r = request_invalidation(cache, &range);
331365790ff9SJoe Thornber if (r)
331465790ff9SJoe Thornber break;
331565790ff9SJoe Thornber }
331665790ff9SJoe Thornber
331765790ff9SJoe Thornber return r;
331865790ff9SJoe Thornber }
331965790ff9SJoe Thornber
332065790ff9SJoe Thornber /*
332165790ff9SJoe Thornber * Supports
332265790ff9SJoe Thornber * "<key> <value>"
332365790ff9SJoe Thornber * and
332465790ff9SJoe Thornber * "invalidate_cblocks [(<begin>)|(<begin>-<end>)]*
3325c6b4fcbaSJoe Thornber *
3326c6b4fcbaSJoe Thornber * The key migration_threshold is supported by the cache target core.
3327c6b4fcbaSJoe Thornber */
cache_message(struct dm_target * ti,unsigned int argc,char ** argv,char * result,unsigned int maxlen)332886a3238cSHeinz Mauelshagen static int cache_message(struct dm_target *ti, unsigned int argc, char **argv,
332986a3238cSHeinz Mauelshagen char *result, unsigned int maxlen)
3330c6b4fcbaSJoe Thornber {
3331c6b4fcbaSJoe Thornber struct cache *cache = ti->private;
3332c6b4fcbaSJoe Thornber
333365790ff9SJoe Thornber if (!argc)
333465790ff9SJoe Thornber return -EINVAL;
333565790ff9SJoe Thornber
3336028ae9f7SJoe Thornber if (get_cache_mode(cache) >= CM_READ_ONLY) {
3337b61d9509SMike Snitzer DMERR("%s: unable to service cache target messages in READ_ONLY or FAIL mode",
3338b61d9509SMike Snitzer cache_device_name(cache));
3339028ae9f7SJoe Thornber return -EOPNOTSUPP;
3340028ae9f7SJoe Thornber }
3341028ae9f7SJoe Thornber
33427b6b2bc9SMike Snitzer if (!strcasecmp(argv[0], "invalidate_cblocks"))
334365790ff9SJoe Thornber return process_invalidate_cblocks_message(cache, argc - 1, (const char **) argv + 1);
334465790ff9SJoe Thornber
3345c6b4fcbaSJoe Thornber if (argc != 2)
3346c6b4fcbaSJoe Thornber return -EINVAL;
3347c6b4fcbaSJoe Thornber
33482f14f4b5SJoe Thornber return set_config_value(cache, argv[0], argv[1]);
3349c6b4fcbaSJoe Thornber }
3350c6b4fcbaSJoe Thornber
cache_iterate_devices(struct dm_target * ti,iterate_devices_callout_fn fn,void * data)3351c6b4fcbaSJoe Thornber static int cache_iterate_devices(struct dm_target *ti,
3352c6b4fcbaSJoe Thornber iterate_devices_callout_fn fn, void *data)
3353c6b4fcbaSJoe Thornber {
3354c6b4fcbaSJoe Thornber int r = 0;
3355c6b4fcbaSJoe Thornber struct cache *cache = ti->private;
3356c6b4fcbaSJoe Thornber
3357c6b4fcbaSJoe Thornber r = fn(ti, cache->cache_dev, 0, get_dev_size(cache->cache_dev), data);
3358c6b4fcbaSJoe Thornber if (!r)
3359c6b4fcbaSJoe Thornber r = fn(ti, cache->origin_dev, 0, ti->len, data);
3360c6b4fcbaSJoe Thornber
3361c6b4fcbaSJoe Thornber return r;
3362c6b4fcbaSJoe Thornber }
3363c6b4fcbaSJoe Thornber
3364de7180ffSMike Snitzer /*
3365de7180ffSMike Snitzer * If discard_passdown was enabled verify that the origin device
3366de7180ffSMike Snitzer * supports discards. Disable discard_passdown if not.
3367de7180ffSMike Snitzer */
disable_passdown_if_not_supported(struct cache * cache)3368de7180ffSMike Snitzer static void disable_passdown_if_not_supported(struct cache *cache)
3369de7180ffSMike Snitzer {
3370de7180ffSMike Snitzer struct block_device *origin_bdev = cache->origin_dev->bdev;
3371de7180ffSMike Snitzer struct queue_limits *origin_limits = &bdev_get_queue(origin_bdev)->limits;
3372de7180ffSMike Snitzer const char *reason = NULL;
3373de7180ffSMike Snitzer
3374de7180ffSMike Snitzer if (!cache->features.discard_passdown)
3375de7180ffSMike Snitzer return;
3376de7180ffSMike Snitzer
337770200574SChristoph Hellwig if (!bdev_max_discard_sectors(origin_bdev))
3378de7180ffSMike Snitzer reason = "discard unsupported";
3379de7180ffSMike Snitzer
3380de7180ffSMike Snitzer else if (origin_limits->max_discard_sectors < cache->sectors_per_block)
3381de7180ffSMike Snitzer reason = "max discard sectors smaller than a block";
3382de7180ffSMike Snitzer
3383de7180ffSMike Snitzer if (reason) {
3384385411ffSChristoph Hellwig DMWARN("Origin device (%pg) %s: Disabling discard passdown.",
3385385411ffSChristoph Hellwig origin_bdev, reason);
3386de7180ffSMike Snitzer cache->features.discard_passdown = false;
3387de7180ffSMike Snitzer }
3388de7180ffSMike Snitzer }
3389de7180ffSMike Snitzer
set_discard_limits(struct cache * cache,struct queue_limits * limits)3390c6b4fcbaSJoe Thornber static void set_discard_limits(struct cache *cache, struct queue_limits *limits)
3391c6b4fcbaSJoe Thornber {
3392de7180ffSMike Snitzer struct block_device *origin_bdev = cache->origin_dev->bdev;
3393de7180ffSMike Snitzer struct queue_limits *origin_limits = &bdev_get_queue(origin_bdev)->limits;
3394de7180ffSMike Snitzer
3395de7180ffSMike Snitzer if (!cache->features.discard_passdown) {
3396de7180ffSMike Snitzer /* No passdown is done so setting own virtual limits */
33977ae34e77SJoe Thornber limits->max_discard_sectors = min_t(sector_t, cache->discard_block_size * 1024,
33987ae34e77SJoe Thornber cache->origin_sectors);
33991bad9bc4SJoe Thornber limits->discard_granularity = cache->discard_block_size << SECTOR_SHIFT;
3400de7180ffSMike Snitzer return;
3401de7180ffSMike Snitzer }
3402de7180ffSMike Snitzer
3403de7180ffSMike Snitzer /*
3404de7180ffSMike Snitzer * cache_iterate_devices() is stacking both origin and fast device limits
3405de7180ffSMike Snitzer * but discards aren't passed to fast device, so inherit origin's limits.
3406de7180ffSMike Snitzer */
3407de7180ffSMike Snitzer limits->max_discard_sectors = origin_limits->max_discard_sectors;
3408de7180ffSMike Snitzer limits->max_hw_discard_sectors = origin_limits->max_hw_discard_sectors;
3409de7180ffSMike Snitzer limits->discard_granularity = origin_limits->discard_granularity;
3410de7180ffSMike Snitzer limits->discard_alignment = origin_limits->discard_alignment;
3411de7180ffSMike Snitzer limits->discard_misaligned = origin_limits->discard_misaligned;
3412c6b4fcbaSJoe Thornber }
3413c6b4fcbaSJoe Thornber
cache_io_hints(struct dm_target * ti,struct queue_limits * limits)3414c6b4fcbaSJoe Thornber static void cache_io_hints(struct dm_target *ti, struct queue_limits *limits)
3415c6b4fcbaSJoe Thornber {
3416c6b4fcbaSJoe Thornber struct cache *cache = ti->private;
3417f6109372SMike Snitzer uint64_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
3418c6b4fcbaSJoe Thornber
3419f6109372SMike Snitzer /*
3420f6109372SMike Snitzer * If the system-determined stacked limits are compatible with the
3421f6109372SMike Snitzer * cache's blocksize (io_opt is a factor) do not override them.
3422f6109372SMike Snitzer */
3423f6109372SMike Snitzer if (io_opt_sectors < cache->sectors_per_block ||
3424f6109372SMike Snitzer do_div(io_opt_sectors, cache->sectors_per_block)) {
3425b0246530SMike Snitzer blk_limits_io_min(limits, cache->sectors_per_block << SECTOR_SHIFT);
3426c6b4fcbaSJoe Thornber blk_limits_io_opt(limits, cache->sectors_per_block << SECTOR_SHIFT);
3427f6109372SMike Snitzer }
3428de7180ffSMike Snitzer
3429de7180ffSMike Snitzer disable_passdown_if_not_supported(cache);
3430c6b4fcbaSJoe Thornber set_discard_limits(cache, limits);
3431c6b4fcbaSJoe Thornber }
3432c6b4fcbaSJoe Thornber
3433c6b4fcbaSJoe Thornber /*----------------------------------------------------------------*/
3434c6b4fcbaSJoe Thornber
3435c6b4fcbaSJoe Thornber static struct target_type cache_target = {
3436c6b4fcbaSJoe Thornber .name = "cache",
3437636be424SMike Snitzer .version = {2, 2, 0},
3438c6b4fcbaSJoe Thornber .module = THIS_MODULE,
3439c6b4fcbaSJoe Thornber .ctr = cache_ctr,
3440c6b4fcbaSJoe Thornber .dtr = cache_dtr,
3441c6b4fcbaSJoe Thornber .map = cache_map,
3442c6b4fcbaSJoe Thornber .end_io = cache_end_io,
3443c6b4fcbaSJoe Thornber .postsuspend = cache_postsuspend,
3444c6b4fcbaSJoe Thornber .preresume = cache_preresume,
3445c6b4fcbaSJoe Thornber .resume = cache_resume,
3446c6b4fcbaSJoe Thornber .status = cache_status,
3447c6b4fcbaSJoe Thornber .message = cache_message,
3448c6b4fcbaSJoe Thornber .iterate_devices = cache_iterate_devices,
3449c6b4fcbaSJoe Thornber .io_hints = cache_io_hints,
3450c6b4fcbaSJoe Thornber };
3451c6b4fcbaSJoe Thornber
dm_cache_init(void)3452c6b4fcbaSJoe Thornber static int __init dm_cache_init(void)
3453c6b4fcbaSJoe Thornber {
3454c6b4fcbaSJoe Thornber int r;
3455c6b4fcbaSJoe Thornber
3456c6b4fcbaSJoe Thornber migration_cache = KMEM_CACHE(dm_cache_migration, 0);
3457c7cd5550SShenghui Wang if (!migration_cache)
3458c6b4fcbaSJoe Thornber return -ENOMEM;
3459c6b4fcbaSJoe Thornber
34607e6358d2Smonty_pavel@sina.com r = dm_register_target(&cache_target);
34617e6358d2Smonty_pavel@sina.com if (r) {
3462c7cd5550SShenghui Wang kmem_cache_destroy(migration_cache);
34637e6358d2Smonty_pavel@sina.com return r;
34647e6358d2Smonty_pavel@sina.com }
34657e6358d2Smonty_pavel@sina.com
3466c6b4fcbaSJoe Thornber return 0;
3467c6b4fcbaSJoe Thornber }
3468c6b4fcbaSJoe Thornber
dm_cache_exit(void)3469c6b4fcbaSJoe Thornber static void __exit dm_cache_exit(void)
3470c6b4fcbaSJoe Thornber {
3471c6b4fcbaSJoe Thornber dm_unregister_target(&cache_target);
3472c6b4fcbaSJoe Thornber kmem_cache_destroy(migration_cache);
3473c6b4fcbaSJoe Thornber }
3474c6b4fcbaSJoe Thornber
3475c6b4fcbaSJoe Thornber module_init(dm_cache_init);
3476c6b4fcbaSJoe Thornber module_exit(dm_cache_exit);
3477c6b4fcbaSJoe Thornber
3478c6b4fcbaSJoe Thornber MODULE_DESCRIPTION(DM_NAME " cache target");
3479c6b4fcbaSJoe Thornber MODULE_AUTHOR("Joe Thornber <ejt@redhat.com>");
3480c6b4fcbaSJoe Thornber MODULE_LICENSE("GPL");
3481