13bd94003SHeinz Mauelshagen // SPDX-License-Identifier: GPL-2.0-only
21da177e4SLinus Torvalds /*
31da177e4SLinus Torvalds * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
4784aae73SMilan Broz * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
51da177e4SLinus Torvalds *
61da177e4SLinus Torvalds * This file is released under the GPL.
71da177e4SLinus Torvalds */
81da177e4SLinus Torvalds
94cc96131SMike Snitzer #include "dm-core.h"
104cc96131SMike Snitzer #include "dm-rq.h"
1151e5b2bdSMike Anderson #include "dm-uevent.h"
1291ccbbacSTushar Sugandhi #include "dm-ima.h"
131da177e4SLinus Torvalds
141da177e4SLinus Torvalds #include <linux/init.h>
151da177e4SLinus Torvalds #include <linux/module.h>
1648c9c27bSArjan van de Ven #include <linux/mutex.h>
176958c1c6SMikulas Patocka #include <linux/sched/mm.h>
18174cd4b1SIngo Molnar #include <linux/sched/signal.h>
191da177e4SLinus Torvalds #include <linux/blkpg.h>
201da177e4SLinus Torvalds #include <linux/bio.h>
211da177e4SLinus Torvalds #include <linux/mempool.h>
22f26c5719SDan Williams #include <linux/dax.h>
231da177e4SLinus Torvalds #include <linux/slab.h>
241da177e4SLinus Torvalds #include <linux/idr.h>
257e026c8cSDan Williams #include <linux/uio.h>
263ac51e74SDarrick J. Wong #include <linux/hdreg.h>
273f77316dSKiyoshi Ueda #include <linux/delay.h>
28ffcc3936SMike Snitzer #include <linux/wait.h>
2971cdb697SChristoph Hellwig #include <linux/pr.h>
30b0b4d7c6SElena Reshetova #include <linux/refcount.h>
31c6a564ffSChristoph Hellwig #include <linux/part_stat.h>
32a892c8d5SSatya Tangirala #include <linux/blk-crypto.h>
331e8d44bdSEric Biggers #include <linux/blk-crypto-profile.h>
3455782138SLi Zefan
3572d94861SAlasdair G Kergon #define DM_MSG_PREFIX "core"
3672d94861SAlasdair G Kergon
3760935eb2SMilan Broz /*
3860935eb2SMilan Broz * Cookies are numeric values sent with CHANGE and REMOVE
3960935eb2SMilan Broz * uevents while resuming, removing or renaming the device.
4060935eb2SMilan Broz */
4160935eb2SMilan Broz #define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE"
4260935eb2SMilan Broz #define DM_COOKIE_LENGTH 24
4360935eb2SMilan Broz
44b99fdcdcSMing Lei /*
45b99fdcdcSMing Lei * For REQ_POLLED fs bio, this flag is set if we link mapped underlying
46b99fdcdcSMing Lei * dm_io into one list, and reuse bio->bi_private as the list head. Before
47b99fdcdcSMing Lei * ending this fs bio, we will recover its ->bi_private.
48b99fdcdcSMing Lei */
49b99fdcdcSMing Lei #define REQ_DM_POLL_LIST REQ_DRV
50b99fdcdcSMing Lei
511da177e4SLinus Torvalds static const char *_name = DM_NAME;
521da177e4SLinus Torvalds
532f06cd12SHeinz Mauelshagen static unsigned int major;
542f06cd12SHeinz Mauelshagen static unsigned int _major;
551da177e4SLinus Torvalds
56d15b774cSAlasdair G Kergon static DEFINE_IDR(_minor_idr);
57d15b774cSAlasdair G Kergon
58f32c10b0SJeff Mahoney static DEFINE_SPINLOCK(_minor_lock);
592c140a24SMikulas Patocka
602c140a24SMikulas Patocka static void do_deferred_remove(struct work_struct *w);
612c140a24SMikulas Patocka
622c140a24SMikulas Patocka static DECLARE_WORK(deferred_remove_work, do_deferred_remove);
632c140a24SMikulas Patocka
64acfe0ad7SMikulas Patocka static struct workqueue_struct *deferred_remove_workqueue;
65acfe0ad7SMikulas Patocka
6693e6442cSMikulas Patocka atomic_t dm_global_event_nr = ATOMIC_INIT(0);
6793e6442cSMikulas Patocka DECLARE_WAIT_QUEUE_HEAD(dm_global_eventq);
6893e6442cSMikulas Patocka
dm_issue_global_event(void)6962e08243SMikulas Patocka void dm_issue_global_event(void)
7062e08243SMikulas Patocka {
7162e08243SMikulas Patocka atomic_inc(&dm_global_event_nr);
7262e08243SMikulas Patocka wake_up(&dm_global_eventq);
7362e08243SMikulas Patocka }
7462e08243SMikulas Patocka
75442761fdSMike Snitzer DEFINE_STATIC_KEY_FALSE(stats_enabled);
76442761fdSMike Snitzer DEFINE_STATIC_KEY_FALSE(swap_bios_enabled);
77442761fdSMike Snitzer DEFINE_STATIC_KEY_FALSE(zoned_enabled);
78442761fdSMike Snitzer
791da177e4SLinus Torvalds /*
8064f52b0eSMike Snitzer * One of these is allocated (on-stack) per original bio.
811da177e4SLinus Torvalds */
8264f52b0eSMike Snitzer struct clone_info {
8364f52b0eSMike Snitzer struct dm_table *map;
8464f52b0eSMike Snitzer struct bio *bio;
8564f52b0eSMike Snitzer struct dm_io *io;
8664f52b0eSMike Snitzer sector_t sector;
8786a3238cSHeinz Mauelshagen unsigned int sector_count;
884edadf6dSMike Snitzer bool is_abnormal_io:1;
894edadf6dSMike Snitzer bool submit_as_polled:1;
9064f52b0eSMike Snitzer };
9164f52b0eSMike Snitzer
clone_to_tio(struct bio * clone)926c23f0bdSChristoph Hellwig static inline struct dm_target_io *clone_to_tio(struct bio *clone)
936c23f0bdSChristoph Hellwig {
946c23f0bdSChristoph Hellwig return container_of(clone, struct dm_target_io, clone);
956c23f0bdSChristoph Hellwig }
966c23f0bdSChristoph Hellwig
dm_per_bio_data(struct bio * bio,size_t data_size)9764f52b0eSMike Snitzer void *dm_per_bio_data(struct bio *bio, size_t data_size)
9864f52b0eSMike Snitzer {
99655f3aadSMike Snitzer if (!dm_tio_flagged(clone_to_tio(bio), DM_TIO_INSIDE_DM_IO))
10062f26317SJeffle Xu return (char *)bio - DM_TARGET_IO_BIO_OFFSET - data_size;
10162f26317SJeffle Xu return (char *)bio - DM_IO_BIO_OFFSET - data_size;
10264f52b0eSMike Snitzer }
10364f52b0eSMike Snitzer EXPORT_SYMBOL_GPL(dm_per_bio_data);
10464f52b0eSMike Snitzer
dm_bio_from_per_bio_data(void * data,size_t data_size)10564f52b0eSMike Snitzer struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size)
10664f52b0eSMike Snitzer {
10764f52b0eSMike Snitzer struct dm_io *io = (struct dm_io *)((char *)data + data_size);
108b30f1607SHeinz Mauelshagen
10964f52b0eSMike Snitzer if (io->magic == DM_IO_MAGIC)
11062f26317SJeffle Xu return (struct bio *)((char *)io + DM_IO_BIO_OFFSET);
11164f52b0eSMike Snitzer BUG_ON(io->magic != DM_TIO_MAGIC);
11262f26317SJeffle Xu return (struct bio *)((char *)io + DM_TARGET_IO_BIO_OFFSET);
11364f52b0eSMike Snitzer }
11464f52b0eSMike Snitzer EXPORT_SYMBOL_GPL(dm_bio_from_per_bio_data);
11564f52b0eSMike Snitzer
dm_bio_get_target_bio_nr(const struct bio * bio)11686a3238cSHeinz Mauelshagen unsigned int dm_bio_get_target_bio_nr(const struct bio *bio)
11764f52b0eSMike Snitzer {
11864f52b0eSMike Snitzer return container_of(bio, struct dm_target_io, clone)->target_bio_nr;
11964f52b0eSMike Snitzer }
12064f52b0eSMike Snitzer EXPORT_SYMBOL_GPL(dm_bio_get_target_bio_nr);
12164f52b0eSMike Snitzer
122ba61fdd1SJeff Mahoney #define MINOR_ALLOCED ((void *)-1)
123ba61fdd1SJeff Mahoney
124115485e8SMike Snitzer #define DM_NUMA_NODE NUMA_NO_NODE
125115485e8SMike Snitzer static int dm_numa_node = DM_NUMA_NODE;
126faad87dfSMike Snitzer
127a666e5c0SMikulas Patocka #define DEFAULT_SWAP_BIOS (8 * 1048576 / PAGE_SIZE)
128a666e5c0SMikulas Patocka static int swap_bios = DEFAULT_SWAP_BIOS;
get_swap_bios(void)129a666e5c0SMikulas Patocka static int get_swap_bios(void)
130a666e5c0SMikulas Patocka {
131a666e5c0SMikulas Patocka int latch = READ_ONCE(swap_bios);
132b30f1607SHeinz Mauelshagen
133a666e5c0SMikulas Patocka if (unlikely(latch <= 0))
134a666e5c0SMikulas Patocka latch = DEFAULT_SWAP_BIOS;
135a666e5c0SMikulas Patocka return latch;
136a666e5c0SMikulas Patocka }
137a666e5c0SMikulas Patocka
13886f1152bSBenjamin Marzinski struct table_device {
13986f1152bSBenjamin Marzinski struct list_head list;
140b0b4d7c6SElena Reshetova refcount_t count;
14186f1152bSBenjamin Marzinski struct dm_dev dm_dev;
14286f1152bSBenjamin Marzinski };
14386f1152bSBenjamin Marzinski
144f4790826SMike Snitzer /*
145e8603136SMike Snitzer * Bio-based DM's mempools' reserved IOs set by the user.
146e8603136SMike Snitzer */
1474cc96131SMike Snitzer #define RESERVED_BIO_BASED_IOS 16
14886a3238cSHeinz Mauelshagen static unsigned int reserved_bio_based_ios = RESERVED_BIO_BASED_IOS;
149e8603136SMike Snitzer
__dm_get_module_param_int(int * module_param,int min,int max)150115485e8SMike Snitzer static int __dm_get_module_param_int(int *module_param, int min, int max)
151115485e8SMike Snitzer {
1526aa7de05SMark Rutland int param = READ_ONCE(*module_param);
153115485e8SMike Snitzer int modified_param = 0;
154115485e8SMike Snitzer bool modified = true;
155115485e8SMike Snitzer
156115485e8SMike Snitzer if (param < min)
157115485e8SMike Snitzer modified_param = min;
158115485e8SMike Snitzer else if (param > max)
159115485e8SMike Snitzer modified_param = max;
160115485e8SMike Snitzer else
161115485e8SMike Snitzer modified = false;
162115485e8SMike Snitzer
163115485e8SMike Snitzer if (modified) {
164115485e8SMike Snitzer (void)cmpxchg(module_param, param, modified_param);
165115485e8SMike Snitzer param = modified_param;
166115485e8SMike Snitzer }
167115485e8SMike Snitzer
168115485e8SMike Snitzer return param;
169115485e8SMike Snitzer }
170115485e8SMike Snitzer
__dm_get_module_param(unsigned int * module_param,unsigned int def,unsigned int max)17186a3238cSHeinz Mauelshagen unsigned int __dm_get_module_param(unsigned int *module_param, unsigned int def, unsigned int max)
172f4790826SMike Snitzer {
17386a3238cSHeinz Mauelshagen unsigned int param = READ_ONCE(*module_param);
17486a3238cSHeinz Mauelshagen unsigned int modified_param = 0;
175f4790826SMike Snitzer
17609c2d531SMike Snitzer if (!param)
17709c2d531SMike Snitzer modified_param = def;
17809c2d531SMike Snitzer else if (param > max)
17909c2d531SMike Snitzer modified_param = max;
180f4790826SMike Snitzer
18109c2d531SMike Snitzer if (modified_param) {
18209c2d531SMike Snitzer (void)cmpxchg(module_param, param, modified_param);
18309c2d531SMike Snitzer param = modified_param;
184f4790826SMike Snitzer }
185f4790826SMike Snitzer
18609c2d531SMike Snitzer return param;
187f4790826SMike Snitzer }
188f4790826SMike Snitzer
dm_get_reserved_bio_based_ios(void)18986a3238cSHeinz Mauelshagen unsigned int dm_get_reserved_bio_based_ios(void)
190e8603136SMike Snitzer {
19109c2d531SMike Snitzer return __dm_get_module_param(&reserved_bio_based_ios,
1924cc96131SMike Snitzer RESERVED_BIO_BASED_IOS, DM_RESERVED_MAX_IOS);
193e8603136SMike Snitzer }
194e8603136SMike Snitzer EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios);
195e8603136SMike Snitzer
dm_get_numa_node(void)19686a3238cSHeinz Mauelshagen static unsigned int dm_get_numa_node(void)
197115485e8SMike Snitzer {
198115485e8SMike Snitzer return __dm_get_module_param_int(&dm_numa_node,
199115485e8SMike Snitzer DM_NUMA_NODE, num_online_nodes() - 1);
200115485e8SMike Snitzer }
201115485e8SMike Snitzer
local_init(void)2021da177e4SLinus Torvalds static int __init local_init(void)
2031da177e4SLinus Torvalds {
204e689fbabSMike Snitzer int r;
2051ae49ea2SMike Snitzer
20651e5b2bdSMike Anderson r = dm_uevent_init();
20751157b4aSKiyoshi Ueda if (r)
208e689fbabSMike Snitzer return r;
20951e5b2bdSMike Anderson
21057bbf99cSTejun Heo deferred_remove_workqueue = alloc_ordered_workqueue("kdmremove", 0);
211acfe0ad7SMikulas Patocka if (!deferred_remove_workqueue) {
212acfe0ad7SMikulas Patocka r = -ENOMEM;
213acfe0ad7SMikulas Patocka goto out_uevent_exit;
214acfe0ad7SMikulas Patocka }
215acfe0ad7SMikulas Patocka
2161da177e4SLinus Torvalds _major = major;
2171da177e4SLinus Torvalds r = register_blkdev(_major, _name);
21851157b4aSKiyoshi Ueda if (r < 0)
219acfe0ad7SMikulas Patocka goto out_free_workqueue;
2201da177e4SLinus Torvalds
2211da177e4SLinus Torvalds if (!_major)
2221da177e4SLinus Torvalds _major = r;
2231da177e4SLinus Torvalds
2241da177e4SLinus Torvalds return 0;
22551157b4aSKiyoshi Ueda
226acfe0ad7SMikulas Patocka out_free_workqueue:
227acfe0ad7SMikulas Patocka destroy_workqueue(deferred_remove_workqueue);
22851157b4aSKiyoshi Ueda out_uevent_exit:
22951157b4aSKiyoshi Ueda dm_uevent_exit();
23051157b4aSKiyoshi Ueda
23151157b4aSKiyoshi Ueda return r;
2321da177e4SLinus Torvalds }
2331da177e4SLinus Torvalds
local_exit(void)2341da177e4SLinus Torvalds static void local_exit(void)
2351da177e4SLinus Torvalds {
236acfe0ad7SMikulas Patocka destroy_workqueue(deferred_remove_workqueue);
2372c140a24SMikulas Patocka
23800d59405SAkinobu Mita unregister_blkdev(_major, _name);
23951e5b2bdSMike Anderson dm_uevent_exit();
2401da177e4SLinus Torvalds
2411da177e4SLinus Torvalds _major = 0;
2421da177e4SLinus Torvalds
2431da177e4SLinus Torvalds DMINFO("cleaned up");
2441da177e4SLinus Torvalds }
2451da177e4SLinus Torvalds
246b9249e55SAlasdair G Kergon static int (*_inits[])(void) __initdata = {
2471da177e4SLinus Torvalds local_init,
2481da177e4SLinus Torvalds dm_target_init,
2491da177e4SLinus Torvalds dm_linear_init,
2501da177e4SLinus Torvalds dm_stripe_init,
251952b3557SMikulas Patocka dm_io_init,
252945fa4d2SMikulas Patocka dm_kcopyd_init,
2531da177e4SLinus Torvalds dm_interface_init,
254fd2ed4d2SMikulas Patocka dm_statistics_init,
2551da177e4SLinus Torvalds };
2561da177e4SLinus Torvalds
257b9249e55SAlasdair G Kergon static void (*_exits[])(void) = {
2581da177e4SLinus Torvalds local_exit,
2591da177e4SLinus Torvalds dm_target_exit,
2601da177e4SLinus Torvalds dm_linear_exit,
2611da177e4SLinus Torvalds dm_stripe_exit,
262952b3557SMikulas Patocka dm_io_exit,
263945fa4d2SMikulas Patocka dm_kcopyd_exit,
2641da177e4SLinus Torvalds dm_interface_exit,
265fd2ed4d2SMikulas Patocka dm_statistics_exit,
2661da177e4SLinus Torvalds };
2671da177e4SLinus Torvalds
dm_init(void)2681da177e4SLinus Torvalds static int __init dm_init(void)
2691da177e4SLinus Torvalds {
2701da177e4SLinus Torvalds const int count = ARRAY_SIZE(_inits);
2711da177e4SLinus Torvalds int r, i;
2721da177e4SLinus Torvalds
273f1cd6cb2STushar Sugandhi #if (IS_ENABLED(CONFIG_IMA) && !IS_ENABLED(CONFIG_IMA_DISABLE_HTABLE))
274f1cd6cb2STushar Sugandhi DMWARN("CONFIG_IMA_DISABLE_HTABLE is disabled."
275f1cd6cb2STushar Sugandhi " Duplicate IMA measurements will not be recorded in the IMA log.");
276f1cd6cb2STushar Sugandhi #endif
277f1cd6cb2STushar Sugandhi
2781da177e4SLinus Torvalds for (i = 0; i < count; i++) {
2791da177e4SLinus Torvalds r = _inits[i]();
2801da177e4SLinus Torvalds if (r)
2811da177e4SLinus Torvalds goto bad;
2821da177e4SLinus Torvalds }
2831da177e4SLinus Torvalds
2841da177e4SLinus Torvalds return 0;
2851da177e4SLinus Torvalds bad:
2861da177e4SLinus Torvalds while (i--)
2871da177e4SLinus Torvalds _exits[i]();
2881da177e4SLinus Torvalds
2891da177e4SLinus Torvalds return r;
2901da177e4SLinus Torvalds }
2911da177e4SLinus Torvalds
dm_exit(void)2921da177e4SLinus Torvalds static void __exit dm_exit(void)
2931da177e4SLinus Torvalds {
2941da177e4SLinus Torvalds int i = ARRAY_SIZE(_exits);
2951da177e4SLinus Torvalds
2961da177e4SLinus Torvalds while (i--)
2971da177e4SLinus Torvalds _exits[i]();
298d15b774cSAlasdair G Kergon
299d15b774cSAlasdair G Kergon /*
300d15b774cSAlasdair G Kergon * Should be empty by this point.
301d15b774cSAlasdair G Kergon */
302d15b774cSAlasdair G Kergon idr_destroy(&_minor_idr);
3031da177e4SLinus Torvalds }
3041da177e4SLinus Torvalds
3051da177e4SLinus Torvalds /*
3061da177e4SLinus Torvalds * Block device functions
3071da177e4SLinus Torvalds */
dm_deleting_md(struct mapped_device * md)308432a212cSMike Anderson int dm_deleting_md(struct mapped_device *md)
309432a212cSMike Anderson {
310432a212cSMike Anderson return test_bit(DMF_DELETING, &md->flags);
311432a212cSMike Anderson }
312432a212cSMike Anderson
dm_blk_open(struct gendisk * disk,blk_mode_t mode)31305bdb996SChristoph Hellwig static int dm_blk_open(struct gendisk *disk, blk_mode_t mode)
3141da177e4SLinus Torvalds {
3151da177e4SLinus Torvalds struct mapped_device *md;
3161da177e4SLinus Torvalds
317fba9f90eSJeff Mahoney spin_lock(&_minor_lock);
318fba9f90eSJeff Mahoney
319d32e2bf8SChristoph Hellwig md = disk->private_data;
320fba9f90eSJeff Mahoney if (!md)
321fba9f90eSJeff Mahoney goto out;
322fba9f90eSJeff Mahoney
3235c6bd75dSAlasdair G Kergon if (test_bit(DMF_FREEING, &md->flags) ||
324432a212cSMike Anderson dm_deleting_md(md)) {
325fba9f90eSJeff Mahoney md = NULL;
326fba9f90eSJeff Mahoney goto out;
327fba9f90eSJeff Mahoney }
328fba9f90eSJeff Mahoney
3291da177e4SLinus Torvalds dm_get(md);
3305c6bd75dSAlasdair G Kergon atomic_inc(&md->open_count);
331fba9f90eSJeff Mahoney out:
332fba9f90eSJeff Mahoney spin_unlock(&_minor_lock);
333fba9f90eSJeff Mahoney
334fba9f90eSJeff Mahoney return md ? 0 : -ENXIO;
3351da177e4SLinus Torvalds }
3361da177e4SLinus Torvalds
dm_blk_close(struct gendisk * disk)337ae220766SChristoph Hellwig static void dm_blk_close(struct gendisk *disk)
3381da177e4SLinus Torvalds {
33963a4f065SMike Snitzer struct mapped_device *md;
3406e9624b8SArnd Bergmann
3414a1aeb98SMilan Broz spin_lock(&_minor_lock);
3424a1aeb98SMilan Broz
34363a4f065SMike Snitzer md = disk->private_data;
34463a4f065SMike Snitzer if (WARN_ON(!md))
34563a4f065SMike Snitzer goto out;
34663a4f065SMike Snitzer
3472c140a24SMikulas Patocka if (atomic_dec_and_test(&md->open_count) &&
3482c140a24SMikulas Patocka (test_bit(DMF_DEFERRED_REMOVE, &md->flags)))
349acfe0ad7SMikulas Patocka queue_work(deferred_remove_workqueue, &deferred_remove_work);
3502c140a24SMikulas Patocka
3511da177e4SLinus Torvalds dm_put(md);
35263a4f065SMike Snitzer out:
3534a1aeb98SMilan Broz spin_unlock(&_minor_lock);
3541da177e4SLinus Torvalds }
3551da177e4SLinus Torvalds
dm_open_count(struct mapped_device * md)3565c6bd75dSAlasdair G Kergon int dm_open_count(struct mapped_device *md)
3575c6bd75dSAlasdair G Kergon {
3585c6bd75dSAlasdair G Kergon return atomic_read(&md->open_count);
3595c6bd75dSAlasdair G Kergon }
3605c6bd75dSAlasdair G Kergon
3615c6bd75dSAlasdair G Kergon /*
3625c6bd75dSAlasdair G Kergon * Guarantees nothing is using the device before it's deleted.
3635c6bd75dSAlasdair G Kergon */
dm_lock_for_deletion(struct mapped_device * md,bool mark_deferred,bool only_deferred)3642c140a24SMikulas Patocka int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred)
3655c6bd75dSAlasdair G Kergon {
3665c6bd75dSAlasdair G Kergon int r = 0;
3675c6bd75dSAlasdair G Kergon
3685c6bd75dSAlasdair G Kergon spin_lock(&_minor_lock);
3695c6bd75dSAlasdair G Kergon
3702c140a24SMikulas Patocka if (dm_open_count(md)) {
3715c6bd75dSAlasdair G Kergon r = -EBUSY;
3722c140a24SMikulas Patocka if (mark_deferred)
3732c140a24SMikulas Patocka set_bit(DMF_DEFERRED_REMOVE, &md->flags);
3742c140a24SMikulas Patocka } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags))
3752c140a24SMikulas Patocka r = -EEXIST;
3765c6bd75dSAlasdair G Kergon else
3775c6bd75dSAlasdair G Kergon set_bit(DMF_DELETING, &md->flags);
3785c6bd75dSAlasdair G Kergon
3795c6bd75dSAlasdair G Kergon spin_unlock(&_minor_lock);
3805c6bd75dSAlasdair G Kergon
3815c6bd75dSAlasdair G Kergon return r;
3825c6bd75dSAlasdair G Kergon }
3835c6bd75dSAlasdair G Kergon
dm_cancel_deferred_remove(struct mapped_device * md)3842c140a24SMikulas Patocka int dm_cancel_deferred_remove(struct mapped_device *md)
3852c140a24SMikulas Patocka {
3862c140a24SMikulas Patocka int r = 0;
3872c140a24SMikulas Patocka
3882c140a24SMikulas Patocka spin_lock(&_minor_lock);
3892c140a24SMikulas Patocka
3902c140a24SMikulas Patocka if (test_bit(DMF_DELETING, &md->flags))
3912c140a24SMikulas Patocka r = -EBUSY;
3922c140a24SMikulas Patocka else
3932c140a24SMikulas Patocka clear_bit(DMF_DEFERRED_REMOVE, &md->flags);
3942c140a24SMikulas Patocka
3952c140a24SMikulas Patocka spin_unlock(&_minor_lock);
3962c140a24SMikulas Patocka
3972c140a24SMikulas Patocka return r;
3982c140a24SMikulas Patocka }
3992c140a24SMikulas Patocka
do_deferred_remove(struct work_struct * w)4002c140a24SMikulas Patocka static void do_deferred_remove(struct work_struct *w)
4012c140a24SMikulas Patocka {
4022c140a24SMikulas Patocka dm_deferred_remove();
4032c140a24SMikulas Patocka }
4042c140a24SMikulas Patocka
dm_blk_getgeo(struct block_device * bdev,struct hd_geometry * geo)4053ac51e74SDarrick J. Wong static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
4063ac51e74SDarrick J. Wong {
4073ac51e74SDarrick J. Wong struct mapped_device *md = bdev->bd_disk->private_data;
4083ac51e74SDarrick J. Wong
4093ac51e74SDarrick J. Wong return dm_get_geometry(md, geo);
4103ac51e74SDarrick J. Wong }
4113ac51e74SDarrick J. Wong
dm_prepare_ioctl(struct mapped_device * md,int * srcu_idx,struct block_device ** bdev)412971888c4SMike Snitzer static int dm_prepare_ioctl(struct mapped_device *md, int *srcu_idx,
4135bd5e8d8SMike Snitzer struct block_device **bdev)
414aa129a22SMilan Broz {
415564b5c54SMike Snitzer struct dm_target *ti;
4166c182cd8SHannes Reinecke struct dm_table *map;
417971888c4SMike Snitzer int r;
418aa129a22SMilan Broz
4196c182cd8SHannes Reinecke retry:
420e56f81e0SChristoph Hellwig r = -ENOTTY;
421971888c4SMike Snitzer map = dm_get_live_table(md, srcu_idx);
422aa129a22SMilan Broz if (!map || !dm_table_get_size(map))
423971888c4SMike Snitzer return r;
424aa129a22SMilan Broz
425aa129a22SMilan Broz /* We only support devices that have a single target */
4262aec377aSMike Snitzer if (map->num_targets != 1)
427971888c4SMike Snitzer return r;
428aa129a22SMilan Broz
429564b5c54SMike Snitzer ti = dm_table_get_target(map, 0);
430564b5c54SMike Snitzer if (!ti->type->prepare_ioctl)
431e56f81e0SChristoph Hellwig return r;
432aa129a22SMilan Broz
433971888c4SMike Snitzer if (dm_suspended_md(md))
434971888c4SMike Snitzer return -EAGAIN;
435971888c4SMike Snitzer
436564b5c54SMike Snitzer r = ti->type->prepare_ioctl(ti, bdev);
4375bbbfdf6SJunichi Nomura if (r == -ENOTCONN && !fatal_signal_pending(current)) {
438971888c4SMike Snitzer dm_put_live_table(md, *srcu_idx);
439238d991fSHeinz Mauelshagen fsleep(10000);
4406c182cd8SHannes Reinecke goto retry;
4416c182cd8SHannes Reinecke }
442971888c4SMike Snitzer
443e56f81e0SChristoph Hellwig return r;
444e56f81e0SChristoph Hellwig }
4456c182cd8SHannes Reinecke
dm_unprepare_ioctl(struct mapped_device * md,int srcu_idx)446971888c4SMike Snitzer static void dm_unprepare_ioctl(struct mapped_device *md, int srcu_idx)
447971888c4SMike Snitzer {
448971888c4SMike Snitzer dm_put_live_table(md, srcu_idx);
449971888c4SMike Snitzer }
450971888c4SMike Snitzer
dm_blk_ioctl(struct block_device * bdev,blk_mode_t mode,unsigned int cmd,unsigned long arg)45105bdb996SChristoph Hellwig static int dm_blk_ioctl(struct block_device *bdev, blk_mode_t mode,
452e56f81e0SChristoph Hellwig unsigned int cmd, unsigned long arg)
453e56f81e0SChristoph Hellwig {
454e56f81e0SChristoph Hellwig struct mapped_device *md = bdev->bd_disk->private_data;
455971888c4SMike Snitzer int r, srcu_idx;
456e56f81e0SChristoph Hellwig
4575bd5e8d8SMike Snitzer r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
458e56f81e0SChristoph Hellwig if (r < 0)
459971888c4SMike Snitzer goto out;
460e56f81e0SChristoph Hellwig
461e56f81e0SChristoph Hellwig if (r > 0) {
462e56f81e0SChristoph Hellwig /*
463e980f623SChristoph Hellwig * Target determined this ioctl is being issued against a
464e980f623SChristoph Hellwig * subset of the parent bdev; require extra privileges.
465e56f81e0SChristoph Hellwig */
466e980f623SChristoph Hellwig if (!capable(CAP_SYS_RAWIO)) {
4670378c625SMike Snitzer DMDEBUG_LIMIT(
468e980f623SChristoph Hellwig "%s: sending ioctl %x to DM device without required privilege.",
469e980f623SChristoph Hellwig current->comm, cmd);
470e980f623SChristoph Hellwig r = -ENOIOCTLCMD;
471e56f81e0SChristoph Hellwig goto out;
472e56f81e0SChristoph Hellwig }
473e980f623SChristoph Hellwig }
474e56f81e0SChristoph Hellwig
475a7cb3d2fSChristoph Hellwig if (!bdev->bd_disk->fops->ioctl)
476a7cb3d2fSChristoph Hellwig r = -ENOTTY;
477a7cb3d2fSChristoph Hellwig else
478a7cb3d2fSChristoph Hellwig r = bdev->bd_disk->fops->ioctl(bdev, mode, cmd, arg);
479e56f81e0SChristoph Hellwig out:
480971888c4SMike Snitzer dm_unprepare_ioctl(md, srcu_idx);
481aa129a22SMilan Broz return r;
482aa129a22SMilan Broz }
483aa129a22SMilan Broz
dm_start_time_ns_from_clone(struct bio * bio)4847465d7acSMike Snitzer u64 dm_start_time_ns_from_clone(struct bio *bio)
4857465d7acSMike Snitzer {
4866c23f0bdSChristoph Hellwig return jiffies_to_nsecs(clone_to_tio(bio)->io->start_time);
4877465d7acSMike Snitzer }
4887465d7acSMike Snitzer EXPORT_SYMBOL_GPL(dm_start_time_ns_from_clone);
4897465d7acSMike Snitzer
bio_is_flush_with_data(struct bio * bio)49006eed768SMike Snitzer static inline bool bio_is_flush_with_data(struct bio *bio)
4917465d7acSMike Snitzer {
4928d394bc4SMike Snitzer return ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size);
4937465d7acSMike Snitzer }
4947465d7acSMike Snitzer
dm_io_sectors(struct dm_io * io,struct bio * bio)49506eed768SMike Snitzer static inline unsigned int dm_io_sectors(struct dm_io *io, struct bio *bio)
4967465d7acSMike Snitzer {
497d3de6d12SMing Lei /*
498d3de6d12SMing Lei * If REQ_PREFLUSH set, don't account payload, it will be
499d3de6d12SMing Lei * submitted (and accounted) after this flush completes.
500d3de6d12SMing Lei */
501d3de6d12SMing Lei if (bio_is_flush_with_data(bio))
50206eed768SMike Snitzer return 0;
50306eed768SMike Snitzer if (unlikely(dm_io_flagged(io, DM_IO_WAS_SPLIT)))
50406eed768SMike Snitzer return io->sectors;
50506eed768SMike Snitzer return bio_sectors(bio);
50606eed768SMike Snitzer }
5078d394bc4SMike Snitzer
dm_io_acct(struct dm_io * io,bool end)50806eed768SMike Snitzer static void dm_io_acct(struct dm_io *io, bool end)
50906eed768SMike Snitzer {
51006eed768SMike Snitzer struct bio *bio = io->orig_bio;
5118d394bc4SMike Snitzer
512526d1006SLi Nan if (dm_io_flagged(io, DM_IO_BLK_STAT)) {
5138d394bc4SMike Snitzer if (!end)
514526d1006SLi Nan bdev_start_io_acct(bio->bi_bdev, bio_op(bio),
51506eed768SMike Snitzer io->start_time);
5168d394bc4SMike Snitzer else
517526d1006SLi Nan bdev_end_io_acct(bio->bi_bdev, bio_op(bio),
51806eed768SMike Snitzer dm_io_sectors(io, bio),
51906eed768SMike Snitzer io->start_time);
520526d1006SLi Nan }
5217465d7acSMike Snitzer
522442761fdSMike Snitzer if (static_branch_unlikely(&stats_enabled) &&
52306eed768SMike Snitzer unlikely(dm_stats_used(&io->md->stats))) {
5247dd76d1fSMing Lei sector_t sector;
5257dd76d1fSMing Lei
52606eed768SMike Snitzer if (unlikely(dm_io_flagged(io, DM_IO_WAS_SPLIT)))
5277dd76d1fSMing Lei sector = bio_end_sector(bio) - io->sector_offset;
52806eed768SMike Snitzer else
52906eed768SMike Snitzer sector = bio->bi_iter.bi_sector;
5307dd76d1fSMing Lei
53106eed768SMike Snitzer dm_stats_account_io(&io->md->stats, bio_data_dir(bio),
53206eed768SMike Snitzer sector, dm_io_sectors(io, bio),
53306eed768SMike Snitzer end, io->start_time, &io->stats_aux);
5348d394bc4SMike Snitzer }
5357dd76d1fSMing Lei }
5368d394bc4SMike Snitzer
__dm_start_io_acct(struct dm_io * io)537b992b40dSMing Lei static void __dm_start_io_acct(struct dm_io *io)
5388d394bc4SMike Snitzer {
539e6926ad0SMing Lei dm_io_acct(io, false);
5408d394bc4SMike Snitzer }
5418d394bc4SMike Snitzer
dm_start_io_acct(struct dm_io * io,struct bio * clone)5420fbb4d93SMike Snitzer static void dm_start_io_acct(struct dm_io *io, struct bio *clone)
5438d394bc4SMike Snitzer {
5440fbb4d93SMike Snitzer /*
5450fbb4d93SMike Snitzer * Ensure IO accounting is only ever started once.
5460fbb4d93SMike Snitzer */
5473b03f7c1SMike Snitzer if (dm_io_flagged(io, DM_IO_ACCOUNTED))
5480fbb4d93SMike Snitzer return;
5493b03f7c1SMike Snitzer
5503b03f7c1SMike Snitzer /* Expect no possibility for race unless DM_TIO_IS_DUPLICATE_BIO. */
5513b03f7c1SMike Snitzer if (!clone || likely(dm_tio_is_normal(clone_to_tio(clone)))) {
55282f6cdccSMike Snitzer dm_io_set_flag(io, DM_IO_ACCOUNTED);
55382f6cdccSMike Snitzer } else {
55482f6cdccSMike Snitzer unsigned long flags;
555655f3aadSMike Snitzer /* Can afford locking given DM_TIO_IS_DUPLICATE_BIO */
5564d7bca13SMike Snitzer spin_lock_irqsave(&io->lock, flags);
55710eb3a0dSBenjamin Marzinski if (dm_io_flagged(io, DM_IO_ACCOUNTED)) {
55810eb3a0dSBenjamin Marzinski spin_unlock_irqrestore(&io->lock, flags);
55910eb3a0dSBenjamin Marzinski return;
56010eb3a0dSBenjamin Marzinski }
56182f6cdccSMike Snitzer dm_io_set_flag(io, DM_IO_ACCOUNTED);
5624d7bca13SMike Snitzer spin_unlock_irqrestore(&io->lock, flags);
56382f6cdccSMike Snitzer }
5640fbb4d93SMike Snitzer
565b992b40dSMing Lei __dm_start_io_acct(io);
5660fbb4d93SMike Snitzer }
5670fbb4d93SMike Snitzer
dm_end_io_acct(struct dm_io * io)568b992b40dSMing Lei static void dm_end_io_acct(struct dm_io *io)
5690fbb4d93SMike Snitzer {
570e6926ad0SMing Lei dm_io_acct(io, true);
5717465d7acSMike Snitzer }
572978e51baSMike Snitzer
alloc_io(struct mapped_device * md,struct bio * bio)573978e51baSMike Snitzer static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio)
5741da177e4SLinus Torvalds {
57564f52b0eSMike Snitzer struct dm_io *io;
57664f52b0eSMike Snitzer struct dm_target_io *tio;
57764f52b0eSMike Snitzer struct bio *clone;
57864f52b0eSMike Snitzer
57929dec90aSChristoph Hellwig clone = bio_alloc_clone(NULL, bio, GFP_NOIO, &md->mempools->io_bs);
5806c23f0bdSChristoph Hellwig tio = clone_to_tio(clone);
581655f3aadSMike Snitzer tio->flags = 0;
582655f3aadSMike Snitzer dm_tio_set_flag(tio, DM_TIO_INSIDE_DM_IO);
58364f52b0eSMike Snitzer tio->io = NULL;
58464f52b0eSMike Snitzer
58564f52b0eSMike Snitzer io = container_of(tio, struct dm_io, tio);
58664f52b0eSMike Snitzer io->magic = DM_IO_MAGIC;
58784b98f4cSMike Snitzer io->status = BLK_STS_OK;
5880f14d60aSMing Lei
5890f14d60aSMing Lei /* one ref is for submission, the other is for completion */
5900f14d60aSMing Lei atomic_set(&io->io_count, 2);
5919f6dc633SMike Snitzer this_cpu_inc(*md->pending_io);
5927dd76d1fSMing Lei io->orig_bio = bio;
593978e51baSMike Snitzer io->md = md;
5944d7bca13SMike Snitzer spin_lock_init(&io->lock);
595b879f915SMike Snitzer io->start_time = jiffies;
59682f6cdccSMike Snitzer io->flags = 0;
597526d1006SLi Nan if (blk_queue_io_stat(md->queue))
598526d1006SLi Nan dm_io_set_flag(io, DM_IO_BLK_STAT);
59964f52b0eSMike Snitzer
600c4f512d2SMike Snitzer if (static_branch_unlikely(&stats_enabled) &&
601c4f512d2SMike Snitzer unlikely(dm_stats_used(&md->stats)))
6020cdb90f0SMike Snitzer dm_stats_record_start(&md->stats, &io->stats_aux);
60364f52b0eSMike Snitzer
60464f52b0eSMike Snitzer return io;
6051da177e4SLinus Torvalds }
6061da177e4SLinus Torvalds
free_io(struct dm_io * io)6070119ab14SMike Snitzer static void free_io(struct dm_io *io)
6081da177e4SLinus Torvalds {
60964f52b0eSMike Snitzer bio_put(&io->tio.clone);
61064f52b0eSMike Snitzer }
61164f52b0eSMike Snitzer
alloc_tio(struct clone_info * ci,struct dm_target * ti,unsigned int target_bio_nr,unsigned int * len,gfp_t gfp_mask)6121d1068ceSChristoph Hellwig static struct bio *alloc_tio(struct clone_info *ci, struct dm_target *ti,
61386a3238cSHeinz Mauelshagen unsigned int target_bio_nr, unsigned int *len, gfp_t gfp_mask)
61464f52b0eSMike Snitzer {
6159dd1cd32SMike Snitzer struct mapped_device *md = ci->io->md;
61664f52b0eSMike Snitzer struct dm_target_io *tio;
617018b05ebSMike Snitzer struct bio *clone;
61864f52b0eSMike Snitzer
61964f52b0eSMike Snitzer if (!ci->io->tio.io) {
62064f52b0eSMike Snitzer /* the dm_target_io embedded in ci->io is available */
62164f52b0eSMike Snitzer tio = &ci->io->tio;
622018b05ebSMike Snitzer /* alloc_io() already initialized embedded clone */
623018b05ebSMike Snitzer clone = &tio->clone;
62464f52b0eSMike Snitzer } else {
62529dec90aSChristoph Hellwig clone = bio_alloc_clone(NULL, ci->bio, gfp_mask,
62629dec90aSChristoph Hellwig &md->mempools->bs);
62764f52b0eSMike Snitzer if (!clone)
62864f52b0eSMike Snitzer return NULL;
62964f52b0eSMike Snitzer
630b99fdcdcSMing Lei /* REQ_DM_POLL_LIST shouldn't be inherited */
631b99fdcdcSMing Lei clone->bi_opf &= ~REQ_DM_POLL_LIST;
632b99fdcdcSMing Lei
6336c23f0bdSChristoph Hellwig tio = clone_to_tio(clone);
634655f3aadSMike Snitzer tio->flags = 0; /* also clears DM_TIO_INSIDE_DM_IO */
63564f52b0eSMike Snitzer }
63664f52b0eSMike Snitzer
63764f52b0eSMike Snitzer tio->magic = DM_TIO_MAGIC;
63864f52b0eSMike Snitzer tio->io = ci->io;
63964f52b0eSMike Snitzer tio->ti = ti;
64064f52b0eSMike Snitzer tio->target_bio_nr = target_bio_nr;
641dc8e2021SChristoph Hellwig tio->len_ptr = len;
642743598f0SMike Snitzer tio->old_sector = 0;
64364f52b0eSMike Snitzer
6449dd1cd32SMike Snitzer /* Set default bdev, but target must bio_set_dev() before issuing IO */
6459dd1cd32SMike Snitzer clone->bi_bdev = md->disk->part0;
6469dd1cd32SMike Snitzer if (unlikely(ti->needs_bio_set_dev))
6479dd1cd32SMike Snitzer bio_set_dev(clone, md->disk->part0);
6489dd1cd32SMike Snitzer
649018b05ebSMike Snitzer if (len) {
650018b05ebSMike Snitzer clone->bi_iter.bi_size = to_bytes(*len);
651018b05ebSMike Snitzer if (bio_integrity(clone))
652018b05ebSMike Snitzer bio_integrity_trim(clone);
653018b05ebSMike Snitzer }
654018b05ebSMike Snitzer
655018b05ebSMike Snitzer return clone;
6561da177e4SLinus Torvalds }
6571da177e4SLinus Torvalds
free_tio(struct bio * clone)6581d1068ceSChristoph Hellwig static void free_tio(struct bio *clone)
6591da177e4SLinus Torvalds {
660655f3aadSMike Snitzer if (dm_tio_flagged(clone_to_tio(clone), DM_TIO_INSIDE_DM_IO))
66164f52b0eSMike Snitzer return;
6621d1068ceSChristoph Hellwig bio_put(clone);
6631da177e4SLinus Torvalds }
6641da177e4SLinus Torvalds
6651da177e4SLinus Torvalds /*
6661da177e4SLinus Torvalds * Add the bio to the list of deferred io.
6671da177e4SLinus Torvalds */
queue_io(struct mapped_device * md,struct bio * bio)66892c63902SMikulas Patocka static void queue_io(struct mapped_device *md, struct bio *bio)
6691da177e4SLinus Torvalds {
67005447420SKiyoshi Ueda unsigned long flags;
6711da177e4SLinus Torvalds
67205447420SKiyoshi Ueda spin_lock_irqsave(&md->deferred_lock, flags);
6731da177e4SLinus Torvalds bio_list_add(&md->deferred, bio);
67405447420SKiyoshi Ueda spin_unlock_irqrestore(&md->deferred_lock, flags);
67592c63902SMikulas Patocka queue_work(md->wq, &md->work);
6761da177e4SLinus Torvalds }
6771da177e4SLinus Torvalds
6781da177e4SLinus Torvalds /*
6791da177e4SLinus Torvalds * Everyone (including functions in this file), should use this
6801da177e4SLinus Torvalds * function to access the md->map field, and make sure they call
68183d5e5b0SMikulas Patocka * dm_put_live_table() when finished.
6821da177e4SLinus Torvalds */
dm_get_live_table(struct mapped_device * md,int * srcu_idx)683563a225cSMike Snitzer struct dm_table *dm_get_live_table(struct mapped_device *md,
684563a225cSMike Snitzer int *srcu_idx) __acquires(md->io_barrier)
6851da177e4SLinus Torvalds {
68683d5e5b0SMikulas Patocka *srcu_idx = srcu_read_lock(&md->io_barrier);
6871da177e4SLinus Torvalds
68883d5e5b0SMikulas Patocka return srcu_dereference(md->map, &md->io_barrier);
68983d5e5b0SMikulas Patocka }
6901da177e4SLinus Torvalds
dm_put_live_table(struct mapped_device * md,int srcu_idx)691563a225cSMike Snitzer void dm_put_live_table(struct mapped_device *md,
692563a225cSMike Snitzer int srcu_idx) __releases(md->io_barrier)
69383d5e5b0SMikulas Patocka {
69483d5e5b0SMikulas Patocka srcu_read_unlock(&md->io_barrier, srcu_idx);
69583d5e5b0SMikulas Patocka }
69683d5e5b0SMikulas Patocka
dm_sync_table(struct mapped_device * md)69783d5e5b0SMikulas Patocka void dm_sync_table(struct mapped_device *md)
69883d5e5b0SMikulas Patocka {
69983d5e5b0SMikulas Patocka synchronize_srcu(&md->io_barrier);
70083d5e5b0SMikulas Patocka synchronize_rcu_expedited();
70183d5e5b0SMikulas Patocka }
70283d5e5b0SMikulas Patocka
70383d5e5b0SMikulas Patocka /*
70483d5e5b0SMikulas Patocka * A fast alternative to dm_get_live_table/dm_put_live_table.
70583d5e5b0SMikulas Patocka * The caller must not block between these two functions.
70683d5e5b0SMikulas Patocka */
dm_get_live_table_fast(struct mapped_device * md)70783d5e5b0SMikulas Patocka static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU)
70883d5e5b0SMikulas Patocka {
70983d5e5b0SMikulas Patocka rcu_read_lock();
71083d5e5b0SMikulas Patocka return rcu_dereference(md->map);
71183d5e5b0SMikulas Patocka }
71283d5e5b0SMikulas Patocka
dm_put_live_table_fast(struct mapped_device * md)71383d5e5b0SMikulas Patocka static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU)
71483d5e5b0SMikulas Patocka {
71583d5e5b0SMikulas Patocka rcu_read_unlock();
7161da177e4SLinus Torvalds }
7171da177e4SLinus Torvalds
718971888c4SMike Snitzer static char *_dm_claim_ptr = "I belong to device-mapper";
719971888c4SMike Snitzer
7203ac51e74SDarrick J. Wong /*
72186f1152bSBenjamin Marzinski * Open a table device so we can use it as a map destination.
72286f1152bSBenjamin Marzinski */
open_table_device(struct mapped_device * md,dev_t dev,blk_mode_t mode)723b9a785d2SChristoph Hellwig static struct table_device *open_table_device(struct mapped_device *md,
72405bdb996SChristoph Hellwig dev_t dev, blk_mode_t mode)
72586f1152bSBenjamin Marzinski {
726b9a785d2SChristoph Hellwig struct table_device *td;
72786f1152bSBenjamin Marzinski struct block_device *bdev;
728cd913c76SChristoph Hellwig u64 part_off;
72986f1152bSBenjamin Marzinski int r;
73086f1152bSBenjamin Marzinski
731b9a785d2SChristoph Hellwig td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id);
732b9a785d2SChristoph Hellwig if (!td)
733b9a785d2SChristoph Hellwig return ERR_PTR(-ENOMEM);
734b9a785d2SChristoph Hellwig refcount_set(&td->count, 1);
73586f1152bSBenjamin Marzinski
7362736e8eeSChristoph Hellwig bdev = blkdev_get_by_dev(dev, mode, _dm_claim_ptr, NULL);
737b9a785d2SChristoph Hellwig if (IS_ERR(bdev)) {
738b9a785d2SChristoph Hellwig r = PTR_ERR(bdev);
739b9a785d2SChristoph Hellwig goto out_free_td;
74086f1152bSBenjamin Marzinski }
74186f1152bSBenjamin Marzinski
7421a581b72SChristoph Hellwig /*
7431a581b72SChristoph Hellwig * We can be called before the dm disk is added. In that case we can't
7441a581b72SChristoph Hellwig * register the holder relation here. It will be done once add_disk was
7451a581b72SChristoph Hellwig * called.
7461a581b72SChristoph Hellwig */
7471a581b72SChristoph Hellwig if (md->disk->slave_dir) {
7481a581b72SChristoph Hellwig r = bd_link_disk_holder(bdev, md->disk);
749b9a785d2SChristoph Hellwig if (r)
750b9a785d2SChristoph Hellwig goto out_blkdev_put;
7511a581b72SChristoph Hellwig }
752b9a785d2SChristoph Hellwig
753b9a785d2SChristoph Hellwig td->dm_dev.mode = mode;
75486f1152bSBenjamin Marzinski td->dm_dev.bdev = bdev;
7558012b866SShiyang Ruan td->dm_dev.dax_dev = fs_dax_get_by_bdev(bdev, &part_off, NULL, NULL);
756b9a785d2SChristoph Hellwig format_dev_t(td->dm_dev.name, dev);
757b9a785d2SChristoph Hellwig list_add(&td->list, &md->table_devices);
758b9a785d2SChristoph Hellwig return td;
759b9a785d2SChristoph Hellwig
760b9a785d2SChristoph Hellwig out_blkdev_put:
7612736e8eeSChristoph Hellwig blkdev_put(bdev, _dm_claim_ptr);
762b9a785d2SChristoph Hellwig out_free_td:
763b9a785d2SChristoph Hellwig kfree(td);
764b9a785d2SChristoph Hellwig return ERR_PTR(r);
76586f1152bSBenjamin Marzinski }
76686f1152bSBenjamin Marzinski
76786f1152bSBenjamin Marzinski /*
76886f1152bSBenjamin Marzinski * Close a table device that we've been using.
76986f1152bSBenjamin Marzinski */
close_table_device(struct table_device * td,struct mapped_device * md)77086f1152bSBenjamin Marzinski static void close_table_device(struct table_device *td, struct mapped_device *md)
77186f1152bSBenjamin Marzinski {
7721a581b72SChristoph Hellwig if (md->disk->slave_dir)
7731a581b72SChristoph Hellwig bd_unlink_disk_holder(td->dm_dev.bdev, md->disk);
7742736e8eeSChristoph Hellwig blkdev_put(td->dm_dev.bdev, _dm_claim_ptr);
775817bf402SDan Williams put_dax(td->dm_dev.dax_dev);
7767b586583SChristoph Hellwig list_del(&td->list);
7777b586583SChristoph Hellwig kfree(td);
77886f1152bSBenjamin Marzinski }
77986f1152bSBenjamin Marzinski
find_table_device(struct list_head * l,dev_t dev,blk_mode_t mode)78086f1152bSBenjamin Marzinski static struct table_device *find_table_device(struct list_head *l, dev_t dev,
78105bdb996SChristoph Hellwig blk_mode_t mode)
7828454fca4SSheetal Singala {
78386f1152bSBenjamin Marzinski struct table_device *td;
78486f1152bSBenjamin Marzinski
78586f1152bSBenjamin Marzinski list_for_each_entry(td, l, list)
78686f1152bSBenjamin Marzinski if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode)
78786f1152bSBenjamin Marzinski return td;
78886f1152bSBenjamin Marzinski
78986f1152bSBenjamin Marzinski return NULL;
79086f1152bSBenjamin Marzinski }
79186f1152bSBenjamin Marzinski
dm_get_table_device(struct mapped_device * md,dev_t dev,blk_mode_t mode,struct dm_dev ** result)79205bdb996SChristoph Hellwig int dm_get_table_device(struct mapped_device *md, dev_t dev, blk_mode_t mode,
7938454fca4SSheetal Singala struct dm_dev **result)
7948454fca4SSheetal Singala {
79586f1152bSBenjamin Marzinski struct table_device *td;
79686f1152bSBenjamin Marzinski
79786f1152bSBenjamin Marzinski mutex_lock(&md->table_devices_lock);
79886f1152bSBenjamin Marzinski td = find_table_device(&md->table_devices, dev, mode);
79986f1152bSBenjamin Marzinski if (!td) {
800b9a785d2SChristoph Hellwig td = open_table_device(md, dev, mode);
801b9a785d2SChristoph Hellwig if (IS_ERR(td)) {
80286f1152bSBenjamin Marzinski mutex_unlock(&md->table_devices_lock);
803b9a785d2SChristoph Hellwig return PTR_ERR(td);
80486f1152bSBenjamin Marzinski }
805b0b4d7c6SElena Reshetova } else {
806b0b4d7c6SElena Reshetova refcount_inc(&td->count);
80786f1152bSBenjamin Marzinski }
80886f1152bSBenjamin Marzinski mutex_unlock(&md->table_devices_lock);
80986f1152bSBenjamin Marzinski
81086f1152bSBenjamin Marzinski *result = &td->dm_dev;
81186f1152bSBenjamin Marzinski return 0;
81286f1152bSBenjamin Marzinski }
81386f1152bSBenjamin Marzinski
dm_put_table_device(struct mapped_device * md,struct dm_dev * d)81486f1152bSBenjamin Marzinski void dm_put_table_device(struct mapped_device *md, struct dm_dev *d)
81586f1152bSBenjamin Marzinski {
81686f1152bSBenjamin Marzinski struct table_device *td = container_of(d, struct table_device, dm_dev);
81786f1152bSBenjamin Marzinski
81886f1152bSBenjamin Marzinski mutex_lock(&md->table_devices_lock);
8197b586583SChristoph Hellwig if (refcount_dec_and_test(&td->count))
82086f1152bSBenjamin Marzinski close_table_device(td, md);
82186f1152bSBenjamin Marzinski mutex_unlock(&md->table_devices_lock);
82286f1152bSBenjamin Marzinski }
82386f1152bSBenjamin Marzinski
82486f1152bSBenjamin Marzinski /*
8253ac51e74SDarrick J. Wong * Get the geometry associated with a dm device
8263ac51e74SDarrick J. Wong */
dm_get_geometry(struct mapped_device * md,struct hd_geometry * geo)8273ac51e74SDarrick J. Wong int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
8283ac51e74SDarrick J. Wong {
8293ac51e74SDarrick J. Wong *geo = md->geometry;
8303ac51e74SDarrick J. Wong
8313ac51e74SDarrick J. Wong return 0;
8323ac51e74SDarrick J. Wong }
8333ac51e74SDarrick J. Wong
8343ac51e74SDarrick J. Wong /*
8353ac51e74SDarrick J. Wong * Set the geometry of a device.
8363ac51e74SDarrick J. Wong */
dm_set_geometry(struct mapped_device * md,struct hd_geometry * geo)8373ac51e74SDarrick J. Wong int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
8383ac51e74SDarrick J. Wong {
8393ac51e74SDarrick J. Wong sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
8403ac51e74SDarrick J. Wong
8413ac51e74SDarrick J. Wong if (geo->start > sz) {
84243e6c111SMikulas Patocka DMERR("Start sector is beyond the geometry limits.");
8433ac51e74SDarrick J. Wong return -EINVAL;
8443ac51e74SDarrick J. Wong }
8453ac51e74SDarrick J. Wong
8463ac51e74SDarrick J. Wong md->geometry = *geo;
8473ac51e74SDarrick J. Wong
8483ac51e74SDarrick J. Wong return 0;
8493ac51e74SDarrick J. Wong }
8503ac51e74SDarrick J. Wong
__noflush_suspending(struct mapped_device * md)8512e93ccc1SKiyoshi Ueda static int __noflush_suspending(struct mapped_device *md)
8522e93ccc1SKiyoshi Ueda {
8532e93ccc1SKiyoshi Ueda return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
8542e93ccc1SKiyoshi Ueda }
8552e93ccc1SKiyoshi Ueda
dm_requeue_add_io(struct dm_io * io,bool first_stage)8568b211aacSMing Lei static void dm_requeue_add_io(struct dm_io *io, bool first_stage)
8571da177e4SLinus Torvalds {
858b35f8caaSMilan Broz struct mapped_device *md = io->md;
8592e93ccc1SKiyoshi Ueda
8608b211aacSMing Lei if (first_stage) {
8618b211aacSMing Lei struct dm_io *next = md->requeue_list;
8628b211aacSMing Lei
8638b211aacSMing Lei md->requeue_list = io;
8648b211aacSMing Lei io->next = next;
8658b211aacSMing Lei } else {
8668b211aacSMing Lei bio_list_add_head(&md->deferred, io->orig_bio);
8678b211aacSMing Lei }
8688b211aacSMing Lei }
8698b211aacSMing Lei
dm_kick_requeue(struct mapped_device * md,bool first_stage)8708b211aacSMing Lei static void dm_kick_requeue(struct mapped_device *md, bool first_stage)
8718b211aacSMing Lei {
8728b211aacSMing Lei if (first_stage)
8738b211aacSMing Lei queue_work(md->wq, &md->requeue_work);
8748b211aacSMing Lei else
8758b211aacSMing Lei queue_work(md->wq, &md->work);
8768b211aacSMing Lei }
8778b211aacSMing Lei
8782e93ccc1SKiyoshi Ueda /*
879444fe04fSMing Lei * Return true if the dm_io's original bio is requeued.
880444fe04fSMing Lei * io->status is updated with error if requeue disallowed.
881444fe04fSMing Lei */
dm_handle_requeue(struct dm_io * io,bool first_stage)8828b211aacSMing Lei static bool dm_handle_requeue(struct dm_io *io, bool first_stage)
883444fe04fSMing Lei {
8848b211aacSMing Lei struct bio *bio = io->orig_bio;
885444fe04fSMing Lei bool handle_requeue = (io->status == BLK_STS_DM_REQUEUE);
886444fe04fSMing Lei bool handle_polled_eagain = ((io->status == BLK_STS_AGAIN) &&
887444fe04fSMing Lei (bio->bi_opf & REQ_POLLED));
888444fe04fSMing Lei struct mapped_device *md = io->md;
889444fe04fSMing Lei bool requeued = false;
890444fe04fSMing Lei
891444fe04fSMing Lei if (handle_requeue || handle_polled_eagain) {
892444fe04fSMing Lei unsigned long flags;
893444fe04fSMing Lei
894444fe04fSMing Lei if (bio->bi_opf & REQ_POLLED) {
895444fe04fSMing Lei /*
896444fe04fSMing Lei * Upper layer won't help us poll split bio
897444fe04fSMing Lei * (io->orig_bio may only reflect a subset of the
898444fe04fSMing Lei * pre-split original) so clear REQ_POLLED.
899444fe04fSMing Lei */
900444fe04fSMing Lei bio_clear_polled(bio);
901444fe04fSMing Lei }
902444fe04fSMing Lei
903444fe04fSMing Lei /*
904444fe04fSMing Lei * Target requested pushing back the I/O or
905444fe04fSMing Lei * polled IO hit BLK_STS_AGAIN.
9062e93ccc1SKiyoshi Ueda */
907022c2611SMikulas Patocka spin_lock_irqsave(&md->deferred_lock, flags);
908444fe04fSMing Lei if ((__noflush_suspending(md) &&
909444fe04fSMing Lei !WARN_ON_ONCE(dm_is_zone_write(md, bio))) ||
9108b211aacSMing Lei handle_polled_eagain || first_stage) {
9118b211aacSMing Lei dm_requeue_add_io(io, first_stage);
912444fe04fSMing Lei requeued = true;
913bf14e2b2SDamien Le Moal } else {
914bf14e2b2SDamien Le Moal /*
915bf14e2b2SDamien Le Moal * noflush suspend was interrupted or this is
916bf14e2b2SDamien Le Moal * a write to a zoned target.
917bf14e2b2SDamien Le Moal */
9184e4cbee9SChristoph Hellwig io->status = BLK_STS_IOERR;
919bf14e2b2SDamien Le Moal }
920022c2611SMikulas Patocka spin_unlock_irqrestore(&md->deferred_lock, flags);
9212e93ccc1SKiyoshi Ueda }
9222e93ccc1SKiyoshi Ueda
923444fe04fSMing Lei if (requeued)
9248b211aacSMing Lei dm_kick_requeue(md, first_stage);
925444fe04fSMing Lei
926444fe04fSMing Lei return requeued;
927444fe04fSMing Lei }
928444fe04fSMing Lei
__dm_io_complete(struct dm_io * io,bool first_stage)9298b211aacSMing Lei static void __dm_io_complete(struct dm_io *io, bool first_stage)
930444fe04fSMing Lei {
9318b211aacSMing Lei struct bio *bio = io->orig_bio;
932444fe04fSMing Lei struct mapped_device *md = io->md;
933444fe04fSMing Lei blk_status_t io_error;
934444fe04fSMing Lei bool requeued;
935444fe04fSMing Lei
9368b211aacSMing Lei requeued = dm_handle_requeue(io, first_stage);
9378b211aacSMing Lei if (requeued && first_stage)
9388b211aacSMing Lei return;
939444fe04fSMing Lei
9404e4cbee9SChristoph Hellwig io_error = io->status;
94182f6cdccSMike Snitzer if (dm_io_flagged(io, DM_IO_ACCOUNTED))
942b992b40dSMing Lei dm_end_io_acct(io);
9430fbb4d93SMike Snitzer else if (!io_error) {
9440fbb4d93SMike Snitzer /*
9450fbb4d93SMike Snitzer * Must handle target that DM_MAPIO_SUBMITTED only to
9460fbb4d93SMike Snitzer * then bio_endio() rather than dm_submit_bio_remap()
9470fbb4d93SMike Snitzer */
948b992b40dSMing Lei __dm_start_io_acct(io);
949b992b40dSMing Lei dm_end_io_acct(io);
9500fbb4d93SMike Snitzer }
9510119ab14SMike Snitzer free_io(io);
9529f6dc633SMike Snitzer smp_wmb();
9539f6dc633SMike Snitzer this_cpu_dec(*md->pending_io);
9542056a782SJens Axboe
9559f6dc633SMike Snitzer /* nudge anyone waiting on suspend queue */
9569f6dc633SMike Snitzer if (unlikely(wq_has_sleeper(&md->wait)))
9579f6dc633SMike Snitzer wake_up(&md->wait);
9581da177e4SLinus Torvalds
959444fe04fSMing Lei /* Return early if the original bio was requeued */
960444fe04fSMing Lei if (requeued)
96178ccef91SMike Snitzer return;
9626a8736d1STejun Heo
9638d394bc4SMike Snitzer if (bio_is_flush_with_data(bio)) {
964af7e466aSMikulas Patocka /*
9656a8736d1STejun Heo * Preflush done for flush with data, reissue
96628a8f0d3SMike Christie * without REQ_PREFLUSH.
967af7e466aSMikulas Patocka */
9681eff9d32SJens Axboe bio->bi_opf &= ~REQ_PREFLUSH;
9696a8736d1STejun Heo queue_io(md, bio);
970af7e466aSMikulas Patocka } else {
971b372d360SMike Snitzer /* done with normal IO or empty flush */
9728dd601faSNeilBrown if (io_error)
9734e4cbee9SChristoph Hellwig bio->bi_status = io_error;
9744246a0b6SChristoph Hellwig bio_endio(bio);
9752e93ccc1SKiyoshi Ueda }
9761da177e4SLinus Torvalds }
977e2736347SMike Snitzer
dm_wq_requeue_work(struct work_struct * work)9788b211aacSMing Lei static void dm_wq_requeue_work(struct work_struct *work)
9798b211aacSMing Lei {
9808b211aacSMing Lei struct mapped_device *md = container_of(work, struct mapped_device,
9818b211aacSMing Lei requeue_work);
9828b211aacSMing Lei unsigned long flags;
9838b211aacSMing Lei struct dm_io *io;
9848b211aacSMing Lei
9858b211aacSMing Lei /* reuse deferred lock to simplify dm_handle_requeue */
9868b211aacSMing Lei spin_lock_irqsave(&md->deferred_lock, flags);
9878b211aacSMing Lei io = md->requeue_list;
9888b211aacSMing Lei md->requeue_list = NULL;
9898b211aacSMing Lei spin_unlock_irqrestore(&md->deferred_lock, flags);
9908b211aacSMing Lei
9918b211aacSMing Lei while (io) {
9928b211aacSMing Lei struct dm_io *next = io->next;
9938b211aacSMing Lei
99446754bd0SChristoph Hellwig dm_io_rewind(io, &md->disk->bio_split);
9958b211aacSMing Lei
9968b211aacSMing Lei io->next = NULL;
9978b211aacSMing Lei __dm_io_complete(io, false);
9988b211aacSMing Lei io = next;
999f77692d6SMike Snitzer cond_resched();
10008b211aacSMing Lei }
10018b211aacSMing Lei }
10028b211aacSMing Lei
10038b211aacSMing Lei /*
10048b211aacSMing Lei * Two staged requeue:
10058b211aacSMing Lei *
10068b211aacSMing Lei * 1) io->orig_bio points to the real original bio, and the part mapped to
10078b211aacSMing Lei * this io must be requeued, instead of other parts of the original bio.
10088b211aacSMing Lei *
10098b211aacSMing Lei * 2) io->orig_bio points to new cloned bio which matches the requeued dm_io.
10108b211aacSMing Lei */
dm_io_complete(struct dm_io * io)10118b211aacSMing Lei static void dm_io_complete(struct dm_io *io)
10128b211aacSMing Lei {
10138b211aacSMing Lei bool first_requeue;
10148b211aacSMing Lei
10158b211aacSMing Lei /*
10168b211aacSMing Lei * Only dm_io that has been split needs two stage requeue, otherwise
10178b211aacSMing Lei * we may run into long bio clone chain during suspend and OOM could
10188b211aacSMing Lei * be triggered.
10198b211aacSMing Lei *
10208b211aacSMing Lei * Also flush data dm_io won't be marked as DM_IO_WAS_SPLIT, so they
10218b211aacSMing Lei * also aren't handled via the first stage requeue.
10228b211aacSMing Lei */
10238b211aacSMing Lei if (dm_io_flagged(io, DM_IO_WAS_SPLIT))
10248b211aacSMing Lei first_requeue = true;
10258b211aacSMing Lei else
10268b211aacSMing Lei first_requeue = false;
10278b211aacSMing Lei
10288b211aacSMing Lei __dm_io_complete(io, first_requeue);
10298b211aacSMing Lei }
10308b211aacSMing Lei
1031e2736347SMike Snitzer /*
1032e2736347SMike Snitzer * Decrements the number of outstanding ios that a bio has been
1033e2736347SMike Snitzer * cloned into, completing the original io if necc.
1034e2736347SMike Snitzer */
__dm_io_dec_pending(struct dm_io * io)103584b98f4cSMike Snitzer static inline void __dm_io_dec_pending(struct dm_io *io)
1036e2736347SMike Snitzer {
103784b98f4cSMike Snitzer if (atomic_dec_and_test(&io->io_count))
103884b98f4cSMike Snitzer dm_io_complete(io);
103984b98f4cSMike Snitzer }
104084b98f4cSMike Snitzer
dm_io_set_error(struct dm_io * io,blk_status_t error)104184b98f4cSMike Snitzer static void dm_io_set_error(struct dm_io *io, blk_status_t error)
104284b98f4cSMike Snitzer {
1043e2736347SMike Snitzer unsigned long flags;
104484b98f4cSMike Snitzer
104584b98f4cSMike Snitzer /* Push-back supersedes any I/O errors */
10464d7bca13SMike Snitzer spin_lock_irqsave(&io->lock, flags);
1047e2736347SMike Snitzer if (!(io->status == BLK_STS_DM_REQUEUE &&
104884b98f4cSMike Snitzer __noflush_suspending(io->md))) {
1049e2736347SMike Snitzer io->status = error;
105084b98f4cSMike Snitzer }
10514d7bca13SMike Snitzer spin_unlock_irqrestore(&io->lock, flags);
1052e2736347SMike Snitzer }
1053e2736347SMike Snitzer
dm_io_dec_pending(struct dm_io * io,blk_status_t error)10542e803cd9SMing Lei static void dm_io_dec_pending(struct dm_io *io, blk_status_t error)
105584b98f4cSMike Snitzer {
105684b98f4cSMike Snitzer if (unlikely(error))
105784b98f4cSMike Snitzer dm_io_set_error(io, error);
105884b98f4cSMike Snitzer
105984b98f4cSMike Snitzer __dm_io_dec_pending(io);
1060af7e466aSMikulas Patocka }
10611da177e4SLinus Torvalds
1062f7995089SMike Snitzer /*
1063f7995089SMike Snitzer * The queue_limits are only valid as long as you have a reference
1064f7995089SMike Snitzer * count on 'md'. But _not_ imposing verification to avoid atomic_read(),
1065f7995089SMike Snitzer */
dm_get_queue_limits(struct mapped_device * md)1066f7995089SMike Snitzer static inline struct queue_limits *dm_get_queue_limits(struct mapped_device *md)
1067f7995089SMike Snitzer {
1068f7995089SMike Snitzer return &md->queue->limits;
1069f7995089SMike Snitzer }
1070f7995089SMike Snitzer
disable_discard(struct mapped_device * md)1071bcb44433SMike Snitzer void disable_discard(struct mapped_device *md)
1072bcb44433SMike Snitzer {
1073bcb44433SMike Snitzer struct queue_limits *limits = dm_get_queue_limits(md);
1074bcb44433SMike Snitzer
1075bcb44433SMike Snitzer /* device doesn't really support DISCARD, disable it */
1076bcb44433SMike Snitzer limits->max_discard_sectors = 0;
1077bcb44433SMike Snitzer }
1078bcb44433SMike Snitzer
disable_write_zeroes(struct mapped_device * md)1079ac62d620SChristoph Hellwig void disable_write_zeroes(struct mapped_device *md)
1080ac62d620SChristoph Hellwig {
1081ac62d620SChristoph Hellwig struct queue_limits *limits = dm_get_queue_limits(md);
1082ac62d620SChristoph Hellwig
1083ac62d620SChristoph Hellwig /* device doesn't really support WRITE ZEROES, disable it */
1084ac62d620SChristoph Hellwig limits->max_write_zeroes_sectors = 0;
1085ac62d620SChristoph Hellwig }
1086ac62d620SChristoph Hellwig
swap_bios_limit(struct dm_target * ti,struct bio * bio)1087a666e5c0SMikulas Patocka static bool swap_bios_limit(struct dm_target *ti, struct bio *bio)
1088a666e5c0SMikulas Patocka {
1089a666e5c0SMikulas Patocka return unlikely((bio->bi_opf & REQ_SWAP) != 0) && unlikely(ti->limit_swap_bios);
1090a666e5c0SMikulas Patocka }
1091a666e5c0SMikulas Patocka
clone_endio(struct bio * bio)10924246a0b6SChristoph Hellwig static void clone_endio(struct bio *bio)
10931da177e4SLinus Torvalds {
10944e4cbee9SChristoph Hellwig blk_status_t error = bio->bi_status;
10956cbce280SMike Snitzer struct dm_target_io *tio = clone_to_tio(bio);
10966cbce280SMike Snitzer struct dm_target *ti = tio->ti;
10976cbce280SMike Snitzer dm_endio_fn endio = ti->type->end_io;
10986cbce280SMike Snitzer struct dm_io *io = tio->io;
10996cbce280SMike Snitzer struct mapped_device *md = io->md;
11001da177e4SLinus Torvalds
11019c37de29SMike Snitzer if (unlikely(error == BLK_STS_TARGET)) {
1102bcb44433SMike Snitzer if (bio_op(bio) == REQ_OP_DISCARD &&
110370200574SChristoph Hellwig !bdev_max_discard_sectors(bio->bi_bdev))
1104bcb44433SMike Snitzer disable_discard(md);
1105bcb44433SMike Snitzer else if (bio_op(bio) == REQ_OP_WRITE_ZEROES &&
1106dddf3056SMike Snitzer !bdev_write_zeroes_sectors(bio->bi_bdev))
1107ac62d620SChristoph Hellwig disable_write_zeroes(md);
1108ac62d620SChristoph Hellwig }
11097eee4ae2SMike Snitzer
1110442761fdSMike Snitzer if (static_branch_unlikely(&zoned_enabled) &&
1111edd1dbc8SChristoph Hellwig unlikely(bdev_is_zoned(bio->bi_bdev)))
1112bb37d772SDamien Le Moal dm_zone_endio(io, bio);
1113415c79e1SJohannes Thumshirn
11141be56909SChristoph Hellwig if (endio) {
11156cbce280SMike Snitzer int r = endio(ti, bio, &error);
1116b30f1607SHeinz Mauelshagen
11171be56909SChristoph Hellwig switch (r) {
11181be56909SChristoph Hellwig case DM_ENDIO_REQUEUE:
1119442761fdSMike Snitzer if (static_branch_unlikely(&zoned_enabled)) {
1120bf14e2b2SDamien Le Moal /*
1121bf14e2b2SDamien Le Moal * Requeuing writes to a sequential zone of a zoned
1122bf14e2b2SDamien Le Moal * target will break the sequential write pattern:
1123bf14e2b2SDamien Le Moal * fail such IO.
1124bf14e2b2SDamien Le Moal */
1125bf14e2b2SDamien Le Moal if (WARN_ON_ONCE(dm_is_zone_write(md, bio)))
1126bf14e2b2SDamien Le Moal error = BLK_STS_IOERR;
1127bf14e2b2SDamien Le Moal else
11284e4cbee9SChristoph Hellwig error = BLK_STS_DM_REQUEUE;
1129442761fdSMike Snitzer } else
1130442761fdSMike Snitzer error = BLK_STS_DM_REQUEUE;
1131df561f66SGustavo A. R. Silva fallthrough;
11321be56909SChristoph Hellwig case DM_ENDIO_DONE:
11331be56909SChristoph Hellwig break;
11341be56909SChristoph Hellwig case DM_ENDIO_INCOMPLETE:
11351be56909SChristoph Hellwig /* The target will handle the io */
11361be56909SChristoph Hellwig return;
11371be56909SChristoph Hellwig default:
113843e6c111SMikulas Patocka DMCRIT("unimplemented target endio return value: %d", r);
11391be56909SChristoph Hellwig BUG();
11401be56909SChristoph Hellwig }
11411be56909SChristoph Hellwig }
11421be56909SChristoph Hellwig
1143442761fdSMike Snitzer if (static_branch_unlikely(&swap_bios_enabled) &&
1144442761fdSMike Snitzer unlikely(swap_bios_limit(ti, bio)))
1145a666e5c0SMikulas Patocka up(&md->swap_bios_semaphore);
1146a666e5c0SMikulas Patocka
11471d1068ceSChristoph Hellwig free_tio(bio);
1148e2118b3cSDamien Le Moal dm_io_dec_pending(io, error);
11491da177e4SLinus Torvalds }
11501da177e4SLinus Torvalds
115178d8e58aSMike Snitzer /*
115256a67df7SMike Snitzer * Return maximum size of I/O possible at the supplied sector up to the current
115356a67df7SMike Snitzer * target boundary.
115456a67df7SMike Snitzer */
max_io_len_target_boundary(struct dm_target * ti,sector_t target_offset)11553720281dSMike Snitzer static inline sector_t max_io_len_target_boundary(struct dm_target *ti,
11563720281dSMike Snitzer sector_t target_offset)
11571da177e4SLinus Torvalds {
115856a67df7SMike Snitzer return ti->len - target_offset;
115956a67df7SMike Snitzer }
116056a67df7SMike Snitzer
__max_io_len(struct dm_target * ti,sector_t sector,unsigned int max_granularity,unsigned int max_sectors)116106961c48SMike Snitzer static sector_t __max_io_len(struct dm_target *ti, sector_t sector,
1162be04c14aSMike Snitzer unsigned int max_granularity,
1163be04c14aSMike Snitzer unsigned int max_sectors)
116456a67df7SMike Snitzer {
11653720281dSMike Snitzer sector_t target_offset = dm_target_offset(ti, sector);
11663720281dSMike Snitzer sector_t len = max_io_len_target_boundary(ti, target_offset);
11671da177e4SLinus Torvalds
11681da177e4SLinus Torvalds /*
11693ee16db3SMike Snitzer * Does the target need to split IO even further?
11703ee16db3SMike Snitzer * - varied (per target) IO splitting is a tenet of DM; this
11713ee16db3SMike Snitzer * explains why stacked chunk_sectors based splitting via
11725a97806fSChristoph Hellwig * bio_split_to_limits() isn't possible here.
11731da177e4SLinus Torvalds */
117406961c48SMike Snitzer if (!max_granularity)
11751da177e4SLinus Torvalds return len;
1176c3949322SChristoph Hellwig return min_t(sector_t, len,
1177be04c14aSMike Snitzer min(max_sectors ? : queue_max_sectors(ti->table->md->queue),
117806961c48SMike Snitzer blk_chunk_sectors_left(target_offset, max_granularity)));
117906961c48SMike Snitzer }
118006961c48SMike Snitzer
max_io_len(struct dm_target * ti,sector_t sector)118106961c48SMike Snitzer static inline sector_t max_io_len(struct dm_target *ti, sector_t sector)
118206961c48SMike Snitzer {
1183be04c14aSMike Snitzer return __max_io_len(ti, sector, ti->max_io_len, 0);
11841da177e4SLinus Torvalds }
11851da177e4SLinus Torvalds
dm_set_target_max_io_len(struct dm_target * ti,sector_t len)1186542f9038SMike Snitzer int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
1187542f9038SMike Snitzer {
1188542f9038SMike Snitzer if (len > UINT_MAX) {
1189542f9038SMike Snitzer DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)",
1190542f9038SMike Snitzer (unsigned long long)len, UINT_MAX);
1191542f9038SMike Snitzer ti->error = "Maximum size of target IO is too large";
1192542f9038SMike Snitzer return -EINVAL;
1193542f9038SMike Snitzer }
1194542f9038SMike Snitzer
119575ae1936SMikulas Patocka ti->max_io_len = (uint32_t) len;
1196542f9038SMike Snitzer
1197542f9038SMike Snitzer return 0;
1198542f9038SMike Snitzer }
1199542f9038SMike Snitzer EXPORT_SYMBOL_GPL(dm_set_target_max_io_len);
1200542f9038SMike Snitzer
dm_dax_get_live_target(struct mapped_device * md,sector_t sector,int * srcu_idx)1201f26c5719SDan Williams static struct dm_target *dm_dax_get_live_target(struct mapped_device *md,
1202f26c5719SDan Williams sector_t sector, int *srcu_idx)
12033d97c829SMike Snitzer __acquires(md->io_barrier)
1204545ed20eSToshi Kani {
1205545ed20eSToshi Kani struct dm_table *map;
1206545ed20eSToshi Kani struct dm_target *ti;
1207545ed20eSToshi Kani
1208f26c5719SDan Williams map = dm_get_live_table(md, srcu_idx);
1209545ed20eSToshi Kani if (!map)
1210f26c5719SDan Williams return NULL;
1211545ed20eSToshi Kani
1212545ed20eSToshi Kani ti = dm_table_find_target(map, sector);
1213123d87d5SMikulas Patocka if (!ti)
1214f26c5719SDan Williams return NULL;
1215f26c5719SDan Williams
1216f26c5719SDan Williams return ti;
1217f26c5719SDan Williams }
1218f26c5719SDan Williams
dm_dax_direct_access(struct dax_device * dax_dev,pgoff_t pgoff,long nr_pages,enum dax_access_mode mode,void ** kaddr,pfn_t * pfn)1219f26c5719SDan Williams static long dm_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
1220e511c4a3SJane Chu long nr_pages, enum dax_access_mode mode, void **kaddr,
1221e511c4a3SJane Chu pfn_t *pfn)
1222f26c5719SDan Williams {
1223f26c5719SDan Williams struct mapped_device *md = dax_get_private(dax_dev);
1224f26c5719SDan Williams sector_t sector = pgoff * PAGE_SECTORS;
1225f26c5719SDan Williams struct dm_target *ti;
1226f26c5719SDan Williams long len, ret = -EIO;
1227f26c5719SDan Williams int srcu_idx;
1228f26c5719SDan Williams
1229f26c5719SDan Williams ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1230f26c5719SDan Williams
1231f26c5719SDan Williams if (!ti)
1232545ed20eSToshi Kani goto out;
1233f26c5719SDan Williams if (!ti->type->direct_access)
1234f26c5719SDan Williams goto out;
12353720281dSMike Snitzer len = max_io_len(ti, sector) / PAGE_SECTORS;
1236f26c5719SDan Williams if (len < 1)
1237f26c5719SDan Williams goto out;
1238f26c5719SDan Williams nr_pages = min(len, nr_pages);
1239e511c4a3SJane Chu ret = ti->type->direct_access(ti, pgoff, nr_pages, mode, kaddr, pfn);
1240817bf402SDan Williams
1241545ed20eSToshi Kani out:
1242545ed20eSToshi Kani dm_put_live_table(md, srcu_idx);
1243f26c5719SDan Williams
1244f26c5719SDan Williams return ret;
1245545ed20eSToshi Kani }
1246545ed20eSToshi Kani
dm_dax_zero_page_range(struct dax_device * dax_dev,pgoff_t pgoff,size_t nr_pages)1247cdf6cdcdSVivek Goyal static int dm_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,
1248cdf6cdcdSVivek Goyal size_t nr_pages)
1249cdf6cdcdSVivek Goyal {
1250cdf6cdcdSVivek Goyal struct mapped_device *md = dax_get_private(dax_dev);
1251cdf6cdcdSVivek Goyal sector_t sector = pgoff * PAGE_SECTORS;
1252cdf6cdcdSVivek Goyal struct dm_target *ti;
1253cdf6cdcdSVivek Goyal int ret = -EIO;
1254cdf6cdcdSVivek Goyal int srcu_idx;
1255cdf6cdcdSVivek Goyal
1256cdf6cdcdSVivek Goyal ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1257cdf6cdcdSVivek Goyal
1258cdf6cdcdSVivek Goyal if (!ti)
1259cdf6cdcdSVivek Goyal goto out;
1260cdf6cdcdSVivek Goyal if (WARN_ON(!ti->type->dax_zero_page_range)) {
1261cdf6cdcdSVivek Goyal /*
1262cdf6cdcdSVivek Goyal * ->zero_page_range() is mandatory dax operation. If we are
1263cdf6cdcdSVivek Goyal * here, something is wrong.
1264cdf6cdcdSVivek Goyal */
1265cdf6cdcdSVivek Goyal goto out;
1266cdf6cdcdSVivek Goyal }
1267cdf6cdcdSVivek Goyal ret = ti->type->dax_zero_page_range(ti, pgoff, nr_pages);
1268cdf6cdcdSVivek Goyal out:
1269cdf6cdcdSVivek Goyal dm_put_live_table(md, srcu_idx);
1270cdf6cdcdSVivek Goyal
1271cdf6cdcdSVivek Goyal return ret;
1272cdf6cdcdSVivek Goyal }
1273cdf6cdcdSVivek Goyal
dm_dax_recovery_write(struct dax_device * dax_dev,pgoff_t pgoff,void * addr,size_t bytes,struct iov_iter * i)1274047218ecSJane Chu static size_t dm_dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff,
1275047218ecSJane Chu void *addr, size_t bytes, struct iov_iter *i)
1276047218ecSJane Chu {
1277047218ecSJane Chu struct mapped_device *md = dax_get_private(dax_dev);
1278047218ecSJane Chu sector_t sector = pgoff * PAGE_SECTORS;
1279047218ecSJane Chu struct dm_target *ti;
1280047218ecSJane Chu int srcu_idx;
1281047218ecSJane Chu long ret = 0;
1282047218ecSJane Chu
1283047218ecSJane Chu ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1284047218ecSJane Chu if (!ti || !ti->type->dax_recovery_write)
1285047218ecSJane Chu goto out;
1286047218ecSJane Chu
1287047218ecSJane Chu ret = ti->type->dax_recovery_write(ti, pgoff, addr, bytes, i);
1288047218ecSJane Chu out:
1289047218ecSJane Chu dm_put_live_table(md, srcu_idx);
1290047218ecSJane Chu return ret;
1291047218ecSJane Chu }
1292047218ecSJane Chu
12931dd40c3eSMikulas Patocka /*
12941dd40c3eSMikulas Patocka * A target may call dm_accept_partial_bio only from the map routine. It is
12956842d264SDamien Le Moal * allowed for all bio types except REQ_PREFLUSH, REQ_OP_ZONE_* zone management
1296e6fc9f62SMike Snitzer * operations, REQ_OP_ZONE_APPEND (zone append writes) and any bio serviced by
1297e6fc9f62SMike Snitzer * __send_duplicate_bios().
12981dd40c3eSMikulas Patocka *
12991dd40c3eSMikulas Patocka * dm_accept_partial_bio informs the dm that the target only wants to process
13001dd40c3eSMikulas Patocka * additional n_sectors sectors of the bio and the rest of the data should be
13011dd40c3eSMikulas Patocka * sent in a next bio.
13021dd40c3eSMikulas Patocka *
13031dd40c3eSMikulas Patocka * A diagram that explains the arithmetics:
13041dd40c3eSMikulas Patocka * +--------------------+---------------+-------+
13051dd40c3eSMikulas Patocka * | 1 | 2 | 3 |
13061dd40c3eSMikulas Patocka * +--------------------+---------------+-------+
13071dd40c3eSMikulas Patocka *
13081dd40c3eSMikulas Patocka * <-------------- *tio->len_ptr --------------->
1309bdb34759SMike Snitzer * <----- bio_sectors ----->
13101dd40c3eSMikulas Patocka * <-- n_sectors -->
13111dd40c3eSMikulas Patocka *
13121dd40c3eSMikulas Patocka * Region 1 was already iterated over with bio_advance or similar function.
13131dd40c3eSMikulas Patocka * (it may be empty if the target doesn't use bio_advance)
13141dd40c3eSMikulas Patocka * Region 2 is the remaining bio size that the target wants to process.
13151dd40c3eSMikulas Patocka * (it may be empty if region 1 is non-empty, although there is no reason
13161dd40c3eSMikulas Patocka * to make it empty)
13171dd40c3eSMikulas Patocka * The target requires that region 3 is to be sent in the next bio.
13181dd40c3eSMikulas Patocka *
13191dd40c3eSMikulas Patocka * If the target wants to receive multiple copies of the bio (via num_*bios, etc),
13201dd40c3eSMikulas Patocka * the partially processed part (the sum of regions 1+2) must be the same for all
13211dd40c3eSMikulas Patocka * copies of the bio.
13221dd40c3eSMikulas Patocka */
dm_accept_partial_bio(struct bio * bio,unsigned int n_sectors)132386a3238cSHeinz Mauelshagen void dm_accept_partial_bio(struct bio *bio, unsigned int n_sectors)
13241dd40c3eSMikulas Patocka {
13256c23f0bdSChristoph Hellwig struct dm_target_io *tio = clone_to_tio(bio);
13268b211aacSMing Lei struct dm_io *io = tio->io;
132786a3238cSHeinz Mauelshagen unsigned int bio_sectors = bio_sectors(bio);
13286842d264SDamien Le Moal
1329655f3aadSMike Snitzer BUG_ON(dm_tio_flagged(tio, DM_TIO_IS_DUPLICATE_BIO));
13306842d264SDamien Le Moal BUG_ON(op_is_zone_mgmt(bio_op(bio)));
13316842d264SDamien Le Moal BUG_ON(bio_op(bio) == REQ_OP_ZONE_APPEND);
1332bdb34759SMike Snitzer BUG_ON(bio_sectors > *tio->len_ptr);
1333bdb34759SMike Snitzer BUG_ON(n_sectors > bio_sectors);
13346842d264SDamien Le Moal
1335bdb34759SMike Snitzer *tio->len_ptr -= bio_sectors - n_sectors;
13361dd40c3eSMikulas Patocka bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT;
13377dd76d1fSMing Lei
13387dd76d1fSMing Lei /*
13397dd76d1fSMing Lei * __split_and_process_bio() may have already saved mapped part
13407dd76d1fSMing Lei * for accounting but it is being reduced so update accordingly.
13417dd76d1fSMing Lei */
13428b211aacSMing Lei dm_io_set_flag(io, DM_IO_WAS_SPLIT);
13438b211aacSMing Lei io->sectors = n_sectors;
13448b211aacSMing Lei io->sector_offset = bio_sectors(io->orig_bio);
13451dd40c3eSMikulas Patocka }
13461dd40c3eSMikulas Patocka EXPORT_SYMBOL_GPL(dm_accept_partial_bio);
13471dd40c3eSMikulas Patocka
13480fbb4d93SMike Snitzer /*
13490fbb4d93SMike Snitzer * @clone: clone bio that DM core passed to target's .map function
13500fbb4d93SMike Snitzer * @tgt_clone: clone of @clone bio that target needs submitted
13510fbb4d93SMike Snitzer *
13520fbb4d93SMike Snitzer * Targets should use this interface to submit bios they take
13530fbb4d93SMike Snitzer * ownership of when returning DM_MAPIO_SUBMITTED.
13540fbb4d93SMike Snitzer *
13550fbb4d93SMike Snitzer * Target should also enable ti->accounts_remapped_io
13560fbb4d93SMike Snitzer */
dm_submit_bio_remap(struct bio * clone,struct bio * tgt_clone)1357b7f8dff0SMike Snitzer void dm_submit_bio_remap(struct bio *clone, struct bio *tgt_clone)
13580fbb4d93SMike Snitzer {
13590fbb4d93SMike Snitzer struct dm_target_io *tio = clone_to_tio(clone);
13600fbb4d93SMike Snitzer struct dm_io *io = tio->io;
13610fbb4d93SMike Snitzer
13620fbb4d93SMike Snitzer /* establish bio that will get submitted */
13630fbb4d93SMike Snitzer if (!tgt_clone)
13640fbb4d93SMike Snitzer tgt_clone = clone;
13650fbb4d93SMike Snitzer
13660fbb4d93SMike Snitzer /*
13670fbb4d93SMike Snitzer * Account io->origin_bio to DM dev on behalf of target
13680fbb4d93SMike Snitzer * that took ownership of IO with DM_MAPIO_SUBMITTED.
13690fbb4d93SMike Snitzer */
13700fbb4d93SMike Snitzer dm_start_io_acct(io, clone);
13710fbb4d93SMike Snitzer
13729d20653fSMike Snitzer trace_block_bio_remap(tgt_clone, disk_devt(io->md->disk),
13730fbb4d93SMike Snitzer tio->old_sector);
13749d20653fSMike Snitzer submit_bio_noacct(tgt_clone);
13750fbb4d93SMike Snitzer }
13760fbb4d93SMike Snitzer EXPORT_SYMBOL_GPL(dm_submit_bio_remap);
13770fbb4d93SMike Snitzer
__set_swap_bios_limit(struct mapped_device * md,int latch)1378a666e5c0SMikulas Patocka static noinline void __set_swap_bios_limit(struct mapped_device *md, int latch)
1379a666e5c0SMikulas Patocka {
1380a666e5c0SMikulas Patocka mutex_lock(&md->swap_bios_lock);
1381a666e5c0SMikulas Patocka while (latch < md->swap_bios) {
1382a666e5c0SMikulas Patocka cond_resched();
1383a666e5c0SMikulas Patocka down(&md->swap_bios_semaphore);
1384a666e5c0SMikulas Patocka md->swap_bios--;
1385a666e5c0SMikulas Patocka }
1386a666e5c0SMikulas Patocka while (latch > md->swap_bios) {
1387a666e5c0SMikulas Patocka cond_resched();
1388a666e5c0SMikulas Patocka up(&md->swap_bios_semaphore);
1389a666e5c0SMikulas Patocka md->swap_bios++;
1390a666e5c0SMikulas Patocka }
1391a666e5c0SMikulas Patocka mutex_unlock(&md->swap_bios_lock);
1392a666e5c0SMikulas Patocka }
1393a666e5c0SMikulas Patocka
__map_bio(struct bio * clone)13941561b396SChristoph Hellwig static void __map_bio(struct bio *clone)
13951da177e4SLinus Torvalds {
13961561b396SChristoph Hellwig struct dm_target_io *tio = clone_to_tio(clone);
1397bd2a49b8SAlasdair G Kergon struct dm_target *ti = tio->ti;
13986cbce280SMike Snitzer struct dm_io *io = tio->io;
13996cbce280SMike Snitzer struct mapped_device *md = io->md;
14006cbce280SMike Snitzer int r;
14011da177e4SLinus Torvalds
14021da177e4SLinus Torvalds clone->bi_end_io = clone_endio;
14031da177e4SLinus Torvalds
14041da177e4SLinus Torvalds /*
14050fbb4d93SMike Snitzer * Map the clone.
14061da177e4SLinus Torvalds */
1407743598f0SMike Snitzer tio->old_sector = clone->bi_iter.bi_sector;
1408d67a5f4bSMikulas Patocka
1409442761fdSMike Snitzer if (static_branch_unlikely(&swap_bios_enabled) &&
1410442761fdSMike Snitzer unlikely(swap_bios_limit(ti, clone))) {
1411a666e5c0SMikulas Patocka int latch = get_swap_bios();
1412b30f1607SHeinz Mauelshagen
1413a666e5c0SMikulas Patocka if (unlikely(latch != md->swap_bios))
1414a666e5c0SMikulas Patocka __set_swap_bios_limit(md, latch);
1415a666e5c0SMikulas Patocka down(&md->swap_bios_semaphore);
1416a666e5c0SMikulas Patocka }
1417a666e5c0SMikulas Patocka
1418442761fdSMike Snitzer if (static_branch_unlikely(&zoned_enabled)) {
1419bb37d772SDamien Le Moal /*
1420442761fdSMike Snitzer * Check if the IO needs a special mapping due to zone append
1421442761fdSMike Snitzer * emulation on zoned target. In this case, dm_zone_map_bio()
1422442761fdSMike Snitzer * calls the target map operation.
1423bb37d772SDamien Le Moal */
14246cbce280SMike Snitzer if (unlikely(dm_emulate_zone_append(md)))
1425bb37d772SDamien Le Moal r = dm_zone_map_bio(tio);
1426bb37d772SDamien Le Moal else
14277de3ee57SMikulas Patocka r = ti->type->map(ti, clone);
1428442761fdSMike Snitzer } else
1429442761fdSMike Snitzer r = ti->type->map(ti, clone);
1430bb37d772SDamien Le Moal
1431846785e6SChristoph Hellwig switch (r) {
1432846785e6SChristoph Hellwig case DM_MAPIO_SUBMITTED:
14330fbb4d93SMike Snitzer /* target has assumed ownership of this io */
14340fbb4d93SMike Snitzer if (!ti->accounts_remapped_io)
14359d20653fSMike Snitzer dm_start_io_acct(io, clone);
1436846785e6SChristoph Hellwig break;
1437846785e6SChristoph Hellwig case DM_MAPIO_REMAPPED:
14389d20653fSMike Snitzer dm_submit_bio_remap(clone, NULL);
1439846785e6SChristoph Hellwig break;
1440846785e6SChristoph Hellwig case DM_MAPIO_KILL:
1441846785e6SChristoph Hellwig case DM_MAPIO_REQUEUE:
1442442761fdSMike Snitzer if (static_branch_unlikely(&swap_bios_enabled) &&
1443442761fdSMike Snitzer unlikely(swap_bios_limit(ti, clone)))
14446cbce280SMike Snitzer up(&md->swap_bios_semaphore);
14451d1068ceSChristoph Hellwig free_tio(clone);
144690a2326eSMike Snitzer if (r == DM_MAPIO_KILL)
144790a2326eSMike Snitzer dm_io_dec_pending(io, BLK_STS_IOERR);
144890a2326eSMike Snitzer else
1449e2118b3cSDamien Le Moal dm_io_dec_pending(io, BLK_STS_DM_REQUEUE);
1450846785e6SChristoph Hellwig break;
1451846785e6SChristoph Hellwig default:
145243e6c111SMikulas Patocka DMCRIT("unimplemented target map return value: %d", r);
145345cbcd79SKiyoshi Ueda BUG();
14541da177e4SLinus Torvalds }
14551da177e4SLinus Torvalds }
14561da177e4SLinus Torvalds
setup_split_accounting(struct clone_info * ci,unsigned int len)145786a3238cSHeinz Mauelshagen static void setup_split_accounting(struct clone_info *ci, unsigned int len)
14587dd76d1fSMing Lei {
14597dd76d1fSMing Lei struct dm_io *io = ci->io;
14607dd76d1fSMing Lei
14617dd76d1fSMing Lei if (ci->sector_count > len) {
14627dd76d1fSMing Lei /*
14637dd76d1fSMing Lei * Split needed, save the mapped part for accounting.
14647dd76d1fSMing Lei * NOTE: dm_accept_partial_bio() will update accordingly.
14657dd76d1fSMing Lei */
14667dd76d1fSMing Lei dm_io_set_flag(io, DM_IO_WAS_SPLIT);
14677dd76d1fSMing Lei io->sectors = len;
14688b211aacSMing Lei io->sector_offset = bio_sectors(ci->bio);
14697dd76d1fSMing Lei }
14707dd76d1fSMing Lei }
14717dd76d1fSMing Lei
alloc_multiple_bios(struct bio_list * blist,struct clone_info * ci,struct dm_target * ti,unsigned int num_bios,unsigned * len)1472318716ddSMike Snitzer static void alloc_multiple_bios(struct bio_list *blist, struct clone_info *ci,
1473f7b58a69SMike Snitzer struct dm_target *ti, unsigned int num_bios,
1474f7b58a69SMike Snitzer unsigned *len)
1475f9ab94ceSMikulas Patocka {
14761d1068ceSChristoph Hellwig struct bio *bio;
1477318716ddSMike Snitzer int try;
1478dba14160SMikulas Patocka
1479318716ddSMike Snitzer for (try = 0; try < 2; try++) {
1480318716ddSMike Snitzer int bio_nr;
1481318716ddSMike Snitzer
1482318716ddSMike Snitzer if (try)
1483bc02cdbeSMike Snitzer mutex_lock(&ci->io->md->table_devices_lock);
1484318716ddSMike Snitzer for (bio_nr = 0; bio_nr < num_bios; bio_nr++) {
1485f7b58a69SMike Snitzer bio = alloc_tio(ci, ti, bio_nr, len,
1486dc8e2021SChristoph Hellwig try ? GFP_NOIO : GFP_NOWAIT);
14871d1068ceSChristoph Hellwig if (!bio)
1488318716ddSMike Snitzer break;
1489318716ddSMike Snitzer
14901d1068ceSChristoph Hellwig bio_list_add(blist, bio);
1491318716ddSMike Snitzer }
1492318716ddSMike Snitzer if (try)
1493bc02cdbeSMike Snitzer mutex_unlock(&ci->io->md->table_devices_lock);
1494318716ddSMike Snitzer if (bio_nr == num_bios)
1495318716ddSMike Snitzer return;
1496318716ddSMike Snitzer
14976c23f0bdSChristoph Hellwig while ((bio = bio_list_pop(blist)))
14981d1068ceSChristoph Hellwig free_tio(bio);
1499318716ddSMike Snitzer }
1500318716ddSMike Snitzer }
1501f9ab94ceSMikulas Patocka
__send_duplicate_bios(struct clone_info * ci,struct dm_target * ti,unsigned int num_bios,unsigned int * len)15020f14d60aSMing Lei static int __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti,
150386a3238cSHeinz Mauelshagen unsigned int num_bios, unsigned int *len)
150406a426ceSMike Snitzer {
1505318716ddSMike Snitzer struct bio_list blist = BIO_EMPTY_LIST;
15068eabf5d0SChristoph Hellwig struct bio *clone;
1507564b5c54SMike Snitzer unsigned int ret = 0;
150806a426ceSMike Snitzer
1509891fced6SChristoph Hellwig switch (num_bios) {
1510891fced6SChristoph Hellwig case 0:
1511891fced6SChristoph Hellwig break;
1512891fced6SChristoph Hellwig case 1:
15137dd76d1fSMing Lei if (len)
15147dd76d1fSMing Lei setup_split_accounting(ci, *len);
1515891fced6SChristoph Hellwig clone = alloc_tio(ci, ti, 0, len, GFP_NOIO);
1516891fced6SChristoph Hellwig __map_bio(clone);
15170f14d60aSMing Lei ret = 1;
1518891fced6SChristoph Hellwig break;
1519891fced6SChristoph Hellwig default:
1520666eed46SMike Snitzer if (len)
1521666eed46SMike Snitzer setup_split_accounting(ci, *len);
15227dd06a25SMike Snitzer /* dm_accept_partial_bio() is not supported with shared tio->len_ptr */
1523f7b58a69SMike Snitzer alloc_multiple_bios(&blist, ci, ti, num_bios, len);
15248eabf5d0SChristoph Hellwig while ((clone = bio_list_pop(&blist))) {
1525655f3aadSMike Snitzer dm_tio_set_flag(clone_to_tio(clone), DM_TIO_IS_DUPLICATE_BIO);
15261561b396SChristoph Hellwig __map_bio(clone);
15270f14d60aSMing Lei ret += 1;
1528f9ab94ceSMikulas Patocka }
1529891fced6SChristoph Hellwig break;
1530318716ddSMike Snitzer }
15310f14d60aSMing Lei
15320f14d60aSMing Lei return ret;
153306a426ceSMike Snitzer }
153406a426ceSMike Snitzer
__send_empty_flush(struct clone_info * ci)1535332f2b1eSMike Snitzer static void __send_empty_flush(struct clone_info *ci)
1536f9ab94ceSMikulas Patocka {
1537564b5c54SMike Snitzer struct dm_table *t = ci->map;
1538828678b8SMike Snitzer struct bio flush_bio;
1539828678b8SMike Snitzer
1540828678b8SMike Snitzer /*
1541828678b8SMike Snitzer * Use an on-stack bio for this, it's safe since we don't
1542828678b8SMike Snitzer * need to reference it after submit. It's just used as
1543828678b8SMike Snitzer * the basis for the clone(s).
1544828678b8SMike Snitzer */
154549add496SChristoph Hellwig bio_init(&flush_bio, ci->io->md->disk->part0, NULL, 0,
154649add496SChristoph Hellwig REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC);
154747d95102SChristoph Hellwig
1548828678b8SMike Snitzer ci->bio = &flush_bio;
1549828678b8SMike Snitzer ci->sector_count = 0;
155092b914e2SShin'ichiro Kawasaki ci->io->tio.clone.bi_iter.bi_size = 0;
1551f9ab94ceSMikulas Patocka
1552564b5c54SMike Snitzer for (unsigned int i = 0; i < t->num_targets; i++) {
1553564b5c54SMike Snitzer unsigned int bios;
1554564b5c54SMike Snitzer struct dm_target *ti = dm_table_get_target(t, i);
15550f14d60aSMing Lei
15560f14d60aSMing Lei atomic_add(ti->num_flush_bios, &ci->io->io_count);
15570f14d60aSMing Lei bios = __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL);
15580f14d60aSMing Lei atomic_sub(ti->num_flush_bios - bios, &ci->io->io_count);
15590f14d60aSMing Lei }
15600f14d60aSMing Lei
15610f14d60aSMing Lei /*
15620f14d60aSMing Lei * alloc_io() takes one extra reference for submission, so the
15630f14d60aSMing Lei * reference won't reach 0 without the following subtraction
15640f14d60aSMing Lei */
15650f14d60aSMing Lei atomic_sub(1, &ci->io->io_count);
1566828678b8SMike Snitzer
1567828678b8SMike Snitzer bio_uninit(ci->bio);
1568f9ab94ceSMikulas Patocka }
1569f9ab94ceSMikulas Patocka
__send_changing_extent_only(struct clone_info * ci,struct dm_target * ti,unsigned int num_bios,unsigned int max_granularity,unsigned int max_sectors)1570e6fc9f62SMike Snitzer static void __send_changing_extent_only(struct clone_info *ci, struct dm_target *ti,
157106961c48SMike Snitzer unsigned int num_bios,
1572be04c14aSMike Snitzer unsigned int max_granularity,
1573be04c14aSMike Snitzer unsigned int max_sectors)
15745ae89a87SMike Snitzer {
157586a3238cSHeinz Mauelshagen unsigned int len, bios;
15765ae89a87SMike Snitzer
15773720281dSMike Snitzer len = min_t(sector_t, ci->sector_count,
1578be04c14aSMike Snitzer __max_io_len(ti, ci->sector, max_granularity, max_sectors));
157951b86f9aSMichael Lass
15800f14d60aSMing Lei atomic_add(num_bios, &ci->io->io_count);
15810f14d60aSMing Lei bios = __send_duplicate_bios(ci, ti, num_bios, &len);
15820f14d60aSMing Lei /*
15830f14d60aSMing Lei * alloc_io() takes one extra reference for submission, so the
15840f14d60aSMing Lei * reference won't reach 0 without the following (+1) subtraction
15850f14d60aSMing Lei */
15860f14d60aSMing Lei atomic_sub(num_bios - bios + 1, &ci->io->io_count);
15877dd06a25SMike Snitzer
1588a79245b3SMike Snitzer ci->sector += len;
15893d7f4562SMike Snitzer ci->sector_count -= len;
15905ae89a87SMike Snitzer }
15915ae89a87SMike Snitzer
is_abnormal_io(struct bio * bio)1592568c73a3SMike Snitzer static bool is_abnormal_io(struct bio *bio)
1593568c73a3SMike Snitzer {
1594a3282b43SBart Van Assche enum req_op op = bio_op(bio);
1595568c73a3SMike Snitzer
15964edadf6dSMike Snitzer if (op != REQ_OP_READ && op != REQ_OP_WRITE && op != REQ_OP_FLUSH) {
15974edadf6dSMike Snitzer switch (op) {
1598568c73a3SMike Snitzer case REQ_OP_DISCARD:
1599568c73a3SMike Snitzer case REQ_OP_SECURE_ERASE:
1600568c73a3SMike Snitzer case REQ_OP_WRITE_ZEROES:
16014edadf6dSMike Snitzer return true;
16024edadf6dSMike Snitzer default:
1603568c73a3SMike Snitzer break;
1604568c73a3SMike Snitzer }
1605568c73a3SMike Snitzer }
1606568c73a3SMike Snitzer
16074edadf6dSMike Snitzer return false;
16084edadf6dSMike Snitzer }
16094edadf6dSMike Snitzer
__process_abnormal_io(struct clone_info * ci,struct dm_target * ti)16104edadf6dSMike Snitzer static blk_status_t __process_abnormal_io(struct clone_info *ci,
16114edadf6dSMike Snitzer struct dm_target *ti)
16120519c71eSMike Snitzer {
161386a3238cSHeinz Mauelshagen unsigned int num_bios = 0;
161406961c48SMike Snitzer unsigned int max_granularity = 0;
1615be04c14aSMike Snitzer unsigned int max_sectors = 0;
161613f6facfSMike Snitzer struct queue_limits *limits = dm_get_queue_limits(ti->table->md);
16170519c71eSMike Snitzer
1618e6fc9f62SMike Snitzer switch (bio_op(ci->bio)) {
16199679b5a7SMike Snitzer case REQ_OP_DISCARD:
16209679b5a7SMike Snitzer num_bios = ti->num_discard_bios;
1621be04c14aSMike Snitzer max_sectors = limits->max_discard_sectors;
162213f6facfSMike Snitzer if (ti->max_discard_granularity)
1623be04c14aSMike Snitzer max_granularity = max_sectors;
16249679b5a7SMike Snitzer break;
16259679b5a7SMike Snitzer case REQ_OP_SECURE_ERASE:
16269679b5a7SMike Snitzer num_bios = ti->num_secure_erase_bios;
1627be04c14aSMike Snitzer max_sectors = limits->max_secure_erase_sectors;
162813f6facfSMike Snitzer if (ti->max_secure_erase_granularity)
1629be04c14aSMike Snitzer max_granularity = max_sectors;
16309679b5a7SMike Snitzer break;
16319679b5a7SMike Snitzer case REQ_OP_WRITE_ZEROES:
16329679b5a7SMike Snitzer num_bios = ti->num_write_zeroes_bios;
1633be04c14aSMike Snitzer max_sectors = limits->max_write_zeroes_sectors;
163413f6facfSMike Snitzer if (ti->max_write_zeroes_granularity)
1635be04c14aSMike Snitzer max_granularity = max_sectors;
16369679b5a7SMike Snitzer break;
16372d9b02beSBart Van Assche default:
16382d9b02beSBart Van Assche break;
16399679b5a7SMike Snitzer }
16400519c71eSMike Snitzer
1641e6fc9f62SMike Snitzer /*
1642e6fc9f62SMike Snitzer * Even though the device advertised support for this type of
1643e6fc9f62SMike Snitzer * request, that does not mean every target supports it, and
1644e6fc9f62SMike Snitzer * reconfiguration might also have changed that since the
1645e6fc9f62SMike Snitzer * check was performed.
1646e6fc9f62SMike Snitzer */
164784b98f4cSMike Snitzer if (unlikely(!num_bios))
16484edadf6dSMike Snitzer return BLK_STS_NOTSUPP;
16494edadf6dSMike Snitzer
1650be04c14aSMike Snitzer __send_changing_extent_only(ci, ti, num_bios,
1651be04c14aSMike Snitzer max_granularity, max_sectors);
16524edadf6dSMike Snitzer return BLK_STS_OK;
16530519c71eSMike Snitzer }
16540519c71eSMike Snitzer
1655e4c93811SAlasdair G Kergon /*
1656ec211631SMing Lei * Reuse ->bi_private as dm_io list head for storing all dm_io instances
1657b99fdcdcSMing Lei * associated with this bio, and this bio's bi_private needs to be
1658b99fdcdcSMing Lei * stored in dm_io->data before the reuse.
1659b99fdcdcSMing Lei *
1660b99fdcdcSMing Lei * bio->bi_private is owned by fs or upper layer, so block layer won't
1661b99fdcdcSMing Lei * touch it after splitting. Meantime it won't be changed by anyone after
1662b99fdcdcSMing Lei * bio is submitted. So this reuse is safe.
1663b99fdcdcSMing Lei */
dm_poll_list_head(struct bio * bio)1664ec211631SMing Lei static inline struct dm_io **dm_poll_list_head(struct bio *bio)
1665b99fdcdcSMing Lei {
1666ec211631SMing Lei return (struct dm_io **)&bio->bi_private;
1667b99fdcdcSMing Lei }
1668b99fdcdcSMing Lei
dm_queue_poll_io(struct bio * bio,struct dm_io * io)1669b99fdcdcSMing Lei static void dm_queue_poll_io(struct bio *bio, struct dm_io *io)
1670b99fdcdcSMing Lei {
1671ec211631SMing Lei struct dm_io **head = dm_poll_list_head(bio);
1672b99fdcdcSMing Lei
1673b99fdcdcSMing Lei if (!(bio->bi_opf & REQ_DM_POLL_LIST)) {
1674b99fdcdcSMing Lei bio->bi_opf |= REQ_DM_POLL_LIST;
1675b99fdcdcSMing Lei /*
1676b99fdcdcSMing Lei * Save .bi_private into dm_io, so that we can reuse
1677ec211631SMing Lei * .bi_private as dm_io list head for storing dm_io list
1678b99fdcdcSMing Lei */
1679b99fdcdcSMing Lei io->data = bio->bi_private;
1680b99fdcdcSMing Lei
1681b99fdcdcSMing Lei /* tell block layer to poll for completion */
1682b99fdcdcSMing Lei bio->bi_cookie = ~BLK_QC_T_NONE;
1683ec211631SMing Lei
1684ec211631SMing Lei io->next = NULL;
1685b99fdcdcSMing Lei } else {
1686b99fdcdcSMing Lei /*
1687b99fdcdcSMing Lei * bio recursed due to split, reuse original poll list,
1688b99fdcdcSMing Lei * and save bio->bi_private too.
1689b99fdcdcSMing Lei */
1690ec211631SMing Lei io->data = (*head)->data;
1691ec211631SMing Lei io->next = *head;
1692b99fdcdcSMing Lei }
1693b99fdcdcSMing Lei
1694ec211631SMing Lei *head = io;
1695b99fdcdcSMing Lei }
1696b99fdcdcSMing Lei
1697b99fdcdcSMing Lei /*
1698e4c93811SAlasdair G Kergon * Select the correct strategy for processing a non-flush bio.
1699e4c93811SAlasdair G Kergon */
__split_and_process_bio(struct clone_info * ci)170084b98f4cSMike Snitzer static blk_status_t __split_and_process_bio(struct clone_info *ci)
1701e4c93811SAlasdair G Kergon {
170266bdaa43SMike Snitzer struct bio *clone;
1703e4c93811SAlasdair G Kergon struct dm_target *ti;
170486a3238cSHeinz Mauelshagen unsigned int len;
1705e4c93811SAlasdair G Kergon
1706e4c93811SAlasdair G Kergon ti = dm_table_find_target(ci->map, ci->sector);
17074edadf6dSMike Snitzer if (unlikely(!ti))
17084edadf6dSMike Snitzer return BLK_STS_IOERR;
17091ee88de3SMikulas Patocka
17101ee88de3SMikulas Patocka if (unlikely((ci->bio->bi_opf & REQ_NOWAIT) != 0) &&
17111ee88de3SMikulas Patocka unlikely(!dm_target_supports_nowait(ti->type)))
17121ee88de3SMikulas Patocka return BLK_STS_NOTSUPP;
17131ee88de3SMikulas Patocka
17141ee88de3SMikulas Patocka if (unlikely(ci->is_abnormal_io))
17154edadf6dSMike Snitzer return __process_abnormal_io(ci, ti);
17163d7f4562SMike Snitzer
1717b99fdcdcSMing Lei /*
1718b99fdcdcSMing Lei * Only support bio polling for normal IO, and the target io is
1719b99fdcdcSMing Lei * exactly inside the dm_io instance (verified in dm_poll_dm_io)
1720b99fdcdcSMing Lei */
1721a3282b43SBart Van Assche ci->submit_as_polled = !!(ci->bio->bi_opf & REQ_POLLED);
1722e4c93811SAlasdair G Kergon
1723e4c93811SAlasdair G Kergon len = min_t(sector_t, max_io_len(ti, ci->sector), ci->sector_count);
17247dd76d1fSMing Lei setup_split_accounting(ci, len);
172566bdaa43SMike Snitzer clone = alloc_tio(ci, ti, 0, &len, GFP_NOIO);
172666bdaa43SMike Snitzer __map_bio(clone);
1727e4c93811SAlasdair G Kergon
1728e4c93811SAlasdair G Kergon ci->sector += len;
1729e4c93811SAlasdair G Kergon ci->sector_count -= len;
1730e4c93811SAlasdair G Kergon
173184b98f4cSMike Snitzer return BLK_STS_OK;
1732e4c93811SAlasdair G Kergon }
1733e4c93811SAlasdair G Kergon
init_clone_info(struct clone_info * ci,struct mapped_device * md,struct dm_table * map,struct bio * bio,bool is_abnormal)1734978e51baSMike Snitzer static void init_clone_info(struct clone_info *ci, struct mapped_device *md,
17354edadf6dSMike Snitzer struct dm_table *map, struct bio *bio, bool is_abnormal)
1736978e51baSMike Snitzer {
1737978e51baSMike Snitzer ci->map = map;
1738978e51baSMike Snitzer ci->io = alloc_io(md, bio);
1739d41e077aSMike Snitzer ci->bio = bio;
17404edadf6dSMike Snitzer ci->is_abnormal_io = is_abnormal;
1741b99fdcdcSMing Lei ci->submit_as_polled = false;
1742978e51baSMike Snitzer ci->sector = bio->bi_iter.bi_sector;
1743d41e077aSMike Snitzer ci->sector_count = bio_sectors(bio);
1744d41e077aSMike Snitzer
1745d41e077aSMike Snitzer /* Shouldn't happen but sector_count was being set to 0 so... */
1746442761fdSMike Snitzer if (static_branch_unlikely(&zoned_enabled) &&
1747442761fdSMike Snitzer WARN_ON_ONCE(op_is_zone_mgmt(bio_op(bio)) && ci->sector_count))
1748d41e077aSMike Snitzer ci->sector_count = 0;
1749978e51baSMike Snitzer }
1750978e51baSMike Snitzer
1751e4c93811SAlasdair G Kergon /*
175214fe594dSAlasdair G Kergon * Entry point to split a bio into clones and submit them to the targets.
17531da177e4SLinus Torvalds */
dm_split_and_process_bio(struct mapped_device * md,struct dm_table * map,struct bio * bio)175496c9865cSMike Snitzer static void dm_split_and_process_bio(struct mapped_device *md,
175583d5e5b0SMikulas Patocka struct dm_table *map, struct bio *bio)
17561da177e4SLinus Torvalds {
17571da177e4SLinus Torvalds struct clone_info ci;
17584857abf6SMike Snitzer struct dm_io *io;
175984b98f4cSMike Snitzer blk_status_t error = BLK_STS_OK;
17604edadf6dSMike Snitzer bool is_abnormal;
17611da177e4SLinus Torvalds
17624edadf6dSMike Snitzer is_abnormal = is_abnormal_io(bio);
17634edadf6dSMike Snitzer if (unlikely(is_abnormal)) {
17644edadf6dSMike Snitzer /*
17655a97806fSChristoph Hellwig * Use bio_split_to_limits() for abnormal IO (e.g. discard, etc)
17664edadf6dSMike Snitzer * otherwise associated queue_limits won't be imposed.
17674edadf6dSMike Snitzer */
17685a97806fSChristoph Hellwig bio = bio_split_to_limits(bio);
1769613b1488SJens Axboe if (!bio)
1770613b1488SJens Axboe return;
17714edadf6dSMike Snitzer }
17724edadf6dSMike Snitzer
17734edadf6dSMike Snitzer init_clone_info(&ci, md, map, bio, is_abnormal);
17744857abf6SMike Snitzer io = ci.io;
1775bd2a49b8SAlasdair G Kergon
17761eff9d32SJens Axboe if (bio->bi_opf & REQ_PREFLUSH) {
1777332f2b1eSMike Snitzer __send_empty_flush(&ci);
1778e2736347SMike Snitzer /* dm_io_complete submits any data associated with flush */
1779d41e077aSMike Snitzer goto out;
1780d41e077aSMike Snitzer }
1781d41e077aSMike Snitzer
178296c9865cSMike Snitzer error = __split_and_process_bio(&ci);
1783d41e077aSMike Snitzer if (error || !ci.sector_count)
1784d41e077aSMike Snitzer goto out;
178518a25da8SNeilBrown /*
1786d41e077aSMike Snitzer * Remainder must be passed to submit_bio_noacct() so it gets handled
1787d41e077aSMike Snitzer * *after* bios already submitted have been completely processed.
178818a25da8SNeilBrown */
17898b211aacSMing Lei bio_trim(bio, io->sectors, ci.sector_count);
17908b211aacSMing Lei trace_block_split(bio, bio->bi_iter.bi_sector);
17918b211aacSMing Lei bio_inc_remaining(bio);
17923e08773cSChristoph Hellwig submit_bio_noacct(bio);
1793d41e077aSMike Snitzer out:
1794b99fdcdcSMing Lei /*
1795b99fdcdcSMing Lei * Drop the extra reference count for non-POLLED bio, and hold one
1796b99fdcdcSMing Lei * reference for POLLED bio, which will be released in dm_poll_bio
1797b99fdcdcSMing Lei *
1798ec211631SMing Lei * Add every dm_io instance into the dm_io list head which is stored
1799ec211631SMing Lei * in bio->bi_private, so that dm_poll_bio can poll them all.
1800b99fdcdcSMing Lei */
18010f14d60aSMing Lei if (error || !ci.submit_as_polled) {
18020f14d60aSMing Lei /*
18030f14d60aSMing Lei * In case of submission failure, the extra reference for
18040f14d60aSMing Lei * submitting io isn't consumed yet
18050f14d60aSMing Lei */
18060f14d60aSMing Lei if (error)
18070f14d60aSMing Lei atomic_dec(&io->io_count);
18080f14d60aSMing Lei dm_io_dec_pending(io, error);
18090f14d60aSMing Lei } else
18104857abf6SMike Snitzer dm_queue_poll_io(bio, io);
18111da177e4SLinus Torvalds }
18121da177e4SLinus Torvalds
dm_submit_bio(struct bio * bio)18133e08773cSChristoph Hellwig static void dm_submit_bio(struct bio *bio)
18141da177e4SLinus Torvalds {
1815309dca30SChristoph Hellwig struct mapped_device *md = bio->bi_bdev->bd_disk->private_data;
181683d5e5b0SMikulas Patocka int srcu_idx;
181783d5e5b0SMikulas Patocka struct dm_table *map;
18181da177e4SLinus Torvalds
1819a9ce3853SJens Axboe map = dm_get_live_table(md, &srcu_idx);
1820*9360d077SMikulas Patocka if (unlikely(!map)) {
1821*9360d077SMikulas Patocka DMERR_LIMIT("%s: mapping table unavailable, erroring io",
1822*9360d077SMikulas Patocka dm_device_name(md));
1823*9360d077SMikulas Patocka bio_io_error(bio);
1824*9360d077SMikulas Patocka goto out;
1825*9360d077SMikulas Patocka }
18268cf7961dSChristoph Hellwig
1827*9360d077SMikulas Patocka /* If suspended, queue this IO for later */
1828*9360d077SMikulas Patocka if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) {
18296abc4946SKonstantin Khlebnikov if (bio->bi_opf & REQ_NOWAIT)
18306abc4946SKonstantin Khlebnikov bio_wouldblock_error(bio);
1831b2abdb1bSMike Snitzer else if (bio->bi_opf & REQ_RAHEAD)
18326a8736d1STejun Heo bio_io_error(bio);
1833b2abdb1bSMike Snitzer else
1834b2abdb1bSMike Snitzer queue_io(md, bio);
1835b2abdb1bSMike Snitzer goto out;
18361da177e4SLinus Torvalds }
18371da177e4SLinus Torvalds
183896c9865cSMike Snitzer dm_split_and_process_bio(md, map, bio);
1839b2abdb1bSMike Snitzer out:
1840a9ce3853SJens Axboe dm_put_live_table(md, srcu_idx);
1841978e51baSMike Snitzer }
1842978e51baSMike Snitzer
dm_poll_dm_io(struct dm_io * io,struct io_comp_batch * iob,unsigned int flags)1843b99fdcdcSMing Lei static bool dm_poll_dm_io(struct dm_io *io, struct io_comp_batch *iob,
1844b99fdcdcSMing Lei unsigned int flags)
1845b99fdcdcSMing Lei {
1846655f3aadSMike Snitzer WARN_ON_ONCE(!dm_tio_is_normal(&io->tio));
1847b99fdcdcSMing Lei
1848b99fdcdcSMing Lei /* don't poll if the mapped io is done */
1849b99fdcdcSMing Lei if (atomic_read(&io->io_count) > 1)
1850b99fdcdcSMing Lei bio_poll(&io->tio.clone, iob, flags);
1851b99fdcdcSMing Lei
1852b99fdcdcSMing Lei /* bio_poll holds the last reference */
1853b99fdcdcSMing Lei return atomic_read(&io->io_count) == 1;
1854b99fdcdcSMing Lei }
1855b99fdcdcSMing Lei
dm_poll_bio(struct bio * bio,struct io_comp_batch * iob,unsigned int flags)1856b99fdcdcSMing Lei static int dm_poll_bio(struct bio *bio, struct io_comp_batch *iob,
1857b99fdcdcSMing Lei unsigned int flags)
1858b99fdcdcSMing Lei {
1859ec211631SMing Lei struct dm_io **head = dm_poll_list_head(bio);
1860ec211631SMing Lei struct dm_io *list = *head;
1861ec211631SMing Lei struct dm_io *tmp = NULL;
1862ec211631SMing Lei struct dm_io *curr, *next;
1863b99fdcdcSMing Lei
1864b99fdcdcSMing Lei /* Only poll normal bio which was marked as REQ_DM_POLL_LIST */
1865b99fdcdcSMing Lei if (!(bio->bi_opf & REQ_DM_POLL_LIST))
1866b99fdcdcSMing Lei return 0;
1867b99fdcdcSMing Lei
1868ec211631SMing Lei WARN_ON_ONCE(!list);
1869b99fdcdcSMing Lei
1870b99fdcdcSMing Lei /*
1871b99fdcdcSMing Lei * Restore .bi_private before possibly completing dm_io.
1872b99fdcdcSMing Lei *
1873b99fdcdcSMing Lei * bio_poll() is only possible once @bio has been completely
1874b99fdcdcSMing Lei * submitted via submit_bio_noacct()'s depth-first submission.
1875b99fdcdcSMing Lei * So there is no dm_queue_poll_io() race associated with
1876b99fdcdcSMing Lei * clearing REQ_DM_POLL_LIST here.
1877b99fdcdcSMing Lei */
1878b99fdcdcSMing Lei bio->bi_opf &= ~REQ_DM_POLL_LIST;
1879ec211631SMing Lei bio->bi_private = list->data;
1880b99fdcdcSMing Lei
1881ec211631SMing Lei for (curr = list, next = curr->next; curr; curr = next, next =
1882ec211631SMing Lei curr ? curr->next : NULL) {
1883ec211631SMing Lei if (dm_poll_dm_io(curr, iob, flags)) {
1884b99fdcdcSMing Lei /*
188584b98f4cSMike Snitzer * clone_endio() has already occurred, so no
188684b98f4cSMike Snitzer * error handling is needed here.
1887b99fdcdcSMing Lei */
1888ec211631SMing Lei __dm_io_dec_pending(curr);
1889ec211631SMing Lei } else {
1890ec211631SMing Lei curr->next = tmp;
1891ec211631SMing Lei tmp = curr;
1892b99fdcdcSMing Lei }
1893b99fdcdcSMing Lei }
1894b99fdcdcSMing Lei
1895b99fdcdcSMing Lei /* Not done? */
1896ec211631SMing Lei if (tmp) {
1897b99fdcdcSMing Lei bio->bi_opf |= REQ_DM_POLL_LIST;
1898b99fdcdcSMing Lei /* Reset bio->bi_private to dm_io list head */
1899ec211631SMing Lei *head = tmp;
1900b99fdcdcSMing Lei return 0;
1901b99fdcdcSMing Lei }
1902b99fdcdcSMing Lei return 1;
1903b99fdcdcSMing Lei }
1904b99fdcdcSMing Lei
1905a4a82ce3SHeinz Mauelshagen /*
1906a4a82ce3SHeinz Mauelshagen *---------------------------------------------------------------
19071da177e4SLinus Torvalds * An IDR is used to keep track of allocated minor numbers.
1908a4a82ce3SHeinz Mauelshagen *---------------------------------------------------------------
1909a4a82ce3SHeinz Mauelshagen */
free_minor(int minor)19102b06cfffSAlasdair G Kergon static void free_minor(int minor)
19111da177e4SLinus Torvalds {
1912f32c10b0SJeff Mahoney spin_lock(&_minor_lock);
19131da177e4SLinus Torvalds idr_remove(&_minor_idr, minor);
1914f32c10b0SJeff Mahoney spin_unlock(&_minor_lock);
19151da177e4SLinus Torvalds }
19161da177e4SLinus Torvalds
19171da177e4SLinus Torvalds /*
19181da177e4SLinus Torvalds * See if the device with a specific minor # is free.
19191da177e4SLinus Torvalds */
specific_minor(int minor)1920cf13ab8eSFrederik Deweerdt static int specific_minor(int minor)
19211da177e4SLinus Torvalds {
1922c9d76be6STejun Heo int r;
19231da177e4SLinus Torvalds
19241da177e4SLinus Torvalds if (minor >= (1 << MINORBITS))
19251da177e4SLinus Torvalds return -EINVAL;
19261da177e4SLinus Torvalds
1927c9d76be6STejun Heo idr_preload(GFP_KERNEL);
1928f32c10b0SJeff Mahoney spin_lock(&_minor_lock);
19291da177e4SLinus Torvalds
1930c9d76be6STejun Heo r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT);
19311da177e4SLinus Torvalds
1932f32c10b0SJeff Mahoney spin_unlock(&_minor_lock);
1933c9d76be6STejun Heo idr_preload_end();
1934c9d76be6STejun Heo if (r < 0)
1935c9d76be6STejun Heo return r == -ENOSPC ? -EBUSY : r;
1936c9d76be6STejun Heo return 0;
19371da177e4SLinus Torvalds }
19381da177e4SLinus Torvalds
next_free_minor(int * minor)1939cf13ab8eSFrederik Deweerdt static int next_free_minor(int *minor)
19401da177e4SLinus Torvalds {
1941c9d76be6STejun Heo int r;
19421da177e4SLinus Torvalds
1943c9d76be6STejun Heo idr_preload(GFP_KERNEL);
1944f32c10b0SJeff Mahoney spin_lock(&_minor_lock);
19451da177e4SLinus Torvalds
1946c9d76be6STejun Heo r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT);
19471da177e4SLinus Torvalds
1948f32c10b0SJeff Mahoney spin_unlock(&_minor_lock);
1949c9d76be6STejun Heo idr_preload_end();
1950c9d76be6STejun Heo if (r < 0)
19511da177e4SLinus Torvalds return r;
1952c9d76be6STejun Heo *minor = r;
1953c9d76be6STejun Heo return 0;
19541da177e4SLinus Torvalds }
19551da177e4SLinus Torvalds
195683d5cde4SAlexey Dobriyan static const struct block_device_operations dm_blk_dops;
1957681cc5e8SMike Snitzer static const struct block_device_operations dm_rq_blk_dops;
1958f26c5719SDan Williams static const struct dax_operations dm_dax_ops;
19591da177e4SLinus Torvalds
196053d5914fSMikulas Patocka static void dm_wq_work(struct work_struct *work);
196153d5914fSMikulas Patocka
1962aa6ce87aSSatya Tangirala #ifdef CONFIG_BLK_INLINE_ENCRYPTION
dm_queue_destroy_crypto_profile(struct request_queue * q)1963cb77cb5aSEric Biggers static void dm_queue_destroy_crypto_profile(struct request_queue *q)
1964aa6ce87aSSatya Tangirala {
1965cb77cb5aSEric Biggers dm_destroy_crypto_profile(q->crypto_profile);
1966aa6ce87aSSatya Tangirala }
1967aa6ce87aSSatya Tangirala
1968aa6ce87aSSatya Tangirala #else /* CONFIG_BLK_INLINE_ENCRYPTION */
1969aa6ce87aSSatya Tangirala
dm_queue_destroy_crypto_profile(struct request_queue * q)1970cb77cb5aSEric Biggers static inline void dm_queue_destroy_crypto_profile(struct request_queue *q)
1971aa6ce87aSSatya Tangirala {
1972aa6ce87aSSatya Tangirala }
1973aa6ce87aSSatya Tangirala #endif /* !CONFIG_BLK_INLINE_ENCRYPTION */
1974aa6ce87aSSatya Tangirala
cleanup_mapped_device(struct mapped_device * md)19750f20972fSMike Snitzer static void cleanup_mapped_device(struct mapped_device *md)
19760f20972fSMike Snitzer {
19770f20972fSMike Snitzer if (md->wq)
19780f20972fSMike Snitzer destroy_workqueue(md->wq);
197929dec90aSChristoph Hellwig dm_free_md_mempools(md->mempools);
19800f20972fSMike Snitzer
1981f26c5719SDan Williams if (md->dax_dev) {
1982fb08a190SChristoph Hellwig dax_remove_host(md->disk);
1983f26c5719SDan Williams kill_dax(md->dax_dev);
1984f26c5719SDan Williams put_dax(md->dax_dev);
1985f26c5719SDan Williams md->dax_dev = NULL;
1986f26c5719SDan Williams }
1987f26c5719SDan Williams
1988588b7f5dSKirill Tkhai dm_cleanup_zoned_dev(md);
19890f20972fSMike Snitzer if (md->disk) {
19900f20972fSMike Snitzer spin_lock(&_minor_lock);
19910f20972fSMike Snitzer md->disk->private_data = NULL;
19920f20972fSMike Snitzer spin_unlock(&_minor_lock);
199389f871afSChristoph Hellwig if (dm_get_md_type(md) != DM_TYPE_NONE) {
19941a581b72SChristoph Hellwig struct table_device *td;
19951a581b72SChristoph Hellwig
199689f871afSChristoph Hellwig dm_sysfs_exit(md);
19971a581b72SChristoph Hellwig list_for_each_entry(td, &md->table_devices, list) {
19981a581b72SChristoph Hellwig bd_unlink_disk_holder(td->dm_dev.bdev,
19991a581b72SChristoph Hellwig md->disk);
20001a581b72SChristoph Hellwig }
2001d563792cSYu Kuai
2002d563792cSYu Kuai /*
2003d563792cSYu Kuai * Hold lock to make sure del_gendisk() won't concurrent
2004d563792cSYu Kuai * with open/close_table_device().
2005d563792cSYu Kuai */
2006d563792cSYu Kuai mutex_lock(&md->table_devices_lock);
20070f20972fSMike Snitzer del_gendisk(md->disk);
2008d563792cSYu Kuai mutex_unlock(&md->table_devices_lock);
200989f871afSChristoph Hellwig }
2010cb77cb5aSEric Biggers dm_queue_destroy_crypto_profile(md->queue);
20118b9ab626SChristoph Hellwig put_disk(md->disk);
201274a2b6ecSChristoph Hellwig }
20130f20972fSMike Snitzer
20149f6dc633SMike Snitzer if (md->pending_io) {
20159f6dc633SMike Snitzer free_percpu(md->pending_io);
20169f6dc633SMike Snitzer md->pending_io = NULL;
20179f6dc633SMike Snitzer }
20189f6dc633SMike Snitzer
2019d09960b0STahsin Erdogan cleanup_srcu_struct(&md->io_barrier);
2020d09960b0STahsin Erdogan
2021d5ffebddSMike Snitzer mutex_destroy(&md->suspend_lock);
2022d5ffebddSMike Snitzer mutex_destroy(&md->type_lock);
2023d5ffebddSMike Snitzer mutex_destroy(&md->table_devices_lock);
2024a666e5c0SMikulas Patocka mutex_destroy(&md->swap_bios_lock);
2025d5ffebddSMike Snitzer
20264cc96131SMike Snitzer dm_mq_cleanup_mapped_device(md);
20270f20972fSMike Snitzer }
20280f20972fSMike Snitzer
20291da177e4SLinus Torvalds /*
20301da177e4SLinus Torvalds * Allocate and initialise a blank device with a given minor.
20311da177e4SLinus Torvalds */
alloc_dev(int minor)20322b06cfffSAlasdair G Kergon static struct mapped_device *alloc_dev(int minor)
20331da177e4SLinus Torvalds {
2034115485e8SMike Snitzer int r, numa_node_id = dm_get_numa_node();
2035115485e8SMike Snitzer struct mapped_device *md;
2036ba61fdd1SJeff Mahoney void *old_md;
20371da177e4SLinus Torvalds
2038856eb091SMikulas Patocka md = kvzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id);
20391da177e4SLinus Torvalds if (!md) {
204043e6c111SMikulas Patocka DMERR("unable to allocate device, out of memory.");
20411da177e4SLinus Torvalds return NULL;
20421da177e4SLinus Torvalds }
20431da177e4SLinus Torvalds
204410da4f79SJeff Mahoney if (!try_module_get(THIS_MODULE))
20456ed7ade8SMilan Broz goto bad_module_get;
204610da4f79SJeff Mahoney
20471da177e4SLinus Torvalds /* get a minor number for the dev */
20482b06cfffSAlasdair G Kergon if (minor == DM_ANY_MINOR)
2049cf13ab8eSFrederik Deweerdt r = next_free_minor(&minor);
20502b06cfffSAlasdair G Kergon else
2051cf13ab8eSFrederik Deweerdt r = specific_minor(minor);
20521da177e4SLinus Torvalds if (r < 0)
20536ed7ade8SMilan Broz goto bad_minor;
20541da177e4SLinus Torvalds
205583d5e5b0SMikulas Patocka r = init_srcu_struct(&md->io_barrier);
205683d5e5b0SMikulas Patocka if (r < 0)
205783d5e5b0SMikulas Patocka goto bad_io_barrier;
205883d5e5b0SMikulas Patocka
2059115485e8SMike Snitzer md->numa_node_id = numa_node_id;
2060591ddcfcSMike Snitzer md->init_tio_pdu = false;
2061a5664dadSMike Snitzer md->type = DM_TYPE_NONE;
2062e61290a4SDaniel Walker mutex_init(&md->suspend_lock);
2063a5664dadSMike Snitzer mutex_init(&md->type_lock);
206486f1152bSBenjamin Marzinski mutex_init(&md->table_devices_lock);
2065022c2611SMikulas Patocka spin_lock_init(&md->deferred_lock);
20661da177e4SLinus Torvalds atomic_set(&md->holders, 1);
20675c6bd75dSAlasdair G Kergon atomic_set(&md->open_count, 0);
20681da177e4SLinus Torvalds atomic_set(&md->event_nr, 0);
20697a8c3d3bSMike Anderson atomic_set(&md->uevent_seq, 0);
20707a8c3d3bSMike Anderson INIT_LIST_HEAD(&md->uevent_list);
207186f1152bSBenjamin Marzinski INIT_LIST_HEAD(&md->table_devices);
20727a8c3d3bSMike Anderson spin_lock_init(&md->uevent_lock);
20731da177e4SLinus Torvalds
207447ace7e0SMike Snitzer /*
2075c62b37d9SChristoph Hellwig * default to bio-based until DM table is loaded and md->type
2076c62b37d9SChristoph Hellwig * established. If request-based table is loaded: blk-mq will
2077c62b37d9SChristoph Hellwig * override accordingly.
207847ace7e0SMike Snitzer */
207974fe6ba9SChristoph Hellwig md->disk = blk_alloc_disk(md->numa_node_id);
20801da177e4SLinus Torvalds if (!md->disk)
20810f20972fSMike Snitzer goto bad;
208274fe6ba9SChristoph Hellwig md->queue = md->disk->queue;
20831da177e4SLinus Torvalds
2084f0b04115SJeff Mahoney init_waitqueue_head(&md->wait);
208553d5914fSMikulas Patocka INIT_WORK(&md->work, dm_wq_work);
20868b211aacSMing Lei INIT_WORK(&md->requeue_work, dm_wq_requeue_work);
2087f0b04115SJeff Mahoney init_waitqueue_head(&md->eventq);
20882995fa78SMikulas Patocka init_completion(&md->kobj_holder.completion);
2089f0b04115SJeff Mahoney
20908b211aacSMing Lei md->requeue_list = NULL;
2091a666e5c0SMikulas Patocka md->swap_bios = get_swap_bios();
2092a666e5c0SMikulas Patocka sema_init(&md->swap_bios_semaphore, md->swap_bios);
2093a666e5c0SMikulas Patocka mutex_init(&md->swap_bios_lock);
2094a666e5c0SMikulas Patocka
20951da177e4SLinus Torvalds md->disk->major = _major;
20961da177e4SLinus Torvalds md->disk->first_minor = minor;
209774fe6ba9SChristoph Hellwig md->disk->minors = 1;
20981ebe2e5fSChristoph Hellwig md->disk->flags |= GENHD_FL_NO_PART;
20991da177e4SLinus Torvalds md->disk->fops = &dm_blk_dops;
21001da177e4SLinus Torvalds md->disk->private_data = md;
21011da177e4SLinus Torvalds sprintf(md->disk->disk_name, "dm-%d", minor);
2102f26c5719SDan Williams
21035d2a228bSChristoph Hellwig if (IS_ENABLED(CONFIG_FS_DAX)) {
210430c6828aSChristoph Hellwig md->dax_dev = alloc_dax(md, &dm_dax_ops);
2105d7519392SChristoph Hellwig if (IS_ERR(md->dax_dev)) {
2106d7519392SChristoph Hellwig md->dax_dev = NULL;
2107f26c5719SDan Williams goto bad;
2108976431b0SDan Williams }
21097ac5360cSChristoph Hellwig set_dax_nocache(md->dax_dev);
21107ac5360cSChristoph Hellwig set_dax_nomc(md->dax_dev);
2111fb08a190SChristoph Hellwig if (dax_add_host(md->dax_dev, md->disk))
2112f26c5719SDan Williams goto bad;
2113f26c5719SDan Williams }
21141da177e4SLinus Torvalds
21157e51f257SMike Anderson format_dev_t(md->name, MKDEV(_major, minor));
21161da177e4SLinus Torvalds
2117c7c879eeSMichał Mirosław md->wq = alloc_workqueue("kdmflush/%s", WQ_MEM_RECLAIM, 0, md->name);
2118304f3f6aSMilan Broz if (!md->wq)
21190f20972fSMike Snitzer goto bad;
2120304f3f6aSMilan Broz
21219f6dc633SMike Snitzer md->pending_io = alloc_percpu(unsigned long);
21229f6dc633SMike Snitzer if (!md->pending_io)
21239f6dc633SMike Snitzer goto bad;
21249f6dc633SMike Snitzer
2125d3aa3e06SJiasheng Jiang r = dm_stats_init(&md->stats);
2126d3aa3e06SJiasheng Jiang if (r < 0)
2127d3aa3e06SJiasheng Jiang goto bad;
2128fd2ed4d2SMikulas Patocka
2129ba61fdd1SJeff Mahoney /* Populate the mapping, nobody knows we exist yet */
2130f32c10b0SJeff Mahoney spin_lock(&_minor_lock);
2131ba61fdd1SJeff Mahoney old_md = idr_replace(&_minor_idr, md, minor);
2132f32c10b0SJeff Mahoney spin_unlock(&_minor_lock);
2133ba61fdd1SJeff Mahoney
2134ba61fdd1SJeff Mahoney BUG_ON(old_md != MINOR_ALLOCED);
2135ba61fdd1SJeff Mahoney
21361da177e4SLinus Torvalds return md;
21371da177e4SLinus Torvalds
21380f20972fSMike Snitzer bad:
21390f20972fSMike Snitzer cleanup_mapped_device(md);
214083d5e5b0SMikulas Patocka bad_io_barrier:
21411da177e4SLinus Torvalds free_minor(minor);
21426ed7ade8SMilan Broz bad_minor:
214310da4f79SJeff Mahoney module_put(THIS_MODULE);
21446ed7ade8SMilan Broz bad_module_get:
2145856eb091SMikulas Patocka kvfree(md);
21461da177e4SLinus Torvalds return NULL;
21471da177e4SLinus Torvalds }
21481da177e4SLinus Torvalds
2149ae9da83fSJun'ichi Nomura static void unlock_fs(struct mapped_device *md);
2150ae9da83fSJun'ichi Nomura
free_dev(struct mapped_device * md)21511da177e4SLinus Torvalds static void free_dev(struct mapped_device *md)
21521da177e4SLinus Torvalds {
2153f331c029STejun Heo int minor = MINOR(disk_devt(md->disk));
215463d94e48SJun'ichi Nomura
2155ae9da83fSJun'ichi Nomura unlock_fs(md);
21562eb6e1e3SKeith Busch
21570f20972fSMike Snitzer cleanup_mapped_device(md);
21580f20972fSMike Snitzer
2159992ec6a9SChristoph Hellwig WARN_ON_ONCE(!list_empty(&md->table_devices));
21600f20972fSMike Snitzer dm_stats_cleanup(&md->stats);
216163a4f065SMike Snitzer free_minor(minor);
216263a4f065SMike Snitzer
216310da4f79SJeff Mahoney module_put(THIS_MODULE);
2164856eb091SMikulas Patocka kvfree(md);
21651da177e4SLinus Torvalds }
21661da177e4SLinus Torvalds
21671da177e4SLinus Torvalds /*
21681da177e4SLinus Torvalds * Bind a table to the device.
21691da177e4SLinus Torvalds */
event_callback(void * context)21701da177e4SLinus Torvalds static void event_callback(void *context)
21711da177e4SLinus Torvalds {
21727a8c3d3bSMike Anderson unsigned long flags;
21737a8c3d3bSMike Anderson LIST_HEAD(uevents);
2174d695e441SXU pengfei struct mapped_device *md = context;
21751da177e4SLinus Torvalds
21767a8c3d3bSMike Anderson spin_lock_irqsave(&md->uevent_lock, flags);
21777a8c3d3bSMike Anderson list_splice_init(&md->uevent_list, &uevents);
21787a8c3d3bSMike Anderson spin_unlock_irqrestore(&md->uevent_lock, flags);
21797a8c3d3bSMike Anderson
2180ed9e1982STejun Heo dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
21817a8c3d3bSMike Anderson
21821da177e4SLinus Torvalds atomic_inc(&md->event_nr);
21831da177e4SLinus Torvalds wake_up(&md->eventq);
218462e08243SMikulas Patocka dm_issue_global_event();
21851da177e4SLinus Torvalds }
21861da177e4SLinus Torvalds
2187c217649bSMike Snitzer /*
2188042d2a9bSAlasdair G Kergon * Returns old map, which caller must destroy.
2189042d2a9bSAlasdair G Kergon */
__bind(struct mapped_device * md,struct dm_table * t,struct queue_limits * limits)2190042d2a9bSAlasdair G Kergon static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
2191754c5fc7SMike Snitzer struct queue_limits *limits)
21921da177e4SLinus Torvalds {
2193042d2a9bSAlasdair G Kergon struct dm_table *old_map;
21941da177e4SLinus Torvalds sector_t size;
21952a2a4c51SJens Axboe int ret;
21961da177e4SLinus Torvalds
21975a8f1f80SBart Van Assche lockdep_assert_held(&md->suspend_lock);
21985a8f1f80SBart Van Assche
21991da177e4SLinus Torvalds size = dm_table_get_size(t);
22003ac51e74SDarrick J. Wong
22013ac51e74SDarrick J. Wong /*
22023ac51e74SDarrick J. Wong * Wipe any geometry if the size of the table changed.
22033ac51e74SDarrick J. Wong */
2204fd2ed4d2SMikulas Patocka if (size != dm_get_size(md))
22053ac51e74SDarrick J. Wong memset(&md->geometry, 0, sizeof(md->geometry));
22063ac51e74SDarrick J. Wong
22075424a0b8SMikulas Patocka set_capacity(md->disk, size);
22081da177e4SLinus Torvalds
2209cf222b37SAlasdair G Kergon dm_table_event_callback(t, event_callback, md);
22102ca3310eSAlasdair G Kergon
2211f5b4aee1SMike Snitzer if (dm_table_request_based(t)) {
221216f12266SMike Snitzer /*
22139c37de29SMike Snitzer * Leverage the fact that request-based DM targets are
22149c37de29SMike Snitzer * immutable singletons - used to optimize dm_mq_queue_rq.
221516f12266SMike Snitzer */
221616f12266SMike Snitzer md->immutable_target = dm_table_get_immutable_target(t);
2217e6ee8c0bSKiyoshi Ueda
221829dec90aSChristoph Hellwig /*
221929dec90aSChristoph Hellwig * There is no need to reload with request-based dm because the
222029dec90aSChristoph Hellwig * size of front_pad doesn't change.
222129dec90aSChristoph Hellwig *
222229dec90aSChristoph Hellwig * Note for future: If you are to reload bioset, prep-ed
222329dec90aSChristoph Hellwig * requests in the queue may refer to bio from the old bioset,
222429dec90aSChristoph Hellwig * so you must walk through the queue to unprep.
222529dec90aSChristoph Hellwig */
222629dec90aSChristoph Hellwig if (!md->mempools) {
222729dec90aSChristoph Hellwig md->mempools = t->mempools;
222829dec90aSChristoph Hellwig t->mempools = NULL;
222929dec90aSChristoph Hellwig }
223029dec90aSChristoph Hellwig } else {
223129dec90aSChristoph Hellwig /*
223229dec90aSChristoph Hellwig * The md may already have mempools that need changing.
223329dec90aSChristoph Hellwig * If so, reload bioset because front_pad may have changed
223429dec90aSChristoph Hellwig * because a different table was loaded.
223529dec90aSChristoph Hellwig */
223629dec90aSChristoph Hellwig dm_free_md_mempools(md->mempools);
223729dec90aSChristoph Hellwig md->mempools = t->mempools;
223829dec90aSChristoph Hellwig t->mempools = NULL;
22392a2a4c51SJens Axboe }
2240e6ee8c0bSKiyoshi Ueda
2241f5b4aee1SMike Snitzer ret = dm_table_set_restrictions(t, md->queue, limits);
2242bb37d772SDamien Le Moal if (ret) {
2243bb37d772SDamien Le Moal old_map = ERR_PTR(ret);
2244bb37d772SDamien Le Moal goto out;
2245bb37d772SDamien Le Moal }
2246bb37d772SDamien Le Moal
2247a12f5d48SEric Dumazet old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
22481d3aa6f6SMike Snitzer rcu_assign_pointer(md->map, (void *)t);
224936a0456fSAlasdair G Kergon md->immutable_target_type = dm_table_get_immutable_target_type(t);
225036a0456fSAlasdair G Kergon
225141abc4e1SHannes Reinecke if (old_map)
225283d5e5b0SMikulas Patocka dm_sync_table(md);
22532a2a4c51SJens Axboe out:
2254042d2a9bSAlasdair G Kergon return old_map;
22551da177e4SLinus Torvalds }
22561da177e4SLinus Torvalds
2257a7940155SAlasdair G Kergon /*
2258a7940155SAlasdair G Kergon * Returns unbound table for the caller to free.
2259a7940155SAlasdair G Kergon */
__unbind(struct mapped_device * md)2260a7940155SAlasdair G Kergon static struct dm_table *__unbind(struct mapped_device *md)
22611da177e4SLinus Torvalds {
2262a12f5d48SEric Dumazet struct dm_table *map = rcu_dereference_protected(md->map, 1);
22631da177e4SLinus Torvalds
22641da177e4SLinus Torvalds if (!map)
2265a7940155SAlasdair G Kergon return NULL;
22661da177e4SLinus Torvalds
22671da177e4SLinus Torvalds dm_table_event_callback(map, NULL, NULL);
22689cdb8520SMonam Agarwal RCU_INIT_POINTER(md->map, NULL);
226983d5e5b0SMikulas Patocka dm_sync_table(md);
2270a7940155SAlasdair G Kergon
2271a7940155SAlasdair G Kergon return map;
22721da177e4SLinus Torvalds }
22731da177e4SLinus Torvalds
22741da177e4SLinus Torvalds /*
22751da177e4SLinus Torvalds * Constructor for a new device.
22761da177e4SLinus Torvalds */
dm_create(int minor,struct mapped_device ** result)22772b06cfffSAlasdair G Kergon int dm_create(int minor, struct mapped_device **result)
22781da177e4SLinus Torvalds {
22791da177e4SLinus Torvalds struct mapped_device *md;
22801da177e4SLinus Torvalds
22812b06cfffSAlasdair G Kergon md = alloc_dev(minor);
22821da177e4SLinus Torvalds if (!md)
22831da177e4SLinus Torvalds return -ENXIO;
22841da177e4SLinus Torvalds
228591ccbbacSTushar Sugandhi dm_ima_reset_data(md);
228691ccbbacSTushar Sugandhi
22871da177e4SLinus Torvalds *result = md;
22881da177e4SLinus Torvalds return 0;
22891da177e4SLinus Torvalds }
22901da177e4SLinus Torvalds
2291a5664dadSMike Snitzer /*
2292a5664dadSMike Snitzer * Functions to manage md->type.
2293a5664dadSMike Snitzer * All are required to hold md->type_lock.
2294a5664dadSMike Snitzer */
dm_lock_md_type(struct mapped_device * md)2295a5664dadSMike Snitzer void dm_lock_md_type(struct mapped_device *md)
2296a5664dadSMike Snitzer {
2297a5664dadSMike Snitzer mutex_lock(&md->type_lock);
2298a5664dadSMike Snitzer }
2299a5664dadSMike Snitzer
dm_unlock_md_type(struct mapped_device * md)2300a5664dadSMike Snitzer void dm_unlock_md_type(struct mapped_device *md)
2301a5664dadSMike Snitzer {
2302a5664dadSMike Snitzer mutex_unlock(&md->type_lock);
2303a5664dadSMike Snitzer }
2304a5664dadSMike Snitzer
dm_set_md_type(struct mapped_device * md,enum dm_queue_mode type)23057e0d574fSBart Van Assche void dm_set_md_type(struct mapped_device *md, enum dm_queue_mode type)
2306a5664dadSMike Snitzer {
230700c4fc3bSMike Snitzer BUG_ON(!mutex_is_locked(&md->type_lock));
2308a5664dadSMike Snitzer md->type = type;
2309a5664dadSMike Snitzer }
2310a5664dadSMike Snitzer
dm_get_md_type(struct mapped_device * md)23117e0d574fSBart Van Assche enum dm_queue_mode dm_get_md_type(struct mapped_device *md)
2312a5664dadSMike Snitzer {
2313a5664dadSMike Snitzer return md->type;
2314a5664dadSMike Snitzer }
2315a5664dadSMike Snitzer
dm_get_immutable_target_type(struct mapped_device * md)231636a0456fSAlasdair G Kergon struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
231736a0456fSAlasdair G Kergon {
231836a0456fSAlasdair G Kergon return md->immutable_target_type;
231936a0456fSAlasdair G Kergon }
232036a0456fSAlasdair G Kergon
23214a0b4ddfSMike Snitzer /*
23224a0b4ddfSMike Snitzer * Setup the DM device's queue based on md's type
23234a0b4ddfSMike Snitzer */
dm_setup_md_queue(struct mapped_device * md,struct dm_table * t)2324591ddcfcSMike Snitzer int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
23254a0b4ddfSMike Snitzer {
2326ba305859SChristoph Hellwig enum dm_queue_mode type = dm_table_get_type(t);
2327c100ec49SMike Snitzer struct queue_limits limits;
23281a581b72SChristoph Hellwig struct table_device *td;
2329ba305859SChristoph Hellwig int r;
2330bfebd1cdSMike Snitzer
2331545ed20eSToshi Kani switch (type) {
2332bfebd1cdSMike Snitzer case DM_TYPE_REQUEST_BASED:
2333681cc5e8SMike Snitzer md->disk->fops = &dm_rq_blk_dops;
2334e83068a5SMike Snitzer r = dm_mq_init_request_queue(md, t);
2335bfebd1cdSMike Snitzer if (r) {
2336681cc5e8SMike Snitzer DMERR("Cannot initialize queue for request-based dm mapped device");
2337bfebd1cdSMike Snitzer return r;
2338bfebd1cdSMike Snitzer }
2339bfebd1cdSMike Snitzer break;
2340bfebd1cdSMike Snitzer case DM_TYPE_BIO_BASED:
2341545ed20eSToshi Kani case DM_TYPE_DAX_BIO_BASED:
2342526d1006SLi Nan blk_queue_flag_set(QUEUE_FLAG_IO_STAT, md->queue);
2343bfebd1cdSMike Snitzer break;
23447e0d574fSBart Van Assche case DM_TYPE_NONE:
23457e0d574fSBart Van Assche WARN_ON_ONCE(true);
23467e0d574fSBart Van Assche break;
2347ff36ab34SMike Snitzer }
23484a0b4ddfSMike Snitzer
2349c100ec49SMike Snitzer r = dm_calculate_queue_limits(t, &limits);
2350c100ec49SMike Snitzer if (r) {
2351c100ec49SMike Snitzer DMERR("Cannot calculate initial queue limits");
2352c100ec49SMike Snitzer return r;
2353c100ec49SMike Snitzer }
2354bb37d772SDamien Le Moal r = dm_table_set_restrictions(t, md->queue, &limits);
2355bb37d772SDamien Le Moal if (r)
2356bb37d772SDamien Le Moal return r;
235789f871afSChristoph Hellwig
2358d563792cSYu Kuai /*
2359d563792cSYu Kuai * Hold lock to make sure add_disk() and del_gendisk() won't concurrent
2360d563792cSYu Kuai * with open_table_device() and close_table_device().
2361d563792cSYu Kuai */
2362d563792cSYu Kuai mutex_lock(&md->table_devices_lock);
2363e7089f65SLuis Chamberlain r = add_disk(md->disk);
2364d563792cSYu Kuai mutex_unlock(&md->table_devices_lock);
2365e7089f65SLuis Chamberlain if (r)
2366e7089f65SLuis Chamberlain return r;
236789f871afSChristoph Hellwig
23681a581b72SChristoph Hellwig /*
23691a581b72SChristoph Hellwig * Register the holder relationship for devices added before the disk
23701a581b72SChristoph Hellwig * was live.
23711a581b72SChristoph Hellwig */
23721a581b72SChristoph Hellwig list_for_each_entry(td, &md->table_devices, list) {
23731a581b72SChristoph Hellwig r = bd_link_disk_holder(td->dm_dev.bdev, md->disk);
23741a581b72SChristoph Hellwig if (r)
23751a581b72SChristoph Hellwig goto out_undo_holders;
23761a581b72SChristoph Hellwig }
23771a581b72SChristoph Hellwig
237889f871afSChristoph Hellwig r = dm_sysfs_init(md);
23791a581b72SChristoph Hellwig if (r)
23801a581b72SChristoph Hellwig goto out_undo_holders;
23811a581b72SChristoph Hellwig
23821a581b72SChristoph Hellwig md->type = type;
23831a581b72SChristoph Hellwig return 0;
23841a581b72SChristoph Hellwig
23851a581b72SChristoph Hellwig out_undo_holders:
23861a581b72SChristoph Hellwig list_for_each_entry_continue_reverse(td, &md->table_devices, list)
23871a581b72SChristoph Hellwig bd_unlink_disk_holder(td->dm_dev.bdev, md->disk);
2388d563792cSYu Kuai mutex_lock(&md->table_devices_lock);
238989f871afSChristoph Hellwig del_gendisk(md->disk);
2390d563792cSYu Kuai mutex_unlock(&md->table_devices_lock);
239189f871afSChristoph Hellwig return r;
239289f871afSChristoph Hellwig }
2393d563792cSYu Kuai
dm_get_md(dev_t dev)23942bec1f4aSMikulas Patocka struct mapped_device *dm_get_md(dev_t dev)
23951da177e4SLinus Torvalds {
23961da177e4SLinus Torvalds struct mapped_device *md;
239786a3238cSHeinz Mauelshagen unsigned int minor = MINOR(dev);
23981da177e4SLinus Torvalds
23991da177e4SLinus Torvalds if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
24001da177e4SLinus Torvalds return NULL;
24011da177e4SLinus Torvalds
2402f32c10b0SJeff Mahoney spin_lock(&_minor_lock);
24031da177e4SLinus Torvalds
24041da177e4SLinus Torvalds md = idr_find(&_minor_idr, minor);
240549de5769SMike Snitzer if (!md || md == MINOR_ALLOCED || (MINOR(disk_devt(dm_disk(md))) != minor) ||
240649de5769SMike Snitzer test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
2407637842cfSDavid Teigland md = NULL;
2408fba9f90eSJeff Mahoney goto out;
2409fba9f90eSJeff Mahoney }
24102bec1f4aSMikulas Patocka dm_get(md);
2411fba9f90eSJeff Mahoney out:
2412f32c10b0SJeff Mahoney spin_unlock(&_minor_lock);
24131da177e4SLinus Torvalds
2414637842cfSDavid Teigland return md;
2415637842cfSDavid Teigland }
24163cf2e4baSAlasdair G Kergon EXPORT_SYMBOL_GPL(dm_get_md);
2417d229a958SDavid Teigland
dm_get_mdptr(struct mapped_device * md)24189ade92a9SAlasdair G Kergon void *dm_get_mdptr(struct mapped_device *md)
2419637842cfSDavid Teigland {
24209ade92a9SAlasdair G Kergon return md->interface_ptr;
24211da177e4SLinus Torvalds }
24221da177e4SLinus Torvalds
dm_set_mdptr(struct mapped_device * md,void * ptr)24231da177e4SLinus Torvalds void dm_set_mdptr(struct mapped_device *md, void *ptr)
24241da177e4SLinus Torvalds {
24251da177e4SLinus Torvalds md->interface_ptr = ptr;
24261da177e4SLinus Torvalds }
24271da177e4SLinus Torvalds
dm_get(struct mapped_device * md)24281da177e4SLinus Torvalds void dm_get(struct mapped_device *md)
24291da177e4SLinus Torvalds {
24301da177e4SLinus Torvalds atomic_inc(&md->holders);
24313f77316dSKiyoshi Ueda BUG_ON(test_bit(DMF_FREEING, &md->flags));
24321da177e4SLinus Torvalds }
24331da177e4SLinus Torvalds
dm_hold(struct mapped_device * md)243409ee96b2SMikulas Patocka int dm_hold(struct mapped_device *md)
243509ee96b2SMikulas Patocka {
243609ee96b2SMikulas Patocka spin_lock(&_minor_lock);
243709ee96b2SMikulas Patocka if (test_bit(DMF_FREEING, &md->flags)) {
243809ee96b2SMikulas Patocka spin_unlock(&_minor_lock);
243909ee96b2SMikulas Patocka return -EBUSY;
244009ee96b2SMikulas Patocka }
244109ee96b2SMikulas Patocka dm_get(md);
244209ee96b2SMikulas Patocka spin_unlock(&_minor_lock);
244309ee96b2SMikulas Patocka return 0;
244409ee96b2SMikulas Patocka }
244509ee96b2SMikulas Patocka EXPORT_SYMBOL_GPL(dm_hold);
244609ee96b2SMikulas Patocka
dm_device_name(struct mapped_device * md)244772d94861SAlasdair G Kergon const char *dm_device_name(struct mapped_device *md)
244872d94861SAlasdair G Kergon {
244972d94861SAlasdair G Kergon return md->name;
245072d94861SAlasdair G Kergon }
245172d94861SAlasdair G Kergon EXPORT_SYMBOL_GPL(dm_device_name);
245272d94861SAlasdair G Kergon
__dm_destroy(struct mapped_device * md,bool wait)24533f77316dSKiyoshi Ueda static void __dm_destroy(struct mapped_device *md, bool wait)
24541da177e4SLinus Torvalds {
24551134e5aeSMike Anderson struct dm_table *map;
245683d5e5b0SMikulas Patocka int srcu_idx;
24571da177e4SLinus Torvalds
24583f77316dSKiyoshi Ueda might_sleep();
2459fba9f90eSJeff Mahoney
246063a4f065SMike Snitzer spin_lock(&_minor_lock);
24613f77316dSKiyoshi Ueda idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));
2462fba9f90eSJeff Mahoney set_bit(DMF_FREEING, &md->flags);
2463f32c10b0SJeff Mahoney spin_unlock(&_minor_lock);
24643f77316dSKiyoshi Ueda
24657a5428dcSChristoph Hellwig blk_mark_disk_dead(md->disk);
24663b785fbcSBart Van Assche
2467ab7c7bb6SMikulas Patocka /*
2468ab7c7bb6SMikulas Patocka * Take suspend_lock so that presuspend and postsuspend methods
2469ab7c7bb6SMikulas Patocka * do not race with internal suspend.
2470ab7c7bb6SMikulas Patocka */
2471ab7c7bb6SMikulas Patocka mutex_lock(&md->suspend_lock);
24722a708cffSJunichi Nomura map = dm_get_live_table(md, &srcu_idx);
24734f186f8bSKiyoshi Ueda if (!dm_suspended_md(md)) {
24741da177e4SLinus Torvalds dm_table_presuspend_targets(map);
2475adc0daadSMikulas Patocka set_bit(DMF_SUSPENDED, &md->flags);
24765df96f2bSMikulas Patocka set_bit(DMF_POST_SUSPENDING, &md->flags);
24771da177e4SLinus Torvalds dm_table_postsuspend_targets(map);
24781da177e4SLinus Torvalds }
2479238d991fSHeinz Mauelshagen /* dm_put_live_table must be before fsleep, otherwise deadlock is possible */
248083d5e5b0SMikulas Patocka dm_put_live_table(md, srcu_idx);
24812a708cffSJunichi Nomura mutex_unlock(&md->suspend_lock);
248283d5e5b0SMikulas Patocka
24833f77316dSKiyoshi Ueda /*
24843f77316dSKiyoshi Ueda * Rare, but there may be I/O requests still going to complete,
24853f77316dSKiyoshi Ueda * for example. Wait for all references to disappear.
24863f77316dSKiyoshi Ueda * No one should increment the reference count of the mapped_device,
24873f77316dSKiyoshi Ueda * after the mapped_device state becomes DMF_FREEING.
24883f77316dSKiyoshi Ueda */
24893f77316dSKiyoshi Ueda if (wait)
24903f77316dSKiyoshi Ueda while (atomic_read(&md->holders))
2491238d991fSHeinz Mauelshagen fsleep(1000);
24923f77316dSKiyoshi Ueda else if (atomic_read(&md->holders))
24933f77316dSKiyoshi Ueda DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)",
24943f77316dSKiyoshi Ueda dm_device_name(md), atomic_read(&md->holders));
24953f77316dSKiyoshi Ueda
2496a7940155SAlasdair G Kergon dm_table_destroy(__unbind(md));
24971da177e4SLinus Torvalds free_dev(md);
24981da177e4SLinus Torvalds }
24993f77316dSKiyoshi Ueda
dm_destroy(struct mapped_device * md)25003f77316dSKiyoshi Ueda void dm_destroy(struct mapped_device *md)
25013f77316dSKiyoshi Ueda {
25023f77316dSKiyoshi Ueda __dm_destroy(md, true);
25033f77316dSKiyoshi Ueda }
25043f77316dSKiyoshi Ueda
dm_destroy_immediate(struct mapped_device * md)25053f77316dSKiyoshi Ueda void dm_destroy_immediate(struct mapped_device *md)
25063f77316dSKiyoshi Ueda {
25073f77316dSKiyoshi Ueda __dm_destroy(md, false);
25083f77316dSKiyoshi Ueda }
25093f77316dSKiyoshi Ueda
dm_put(struct mapped_device * md)25103f77316dSKiyoshi Ueda void dm_put(struct mapped_device *md)
25113f77316dSKiyoshi Ueda {
25123f77316dSKiyoshi Ueda atomic_dec(&md->holders);
25131da177e4SLinus Torvalds }
251479eb885cSEdward Goggin EXPORT_SYMBOL_GPL(dm_put);
25151da177e4SLinus Torvalds
dm_in_flight_bios(struct mapped_device * md)25169f6dc633SMike Snitzer static bool dm_in_flight_bios(struct mapped_device *md)
251785067747SMing Lei {
251885067747SMing Lei int cpu;
25199f6dc633SMike Snitzer unsigned long sum = 0;
252085067747SMing Lei
25219f6dc633SMike Snitzer for_each_possible_cpu(cpu)
25229f6dc633SMike Snitzer sum += *per_cpu_ptr(md->pending_io, cpu);
252385067747SMing Lei
252485067747SMing Lei return sum != 0;
252585067747SMing Lei }
252685067747SMing Lei
dm_wait_for_bios_completion(struct mapped_device * md,unsigned int task_state)25272f064a59SPeter Zijlstra static int dm_wait_for_bios_completion(struct mapped_device *md, unsigned int task_state)
252846125c1cSMilan Broz {
252946125c1cSMilan Broz int r = 0;
25309f4c3f87SBart Van Assche DEFINE_WAIT(wait);
253146125c1cSMilan Broz
253285067747SMing Lei while (true) {
25339f4c3f87SBart Van Assche prepare_to_wait(&md->wait, &wait, task_state);
253446125c1cSMilan Broz
25359f6dc633SMike Snitzer if (!dm_in_flight_bios(md))
253646125c1cSMilan Broz break;
253746125c1cSMilan Broz
2538e3fabdfdSBart Van Assche if (signal_pending_state(task_state, current)) {
253900b395e9SMikulas Patocka r = -ERESTARTSYS;
254046125c1cSMilan Broz break;
254146125c1cSMilan Broz }
254246125c1cSMilan Broz
254346125c1cSMilan Broz io_schedule();
254446125c1cSMilan Broz }
25459f4c3f87SBart Van Assche finish_wait(&md->wait, &wait);
2546b44ebeb0SMikulas Patocka
25479f6dc633SMike Snitzer smp_rmb();
25489f6dc633SMike Snitzer
254946125c1cSMilan Broz return r;
255046125c1cSMilan Broz }
255146125c1cSMilan Broz
dm_wait_for_completion(struct mapped_device * md,unsigned int task_state)25522f064a59SPeter Zijlstra static int dm_wait_for_completion(struct mapped_device *md, unsigned int task_state)
255385067747SMing Lei {
255485067747SMing Lei int r = 0;
255585067747SMing Lei
255685067747SMing Lei if (!queue_is_mq(md->queue))
255785067747SMing Lei return dm_wait_for_bios_completion(md, task_state);
255885067747SMing Lei
255985067747SMing Lei while (true) {
256085067747SMing Lei if (!blk_mq_queue_inflight(md->queue))
256185067747SMing Lei break;
256285067747SMing Lei
256385067747SMing Lei if (signal_pending_state(task_state, current)) {
256400b395e9SMikulas Patocka r = -ERESTARTSYS;
256585067747SMing Lei break;
256685067747SMing Lei }
256785067747SMing Lei
2568238d991fSHeinz Mauelshagen fsleep(5000);
256985067747SMing Lei }
257085067747SMing Lei
257185067747SMing Lei return r;
257285067747SMing Lei }
257385067747SMing Lei
25741da177e4SLinus Torvalds /*
25751da177e4SLinus Torvalds * Process the deferred bios
25761da177e4SLinus Torvalds */
dm_wq_work(struct work_struct * work)2577ef208587SMikulas Patocka static void dm_wq_work(struct work_struct *work)
25781da177e4SLinus Torvalds {
25790c2915b8SMike Snitzer struct mapped_device *md = container_of(work, struct mapped_device, work);
25800c2915b8SMike Snitzer struct bio *bio;
2581ef208587SMikulas Patocka
25823b00b203SMikulas Patocka while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
2583022c2611SMikulas Patocka spin_lock_irq(&md->deferred_lock);
25840c2915b8SMike Snitzer bio = bio_list_pop(&md->deferred);
2585022c2611SMikulas Patocka spin_unlock_irq(&md->deferred_lock);
2586022c2611SMikulas Patocka
25870c2915b8SMike Snitzer if (!bio)
2588df12ee99SAlasdair G Kergon break;
258973d410c0SMilan Broz
25900c2915b8SMike Snitzer submit_bio_noacct(bio);
25910ca44fceSPingfan Liu cond_resched();
2592e6ee8c0bSKiyoshi Ueda }
25931da177e4SLinus Torvalds }
25941da177e4SLinus Torvalds
dm_queue_flush(struct mapped_device * md)25959a1fb464SMikulas Patocka static void dm_queue_flush(struct mapped_device *md)
2596304f3f6aSMilan Broz {
25973b00b203SMikulas Patocka clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
25984e857c58SPeter Zijlstra smp_mb__after_atomic();
259953d5914fSMikulas Patocka queue_work(md->wq, &md->work);
2600304f3f6aSMilan Broz }
2601304f3f6aSMilan Broz
26021da177e4SLinus Torvalds /*
2603042d2a9bSAlasdair G Kergon * Swap in a new table, returning the old one for the caller to destroy.
26041da177e4SLinus Torvalds */
dm_swap_table(struct mapped_device * md,struct dm_table * table)2605042d2a9bSAlasdair G Kergon struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table)
26061da177e4SLinus Torvalds {
260787eb5b21SMike Christie struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL);
2608754c5fc7SMike Snitzer struct queue_limits limits;
2609042d2a9bSAlasdair G Kergon int r;
26101da177e4SLinus Torvalds
2611e61290a4SDaniel Walker mutex_lock(&md->suspend_lock);
26121da177e4SLinus Torvalds
26131da177e4SLinus Torvalds /* device must be suspended */
26144f186f8bSKiyoshi Ueda if (!dm_suspended_md(md))
261593c534aeSAlasdair G Kergon goto out;
26161da177e4SLinus Torvalds
26173ae70656SMike Snitzer /*
26183ae70656SMike Snitzer * If the new table has no data devices, retain the existing limits.
26193ae70656SMike Snitzer * This helps multipath with queue_if_no_path if all paths disappear,
26203ae70656SMike Snitzer * then new I/O is queued based on these limits, and then some paths
26213ae70656SMike Snitzer * reappear.
26223ae70656SMike Snitzer */
26233ae70656SMike Snitzer if (dm_table_has_no_data_devices(table)) {
262483d5e5b0SMikulas Patocka live_map = dm_get_live_table_fast(md);
26253ae70656SMike Snitzer if (live_map)
26263ae70656SMike Snitzer limits = md->queue->limits;
262783d5e5b0SMikulas Patocka dm_put_live_table_fast(md);
26283ae70656SMike Snitzer }
26293ae70656SMike Snitzer
263087eb5b21SMike Christie if (!live_map) {
2631754c5fc7SMike Snitzer r = dm_calculate_queue_limits(table, &limits);
2632042d2a9bSAlasdair G Kergon if (r) {
2633042d2a9bSAlasdair G Kergon map = ERR_PTR(r);
2634754c5fc7SMike Snitzer goto out;
2635042d2a9bSAlasdair G Kergon }
263687eb5b21SMike Christie }
2637754c5fc7SMike Snitzer
2638042d2a9bSAlasdair G Kergon map = __bind(md, table, &limits);
263962e08243SMikulas Patocka dm_issue_global_event();
26401da177e4SLinus Torvalds
264193c534aeSAlasdair G Kergon out:
2642e61290a4SDaniel Walker mutex_unlock(&md->suspend_lock);
2643042d2a9bSAlasdair G Kergon return map;
26441da177e4SLinus Torvalds }
26451da177e4SLinus Torvalds
26461da177e4SLinus Torvalds /*
26471da177e4SLinus Torvalds * Functions to lock and unlock any filesystem running on the
26481da177e4SLinus Torvalds * device.
26491da177e4SLinus Torvalds */
lock_fs(struct mapped_device * md)26502ca3310eSAlasdair G Kergon static int lock_fs(struct mapped_device *md)
26511da177e4SLinus Torvalds {
2652e39e2e95SAlasdair G Kergon int r;
26531da177e4SLinus Torvalds
2654040f04bdSChristoph Hellwig WARN_ON(test_bit(DMF_FROZEN, &md->flags));
2655dfbe03f6SAlasdair G Kergon
2656977115c0SChristoph Hellwig r = freeze_bdev(md->disk->part0);
2657040f04bdSChristoph Hellwig if (!r)
2658aa8d7c2fSAlasdair G Kergon set_bit(DMF_FROZEN, &md->flags);
2659040f04bdSChristoph Hellwig return r;
26601da177e4SLinus Torvalds }
26611da177e4SLinus Torvalds
unlock_fs(struct mapped_device * md)26622ca3310eSAlasdair G Kergon static void unlock_fs(struct mapped_device *md)
26631da177e4SLinus Torvalds {
2664aa8d7c2fSAlasdair G Kergon if (!test_bit(DMF_FROZEN, &md->flags))
2665aa8d7c2fSAlasdair G Kergon return;
2666977115c0SChristoph Hellwig thaw_bdev(md->disk->part0);
2667aa8d7c2fSAlasdair G Kergon clear_bit(DMF_FROZEN, &md->flags);
26681da177e4SLinus Torvalds }
26691da177e4SLinus Torvalds
26701da177e4SLinus Torvalds /*
2671b48633f8SBart Van Assche * @suspend_flags: DM_SUSPEND_LOCKFS_FLAG and/or DM_SUSPEND_NOFLUSH_FLAG
2672b48633f8SBart Van Assche * @task_state: e.g. TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE
2673b48633f8SBart Van Assche * @dmf_suspended_flag: DMF_SUSPENDED or DMF_SUSPENDED_INTERNALLY
2674b48633f8SBart Van Assche *
2675ffcc3936SMike Snitzer * If __dm_suspend returns 0, the device is completely quiescent
2676ffcc3936SMike Snitzer * now. There is no request-processing activity. All new requests
2677ffcc3936SMike Snitzer * are being added to md->deferred list.
2678cec47e3dSKiyoshi Ueda */
__dm_suspend(struct mapped_device * md,struct dm_table * map,unsigned int suspend_flags,unsigned int task_state,int dmf_suspended_flag)2679ffcc3936SMike Snitzer static int __dm_suspend(struct mapped_device *md, struct dm_table *map,
268086a3238cSHeinz Mauelshagen unsigned int suspend_flags, unsigned int task_state,
2681eaf9a736SMike Snitzer int dmf_suspended_flag)
26821da177e4SLinus Torvalds {
2683ffcc3936SMike Snitzer bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG;
2684ffcc3936SMike Snitzer bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG;
2685ffcc3936SMike Snitzer int r;
2686cf222b37SAlasdair G Kergon
26875a8f1f80SBart Van Assche lockdep_assert_held(&md->suspend_lock);
26885a8f1f80SBart Van Assche
26892e93ccc1SKiyoshi Ueda /*
26902e93ccc1SKiyoshi Ueda * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
26912e93ccc1SKiyoshi Ueda * This flag is cleared before dm_suspend returns.
26922e93ccc1SKiyoshi Ueda */
26932e93ccc1SKiyoshi Ueda if (noflush)
26942e93ccc1SKiyoshi Ueda set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
269586331f39SBart Van Assche else
2696ac75b09fSMike Snitzer DMDEBUG("%s: suspending with flush", dm_device_name(md));
26972e93ccc1SKiyoshi Ueda
2698d67ee213SMike Snitzer /*
2699d67ee213SMike Snitzer * This gets reverted if there's an error later and the targets
2700d67ee213SMike Snitzer * provide the .presuspend_undo hook.
2701d67ee213SMike Snitzer */
27021da177e4SLinus Torvalds dm_table_presuspend_targets(map);
27031da177e4SLinus Torvalds
27042e93ccc1SKiyoshi Ueda /*
27059f518b27SKiyoshi Ueda * Flush I/O to the device.
27069f518b27SKiyoshi Ueda * Any I/O submitted after lock_fs() may not be flushed.
27079f518b27SKiyoshi Ueda * noflush takes precedence over do_lockfs.
27089f518b27SKiyoshi Ueda * (lock_fs() flushes I/Os and waits for them to complete.)
27092e93ccc1SKiyoshi Ueda */
271032a926daSMikulas Patocka if (!noflush && do_lockfs) {
27112ca3310eSAlasdair G Kergon r = lock_fs(md);
2712d67ee213SMike Snitzer if (r) {
2713d67ee213SMike Snitzer dm_table_presuspend_undo_targets(map);
2714ffcc3936SMike Snitzer return r;
2715aa8d7c2fSAlasdair G Kergon }
2716d67ee213SMike Snitzer }
27171da177e4SLinus Torvalds
27181da177e4SLinus Torvalds /*
27193b00b203SMikulas Patocka * Here we must make sure that no processes are submitting requests
27203b00b203SMikulas Patocka * to target drivers i.e. no one may be executing
272196c9865cSMike Snitzer * dm_split_and_process_bio from dm_submit_bio.
27223b00b203SMikulas Patocka *
272396c9865cSMike Snitzer * To get all processes out of dm_split_and_process_bio in dm_submit_bio,
27243b00b203SMikulas Patocka * we take the write lock. To prevent any process from reentering
272596c9865cSMike Snitzer * dm_split_and_process_bio from dm_submit_bio and quiesce the thread
27260cede372SMike Snitzer * (dm_wq_work), we set DMF_BLOCK_IO_FOR_SUSPEND and call
27276a8736d1STejun Heo * flush_workqueue(md->wq).
27281da177e4SLinus Torvalds */
27291eb787ecSAlasdair G Kergon set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
273041abc4e1SHannes Reinecke if (map)
273183d5e5b0SMikulas Patocka synchronize_srcu(&md->io_barrier);
27321da177e4SLinus Torvalds
2733d0bcb878SKiyoshi Ueda /*
273429e4013dSTejun Heo * Stop md->queue before flushing md->wq in case request-based
273529e4013dSTejun Heo * dm defers requests to md->wq from md->queue.
2736d0bcb878SKiyoshi Ueda */
27376a23e05cSJens Axboe if (dm_request_based(md))
2738eca7ee6dSMike Snitzer dm_stop_queue(md->queue);
2739cec47e3dSKiyoshi Ueda
2740d0bcb878SKiyoshi Ueda flush_workqueue(md->wq);
2741d0bcb878SKiyoshi Ueda
27421da177e4SLinus Torvalds /*
27433b00b203SMikulas Patocka * At this point no more requests are entering target request routines.
27443b00b203SMikulas Patocka * We call dm_wait_for_completion to wait for all existing requests
27453b00b203SMikulas Patocka * to finish.
27461da177e4SLinus Torvalds */
2747b48633f8SBart Van Assche r = dm_wait_for_completion(md, task_state);
2748eaf9a736SMike Snitzer if (!r)
2749eaf9a736SMike Snitzer set_bit(dmf_suspended_flag, &md->flags);
27501da177e4SLinus Torvalds
27516d6f10dfSMilan Broz if (noflush)
2752022c2611SMikulas Patocka clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
275341abc4e1SHannes Reinecke if (map)
275483d5e5b0SMikulas Patocka synchronize_srcu(&md->io_barrier);
27552e93ccc1SKiyoshi Ueda
27561da177e4SLinus Torvalds /* were we interrupted ? */
275746125c1cSMilan Broz if (r < 0) {
27589a1fb464SMikulas Patocka dm_queue_flush(md);
275973d410c0SMilan Broz
2760cec47e3dSKiyoshi Ueda if (dm_request_based(md))
2761eca7ee6dSMike Snitzer dm_start_queue(md->queue);
2762cec47e3dSKiyoshi Ueda
27632ca3310eSAlasdair G Kergon unlock_fs(md);
2764d67ee213SMike Snitzer dm_table_presuspend_undo_targets(map);
2765ffcc3936SMike Snitzer /* pushback list is already flushed, so skip flush */
2766ffcc3936SMike Snitzer }
2767ffcc3936SMike Snitzer
2768ffcc3936SMike Snitzer return r;
27692ca3310eSAlasdair G Kergon }
27702ca3310eSAlasdair G Kergon
27713b00b203SMikulas Patocka /*
2772ffcc3936SMike Snitzer * We need to be able to change a mapping table under a mounted
2773ffcc3936SMike Snitzer * filesystem. For example we might want to move some data in
2774ffcc3936SMike Snitzer * the background. Before the table can be swapped with
2775ffcc3936SMike Snitzer * dm_bind_table, dm_suspend must be called to flush any in
2776ffcc3936SMike Snitzer * flight bios and ensure that any further io gets deferred.
27773b00b203SMikulas Patocka */
2778ffcc3936SMike Snitzer /*
2779ffcc3936SMike Snitzer * Suspend mechanism in request-based dm.
2780ffcc3936SMike Snitzer *
2781ffcc3936SMike Snitzer * 1. Flush all I/Os by lock_fs() if needed.
2782ffcc3936SMike Snitzer * 2. Stop dispatching any I/O by stopping the request_queue.
2783ffcc3936SMike Snitzer * 3. Wait for all in-flight I/Os to be completed or requeued.
2784ffcc3936SMike Snitzer *
2785ffcc3936SMike Snitzer * To abort suspend, start the request_queue.
2786ffcc3936SMike Snitzer */
dm_suspend(struct mapped_device * md,unsigned int suspend_flags)278786a3238cSHeinz Mauelshagen int dm_suspend(struct mapped_device *md, unsigned int suspend_flags)
2788ffcc3936SMike Snitzer {
2789ffcc3936SMike Snitzer struct dm_table *map = NULL;
2790ffcc3936SMike Snitzer int r = 0;
2791ffcc3936SMike Snitzer
2792ffcc3936SMike Snitzer retry:
2793ffcc3936SMike Snitzer mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2794ffcc3936SMike Snitzer
2795ffcc3936SMike Snitzer if (dm_suspended_md(md)) {
2796ffcc3936SMike Snitzer r = -EINVAL;
2797ffcc3936SMike Snitzer goto out_unlock;
2798ffcc3936SMike Snitzer }
2799ffcc3936SMike Snitzer
2800ffcc3936SMike Snitzer if (dm_suspended_internally_md(md)) {
2801ffcc3936SMike Snitzer /* already internally suspended, wait for internal resume */
2802ffcc3936SMike Snitzer mutex_unlock(&md->suspend_lock);
2803ffcc3936SMike Snitzer r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2804ffcc3936SMike Snitzer if (r)
2805ffcc3936SMike Snitzer return r;
2806ffcc3936SMike Snitzer goto retry;
2807ffcc3936SMike Snitzer }
2808ffcc3936SMike Snitzer
2809a12f5d48SEric Dumazet map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
28102760904dSLi Lingfeng if (!map) {
28112760904dSLi Lingfeng /* avoid deadlock with fs/namespace.c:do_mount() */
28122760904dSLi Lingfeng suspend_flags &= ~DM_SUSPEND_LOCKFS_FLAG;
28132760904dSLi Lingfeng }
2814ffcc3936SMike Snitzer
2815eaf9a736SMike Snitzer r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED);
2816ffcc3936SMike Snitzer if (r)
2817ffcc3936SMike Snitzer goto out_unlock;
28183b00b203SMikulas Patocka
28195df96f2bSMikulas Patocka set_bit(DMF_POST_SUSPENDING, &md->flags);
28204d4471cbSKiyoshi Ueda dm_table_postsuspend_targets(map);
28215df96f2bSMikulas Patocka clear_bit(DMF_POST_SUSPENDING, &md->flags);
28224d4471cbSKiyoshi Ueda
2823d287483dSAlasdair G Kergon out_unlock:
2824e61290a4SDaniel Walker mutex_unlock(&md->suspend_lock);
2825cf222b37SAlasdair G Kergon return r;
28261da177e4SLinus Torvalds }
28271da177e4SLinus Torvalds
__dm_resume(struct mapped_device * md,struct dm_table * map)2828ffcc3936SMike Snitzer static int __dm_resume(struct mapped_device *md, struct dm_table *map)
28291da177e4SLinus Torvalds {
2830ffcc3936SMike Snitzer if (map) {
2831ffcc3936SMike Snitzer int r = dm_table_resume_targets(map);
2832b30f1607SHeinz Mauelshagen
28338757b776SMilan Broz if (r)
2834ffcc3936SMike Snitzer return r;
2835ffcc3936SMike Snitzer }
28362ca3310eSAlasdair G Kergon
28379a1fb464SMikulas Patocka dm_queue_flush(md);
28382ca3310eSAlasdair G Kergon
2839cec47e3dSKiyoshi Ueda /*
2840cec47e3dSKiyoshi Ueda * Flushing deferred I/Os must be done after targets are resumed
2841cec47e3dSKiyoshi Ueda * so that mapping of targets can work correctly.
2842cec47e3dSKiyoshi Ueda * Request-based dm is queueing the deferred I/Os in its request_queue.
2843cec47e3dSKiyoshi Ueda */
2844cec47e3dSKiyoshi Ueda if (dm_request_based(md))
2845eca7ee6dSMike Snitzer dm_start_queue(md->queue);
2846cec47e3dSKiyoshi Ueda
28472ca3310eSAlasdair G Kergon unlock_fs(md);
28482ca3310eSAlasdair G Kergon
2849ffcc3936SMike Snitzer return 0;
2850ffcc3936SMike Snitzer }
2851ffcc3936SMike Snitzer
dm_resume(struct mapped_device * md)2852ffcc3936SMike Snitzer int dm_resume(struct mapped_device *md)
2853ffcc3936SMike Snitzer {
28548dc23658SMinfei Huang int r;
2855ffcc3936SMike Snitzer struct dm_table *map = NULL;
2856ffcc3936SMike Snitzer
2857ffcc3936SMike Snitzer retry:
28588dc23658SMinfei Huang r = -EINVAL;
2859ffcc3936SMike Snitzer mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2860ffcc3936SMike Snitzer
2861ffcc3936SMike Snitzer if (!dm_suspended_md(md))
2862ffcc3936SMike Snitzer goto out;
2863ffcc3936SMike Snitzer
2864ffcc3936SMike Snitzer if (dm_suspended_internally_md(md)) {
2865ffcc3936SMike Snitzer /* already internally suspended, wait for internal resume */
2866ffcc3936SMike Snitzer mutex_unlock(&md->suspend_lock);
2867ffcc3936SMike Snitzer r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2868ffcc3936SMike Snitzer if (r)
2869ffcc3936SMike Snitzer return r;
2870ffcc3936SMike Snitzer goto retry;
2871ffcc3936SMike Snitzer }
2872ffcc3936SMike Snitzer
2873a12f5d48SEric Dumazet map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2874ffcc3936SMike Snitzer if (!map || !dm_table_get_size(map))
2875ffcc3936SMike Snitzer goto out;
2876ffcc3936SMike Snitzer
2877ffcc3936SMike Snitzer r = __dm_resume(md, map);
2878ffcc3936SMike Snitzer if (r)
2879ffcc3936SMike Snitzer goto out;
2880ffcc3936SMike Snitzer
28812ca3310eSAlasdair G Kergon clear_bit(DMF_SUSPENDED, &md->flags);
2882cf222b37SAlasdair G Kergon out:
2883e61290a4SDaniel Walker mutex_unlock(&md->suspend_lock);
28842ca3310eSAlasdair G Kergon
2885cf222b37SAlasdair G Kergon return r;
28861da177e4SLinus Torvalds }
28871da177e4SLinus Torvalds
2888fd2ed4d2SMikulas Patocka /*
2889fd2ed4d2SMikulas Patocka * Internal suspend/resume works like userspace-driven suspend. It waits
2890fd2ed4d2SMikulas Patocka * until all bios finish and prevents issuing new bios to the target drivers.
2891fd2ed4d2SMikulas Patocka * It may be used only from the kernel.
2892fd2ed4d2SMikulas Patocka */
2893fd2ed4d2SMikulas Patocka
__dm_internal_suspend(struct mapped_device * md,unsigned int suspend_flags)289486a3238cSHeinz Mauelshagen static void __dm_internal_suspend(struct mapped_device *md, unsigned int suspend_flags)
2895ffcc3936SMike Snitzer {
2896ffcc3936SMike Snitzer struct dm_table *map = NULL;
2897ffcc3936SMike Snitzer
28981ea0654eSBart Van Assche lockdep_assert_held(&md->suspend_lock);
28991ea0654eSBart Van Assche
290096b26c8cSMikulas Patocka if (md->internal_suspend_count++)
2901ffcc3936SMike Snitzer return; /* nested internal suspend */
2902ffcc3936SMike Snitzer
2903ffcc3936SMike Snitzer if (dm_suspended_md(md)) {
2904ffcc3936SMike Snitzer set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2905ffcc3936SMike Snitzer return; /* nest suspend */
2906ffcc3936SMike Snitzer }
2907ffcc3936SMike Snitzer
2908a12f5d48SEric Dumazet map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2909ffcc3936SMike Snitzer
2910ffcc3936SMike Snitzer /*
2911ffcc3936SMike Snitzer * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is
2912ffcc3936SMike Snitzer * supported. Properly supporting a TASK_INTERRUPTIBLE internal suspend
2913ffcc3936SMike Snitzer * would require changing .presuspend to return an error -- avoid this
2914ffcc3936SMike Snitzer * until there is a need for more elaborate variants of internal suspend.
2915ffcc3936SMike Snitzer */
2916eaf9a736SMike Snitzer (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE,
2917eaf9a736SMike Snitzer DMF_SUSPENDED_INTERNALLY);
2918ffcc3936SMike Snitzer
29195df96f2bSMikulas Patocka set_bit(DMF_POST_SUSPENDING, &md->flags);
2920ffcc3936SMike Snitzer dm_table_postsuspend_targets(map);
29215df96f2bSMikulas Patocka clear_bit(DMF_POST_SUSPENDING, &md->flags);
2922ffcc3936SMike Snitzer }
2923ffcc3936SMike Snitzer
__dm_internal_resume(struct mapped_device * md)2924ffcc3936SMike Snitzer static void __dm_internal_resume(struct mapped_device *md)
2925ffcc3936SMike Snitzer {
292615a3fc5cSMikulas Patocka int r;
292715a3fc5cSMikulas Patocka struct dm_table *map;
292815a3fc5cSMikulas Patocka
292996b26c8cSMikulas Patocka BUG_ON(!md->internal_suspend_count);
293096b26c8cSMikulas Patocka
293196b26c8cSMikulas Patocka if (--md->internal_suspend_count)
2932ffcc3936SMike Snitzer return; /* resume from nested internal suspend */
2933ffcc3936SMike Snitzer
2934ffcc3936SMike Snitzer if (dm_suspended_md(md))
2935ffcc3936SMike Snitzer goto done; /* resume from nested suspend */
2936ffcc3936SMike Snitzer
293715a3fc5cSMikulas Patocka map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
293815a3fc5cSMikulas Patocka r = __dm_resume(md, map);
293915a3fc5cSMikulas Patocka if (r) {
2940ffcc3936SMike Snitzer /*
294115a3fc5cSMikulas Patocka * If a preresume method of some target failed, we are in a
294215a3fc5cSMikulas Patocka * tricky situation. We can't return an error to the caller. We
294315a3fc5cSMikulas Patocka * can't fake success because then the "resume" and
294415a3fc5cSMikulas Patocka * "postsuspend" methods would not be paired correctly, and it
294515a3fc5cSMikulas Patocka * would break various targets, for example it would cause list
294615a3fc5cSMikulas Patocka * corruption in the "origin" target.
294715a3fc5cSMikulas Patocka *
294815a3fc5cSMikulas Patocka * So, we fake normal suspend here, to make sure that the
294915a3fc5cSMikulas Patocka * "resume" and "postsuspend" methods will be paired correctly.
2950ffcc3936SMike Snitzer */
295115a3fc5cSMikulas Patocka DMERR("Preresume method failed: %d", r);
295215a3fc5cSMikulas Patocka set_bit(DMF_SUSPENDED, &md->flags);
295315a3fc5cSMikulas Patocka }
2954ffcc3936SMike Snitzer done:
2955ffcc3936SMike Snitzer clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2956ffcc3936SMike Snitzer smp_mb__after_atomic();
2957ffcc3936SMike Snitzer wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY);
2958ffcc3936SMike Snitzer }
2959ffcc3936SMike Snitzer
dm_internal_suspend_noflush(struct mapped_device * md)2960ffcc3936SMike Snitzer void dm_internal_suspend_noflush(struct mapped_device *md)
2961fd2ed4d2SMikulas Patocka {
2962fd2ed4d2SMikulas Patocka mutex_lock(&md->suspend_lock);
2963ffcc3936SMike Snitzer __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG);
2964ffcc3936SMike Snitzer mutex_unlock(&md->suspend_lock);
2965ffcc3936SMike Snitzer }
2966ffcc3936SMike Snitzer EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush);
2967ffcc3936SMike Snitzer
dm_internal_resume(struct mapped_device * md)2968ffcc3936SMike Snitzer void dm_internal_resume(struct mapped_device *md)
2969ffcc3936SMike Snitzer {
2970ffcc3936SMike Snitzer mutex_lock(&md->suspend_lock);
2971ffcc3936SMike Snitzer __dm_internal_resume(md);
2972ffcc3936SMike Snitzer mutex_unlock(&md->suspend_lock);
2973ffcc3936SMike Snitzer }
2974ffcc3936SMike Snitzer EXPORT_SYMBOL_GPL(dm_internal_resume);
2975ffcc3936SMike Snitzer
2976ffcc3936SMike Snitzer /*
2977ffcc3936SMike Snitzer * Fast variants of internal suspend/resume hold md->suspend_lock,
2978ffcc3936SMike Snitzer * which prevents interaction with userspace-driven suspend.
2979ffcc3936SMike Snitzer */
2980ffcc3936SMike Snitzer
dm_internal_suspend_fast(struct mapped_device * md)2981ffcc3936SMike Snitzer void dm_internal_suspend_fast(struct mapped_device *md)
2982ffcc3936SMike Snitzer {
2983ffcc3936SMike Snitzer mutex_lock(&md->suspend_lock);
2984ffcc3936SMike Snitzer if (dm_suspended_md(md) || dm_suspended_internally_md(md))
2985fd2ed4d2SMikulas Patocka return;
2986fd2ed4d2SMikulas Patocka
2987fd2ed4d2SMikulas Patocka set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2988fd2ed4d2SMikulas Patocka synchronize_srcu(&md->io_barrier);
2989fd2ed4d2SMikulas Patocka flush_workqueue(md->wq);
2990fd2ed4d2SMikulas Patocka dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
2991fd2ed4d2SMikulas Patocka }
2992b735fedeSMikulas Patocka EXPORT_SYMBOL_GPL(dm_internal_suspend_fast);
2993fd2ed4d2SMikulas Patocka
dm_internal_resume_fast(struct mapped_device * md)2994ffcc3936SMike Snitzer void dm_internal_resume_fast(struct mapped_device *md)
2995fd2ed4d2SMikulas Patocka {
2996ffcc3936SMike Snitzer if (dm_suspended_md(md) || dm_suspended_internally_md(md))
2997fd2ed4d2SMikulas Patocka goto done;
2998fd2ed4d2SMikulas Patocka
2999fd2ed4d2SMikulas Patocka dm_queue_flush(md);
3000fd2ed4d2SMikulas Patocka
3001fd2ed4d2SMikulas Patocka done:
3002fd2ed4d2SMikulas Patocka mutex_unlock(&md->suspend_lock);
3003fd2ed4d2SMikulas Patocka }
3004b735fedeSMikulas Patocka EXPORT_SYMBOL_GPL(dm_internal_resume_fast);
3005fd2ed4d2SMikulas Patocka
3006a4a82ce3SHeinz Mauelshagen /*
3007a4a82ce3SHeinz Mauelshagen *---------------------------------------------------------------
30081da177e4SLinus Torvalds * Event notification.
3009a4a82ce3SHeinz Mauelshagen *---------------------------------------------------------------
3010a4a82ce3SHeinz Mauelshagen */
dm_kobject_uevent(struct mapped_device * md,enum kobject_action action,unsigned int cookie,bool need_resize_uevent)30113abf85b5SPeter Rajnoha int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action,
301286a3238cSHeinz Mauelshagen unsigned int cookie, bool need_resize_uevent)
301369267a30SAlasdair G Kergon {
30146958c1c6SMikulas Patocka int r;
301586a3238cSHeinz Mauelshagen unsigned int noio_flag;
301660935eb2SMilan Broz char udev_cookie[DM_COOKIE_LENGTH];
30177533afa1SMikulas Patocka char *envp[3] = { NULL, NULL, NULL };
30187533afa1SMikulas Patocka char **envpp = envp;
30197533afa1SMikulas Patocka if (cookie) {
30207533afa1SMikulas Patocka snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u",
30217533afa1SMikulas Patocka DM_COOKIE_ENV_VAR_NAME, cookie);
30227533afa1SMikulas Patocka *envpp++ = udev_cookie;
30237533afa1SMikulas Patocka }
30247533afa1SMikulas Patocka if (need_resize_uevent) {
30257533afa1SMikulas Patocka *envpp++ = "RESIZE=1";
30267533afa1SMikulas Patocka }
302760935eb2SMilan Broz
30286958c1c6SMikulas Patocka noio_flag = memalloc_noio_save();
30296958c1c6SMikulas Patocka
30307533afa1SMikulas Patocka r = kobject_uevent_env(&disk_to_dev(md->disk)->kobj, action, envp);
30316958c1c6SMikulas Patocka
30326958c1c6SMikulas Patocka memalloc_noio_restore(noio_flag);
30336958c1c6SMikulas Patocka
30346958c1c6SMikulas Patocka return r;
303569267a30SAlasdair G Kergon }
303669267a30SAlasdair G Kergon
dm_next_uevent_seq(struct mapped_device * md)30377a8c3d3bSMike Anderson uint32_t dm_next_uevent_seq(struct mapped_device *md)
30387a8c3d3bSMike Anderson {
30397a8c3d3bSMike Anderson return atomic_add_return(1, &md->uevent_seq);
30407a8c3d3bSMike Anderson }
30417a8c3d3bSMike Anderson
dm_get_event_nr(struct mapped_device * md)30421da177e4SLinus Torvalds uint32_t dm_get_event_nr(struct mapped_device *md)
30431da177e4SLinus Torvalds {
30441da177e4SLinus Torvalds return atomic_read(&md->event_nr);
30451da177e4SLinus Torvalds }
30461da177e4SLinus Torvalds
dm_wait_event(struct mapped_device * md,int event_nr)30471da177e4SLinus Torvalds int dm_wait_event(struct mapped_device *md, int event_nr)
30481da177e4SLinus Torvalds {
30491da177e4SLinus Torvalds return wait_event_interruptible(md->eventq,
30501da177e4SLinus Torvalds (event_nr != atomic_read(&md->event_nr)));
30511da177e4SLinus Torvalds }
30521da177e4SLinus Torvalds
dm_uevent_add(struct mapped_device * md,struct list_head * elist)30537a8c3d3bSMike Anderson void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
30547a8c3d3bSMike Anderson {
30557a8c3d3bSMike Anderson unsigned long flags;
30567a8c3d3bSMike Anderson
30577a8c3d3bSMike Anderson spin_lock_irqsave(&md->uevent_lock, flags);
30587a8c3d3bSMike Anderson list_add(elist, &md->uevent_list);
30597a8c3d3bSMike Anderson spin_unlock_irqrestore(&md->uevent_lock, flags);
30607a8c3d3bSMike Anderson }
30617a8c3d3bSMike Anderson
30621da177e4SLinus Torvalds /*
30631da177e4SLinus Torvalds * The gendisk is only valid as long as you have a reference
30641da177e4SLinus Torvalds * count on 'md'.
30651da177e4SLinus Torvalds */
dm_disk(struct mapped_device * md)30661da177e4SLinus Torvalds struct gendisk *dm_disk(struct mapped_device *md)
30671da177e4SLinus Torvalds {
30681da177e4SLinus Torvalds return md->disk;
30691da177e4SLinus Torvalds }
307065ff5b7dSSami Tolvanen EXPORT_SYMBOL_GPL(dm_disk);
30711da177e4SLinus Torvalds
dm_kobject(struct mapped_device * md)3072784aae73SMilan Broz struct kobject *dm_kobject(struct mapped_device *md)
3073784aae73SMilan Broz {
30742995fa78SMikulas Patocka return &md->kobj_holder.kobj;
3075784aae73SMilan Broz }
3076784aae73SMilan Broz
dm_get_from_kobject(struct kobject * kobj)3077784aae73SMilan Broz struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
3078784aae73SMilan Broz {
3079784aae73SMilan Broz struct mapped_device *md;
3080784aae73SMilan Broz
30812995fa78SMikulas Patocka md = container_of(kobj, struct mapped_device, kobj_holder.kobj);
3082784aae73SMilan Broz
3083b9a41d21SHou Tao spin_lock(&_minor_lock);
3084b9a41d21SHou Tao if (test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
3085b9a41d21SHou Tao md = NULL;
3086b9a41d21SHou Tao goto out;
3087b9a41d21SHou Tao }
3088784aae73SMilan Broz dm_get(md);
3089b9a41d21SHou Tao out:
3090b9a41d21SHou Tao spin_unlock(&_minor_lock);
3091b9a41d21SHou Tao
3092784aae73SMilan Broz return md;
3093784aae73SMilan Broz }
3094784aae73SMilan Broz
dm_suspended_md(struct mapped_device * md)30954f186f8bSKiyoshi Ueda int dm_suspended_md(struct mapped_device *md)
30961da177e4SLinus Torvalds {
30971da177e4SLinus Torvalds return test_bit(DMF_SUSPENDED, &md->flags);
30981da177e4SLinus Torvalds }
30991da177e4SLinus Torvalds
dm_post_suspending_md(struct mapped_device * md)31005df96f2bSMikulas Patocka static int dm_post_suspending_md(struct mapped_device *md)
31015df96f2bSMikulas Patocka {
31025df96f2bSMikulas Patocka return test_bit(DMF_POST_SUSPENDING, &md->flags);
31035df96f2bSMikulas Patocka }
31045df96f2bSMikulas Patocka
dm_suspended_internally_md(struct mapped_device * md)3105ffcc3936SMike Snitzer int dm_suspended_internally_md(struct mapped_device *md)
3106ffcc3936SMike Snitzer {
3107ffcc3936SMike Snitzer return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
3108ffcc3936SMike Snitzer }
3109ffcc3936SMike Snitzer
dm_test_deferred_remove_flag(struct mapped_device * md)31102c140a24SMikulas Patocka int dm_test_deferred_remove_flag(struct mapped_device *md)
31112c140a24SMikulas Patocka {
31122c140a24SMikulas Patocka return test_bit(DMF_DEFERRED_REMOVE, &md->flags);
31132c140a24SMikulas Patocka }
31142c140a24SMikulas Patocka
dm_suspended(struct dm_target * ti)311564dbce58SKiyoshi Ueda int dm_suspended(struct dm_target *ti)
311664dbce58SKiyoshi Ueda {
311733bd6f06SMike Snitzer return dm_suspended_md(ti->table->md);
311864dbce58SKiyoshi Ueda }
311964dbce58SKiyoshi Ueda EXPORT_SYMBOL_GPL(dm_suspended);
312064dbce58SKiyoshi Ueda
dm_post_suspending(struct dm_target * ti)31215df96f2bSMikulas Patocka int dm_post_suspending(struct dm_target *ti)
31225df96f2bSMikulas Patocka {
312333bd6f06SMike Snitzer return dm_post_suspending_md(ti->table->md);
31245df96f2bSMikulas Patocka }
31255df96f2bSMikulas Patocka EXPORT_SYMBOL_GPL(dm_post_suspending);
31265df96f2bSMikulas Patocka
dm_noflush_suspending(struct dm_target * ti)31272e93ccc1SKiyoshi Ueda int dm_noflush_suspending(struct dm_target *ti)
31282e93ccc1SKiyoshi Ueda {
312933bd6f06SMike Snitzer return __noflush_suspending(ti->table->md);
31302e93ccc1SKiyoshi Ueda }
31312e93ccc1SKiyoshi Ueda EXPORT_SYMBOL_GPL(dm_noflush_suspending);
31322e93ccc1SKiyoshi Ueda
dm_free_md_mempools(struct dm_md_mempools * pools)3133e6ee8c0bSKiyoshi Ueda void dm_free_md_mempools(struct dm_md_mempools *pools)
3134e6ee8c0bSKiyoshi Ueda {
3135e6ee8c0bSKiyoshi Ueda if (!pools)
3136e6ee8c0bSKiyoshi Ueda return;
3137e6ee8c0bSKiyoshi Ueda
31386f1c819cSKent Overstreet bioset_exit(&pools->bs);
31396f1c819cSKent Overstreet bioset_exit(&pools->io_bs);
3140e6ee8c0bSKiyoshi Ueda
3141e6ee8c0bSKiyoshi Ueda kfree(pools);
3142e6ee8c0bSKiyoshi Ueda }
3143e6ee8c0bSKiyoshi Ueda
31449c72bad1SChristoph Hellwig struct dm_pr {
31459c72bad1SChristoph Hellwig u64 old_key;
31469c72bad1SChristoph Hellwig u64 new_key;
31479c72bad1SChristoph Hellwig u32 flags;
3148c6adada5SMike Christie bool abort;
31499c72bad1SChristoph Hellwig bool fail_early;
31508dd87f3cSMike Christie int ret;
315170151087SMike Christie enum pr_type type;
31528a8da082SMike Christie struct pr_keys *read_keys;
31538a8da082SMike Christie struct pr_held_reservation *rsv;
31549c72bad1SChristoph Hellwig };
31559c72bad1SChristoph Hellwig
dm_call_pr(struct block_device * bdev,iterate_devices_callout_fn fn,struct dm_pr * pr)31569c72bad1SChristoph Hellwig static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn,
31578dd87f3cSMike Christie struct dm_pr *pr)
31589c72bad1SChristoph Hellwig {
31599c72bad1SChristoph Hellwig struct mapped_device *md = bdev->bd_disk->private_data;
31609c72bad1SChristoph Hellwig struct dm_table *table;
31619c72bad1SChristoph Hellwig struct dm_target *ti;
31629c72bad1SChristoph Hellwig int ret = -ENOTTY, srcu_idx;
31639c72bad1SChristoph Hellwig
31649c72bad1SChristoph Hellwig table = dm_get_live_table(md, &srcu_idx);
31659c72bad1SChristoph Hellwig if (!table || !dm_table_get_size(table))
31669c72bad1SChristoph Hellwig goto out;
31679c72bad1SChristoph Hellwig
31689c72bad1SChristoph Hellwig /* We only support devices that have a single target */
31692aec377aSMike Snitzer if (table->num_targets != 1)
31709c72bad1SChristoph Hellwig goto out;
31719c72bad1SChristoph Hellwig ti = dm_table_get_target(table, 0);
31729c72bad1SChristoph Hellwig
3173e120a5f1SMike Snitzer if (dm_suspended_md(md)) {
3174e120a5f1SMike Snitzer ret = -EAGAIN;
3175e120a5f1SMike Snitzer goto out;
3176e120a5f1SMike Snitzer }
3177e120a5f1SMike Snitzer
31789c72bad1SChristoph Hellwig ret = -EINVAL;
31799c72bad1SChristoph Hellwig if (!ti->type->iterate_devices)
31809c72bad1SChristoph Hellwig goto out;
31819c72bad1SChristoph Hellwig
31828dd87f3cSMike Christie ti->type->iterate_devices(ti, fn, pr);
31838dd87f3cSMike Christie ret = 0;
31849c72bad1SChristoph Hellwig out:
31859c72bad1SChristoph Hellwig dm_put_live_table(md, srcu_idx);
31869c72bad1SChristoph Hellwig return ret;
31879c72bad1SChristoph Hellwig }
31889c72bad1SChristoph Hellwig
31899c72bad1SChristoph Hellwig /*
31909c72bad1SChristoph Hellwig * For register / unregister we need to manually call out to every path.
31919c72bad1SChristoph Hellwig */
__dm_pr_register(struct dm_target * ti,struct dm_dev * dev,sector_t start,sector_t len,void * data)31929c72bad1SChristoph Hellwig static int __dm_pr_register(struct dm_target *ti, struct dm_dev *dev,
31939c72bad1SChristoph Hellwig sector_t start, sector_t len, void *data)
31949c72bad1SChristoph Hellwig {
31959c72bad1SChristoph Hellwig struct dm_pr *pr = data;
31969c72bad1SChristoph Hellwig const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
31978dd87f3cSMike Christie int ret;
31989c72bad1SChristoph Hellwig
31998dd87f3cSMike Christie if (!ops || !ops->pr_register) {
32008dd87f3cSMike Christie pr->ret = -EOPNOTSUPP;
32018dd87f3cSMike Christie return -1;
32028dd87f3cSMike Christie }
32038dd87f3cSMike Christie
32048dd87f3cSMike Christie ret = ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags);
32058dd87f3cSMike Christie if (!ret)
32068dd87f3cSMike Christie return 0;
32078dd87f3cSMike Christie
32088dd87f3cSMike Christie if (!pr->ret)
32098dd87f3cSMike Christie pr->ret = ret;
32108dd87f3cSMike Christie
32118dd87f3cSMike Christie if (pr->fail_early)
32128dd87f3cSMike Christie return -1;
32138dd87f3cSMike Christie
32148dd87f3cSMike Christie return 0;
32159c72bad1SChristoph Hellwig }
32169c72bad1SChristoph Hellwig
dm_pr_register(struct block_device * bdev,u64 old_key,u64 new_key,u32 flags)321771cdb697SChristoph Hellwig static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key,
321871cdb697SChristoph Hellwig u32 flags)
321971cdb697SChristoph Hellwig {
32209c72bad1SChristoph Hellwig struct dm_pr pr = {
32219c72bad1SChristoph Hellwig .old_key = old_key,
32229c72bad1SChristoph Hellwig .new_key = new_key,
32239c72bad1SChristoph Hellwig .flags = flags,
32249c72bad1SChristoph Hellwig .fail_early = true,
32258dd87f3cSMike Christie .ret = 0,
32269c72bad1SChristoph Hellwig };
32279c72bad1SChristoph Hellwig int ret;
322871cdb697SChristoph Hellwig
32299c72bad1SChristoph Hellwig ret = dm_call_pr(bdev, __dm_pr_register, &pr);
32308dd87f3cSMike Christie if (ret) {
32318dd87f3cSMike Christie /* Didn't even get to register a path */
32328dd87f3cSMike Christie return ret;
32338dd87f3cSMike Christie }
32348dd87f3cSMike Christie
32358dd87f3cSMike Christie if (!pr.ret)
32368dd87f3cSMike Christie return 0;
32378dd87f3cSMike Christie ret = pr.ret;
32388dd87f3cSMike Christie
32398dd87f3cSMike Christie if (!new_key)
32408dd87f3cSMike Christie return ret;
32418dd87f3cSMike Christie
32429c72bad1SChristoph Hellwig /* unregister all paths if we failed to register any path */
32439c72bad1SChristoph Hellwig pr.old_key = new_key;
32449c72bad1SChristoph Hellwig pr.new_key = 0;
32459c72bad1SChristoph Hellwig pr.flags = 0;
32469c72bad1SChristoph Hellwig pr.fail_early = false;
32478dd87f3cSMike Christie (void) dm_call_pr(bdev, __dm_pr_register, &pr);
32489c72bad1SChristoph Hellwig return ret;
324971cdb697SChristoph Hellwig }
3250956a4025SMike Snitzer
325170151087SMike Christie
__dm_pr_reserve(struct dm_target * ti,struct dm_dev * dev,sector_t start,sector_t len,void * data)325270151087SMike Christie static int __dm_pr_reserve(struct dm_target *ti, struct dm_dev *dev,
325370151087SMike Christie sector_t start, sector_t len, void *data)
325470151087SMike Christie {
325570151087SMike Christie struct dm_pr *pr = data;
325670151087SMike Christie const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
325770151087SMike Christie
325870151087SMike Christie if (!ops || !ops->pr_reserve) {
325970151087SMike Christie pr->ret = -EOPNOTSUPP;
326070151087SMike Christie return -1;
326170151087SMike Christie }
326270151087SMike Christie
326370151087SMike Christie pr->ret = ops->pr_reserve(dev->bdev, pr->old_key, pr->type, pr->flags);
326470151087SMike Christie if (!pr->ret)
326570151087SMike Christie return -1;
326670151087SMike Christie
326770151087SMike Christie return 0;
326871cdb697SChristoph Hellwig }
326971cdb697SChristoph Hellwig
dm_pr_reserve(struct block_device * bdev,u64 key,enum pr_type type,u32 flags)327071cdb697SChristoph Hellwig static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type,
327171cdb697SChristoph Hellwig u32 flags)
327271cdb697SChristoph Hellwig {
327370151087SMike Christie struct dm_pr pr = {
327470151087SMike Christie .old_key = key,
327570151087SMike Christie .flags = flags,
327670151087SMike Christie .type = type,
327770151087SMike Christie .fail_early = false,
327870151087SMike Christie .ret = 0,
327970151087SMike Christie };
328070151087SMike Christie int ret;
328171cdb697SChristoph Hellwig
328270151087SMike Christie ret = dm_call_pr(bdev, __dm_pr_reserve, &pr);
328370151087SMike Christie if (ret)
328470151087SMike Christie return ret;
328571cdb697SChristoph Hellwig
328670151087SMike Christie return pr.ret;
328771cdb697SChristoph Hellwig }
328871cdb697SChristoph Hellwig
328908a3c338SMike Christie /*
329008a3c338SMike Christie * If there is a non-All Registrants type of reservation, the release must be
329108a3c338SMike Christie * sent down the holding path. For the cases where there is no reservation or
329208a3c338SMike Christie * the path is not the holder the device will also return success, so we must
329308a3c338SMike Christie * try each path to make sure we got the correct path.
329408a3c338SMike Christie */
__dm_pr_release(struct dm_target * ti,struct dm_dev * dev,sector_t start,sector_t len,void * data)329508a3c338SMike Christie static int __dm_pr_release(struct dm_target *ti, struct dm_dev *dev,
329608a3c338SMike Christie sector_t start, sector_t len, void *data)
329708a3c338SMike Christie {
329808a3c338SMike Christie struct dm_pr *pr = data;
329908a3c338SMike Christie const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
330008a3c338SMike Christie
330108a3c338SMike Christie if (!ops || !ops->pr_release) {
330208a3c338SMike Christie pr->ret = -EOPNOTSUPP;
330308a3c338SMike Christie return -1;
330408a3c338SMike Christie }
330508a3c338SMike Christie
330608a3c338SMike Christie pr->ret = ops->pr_release(dev->bdev, pr->old_key, pr->type);
330708a3c338SMike Christie if (pr->ret)
330808a3c338SMike Christie return -1;
330908a3c338SMike Christie
331008a3c338SMike Christie return 0;
331171cdb697SChristoph Hellwig }
331271cdb697SChristoph Hellwig
dm_pr_release(struct block_device * bdev,u64 key,enum pr_type type)331371cdb697SChristoph Hellwig static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
331471cdb697SChristoph Hellwig {
331508a3c338SMike Christie struct dm_pr pr = {
331608a3c338SMike Christie .old_key = key,
331708a3c338SMike Christie .type = type,
331808a3c338SMike Christie .fail_early = false,
331908a3c338SMike Christie };
332008a3c338SMike Christie int ret;
332171cdb697SChristoph Hellwig
332208a3c338SMike Christie ret = dm_call_pr(bdev, __dm_pr_release, &pr);
332308a3c338SMike Christie if (ret)
332408a3c338SMike Christie return ret;
332571cdb697SChristoph Hellwig
332608a3c338SMike Christie return pr.ret;
332771cdb697SChristoph Hellwig }
332871cdb697SChristoph Hellwig
__dm_pr_preempt(struct dm_target * ti,struct dm_dev * dev,sector_t start,sector_t len,void * data)3329c6adada5SMike Christie static int __dm_pr_preempt(struct dm_target *ti, struct dm_dev *dev,
3330c6adada5SMike Christie sector_t start, sector_t len, void *data)
3331c6adada5SMike Christie {
3332c6adada5SMike Christie struct dm_pr *pr = data;
3333c6adada5SMike Christie const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
3334c6adada5SMike Christie
3335c6adada5SMike Christie if (!ops || !ops->pr_preempt) {
3336c6adada5SMike Christie pr->ret = -EOPNOTSUPP;
3337c6adada5SMike Christie return -1;
3338c6adada5SMike Christie }
3339c6adada5SMike Christie
3340c6adada5SMike Christie pr->ret = ops->pr_preempt(dev->bdev, pr->old_key, pr->new_key, pr->type,
3341c6adada5SMike Christie pr->abort);
3342c6adada5SMike Christie if (!pr->ret)
3343c6adada5SMike Christie return -1;
3344c6adada5SMike Christie
3345c6adada5SMike Christie return 0;
334671cdb697SChristoph Hellwig }
334771cdb697SChristoph Hellwig
dm_pr_preempt(struct block_device * bdev,u64 old_key,u64 new_key,enum pr_type type,bool abort)334871cdb697SChristoph Hellwig static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key,
334971cdb697SChristoph Hellwig enum pr_type type, bool abort)
335071cdb697SChristoph Hellwig {
3351c6adada5SMike Christie struct dm_pr pr = {
3352c6adada5SMike Christie .new_key = new_key,
3353c6adada5SMike Christie .old_key = old_key,
3354c6adada5SMike Christie .type = type,
3355c6adada5SMike Christie .fail_early = false,
3356c6adada5SMike Christie };
3357c6adada5SMike Christie int ret;
335871cdb697SChristoph Hellwig
3359c6adada5SMike Christie ret = dm_call_pr(bdev, __dm_pr_preempt, &pr);
3360c6adada5SMike Christie if (ret)
3361c6adada5SMike Christie return ret;
336271cdb697SChristoph Hellwig
3363c6adada5SMike Christie return pr.ret;
336471cdb697SChristoph Hellwig }
336571cdb697SChristoph Hellwig
dm_pr_clear(struct block_device * bdev,u64 key)336671cdb697SChristoph Hellwig static int dm_pr_clear(struct block_device *bdev, u64 key)
336771cdb697SChristoph Hellwig {
336871cdb697SChristoph Hellwig struct mapped_device *md = bdev->bd_disk->private_data;
336971cdb697SChristoph Hellwig const struct pr_ops *ops;
3370971888c4SMike Snitzer int r, srcu_idx;
337171cdb697SChristoph Hellwig
33725bd5e8d8SMike Snitzer r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
337371cdb697SChristoph Hellwig if (r < 0)
3374971888c4SMike Snitzer goto out;
337571cdb697SChristoph Hellwig
337671cdb697SChristoph Hellwig ops = bdev->bd_disk->fops->pr_ops;
337771cdb697SChristoph Hellwig if (ops && ops->pr_clear)
337871cdb697SChristoph Hellwig r = ops->pr_clear(bdev, key);
337971cdb697SChristoph Hellwig else
338071cdb697SChristoph Hellwig r = -EOPNOTSUPP;
3381971888c4SMike Snitzer out:
3382971888c4SMike Snitzer dm_unprepare_ioctl(md, srcu_idx);
338371cdb697SChristoph Hellwig return r;
338471cdb697SChristoph Hellwig }
338571cdb697SChristoph Hellwig
__dm_pr_read_keys(struct dm_target * ti,struct dm_dev * dev,sector_t start,sector_t len,void * data)33868a8da082SMike Christie static int __dm_pr_read_keys(struct dm_target *ti, struct dm_dev *dev,
33878a8da082SMike Christie sector_t start, sector_t len, void *data)
33888a8da082SMike Christie {
33898a8da082SMike Christie struct dm_pr *pr = data;
33908a8da082SMike Christie const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
33918a8da082SMike Christie
33928a8da082SMike Christie if (!ops || !ops->pr_read_keys) {
33938a8da082SMike Christie pr->ret = -EOPNOTSUPP;
33948a8da082SMike Christie return -1;
33958a8da082SMike Christie }
33968a8da082SMike Christie
33978a8da082SMike Christie pr->ret = ops->pr_read_keys(dev->bdev, pr->read_keys);
33988a8da082SMike Christie if (!pr->ret)
33998a8da082SMike Christie return -1;
34008a8da082SMike Christie
34018a8da082SMike Christie return 0;
34028a8da082SMike Christie }
34038a8da082SMike Christie
dm_pr_read_keys(struct block_device * bdev,struct pr_keys * keys)34048a8da082SMike Christie static int dm_pr_read_keys(struct block_device *bdev, struct pr_keys *keys)
34058a8da082SMike Christie {
34068a8da082SMike Christie struct dm_pr pr = {
34078a8da082SMike Christie .read_keys = keys,
34088a8da082SMike Christie };
34098a8da082SMike Christie int ret;
34108a8da082SMike Christie
34118a8da082SMike Christie ret = dm_call_pr(bdev, __dm_pr_read_keys, &pr);
34128a8da082SMike Christie if (ret)
34138a8da082SMike Christie return ret;
34148a8da082SMike Christie
34158a8da082SMike Christie return pr.ret;
34168a8da082SMike Christie }
34178a8da082SMike Christie
__dm_pr_read_reservation(struct dm_target * ti,struct dm_dev * dev,sector_t start,sector_t len,void * data)34188a8da082SMike Christie static int __dm_pr_read_reservation(struct dm_target *ti, struct dm_dev *dev,
34198a8da082SMike Christie sector_t start, sector_t len, void *data)
34208a8da082SMike Christie {
34218a8da082SMike Christie struct dm_pr *pr = data;
34228a8da082SMike Christie const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
34238a8da082SMike Christie
34248a8da082SMike Christie if (!ops || !ops->pr_read_reservation) {
34258a8da082SMike Christie pr->ret = -EOPNOTSUPP;
34268a8da082SMike Christie return -1;
34278a8da082SMike Christie }
34288a8da082SMike Christie
34298a8da082SMike Christie pr->ret = ops->pr_read_reservation(dev->bdev, pr->rsv);
34308a8da082SMike Christie if (!pr->ret)
34318a8da082SMike Christie return -1;
34328a8da082SMike Christie
34338a8da082SMike Christie return 0;
34348a8da082SMike Christie }
34358a8da082SMike Christie
dm_pr_read_reservation(struct block_device * bdev,struct pr_held_reservation * rsv)34368a8da082SMike Christie static int dm_pr_read_reservation(struct block_device *bdev,
34378a8da082SMike Christie struct pr_held_reservation *rsv)
34388a8da082SMike Christie {
34398a8da082SMike Christie struct dm_pr pr = {
34408a8da082SMike Christie .rsv = rsv,
34418a8da082SMike Christie };
34428a8da082SMike Christie int ret;
34438a8da082SMike Christie
34448a8da082SMike Christie ret = dm_call_pr(bdev, __dm_pr_read_reservation, &pr);
34458a8da082SMike Christie if (ret)
34468a8da082SMike Christie return ret;
34478a8da082SMike Christie
34488a8da082SMike Christie return pr.ret;
34498a8da082SMike Christie }
34508a8da082SMike Christie
345171cdb697SChristoph Hellwig static const struct pr_ops dm_pr_ops = {
345271cdb697SChristoph Hellwig .pr_register = dm_pr_register,
345371cdb697SChristoph Hellwig .pr_reserve = dm_pr_reserve,
345471cdb697SChristoph Hellwig .pr_release = dm_pr_release,
345571cdb697SChristoph Hellwig .pr_preempt = dm_pr_preempt,
345671cdb697SChristoph Hellwig .pr_clear = dm_pr_clear,
34578a8da082SMike Christie .pr_read_keys = dm_pr_read_keys,
34588a8da082SMike Christie .pr_read_reservation = dm_pr_read_reservation,
345971cdb697SChristoph Hellwig };
346071cdb697SChristoph Hellwig
346183d5cde4SAlexey Dobriyan static const struct block_device_operations dm_blk_dops = {
3462c62b37d9SChristoph Hellwig .submit_bio = dm_submit_bio,
3463b99fdcdcSMing Lei .poll_bio = dm_poll_bio,
34641da177e4SLinus Torvalds .open = dm_blk_open,
34651da177e4SLinus Torvalds .release = dm_blk_close,
3466aa129a22SMilan Broz .ioctl = dm_blk_ioctl,
34673ac51e74SDarrick J. Wong .getgeo = dm_blk_getgeo,
3468e76239a3SChristoph Hellwig .report_zones = dm_blk_report_zones,
346971cdb697SChristoph Hellwig .pr_ops = &dm_pr_ops,
34701da177e4SLinus Torvalds .owner = THIS_MODULE
34711da177e4SLinus Torvalds };
34721da177e4SLinus Torvalds
3473681cc5e8SMike Snitzer static const struct block_device_operations dm_rq_blk_dops = {
3474681cc5e8SMike Snitzer .open = dm_blk_open,
3475681cc5e8SMike Snitzer .release = dm_blk_close,
3476681cc5e8SMike Snitzer .ioctl = dm_blk_ioctl,
3477681cc5e8SMike Snitzer .getgeo = dm_blk_getgeo,
3478681cc5e8SMike Snitzer .pr_ops = &dm_pr_ops,
3479681cc5e8SMike Snitzer .owner = THIS_MODULE
3480681cc5e8SMike Snitzer };
3481681cc5e8SMike Snitzer
3482f26c5719SDan Williams static const struct dax_operations dm_dax_ops = {
3483f26c5719SDan Williams .direct_access = dm_dax_direct_access,
3484cdf6cdcdSVivek Goyal .zero_page_range = dm_dax_zero_page_range,
3485047218ecSJane Chu .recovery_write = dm_dax_recovery_write,
3486f26c5719SDan Williams };
3487f26c5719SDan Williams
34881da177e4SLinus Torvalds /*
34891da177e4SLinus Torvalds * module hooks
34901da177e4SLinus Torvalds */
34911da177e4SLinus Torvalds module_init(dm_init);
34921da177e4SLinus Torvalds module_exit(dm_exit);
34931da177e4SLinus Torvalds
34941da177e4SLinus Torvalds module_param(major, uint, 0);
34951da177e4SLinus Torvalds MODULE_PARM_DESC(major, "The major number of the device mapper");
3496f4790826SMike Snitzer
34976a808034SHeinz Mauelshagen module_param(reserved_bio_based_ios, uint, 0644);
3498e8603136SMike Snitzer MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools");
3499e8603136SMike Snitzer
35006a808034SHeinz Mauelshagen module_param(dm_numa_node, int, 0644);
3501115485e8SMike Snitzer MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations");
3502115485e8SMike Snitzer
35036a808034SHeinz Mauelshagen module_param(swap_bios, int, 0644);
3504a666e5c0SMikulas Patocka MODULE_PARM_DESC(swap_bios, "Maximum allowed inflight swap IOs");
3505a666e5c0SMikulas Patocka
35061da177e4SLinus Torvalds MODULE_DESCRIPTION(DM_NAME " driver");
35071da177e4SLinus Torvalds MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
35081da177e4SLinus Torvalds MODULE_LICENSE("GPL");
3509