xref: /openbmc/linux/drivers/md/dm.c (revision 743598f049df473869138d599d6ad3b7c740a73f)
1 /*
2  * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3  * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
4  *
5  * This file is released under the GPL.
6  */
7 
8 #include "dm-core.h"
9 #include "dm-rq.h"
10 #include "dm-uevent.h"
11 #include "dm-ima.h"
12 
13 #include <linux/init.h>
14 #include <linux/module.h>
15 #include <linux/mutex.h>
16 #include <linux/sched/mm.h>
17 #include <linux/sched/signal.h>
18 #include <linux/blkpg.h>
19 #include <linux/bio.h>
20 #include <linux/mempool.h>
21 #include <linux/dax.h>
22 #include <linux/slab.h>
23 #include <linux/idr.h>
24 #include <linux/uio.h>
25 #include <linux/hdreg.h>
26 #include <linux/delay.h>
27 #include <linux/wait.h>
28 #include <linux/pr.h>
29 #include <linux/refcount.h>
30 #include <linux/part_stat.h>
31 #include <linux/blk-crypto.h>
32 #include <linux/blk-crypto-profile.h>
33 
34 #define DM_MSG_PREFIX "core"
35 
36 /*
37  * Cookies are numeric values sent with CHANGE and REMOVE
38  * uevents while resuming, removing or renaming the device.
39  */
40 #define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE"
41 #define DM_COOKIE_LENGTH 24
42 
43 static const char *_name = DM_NAME;
44 
45 static unsigned int major = 0;
46 static unsigned int _major = 0;
47 
48 static DEFINE_IDR(_minor_idr);
49 
50 static DEFINE_SPINLOCK(_minor_lock);
51 
52 static void do_deferred_remove(struct work_struct *w);
53 
54 static DECLARE_WORK(deferred_remove_work, do_deferred_remove);
55 
56 static struct workqueue_struct *deferred_remove_workqueue;
57 
58 atomic_t dm_global_event_nr = ATOMIC_INIT(0);
59 DECLARE_WAIT_QUEUE_HEAD(dm_global_eventq);
60 
61 void dm_issue_global_event(void)
62 {
63 	atomic_inc(&dm_global_event_nr);
64 	wake_up(&dm_global_eventq);
65 }
66 
67 /*
68  * One of these is allocated (on-stack) per original bio.
69  */
70 struct clone_info {
71 	struct dm_table *map;
72 	struct bio *bio;
73 	struct dm_io *io;
74 	sector_t sector;
75 	unsigned sector_count;
76 };
77 
78 #define DM_TARGET_IO_BIO_OFFSET (offsetof(struct dm_target_io, clone))
79 #define DM_IO_BIO_OFFSET \
80 	(offsetof(struct dm_target_io, clone) + offsetof(struct dm_io, tio))
81 
82 static inline struct dm_target_io *clone_to_tio(struct bio *clone)
83 {
84 	return container_of(clone, struct dm_target_io, clone);
85 }
86 
87 void *dm_per_bio_data(struct bio *bio, size_t data_size)
88 {
89 	if (!clone_to_tio(bio)->inside_dm_io)
90 		return (char *)bio - DM_TARGET_IO_BIO_OFFSET - data_size;
91 	return (char *)bio - DM_IO_BIO_OFFSET - data_size;
92 }
93 EXPORT_SYMBOL_GPL(dm_per_bio_data);
94 
95 struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size)
96 {
97 	struct dm_io *io = (struct dm_io *)((char *)data + data_size);
98 	if (io->magic == DM_IO_MAGIC)
99 		return (struct bio *)((char *)io + DM_IO_BIO_OFFSET);
100 	BUG_ON(io->magic != DM_TIO_MAGIC);
101 	return (struct bio *)((char *)io + DM_TARGET_IO_BIO_OFFSET);
102 }
103 EXPORT_SYMBOL_GPL(dm_bio_from_per_bio_data);
104 
105 unsigned dm_bio_get_target_bio_nr(const struct bio *bio)
106 {
107 	return container_of(bio, struct dm_target_io, clone)->target_bio_nr;
108 }
109 EXPORT_SYMBOL_GPL(dm_bio_get_target_bio_nr);
110 
111 #define MINOR_ALLOCED ((void *)-1)
112 
113 #define DM_NUMA_NODE NUMA_NO_NODE
114 static int dm_numa_node = DM_NUMA_NODE;
115 
116 #define DEFAULT_SWAP_BIOS	(8 * 1048576 / PAGE_SIZE)
117 static int swap_bios = DEFAULT_SWAP_BIOS;
118 static int get_swap_bios(void)
119 {
120 	int latch = READ_ONCE(swap_bios);
121 	if (unlikely(latch <= 0))
122 		latch = DEFAULT_SWAP_BIOS;
123 	return latch;
124 }
125 
126 /*
127  * For mempools pre-allocation at the table loading time.
128  */
129 struct dm_md_mempools {
130 	struct bio_set bs;
131 	struct bio_set io_bs;
132 };
133 
134 struct table_device {
135 	struct list_head list;
136 	refcount_t count;
137 	struct dm_dev dm_dev;
138 };
139 
140 /*
141  * Bio-based DM's mempools' reserved IOs set by the user.
142  */
143 #define RESERVED_BIO_BASED_IOS		16
144 static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS;
145 
146 static int __dm_get_module_param_int(int *module_param, int min, int max)
147 {
148 	int param = READ_ONCE(*module_param);
149 	int modified_param = 0;
150 	bool modified = true;
151 
152 	if (param < min)
153 		modified_param = min;
154 	else if (param > max)
155 		modified_param = max;
156 	else
157 		modified = false;
158 
159 	if (modified) {
160 		(void)cmpxchg(module_param, param, modified_param);
161 		param = modified_param;
162 	}
163 
164 	return param;
165 }
166 
167 unsigned __dm_get_module_param(unsigned *module_param,
168 			       unsigned def, unsigned max)
169 {
170 	unsigned param = READ_ONCE(*module_param);
171 	unsigned modified_param = 0;
172 
173 	if (!param)
174 		modified_param = def;
175 	else if (param > max)
176 		modified_param = max;
177 
178 	if (modified_param) {
179 		(void)cmpxchg(module_param, param, modified_param);
180 		param = modified_param;
181 	}
182 
183 	return param;
184 }
185 
186 unsigned dm_get_reserved_bio_based_ios(void)
187 {
188 	return __dm_get_module_param(&reserved_bio_based_ios,
189 				     RESERVED_BIO_BASED_IOS, DM_RESERVED_MAX_IOS);
190 }
191 EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios);
192 
193 static unsigned dm_get_numa_node(void)
194 {
195 	return __dm_get_module_param_int(&dm_numa_node,
196 					 DM_NUMA_NODE, num_online_nodes() - 1);
197 }
198 
199 static int __init local_init(void)
200 {
201 	int r;
202 
203 	r = dm_uevent_init();
204 	if (r)
205 		return r;
206 
207 	deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1);
208 	if (!deferred_remove_workqueue) {
209 		r = -ENOMEM;
210 		goto out_uevent_exit;
211 	}
212 
213 	_major = major;
214 	r = register_blkdev(_major, _name);
215 	if (r < 0)
216 		goto out_free_workqueue;
217 
218 	if (!_major)
219 		_major = r;
220 
221 	return 0;
222 
223 out_free_workqueue:
224 	destroy_workqueue(deferred_remove_workqueue);
225 out_uevent_exit:
226 	dm_uevent_exit();
227 
228 	return r;
229 }
230 
231 static void local_exit(void)
232 {
233 	flush_scheduled_work();
234 	destroy_workqueue(deferred_remove_workqueue);
235 
236 	unregister_blkdev(_major, _name);
237 	dm_uevent_exit();
238 
239 	_major = 0;
240 
241 	DMINFO("cleaned up");
242 }
243 
244 static int (*_inits[])(void) __initdata = {
245 	local_init,
246 	dm_target_init,
247 	dm_linear_init,
248 	dm_stripe_init,
249 	dm_io_init,
250 	dm_kcopyd_init,
251 	dm_interface_init,
252 	dm_statistics_init,
253 };
254 
255 static void (*_exits[])(void) = {
256 	local_exit,
257 	dm_target_exit,
258 	dm_linear_exit,
259 	dm_stripe_exit,
260 	dm_io_exit,
261 	dm_kcopyd_exit,
262 	dm_interface_exit,
263 	dm_statistics_exit,
264 };
265 
266 static int __init dm_init(void)
267 {
268 	const int count = ARRAY_SIZE(_inits);
269 	int r, i;
270 
271 #if (IS_ENABLED(CONFIG_IMA) && !IS_ENABLED(CONFIG_IMA_DISABLE_HTABLE))
272 	DMWARN("CONFIG_IMA_DISABLE_HTABLE is disabled."
273 	       " Duplicate IMA measurements will not be recorded in the IMA log.");
274 #endif
275 
276 	for (i = 0; i < count; i++) {
277 		r = _inits[i]();
278 		if (r)
279 			goto bad;
280 	}
281 
282 	return 0;
283 bad:
284 	while (i--)
285 		_exits[i]();
286 
287 	return r;
288 }
289 
290 static void __exit dm_exit(void)
291 {
292 	int i = ARRAY_SIZE(_exits);
293 
294 	while (i--)
295 		_exits[i]();
296 
297 	/*
298 	 * Should be empty by this point.
299 	 */
300 	idr_destroy(&_minor_idr);
301 }
302 
303 /*
304  * Block device functions
305  */
306 int dm_deleting_md(struct mapped_device *md)
307 {
308 	return test_bit(DMF_DELETING, &md->flags);
309 }
310 
311 static int dm_blk_open(struct block_device *bdev, fmode_t mode)
312 {
313 	struct mapped_device *md;
314 
315 	spin_lock(&_minor_lock);
316 
317 	md = bdev->bd_disk->private_data;
318 	if (!md)
319 		goto out;
320 
321 	if (test_bit(DMF_FREEING, &md->flags) ||
322 	    dm_deleting_md(md)) {
323 		md = NULL;
324 		goto out;
325 	}
326 
327 	dm_get(md);
328 	atomic_inc(&md->open_count);
329 out:
330 	spin_unlock(&_minor_lock);
331 
332 	return md ? 0 : -ENXIO;
333 }
334 
335 static void dm_blk_close(struct gendisk *disk, fmode_t mode)
336 {
337 	struct mapped_device *md;
338 
339 	spin_lock(&_minor_lock);
340 
341 	md = disk->private_data;
342 	if (WARN_ON(!md))
343 		goto out;
344 
345 	if (atomic_dec_and_test(&md->open_count) &&
346 	    (test_bit(DMF_DEFERRED_REMOVE, &md->flags)))
347 		queue_work(deferred_remove_workqueue, &deferred_remove_work);
348 
349 	dm_put(md);
350 out:
351 	spin_unlock(&_minor_lock);
352 }
353 
354 int dm_open_count(struct mapped_device *md)
355 {
356 	return atomic_read(&md->open_count);
357 }
358 
359 /*
360  * Guarantees nothing is using the device before it's deleted.
361  */
362 int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred)
363 {
364 	int r = 0;
365 
366 	spin_lock(&_minor_lock);
367 
368 	if (dm_open_count(md)) {
369 		r = -EBUSY;
370 		if (mark_deferred)
371 			set_bit(DMF_DEFERRED_REMOVE, &md->flags);
372 	} else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags))
373 		r = -EEXIST;
374 	else
375 		set_bit(DMF_DELETING, &md->flags);
376 
377 	spin_unlock(&_minor_lock);
378 
379 	return r;
380 }
381 
382 int dm_cancel_deferred_remove(struct mapped_device *md)
383 {
384 	int r = 0;
385 
386 	spin_lock(&_minor_lock);
387 
388 	if (test_bit(DMF_DELETING, &md->flags))
389 		r = -EBUSY;
390 	else
391 		clear_bit(DMF_DEFERRED_REMOVE, &md->flags);
392 
393 	spin_unlock(&_minor_lock);
394 
395 	return r;
396 }
397 
398 static void do_deferred_remove(struct work_struct *w)
399 {
400 	dm_deferred_remove();
401 }
402 
403 static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
404 {
405 	struct mapped_device *md = bdev->bd_disk->private_data;
406 
407 	return dm_get_geometry(md, geo);
408 }
409 
410 static int dm_prepare_ioctl(struct mapped_device *md, int *srcu_idx,
411 			    struct block_device **bdev)
412 {
413 	struct dm_target *tgt;
414 	struct dm_table *map;
415 	int r;
416 
417 retry:
418 	r = -ENOTTY;
419 	map = dm_get_live_table(md, srcu_idx);
420 	if (!map || !dm_table_get_size(map))
421 		return r;
422 
423 	/* We only support devices that have a single target */
424 	if (dm_table_get_num_targets(map) != 1)
425 		return r;
426 
427 	tgt = dm_table_get_target(map, 0);
428 	if (!tgt->type->prepare_ioctl)
429 		return r;
430 
431 	if (dm_suspended_md(md))
432 		return -EAGAIN;
433 
434 	r = tgt->type->prepare_ioctl(tgt, bdev);
435 	if (r == -ENOTCONN && !fatal_signal_pending(current)) {
436 		dm_put_live_table(md, *srcu_idx);
437 		msleep(10);
438 		goto retry;
439 	}
440 
441 	return r;
442 }
443 
444 static void dm_unprepare_ioctl(struct mapped_device *md, int srcu_idx)
445 {
446 	dm_put_live_table(md, srcu_idx);
447 }
448 
449 static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
450 			unsigned int cmd, unsigned long arg)
451 {
452 	struct mapped_device *md = bdev->bd_disk->private_data;
453 	int r, srcu_idx;
454 
455 	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
456 	if (r < 0)
457 		goto out;
458 
459 	if (r > 0) {
460 		/*
461 		 * Target determined this ioctl is being issued against a
462 		 * subset of the parent bdev; require extra privileges.
463 		 */
464 		if (!capable(CAP_SYS_RAWIO)) {
465 			DMDEBUG_LIMIT(
466 	"%s: sending ioctl %x to DM device without required privilege.",
467 				current->comm, cmd);
468 			r = -ENOIOCTLCMD;
469 			goto out;
470 		}
471 	}
472 
473 	if (!bdev->bd_disk->fops->ioctl)
474 		r = -ENOTTY;
475 	else
476 		r = bdev->bd_disk->fops->ioctl(bdev, mode, cmd, arg);
477 out:
478 	dm_unprepare_ioctl(md, srcu_idx);
479 	return r;
480 }
481 
482 u64 dm_start_time_ns_from_clone(struct bio *bio)
483 {
484 	return jiffies_to_nsecs(clone_to_tio(bio)->io->start_time);
485 }
486 EXPORT_SYMBOL_GPL(dm_start_time_ns_from_clone);
487 
488 static bool bio_is_flush_with_data(struct bio *bio)
489 {
490 	return ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size);
491 }
492 
493 static void dm_io_acct(bool end, struct mapped_device *md, struct bio *bio,
494 		       unsigned long start_time, struct dm_stats_aux *stats_aux)
495 {
496 	bool is_flush_with_data;
497 	unsigned int bi_size;
498 
499 	/* If REQ_PREFLUSH set save any payload but do not account it */
500 	is_flush_with_data = bio_is_flush_with_data(bio);
501 	if (is_flush_with_data) {
502 		bi_size = bio->bi_iter.bi_size;
503 		bio->bi_iter.bi_size = 0;
504 	}
505 
506 	if (!end)
507 		bio_start_io_acct_time(bio, start_time);
508 	else
509 		bio_end_io_acct(bio, start_time);
510 
511 	if (unlikely(dm_stats_used(&md->stats)))
512 		dm_stats_account_io(&md->stats, bio_data_dir(bio),
513 				    bio->bi_iter.bi_sector, bio_sectors(bio),
514 				    end, start_time, stats_aux);
515 
516 	/* Restore bio's payload so it does get accounted upon requeue */
517 	if (is_flush_with_data)
518 		bio->bi_iter.bi_size = bi_size;
519 }
520 
521 static void dm_start_io_acct(struct dm_io *io)
522 {
523 	dm_io_acct(false, io->md, io->orig_bio, io->start_time, &io->stats_aux);
524 }
525 
526 static void dm_end_io_acct(struct dm_io *io)
527 {
528 	dm_io_acct(true, io->md, io->orig_bio, io->start_time, &io->stats_aux);
529 }
530 
531 static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio)
532 {
533 	struct dm_io *io;
534 	struct dm_target_io *tio;
535 	struct bio *clone;
536 
537 	clone = bio_alloc_clone(bio->bi_bdev, bio, GFP_NOIO, &md->io_bs);
538 
539 	tio = clone_to_tio(clone);
540 	tio->inside_dm_io = true;
541 	tio->io = NULL;
542 
543 	io = container_of(tio, struct dm_io, tio);
544 	io->magic = DM_IO_MAGIC;
545 	io->status = 0;
546 	atomic_set(&io->io_count, 1);
547 	this_cpu_inc(*md->pending_io);
548 	io->orig_bio = bio;
549 	io->md = md;
550 	spin_lock_init(&io->endio_lock);
551 
552 	io->start_time = jiffies;
553 
554 	dm_stats_record_start(&md->stats, &io->stats_aux);
555 
556 	return io;
557 }
558 
559 static void free_io(struct dm_io *io)
560 {
561 	bio_put(&io->tio.clone);
562 }
563 
564 static struct bio *alloc_tio(struct clone_info *ci, struct dm_target *ti,
565 		unsigned target_bio_nr, unsigned *len, gfp_t gfp_mask)
566 {
567 	struct dm_target_io *tio;
568 
569 	if (!ci->io->tio.io) {
570 		/* the dm_target_io embedded in ci->io is available */
571 		tio = &ci->io->tio;
572 	} else {
573 		struct bio *clone = bio_alloc_clone(ci->bio->bi_bdev, ci->bio,
574 						    gfp_mask, &ci->io->md->bs);
575 		if (!clone)
576 			return NULL;
577 
578 		tio = clone_to_tio(clone);
579 		tio->inside_dm_io = false;
580 	}
581 
582 	tio->magic = DM_TIO_MAGIC;
583 	tio->io = ci->io;
584 	tio->ti = ti;
585 	tio->target_bio_nr = target_bio_nr;
586 	tio->len_ptr = len;
587 	tio->old_sector = 0;
588 
589 	return &tio->clone;
590 }
591 
592 static void free_tio(struct bio *clone)
593 {
594 	if (clone_to_tio(clone)->inside_dm_io)
595 		return;
596 	bio_put(clone);
597 }
598 
599 /*
600  * Add the bio to the list of deferred io.
601  */
602 static void queue_io(struct mapped_device *md, struct bio *bio)
603 {
604 	unsigned long flags;
605 
606 	spin_lock_irqsave(&md->deferred_lock, flags);
607 	bio_list_add(&md->deferred, bio);
608 	spin_unlock_irqrestore(&md->deferred_lock, flags);
609 	queue_work(md->wq, &md->work);
610 }
611 
612 /*
613  * Everyone (including functions in this file), should use this
614  * function to access the md->map field, and make sure they call
615  * dm_put_live_table() when finished.
616  */
617 struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier)
618 {
619 	*srcu_idx = srcu_read_lock(&md->io_barrier);
620 
621 	return srcu_dereference(md->map, &md->io_barrier);
622 }
623 
624 void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier)
625 {
626 	srcu_read_unlock(&md->io_barrier, srcu_idx);
627 }
628 
629 void dm_sync_table(struct mapped_device *md)
630 {
631 	synchronize_srcu(&md->io_barrier);
632 	synchronize_rcu_expedited();
633 }
634 
635 /*
636  * A fast alternative to dm_get_live_table/dm_put_live_table.
637  * The caller must not block between these two functions.
638  */
639 static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU)
640 {
641 	rcu_read_lock();
642 	return rcu_dereference(md->map);
643 }
644 
645 static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU)
646 {
647 	rcu_read_unlock();
648 }
649 
650 static char *_dm_claim_ptr = "I belong to device-mapper";
651 
652 /*
653  * Open a table device so we can use it as a map destination.
654  */
655 static int open_table_device(struct table_device *td, dev_t dev,
656 			     struct mapped_device *md)
657 {
658 	struct block_device *bdev;
659 	u64 part_off;
660 	int r;
661 
662 	BUG_ON(td->dm_dev.bdev);
663 
664 	bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _dm_claim_ptr);
665 	if (IS_ERR(bdev))
666 		return PTR_ERR(bdev);
667 
668 	r = bd_link_disk_holder(bdev, dm_disk(md));
669 	if (r) {
670 		blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL);
671 		return r;
672 	}
673 
674 	td->dm_dev.bdev = bdev;
675 	td->dm_dev.dax_dev = fs_dax_get_by_bdev(bdev, &part_off);
676 	return 0;
677 }
678 
679 /*
680  * Close a table device that we've been using.
681  */
682 static void close_table_device(struct table_device *td, struct mapped_device *md)
683 {
684 	if (!td->dm_dev.bdev)
685 		return;
686 
687 	bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md));
688 	blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL);
689 	put_dax(td->dm_dev.dax_dev);
690 	td->dm_dev.bdev = NULL;
691 	td->dm_dev.dax_dev = NULL;
692 }
693 
694 static struct table_device *find_table_device(struct list_head *l, dev_t dev,
695 					      fmode_t mode)
696 {
697 	struct table_device *td;
698 
699 	list_for_each_entry(td, l, list)
700 		if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode)
701 			return td;
702 
703 	return NULL;
704 }
705 
706 int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode,
707 			struct dm_dev **result)
708 {
709 	int r;
710 	struct table_device *td;
711 
712 	mutex_lock(&md->table_devices_lock);
713 	td = find_table_device(&md->table_devices, dev, mode);
714 	if (!td) {
715 		td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id);
716 		if (!td) {
717 			mutex_unlock(&md->table_devices_lock);
718 			return -ENOMEM;
719 		}
720 
721 		td->dm_dev.mode = mode;
722 		td->dm_dev.bdev = NULL;
723 
724 		if ((r = open_table_device(td, dev, md))) {
725 			mutex_unlock(&md->table_devices_lock);
726 			kfree(td);
727 			return r;
728 		}
729 
730 		format_dev_t(td->dm_dev.name, dev);
731 
732 		refcount_set(&td->count, 1);
733 		list_add(&td->list, &md->table_devices);
734 	} else {
735 		refcount_inc(&td->count);
736 	}
737 	mutex_unlock(&md->table_devices_lock);
738 
739 	*result = &td->dm_dev;
740 	return 0;
741 }
742 
743 void dm_put_table_device(struct mapped_device *md, struct dm_dev *d)
744 {
745 	struct table_device *td = container_of(d, struct table_device, dm_dev);
746 
747 	mutex_lock(&md->table_devices_lock);
748 	if (refcount_dec_and_test(&td->count)) {
749 		close_table_device(td, md);
750 		list_del(&td->list);
751 		kfree(td);
752 	}
753 	mutex_unlock(&md->table_devices_lock);
754 }
755 
756 static void free_table_devices(struct list_head *devices)
757 {
758 	struct list_head *tmp, *next;
759 
760 	list_for_each_safe(tmp, next, devices) {
761 		struct table_device *td = list_entry(tmp, struct table_device, list);
762 
763 		DMWARN("dm_destroy: %s still exists with %d references",
764 		       td->dm_dev.name, refcount_read(&td->count));
765 		kfree(td);
766 	}
767 }
768 
769 /*
770  * Get the geometry associated with a dm device
771  */
772 int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
773 {
774 	*geo = md->geometry;
775 
776 	return 0;
777 }
778 
779 /*
780  * Set the geometry of a device.
781  */
782 int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
783 {
784 	sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
785 
786 	if (geo->start > sz) {
787 		DMWARN("Start sector is beyond the geometry limits.");
788 		return -EINVAL;
789 	}
790 
791 	md->geometry = *geo;
792 
793 	return 0;
794 }
795 
796 static int __noflush_suspending(struct mapped_device *md)
797 {
798 	return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
799 }
800 
801 /*
802  * Decrements the number of outstanding ios that a bio has been
803  * cloned into, completing the original io if necc.
804  */
805 void dm_io_dec_pending(struct dm_io *io, blk_status_t error)
806 {
807 	unsigned long flags;
808 	blk_status_t io_error;
809 	struct bio *bio;
810 	struct mapped_device *md = io->md;
811 
812 	/* Push-back supersedes any I/O errors */
813 	if (unlikely(error)) {
814 		spin_lock_irqsave(&io->endio_lock, flags);
815 		if (!(io->status == BLK_STS_DM_REQUEUE && __noflush_suspending(md)))
816 			io->status = error;
817 		spin_unlock_irqrestore(&io->endio_lock, flags);
818 	}
819 
820 	if (atomic_dec_and_test(&io->io_count)) {
821 		bio = io->orig_bio;
822 		if (io->status == BLK_STS_DM_REQUEUE) {
823 			/*
824 			 * Target requested pushing back the I/O.
825 			 */
826 			spin_lock_irqsave(&md->deferred_lock, flags);
827 			if (__noflush_suspending(md) &&
828 			    !WARN_ON_ONCE(dm_is_zone_write(md, bio))) {
829 				/* NOTE early return due to BLK_STS_DM_REQUEUE below */
830 				bio_list_add_head(&md->deferred, bio);
831 			} else {
832 				/*
833 				 * noflush suspend was interrupted or this is
834 				 * a write to a zoned target.
835 				 */
836 				io->status = BLK_STS_IOERR;
837 			}
838 			spin_unlock_irqrestore(&md->deferred_lock, flags);
839 		}
840 
841 		io_error = io->status;
842 		dm_end_io_acct(io);
843 		free_io(io);
844 		smp_wmb();
845 		this_cpu_dec(*md->pending_io);
846 
847 		/* nudge anyone waiting on suspend queue */
848 		if (unlikely(wq_has_sleeper(&md->wait)))
849 			wake_up(&md->wait);
850 
851 		if (io_error == BLK_STS_DM_REQUEUE)
852 			return;
853 
854 		if (bio_is_flush_with_data(bio)) {
855 			/*
856 			 * Preflush done for flush with data, reissue
857 			 * without REQ_PREFLUSH.
858 			 */
859 			bio->bi_opf &= ~REQ_PREFLUSH;
860 			queue_io(md, bio);
861 		} else {
862 			/* done with normal IO or empty flush */
863 			if (io_error)
864 				bio->bi_status = io_error;
865 			bio_endio(bio);
866 		}
867 	}
868 }
869 
870 void disable_discard(struct mapped_device *md)
871 {
872 	struct queue_limits *limits = dm_get_queue_limits(md);
873 
874 	/* device doesn't really support DISCARD, disable it */
875 	limits->max_discard_sectors = 0;
876 	blk_queue_flag_clear(QUEUE_FLAG_DISCARD, md->queue);
877 }
878 
879 void disable_write_same(struct mapped_device *md)
880 {
881 	struct queue_limits *limits = dm_get_queue_limits(md);
882 
883 	/* device doesn't really support WRITE SAME, disable it */
884 	limits->max_write_same_sectors = 0;
885 }
886 
887 void disable_write_zeroes(struct mapped_device *md)
888 {
889 	struct queue_limits *limits = dm_get_queue_limits(md);
890 
891 	/* device doesn't really support WRITE ZEROES, disable it */
892 	limits->max_write_zeroes_sectors = 0;
893 }
894 
895 static bool swap_bios_limit(struct dm_target *ti, struct bio *bio)
896 {
897 	return unlikely((bio->bi_opf & REQ_SWAP) != 0) && unlikely(ti->limit_swap_bios);
898 }
899 
900 static void clone_endio(struct bio *bio)
901 {
902 	blk_status_t error = bio->bi_status;
903 	struct dm_target_io *tio = clone_to_tio(bio);
904 	struct dm_io *io = tio->io;
905 	struct mapped_device *md = tio->io->md;
906 	dm_endio_fn endio = tio->ti->type->end_io;
907 	struct request_queue *q = bio->bi_bdev->bd_disk->queue;
908 
909 	if (unlikely(error == BLK_STS_TARGET)) {
910 		if (bio_op(bio) == REQ_OP_DISCARD &&
911 		    !q->limits.max_discard_sectors)
912 			disable_discard(md);
913 		else if (bio_op(bio) == REQ_OP_WRITE_SAME &&
914 			 !q->limits.max_write_same_sectors)
915 			disable_write_same(md);
916 		else if (bio_op(bio) == REQ_OP_WRITE_ZEROES &&
917 			 !q->limits.max_write_zeroes_sectors)
918 			disable_write_zeroes(md);
919 	}
920 
921 	if (blk_queue_is_zoned(q))
922 		dm_zone_endio(io, bio);
923 
924 	if (endio) {
925 		int r = endio(tio->ti, bio, &error);
926 		switch (r) {
927 		case DM_ENDIO_REQUEUE:
928 			/*
929 			 * Requeuing writes to a sequential zone of a zoned
930 			 * target will break the sequential write pattern:
931 			 * fail such IO.
932 			 */
933 			if (WARN_ON_ONCE(dm_is_zone_write(md, bio)))
934 				error = BLK_STS_IOERR;
935 			else
936 				error = BLK_STS_DM_REQUEUE;
937 			fallthrough;
938 		case DM_ENDIO_DONE:
939 			break;
940 		case DM_ENDIO_INCOMPLETE:
941 			/* The target will handle the io */
942 			return;
943 		default:
944 			DMWARN("unimplemented target endio return value: %d", r);
945 			BUG();
946 		}
947 	}
948 
949 	if (unlikely(swap_bios_limit(tio->ti, bio))) {
950 		struct mapped_device *md = io->md;
951 		up(&md->swap_bios_semaphore);
952 	}
953 
954 	free_tio(bio);
955 	dm_io_dec_pending(io, error);
956 }
957 
958 /*
959  * Return maximum size of I/O possible at the supplied sector up to the current
960  * target boundary.
961  */
962 static inline sector_t max_io_len_target_boundary(struct dm_target *ti,
963 						  sector_t target_offset)
964 {
965 	return ti->len - target_offset;
966 }
967 
968 static sector_t max_io_len(struct dm_target *ti, sector_t sector)
969 {
970 	sector_t target_offset = dm_target_offset(ti, sector);
971 	sector_t len = max_io_len_target_boundary(ti, target_offset);
972 	sector_t max_len;
973 
974 	/*
975 	 * Does the target need to split IO even further?
976 	 * - varied (per target) IO splitting is a tenet of DM; this
977 	 *   explains why stacked chunk_sectors based splitting via
978 	 *   blk_max_size_offset() isn't possible here. So pass in
979 	 *   ti->max_io_len to override stacked chunk_sectors.
980 	 */
981 	if (ti->max_io_len) {
982 		max_len = blk_max_size_offset(ti->table->md->queue,
983 					      target_offset, ti->max_io_len);
984 		if (len > max_len)
985 			len = max_len;
986 	}
987 
988 	return len;
989 }
990 
991 int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
992 {
993 	if (len > UINT_MAX) {
994 		DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)",
995 		      (unsigned long long)len, UINT_MAX);
996 		ti->error = "Maximum size of target IO is too large";
997 		return -EINVAL;
998 	}
999 
1000 	ti->max_io_len = (uint32_t) len;
1001 
1002 	return 0;
1003 }
1004 EXPORT_SYMBOL_GPL(dm_set_target_max_io_len);
1005 
1006 static struct dm_target *dm_dax_get_live_target(struct mapped_device *md,
1007 						sector_t sector, int *srcu_idx)
1008 	__acquires(md->io_barrier)
1009 {
1010 	struct dm_table *map;
1011 	struct dm_target *ti;
1012 
1013 	map = dm_get_live_table(md, srcu_idx);
1014 	if (!map)
1015 		return NULL;
1016 
1017 	ti = dm_table_find_target(map, sector);
1018 	if (!ti)
1019 		return NULL;
1020 
1021 	return ti;
1022 }
1023 
1024 static long dm_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
1025 				 long nr_pages, void **kaddr, pfn_t *pfn)
1026 {
1027 	struct mapped_device *md = dax_get_private(dax_dev);
1028 	sector_t sector = pgoff * PAGE_SECTORS;
1029 	struct dm_target *ti;
1030 	long len, ret = -EIO;
1031 	int srcu_idx;
1032 
1033 	ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1034 
1035 	if (!ti)
1036 		goto out;
1037 	if (!ti->type->direct_access)
1038 		goto out;
1039 	len = max_io_len(ti, sector) / PAGE_SECTORS;
1040 	if (len < 1)
1041 		goto out;
1042 	nr_pages = min(len, nr_pages);
1043 	ret = ti->type->direct_access(ti, pgoff, nr_pages, kaddr, pfn);
1044 
1045  out:
1046 	dm_put_live_table(md, srcu_idx);
1047 
1048 	return ret;
1049 }
1050 
1051 static int dm_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,
1052 				  size_t nr_pages)
1053 {
1054 	struct mapped_device *md = dax_get_private(dax_dev);
1055 	sector_t sector = pgoff * PAGE_SECTORS;
1056 	struct dm_target *ti;
1057 	int ret = -EIO;
1058 	int srcu_idx;
1059 
1060 	ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1061 
1062 	if (!ti)
1063 		goto out;
1064 	if (WARN_ON(!ti->type->dax_zero_page_range)) {
1065 		/*
1066 		 * ->zero_page_range() is mandatory dax operation. If we are
1067 		 *  here, something is wrong.
1068 		 */
1069 		goto out;
1070 	}
1071 	ret = ti->type->dax_zero_page_range(ti, pgoff, nr_pages);
1072  out:
1073 	dm_put_live_table(md, srcu_idx);
1074 
1075 	return ret;
1076 }
1077 
1078 /*
1079  * A target may call dm_accept_partial_bio only from the map routine.  It is
1080  * allowed for all bio types except REQ_PREFLUSH, REQ_OP_ZONE_* zone management
1081  * operations and REQ_OP_ZONE_APPEND (zone append writes).
1082  *
1083  * dm_accept_partial_bio informs the dm that the target only wants to process
1084  * additional n_sectors sectors of the bio and the rest of the data should be
1085  * sent in a next bio.
1086  *
1087  * A diagram that explains the arithmetics:
1088  * +--------------------+---------------+-------+
1089  * |         1          |       2       |   3   |
1090  * +--------------------+---------------+-------+
1091  *
1092  * <-------------- *tio->len_ptr --------------->
1093  *                      <------- bi_size ------->
1094  *                      <-- n_sectors -->
1095  *
1096  * Region 1 was already iterated over with bio_advance or similar function.
1097  *	(it may be empty if the target doesn't use bio_advance)
1098  * Region 2 is the remaining bio size that the target wants to process.
1099  *	(it may be empty if region 1 is non-empty, although there is no reason
1100  *	 to make it empty)
1101  * The target requires that region 3 is to be sent in the next bio.
1102  *
1103  * If the target wants to receive multiple copies of the bio (via num_*bios, etc),
1104  * the partially processed part (the sum of regions 1+2) must be the same for all
1105  * copies of the bio.
1106  */
1107 void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors)
1108 {
1109 	struct dm_target_io *tio = clone_to_tio(bio);
1110 	unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT;
1111 
1112 	BUG_ON(bio->bi_opf & REQ_PREFLUSH);
1113 	BUG_ON(op_is_zone_mgmt(bio_op(bio)));
1114 	BUG_ON(bio_op(bio) == REQ_OP_ZONE_APPEND);
1115 	BUG_ON(bi_size > *tio->len_ptr);
1116 	BUG_ON(n_sectors > bi_size);
1117 
1118 	*tio->len_ptr -= bi_size - n_sectors;
1119 	bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT;
1120 }
1121 EXPORT_SYMBOL_GPL(dm_accept_partial_bio);
1122 
1123 static noinline void __set_swap_bios_limit(struct mapped_device *md, int latch)
1124 {
1125 	mutex_lock(&md->swap_bios_lock);
1126 	while (latch < md->swap_bios) {
1127 		cond_resched();
1128 		down(&md->swap_bios_semaphore);
1129 		md->swap_bios--;
1130 	}
1131 	while (latch > md->swap_bios) {
1132 		cond_resched();
1133 		up(&md->swap_bios_semaphore);
1134 		md->swap_bios++;
1135 	}
1136 	mutex_unlock(&md->swap_bios_lock);
1137 }
1138 
1139 static void __map_bio(struct bio *clone)
1140 {
1141 	struct dm_target_io *tio = clone_to_tio(clone);
1142 	int r;
1143 	struct dm_io *io = tio->io;
1144 	struct dm_target *ti = tio->ti;
1145 
1146 	clone->bi_end_io = clone_endio;
1147 
1148 	/*
1149 	 * Map the clone.  If r == 0 we don't need to do
1150 	 * anything, the target has assumed ownership of
1151 	 * this io.
1152 	 */
1153 	dm_io_inc_pending(io);
1154 	tio->old_sector = clone->bi_iter.bi_sector;
1155 
1156 	if (unlikely(swap_bios_limit(ti, clone))) {
1157 		struct mapped_device *md = io->md;
1158 		int latch = get_swap_bios();
1159 		if (unlikely(latch != md->swap_bios))
1160 			__set_swap_bios_limit(md, latch);
1161 		down(&md->swap_bios_semaphore);
1162 	}
1163 
1164 	/*
1165 	 * Check if the IO needs a special mapping due to zone append emulation
1166 	 * on zoned target. In this case, dm_zone_map_bio() calls the target
1167 	 * map operation.
1168 	 */
1169 	if (dm_emulate_zone_append(io->md))
1170 		r = dm_zone_map_bio(tio);
1171 	else
1172 		r = ti->type->map(ti, clone);
1173 
1174 	switch (r) {
1175 	case DM_MAPIO_SUBMITTED:
1176 		break;
1177 	case DM_MAPIO_REMAPPED:
1178 		/* the bio has been remapped so dispatch it */
1179 		trace_block_bio_remap(clone, bio_dev(io->orig_bio),
1180 				      tio->old_sector);
1181 		submit_bio_noacct(clone);
1182 		break;
1183 	case DM_MAPIO_KILL:
1184 	case DM_MAPIO_REQUEUE:
1185 		if (unlikely(swap_bios_limit(ti, clone)))
1186 			up(&io->md->swap_bios_semaphore);
1187 		free_tio(clone);
1188 		if (r == DM_MAPIO_KILL)
1189 			dm_io_dec_pending(io, BLK_STS_IOERR);
1190 		else
1191 			dm_io_dec_pending(io, BLK_STS_DM_REQUEUE);
1192 		break;
1193 	default:
1194 		DMWARN("unimplemented target map return value: %d", r);
1195 		BUG();
1196 	}
1197 }
1198 
1199 static void alloc_multiple_bios(struct bio_list *blist, struct clone_info *ci,
1200 				struct dm_target *ti, unsigned num_bios,
1201 				unsigned *len)
1202 {
1203 	struct bio *bio;
1204 	int try;
1205 
1206 	for (try = 0; try < 2; try++) {
1207 		int bio_nr;
1208 
1209 		if (try)
1210 			mutex_lock(&ci->io->md->table_devices_lock);
1211 		for (bio_nr = 0; bio_nr < num_bios; bio_nr++) {
1212 			bio = alloc_tio(ci, ti, bio_nr, len,
1213 					try ? GFP_NOIO : GFP_NOWAIT);
1214 			if (!bio)
1215 				break;
1216 
1217 			bio_list_add(blist, bio);
1218 		}
1219 		if (try)
1220 			mutex_unlock(&ci->io->md->table_devices_lock);
1221 		if (bio_nr == num_bios)
1222 			return;
1223 
1224 		while ((bio = bio_list_pop(blist)))
1225 			free_tio(bio);
1226 	}
1227 }
1228 
1229 static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti,
1230 				  unsigned num_bios, unsigned *len)
1231 {
1232 	struct bio_list blist = BIO_EMPTY_LIST;
1233 	struct bio *clone;
1234 
1235 	switch (num_bios) {
1236 	case 0:
1237 		break;
1238 	case 1:
1239 		clone = alloc_tio(ci, ti, 0, len, GFP_NOIO);
1240 		if (len)
1241 			clone->bi_iter.bi_size = to_bytes(*len);
1242 		__map_bio(clone);
1243 		break;
1244 	default:
1245 		alloc_multiple_bios(&blist, ci, ti, num_bios, len);
1246 		while ((clone = bio_list_pop(&blist))) {
1247 			if (len)
1248 				clone->bi_iter.bi_size = to_bytes(*len);
1249 			__map_bio(clone);
1250 		}
1251 		break;
1252 	}
1253 }
1254 
1255 static int __send_empty_flush(struct clone_info *ci)
1256 {
1257 	unsigned target_nr = 0;
1258 	struct dm_target *ti;
1259 	struct bio flush_bio;
1260 
1261 	/*
1262 	 * Use an on-stack bio for this, it's safe since we don't
1263 	 * need to reference it after submit. It's just used as
1264 	 * the basis for the clone(s).
1265 	 */
1266 	bio_init(&flush_bio, ci->io->md->disk->part0, NULL, 0,
1267 		 REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC);
1268 
1269 	ci->bio = &flush_bio;
1270 	ci->sector_count = 0;
1271 
1272 	while ((ti = dm_table_get_target(ci->map, target_nr++)))
1273 		__send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL);
1274 
1275 	bio_uninit(ci->bio);
1276 	return 0;
1277 }
1278 
1279 static int __send_changing_extent_only(struct clone_info *ci, struct dm_target *ti,
1280 				       unsigned num_bios)
1281 {
1282 	unsigned len;
1283 
1284 	/*
1285 	 * Even though the device advertised support for this type of
1286 	 * request, that does not mean every target supports it, and
1287 	 * reconfiguration might also have changed that since the
1288 	 * check was performed.
1289 	 */
1290 	if (!num_bios)
1291 		return -EOPNOTSUPP;
1292 
1293 	len = min_t(sector_t, ci->sector_count,
1294 		    max_io_len_target_boundary(ti, dm_target_offset(ti, ci->sector)));
1295 
1296 	__send_duplicate_bios(ci, ti, num_bios, &len);
1297 
1298 	ci->sector += len;
1299 	ci->sector_count -= len;
1300 
1301 	return 0;
1302 }
1303 
1304 static bool is_abnormal_io(struct bio *bio)
1305 {
1306 	bool r = false;
1307 
1308 	switch (bio_op(bio)) {
1309 	case REQ_OP_DISCARD:
1310 	case REQ_OP_SECURE_ERASE:
1311 	case REQ_OP_WRITE_SAME:
1312 	case REQ_OP_WRITE_ZEROES:
1313 		r = true;
1314 		break;
1315 	}
1316 
1317 	return r;
1318 }
1319 
1320 static bool __process_abnormal_io(struct clone_info *ci, struct dm_target *ti,
1321 				  int *result)
1322 {
1323 	struct bio *bio = ci->bio;
1324 	unsigned num_bios = 0;
1325 
1326 	switch (bio_op(bio)) {
1327 	case REQ_OP_DISCARD:
1328 		num_bios = ti->num_discard_bios;
1329 		break;
1330 	case REQ_OP_SECURE_ERASE:
1331 		num_bios = ti->num_secure_erase_bios;
1332 		break;
1333 	case REQ_OP_WRITE_SAME:
1334 		num_bios = ti->num_write_same_bios;
1335 		break;
1336 	case REQ_OP_WRITE_ZEROES:
1337 		num_bios = ti->num_write_zeroes_bios;
1338 		break;
1339 	default:
1340 		return false;
1341 	}
1342 
1343 	*result = __send_changing_extent_only(ci, ti, num_bios);
1344 	return true;
1345 }
1346 
1347 /*
1348  * Select the correct strategy for processing a non-flush bio.
1349  */
1350 static int __split_and_process_bio(struct clone_info *ci)
1351 {
1352 	struct bio *clone;
1353 	struct dm_target *ti;
1354 	unsigned len;
1355 	int r;
1356 
1357 	ti = dm_table_find_target(ci->map, ci->sector);
1358 	if (!ti)
1359 		return -EIO;
1360 
1361 	if (__process_abnormal_io(ci, ti, &r))
1362 		return r;
1363 
1364 	len = min_t(sector_t, max_io_len(ti, ci->sector), ci->sector_count);
1365 
1366 	clone = alloc_tio(ci, ti, 0, &len, GFP_NOIO);
1367 	clone->bi_iter.bi_size = to_bytes(len);
1368 	if (bio_integrity(clone))
1369 		bio_integrity_trim(clone);
1370 
1371 	__map_bio(clone);
1372 
1373 	ci->sector += len;
1374 	ci->sector_count -= len;
1375 
1376 	return 0;
1377 }
1378 
1379 static void init_clone_info(struct clone_info *ci, struct mapped_device *md,
1380 			    struct dm_table *map, struct bio *bio)
1381 {
1382 	ci->map = map;
1383 	ci->io = alloc_io(md, bio);
1384 	ci->bio = bio;
1385 	ci->sector = bio->bi_iter.bi_sector;
1386 	ci->sector_count = bio_sectors(bio);
1387 
1388 	/* Shouldn't happen but sector_count was being set to 0 so... */
1389 	if (WARN_ON_ONCE(op_is_zone_mgmt(bio_op(bio)) && ci->sector_count))
1390 		ci->sector_count = 0;
1391 }
1392 
1393 /*
1394  * Entry point to split a bio into clones and submit them to the targets.
1395  */
1396 static void dm_split_and_process_bio(struct mapped_device *md,
1397 				     struct dm_table *map, struct bio *bio)
1398 {
1399 	struct clone_info ci;
1400 	struct bio *b;
1401 	int error = 0;
1402 
1403 	init_clone_info(&ci, md, map, bio);
1404 
1405 	if (bio->bi_opf & REQ_PREFLUSH) {
1406 		error = __send_empty_flush(&ci);
1407 		/* dm_io_dec_pending submits any data associated with flush */
1408 		goto out;
1409 	}
1410 
1411 	error = __split_and_process_bio(&ci);
1412 	if (error || !ci.sector_count)
1413 		goto out;
1414 
1415 	/*
1416 	 * Remainder must be passed to submit_bio_noacct() so it gets handled
1417 	 * *after* bios already submitted have been completely processed.
1418 	 * We take a clone of the original to store in ci.io->orig_bio to be
1419 	 * used by dm_end_io_acct() and for dm_io_dec_pending() to use for
1420 	 * completion handling.
1421 	 */
1422 	b = bio_split(bio, bio_sectors(bio) - ci.sector_count,
1423 		      GFP_NOIO, &md->queue->bio_split);
1424 	ci.io->orig_bio = b;
1425 
1426 	bio_chain(b, bio);
1427 	trace_block_split(b, bio->bi_iter.bi_sector);
1428 	submit_bio_noacct(bio);
1429 out:
1430 	dm_start_io_acct(ci.io);
1431 	/* drop the extra reference count */
1432 	dm_io_dec_pending(ci.io, errno_to_blk_status(error));
1433 }
1434 
1435 static void dm_submit_bio(struct bio *bio)
1436 {
1437 	struct mapped_device *md = bio->bi_bdev->bd_disk->private_data;
1438 	int srcu_idx;
1439 	struct dm_table *map;
1440 
1441 	map = dm_get_live_table(md, &srcu_idx);
1442 	if (unlikely(!map)) {
1443 		DMERR_LIMIT("%s: mapping table unavailable, erroring io",
1444 			    dm_device_name(md));
1445 		bio_io_error(bio);
1446 		goto out;
1447 	}
1448 
1449 	/* If suspended, queue this IO for later */
1450 	if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) {
1451 		if (bio->bi_opf & REQ_NOWAIT)
1452 			bio_wouldblock_error(bio);
1453 		else if (bio->bi_opf & REQ_RAHEAD)
1454 			bio_io_error(bio);
1455 		else
1456 			queue_io(md, bio);
1457 		goto out;
1458 	}
1459 
1460 	/*
1461 	 * Use blk_queue_split() for abnormal IO (e.g. discard, writesame, etc)
1462 	 * otherwise associated queue_limits won't be imposed.
1463 	 */
1464 	if (is_abnormal_io(bio))
1465 		blk_queue_split(&bio);
1466 
1467 	dm_split_and_process_bio(md, map, bio);
1468 out:
1469 	dm_put_live_table(md, srcu_idx);
1470 }
1471 
1472 /*-----------------------------------------------------------------
1473  * An IDR is used to keep track of allocated minor numbers.
1474  *---------------------------------------------------------------*/
1475 static void free_minor(int minor)
1476 {
1477 	spin_lock(&_minor_lock);
1478 	idr_remove(&_minor_idr, minor);
1479 	spin_unlock(&_minor_lock);
1480 }
1481 
1482 /*
1483  * See if the device with a specific minor # is free.
1484  */
1485 static int specific_minor(int minor)
1486 {
1487 	int r;
1488 
1489 	if (minor >= (1 << MINORBITS))
1490 		return -EINVAL;
1491 
1492 	idr_preload(GFP_KERNEL);
1493 	spin_lock(&_minor_lock);
1494 
1495 	r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT);
1496 
1497 	spin_unlock(&_minor_lock);
1498 	idr_preload_end();
1499 	if (r < 0)
1500 		return r == -ENOSPC ? -EBUSY : r;
1501 	return 0;
1502 }
1503 
1504 static int next_free_minor(int *minor)
1505 {
1506 	int r;
1507 
1508 	idr_preload(GFP_KERNEL);
1509 	spin_lock(&_minor_lock);
1510 
1511 	r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT);
1512 
1513 	spin_unlock(&_minor_lock);
1514 	idr_preload_end();
1515 	if (r < 0)
1516 		return r;
1517 	*minor = r;
1518 	return 0;
1519 }
1520 
1521 static const struct block_device_operations dm_blk_dops;
1522 static const struct block_device_operations dm_rq_blk_dops;
1523 static const struct dax_operations dm_dax_ops;
1524 
1525 static void dm_wq_work(struct work_struct *work);
1526 
1527 #ifdef CONFIG_BLK_INLINE_ENCRYPTION
1528 static void dm_queue_destroy_crypto_profile(struct request_queue *q)
1529 {
1530 	dm_destroy_crypto_profile(q->crypto_profile);
1531 }
1532 
1533 #else /* CONFIG_BLK_INLINE_ENCRYPTION */
1534 
1535 static inline void dm_queue_destroy_crypto_profile(struct request_queue *q)
1536 {
1537 }
1538 #endif /* !CONFIG_BLK_INLINE_ENCRYPTION */
1539 
1540 static void cleanup_mapped_device(struct mapped_device *md)
1541 {
1542 	if (md->wq)
1543 		destroy_workqueue(md->wq);
1544 	bioset_exit(&md->bs);
1545 	bioset_exit(&md->io_bs);
1546 
1547 	if (md->dax_dev) {
1548 		dax_remove_host(md->disk);
1549 		kill_dax(md->dax_dev);
1550 		put_dax(md->dax_dev);
1551 		md->dax_dev = NULL;
1552 	}
1553 
1554 	if (md->disk) {
1555 		spin_lock(&_minor_lock);
1556 		md->disk->private_data = NULL;
1557 		spin_unlock(&_minor_lock);
1558 		if (dm_get_md_type(md) != DM_TYPE_NONE) {
1559 			dm_sysfs_exit(md);
1560 			del_gendisk(md->disk);
1561 		}
1562 		dm_queue_destroy_crypto_profile(md->queue);
1563 		blk_cleanup_disk(md->disk);
1564 	}
1565 
1566 	if (md->pending_io) {
1567 		free_percpu(md->pending_io);
1568 		md->pending_io = NULL;
1569 	}
1570 
1571 	cleanup_srcu_struct(&md->io_barrier);
1572 
1573 	mutex_destroy(&md->suspend_lock);
1574 	mutex_destroy(&md->type_lock);
1575 	mutex_destroy(&md->table_devices_lock);
1576 	mutex_destroy(&md->swap_bios_lock);
1577 
1578 	dm_mq_cleanup_mapped_device(md);
1579 	dm_cleanup_zoned_dev(md);
1580 }
1581 
1582 /*
1583  * Allocate and initialise a blank device with a given minor.
1584  */
1585 static struct mapped_device *alloc_dev(int minor)
1586 {
1587 	int r, numa_node_id = dm_get_numa_node();
1588 	struct mapped_device *md;
1589 	void *old_md;
1590 
1591 	md = kvzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id);
1592 	if (!md) {
1593 		DMWARN("unable to allocate device, out of memory.");
1594 		return NULL;
1595 	}
1596 
1597 	if (!try_module_get(THIS_MODULE))
1598 		goto bad_module_get;
1599 
1600 	/* get a minor number for the dev */
1601 	if (minor == DM_ANY_MINOR)
1602 		r = next_free_minor(&minor);
1603 	else
1604 		r = specific_minor(minor);
1605 	if (r < 0)
1606 		goto bad_minor;
1607 
1608 	r = init_srcu_struct(&md->io_barrier);
1609 	if (r < 0)
1610 		goto bad_io_barrier;
1611 
1612 	md->numa_node_id = numa_node_id;
1613 	md->init_tio_pdu = false;
1614 	md->type = DM_TYPE_NONE;
1615 	mutex_init(&md->suspend_lock);
1616 	mutex_init(&md->type_lock);
1617 	mutex_init(&md->table_devices_lock);
1618 	spin_lock_init(&md->deferred_lock);
1619 	atomic_set(&md->holders, 1);
1620 	atomic_set(&md->open_count, 0);
1621 	atomic_set(&md->event_nr, 0);
1622 	atomic_set(&md->uevent_seq, 0);
1623 	INIT_LIST_HEAD(&md->uevent_list);
1624 	INIT_LIST_HEAD(&md->table_devices);
1625 	spin_lock_init(&md->uevent_lock);
1626 
1627 	/*
1628 	 * default to bio-based until DM table is loaded and md->type
1629 	 * established. If request-based table is loaded: blk-mq will
1630 	 * override accordingly.
1631 	 */
1632 	md->disk = blk_alloc_disk(md->numa_node_id);
1633 	if (!md->disk)
1634 		goto bad;
1635 	md->queue = md->disk->queue;
1636 
1637 	init_waitqueue_head(&md->wait);
1638 	INIT_WORK(&md->work, dm_wq_work);
1639 	init_waitqueue_head(&md->eventq);
1640 	init_completion(&md->kobj_holder.completion);
1641 
1642 	md->swap_bios = get_swap_bios();
1643 	sema_init(&md->swap_bios_semaphore, md->swap_bios);
1644 	mutex_init(&md->swap_bios_lock);
1645 
1646 	md->disk->major = _major;
1647 	md->disk->first_minor = minor;
1648 	md->disk->minors = 1;
1649 	md->disk->flags |= GENHD_FL_NO_PART;
1650 	md->disk->fops = &dm_blk_dops;
1651 	md->disk->queue = md->queue;
1652 	md->disk->private_data = md;
1653 	sprintf(md->disk->disk_name, "dm-%d", minor);
1654 
1655 	if (IS_ENABLED(CONFIG_FS_DAX)) {
1656 		md->dax_dev = alloc_dax(md, &dm_dax_ops);
1657 		if (IS_ERR(md->dax_dev)) {
1658 			md->dax_dev = NULL;
1659 			goto bad;
1660 		}
1661 		set_dax_nocache(md->dax_dev);
1662 		set_dax_nomc(md->dax_dev);
1663 		if (dax_add_host(md->dax_dev, md->disk))
1664 			goto bad;
1665 	}
1666 
1667 	format_dev_t(md->name, MKDEV(_major, minor));
1668 
1669 	md->wq = alloc_workqueue("kdmflush/%s", WQ_MEM_RECLAIM, 0, md->name);
1670 	if (!md->wq)
1671 		goto bad;
1672 
1673 	md->pending_io = alloc_percpu(unsigned long);
1674 	if (!md->pending_io)
1675 		goto bad;
1676 
1677 	dm_stats_init(&md->stats);
1678 
1679 	/* Populate the mapping, nobody knows we exist yet */
1680 	spin_lock(&_minor_lock);
1681 	old_md = idr_replace(&_minor_idr, md, minor);
1682 	spin_unlock(&_minor_lock);
1683 
1684 	BUG_ON(old_md != MINOR_ALLOCED);
1685 
1686 	return md;
1687 
1688 bad:
1689 	cleanup_mapped_device(md);
1690 bad_io_barrier:
1691 	free_minor(minor);
1692 bad_minor:
1693 	module_put(THIS_MODULE);
1694 bad_module_get:
1695 	kvfree(md);
1696 	return NULL;
1697 }
1698 
1699 static void unlock_fs(struct mapped_device *md);
1700 
1701 static void free_dev(struct mapped_device *md)
1702 {
1703 	int minor = MINOR(disk_devt(md->disk));
1704 
1705 	unlock_fs(md);
1706 
1707 	cleanup_mapped_device(md);
1708 
1709 	free_table_devices(&md->table_devices);
1710 	dm_stats_cleanup(&md->stats);
1711 	free_minor(minor);
1712 
1713 	module_put(THIS_MODULE);
1714 	kvfree(md);
1715 }
1716 
1717 static int __bind_mempools(struct mapped_device *md, struct dm_table *t)
1718 {
1719 	struct dm_md_mempools *p = dm_table_get_md_mempools(t);
1720 	int ret = 0;
1721 
1722 	if (dm_table_bio_based(t)) {
1723 		/*
1724 		 * The md may already have mempools that need changing.
1725 		 * If so, reload bioset because front_pad may have changed
1726 		 * because a different table was loaded.
1727 		 */
1728 		bioset_exit(&md->bs);
1729 		bioset_exit(&md->io_bs);
1730 
1731 	} else if (bioset_initialized(&md->bs)) {
1732 		/*
1733 		 * There's no need to reload with request-based dm
1734 		 * because the size of front_pad doesn't change.
1735 		 * Note for future: If you are to reload bioset,
1736 		 * prep-ed requests in the queue may refer
1737 		 * to bio from the old bioset, so you must walk
1738 		 * through the queue to unprep.
1739 		 */
1740 		goto out;
1741 	}
1742 
1743 	BUG_ON(!p ||
1744 	       bioset_initialized(&md->bs) ||
1745 	       bioset_initialized(&md->io_bs));
1746 
1747 	ret = bioset_init_from_src(&md->bs, &p->bs);
1748 	if (ret)
1749 		goto out;
1750 	ret = bioset_init_from_src(&md->io_bs, &p->io_bs);
1751 	if (ret)
1752 		bioset_exit(&md->bs);
1753 out:
1754 	/* mempool bind completed, no longer need any mempools in the table */
1755 	dm_table_free_md_mempools(t);
1756 	return ret;
1757 }
1758 
1759 /*
1760  * Bind a table to the device.
1761  */
1762 static void event_callback(void *context)
1763 {
1764 	unsigned long flags;
1765 	LIST_HEAD(uevents);
1766 	struct mapped_device *md = (struct mapped_device *) context;
1767 
1768 	spin_lock_irqsave(&md->uevent_lock, flags);
1769 	list_splice_init(&md->uevent_list, &uevents);
1770 	spin_unlock_irqrestore(&md->uevent_lock, flags);
1771 
1772 	dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
1773 
1774 	atomic_inc(&md->event_nr);
1775 	wake_up(&md->eventq);
1776 	dm_issue_global_event();
1777 }
1778 
1779 /*
1780  * Returns old map, which caller must destroy.
1781  */
1782 static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
1783 			       struct queue_limits *limits)
1784 {
1785 	struct dm_table *old_map;
1786 	struct request_queue *q = md->queue;
1787 	bool request_based = dm_table_request_based(t);
1788 	sector_t size;
1789 	int ret;
1790 
1791 	lockdep_assert_held(&md->suspend_lock);
1792 
1793 	size = dm_table_get_size(t);
1794 
1795 	/*
1796 	 * Wipe any geometry if the size of the table changed.
1797 	 */
1798 	if (size != dm_get_size(md))
1799 		memset(&md->geometry, 0, sizeof(md->geometry));
1800 
1801 	if (!get_capacity(md->disk))
1802 		set_capacity(md->disk, size);
1803 	else
1804 		set_capacity_and_notify(md->disk, size);
1805 
1806 	dm_table_event_callback(t, event_callback, md);
1807 
1808 	if (request_based) {
1809 		/*
1810 		 * Leverage the fact that request-based DM targets are
1811 		 * immutable singletons - used to optimize dm_mq_queue_rq.
1812 		 */
1813 		md->immutable_target = dm_table_get_immutable_target(t);
1814 	}
1815 
1816 	ret = __bind_mempools(md, t);
1817 	if (ret) {
1818 		old_map = ERR_PTR(ret);
1819 		goto out;
1820 	}
1821 
1822 	ret = dm_table_set_restrictions(t, q, limits);
1823 	if (ret) {
1824 		old_map = ERR_PTR(ret);
1825 		goto out;
1826 	}
1827 
1828 	old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
1829 	rcu_assign_pointer(md->map, (void *)t);
1830 	md->immutable_target_type = dm_table_get_immutable_target_type(t);
1831 
1832 	if (old_map)
1833 		dm_sync_table(md);
1834 
1835 out:
1836 	return old_map;
1837 }
1838 
1839 /*
1840  * Returns unbound table for the caller to free.
1841  */
1842 static struct dm_table *__unbind(struct mapped_device *md)
1843 {
1844 	struct dm_table *map = rcu_dereference_protected(md->map, 1);
1845 
1846 	if (!map)
1847 		return NULL;
1848 
1849 	dm_table_event_callback(map, NULL, NULL);
1850 	RCU_INIT_POINTER(md->map, NULL);
1851 	dm_sync_table(md);
1852 
1853 	return map;
1854 }
1855 
1856 /*
1857  * Constructor for a new device.
1858  */
1859 int dm_create(int minor, struct mapped_device **result)
1860 {
1861 	struct mapped_device *md;
1862 
1863 	md = alloc_dev(minor);
1864 	if (!md)
1865 		return -ENXIO;
1866 
1867 	dm_ima_reset_data(md);
1868 
1869 	*result = md;
1870 	return 0;
1871 }
1872 
1873 /*
1874  * Functions to manage md->type.
1875  * All are required to hold md->type_lock.
1876  */
1877 void dm_lock_md_type(struct mapped_device *md)
1878 {
1879 	mutex_lock(&md->type_lock);
1880 }
1881 
1882 void dm_unlock_md_type(struct mapped_device *md)
1883 {
1884 	mutex_unlock(&md->type_lock);
1885 }
1886 
1887 void dm_set_md_type(struct mapped_device *md, enum dm_queue_mode type)
1888 {
1889 	BUG_ON(!mutex_is_locked(&md->type_lock));
1890 	md->type = type;
1891 }
1892 
1893 enum dm_queue_mode dm_get_md_type(struct mapped_device *md)
1894 {
1895 	return md->type;
1896 }
1897 
1898 struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
1899 {
1900 	return md->immutable_target_type;
1901 }
1902 
1903 /*
1904  * The queue_limits are only valid as long as you have a reference
1905  * count on 'md'.
1906  */
1907 struct queue_limits *dm_get_queue_limits(struct mapped_device *md)
1908 {
1909 	BUG_ON(!atomic_read(&md->holders));
1910 	return &md->queue->limits;
1911 }
1912 EXPORT_SYMBOL_GPL(dm_get_queue_limits);
1913 
1914 /*
1915  * Setup the DM device's queue based on md's type
1916  */
1917 int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
1918 {
1919 	enum dm_queue_mode type = dm_table_get_type(t);
1920 	struct queue_limits limits;
1921 	int r;
1922 
1923 	switch (type) {
1924 	case DM_TYPE_REQUEST_BASED:
1925 		md->disk->fops = &dm_rq_blk_dops;
1926 		r = dm_mq_init_request_queue(md, t);
1927 		if (r) {
1928 			DMERR("Cannot initialize queue for request-based dm mapped device");
1929 			return r;
1930 		}
1931 		break;
1932 	case DM_TYPE_BIO_BASED:
1933 	case DM_TYPE_DAX_BIO_BASED:
1934 		break;
1935 	case DM_TYPE_NONE:
1936 		WARN_ON_ONCE(true);
1937 		break;
1938 	}
1939 
1940 	r = dm_calculate_queue_limits(t, &limits);
1941 	if (r) {
1942 		DMERR("Cannot calculate initial queue limits");
1943 		return r;
1944 	}
1945 	r = dm_table_set_restrictions(t, md->queue, &limits);
1946 	if (r)
1947 		return r;
1948 
1949 	r = add_disk(md->disk);
1950 	if (r)
1951 		return r;
1952 
1953 	r = dm_sysfs_init(md);
1954 	if (r) {
1955 		del_gendisk(md->disk);
1956 		return r;
1957 	}
1958 	md->type = type;
1959 	return 0;
1960 }
1961 
1962 struct mapped_device *dm_get_md(dev_t dev)
1963 {
1964 	struct mapped_device *md;
1965 	unsigned minor = MINOR(dev);
1966 
1967 	if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
1968 		return NULL;
1969 
1970 	spin_lock(&_minor_lock);
1971 
1972 	md = idr_find(&_minor_idr, minor);
1973 	if (!md || md == MINOR_ALLOCED || (MINOR(disk_devt(dm_disk(md))) != minor) ||
1974 	    test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
1975 		md = NULL;
1976 		goto out;
1977 	}
1978 	dm_get(md);
1979 out:
1980 	spin_unlock(&_minor_lock);
1981 
1982 	return md;
1983 }
1984 EXPORT_SYMBOL_GPL(dm_get_md);
1985 
1986 void *dm_get_mdptr(struct mapped_device *md)
1987 {
1988 	return md->interface_ptr;
1989 }
1990 
1991 void dm_set_mdptr(struct mapped_device *md, void *ptr)
1992 {
1993 	md->interface_ptr = ptr;
1994 }
1995 
1996 void dm_get(struct mapped_device *md)
1997 {
1998 	atomic_inc(&md->holders);
1999 	BUG_ON(test_bit(DMF_FREEING, &md->flags));
2000 }
2001 
2002 int dm_hold(struct mapped_device *md)
2003 {
2004 	spin_lock(&_minor_lock);
2005 	if (test_bit(DMF_FREEING, &md->flags)) {
2006 		spin_unlock(&_minor_lock);
2007 		return -EBUSY;
2008 	}
2009 	dm_get(md);
2010 	spin_unlock(&_minor_lock);
2011 	return 0;
2012 }
2013 EXPORT_SYMBOL_GPL(dm_hold);
2014 
2015 const char *dm_device_name(struct mapped_device *md)
2016 {
2017 	return md->name;
2018 }
2019 EXPORT_SYMBOL_GPL(dm_device_name);
2020 
2021 static void __dm_destroy(struct mapped_device *md, bool wait)
2022 {
2023 	struct dm_table *map;
2024 	int srcu_idx;
2025 
2026 	might_sleep();
2027 
2028 	spin_lock(&_minor_lock);
2029 	idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));
2030 	set_bit(DMF_FREEING, &md->flags);
2031 	spin_unlock(&_minor_lock);
2032 
2033 	blk_set_queue_dying(md->queue);
2034 
2035 	/*
2036 	 * Take suspend_lock so that presuspend and postsuspend methods
2037 	 * do not race with internal suspend.
2038 	 */
2039 	mutex_lock(&md->suspend_lock);
2040 	map = dm_get_live_table(md, &srcu_idx);
2041 	if (!dm_suspended_md(md)) {
2042 		dm_table_presuspend_targets(map);
2043 		set_bit(DMF_SUSPENDED, &md->flags);
2044 		set_bit(DMF_POST_SUSPENDING, &md->flags);
2045 		dm_table_postsuspend_targets(map);
2046 	}
2047 	/* dm_put_live_table must be before msleep, otherwise deadlock is possible */
2048 	dm_put_live_table(md, srcu_idx);
2049 	mutex_unlock(&md->suspend_lock);
2050 
2051 	/*
2052 	 * Rare, but there may be I/O requests still going to complete,
2053 	 * for example.  Wait for all references to disappear.
2054 	 * No one should increment the reference count of the mapped_device,
2055 	 * after the mapped_device state becomes DMF_FREEING.
2056 	 */
2057 	if (wait)
2058 		while (atomic_read(&md->holders))
2059 			msleep(1);
2060 	else if (atomic_read(&md->holders))
2061 		DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)",
2062 		       dm_device_name(md), atomic_read(&md->holders));
2063 
2064 	dm_table_destroy(__unbind(md));
2065 	free_dev(md);
2066 }
2067 
2068 void dm_destroy(struct mapped_device *md)
2069 {
2070 	__dm_destroy(md, true);
2071 }
2072 
2073 void dm_destroy_immediate(struct mapped_device *md)
2074 {
2075 	__dm_destroy(md, false);
2076 }
2077 
2078 void dm_put(struct mapped_device *md)
2079 {
2080 	atomic_dec(&md->holders);
2081 }
2082 EXPORT_SYMBOL_GPL(dm_put);
2083 
2084 static bool dm_in_flight_bios(struct mapped_device *md)
2085 {
2086 	int cpu;
2087 	unsigned long sum = 0;
2088 
2089 	for_each_possible_cpu(cpu)
2090 		sum += *per_cpu_ptr(md->pending_io, cpu);
2091 
2092 	return sum != 0;
2093 }
2094 
2095 static int dm_wait_for_bios_completion(struct mapped_device *md, unsigned int task_state)
2096 {
2097 	int r = 0;
2098 	DEFINE_WAIT(wait);
2099 
2100 	while (true) {
2101 		prepare_to_wait(&md->wait, &wait, task_state);
2102 
2103 		if (!dm_in_flight_bios(md))
2104 			break;
2105 
2106 		if (signal_pending_state(task_state, current)) {
2107 			r = -EINTR;
2108 			break;
2109 		}
2110 
2111 		io_schedule();
2112 	}
2113 	finish_wait(&md->wait, &wait);
2114 
2115 	smp_rmb();
2116 
2117 	return r;
2118 }
2119 
2120 static int dm_wait_for_completion(struct mapped_device *md, unsigned int task_state)
2121 {
2122 	int r = 0;
2123 
2124 	if (!queue_is_mq(md->queue))
2125 		return dm_wait_for_bios_completion(md, task_state);
2126 
2127 	while (true) {
2128 		if (!blk_mq_queue_inflight(md->queue))
2129 			break;
2130 
2131 		if (signal_pending_state(task_state, current)) {
2132 			r = -EINTR;
2133 			break;
2134 		}
2135 
2136 		msleep(5);
2137 	}
2138 
2139 	return r;
2140 }
2141 
2142 /*
2143  * Process the deferred bios
2144  */
2145 static void dm_wq_work(struct work_struct *work)
2146 {
2147 	struct mapped_device *md = container_of(work, struct mapped_device, work);
2148 	struct bio *bio;
2149 
2150 	while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
2151 		spin_lock_irq(&md->deferred_lock);
2152 		bio = bio_list_pop(&md->deferred);
2153 		spin_unlock_irq(&md->deferred_lock);
2154 
2155 		if (!bio)
2156 			break;
2157 
2158 		submit_bio_noacct(bio);
2159 	}
2160 }
2161 
2162 static void dm_queue_flush(struct mapped_device *md)
2163 {
2164 	clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2165 	smp_mb__after_atomic();
2166 	queue_work(md->wq, &md->work);
2167 }
2168 
2169 /*
2170  * Swap in a new table, returning the old one for the caller to destroy.
2171  */
2172 struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table)
2173 {
2174 	struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL);
2175 	struct queue_limits limits;
2176 	int r;
2177 
2178 	mutex_lock(&md->suspend_lock);
2179 
2180 	/* device must be suspended */
2181 	if (!dm_suspended_md(md))
2182 		goto out;
2183 
2184 	/*
2185 	 * If the new table has no data devices, retain the existing limits.
2186 	 * This helps multipath with queue_if_no_path if all paths disappear,
2187 	 * then new I/O is queued based on these limits, and then some paths
2188 	 * reappear.
2189 	 */
2190 	if (dm_table_has_no_data_devices(table)) {
2191 		live_map = dm_get_live_table_fast(md);
2192 		if (live_map)
2193 			limits = md->queue->limits;
2194 		dm_put_live_table_fast(md);
2195 	}
2196 
2197 	if (!live_map) {
2198 		r = dm_calculate_queue_limits(table, &limits);
2199 		if (r) {
2200 			map = ERR_PTR(r);
2201 			goto out;
2202 		}
2203 	}
2204 
2205 	map = __bind(md, table, &limits);
2206 	dm_issue_global_event();
2207 
2208 out:
2209 	mutex_unlock(&md->suspend_lock);
2210 	return map;
2211 }
2212 
2213 /*
2214  * Functions to lock and unlock any filesystem running on the
2215  * device.
2216  */
2217 static int lock_fs(struct mapped_device *md)
2218 {
2219 	int r;
2220 
2221 	WARN_ON(test_bit(DMF_FROZEN, &md->flags));
2222 
2223 	r = freeze_bdev(md->disk->part0);
2224 	if (!r)
2225 		set_bit(DMF_FROZEN, &md->flags);
2226 	return r;
2227 }
2228 
2229 static void unlock_fs(struct mapped_device *md)
2230 {
2231 	if (!test_bit(DMF_FROZEN, &md->flags))
2232 		return;
2233 	thaw_bdev(md->disk->part0);
2234 	clear_bit(DMF_FROZEN, &md->flags);
2235 }
2236 
2237 /*
2238  * @suspend_flags: DM_SUSPEND_LOCKFS_FLAG and/or DM_SUSPEND_NOFLUSH_FLAG
2239  * @task_state: e.g. TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE
2240  * @dmf_suspended_flag: DMF_SUSPENDED or DMF_SUSPENDED_INTERNALLY
2241  *
2242  * If __dm_suspend returns 0, the device is completely quiescent
2243  * now. There is no request-processing activity. All new requests
2244  * are being added to md->deferred list.
2245  */
2246 static int __dm_suspend(struct mapped_device *md, struct dm_table *map,
2247 			unsigned suspend_flags, unsigned int task_state,
2248 			int dmf_suspended_flag)
2249 {
2250 	bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG;
2251 	bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG;
2252 	int r;
2253 
2254 	lockdep_assert_held(&md->suspend_lock);
2255 
2256 	/*
2257 	 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
2258 	 * This flag is cleared before dm_suspend returns.
2259 	 */
2260 	if (noflush)
2261 		set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2262 	else
2263 		DMDEBUG("%s: suspending with flush", dm_device_name(md));
2264 
2265 	/*
2266 	 * This gets reverted if there's an error later and the targets
2267 	 * provide the .presuspend_undo hook.
2268 	 */
2269 	dm_table_presuspend_targets(map);
2270 
2271 	/*
2272 	 * Flush I/O to the device.
2273 	 * Any I/O submitted after lock_fs() may not be flushed.
2274 	 * noflush takes precedence over do_lockfs.
2275 	 * (lock_fs() flushes I/Os and waits for them to complete.)
2276 	 */
2277 	if (!noflush && do_lockfs) {
2278 		r = lock_fs(md);
2279 		if (r) {
2280 			dm_table_presuspend_undo_targets(map);
2281 			return r;
2282 		}
2283 	}
2284 
2285 	/*
2286 	 * Here we must make sure that no processes are submitting requests
2287 	 * to target drivers i.e. no one may be executing
2288 	 * dm_split_and_process_bio from dm_submit_bio.
2289 	 *
2290 	 * To get all processes out of dm_split_and_process_bio in dm_submit_bio,
2291 	 * we take the write lock. To prevent any process from reentering
2292 	 * dm_split_and_process_bio from dm_submit_bio and quiesce the thread
2293 	 * (dm_wq_work), we set DMF_BLOCK_IO_FOR_SUSPEND and call
2294 	 * flush_workqueue(md->wq).
2295 	 */
2296 	set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2297 	if (map)
2298 		synchronize_srcu(&md->io_barrier);
2299 
2300 	/*
2301 	 * Stop md->queue before flushing md->wq in case request-based
2302 	 * dm defers requests to md->wq from md->queue.
2303 	 */
2304 	if (dm_request_based(md))
2305 		dm_stop_queue(md->queue);
2306 
2307 	flush_workqueue(md->wq);
2308 
2309 	/*
2310 	 * At this point no more requests are entering target request routines.
2311 	 * We call dm_wait_for_completion to wait for all existing requests
2312 	 * to finish.
2313 	 */
2314 	r = dm_wait_for_completion(md, task_state);
2315 	if (!r)
2316 		set_bit(dmf_suspended_flag, &md->flags);
2317 
2318 	if (noflush)
2319 		clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2320 	if (map)
2321 		synchronize_srcu(&md->io_barrier);
2322 
2323 	/* were we interrupted ? */
2324 	if (r < 0) {
2325 		dm_queue_flush(md);
2326 
2327 		if (dm_request_based(md))
2328 			dm_start_queue(md->queue);
2329 
2330 		unlock_fs(md);
2331 		dm_table_presuspend_undo_targets(map);
2332 		/* pushback list is already flushed, so skip flush */
2333 	}
2334 
2335 	return r;
2336 }
2337 
2338 /*
2339  * We need to be able to change a mapping table under a mounted
2340  * filesystem.  For example we might want to move some data in
2341  * the background.  Before the table can be swapped with
2342  * dm_bind_table, dm_suspend must be called to flush any in
2343  * flight bios and ensure that any further io gets deferred.
2344  */
2345 /*
2346  * Suspend mechanism in request-based dm.
2347  *
2348  * 1. Flush all I/Os by lock_fs() if needed.
2349  * 2. Stop dispatching any I/O by stopping the request_queue.
2350  * 3. Wait for all in-flight I/Os to be completed or requeued.
2351  *
2352  * To abort suspend, start the request_queue.
2353  */
2354 int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
2355 {
2356 	struct dm_table *map = NULL;
2357 	int r = 0;
2358 
2359 retry:
2360 	mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2361 
2362 	if (dm_suspended_md(md)) {
2363 		r = -EINVAL;
2364 		goto out_unlock;
2365 	}
2366 
2367 	if (dm_suspended_internally_md(md)) {
2368 		/* already internally suspended, wait for internal resume */
2369 		mutex_unlock(&md->suspend_lock);
2370 		r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2371 		if (r)
2372 			return r;
2373 		goto retry;
2374 	}
2375 
2376 	map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2377 
2378 	r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED);
2379 	if (r)
2380 		goto out_unlock;
2381 
2382 	set_bit(DMF_POST_SUSPENDING, &md->flags);
2383 	dm_table_postsuspend_targets(map);
2384 	clear_bit(DMF_POST_SUSPENDING, &md->flags);
2385 
2386 out_unlock:
2387 	mutex_unlock(&md->suspend_lock);
2388 	return r;
2389 }
2390 
2391 static int __dm_resume(struct mapped_device *md, struct dm_table *map)
2392 {
2393 	if (map) {
2394 		int r = dm_table_resume_targets(map);
2395 		if (r)
2396 			return r;
2397 	}
2398 
2399 	dm_queue_flush(md);
2400 
2401 	/*
2402 	 * Flushing deferred I/Os must be done after targets are resumed
2403 	 * so that mapping of targets can work correctly.
2404 	 * Request-based dm is queueing the deferred I/Os in its request_queue.
2405 	 */
2406 	if (dm_request_based(md))
2407 		dm_start_queue(md->queue);
2408 
2409 	unlock_fs(md);
2410 
2411 	return 0;
2412 }
2413 
2414 int dm_resume(struct mapped_device *md)
2415 {
2416 	int r;
2417 	struct dm_table *map = NULL;
2418 
2419 retry:
2420 	r = -EINVAL;
2421 	mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2422 
2423 	if (!dm_suspended_md(md))
2424 		goto out;
2425 
2426 	if (dm_suspended_internally_md(md)) {
2427 		/* already internally suspended, wait for internal resume */
2428 		mutex_unlock(&md->suspend_lock);
2429 		r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2430 		if (r)
2431 			return r;
2432 		goto retry;
2433 	}
2434 
2435 	map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2436 	if (!map || !dm_table_get_size(map))
2437 		goto out;
2438 
2439 	r = __dm_resume(md, map);
2440 	if (r)
2441 		goto out;
2442 
2443 	clear_bit(DMF_SUSPENDED, &md->flags);
2444 out:
2445 	mutex_unlock(&md->suspend_lock);
2446 
2447 	return r;
2448 }
2449 
2450 /*
2451  * Internal suspend/resume works like userspace-driven suspend. It waits
2452  * until all bios finish and prevents issuing new bios to the target drivers.
2453  * It may be used only from the kernel.
2454  */
2455 
2456 static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags)
2457 {
2458 	struct dm_table *map = NULL;
2459 
2460 	lockdep_assert_held(&md->suspend_lock);
2461 
2462 	if (md->internal_suspend_count++)
2463 		return; /* nested internal suspend */
2464 
2465 	if (dm_suspended_md(md)) {
2466 		set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2467 		return; /* nest suspend */
2468 	}
2469 
2470 	map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2471 
2472 	/*
2473 	 * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is
2474 	 * supported.  Properly supporting a TASK_INTERRUPTIBLE internal suspend
2475 	 * would require changing .presuspend to return an error -- avoid this
2476 	 * until there is a need for more elaborate variants of internal suspend.
2477 	 */
2478 	(void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE,
2479 			    DMF_SUSPENDED_INTERNALLY);
2480 
2481 	set_bit(DMF_POST_SUSPENDING, &md->flags);
2482 	dm_table_postsuspend_targets(map);
2483 	clear_bit(DMF_POST_SUSPENDING, &md->flags);
2484 }
2485 
2486 static void __dm_internal_resume(struct mapped_device *md)
2487 {
2488 	BUG_ON(!md->internal_suspend_count);
2489 
2490 	if (--md->internal_suspend_count)
2491 		return; /* resume from nested internal suspend */
2492 
2493 	if (dm_suspended_md(md))
2494 		goto done; /* resume from nested suspend */
2495 
2496 	/*
2497 	 * NOTE: existing callers don't need to call dm_table_resume_targets
2498 	 * (which may fail -- so best to avoid it for now by passing NULL map)
2499 	 */
2500 	(void) __dm_resume(md, NULL);
2501 
2502 done:
2503 	clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2504 	smp_mb__after_atomic();
2505 	wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY);
2506 }
2507 
2508 void dm_internal_suspend_noflush(struct mapped_device *md)
2509 {
2510 	mutex_lock(&md->suspend_lock);
2511 	__dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG);
2512 	mutex_unlock(&md->suspend_lock);
2513 }
2514 EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush);
2515 
2516 void dm_internal_resume(struct mapped_device *md)
2517 {
2518 	mutex_lock(&md->suspend_lock);
2519 	__dm_internal_resume(md);
2520 	mutex_unlock(&md->suspend_lock);
2521 }
2522 EXPORT_SYMBOL_GPL(dm_internal_resume);
2523 
2524 /*
2525  * Fast variants of internal suspend/resume hold md->suspend_lock,
2526  * which prevents interaction with userspace-driven suspend.
2527  */
2528 
2529 void dm_internal_suspend_fast(struct mapped_device *md)
2530 {
2531 	mutex_lock(&md->suspend_lock);
2532 	if (dm_suspended_md(md) || dm_suspended_internally_md(md))
2533 		return;
2534 
2535 	set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2536 	synchronize_srcu(&md->io_barrier);
2537 	flush_workqueue(md->wq);
2538 	dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
2539 }
2540 EXPORT_SYMBOL_GPL(dm_internal_suspend_fast);
2541 
2542 void dm_internal_resume_fast(struct mapped_device *md)
2543 {
2544 	if (dm_suspended_md(md) || dm_suspended_internally_md(md))
2545 		goto done;
2546 
2547 	dm_queue_flush(md);
2548 
2549 done:
2550 	mutex_unlock(&md->suspend_lock);
2551 }
2552 EXPORT_SYMBOL_GPL(dm_internal_resume_fast);
2553 
2554 /*-----------------------------------------------------------------
2555  * Event notification.
2556  *---------------------------------------------------------------*/
2557 int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action,
2558 		       unsigned cookie)
2559 {
2560 	int r;
2561 	unsigned noio_flag;
2562 	char udev_cookie[DM_COOKIE_LENGTH];
2563 	char *envp[] = { udev_cookie, NULL };
2564 
2565 	noio_flag = memalloc_noio_save();
2566 
2567 	if (!cookie)
2568 		r = kobject_uevent(&disk_to_dev(md->disk)->kobj, action);
2569 	else {
2570 		snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u",
2571 			 DM_COOKIE_ENV_VAR_NAME, cookie);
2572 		r = kobject_uevent_env(&disk_to_dev(md->disk)->kobj,
2573 				       action, envp);
2574 	}
2575 
2576 	memalloc_noio_restore(noio_flag);
2577 
2578 	return r;
2579 }
2580 
2581 uint32_t dm_next_uevent_seq(struct mapped_device *md)
2582 {
2583 	return atomic_add_return(1, &md->uevent_seq);
2584 }
2585 
2586 uint32_t dm_get_event_nr(struct mapped_device *md)
2587 {
2588 	return atomic_read(&md->event_nr);
2589 }
2590 
2591 int dm_wait_event(struct mapped_device *md, int event_nr)
2592 {
2593 	return wait_event_interruptible(md->eventq,
2594 			(event_nr != atomic_read(&md->event_nr)));
2595 }
2596 
2597 void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
2598 {
2599 	unsigned long flags;
2600 
2601 	spin_lock_irqsave(&md->uevent_lock, flags);
2602 	list_add(elist, &md->uevent_list);
2603 	spin_unlock_irqrestore(&md->uevent_lock, flags);
2604 }
2605 
2606 /*
2607  * The gendisk is only valid as long as you have a reference
2608  * count on 'md'.
2609  */
2610 struct gendisk *dm_disk(struct mapped_device *md)
2611 {
2612 	return md->disk;
2613 }
2614 EXPORT_SYMBOL_GPL(dm_disk);
2615 
2616 struct kobject *dm_kobject(struct mapped_device *md)
2617 {
2618 	return &md->kobj_holder.kobj;
2619 }
2620 
2621 struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
2622 {
2623 	struct mapped_device *md;
2624 
2625 	md = container_of(kobj, struct mapped_device, kobj_holder.kobj);
2626 
2627 	spin_lock(&_minor_lock);
2628 	if (test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
2629 		md = NULL;
2630 		goto out;
2631 	}
2632 	dm_get(md);
2633 out:
2634 	spin_unlock(&_minor_lock);
2635 
2636 	return md;
2637 }
2638 
2639 int dm_suspended_md(struct mapped_device *md)
2640 {
2641 	return test_bit(DMF_SUSPENDED, &md->flags);
2642 }
2643 
2644 static int dm_post_suspending_md(struct mapped_device *md)
2645 {
2646 	return test_bit(DMF_POST_SUSPENDING, &md->flags);
2647 }
2648 
2649 int dm_suspended_internally_md(struct mapped_device *md)
2650 {
2651 	return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2652 }
2653 
2654 int dm_test_deferred_remove_flag(struct mapped_device *md)
2655 {
2656 	return test_bit(DMF_DEFERRED_REMOVE, &md->flags);
2657 }
2658 
2659 int dm_suspended(struct dm_target *ti)
2660 {
2661 	return dm_suspended_md(ti->table->md);
2662 }
2663 EXPORT_SYMBOL_GPL(dm_suspended);
2664 
2665 int dm_post_suspending(struct dm_target *ti)
2666 {
2667 	return dm_post_suspending_md(ti->table->md);
2668 }
2669 EXPORT_SYMBOL_GPL(dm_post_suspending);
2670 
2671 int dm_noflush_suspending(struct dm_target *ti)
2672 {
2673 	return __noflush_suspending(ti->table->md);
2674 }
2675 EXPORT_SYMBOL_GPL(dm_noflush_suspending);
2676 
2677 struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_queue_mode type,
2678 					    unsigned integrity, unsigned per_io_data_size,
2679 					    unsigned min_pool_size)
2680 {
2681 	struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id);
2682 	unsigned int pool_size = 0;
2683 	unsigned int front_pad, io_front_pad;
2684 	int ret;
2685 
2686 	if (!pools)
2687 		return NULL;
2688 
2689 	switch (type) {
2690 	case DM_TYPE_BIO_BASED:
2691 	case DM_TYPE_DAX_BIO_BASED:
2692 		pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size);
2693 		front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + DM_TARGET_IO_BIO_OFFSET;
2694 		io_front_pad = roundup(per_io_data_size,  __alignof__(struct dm_io)) + DM_IO_BIO_OFFSET;
2695 		ret = bioset_init(&pools->io_bs, pool_size, io_front_pad, 0);
2696 		if (ret)
2697 			goto out;
2698 		if (integrity && bioset_integrity_create(&pools->io_bs, pool_size))
2699 			goto out;
2700 		break;
2701 	case DM_TYPE_REQUEST_BASED:
2702 		pool_size = max(dm_get_reserved_rq_based_ios(), min_pool_size);
2703 		front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
2704 		/* per_io_data_size is used for blk-mq pdu at queue allocation */
2705 		break;
2706 	default:
2707 		BUG();
2708 	}
2709 
2710 	ret = bioset_init(&pools->bs, pool_size, front_pad, 0);
2711 	if (ret)
2712 		goto out;
2713 
2714 	if (integrity && bioset_integrity_create(&pools->bs, pool_size))
2715 		goto out;
2716 
2717 	return pools;
2718 
2719 out:
2720 	dm_free_md_mempools(pools);
2721 
2722 	return NULL;
2723 }
2724 
2725 void dm_free_md_mempools(struct dm_md_mempools *pools)
2726 {
2727 	if (!pools)
2728 		return;
2729 
2730 	bioset_exit(&pools->bs);
2731 	bioset_exit(&pools->io_bs);
2732 
2733 	kfree(pools);
2734 }
2735 
2736 struct dm_pr {
2737 	u64	old_key;
2738 	u64	new_key;
2739 	u32	flags;
2740 	bool	fail_early;
2741 };
2742 
2743 static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn,
2744 		      void *data)
2745 {
2746 	struct mapped_device *md = bdev->bd_disk->private_data;
2747 	struct dm_table *table;
2748 	struct dm_target *ti;
2749 	int ret = -ENOTTY, srcu_idx;
2750 
2751 	table = dm_get_live_table(md, &srcu_idx);
2752 	if (!table || !dm_table_get_size(table))
2753 		goto out;
2754 
2755 	/* We only support devices that have a single target */
2756 	if (dm_table_get_num_targets(table) != 1)
2757 		goto out;
2758 	ti = dm_table_get_target(table, 0);
2759 
2760 	ret = -EINVAL;
2761 	if (!ti->type->iterate_devices)
2762 		goto out;
2763 
2764 	ret = ti->type->iterate_devices(ti, fn, data);
2765 out:
2766 	dm_put_live_table(md, srcu_idx);
2767 	return ret;
2768 }
2769 
2770 /*
2771  * For register / unregister we need to manually call out to every path.
2772  */
2773 static int __dm_pr_register(struct dm_target *ti, struct dm_dev *dev,
2774 			    sector_t start, sector_t len, void *data)
2775 {
2776 	struct dm_pr *pr = data;
2777 	const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
2778 
2779 	if (!ops || !ops->pr_register)
2780 		return -EOPNOTSUPP;
2781 	return ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags);
2782 }
2783 
2784 static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key,
2785 			  u32 flags)
2786 {
2787 	struct dm_pr pr = {
2788 		.old_key	= old_key,
2789 		.new_key	= new_key,
2790 		.flags		= flags,
2791 		.fail_early	= true,
2792 	};
2793 	int ret;
2794 
2795 	ret = dm_call_pr(bdev, __dm_pr_register, &pr);
2796 	if (ret && new_key) {
2797 		/* unregister all paths if we failed to register any path */
2798 		pr.old_key = new_key;
2799 		pr.new_key = 0;
2800 		pr.flags = 0;
2801 		pr.fail_early = false;
2802 		dm_call_pr(bdev, __dm_pr_register, &pr);
2803 	}
2804 
2805 	return ret;
2806 }
2807 
2808 static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type,
2809 			 u32 flags)
2810 {
2811 	struct mapped_device *md = bdev->bd_disk->private_data;
2812 	const struct pr_ops *ops;
2813 	int r, srcu_idx;
2814 
2815 	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
2816 	if (r < 0)
2817 		goto out;
2818 
2819 	ops = bdev->bd_disk->fops->pr_ops;
2820 	if (ops && ops->pr_reserve)
2821 		r = ops->pr_reserve(bdev, key, type, flags);
2822 	else
2823 		r = -EOPNOTSUPP;
2824 out:
2825 	dm_unprepare_ioctl(md, srcu_idx);
2826 	return r;
2827 }
2828 
2829 static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
2830 {
2831 	struct mapped_device *md = bdev->bd_disk->private_data;
2832 	const struct pr_ops *ops;
2833 	int r, srcu_idx;
2834 
2835 	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
2836 	if (r < 0)
2837 		goto out;
2838 
2839 	ops = bdev->bd_disk->fops->pr_ops;
2840 	if (ops && ops->pr_release)
2841 		r = ops->pr_release(bdev, key, type);
2842 	else
2843 		r = -EOPNOTSUPP;
2844 out:
2845 	dm_unprepare_ioctl(md, srcu_idx);
2846 	return r;
2847 }
2848 
2849 static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key,
2850 			 enum pr_type type, bool abort)
2851 {
2852 	struct mapped_device *md = bdev->bd_disk->private_data;
2853 	const struct pr_ops *ops;
2854 	int r, srcu_idx;
2855 
2856 	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
2857 	if (r < 0)
2858 		goto out;
2859 
2860 	ops = bdev->bd_disk->fops->pr_ops;
2861 	if (ops && ops->pr_preempt)
2862 		r = ops->pr_preempt(bdev, old_key, new_key, type, abort);
2863 	else
2864 		r = -EOPNOTSUPP;
2865 out:
2866 	dm_unprepare_ioctl(md, srcu_idx);
2867 	return r;
2868 }
2869 
2870 static int dm_pr_clear(struct block_device *bdev, u64 key)
2871 {
2872 	struct mapped_device *md = bdev->bd_disk->private_data;
2873 	const struct pr_ops *ops;
2874 	int r, srcu_idx;
2875 
2876 	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
2877 	if (r < 0)
2878 		goto out;
2879 
2880 	ops = bdev->bd_disk->fops->pr_ops;
2881 	if (ops && ops->pr_clear)
2882 		r = ops->pr_clear(bdev, key);
2883 	else
2884 		r = -EOPNOTSUPP;
2885 out:
2886 	dm_unprepare_ioctl(md, srcu_idx);
2887 	return r;
2888 }
2889 
2890 static const struct pr_ops dm_pr_ops = {
2891 	.pr_register	= dm_pr_register,
2892 	.pr_reserve	= dm_pr_reserve,
2893 	.pr_release	= dm_pr_release,
2894 	.pr_preempt	= dm_pr_preempt,
2895 	.pr_clear	= dm_pr_clear,
2896 };
2897 
2898 static const struct block_device_operations dm_blk_dops = {
2899 	.submit_bio = dm_submit_bio,
2900 	.open = dm_blk_open,
2901 	.release = dm_blk_close,
2902 	.ioctl = dm_blk_ioctl,
2903 	.getgeo = dm_blk_getgeo,
2904 	.report_zones = dm_blk_report_zones,
2905 	.pr_ops = &dm_pr_ops,
2906 	.owner = THIS_MODULE
2907 };
2908 
2909 static const struct block_device_operations dm_rq_blk_dops = {
2910 	.open = dm_blk_open,
2911 	.release = dm_blk_close,
2912 	.ioctl = dm_blk_ioctl,
2913 	.getgeo = dm_blk_getgeo,
2914 	.pr_ops = &dm_pr_ops,
2915 	.owner = THIS_MODULE
2916 };
2917 
2918 static const struct dax_operations dm_dax_ops = {
2919 	.direct_access = dm_dax_direct_access,
2920 	.zero_page_range = dm_dax_zero_page_range,
2921 };
2922 
2923 /*
2924  * module hooks
2925  */
2926 module_init(dm_init);
2927 module_exit(dm_exit);
2928 
2929 module_param(major, uint, 0);
2930 MODULE_PARM_DESC(major, "The major number of the device mapper");
2931 
2932 module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR);
2933 MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools");
2934 
2935 module_param(dm_numa_node, int, S_IRUGO | S_IWUSR);
2936 MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations");
2937 
2938 module_param(swap_bios, int, S_IRUGO | S_IWUSR);
2939 MODULE_PARM_DESC(swap_bios, "Maximum allowed inflight swap IOs");
2940 
2941 MODULE_DESCRIPTION(DM_NAME " driver");
2942 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
2943 MODULE_LICENSE("GPL");
2944