xref: /openbmc/linux/drivers/md/dm.c (revision ff4a7481c3898ffc3cc271d6aca431d190c37247)
1 /*
2  * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3  * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
4  *
5  * This file is released under the GPL.
6  */
7 
8 #include "dm-core.h"
9 #include "dm-rq.h"
10 #include "dm-uevent.h"
11 
12 #include <linux/init.h>
13 #include <linux/module.h>
14 #include <linux/mutex.h>
15 #include <linux/sched/signal.h>
16 #include <linux/blkpg.h>
17 #include <linux/bio.h>
18 #include <linux/mempool.h>
19 #include <linux/dax.h>
20 #include <linux/slab.h>
21 #include <linux/idr.h>
22 #include <linux/uio.h>
23 #include <linux/hdreg.h>
24 #include <linux/delay.h>
25 #include <linux/wait.h>
26 #include <linux/pr.h>
27 #include <linux/refcount.h>
28 
29 #define DM_MSG_PREFIX "core"
30 
31 /*
32  * Cookies are numeric values sent with CHANGE and REMOVE
33  * uevents while resuming, removing or renaming the device.
34  */
35 #define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE"
36 #define DM_COOKIE_LENGTH 24
37 
38 static const char *_name = DM_NAME;
39 
40 static unsigned int major = 0;
41 static unsigned int _major = 0;
42 
43 static DEFINE_IDR(_minor_idr);
44 
45 static DEFINE_SPINLOCK(_minor_lock);
46 
47 static void do_deferred_remove(struct work_struct *w);
48 
49 static DECLARE_WORK(deferred_remove_work, do_deferred_remove);
50 
51 static struct workqueue_struct *deferred_remove_workqueue;
52 
53 atomic_t dm_global_event_nr = ATOMIC_INIT(0);
54 DECLARE_WAIT_QUEUE_HEAD(dm_global_eventq);
55 
56 void dm_issue_global_event(void)
57 {
58 	atomic_inc(&dm_global_event_nr);
59 	wake_up(&dm_global_eventq);
60 }
61 
62 /*
63  * One of these is allocated (on-stack) per original bio.
64  */
65 struct clone_info {
66 	struct dm_table *map;
67 	struct bio *bio;
68 	struct dm_io *io;
69 	sector_t sector;
70 	unsigned sector_count;
71 };
72 
73 /*
74  * One of these is allocated per clone bio.
75  */
76 #define DM_TIO_MAGIC 7282014
77 struct dm_target_io {
78 	unsigned magic;
79 	struct dm_io *io;
80 	struct dm_target *ti;
81 	unsigned target_bio_nr;
82 	unsigned *len_ptr;
83 	bool inside_dm_io;
84 	struct bio clone;
85 };
86 
87 /*
88  * One of these is allocated per original bio.
89  * It contains the first clone used for that original.
90  */
91 #define DM_IO_MAGIC 5191977
92 struct dm_io {
93 	unsigned magic;
94 	struct mapped_device *md;
95 	blk_status_t status;
96 	atomic_t io_count;
97 	struct bio *orig_bio;
98 	unsigned long start_time;
99 	spinlock_t endio_lock;
100 	struct dm_stats_aux stats_aux;
101 	/* last member of dm_target_io is 'struct bio' */
102 	struct dm_target_io tio;
103 };
104 
105 void *dm_per_bio_data(struct bio *bio, size_t data_size)
106 {
107 	struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
108 	if (!tio->inside_dm_io)
109 		return (char *)bio - offsetof(struct dm_target_io, clone) - data_size;
110 	return (char *)bio - offsetof(struct dm_target_io, clone) - offsetof(struct dm_io, tio) - data_size;
111 }
112 EXPORT_SYMBOL_GPL(dm_per_bio_data);
113 
114 struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size)
115 {
116 	struct dm_io *io = (struct dm_io *)((char *)data + data_size);
117 	if (io->magic == DM_IO_MAGIC)
118 		return (struct bio *)((char *)io + offsetof(struct dm_io, tio) + offsetof(struct dm_target_io, clone));
119 	BUG_ON(io->magic != DM_TIO_MAGIC);
120 	return (struct bio *)((char *)io + offsetof(struct dm_target_io, clone));
121 }
122 EXPORT_SYMBOL_GPL(dm_bio_from_per_bio_data);
123 
124 unsigned dm_bio_get_target_bio_nr(const struct bio *bio)
125 {
126 	return container_of(bio, struct dm_target_io, clone)->target_bio_nr;
127 }
128 EXPORT_SYMBOL_GPL(dm_bio_get_target_bio_nr);
129 
130 #define MINOR_ALLOCED ((void *)-1)
131 
132 /*
133  * Bits for the md->flags field.
134  */
135 #define DMF_BLOCK_IO_FOR_SUSPEND 0
136 #define DMF_SUSPENDED 1
137 #define DMF_FROZEN 2
138 #define DMF_FREEING 3
139 #define DMF_DELETING 4
140 #define DMF_NOFLUSH_SUSPENDING 5
141 #define DMF_DEFERRED_REMOVE 6
142 #define DMF_SUSPENDED_INTERNALLY 7
143 
144 #define DM_NUMA_NODE NUMA_NO_NODE
145 static int dm_numa_node = DM_NUMA_NODE;
146 
147 /*
148  * For mempools pre-allocation at the table loading time.
149  */
150 struct dm_md_mempools {
151 	struct bio_set bs;
152 	struct bio_set io_bs;
153 };
154 
155 struct table_device {
156 	struct list_head list;
157 	refcount_t count;
158 	struct dm_dev dm_dev;
159 };
160 
161 static struct kmem_cache *_rq_tio_cache;
162 static struct kmem_cache *_rq_cache;
163 
164 /*
165  * Bio-based DM's mempools' reserved IOs set by the user.
166  */
167 #define RESERVED_BIO_BASED_IOS		16
168 static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS;
169 
170 static int __dm_get_module_param_int(int *module_param, int min, int max)
171 {
172 	int param = READ_ONCE(*module_param);
173 	int modified_param = 0;
174 	bool modified = true;
175 
176 	if (param < min)
177 		modified_param = min;
178 	else if (param > max)
179 		modified_param = max;
180 	else
181 		modified = false;
182 
183 	if (modified) {
184 		(void)cmpxchg(module_param, param, modified_param);
185 		param = modified_param;
186 	}
187 
188 	return param;
189 }
190 
191 unsigned __dm_get_module_param(unsigned *module_param,
192 			       unsigned def, unsigned max)
193 {
194 	unsigned param = READ_ONCE(*module_param);
195 	unsigned modified_param = 0;
196 
197 	if (!param)
198 		modified_param = def;
199 	else if (param > max)
200 		modified_param = max;
201 
202 	if (modified_param) {
203 		(void)cmpxchg(module_param, param, modified_param);
204 		param = modified_param;
205 	}
206 
207 	return param;
208 }
209 
210 unsigned dm_get_reserved_bio_based_ios(void)
211 {
212 	return __dm_get_module_param(&reserved_bio_based_ios,
213 				     RESERVED_BIO_BASED_IOS, DM_RESERVED_MAX_IOS);
214 }
215 EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios);
216 
217 static unsigned dm_get_numa_node(void)
218 {
219 	return __dm_get_module_param_int(&dm_numa_node,
220 					 DM_NUMA_NODE, num_online_nodes() - 1);
221 }
222 
223 static int __init local_init(void)
224 {
225 	int r = -ENOMEM;
226 
227 	_rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0);
228 	if (!_rq_tio_cache)
229 		return r;
230 
231 	_rq_cache = kmem_cache_create("dm_old_clone_request", sizeof(struct request),
232 				      __alignof__(struct request), 0, NULL);
233 	if (!_rq_cache)
234 		goto out_free_rq_tio_cache;
235 
236 	r = dm_uevent_init();
237 	if (r)
238 		goto out_free_rq_cache;
239 
240 	deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1);
241 	if (!deferred_remove_workqueue) {
242 		r = -ENOMEM;
243 		goto out_uevent_exit;
244 	}
245 
246 	_major = major;
247 	r = register_blkdev(_major, _name);
248 	if (r < 0)
249 		goto out_free_workqueue;
250 
251 	if (!_major)
252 		_major = r;
253 
254 	return 0;
255 
256 out_free_workqueue:
257 	destroy_workqueue(deferred_remove_workqueue);
258 out_uevent_exit:
259 	dm_uevent_exit();
260 out_free_rq_cache:
261 	kmem_cache_destroy(_rq_cache);
262 out_free_rq_tio_cache:
263 	kmem_cache_destroy(_rq_tio_cache);
264 
265 	return r;
266 }
267 
268 static void local_exit(void)
269 {
270 	flush_scheduled_work();
271 	destroy_workqueue(deferred_remove_workqueue);
272 
273 	kmem_cache_destroy(_rq_cache);
274 	kmem_cache_destroy(_rq_tio_cache);
275 	unregister_blkdev(_major, _name);
276 	dm_uevent_exit();
277 
278 	_major = 0;
279 
280 	DMINFO("cleaned up");
281 }
282 
283 static int (*_inits[])(void) __initdata = {
284 	local_init,
285 	dm_target_init,
286 	dm_linear_init,
287 	dm_stripe_init,
288 	dm_io_init,
289 	dm_kcopyd_init,
290 	dm_interface_init,
291 	dm_statistics_init,
292 };
293 
294 static void (*_exits[])(void) = {
295 	local_exit,
296 	dm_target_exit,
297 	dm_linear_exit,
298 	dm_stripe_exit,
299 	dm_io_exit,
300 	dm_kcopyd_exit,
301 	dm_interface_exit,
302 	dm_statistics_exit,
303 };
304 
305 static int __init dm_init(void)
306 {
307 	const int count = ARRAY_SIZE(_inits);
308 
309 	int r, i;
310 
311 	for (i = 0; i < count; i++) {
312 		r = _inits[i]();
313 		if (r)
314 			goto bad;
315 	}
316 
317 	return 0;
318 
319       bad:
320 	while (i--)
321 		_exits[i]();
322 
323 	return r;
324 }
325 
326 static void __exit dm_exit(void)
327 {
328 	int i = ARRAY_SIZE(_exits);
329 
330 	while (i--)
331 		_exits[i]();
332 
333 	/*
334 	 * Should be empty by this point.
335 	 */
336 	idr_destroy(&_minor_idr);
337 }
338 
339 /*
340  * Block device functions
341  */
342 int dm_deleting_md(struct mapped_device *md)
343 {
344 	return test_bit(DMF_DELETING, &md->flags);
345 }
346 
347 static int dm_blk_open(struct block_device *bdev, fmode_t mode)
348 {
349 	struct mapped_device *md;
350 
351 	spin_lock(&_minor_lock);
352 
353 	md = bdev->bd_disk->private_data;
354 	if (!md)
355 		goto out;
356 
357 	if (test_bit(DMF_FREEING, &md->flags) ||
358 	    dm_deleting_md(md)) {
359 		md = NULL;
360 		goto out;
361 	}
362 
363 	dm_get(md);
364 	atomic_inc(&md->open_count);
365 out:
366 	spin_unlock(&_minor_lock);
367 
368 	return md ? 0 : -ENXIO;
369 }
370 
371 static void dm_blk_close(struct gendisk *disk, fmode_t mode)
372 {
373 	struct mapped_device *md;
374 
375 	spin_lock(&_minor_lock);
376 
377 	md = disk->private_data;
378 	if (WARN_ON(!md))
379 		goto out;
380 
381 	if (atomic_dec_and_test(&md->open_count) &&
382 	    (test_bit(DMF_DEFERRED_REMOVE, &md->flags)))
383 		queue_work(deferred_remove_workqueue, &deferred_remove_work);
384 
385 	dm_put(md);
386 out:
387 	spin_unlock(&_minor_lock);
388 }
389 
390 int dm_open_count(struct mapped_device *md)
391 {
392 	return atomic_read(&md->open_count);
393 }
394 
395 /*
396  * Guarantees nothing is using the device before it's deleted.
397  */
398 int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred)
399 {
400 	int r = 0;
401 
402 	spin_lock(&_minor_lock);
403 
404 	if (dm_open_count(md)) {
405 		r = -EBUSY;
406 		if (mark_deferred)
407 			set_bit(DMF_DEFERRED_REMOVE, &md->flags);
408 	} else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags))
409 		r = -EEXIST;
410 	else
411 		set_bit(DMF_DELETING, &md->flags);
412 
413 	spin_unlock(&_minor_lock);
414 
415 	return r;
416 }
417 
418 int dm_cancel_deferred_remove(struct mapped_device *md)
419 {
420 	int r = 0;
421 
422 	spin_lock(&_minor_lock);
423 
424 	if (test_bit(DMF_DELETING, &md->flags))
425 		r = -EBUSY;
426 	else
427 		clear_bit(DMF_DEFERRED_REMOVE, &md->flags);
428 
429 	spin_unlock(&_minor_lock);
430 
431 	return r;
432 }
433 
434 static void do_deferred_remove(struct work_struct *w)
435 {
436 	dm_deferred_remove();
437 }
438 
439 sector_t dm_get_size(struct mapped_device *md)
440 {
441 	return get_capacity(md->disk);
442 }
443 
444 struct request_queue *dm_get_md_queue(struct mapped_device *md)
445 {
446 	return md->queue;
447 }
448 
449 struct dm_stats *dm_get_stats(struct mapped_device *md)
450 {
451 	return &md->stats;
452 }
453 
454 static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
455 {
456 	struct mapped_device *md = bdev->bd_disk->private_data;
457 
458 	return dm_get_geometry(md, geo);
459 }
460 
461 static int dm_blk_report_zones(struct gendisk *disk, sector_t sector,
462 			       struct blk_zone *zones, unsigned int *nr_zones,
463 			       gfp_t gfp_mask)
464 {
465 #ifdef CONFIG_BLK_DEV_ZONED
466 	struct mapped_device *md = disk->private_data;
467 	struct dm_target *tgt;
468 	struct dm_table *map;
469 	int srcu_idx, ret;
470 
471 	if (dm_suspended_md(md))
472 		return -EAGAIN;
473 
474 	map = dm_get_live_table(md, &srcu_idx);
475 	if (!map)
476 		return -EIO;
477 
478 	tgt = dm_table_find_target(map, sector);
479 	if (!dm_target_is_valid(tgt)) {
480 		ret = -EIO;
481 		goto out;
482 	}
483 
484 	/*
485 	 * If we are executing this, we already know that the block device
486 	 * is a zoned device and so each target should have support for that
487 	 * type of drive. A missing report_zones method means that the target
488 	 * driver has a problem.
489 	 */
490 	if (WARN_ON(!tgt->type->report_zones)) {
491 		ret = -EIO;
492 		goto out;
493 	}
494 
495 	/*
496 	 * blkdev_report_zones() will loop and call this again to cover all the
497 	 * zones of the target, eventually moving on to the next target.
498 	 * So there is no need to loop here trying to fill the entire array
499 	 * of zones.
500 	 */
501 	ret = tgt->type->report_zones(tgt, sector, zones,
502 				      nr_zones, gfp_mask);
503 
504 out:
505 	dm_put_live_table(md, srcu_idx);
506 	return ret;
507 #else
508 	return -ENOTSUPP;
509 #endif
510 }
511 
512 static int dm_prepare_ioctl(struct mapped_device *md, int *srcu_idx,
513 			    struct block_device **bdev)
514 	__acquires(md->io_barrier)
515 {
516 	struct dm_target *tgt;
517 	struct dm_table *map;
518 	int r;
519 
520 retry:
521 	r = -ENOTTY;
522 	map = dm_get_live_table(md, srcu_idx);
523 	if (!map || !dm_table_get_size(map))
524 		return r;
525 
526 	/* We only support devices that have a single target */
527 	if (dm_table_get_num_targets(map) != 1)
528 		return r;
529 
530 	tgt = dm_table_get_target(map, 0);
531 	if (!tgt->type->prepare_ioctl)
532 		return r;
533 
534 	if (dm_suspended_md(md))
535 		return -EAGAIN;
536 
537 	r = tgt->type->prepare_ioctl(tgt, bdev);
538 	if (r == -ENOTCONN && !fatal_signal_pending(current)) {
539 		dm_put_live_table(md, *srcu_idx);
540 		msleep(10);
541 		goto retry;
542 	}
543 
544 	return r;
545 }
546 
547 static void dm_unprepare_ioctl(struct mapped_device *md, int srcu_idx)
548 	__releases(md->io_barrier)
549 {
550 	dm_put_live_table(md, srcu_idx);
551 }
552 
553 static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
554 			unsigned int cmd, unsigned long arg)
555 {
556 	struct mapped_device *md = bdev->bd_disk->private_data;
557 	int r, srcu_idx;
558 
559 	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
560 	if (r < 0)
561 		goto out;
562 
563 	if (r > 0) {
564 		/*
565 		 * Target determined this ioctl is being issued against a
566 		 * subset of the parent bdev; require extra privileges.
567 		 */
568 		if (!capable(CAP_SYS_RAWIO)) {
569 			DMWARN_LIMIT(
570 	"%s: sending ioctl %x to DM device without required privilege.",
571 				current->comm, cmd);
572 			r = -ENOIOCTLCMD;
573 			goto out;
574 		}
575 	}
576 
577 	r =  __blkdev_driver_ioctl(bdev, mode, cmd, arg);
578 out:
579 	dm_unprepare_ioctl(md, srcu_idx);
580 	return r;
581 }
582 
583 static void start_io_acct(struct dm_io *io);
584 
585 static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio)
586 {
587 	struct dm_io *io;
588 	struct dm_target_io *tio;
589 	struct bio *clone;
590 
591 	clone = bio_alloc_bioset(GFP_NOIO, 0, &md->io_bs);
592 	if (!clone)
593 		return NULL;
594 
595 	tio = container_of(clone, struct dm_target_io, clone);
596 	tio->inside_dm_io = true;
597 	tio->io = NULL;
598 
599 	io = container_of(tio, struct dm_io, tio);
600 	io->magic = DM_IO_MAGIC;
601 	io->status = 0;
602 	atomic_set(&io->io_count, 1);
603 	io->orig_bio = bio;
604 	io->md = md;
605 	spin_lock_init(&io->endio_lock);
606 
607 	start_io_acct(io);
608 
609 	return io;
610 }
611 
612 static void free_io(struct mapped_device *md, struct dm_io *io)
613 {
614 	bio_put(&io->tio.clone);
615 }
616 
617 static struct dm_target_io *alloc_tio(struct clone_info *ci, struct dm_target *ti,
618 				      unsigned target_bio_nr, gfp_t gfp_mask)
619 {
620 	struct dm_target_io *tio;
621 
622 	if (!ci->io->tio.io) {
623 		/* the dm_target_io embedded in ci->io is available */
624 		tio = &ci->io->tio;
625 	} else {
626 		struct bio *clone = bio_alloc_bioset(gfp_mask, 0, &ci->io->md->bs);
627 		if (!clone)
628 			return NULL;
629 
630 		tio = container_of(clone, struct dm_target_io, clone);
631 		tio->inside_dm_io = false;
632 	}
633 
634 	tio->magic = DM_TIO_MAGIC;
635 	tio->io = ci->io;
636 	tio->ti = ti;
637 	tio->target_bio_nr = target_bio_nr;
638 
639 	return tio;
640 }
641 
642 static void free_tio(struct dm_target_io *tio)
643 {
644 	if (tio->inside_dm_io)
645 		return;
646 	bio_put(&tio->clone);
647 }
648 
649 int md_in_flight(struct mapped_device *md)
650 {
651 	return atomic_read(&md->pending[READ]) +
652 	       atomic_read(&md->pending[WRITE]);
653 }
654 
655 static void start_io_acct(struct dm_io *io)
656 {
657 	struct mapped_device *md = io->md;
658 	struct bio *bio = io->orig_bio;
659 	int rw = bio_data_dir(bio);
660 
661 	io->start_time = jiffies;
662 
663 	generic_start_io_acct(md->queue, bio_op(bio), bio_sectors(bio),
664 			      &dm_disk(md)->part0);
665 
666 	atomic_set(&dm_disk(md)->part0.in_flight[rw],
667 		   atomic_inc_return(&md->pending[rw]));
668 
669 	if (unlikely(dm_stats_used(&md->stats)))
670 		dm_stats_account_io(&md->stats, bio_data_dir(bio),
671 				    bio->bi_iter.bi_sector, bio_sectors(bio),
672 				    false, 0, &io->stats_aux);
673 }
674 
675 static void end_io_acct(struct dm_io *io)
676 {
677 	struct mapped_device *md = io->md;
678 	struct bio *bio = io->orig_bio;
679 	unsigned long duration = jiffies - io->start_time;
680 	int pending;
681 	int rw = bio_data_dir(bio);
682 
683 	generic_end_io_acct(md->queue, bio_op(bio), &dm_disk(md)->part0,
684 			    io->start_time);
685 
686 	if (unlikely(dm_stats_used(&md->stats)))
687 		dm_stats_account_io(&md->stats, bio_data_dir(bio),
688 				    bio->bi_iter.bi_sector, bio_sectors(bio),
689 				    true, duration, &io->stats_aux);
690 
691 	/*
692 	 * After this is decremented the bio must not be touched if it is
693 	 * a flush.
694 	 */
695 	pending = atomic_dec_return(&md->pending[rw]);
696 	atomic_set(&dm_disk(md)->part0.in_flight[rw], pending);
697 	pending += atomic_read(&md->pending[rw^0x1]);
698 
699 	/* nudge anyone waiting on suspend queue */
700 	if (!pending)
701 		wake_up(&md->wait);
702 }
703 
704 /*
705  * Add the bio to the list of deferred io.
706  */
707 static void queue_io(struct mapped_device *md, struct bio *bio)
708 {
709 	unsigned long flags;
710 
711 	spin_lock_irqsave(&md->deferred_lock, flags);
712 	bio_list_add(&md->deferred, bio);
713 	spin_unlock_irqrestore(&md->deferred_lock, flags);
714 	queue_work(md->wq, &md->work);
715 }
716 
717 /*
718  * Everyone (including functions in this file), should use this
719  * function to access the md->map field, and make sure they call
720  * dm_put_live_table() when finished.
721  */
722 struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier)
723 {
724 	*srcu_idx = srcu_read_lock(&md->io_barrier);
725 
726 	return srcu_dereference(md->map, &md->io_barrier);
727 }
728 
729 void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier)
730 {
731 	srcu_read_unlock(&md->io_barrier, srcu_idx);
732 }
733 
734 void dm_sync_table(struct mapped_device *md)
735 {
736 	synchronize_srcu(&md->io_barrier);
737 	synchronize_rcu_expedited();
738 }
739 
740 /*
741  * A fast alternative to dm_get_live_table/dm_put_live_table.
742  * The caller must not block between these two functions.
743  */
744 static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU)
745 {
746 	rcu_read_lock();
747 	return rcu_dereference(md->map);
748 }
749 
750 static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU)
751 {
752 	rcu_read_unlock();
753 }
754 
755 static char *_dm_claim_ptr = "I belong to device-mapper";
756 
757 /*
758  * Open a table device so we can use it as a map destination.
759  */
760 static int open_table_device(struct table_device *td, dev_t dev,
761 			     struct mapped_device *md)
762 {
763 	struct block_device *bdev;
764 
765 	int r;
766 
767 	BUG_ON(td->dm_dev.bdev);
768 
769 	bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _dm_claim_ptr);
770 	if (IS_ERR(bdev))
771 		return PTR_ERR(bdev);
772 
773 	r = bd_link_disk_holder(bdev, dm_disk(md));
774 	if (r) {
775 		blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL);
776 		return r;
777 	}
778 
779 	td->dm_dev.bdev = bdev;
780 	td->dm_dev.dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);
781 	return 0;
782 }
783 
784 /*
785  * Close a table device that we've been using.
786  */
787 static void close_table_device(struct table_device *td, struct mapped_device *md)
788 {
789 	if (!td->dm_dev.bdev)
790 		return;
791 
792 	bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md));
793 	blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL);
794 	put_dax(td->dm_dev.dax_dev);
795 	td->dm_dev.bdev = NULL;
796 	td->dm_dev.dax_dev = NULL;
797 }
798 
799 static struct table_device *find_table_device(struct list_head *l, dev_t dev,
800 					      fmode_t mode) {
801 	struct table_device *td;
802 
803 	list_for_each_entry(td, l, list)
804 		if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode)
805 			return td;
806 
807 	return NULL;
808 }
809 
810 int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode,
811 			struct dm_dev **result) {
812 	int r;
813 	struct table_device *td;
814 
815 	mutex_lock(&md->table_devices_lock);
816 	td = find_table_device(&md->table_devices, dev, mode);
817 	if (!td) {
818 		td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id);
819 		if (!td) {
820 			mutex_unlock(&md->table_devices_lock);
821 			return -ENOMEM;
822 		}
823 
824 		td->dm_dev.mode = mode;
825 		td->dm_dev.bdev = NULL;
826 
827 		if ((r = open_table_device(td, dev, md))) {
828 			mutex_unlock(&md->table_devices_lock);
829 			kfree(td);
830 			return r;
831 		}
832 
833 		format_dev_t(td->dm_dev.name, dev);
834 
835 		refcount_set(&td->count, 1);
836 		list_add(&td->list, &md->table_devices);
837 	} else {
838 		refcount_inc(&td->count);
839 	}
840 	mutex_unlock(&md->table_devices_lock);
841 
842 	*result = &td->dm_dev;
843 	return 0;
844 }
845 EXPORT_SYMBOL_GPL(dm_get_table_device);
846 
847 void dm_put_table_device(struct mapped_device *md, struct dm_dev *d)
848 {
849 	struct table_device *td = container_of(d, struct table_device, dm_dev);
850 
851 	mutex_lock(&md->table_devices_lock);
852 	if (refcount_dec_and_test(&td->count)) {
853 		close_table_device(td, md);
854 		list_del(&td->list);
855 		kfree(td);
856 	}
857 	mutex_unlock(&md->table_devices_lock);
858 }
859 EXPORT_SYMBOL(dm_put_table_device);
860 
861 static void free_table_devices(struct list_head *devices)
862 {
863 	struct list_head *tmp, *next;
864 
865 	list_for_each_safe(tmp, next, devices) {
866 		struct table_device *td = list_entry(tmp, struct table_device, list);
867 
868 		DMWARN("dm_destroy: %s still exists with %d references",
869 		       td->dm_dev.name, refcount_read(&td->count));
870 		kfree(td);
871 	}
872 }
873 
874 /*
875  * Get the geometry associated with a dm device
876  */
877 int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
878 {
879 	*geo = md->geometry;
880 
881 	return 0;
882 }
883 
884 /*
885  * Set the geometry of a device.
886  */
887 int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
888 {
889 	sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
890 
891 	if (geo->start > sz) {
892 		DMWARN("Start sector is beyond the geometry limits.");
893 		return -EINVAL;
894 	}
895 
896 	md->geometry = *geo;
897 
898 	return 0;
899 }
900 
901 static int __noflush_suspending(struct mapped_device *md)
902 {
903 	return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
904 }
905 
906 /*
907  * Decrements the number of outstanding ios that a bio has been
908  * cloned into, completing the original io if necc.
909  */
910 static void dec_pending(struct dm_io *io, blk_status_t error)
911 {
912 	unsigned long flags;
913 	blk_status_t io_error;
914 	struct bio *bio;
915 	struct mapped_device *md = io->md;
916 
917 	/* Push-back supersedes any I/O errors */
918 	if (unlikely(error)) {
919 		spin_lock_irqsave(&io->endio_lock, flags);
920 		if (!(io->status == BLK_STS_DM_REQUEUE && __noflush_suspending(md)))
921 			io->status = error;
922 		spin_unlock_irqrestore(&io->endio_lock, flags);
923 	}
924 
925 	if (atomic_dec_and_test(&io->io_count)) {
926 		if (io->status == BLK_STS_DM_REQUEUE) {
927 			/*
928 			 * Target requested pushing back the I/O.
929 			 */
930 			spin_lock_irqsave(&md->deferred_lock, flags);
931 			if (__noflush_suspending(md))
932 				/* NOTE early return due to BLK_STS_DM_REQUEUE below */
933 				bio_list_add_head(&md->deferred, io->orig_bio);
934 			else
935 				/* noflush suspend was interrupted. */
936 				io->status = BLK_STS_IOERR;
937 			spin_unlock_irqrestore(&md->deferred_lock, flags);
938 		}
939 
940 		io_error = io->status;
941 		bio = io->orig_bio;
942 		end_io_acct(io);
943 		free_io(md, io);
944 
945 		if (io_error == BLK_STS_DM_REQUEUE)
946 			return;
947 
948 		if ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size) {
949 			/*
950 			 * Preflush done for flush with data, reissue
951 			 * without REQ_PREFLUSH.
952 			 */
953 			bio->bi_opf &= ~REQ_PREFLUSH;
954 			queue_io(md, bio);
955 		} else {
956 			/* done with normal IO or empty flush */
957 			if (io_error)
958 				bio->bi_status = io_error;
959 			bio_endio(bio);
960 		}
961 	}
962 }
963 
964 void disable_write_same(struct mapped_device *md)
965 {
966 	struct queue_limits *limits = dm_get_queue_limits(md);
967 
968 	/* device doesn't really support WRITE SAME, disable it */
969 	limits->max_write_same_sectors = 0;
970 }
971 
972 void disable_write_zeroes(struct mapped_device *md)
973 {
974 	struct queue_limits *limits = dm_get_queue_limits(md);
975 
976 	/* device doesn't really support WRITE ZEROES, disable it */
977 	limits->max_write_zeroes_sectors = 0;
978 }
979 
980 static void clone_endio(struct bio *bio)
981 {
982 	blk_status_t error = bio->bi_status;
983 	struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
984 	struct dm_io *io = tio->io;
985 	struct mapped_device *md = tio->io->md;
986 	dm_endio_fn endio = tio->ti->type->end_io;
987 
988 	if (unlikely(error == BLK_STS_TARGET) && md->type != DM_TYPE_NVME_BIO_BASED) {
989 		if (bio_op(bio) == REQ_OP_WRITE_SAME &&
990 		    !bio->bi_disk->queue->limits.max_write_same_sectors)
991 			disable_write_same(md);
992 		if (bio_op(bio) == REQ_OP_WRITE_ZEROES &&
993 		    !bio->bi_disk->queue->limits.max_write_zeroes_sectors)
994 			disable_write_zeroes(md);
995 	}
996 
997 	if (endio) {
998 		int r = endio(tio->ti, bio, &error);
999 		switch (r) {
1000 		case DM_ENDIO_REQUEUE:
1001 			error = BLK_STS_DM_REQUEUE;
1002 			/*FALLTHRU*/
1003 		case DM_ENDIO_DONE:
1004 			break;
1005 		case DM_ENDIO_INCOMPLETE:
1006 			/* The target will handle the io */
1007 			return;
1008 		default:
1009 			DMWARN("unimplemented target endio return value: %d", r);
1010 			BUG();
1011 		}
1012 	}
1013 
1014 	free_tio(tio);
1015 	dec_pending(io, error);
1016 }
1017 
1018 /*
1019  * Return maximum size of I/O possible at the supplied sector up to the current
1020  * target boundary.
1021  */
1022 static sector_t max_io_len_target_boundary(sector_t sector, struct dm_target *ti)
1023 {
1024 	sector_t target_offset = dm_target_offset(ti, sector);
1025 
1026 	return ti->len - target_offset;
1027 }
1028 
1029 static sector_t max_io_len(sector_t sector, struct dm_target *ti)
1030 {
1031 	sector_t len = max_io_len_target_boundary(sector, ti);
1032 	sector_t offset, max_len;
1033 
1034 	/*
1035 	 * Does the target need to split even further?
1036 	 */
1037 	if (ti->max_io_len) {
1038 		offset = dm_target_offset(ti, sector);
1039 		if (unlikely(ti->max_io_len & (ti->max_io_len - 1)))
1040 			max_len = sector_div(offset, ti->max_io_len);
1041 		else
1042 			max_len = offset & (ti->max_io_len - 1);
1043 		max_len = ti->max_io_len - max_len;
1044 
1045 		if (len > max_len)
1046 			len = max_len;
1047 	}
1048 
1049 	return len;
1050 }
1051 
1052 int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
1053 {
1054 	if (len > UINT_MAX) {
1055 		DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)",
1056 		      (unsigned long long)len, UINT_MAX);
1057 		ti->error = "Maximum size of target IO is too large";
1058 		return -EINVAL;
1059 	}
1060 
1061 	/*
1062 	 * BIO based queue uses its own splitting. When multipage bvecs
1063 	 * is switched on, size of the incoming bio may be too big to
1064 	 * be handled in some targets, such as crypt.
1065 	 *
1066 	 * When these targets are ready for the big bio, we can remove
1067 	 * the limit.
1068 	 */
1069 	ti->max_io_len = min_t(uint32_t, len, BIO_MAX_PAGES * PAGE_SIZE);
1070 
1071 	return 0;
1072 }
1073 EXPORT_SYMBOL_GPL(dm_set_target_max_io_len);
1074 
1075 static struct dm_target *dm_dax_get_live_target(struct mapped_device *md,
1076 						sector_t sector, int *srcu_idx)
1077 	__acquires(md->io_barrier)
1078 {
1079 	struct dm_table *map;
1080 	struct dm_target *ti;
1081 
1082 	map = dm_get_live_table(md, srcu_idx);
1083 	if (!map)
1084 		return NULL;
1085 
1086 	ti = dm_table_find_target(map, sector);
1087 	if (!dm_target_is_valid(ti))
1088 		return NULL;
1089 
1090 	return ti;
1091 }
1092 
1093 static long dm_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
1094 				 long nr_pages, void **kaddr, pfn_t *pfn)
1095 {
1096 	struct mapped_device *md = dax_get_private(dax_dev);
1097 	sector_t sector = pgoff * PAGE_SECTORS;
1098 	struct dm_target *ti;
1099 	long len, ret = -EIO;
1100 	int srcu_idx;
1101 
1102 	ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1103 
1104 	if (!ti)
1105 		goto out;
1106 	if (!ti->type->direct_access)
1107 		goto out;
1108 	len = max_io_len(sector, ti) / PAGE_SECTORS;
1109 	if (len < 1)
1110 		goto out;
1111 	nr_pages = min(len, nr_pages);
1112 	ret = ti->type->direct_access(ti, pgoff, nr_pages, kaddr, pfn);
1113 
1114  out:
1115 	dm_put_live_table(md, srcu_idx);
1116 
1117 	return ret;
1118 }
1119 
1120 static size_t dm_dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff,
1121 				    void *addr, size_t bytes, struct iov_iter *i)
1122 {
1123 	struct mapped_device *md = dax_get_private(dax_dev);
1124 	sector_t sector = pgoff * PAGE_SECTORS;
1125 	struct dm_target *ti;
1126 	long ret = 0;
1127 	int srcu_idx;
1128 
1129 	ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1130 
1131 	if (!ti)
1132 		goto out;
1133 	if (!ti->type->dax_copy_from_iter) {
1134 		ret = copy_from_iter(addr, bytes, i);
1135 		goto out;
1136 	}
1137 	ret = ti->type->dax_copy_from_iter(ti, pgoff, addr, bytes, i);
1138  out:
1139 	dm_put_live_table(md, srcu_idx);
1140 
1141 	return ret;
1142 }
1143 
1144 static size_t dm_dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff,
1145 		void *addr, size_t bytes, struct iov_iter *i)
1146 {
1147 	struct mapped_device *md = dax_get_private(dax_dev);
1148 	sector_t sector = pgoff * PAGE_SECTORS;
1149 	struct dm_target *ti;
1150 	long ret = 0;
1151 	int srcu_idx;
1152 
1153 	ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1154 
1155 	if (!ti)
1156 		goto out;
1157 	if (!ti->type->dax_copy_to_iter) {
1158 		ret = copy_to_iter(addr, bytes, i);
1159 		goto out;
1160 	}
1161 	ret = ti->type->dax_copy_to_iter(ti, pgoff, addr, bytes, i);
1162  out:
1163 	dm_put_live_table(md, srcu_idx);
1164 
1165 	return ret;
1166 }
1167 
1168 /*
1169  * A target may call dm_accept_partial_bio only from the map routine.  It is
1170  * allowed for all bio types except REQ_PREFLUSH and REQ_OP_ZONE_RESET.
1171  *
1172  * dm_accept_partial_bio informs the dm that the target only wants to process
1173  * additional n_sectors sectors of the bio and the rest of the data should be
1174  * sent in a next bio.
1175  *
1176  * A diagram that explains the arithmetics:
1177  * +--------------------+---------------+-------+
1178  * |         1          |       2       |   3   |
1179  * +--------------------+---------------+-------+
1180  *
1181  * <-------------- *tio->len_ptr --------------->
1182  *                      <------- bi_size ------->
1183  *                      <-- n_sectors -->
1184  *
1185  * Region 1 was already iterated over with bio_advance or similar function.
1186  *	(it may be empty if the target doesn't use bio_advance)
1187  * Region 2 is the remaining bio size that the target wants to process.
1188  *	(it may be empty if region 1 is non-empty, although there is no reason
1189  *	 to make it empty)
1190  * The target requires that region 3 is to be sent in the next bio.
1191  *
1192  * If the target wants to receive multiple copies of the bio (via num_*bios, etc),
1193  * the partially processed part (the sum of regions 1+2) must be the same for all
1194  * copies of the bio.
1195  */
1196 void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors)
1197 {
1198 	struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
1199 	unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT;
1200 	BUG_ON(bio->bi_opf & REQ_PREFLUSH);
1201 	BUG_ON(bi_size > *tio->len_ptr);
1202 	BUG_ON(n_sectors > bi_size);
1203 	*tio->len_ptr -= bi_size - n_sectors;
1204 	bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT;
1205 }
1206 EXPORT_SYMBOL_GPL(dm_accept_partial_bio);
1207 
1208 /*
1209  * The zone descriptors obtained with a zone report indicate
1210  * zone positions within the underlying device of the target. The zone
1211  * descriptors must be remapped to match their position within the dm device.
1212  * The caller target should obtain the zones information using
1213  * blkdev_report_zones() to ensure that remapping for partition offset is
1214  * already handled.
1215  */
1216 void dm_remap_zone_report(struct dm_target *ti, sector_t start,
1217 			  struct blk_zone *zones, unsigned int *nr_zones)
1218 {
1219 #ifdef CONFIG_BLK_DEV_ZONED
1220 	struct blk_zone *zone;
1221 	unsigned int nrz = *nr_zones;
1222 	int i;
1223 
1224 	/*
1225 	 * Remap the start sector and write pointer position of the zones in
1226 	 * the array. Since we may have obtained from the target underlying
1227 	 * device more zones that the target size, also adjust the number
1228 	 * of zones.
1229 	 */
1230 	for (i = 0; i < nrz; i++) {
1231 		zone = zones + i;
1232 		if (zone->start >= start + ti->len) {
1233 			memset(zone, 0, sizeof(struct blk_zone) * (nrz - i));
1234 			break;
1235 		}
1236 
1237 		zone->start = zone->start + ti->begin - start;
1238 		if (zone->type == BLK_ZONE_TYPE_CONVENTIONAL)
1239 			continue;
1240 
1241 		if (zone->cond == BLK_ZONE_COND_FULL)
1242 			zone->wp = zone->start + zone->len;
1243 		else if (zone->cond == BLK_ZONE_COND_EMPTY)
1244 			zone->wp = zone->start;
1245 		else
1246 			zone->wp = zone->wp + ti->begin - start;
1247 	}
1248 
1249 	*nr_zones = i;
1250 #else /* !CONFIG_BLK_DEV_ZONED */
1251 	*nr_zones = 0;
1252 #endif
1253 }
1254 EXPORT_SYMBOL_GPL(dm_remap_zone_report);
1255 
1256 static blk_qc_t __map_bio(struct dm_target_io *tio)
1257 {
1258 	int r;
1259 	sector_t sector;
1260 	struct bio *clone = &tio->clone;
1261 	struct dm_io *io = tio->io;
1262 	struct mapped_device *md = io->md;
1263 	struct dm_target *ti = tio->ti;
1264 	blk_qc_t ret = BLK_QC_T_NONE;
1265 
1266 	clone->bi_end_io = clone_endio;
1267 
1268 	/*
1269 	 * Map the clone.  If r == 0 we don't need to do
1270 	 * anything, the target has assumed ownership of
1271 	 * this io.
1272 	 */
1273 	atomic_inc(&io->io_count);
1274 	sector = clone->bi_iter.bi_sector;
1275 
1276 	r = ti->type->map(ti, clone);
1277 	switch (r) {
1278 	case DM_MAPIO_SUBMITTED:
1279 		break;
1280 	case DM_MAPIO_REMAPPED:
1281 		/* the bio has been remapped so dispatch it */
1282 		trace_block_bio_remap(clone->bi_disk->queue, clone,
1283 				      bio_dev(io->orig_bio), sector);
1284 		if (md->type == DM_TYPE_NVME_BIO_BASED)
1285 			ret = direct_make_request(clone);
1286 		else
1287 			ret = generic_make_request(clone);
1288 		break;
1289 	case DM_MAPIO_KILL:
1290 		free_tio(tio);
1291 		dec_pending(io, BLK_STS_IOERR);
1292 		break;
1293 	case DM_MAPIO_REQUEUE:
1294 		free_tio(tio);
1295 		dec_pending(io, BLK_STS_DM_REQUEUE);
1296 		break;
1297 	default:
1298 		DMWARN("unimplemented target map return value: %d", r);
1299 		BUG();
1300 	}
1301 
1302 	return ret;
1303 }
1304 
1305 static void bio_setup_sector(struct bio *bio, sector_t sector, unsigned len)
1306 {
1307 	bio->bi_iter.bi_sector = sector;
1308 	bio->bi_iter.bi_size = to_bytes(len);
1309 }
1310 
1311 /*
1312  * Creates a bio that consists of range of complete bvecs.
1313  */
1314 static int clone_bio(struct dm_target_io *tio, struct bio *bio,
1315 		     sector_t sector, unsigned len)
1316 {
1317 	struct bio *clone = &tio->clone;
1318 
1319 	__bio_clone_fast(clone, bio);
1320 
1321 	if (unlikely(bio_integrity(bio) != NULL)) {
1322 		int r;
1323 
1324 		if (unlikely(!dm_target_has_integrity(tio->ti->type) &&
1325 			     !dm_target_passes_integrity(tio->ti->type))) {
1326 			DMWARN("%s: the target %s doesn't support integrity data.",
1327 				dm_device_name(tio->io->md),
1328 				tio->ti->type->name);
1329 			return -EIO;
1330 		}
1331 
1332 		r = bio_integrity_clone(clone, bio, GFP_NOIO);
1333 		if (r < 0)
1334 			return r;
1335 	}
1336 
1337 	bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector));
1338 	clone->bi_iter.bi_size = to_bytes(len);
1339 
1340 	if (unlikely(bio_integrity(bio) != NULL))
1341 		bio_integrity_trim(clone);
1342 
1343 	return 0;
1344 }
1345 
1346 static void alloc_multiple_bios(struct bio_list *blist, struct clone_info *ci,
1347 				struct dm_target *ti, unsigned num_bios)
1348 {
1349 	struct dm_target_io *tio;
1350 	int try;
1351 
1352 	if (!num_bios)
1353 		return;
1354 
1355 	if (num_bios == 1) {
1356 		tio = alloc_tio(ci, ti, 0, GFP_NOIO);
1357 		bio_list_add(blist, &tio->clone);
1358 		return;
1359 	}
1360 
1361 	for (try = 0; try < 2; try++) {
1362 		int bio_nr;
1363 		struct bio *bio;
1364 
1365 		if (try)
1366 			mutex_lock(&ci->io->md->table_devices_lock);
1367 		for (bio_nr = 0; bio_nr < num_bios; bio_nr++) {
1368 			tio = alloc_tio(ci, ti, bio_nr, try ? GFP_NOIO : GFP_NOWAIT);
1369 			if (!tio)
1370 				break;
1371 
1372 			bio_list_add(blist, &tio->clone);
1373 		}
1374 		if (try)
1375 			mutex_unlock(&ci->io->md->table_devices_lock);
1376 		if (bio_nr == num_bios)
1377 			return;
1378 
1379 		while ((bio = bio_list_pop(blist))) {
1380 			tio = container_of(bio, struct dm_target_io, clone);
1381 			free_tio(tio);
1382 		}
1383 	}
1384 }
1385 
1386 static blk_qc_t __clone_and_map_simple_bio(struct clone_info *ci,
1387 					   struct dm_target_io *tio, unsigned *len)
1388 {
1389 	struct bio *clone = &tio->clone;
1390 
1391 	tio->len_ptr = len;
1392 
1393 	__bio_clone_fast(clone, ci->bio);
1394 	if (len)
1395 		bio_setup_sector(clone, ci->sector, *len);
1396 
1397 	return __map_bio(tio);
1398 }
1399 
1400 static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti,
1401 				  unsigned num_bios, unsigned *len)
1402 {
1403 	struct bio_list blist = BIO_EMPTY_LIST;
1404 	struct bio *bio;
1405 	struct dm_target_io *tio;
1406 
1407 	alloc_multiple_bios(&blist, ci, ti, num_bios);
1408 
1409 	while ((bio = bio_list_pop(&blist))) {
1410 		tio = container_of(bio, struct dm_target_io, clone);
1411 		(void) __clone_and_map_simple_bio(ci, tio, len);
1412 	}
1413 }
1414 
1415 static int __send_empty_flush(struct clone_info *ci)
1416 {
1417 	unsigned target_nr = 0;
1418 	struct dm_target *ti;
1419 
1420 	BUG_ON(bio_has_data(ci->bio));
1421 	while ((ti = dm_table_get_target(ci->map, target_nr++)))
1422 		__send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL);
1423 
1424 	return 0;
1425 }
1426 
1427 static int __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti,
1428 				    sector_t sector, unsigned *len)
1429 {
1430 	struct bio *bio = ci->bio;
1431 	struct dm_target_io *tio;
1432 	int r;
1433 
1434 	tio = alloc_tio(ci, ti, 0, GFP_NOIO);
1435 	tio->len_ptr = len;
1436 	r = clone_bio(tio, bio, sector, *len);
1437 	if (r < 0) {
1438 		free_tio(tio);
1439 		return r;
1440 	}
1441 	(void) __map_bio(tio);
1442 
1443 	return 0;
1444 }
1445 
1446 typedef unsigned (*get_num_bios_fn)(struct dm_target *ti);
1447 
1448 static unsigned get_num_discard_bios(struct dm_target *ti)
1449 {
1450 	return ti->num_discard_bios;
1451 }
1452 
1453 static unsigned get_num_secure_erase_bios(struct dm_target *ti)
1454 {
1455 	return ti->num_secure_erase_bios;
1456 }
1457 
1458 static unsigned get_num_write_same_bios(struct dm_target *ti)
1459 {
1460 	return ti->num_write_same_bios;
1461 }
1462 
1463 static unsigned get_num_write_zeroes_bios(struct dm_target *ti)
1464 {
1465 	return ti->num_write_zeroes_bios;
1466 }
1467 
1468 typedef bool (*is_split_required_fn)(struct dm_target *ti);
1469 
1470 static bool is_split_required_for_discard(struct dm_target *ti)
1471 {
1472 	return ti->split_discard_bios;
1473 }
1474 
1475 static int __send_changing_extent_only(struct clone_info *ci, struct dm_target *ti,
1476 				       get_num_bios_fn get_num_bios,
1477 				       is_split_required_fn is_split_required)
1478 {
1479 	unsigned len;
1480 	unsigned num_bios;
1481 
1482 	/*
1483 	 * Even though the device advertised support for this type of
1484 	 * request, that does not mean every target supports it, and
1485 	 * reconfiguration might also have changed that since the
1486 	 * check was performed.
1487 	 */
1488 	num_bios = get_num_bios ? get_num_bios(ti) : 0;
1489 	if (!num_bios)
1490 		return -EOPNOTSUPP;
1491 
1492 	if (is_split_required && !is_split_required(ti))
1493 		len = min((sector_t)ci->sector_count, max_io_len_target_boundary(ci->sector, ti));
1494 	else
1495 		len = min((sector_t)ci->sector_count, max_io_len(ci->sector, ti));
1496 
1497 	__send_duplicate_bios(ci, ti, num_bios, &len);
1498 
1499 	ci->sector += len;
1500 	ci->sector_count -= len;
1501 
1502 	return 0;
1503 }
1504 
1505 static int __send_discard(struct clone_info *ci, struct dm_target *ti)
1506 {
1507 	return __send_changing_extent_only(ci, ti, get_num_discard_bios,
1508 					   is_split_required_for_discard);
1509 }
1510 
1511 static int __send_secure_erase(struct clone_info *ci, struct dm_target *ti)
1512 {
1513 	return __send_changing_extent_only(ci, ti, get_num_secure_erase_bios, NULL);
1514 }
1515 
1516 static int __send_write_same(struct clone_info *ci, struct dm_target *ti)
1517 {
1518 	return __send_changing_extent_only(ci, ti, get_num_write_same_bios, NULL);
1519 }
1520 
1521 static int __send_write_zeroes(struct clone_info *ci, struct dm_target *ti)
1522 {
1523 	return __send_changing_extent_only(ci, ti, get_num_write_zeroes_bios, NULL);
1524 }
1525 
1526 static bool __process_abnormal_io(struct clone_info *ci, struct dm_target *ti,
1527 				  int *result)
1528 {
1529 	struct bio *bio = ci->bio;
1530 
1531 	if (bio_op(bio) == REQ_OP_DISCARD)
1532 		*result = __send_discard(ci, ti);
1533 	else if (bio_op(bio) == REQ_OP_SECURE_ERASE)
1534 		*result = __send_secure_erase(ci, ti);
1535 	else if (bio_op(bio) == REQ_OP_WRITE_SAME)
1536 		*result = __send_write_same(ci, ti);
1537 	else if (bio_op(bio) == REQ_OP_WRITE_ZEROES)
1538 		*result = __send_write_zeroes(ci, ti);
1539 	else
1540 		return false;
1541 
1542 	return true;
1543 }
1544 
1545 /*
1546  * Select the correct strategy for processing a non-flush bio.
1547  */
1548 static int __split_and_process_non_flush(struct clone_info *ci)
1549 {
1550 	struct dm_target *ti;
1551 	unsigned len;
1552 	int r;
1553 
1554 	ti = dm_table_find_target(ci->map, ci->sector);
1555 	if (!dm_target_is_valid(ti))
1556 		return -EIO;
1557 
1558 	if (unlikely(__process_abnormal_io(ci, ti, &r)))
1559 		return r;
1560 
1561 	len = min_t(sector_t, max_io_len(ci->sector, ti), ci->sector_count);
1562 
1563 	r = __clone_and_map_data_bio(ci, ti, ci->sector, &len);
1564 	if (r < 0)
1565 		return r;
1566 
1567 	ci->sector += len;
1568 	ci->sector_count -= len;
1569 
1570 	return 0;
1571 }
1572 
1573 static void init_clone_info(struct clone_info *ci, struct mapped_device *md,
1574 			    struct dm_table *map, struct bio *bio)
1575 {
1576 	ci->map = map;
1577 	ci->io = alloc_io(md, bio);
1578 	ci->sector = bio->bi_iter.bi_sector;
1579 }
1580 
1581 /*
1582  * Entry point to split a bio into clones and submit them to the targets.
1583  */
1584 static blk_qc_t __split_and_process_bio(struct mapped_device *md,
1585 					struct dm_table *map, struct bio *bio)
1586 {
1587 	struct clone_info ci;
1588 	blk_qc_t ret = BLK_QC_T_NONE;
1589 	int error = 0;
1590 
1591 	if (unlikely(!map)) {
1592 		bio_io_error(bio);
1593 		return ret;
1594 	}
1595 
1596 	blk_queue_split(md->queue, &bio);
1597 
1598 	init_clone_info(&ci, md, map, bio);
1599 
1600 	if (bio->bi_opf & REQ_PREFLUSH) {
1601 		ci.bio = &ci.io->md->flush_bio;
1602 		ci.sector_count = 0;
1603 		error = __send_empty_flush(&ci);
1604 		/* dec_pending submits any data associated with flush */
1605 	} else if (bio_op(bio) == REQ_OP_ZONE_RESET) {
1606 		ci.bio = bio;
1607 		ci.sector_count = 0;
1608 		error = __split_and_process_non_flush(&ci);
1609 	} else {
1610 		ci.bio = bio;
1611 		ci.sector_count = bio_sectors(bio);
1612 		while (ci.sector_count && !error) {
1613 			error = __split_and_process_non_flush(&ci);
1614 			if (current->bio_list && ci.sector_count && !error) {
1615 				/*
1616 				 * Remainder must be passed to generic_make_request()
1617 				 * so that it gets handled *after* bios already submitted
1618 				 * have been completely processed.
1619 				 * We take a clone of the original to store in
1620 				 * ci.io->orig_bio to be used by end_io_acct() and
1621 				 * for dec_pending to use for completion handling.
1622 				 */
1623 				struct bio *b = bio_split(bio, bio_sectors(bio) - ci.sector_count,
1624 							  GFP_NOIO, &md->queue->bio_split);
1625 				ci.io->orig_bio = b;
1626 				bio_chain(b, bio);
1627 				ret = generic_make_request(bio);
1628 				break;
1629 			}
1630 		}
1631 	}
1632 
1633 	/* drop the extra reference count */
1634 	dec_pending(ci.io, errno_to_blk_status(error));
1635 	return ret;
1636 }
1637 
1638 /*
1639  * Optimized variant of __split_and_process_bio that leverages the
1640  * fact that targets that use it do _not_ have a need to split bios.
1641  */
1642 static blk_qc_t __process_bio(struct mapped_device *md,
1643 			      struct dm_table *map, struct bio *bio)
1644 {
1645 	struct clone_info ci;
1646 	blk_qc_t ret = BLK_QC_T_NONE;
1647 	int error = 0;
1648 
1649 	if (unlikely(!map)) {
1650 		bio_io_error(bio);
1651 		return ret;
1652 	}
1653 
1654 	init_clone_info(&ci, md, map, bio);
1655 
1656 	if (bio->bi_opf & REQ_PREFLUSH) {
1657 		ci.bio = &ci.io->md->flush_bio;
1658 		ci.sector_count = 0;
1659 		error = __send_empty_flush(&ci);
1660 		/* dec_pending submits any data associated with flush */
1661 	} else {
1662 		struct dm_target *ti = md->immutable_target;
1663 		struct dm_target_io *tio;
1664 
1665 		/*
1666 		 * Defend against IO still getting in during teardown
1667 		 * - as was seen for a time with nvme-fcloop
1668 		 */
1669 		if (WARN_ON_ONCE(!ti || !dm_target_is_valid(ti))) {
1670 			error = -EIO;
1671 			goto out;
1672 		}
1673 
1674 		ci.bio = bio;
1675 		ci.sector_count = bio_sectors(bio);
1676 		if (unlikely(__process_abnormal_io(&ci, ti, &error)))
1677 			goto out;
1678 
1679 		tio = alloc_tio(&ci, ti, 0, GFP_NOIO);
1680 		ret = __clone_and_map_simple_bio(&ci, tio, NULL);
1681 	}
1682 out:
1683 	/* drop the extra reference count */
1684 	dec_pending(ci.io, errno_to_blk_status(error));
1685 	return ret;
1686 }
1687 
1688 typedef blk_qc_t (process_bio_fn)(struct mapped_device *, struct dm_table *, struct bio *);
1689 
1690 static blk_qc_t __dm_make_request(struct request_queue *q, struct bio *bio,
1691 				  process_bio_fn process_bio)
1692 {
1693 	struct mapped_device *md = q->queuedata;
1694 	blk_qc_t ret = BLK_QC_T_NONE;
1695 	int srcu_idx;
1696 	struct dm_table *map;
1697 
1698 	map = dm_get_live_table(md, &srcu_idx);
1699 
1700 	/* if we're suspended, we have to queue this io for later */
1701 	if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) {
1702 		dm_put_live_table(md, srcu_idx);
1703 
1704 		if (!(bio->bi_opf & REQ_RAHEAD))
1705 			queue_io(md, bio);
1706 		else
1707 			bio_io_error(bio);
1708 		return ret;
1709 	}
1710 
1711 	ret = process_bio(md, map, bio);
1712 
1713 	dm_put_live_table(md, srcu_idx);
1714 	return ret;
1715 }
1716 
1717 /*
1718  * The request function that remaps the bio to one target and
1719  * splits off any remainder.
1720  */
1721 static blk_qc_t dm_make_request(struct request_queue *q, struct bio *bio)
1722 {
1723 	return __dm_make_request(q, bio, __split_and_process_bio);
1724 }
1725 
1726 static blk_qc_t dm_make_request_nvme(struct request_queue *q, struct bio *bio)
1727 {
1728 	return __dm_make_request(q, bio, __process_bio);
1729 }
1730 
1731 static int dm_any_congested(void *congested_data, int bdi_bits)
1732 {
1733 	int r = bdi_bits;
1734 	struct mapped_device *md = congested_data;
1735 	struct dm_table *map;
1736 
1737 	if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
1738 		if (dm_request_based(md)) {
1739 			/*
1740 			 * With request-based DM we only need to check the
1741 			 * top-level queue for congestion.
1742 			 */
1743 			r = md->queue->backing_dev_info->wb.state & bdi_bits;
1744 		} else {
1745 			map = dm_get_live_table_fast(md);
1746 			if (map)
1747 				r = dm_table_any_congested(map, bdi_bits);
1748 			dm_put_live_table_fast(md);
1749 		}
1750 	}
1751 
1752 	return r;
1753 }
1754 
1755 /*-----------------------------------------------------------------
1756  * An IDR is used to keep track of allocated minor numbers.
1757  *---------------------------------------------------------------*/
1758 static void free_minor(int minor)
1759 {
1760 	spin_lock(&_minor_lock);
1761 	idr_remove(&_minor_idr, minor);
1762 	spin_unlock(&_minor_lock);
1763 }
1764 
1765 /*
1766  * See if the device with a specific minor # is free.
1767  */
1768 static int specific_minor(int minor)
1769 {
1770 	int r;
1771 
1772 	if (minor >= (1 << MINORBITS))
1773 		return -EINVAL;
1774 
1775 	idr_preload(GFP_KERNEL);
1776 	spin_lock(&_minor_lock);
1777 
1778 	r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT);
1779 
1780 	spin_unlock(&_minor_lock);
1781 	idr_preload_end();
1782 	if (r < 0)
1783 		return r == -ENOSPC ? -EBUSY : r;
1784 	return 0;
1785 }
1786 
1787 static int next_free_minor(int *minor)
1788 {
1789 	int r;
1790 
1791 	idr_preload(GFP_KERNEL);
1792 	spin_lock(&_minor_lock);
1793 
1794 	r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT);
1795 
1796 	spin_unlock(&_minor_lock);
1797 	idr_preload_end();
1798 	if (r < 0)
1799 		return r;
1800 	*minor = r;
1801 	return 0;
1802 }
1803 
1804 static const struct block_device_operations dm_blk_dops;
1805 static const struct dax_operations dm_dax_ops;
1806 
1807 static void dm_wq_work(struct work_struct *work);
1808 
1809 static void dm_init_normal_md_queue(struct mapped_device *md)
1810 {
1811 	/*
1812 	 * Initialize aspects of queue that aren't relevant for blk-mq
1813 	 */
1814 	md->queue->backing_dev_info->congested_fn = dm_any_congested;
1815 }
1816 
1817 static void cleanup_mapped_device(struct mapped_device *md)
1818 {
1819 	if (md->wq)
1820 		destroy_workqueue(md->wq);
1821 	bioset_exit(&md->bs);
1822 	bioset_exit(&md->io_bs);
1823 
1824 	if (md->dax_dev) {
1825 		kill_dax(md->dax_dev);
1826 		put_dax(md->dax_dev);
1827 		md->dax_dev = NULL;
1828 	}
1829 
1830 	if (md->disk) {
1831 		spin_lock(&_minor_lock);
1832 		md->disk->private_data = NULL;
1833 		spin_unlock(&_minor_lock);
1834 		del_gendisk(md->disk);
1835 		put_disk(md->disk);
1836 	}
1837 
1838 	if (md->queue)
1839 		blk_cleanup_queue(md->queue);
1840 
1841 	cleanup_srcu_struct(&md->io_barrier);
1842 
1843 	if (md->bdev) {
1844 		bdput(md->bdev);
1845 		md->bdev = NULL;
1846 	}
1847 
1848 	mutex_destroy(&md->suspend_lock);
1849 	mutex_destroy(&md->type_lock);
1850 	mutex_destroy(&md->table_devices_lock);
1851 
1852 	dm_mq_cleanup_mapped_device(md);
1853 }
1854 
1855 /*
1856  * Allocate and initialise a blank device with a given minor.
1857  */
1858 static struct mapped_device *alloc_dev(int minor)
1859 {
1860 	int r, numa_node_id = dm_get_numa_node();
1861 	struct dax_device *dax_dev = NULL;
1862 	struct mapped_device *md;
1863 	void *old_md;
1864 
1865 	md = kvzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id);
1866 	if (!md) {
1867 		DMWARN("unable to allocate device, out of memory.");
1868 		return NULL;
1869 	}
1870 
1871 	if (!try_module_get(THIS_MODULE))
1872 		goto bad_module_get;
1873 
1874 	/* get a minor number for the dev */
1875 	if (minor == DM_ANY_MINOR)
1876 		r = next_free_minor(&minor);
1877 	else
1878 		r = specific_minor(minor);
1879 	if (r < 0)
1880 		goto bad_minor;
1881 
1882 	r = init_srcu_struct(&md->io_barrier);
1883 	if (r < 0)
1884 		goto bad_io_barrier;
1885 
1886 	md->numa_node_id = numa_node_id;
1887 	md->init_tio_pdu = false;
1888 	md->type = DM_TYPE_NONE;
1889 	mutex_init(&md->suspend_lock);
1890 	mutex_init(&md->type_lock);
1891 	mutex_init(&md->table_devices_lock);
1892 	spin_lock_init(&md->deferred_lock);
1893 	atomic_set(&md->holders, 1);
1894 	atomic_set(&md->open_count, 0);
1895 	atomic_set(&md->event_nr, 0);
1896 	atomic_set(&md->uevent_seq, 0);
1897 	INIT_LIST_HEAD(&md->uevent_list);
1898 	INIT_LIST_HEAD(&md->table_devices);
1899 	spin_lock_init(&md->uevent_lock);
1900 
1901 	md->queue = blk_alloc_queue_node(GFP_KERNEL, numa_node_id, NULL);
1902 	if (!md->queue)
1903 		goto bad;
1904 	md->queue->queuedata = md;
1905 	md->queue->backing_dev_info->congested_data = md;
1906 
1907 	md->disk = alloc_disk_node(1, md->numa_node_id);
1908 	if (!md->disk)
1909 		goto bad;
1910 
1911 	atomic_set(&md->pending[0], 0);
1912 	atomic_set(&md->pending[1], 0);
1913 	init_waitqueue_head(&md->wait);
1914 	INIT_WORK(&md->work, dm_wq_work);
1915 	init_waitqueue_head(&md->eventq);
1916 	init_completion(&md->kobj_holder.completion);
1917 
1918 	md->disk->major = _major;
1919 	md->disk->first_minor = minor;
1920 	md->disk->fops = &dm_blk_dops;
1921 	md->disk->queue = md->queue;
1922 	md->disk->private_data = md;
1923 	sprintf(md->disk->disk_name, "dm-%d", minor);
1924 
1925 	if (IS_ENABLED(CONFIG_DAX_DRIVER)) {
1926 		dax_dev = alloc_dax(md, md->disk->disk_name, &dm_dax_ops);
1927 		if (!dax_dev)
1928 			goto bad;
1929 	}
1930 	md->dax_dev = dax_dev;
1931 
1932 	add_disk_no_queue_reg(md->disk);
1933 	format_dev_t(md->name, MKDEV(_major, minor));
1934 
1935 	md->wq = alloc_workqueue("kdmflush", WQ_MEM_RECLAIM, 0);
1936 	if (!md->wq)
1937 		goto bad;
1938 
1939 	md->bdev = bdget_disk(md->disk, 0);
1940 	if (!md->bdev)
1941 		goto bad;
1942 
1943 	bio_init(&md->flush_bio, NULL, 0);
1944 	bio_set_dev(&md->flush_bio, md->bdev);
1945 	md->flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC;
1946 
1947 	dm_stats_init(&md->stats);
1948 
1949 	/* Populate the mapping, nobody knows we exist yet */
1950 	spin_lock(&_minor_lock);
1951 	old_md = idr_replace(&_minor_idr, md, minor);
1952 	spin_unlock(&_minor_lock);
1953 
1954 	BUG_ON(old_md != MINOR_ALLOCED);
1955 
1956 	return md;
1957 
1958 bad:
1959 	cleanup_mapped_device(md);
1960 bad_io_barrier:
1961 	free_minor(minor);
1962 bad_minor:
1963 	module_put(THIS_MODULE);
1964 bad_module_get:
1965 	kvfree(md);
1966 	return NULL;
1967 }
1968 
1969 static void unlock_fs(struct mapped_device *md);
1970 
1971 static void free_dev(struct mapped_device *md)
1972 {
1973 	int minor = MINOR(disk_devt(md->disk));
1974 
1975 	unlock_fs(md);
1976 
1977 	cleanup_mapped_device(md);
1978 
1979 	free_table_devices(&md->table_devices);
1980 	dm_stats_cleanup(&md->stats);
1981 	free_minor(minor);
1982 
1983 	module_put(THIS_MODULE);
1984 	kvfree(md);
1985 }
1986 
1987 static int __bind_mempools(struct mapped_device *md, struct dm_table *t)
1988 {
1989 	struct dm_md_mempools *p = dm_table_get_md_mempools(t);
1990 	int ret = 0;
1991 
1992 	if (dm_table_bio_based(t)) {
1993 		/*
1994 		 * The md may already have mempools that need changing.
1995 		 * If so, reload bioset because front_pad may have changed
1996 		 * because a different table was loaded.
1997 		 */
1998 		bioset_exit(&md->bs);
1999 		bioset_exit(&md->io_bs);
2000 
2001 	} else if (bioset_initialized(&md->bs)) {
2002 		/*
2003 		 * There's no need to reload with request-based dm
2004 		 * because the size of front_pad doesn't change.
2005 		 * Note for future: If you are to reload bioset,
2006 		 * prep-ed requests in the queue may refer
2007 		 * to bio from the old bioset, so you must walk
2008 		 * through the queue to unprep.
2009 		 */
2010 		goto out;
2011 	}
2012 
2013 	BUG_ON(!p ||
2014 	       bioset_initialized(&md->bs) ||
2015 	       bioset_initialized(&md->io_bs));
2016 
2017 	ret = bioset_init_from_src(&md->bs, &p->bs);
2018 	if (ret)
2019 		goto out;
2020 	ret = bioset_init_from_src(&md->io_bs, &p->io_bs);
2021 	if (ret)
2022 		bioset_exit(&md->bs);
2023 out:
2024 	/* mempool bind completed, no longer need any mempools in the table */
2025 	dm_table_free_md_mempools(t);
2026 	return ret;
2027 }
2028 
2029 /*
2030  * Bind a table to the device.
2031  */
2032 static void event_callback(void *context)
2033 {
2034 	unsigned long flags;
2035 	LIST_HEAD(uevents);
2036 	struct mapped_device *md = (struct mapped_device *) context;
2037 
2038 	spin_lock_irqsave(&md->uevent_lock, flags);
2039 	list_splice_init(&md->uevent_list, &uevents);
2040 	spin_unlock_irqrestore(&md->uevent_lock, flags);
2041 
2042 	dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
2043 
2044 	atomic_inc(&md->event_nr);
2045 	wake_up(&md->eventq);
2046 	dm_issue_global_event();
2047 }
2048 
2049 /*
2050  * Protected by md->suspend_lock obtained by dm_swap_table().
2051  */
2052 static void __set_size(struct mapped_device *md, sector_t size)
2053 {
2054 	lockdep_assert_held(&md->suspend_lock);
2055 
2056 	set_capacity(md->disk, size);
2057 
2058 	i_size_write(md->bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
2059 }
2060 
2061 /*
2062  * Returns old map, which caller must destroy.
2063  */
2064 static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
2065 			       struct queue_limits *limits)
2066 {
2067 	struct dm_table *old_map;
2068 	struct request_queue *q = md->queue;
2069 	bool request_based = dm_table_request_based(t);
2070 	sector_t size;
2071 	int ret;
2072 
2073 	lockdep_assert_held(&md->suspend_lock);
2074 
2075 	size = dm_table_get_size(t);
2076 
2077 	/*
2078 	 * Wipe any geometry if the size of the table changed.
2079 	 */
2080 	if (size != dm_get_size(md))
2081 		memset(&md->geometry, 0, sizeof(md->geometry));
2082 
2083 	__set_size(md, size);
2084 
2085 	dm_table_event_callback(t, event_callback, md);
2086 
2087 	/*
2088 	 * The queue hasn't been stopped yet, if the old table type wasn't
2089 	 * for request-based during suspension.  So stop it to prevent
2090 	 * I/O mapping before resume.
2091 	 * This must be done before setting the queue restrictions,
2092 	 * because request-based dm may be run just after the setting.
2093 	 */
2094 	if (request_based)
2095 		dm_stop_queue(q);
2096 
2097 	if (request_based || md->type == DM_TYPE_NVME_BIO_BASED) {
2098 		/*
2099 		 * Leverage the fact that request-based DM targets and
2100 		 * NVMe bio based targets are immutable singletons
2101 		 * - used to optimize both dm_request_fn and dm_mq_queue_rq;
2102 		 *   and __process_bio.
2103 		 */
2104 		md->immutable_target = dm_table_get_immutable_target(t);
2105 	}
2106 
2107 	ret = __bind_mempools(md, t);
2108 	if (ret) {
2109 		old_map = ERR_PTR(ret);
2110 		goto out;
2111 	}
2112 
2113 	old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2114 	rcu_assign_pointer(md->map, (void *)t);
2115 	md->immutable_target_type = dm_table_get_immutable_target_type(t);
2116 
2117 	dm_table_set_restrictions(t, q, limits);
2118 	if (old_map)
2119 		dm_sync_table(md);
2120 
2121 out:
2122 	return old_map;
2123 }
2124 
2125 /*
2126  * Returns unbound table for the caller to free.
2127  */
2128 static struct dm_table *__unbind(struct mapped_device *md)
2129 {
2130 	struct dm_table *map = rcu_dereference_protected(md->map, 1);
2131 
2132 	if (!map)
2133 		return NULL;
2134 
2135 	dm_table_event_callback(map, NULL, NULL);
2136 	RCU_INIT_POINTER(md->map, NULL);
2137 	dm_sync_table(md);
2138 
2139 	return map;
2140 }
2141 
2142 /*
2143  * Constructor for a new device.
2144  */
2145 int dm_create(int minor, struct mapped_device **result)
2146 {
2147 	int r;
2148 	struct mapped_device *md;
2149 
2150 	md = alloc_dev(minor);
2151 	if (!md)
2152 		return -ENXIO;
2153 
2154 	r = dm_sysfs_init(md);
2155 	if (r) {
2156 		free_dev(md);
2157 		return r;
2158 	}
2159 
2160 	*result = md;
2161 	return 0;
2162 }
2163 
2164 /*
2165  * Functions to manage md->type.
2166  * All are required to hold md->type_lock.
2167  */
2168 void dm_lock_md_type(struct mapped_device *md)
2169 {
2170 	mutex_lock(&md->type_lock);
2171 }
2172 
2173 void dm_unlock_md_type(struct mapped_device *md)
2174 {
2175 	mutex_unlock(&md->type_lock);
2176 }
2177 
2178 void dm_set_md_type(struct mapped_device *md, enum dm_queue_mode type)
2179 {
2180 	BUG_ON(!mutex_is_locked(&md->type_lock));
2181 	md->type = type;
2182 }
2183 
2184 enum dm_queue_mode dm_get_md_type(struct mapped_device *md)
2185 {
2186 	return md->type;
2187 }
2188 
2189 struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
2190 {
2191 	return md->immutable_target_type;
2192 }
2193 
2194 /*
2195  * The queue_limits are only valid as long as you have a reference
2196  * count on 'md'.
2197  */
2198 struct queue_limits *dm_get_queue_limits(struct mapped_device *md)
2199 {
2200 	BUG_ON(!atomic_read(&md->holders));
2201 	return &md->queue->limits;
2202 }
2203 EXPORT_SYMBOL_GPL(dm_get_queue_limits);
2204 
2205 /*
2206  * Setup the DM device's queue based on md's type
2207  */
2208 int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
2209 {
2210 	int r;
2211 	struct queue_limits limits;
2212 	enum dm_queue_mode type = dm_get_md_type(md);
2213 
2214 	switch (type) {
2215 	case DM_TYPE_REQUEST_BASED:
2216 		r = dm_mq_init_request_queue(md, t);
2217 		if (r) {
2218 			DMERR("Cannot initialize queue for request-based dm-mq mapped device");
2219 			return r;
2220 		}
2221 		break;
2222 	case DM_TYPE_BIO_BASED:
2223 	case DM_TYPE_DAX_BIO_BASED:
2224 		dm_init_normal_md_queue(md);
2225 		blk_queue_make_request(md->queue, dm_make_request);
2226 		break;
2227 	case DM_TYPE_NVME_BIO_BASED:
2228 		dm_init_normal_md_queue(md);
2229 		blk_queue_make_request(md->queue, dm_make_request_nvme);
2230 		break;
2231 	case DM_TYPE_NONE:
2232 		WARN_ON_ONCE(true);
2233 		break;
2234 	}
2235 
2236 	r = dm_calculate_queue_limits(t, &limits);
2237 	if (r) {
2238 		DMERR("Cannot calculate initial queue limits");
2239 		return r;
2240 	}
2241 	dm_table_set_restrictions(t, md->queue, &limits);
2242 	blk_register_queue(md->disk);
2243 
2244 	return 0;
2245 }
2246 
2247 struct mapped_device *dm_get_md(dev_t dev)
2248 {
2249 	struct mapped_device *md;
2250 	unsigned minor = MINOR(dev);
2251 
2252 	if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
2253 		return NULL;
2254 
2255 	spin_lock(&_minor_lock);
2256 
2257 	md = idr_find(&_minor_idr, minor);
2258 	if (!md || md == MINOR_ALLOCED || (MINOR(disk_devt(dm_disk(md))) != minor) ||
2259 	    test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
2260 		md = NULL;
2261 		goto out;
2262 	}
2263 	dm_get(md);
2264 out:
2265 	spin_unlock(&_minor_lock);
2266 
2267 	return md;
2268 }
2269 EXPORT_SYMBOL_GPL(dm_get_md);
2270 
2271 void *dm_get_mdptr(struct mapped_device *md)
2272 {
2273 	return md->interface_ptr;
2274 }
2275 
2276 void dm_set_mdptr(struct mapped_device *md, void *ptr)
2277 {
2278 	md->interface_ptr = ptr;
2279 }
2280 
2281 void dm_get(struct mapped_device *md)
2282 {
2283 	atomic_inc(&md->holders);
2284 	BUG_ON(test_bit(DMF_FREEING, &md->flags));
2285 }
2286 
2287 int dm_hold(struct mapped_device *md)
2288 {
2289 	spin_lock(&_minor_lock);
2290 	if (test_bit(DMF_FREEING, &md->flags)) {
2291 		spin_unlock(&_minor_lock);
2292 		return -EBUSY;
2293 	}
2294 	dm_get(md);
2295 	spin_unlock(&_minor_lock);
2296 	return 0;
2297 }
2298 EXPORT_SYMBOL_GPL(dm_hold);
2299 
2300 const char *dm_device_name(struct mapped_device *md)
2301 {
2302 	return md->name;
2303 }
2304 EXPORT_SYMBOL_GPL(dm_device_name);
2305 
2306 static void __dm_destroy(struct mapped_device *md, bool wait)
2307 {
2308 	struct dm_table *map;
2309 	int srcu_idx;
2310 
2311 	might_sleep();
2312 
2313 	spin_lock(&_minor_lock);
2314 	idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));
2315 	set_bit(DMF_FREEING, &md->flags);
2316 	spin_unlock(&_minor_lock);
2317 
2318 	blk_set_queue_dying(md->queue);
2319 
2320 	/*
2321 	 * Take suspend_lock so that presuspend and postsuspend methods
2322 	 * do not race with internal suspend.
2323 	 */
2324 	mutex_lock(&md->suspend_lock);
2325 	map = dm_get_live_table(md, &srcu_idx);
2326 	if (!dm_suspended_md(md)) {
2327 		dm_table_presuspend_targets(map);
2328 		dm_table_postsuspend_targets(map);
2329 	}
2330 	/* dm_put_live_table must be before msleep, otherwise deadlock is possible */
2331 	dm_put_live_table(md, srcu_idx);
2332 	mutex_unlock(&md->suspend_lock);
2333 
2334 	/*
2335 	 * Rare, but there may be I/O requests still going to complete,
2336 	 * for example.  Wait for all references to disappear.
2337 	 * No one should increment the reference count of the mapped_device,
2338 	 * after the mapped_device state becomes DMF_FREEING.
2339 	 */
2340 	if (wait)
2341 		while (atomic_read(&md->holders))
2342 			msleep(1);
2343 	else if (atomic_read(&md->holders))
2344 		DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)",
2345 		       dm_device_name(md), atomic_read(&md->holders));
2346 
2347 	dm_sysfs_exit(md);
2348 	dm_table_destroy(__unbind(md));
2349 	free_dev(md);
2350 }
2351 
2352 void dm_destroy(struct mapped_device *md)
2353 {
2354 	__dm_destroy(md, true);
2355 }
2356 
2357 void dm_destroy_immediate(struct mapped_device *md)
2358 {
2359 	__dm_destroy(md, false);
2360 }
2361 
2362 void dm_put(struct mapped_device *md)
2363 {
2364 	atomic_dec(&md->holders);
2365 }
2366 EXPORT_SYMBOL_GPL(dm_put);
2367 
2368 static int dm_wait_for_completion(struct mapped_device *md, long task_state)
2369 {
2370 	int r = 0;
2371 	DEFINE_WAIT(wait);
2372 
2373 	while (1) {
2374 		prepare_to_wait(&md->wait, &wait, task_state);
2375 
2376 		if (!md_in_flight(md))
2377 			break;
2378 
2379 		if (signal_pending_state(task_state, current)) {
2380 			r = -EINTR;
2381 			break;
2382 		}
2383 
2384 		io_schedule();
2385 	}
2386 	finish_wait(&md->wait, &wait);
2387 
2388 	return r;
2389 }
2390 
2391 /*
2392  * Process the deferred bios
2393  */
2394 static void dm_wq_work(struct work_struct *work)
2395 {
2396 	struct mapped_device *md = container_of(work, struct mapped_device,
2397 						work);
2398 	struct bio *c;
2399 	int srcu_idx;
2400 	struct dm_table *map;
2401 
2402 	map = dm_get_live_table(md, &srcu_idx);
2403 
2404 	while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
2405 		spin_lock_irq(&md->deferred_lock);
2406 		c = bio_list_pop(&md->deferred);
2407 		spin_unlock_irq(&md->deferred_lock);
2408 
2409 		if (!c)
2410 			break;
2411 
2412 		if (dm_request_based(md))
2413 			generic_make_request(c);
2414 		else
2415 			__split_and_process_bio(md, map, c);
2416 	}
2417 
2418 	dm_put_live_table(md, srcu_idx);
2419 }
2420 
2421 static void dm_queue_flush(struct mapped_device *md)
2422 {
2423 	clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2424 	smp_mb__after_atomic();
2425 	queue_work(md->wq, &md->work);
2426 }
2427 
2428 /*
2429  * Swap in a new table, returning the old one for the caller to destroy.
2430  */
2431 struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table)
2432 {
2433 	struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL);
2434 	struct queue_limits limits;
2435 	int r;
2436 
2437 	mutex_lock(&md->suspend_lock);
2438 
2439 	/* device must be suspended */
2440 	if (!dm_suspended_md(md))
2441 		goto out;
2442 
2443 	/*
2444 	 * If the new table has no data devices, retain the existing limits.
2445 	 * This helps multipath with queue_if_no_path if all paths disappear,
2446 	 * then new I/O is queued based on these limits, and then some paths
2447 	 * reappear.
2448 	 */
2449 	if (dm_table_has_no_data_devices(table)) {
2450 		live_map = dm_get_live_table_fast(md);
2451 		if (live_map)
2452 			limits = md->queue->limits;
2453 		dm_put_live_table_fast(md);
2454 	}
2455 
2456 	if (!live_map) {
2457 		r = dm_calculate_queue_limits(table, &limits);
2458 		if (r) {
2459 			map = ERR_PTR(r);
2460 			goto out;
2461 		}
2462 	}
2463 
2464 	map = __bind(md, table, &limits);
2465 	dm_issue_global_event();
2466 
2467 out:
2468 	mutex_unlock(&md->suspend_lock);
2469 	return map;
2470 }
2471 
2472 /*
2473  * Functions to lock and unlock any filesystem running on the
2474  * device.
2475  */
2476 static int lock_fs(struct mapped_device *md)
2477 {
2478 	int r;
2479 
2480 	WARN_ON(md->frozen_sb);
2481 
2482 	md->frozen_sb = freeze_bdev(md->bdev);
2483 	if (IS_ERR(md->frozen_sb)) {
2484 		r = PTR_ERR(md->frozen_sb);
2485 		md->frozen_sb = NULL;
2486 		return r;
2487 	}
2488 
2489 	set_bit(DMF_FROZEN, &md->flags);
2490 
2491 	return 0;
2492 }
2493 
2494 static void unlock_fs(struct mapped_device *md)
2495 {
2496 	if (!test_bit(DMF_FROZEN, &md->flags))
2497 		return;
2498 
2499 	thaw_bdev(md->bdev, md->frozen_sb);
2500 	md->frozen_sb = NULL;
2501 	clear_bit(DMF_FROZEN, &md->flags);
2502 }
2503 
2504 /*
2505  * @suspend_flags: DM_SUSPEND_LOCKFS_FLAG and/or DM_SUSPEND_NOFLUSH_FLAG
2506  * @task_state: e.g. TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE
2507  * @dmf_suspended_flag: DMF_SUSPENDED or DMF_SUSPENDED_INTERNALLY
2508  *
2509  * If __dm_suspend returns 0, the device is completely quiescent
2510  * now. There is no request-processing activity. All new requests
2511  * are being added to md->deferred list.
2512  */
2513 static int __dm_suspend(struct mapped_device *md, struct dm_table *map,
2514 			unsigned suspend_flags, long task_state,
2515 			int dmf_suspended_flag)
2516 {
2517 	bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG;
2518 	bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG;
2519 	int r;
2520 
2521 	lockdep_assert_held(&md->suspend_lock);
2522 
2523 	/*
2524 	 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
2525 	 * This flag is cleared before dm_suspend returns.
2526 	 */
2527 	if (noflush)
2528 		set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2529 	else
2530 		pr_debug("%s: suspending with flush\n", dm_device_name(md));
2531 
2532 	/*
2533 	 * This gets reverted if there's an error later and the targets
2534 	 * provide the .presuspend_undo hook.
2535 	 */
2536 	dm_table_presuspend_targets(map);
2537 
2538 	/*
2539 	 * Flush I/O to the device.
2540 	 * Any I/O submitted after lock_fs() may not be flushed.
2541 	 * noflush takes precedence over do_lockfs.
2542 	 * (lock_fs() flushes I/Os and waits for them to complete.)
2543 	 */
2544 	if (!noflush && do_lockfs) {
2545 		r = lock_fs(md);
2546 		if (r) {
2547 			dm_table_presuspend_undo_targets(map);
2548 			return r;
2549 		}
2550 	}
2551 
2552 	/*
2553 	 * Here we must make sure that no processes are submitting requests
2554 	 * to target drivers i.e. no one may be executing
2555 	 * __split_and_process_bio. This is called from dm_request and
2556 	 * dm_wq_work.
2557 	 *
2558 	 * To get all processes out of __split_and_process_bio in dm_request,
2559 	 * we take the write lock. To prevent any process from reentering
2560 	 * __split_and_process_bio from dm_request and quiesce the thread
2561 	 * (dm_wq_work), we set BMF_BLOCK_IO_FOR_SUSPEND and call
2562 	 * flush_workqueue(md->wq).
2563 	 */
2564 	set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2565 	if (map)
2566 		synchronize_srcu(&md->io_barrier);
2567 
2568 	/*
2569 	 * Stop md->queue before flushing md->wq in case request-based
2570 	 * dm defers requests to md->wq from md->queue.
2571 	 */
2572 	if (dm_request_based(md))
2573 		dm_stop_queue(md->queue);
2574 
2575 	flush_workqueue(md->wq);
2576 
2577 	/*
2578 	 * At this point no more requests are entering target request routines.
2579 	 * We call dm_wait_for_completion to wait for all existing requests
2580 	 * to finish.
2581 	 */
2582 	r = dm_wait_for_completion(md, task_state);
2583 	if (!r)
2584 		set_bit(dmf_suspended_flag, &md->flags);
2585 
2586 	if (noflush)
2587 		clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2588 	if (map)
2589 		synchronize_srcu(&md->io_barrier);
2590 
2591 	/* were we interrupted ? */
2592 	if (r < 0) {
2593 		dm_queue_flush(md);
2594 
2595 		if (dm_request_based(md))
2596 			dm_start_queue(md->queue);
2597 
2598 		unlock_fs(md);
2599 		dm_table_presuspend_undo_targets(map);
2600 		/* pushback list is already flushed, so skip flush */
2601 	}
2602 
2603 	return r;
2604 }
2605 
2606 /*
2607  * We need to be able to change a mapping table under a mounted
2608  * filesystem.  For example we might want to move some data in
2609  * the background.  Before the table can be swapped with
2610  * dm_bind_table, dm_suspend must be called to flush any in
2611  * flight bios and ensure that any further io gets deferred.
2612  */
2613 /*
2614  * Suspend mechanism in request-based dm.
2615  *
2616  * 1. Flush all I/Os by lock_fs() if needed.
2617  * 2. Stop dispatching any I/O by stopping the request_queue.
2618  * 3. Wait for all in-flight I/Os to be completed or requeued.
2619  *
2620  * To abort suspend, start the request_queue.
2621  */
2622 int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
2623 {
2624 	struct dm_table *map = NULL;
2625 	int r = 0;
2626 
2627 retry:
2628 	mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2629 
2630 	if (dm_suspended_md(md)) {
2631 		r = -EINVAL;
2632 		goto out_unlock;
2633 	}
2634 
2635 	if (dm_suspended_internally_md(md)) {
2636 		/* already internally suspended, wait for internal resume */
2637 		mutex_unlock(&md->suspend_lock);
2638 		r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2639 		if (r)
2640 			return r;
2641 		goto retry;
2642 	}
2643 
2644 	map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2645 
2646 	r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED);
2647 	if (r)
2648 		goto out_unlock;
2649 
2650 	dm_table_postsuspend_targets(map);
2651 
2652 out_unlock:
2653 	mutex_unlock(&md->suspend_lock);
2654 	return r;
2655 }
2656 
2657 static int __dm_resume(struct mapped_device *md, struct dm_table *map)
2658 {
2659 	if (map) {
2660 		int r = dm_table_resume_targets(map);
2661 		if (r)
2662 			return r;
2663 	}
2664 
2665 	dm_queue_flush(md);
2666 
2667 	/*
2668 	 * Flushing deferred I/Os must be done after targets are resumed
2669 	 * so that mapping of targets can work correctly.
2670 	 * Request-based dm is queueing the deferred I/Os in its request_queue.
2671 	 */
2672 	if (dm_request_based(md))
2673 		dm_start_queue(md->queue);
2674 
2675 	unlock_fs(md);
2676 
2677 	return 0;
2678 }
2679 
2680 int dm_resume(struct mapped_device *md)
2681 {
2682 	int r;
2683 	struct dm_table *map = NULL;
2684 
2685 retry:
2686 	r = -EINVAL;
2687 	mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2688 
2689 	if (!dm_suspended_md(md))
2690 		goto out;
2691 
2692 	if (dm_suspended_internally_md(md)) {
2693 		/* already internally suspended, wait for internal resume */
2694 		mutex_unlock(&md->suspend_lock);
2695 		r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2696 		if (r)
2697 			return r;
2698 		goto retry;
2699 	}
2700 
2701 	map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2702 	if (!map || !dm_table_get_size(map))
2703 		goto out;
2704 
2705 	r = __dm_resume(md, map);
2706 	if (r)
2707 		goto out;
2708 
2709 	clear_bit(DMF_SUSPENDED, &md->flags);
2710 out:
2711 	mutex_unlock(&md->suspend_lock);
2712 
2713 	return r;
2714 }
2715 
2716 /*
2717  * Internal suspend/resume works like userspace-driven suspend. It waits
2718  * until all bios finish and prevents issuing new bios to the target drivers.
2719  * It may be used only from the kernel.
2720  */
2721 
2722 static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags)
2723 {
2724 	struct dm_table *map = NULL;
2725 
2726 	lockdep_assert_held(&md->suspend_lock);
2727 
2728 	if (md->internal_suspend_count++)
2729 		return; /* nested internal suspend */
2730 
2731 	if (dm_suspended_md(md)) {
2732 		set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2733 		return; /* nest suspend */
2734 	}
2735 
2736 	map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2737 
2738 	/*
2739 	 * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is
2740 	 * supported.  Properly supporting a TASK_INTERRUPTIBLE internal suspend
2741 	 * would require changing .presuspend to return an error -- avoid this
2742 	 * until there is a need for more elaborate variants of internal suspend.
2743 	 */
2744 	(void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE,
2745 			    DMF_SUSPENDED_INTERNALLY);
2746 
2747 	dm_table_postsuspend_targets(map);
2748 }
2749 
2750 static void __dm_internal_resume(struct mapped_device *md)
2751 {
2752 	BUG_ON(!md->internal_suspend_count);
2753 
2754 	if (--md->internal_suspend_count)
2755 		return; /* resume from nested internal suspend */
2756 
2757 	if (dm_suspended_md(md))
2758 		goto done; /* resume from nested suspend */
2759 
2760 	/*
2761 	 * NOTE: existing callers don't need to call dm_table_resume_targets
2762 	 * (which may fail -- so best to avoid it for now by passing NULL map)
2763 	 */
2764 	(void) __dm_resume(md, NULL);
2765 
2766 done:
2767 	clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2768 	smp_mb__after_atomic();
2769 	wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY);
2770 }
2771 
2772 void dm_internal_suspend_noflush(struct mapped_device *md)
2773 {
2774 	mutex_lock(&md->suspend_lock);
2775 	__dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG);
2776 	mutex_unlock(&md->suspend_lock);
2777 }
2778 EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush);
2779 
2780 void dm_internal_resume(struct mapped_device *md)
2781 {
2782 	mutex_lock(&md->suspend_lock);
2783 	__dm_internal_resume(md);
2784 	mutex_unlock(&md->suspend_lock);
2785 }
2786 EXPORT_SYMBOL_GPL(dm_internal_resume);
2787 
2788 /*
2789  * Fast variants of internal suspend/resume hold md->suspend_lock,
2790  * which prevents interaction with userspace-driven suspend.
2791  */
2792 
2793 void dm_internal_suspend_fast(struct mapped_device *md)
2794 {
2795 	mutex_lock(&md->suspend_lock);
2796 	if (dm_suspended_md(md) || dm_suspended_internally_md(md))
2797 		return;
2798 
2799 	set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2800 	synchronize_srcu(&md->io_barrier);
2801 	flush_workqueue(md->wq);
2802 	dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
2803 }
2804 EXPORT_SYMBOL_GPL(dm_internal_suspend_fast);
2805 
2806 void dm_internal_resume_fast(struct mapped_device *md)
2807 {
2808 	if (dm_suspended_md(md) || dm_suspended_internally_md(md))
2809 		goto done;
2810 
2811 	dm_queue_flush(md);
2812 
2813 done:
2814 	mutex_unlock(&md->suspend_lock);
2815 }
2816 EXPORT_SYMBOL_GPL(dm_internal_resume_fast);
2817 
2818 /*-----------------------------------------------------------------
2819  * Event notification.
2820  *---------------------------------------------------------------*/
2821 int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action,
2822 		       unsigned cookie)
2823 {
2824 	char udev_cookie[DM_COOKIE_LENGTH];
2825 	char *envp[] = { udev_cookie, NULL };
2826 
2827 	if (!cookie)
2828 		return kobject_uevent(&disk_to_dev(md->disk)->kobj, action);
2829 	else {
2830 		snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u",
2831 			 DM_COOKIE_ENV_VAR_NAME, cookie);
2832 		return kobject_uevent_env(&disk_to_dev(md->disk)->kobj,
2833 					  action, envp);
2834 	}
2835 }
2836 
2837 uint32_t dm_next_uevent_seq(struct mapped_device *md)
2838 {
2839 	return atomic_add_return(1, &md->uevent_seq);
2840 }
2841 
2842 uint32_t dm_get_event_nr(struct mapped_device *md)
2843 {
2844 	return atomic_read(&md->event_nr);
2845 }
2846 
2847 int dm_wait_event(struct mapped_device *md, int event_nr)
2848 {
2849 	return wait_event_interruptible(md->eventq,
2850 			(event_nr != atomic_read(&md->event_nr)));
2851 }
2852 
2853 void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
2854 {
2855 	unsigned long flags;
2856 
2857 	spin_lock_irqsave(&md->uevent_lock, flags);
2858 	list_add(elist, &md->uevent_list);
2859 	spin_unlock_irqrestore(&md->uevent_lock, flags);
2860 }
2861 
2862 /*
2863  * The gendisk is only valid as long as you have a reference
2864  * count on 'md'.
2865  */
2866 struct gendisk *dm_disk(struct mapped_device *md)
2867 {
2868 	return md->disk;
2869 }
2870 EXPORT_SYMBOL_GPL(dm_disk);
2871 
2872 struct kobject *dm_kobject(struct mapped_device *md)
2873 {
2874 	return &md->kobj_holder.kobj;
2875 }
2876 
2877 struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
2878 {
2879 	struct mapped_device *md;
2880 
2881 	md = container_of(kobj, struct mapped_device, kobj_holder.kobj);
2882 
2883 	spin_lock(&_minor_lock);
2884 	if (test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
2885 		md = NULL;
2886 		goto out;
2887 	}
2888 	dm_get(md);
2889 out:
2890 	spin_unlock(&_minor_lock);
2891 
2892 	return md;
2893 }
2894 
2895 int dm_suspended_md(struct mapped_device *md)
2896 {
2897 	return test_bit(DMF_SUSPENDED, &md->flags);
2898 }
2899 
2900 int dm_suspended_internally_md(struct mapped_device *md)
2901 {
2902 	return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2903 }
2904 
2905 int dm_test_deferred_remove_flag(struct mapped_device *md)
2906 {
2907 	return test_bit(DMF_DEFERRED_REMOVE, &md->flags);
2908 }
2909 
2910 int dm_suspended(struct dm_target *ti)
2911 {
2912 	return dm_suspended_md(dm_table_get_md(ti->table));
2913 }
2914 EXPORT_SYMBOL_GPL(dm_suspended);
2915 
2916 int dm_noflush_suspending(struct dm_target *ti)
2917 {
2918 	return __noflush_suspending(dm_table_get_md(ti->table));
2919 }
2920 EXPORT_SYMBOL_GPL(dm_noflush_suspending);
2921 
2922 struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_queue_mode type,
2923 					    unsigned integrity, unsigned per_io_data_size,
2924 					    unsigned min_pool_size)
2925 {
2926 	struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id);
2927 	unsigned int pool_size = 0;
2928 	unsigned int front_pad, io_front_pad;
2929 	int ret;
2930 
2931 	if (!pools)
2932 		return NULL;
2933 
2934 	switch (type) {
2935 	case DM_TYPE_BIO_BASED:
2936 	case DM_TYPE_DAX_BIO_BASED:
2937 	case DM_TYPE_NVME_BIO_BASED:
2938 		pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size);
2939 		front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);
2940 		io_front_pad = roundup(front_pad,  __alignof__(struct dm_io)) + offsetof(struct dm_io, tio);
2941 		ret = bioset_init(&pools->io_bs, pool_size, io_front_pad, 0);
2942 		if (ret)
2943 			goto out;
2944 		if (integrity && bioset_integrity_create(&pools->io_bs, pool_size))
2945 			goto out;
2946 		break;
2947 	case DM_TYPE_REQUEST_BASED:
2948 		pool_size = max(dm_get_reserved_rq_based_ios(), min_pool_size);
2949 		front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
2950 		/* per_io_data_size is used for blk-mq pdu at queue allocation */
2951 		break;
2952 	default:
2953 		BUG();
2954 	}
2955 
2956 	ret = bioset_init(&pools->bs, pool_size, front_pad, 0);
2957 	if (ret)
2958 		goto out;
2959 
2960 	if (integrity && bioset_integrity_create(&pools->bs, pool_size))
2961 		goto out;
2962 
2963 	return pools;
2964 
2965 out:
2966 	dm_free_md_mempools(pools);
2967 
2968 	return NULL;
2969 }
2970 
2971 void dm_free_md_mempools(struct dm_md_mempools *pools)
2972 {
2973 	if (!pools)
2974 		return;
2975 
2976 	bioset_exit(&pools->bs);
2977 	bioset_exit(&pools->io_bs);
2978 
2979 	kfree(pools);
2980 }
2981 
2982 struct dm_pr {
2983 	u64	old_key;
2984 	u64	new_key;
2985 	u32	flags;
2986 	bool	fail_early;
2987 };
2988 
2989 static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn,
2990 		      void *data)
2991 {
2992 	struct mapped_device *md = bdev->bd_disk->private_data;
2993 	struct dm_table *table;
2994 	struct dm_target *ti;
2995 	int ret = -ENOTTY, srcu_idx;
2996 
2997 	table = dm_get_live_table(md, &srcu_idx);
2998 	if (!table || !dm_table_get_size(table))
2999 		goto out;
3000 
3001 	/* We only support devices that have a single target */
3002 	if (dm_table_get_num_targets(table) != 1)
3003 		goto out;
3004 	ti = dm_table_get_target(table, 0);
3005 
3006 	ret = -EINVAL;
3007 	if (!ti->type->iterate_devices)
3008 		goto out;
3009 
3010 	ret = ti->type->iterate_devices(ti, fn, data);
3011 out:
3012 	dm_put_live_table(md, srcu_idx);
3013 	return ret;
3014 }
3015 
3016 /*
3017  * For register / unregister we need to manually call out to every path.
3018  */
3019 static int __dm_pr_register(struct dm_target *ti, struct dm_dev *dev,
3020 			    sector_t start, sector_t len, void *data)
3021 {
3022 	struct dm_pr *pr = data;
3023 	const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
3024 
3025 	if (!ops || !ops->pr_register)
3026 		return -EOPNOTSUPP;
3027 	return ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags);
3028 }
3029 
3030 static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key,
3031 			  u32 flags)
3032 {
3033 	struct dm_pr pr = {
3034 		.old_key	= old_key,
3035 		.new_key	= new_key,
3036 		.flags		= flags,
3037 		.fail_early	= true,
3038 	};
3039 	int ret;
3040 
3041 	ret = dm_call_pr(bdev, __dm_pr_register, &pr);
3042 	if (ret && new_key) {
3043 		/* unregister all paths if we failed to register any path */
3044 		pr.old_key = new_key;
3045 		pr.new_key = 0;
3046 		pr.flags = 0;
3047 		pr.fail_early = false;
3048 		dm_call_pr(bdev, __dm_pr_register, &pr);
3049 	}
3050 
3051 	return ret;
3052 }
3053 
3054 static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type,
3055 			 u32 flags)
3056 {
3057 	struct mapped_device *md = bdev->bd_disk->private_data;
3058 	const struct pr_ops *ops;
3059 	int r, srcu_idx;
3060 
3061 	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
3062 	if (r < 0)
3063 		goto out;
3064 
3065 	ops = bdev->bd_disk->fops->pr_ops;
3066 	if (ops && ops->pr_reserve)
3067 		r = ops->pr_reserve(bdev, key, type, flags);
3068 	else
3069 		r = -EOPNOTSUPP;
3070 out:
3071 	dm_unprepare_ioctl(md, srcu_idx);
3072 	return r;
3073 }
3074 
3075 static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
3076 {
3077 	struct mapped_device *md = bdev->bd_disk->private_data;
3078 	const struct pr_ops *ops;
3079 	int r, srcu_idx;
3080 
3081 	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
3082 	if (r < 0)
3083 		goto out;
3084 
3085 	ops = bdev->bd_disk->fops->pr_ops;
3086 	if (ops && ops->pr_release)
3087 		r = ops->pr_release(bdev, key, type);
3088 	else
3089 		r = -EOPNOTSUPP;
3090 out:
3091 	dm_unprepare_ioctl(md, srcu_idx);
3092 	return r;
3093 }
3094 
3095 static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key,
3096 			 enum pr_type type, bool abort)
3097 {
3098 	struct mapped_device *md = bdev->bd_disk->private_data;
3099 	const struct pr_ops *ops;
3100 	int r, srcu_idx;
3101 
3102 	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
3103 	if (r < 0)
3104 		goto out;
3105 
3106 	ops = bdev->bd_disk->fops->pr_ops;
3107 	if (ops && ops->pr_preempt)
3108 		r = ops->pr_preempt(bdev, old_key, new_key, type, abort);
3109 	else
3110 		r = -EOPNOTSUPP;
3111 out:
3112 	dm_unprepare_ioctl(md, srcu_idx);
3113 	return r;
3114 }
3115 
3116 static int dm_pr_clear(struct block_device *bdev, u64 key)
3117 {
3118 	struct mapped_device *md = bdev->bd_disk->private_data;
3119 	const struct pr_ops *ops;
3120 	int r, srcu_idx;
3121 
3122 	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
3123 	if (r < 0)
3124 		goto out;
3125 
3126 	ops = bdev->bd_disk->fops->pr_ops;
3127 	if (ops && ops->pr_clear)
3128 		r = ops->pr_clear(bdev, key);
3129 	else
3130 		r = -EOPNOTSUPP;
3131 out:
3132 	dm_unprepare_ioctl(md, srcu_idx);
3133 	return r;
3134 }
3135 
3136 static const struct pr_ops dm_pr_ops = {
3137 	.pr_register	= dm_pr_register,
3138 	.pr_reserve	= dm_pr_reserve,
3139 	.pr_release	= dm_pr_release,
3140 	.pr_preempt	= dm_pr_preempt,
3141 	.pr_clear	= dm_pr_clear,
3142 };
3143 
3144 static const struct block_device_operations dm_blk_dops = {
3145 	.open = dm_blk_open,
3146 	.release = dm_blk_close,
3147 	.ioctl = dm_blk_ioctl,
3148 	.getgeo = dm_blk_getgeo,
3149 	.report_zones = dm_blk_report_zones,
3150 	.pr_ops = &dm_pr_ops,
3151 	.owner = THIS_MODULE
3152 };
3153 
3154 static const struct dax_operations dm_dax_ops = {
3155 	.direct_access = dm_dax_direct_access,
3156 	.copy_from_iter = dm_dax_copy_from_iter,
3157 	.copy_to_iter = dm_dax_copy_to_iter,
3158 };
3159 
3160 /*
3161  * module hooks
3162  */
3163 module_init(dm_init);
3164 module_exit(dm_exit);
3165 
3166 module_param(major, uint, 0);
3167 MODULE_PARM_DESC(major, "The major number of the device mapper");
3168 
3169 module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR);
3170 MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools");
3171 
3172 module_param(dm_numa_node, int, S_IRUGO | S_IWUSR);
3173 MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations");
3174 
3175 MODULE_DESCRIPTION(DM_NAME " driver");
3176 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
3177 MODULE_LICENSE("GPL");
3178