xref: /openbmc/linux/drivers/md/dm.c (revision ccb01374)
1 /*
2  * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3  * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
4  *
5  * This file is released under the GPL.
6  */
7 
8 #include "dm-core.h"
9 #include "dm-rq.h"
10 #include "dm-uevent.h"
11 
12 #include <linux/init.h>
13 #include <linux/module.h>
14 #include <linux/mutex.h>
15 #include <linux/sched/signal.h>
16 #include <linux/blkpg.h>
17 #include <linux/bio.h>
18 #include <linux/mempool.h>
19 #include <linux/dax.h>
20 #include <linux/slab.h>
21 #include <linux/idr.h>
22 #include <linux/uio.h>
23 #include <linux/hdreg.h>
24 #include <linux/delay.h>
25 #include <linux/wait.h>
26 #include <linux/pr.h>
27 #include <linux/refcount.h>
28 
29 #define DM_MSG_PREFIX "core"
30 
31 /*
32  * Cookies are numeric values sent with CHANGE and REMOVE
33  * uevents while resuming, removing or renaming the device.
34  */
35 #define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE"
36 #define DM_COOKIE_LENGTH 24
37 
38 static const char *_name = DM_NAME;
39 
40 static unsigned int major = 0;
41 static unsigned int _major = 0;
42 
43 static DEFINE_IDR(_minor_idr);
44 
45 static DEFINE_SPINLOCK(_minor_lock);
46 
47 static void do_deferred_remove(struct work_struct *w);
48 
49 static DECLARE_WORK(deferred_remove_work, do_deferred_remove);
50 
51 static struct workqueue_struct *deferred_remove_workqueue;
52 
53 atomic_t dm_global_event_nr = ATOMIC_INIT(0);
54 DECLARE_WAIT_QUEUE_HEAD(dm_global_eventq);
55 
56 void dm_issue_global_event(void)
57 {
58 	atomic_inc(&dm_global_event_nr);
59 	wake_up(&dm_global_eventq);
60 }
61 
62 /*
63  * One of these is allocated (on-stack) per original bio.
64  */
65 struct clone_info {
66 	struct dm_table *map;
67 	struct bio *bio;
68 	struct dm_io *io;
69 	sector_t sector;
70 	unsigned sector_count;
71 };
72 
73 /*
74  * One of these is allocated per clone bio.
75  */
76 #define DM_TIO_MAGIC 7282014
77 struct dm_target_io {
78 	unsigned magic;
79 	struct dm_io *io;
80 	struct dm_target *ti;
81 	unsigned target_bio_nr;
82 	unsigned *len_ptr;
83 	bool inside_dm_io;
84 	struct bio clone;
85 };
86 
87 /*
88  * One of these is allocated per original bio.
89  * It contains the first clone used for that original.
90  */
91 #define DM_IO_MAGIC 5191977
92 struct dm_io {
93 	unsigned magic;
94 	struct mapped_device *md;
95 	blk_status_t status;
96 	atomic_t io_count;
97 	struct bio *orig_bio;
98 	unsigned long start_time;
99 	spinlock_t endio_lock;
100 	struct dm_stats_aux stats_aux;
101 	/* last member of dm_target_io is 'struct bio' */
102 	struct dm_target_io tio;
103 };
104 
105 void *dm_per_bio_data(struct bio *bio, size_t data_size)
106 {
107 	struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
108 	if (!tio->inside_dm_io)
109 		return (char *)bio - offsetof(struct dm_target_io, clone) - data_size;
110 	return (char *)bio - offsetof(struct dm_target_io, clone) - offsetof(struct dm_io, tio) - data_size;
111 }
112 EXPORT_SYMBOL_GPL(dm_per_bio_data);
113 
114 struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size)
115 {
116 	struct dm_io *io = (struct dm_io *)((char *)data + data_size);
117 	if (io->magic == DM_IO_MAGIC)
118 		return (struct bio *)((char *)io + offsetof(struct dm_io, tio) + offsetof(struct dm_target_io, clone));
119 	BUG_ON(io->magic != DM_TIO_MAGIC);
120 	return (struct bio *)((char *)io + offsetof(struct dm_target_io, clone));
121 }
122 EXPORT_SYMBOL_GPL(dm_bio_from_per_bio_data);
123 
124 unsigned dm_bio_get_target_bio_nr(const struct bio *bio)
125 {
126 	return container_of(bio, struct dm_target_io, clone)->target_bio_nr;
127 }
128 EXPORT_SYMBOL_GPL(dm_bio_get_target_bio_nr);
129 
130 #define MINOR_ALLOCED ((void *)-1)
131 
132 /*
133  * Bits for the md->flags field.
134  */
135 #define DMF_BLOCK_IO_FOR_SUSPEND 0
136 #define DMF_SUSPENDED 1
137 #define DMF_FROZEN 2
138 #define DMF_FREEING 3
139 #define DMF_DELETING 4
140 #define DMF_NOFLUSH_SUSPENDING 5
141 #define DMF_DEFERRED_REMOVE 6
142 #define DMF_SUSPENDED_INTERNALLY 7
143 
144 #define DM_NUMA_NODE NUMA_NO_NODE
145 static int dm_numa_node = DM_NUMA_NODE;
146 
147 /*
148  * For mempools pre-allocation at the table loading time.
149  */
150 struct dm_md_mempools {
151 	struct bio_set bs;
152 	struct bio_set io_bs;
153 };
154 
155 struct table_device {
156 	struct list_head list;
157 	refcount_t count;
158 	struct dm_dev dm_dev;
159 };
160 
161 static struct kmem_cache *_rq_tio_cache;
162 static struct kmem_cache *_rq_cache;
163 
164 /*
165  * Bio-based DM's mempools' reserved IOs set by the user.
166  */
167 #define RESERVED_BIO_BASED_IOS		16
168 static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS;
169 
170 static int __dm_get_module_param_int(int *module_param, int min, int max)
171 {
172 	int param = READ_ONCE(*module_param);
173 	int modified_param = 0;
174 	bool modified = true;
175 
176 	if (param < min)
177 		modified_param = min;
178 	else if (param > max)
179 		modified_param = max;
180 	else
181 		modified = false;
182 
183 	if (modified) {
184 		(void)cmpxchg(module_param, param, modified_param);
185 		param = modified_param;
186 	}
187 
188 	return param;
189 }
190 
191 unsigned __dm_get_module_param(unsigned *module_param,
192 			       unsigned def, unsigned max)
193 {
194 	unsigned param = READ_ONCE(*module_param);
195 	unsigned modified_param = 0;
196 
197 	if (!param)
198 		modified_param = def;
199 	else if (param > max)
200 		modified_param = max;
201 
202 	if (modified_param) {
203 		(void)cmpxchg(module_param, param, modified_param);
204 		param = modified_param;
205 	}
206 
207 	return param;
208 }
209 
210 unsigned dm_get_reserved_bio_based_ios(void)
211 {
212 	return __dm_get_module_param(&reserved_bio_based_ios,
213 				     RESERVED_BIO_BASED_IOS, DM_RESERVED_MAX_IOS);
214 }
215 EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios);
216 
217 static unsigned dm_get_numa_node(void)
218 {
219 	return __dm_get_module_param_int(&dm_numa_node,
220 					 DM_NUMA_NODE, num_online_nodes() - 1);
221 }
222 
223 static int __init local_init(void)
224 {
225 	int r = -ENOMEM;
226 
227 	_rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0);
228 	if (!_rq_tio_cache)
229 		return r;
230 
231 	_rq_cache = kmem_cache_create("dm_old_clone_request", sizeof(struct request),
232 				      __alignof__(struct request), 0, NULL);
233 	if (!_rq_cache)
234 		goto out_free_rq_tio_cache;
235 
236 	r = dm_uevent_init();
237 	if (r)
238 		goto out_free_rq_cache;
239 
240 	deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1);
241 	if (!deferred_remove_workqueue) {
242 		r = -ENOMEM;
243 		goto out_uevent_exit;
244 	}
245 
246 	_major = major;
247 	r = register_blkdev(_major, _name);
248 	if (r < 0)
249 		goto out_free_workqueue;
250 
251 	if (!_major)
252 		_major = r;
253 
254 	return 0;
255 
256 out_free_workqueue:
257 	destroy_workqueue(deferred_remove_workqueue);
258 out_uevent_exit:
259 	dm_uevent_exit();
260 out_free_rq_cache:
261 	kmem_cache_destroy(_rq_cache);
262 out_free_rq_tio_cache:
263 	kmem_cache_destroy(_rq_tio_cache);
264 
265 	return r;
266 }
267 
268 static void local_exit(void)
269 {
270 	flush_scheduled_work();
271 	destroy_workqueue(deferred_remove_workqueue);
272 
273 	kmem_cache_destroy(_rq_cache);
274 	kmem_cache_destroy(_rq_tio_cache);
275 	unregister_blkdev(_major, _name);
276 	dm_uevent_exit();
277 
278 	_major = 0;
279 
280 	DMINFO("cleaned up");
281 }
282 
283 static int (*_inits[])(void) __initdata = {
284 	local_init,
285 	dm_target_init,
286 	dm_linear_init,
287 	dm_stripe_init,
288 	dm_io_init,
289 	dm_kcopyd_init,
290 	dm_interface_init,
291 	dm_statistics_init,
292 };
293 
294 static void (*_exits[])(void) = {
295 	local_exit,
296 	dm_target_exit,
297 	dm_linear_exit,
298 	dm_stripe_exit,
299 	dm_io_exit,
300 	dm_kcopyd_exit,
301 	dm_interface_exit,
302 	dm_statistics_exit,
303 };
304 
305 static int __init dm_init(void)
306 {
307 	const int count = ARRAY_SIZE(_inits);
308 
309 	int r, i;
310 
311 	for (i = 0; i < count; i++) {
312 		r = _inits[i]();
313 		if (r)
314 			goto bad;
315 	}
316 
317 	return 0;
318 
319       bad:
320 	while (i--)
321 		_exits[i]();
322 
323 	return r;
324 }
325 
326 static void __exit dm_exit(void)
327 {
328 	int i = ARRAY_SIZE(_exits);
329 
330 	while (i--)
331 		_exits[i]();
332 
333 	/*
334 	 * Should be empty by this point.
335 	 */
336 	idr_destroy(&_minor_idr);
337 }
338 
339 /*
340  * Block device functions
341  */
342 int dm_deleting_md(struct mapped_device *md)
343 {
344 	return test_bit(DMF_DELETING, &md->flags);
345 }
346 
347 static int dm_blk_open(struct block_device *bdev, fmode_t mode)
348 {
349 	struct mapped_device *md;
350 
351 	spin_lock(&_minor_lock);
352 
353 	md = bdev->bd_disk->private_data;
354 	if (!md)
355 		goto out;
356 
357 	if (test_bit(DMF_FREEING, &md->flags) ||
358 	    dm_deleting_md(md)) {
359 		md = NULL;
360 		goto out;
361 	}
362 
363 	dm_get(md);
364 	atomic_inc(&md->open_count);
365 out:
366 	spin_unlock(&_minor_lock);
367 
368 	return md ? 0 : -ENXIO;
369 }
370 
371 static void dm_blk_close(struct gendisk *disk, fmode_t mode)
372 {
373 	struct mapped_device *md;
374 
375 	spin_lock(&_minor_lock);
376 
377 	md = disk->private_data;
378 	if (WARN_ON(!md))
379 		goto out;
380 
381 	if (atomic_dec_and_test(&md->open_count) &&
382 	    (test_bit(DMF_DEFERRED_REMOVE, &md->flags)))
383 		queue_work(deferred_remove_workqueue, &deferred_remove_work);
384 
385 	dm_put(md);
386 out:
387 	spin_unlock(&_minor_lock);
388 }
389 
390 int dm_open_count(struct mapped_device *md)
391 {
392 	return atomic_read(&md->open_count);
393 }
394 
395 /*
396  * Guarantees nothing is using the device before it's deleted.
397  */
398 int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred)
399 {
400 	int r = 0;
401 
402 	spin_lock(&_minor_lock);
403 
404 	if (dm_open_count(md)) {
405 		r = -EBUSY;
406 		if (mark_deferred)
407 			set_bit(DMF_DEFERRED_REMOVE, &md->flags);
408 	} else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags))
409 		r = -EEXIST;
410 	else
411 		set_bit(DMF_DELETING, &md->flags);
412 
413 	spin_unlock(&_minor_lock);
414 
415 	return r;
416 }
417 
418 int dm_cancel_deferred_remove(struct mapped_device *md)
419 {
420 	int r = 0;
421 
422 	spin_lock(&_minor_lock);
423 
424 	if (test_bit(DMF_DELETING, &md->flags))
425 		r = -EBUSY;
426 	else
427 		clear_bit(DMF_DEFERRED_REMOVE, &md->flags);
428 
429 	spin_unlock(&_minor_lock);
430 
431 	return r;
432 }
433 
434 static void do_deferred_remove(struct work_struct *w)
435 {
436 	dm_deferred_remove();
437 }
438 
439 sector_t dm_get_size(struct mapped_device *md)
440 {
441 	return get_capacity(md->disk);
442 }
443 
444 struct request_queue *dm_get_md_queue(struct mapped_device *md)
445 {
446 	return md->queue;
447 }
448 
449 struct dm_stats *dm_get_stats(struct mapped_device *md)
450 {
451 	return &md->stats;
452 }
453 
454 static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
455 {
456 	struct mapped_device *md = bdev->bd_disk->private_data;
457 
458 	return dm_get_geometry(md, geo);
459 }
460 
461 static int dm_blk_report_zones(struct gendisk *disk, sector_t sector,
462 			       struct blk_zone *zones, unsigned int *nr_zones,
463 			       gfp_t gfp_mask)
464 {
465 #ifdef CONFIG_BLK_DEV_ZONED
466 	struct mapped_device *md = disk->private_data;
467 	struct dm_target *tgt;
468 	struct dm_table *map;
469 	int srcu_idx, ret;
470 
471 	if (dm_suspended_md(md))
472 		return -EAGAIN;
473 
474 	map = dm_get_live_table(md, &srcu_idx);
475 	if (!map)
476 		return -EIO;
477 
478 	tgt = dm_table_find_target(map, sector);
479 	if (!dm_target_is_valid(tgt)) {
480 		ret = -EIO;
481 		goto out;
482 	}
483 
484 	/*
485 	 * If we are executing this, we already know that the block device
486 	 * is a zoned device and so each target should have support for that
487 	 * type of drive. A missing report_zones method means that the target
488 	 * driver has a problem.
489 	 */
490 	if (WARN_ON(!tgt->type->report_zones)) {
491 		ret = -EIO;
492 		goto out;
493 	}
494 
495 	/*
496 	 * blkdev_report_zones() will loop and call this again to cover all the
497 	 * zones of the target, eventually moving on to the next target.
498 	 * So there is no need to loop here trying to fill the entire array
499 	 * of zones.
500 	 */
501 	ret = tgt->type->report_zones(tgt, sector, zones,
502 				      nr_zones, gfp_mask);
503 
504 out:
505 	dm_put_live_table(md, srcu_idx);
506 	return ret;
507 #else
508 	return -ENOTSUPP;
509 #endif
510 }
511 
512 static int dm_prepare_ioctl(struct mapped_device *md, int *srcu_idx,
513 			    struct block_device **bdev)
514 	__acquires(md->io_barrier)
515 {
516 	struct dm_target *tgt;
517 	struct dm_table *map;
518 	int r;
519 
520 retry:
521 	r = -ENOTTY;
522 	map = dm_get_live_table(md, srcu_idx);
523 	if (!map || !dm_table_get_size(map))
524 		return r;
525 
526 	/* We only support devices that have a single target */
527 	if (dm_table_get_num_targets(map) != 1)
528 		return r;
529 
530 	tgt = dm_table_get_target(map, 0);
531 	if (!tgt->type->prepare_ioctl)
532 		return r;
533 
534 	if (dm_suspended_md(md))
535 		return -EAGAIN;
536 
537 	r = tgt->type->prepare_ioctl(tgt, bdev);
538 	if (r == -ENOTCONN && !fatal_signal_pending(current)) {
539 		dm_put_live_table(md, *srcu_idx);
540 		msleep(10);
541 		goto retry;
542 	}
543 
544 	return r;
545 }
546 
547 static void dm_unprepare_ioctl(struct mapped_device *md, int srcu_idx)
548 	__releases(md->io_barrier)
549 {
550 	dm_put_live_table(md, srcu_idx);
551 }
552 
553 static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
554 			unsigned int cmd, unsigned long arg)
555 {
556 	struct mapped_device *md = bdev->bd_disk->private_data;
557 	int r, srcu_idx;
558 
559 	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
560 	if (r < 0)
561 		goto out;
562 
563 	if (r > 0) {
564 		/*
565 		 * Target determined this ioctl is being issued against a
566 		 * subset of the parent bdev; require extra privileges.
567 		 */
568 		if (!capable(CAP_SYS_RAWIO)) {
569 			DMWARN_LIMIT(
570 	"%s: sending ioctl %x to DM device without required privilege.",
571 				current->comm, cmd);
572 			r = -ENOIOCTLCMD;
573 			goto out;
574 		}
575 	}
576 
577 	r =  __blkdev_driver_ioctl(bdev, mode, cmd, arg);
578 out:
579 	dm_unprepare_ioctl(md, srcu_idx);
580 	return r;
581 }
582 
583 static void start_io_acct(struct dm_io *io);
584 
585 static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio)
586 {
587 	struct dm_io *io;
588 	struct dm_target_io *tio;
589 	struct bio *clone;
590 
591 	clone = bio_alloc_bioset(GFP_NOIO, 0, &md->io_bs);
592 	if (!clone)
593 		return NULL;
594 
595 	tio = container_of(clone, struct dm_target_io, clone);
596 	tio->inside_dm_io = true;
597 	tio->io = NULL;
598 
599 	io = container_of(tio, struct dm_io, tio);
600 	io->magic = DM_IO_MAGIC;
601 	io->status = 0;
602 	atomic_set(&io->io_count, 1);
603 	io->orig_bio = bio;
604 	io->md = md;
605 	spin_lock_init(&io->endio_lock);
606 
607 	start_io_acct(io);
608 
609 	return io;
610 }
611 
612 static void free_io(struct mapped_device *md, struct dm_io *io)
613 {
614 	bio_put(&io->tio.clone);
615 }
616 
617 static struct dm_target_io *alloc_tio(struct clone_info *ci, struct dm_target *ti,
618 				      unsigned target_bio_nr, gfp_t gfp_mask)
619 {
620 	struct dm_target_io *tio;
621 
622 	if (!ci->io->tio.io) {
623 		/* the dm_target_io embedded in ci->io is available */
624 		tio = &ci->io->tio;
625 	} else {
626 		struct bio *clone = bio_alloc_bioset(gfp_mask, 0, &ci->io->md->bs);
627 		if (!clone)
628 			return NULL;
629 
630 		tio = container_of(clone, struct dm_target_io, clone);
631 		tio->inside_dm_io = false;
632 	}
633 
634 	tio->magic = DM_TIO_MAGIC;
635 	tio->io = ci->io;
636 	tio->ti = ti;
637 	tio->target_bio_nr = target_bio_nr;
638 
639 	return tio;
640 }
641 
642 static void free_tio(struct dm_target_io *tio)
643 {
644 	if (tio->inside_dm_io)
645 		return;
646 	bio_put(&tio->clone);
647 }
648 
649 static bool md_in_flight_bios(struct mapped_device *md)
650 {
651 	int cpu;
652 	struct hd_struct *part = &dm_disk(md)->part0;
653 	long sum = 0;
654 
655 	for_each_possible_cpu(cpu) {
656 		sum += part_stat_local_read_cpu(part, in_flight[0], cpu);
657 		sum += part_stat_local_read_cpu(part, in_flight[1], cpu);
658 	}
659 
660 	return sum != 0;
661 }
662 
663 static bool md_in_flight(struct mapped_device *md)
664 {
665 	if (queue_is_mq(md->queue))
666 		return blk_mq_queue_inflight(md->queue);
667 	else
668 		return md_in_flight_bios(md);
669 }
670 
671 static void start_io_acct(struct dm_io *io)
672 {
673 	struct mapped_device *md = io->md;
674 	struct bio *bio = io->orig_bio;
675 
676 	io->start_time = jiffies;
677 
678 	generic_start_io_acct(md->queue, bio_op(bio), bio_sectors(bio),
679 			      &dm_disk(md)->part0);
680 
681 	if (unlikely(dm_stats_used(&md->stats)))
682 		dm_stats_account_io(&md->stats, bio_data_dir(bio),
683 				    bio->bi_iter.bi_sector, bio_sectors(bio),
684 				    false, 0, &io->stats_aux);
685 }
686 
687 static void end_io_acct(struct dm_io *io)
688 {
689 	struct mapped_device *md = io->md;
690 	struct bio *bio = io->orig_bio;
691 	unsigned long duration = jiffies - io->start_time;
692 
693 	generic_end_io_acct(md->queue, bio_op(bio), &dm_disk(md)->part0,
694 			    io->start_time);
695 
696 	if (unlikely(dm_stats_used(&md->stats)))
697 		dm_stats_account_io(&md->stats, bio_data_dir(bio),
698 				    bio->bi_iter.bi_sector, bio_sectors(bio),
699 				    true, duration, &io->stats_aux);
700 
701 	/* nudge anyone waiting on suspend queue */
702 	if (unlikely(waitqueue_active(&md->wait)))
703 		wake_up(&md->wait);
704 }
705 
706 /*
707  * Add the bio to the list of deferred io.
708  */
709 static void queue_io(struct mapped_device *md, struct bio *bio)
710 {
711 	unsigned long flags;
712 
713 	spin_lock_irqsave(&md->deferred_lock, flags);
714 	bio_list_add(&md->deferred, bio);
715 	spin_unlock_irqrestore(&md->deferred_lock, flags);
716 	queue_work(md->wq, &md->work);
717 }
718 
719 /*
720  * Everyone (including functions in this file), should use this
721  * function to access the md->map field, and make sure they call
722  * dm_put_live_table() when finished.
723  */
724 struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier)
725 {
726 	*srcu_idx = srcu_read_lock(&md->io_barrier);
727 
728 	return srcu_dereference(md->map, &md->io_barrier);
729 }
730 
731 void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier)
732 {
733 	srcu_read_unlock(&md->io_barrier, srcu_idx);
734 }
735 
736 void dm_sync_table(struct mapped_device *md)
737 {
738 	synchronize_srcu(&md->io_barrier);
739 	synchronize_rcu_expedited();
740 }
741 
742 /*
743  * A fast alternative to dm_get_live_table/dm_put_live_table.
744  * The caller must not block between these two functions.
745  */
746 static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU)
747 {
748 	rcu_read_lock();
749 	return rcu_dereference(md->map);
750 }
751 
752 static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU)
753 {
754 	rcu_read_unlock();
755 }
756 
757 static char *_dm_claim_ptr = "I belong to device-mapper";
758 
759 /*
760  * Open a table device so we can use it as a map destination.
761  */
762 static int open_table_device(struct table_device *td, dev_t dev,
763 			     struct mapped_device *md)
764 {
765 	struct block_device *bdev;
766 
767 	int r;
768 
769 	BUG_ON(td->dm_dev.bdev);
770 
771 	bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _dm_claim_ptr);
772 	if (IS_ERR(bdev))
773 		return PTR_ERR(bdev);
774 
775 	r = bd_link_disk_holder(bdev, dm_disk(md));
776 	if (r) {
777 		blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL);
778 		return r;
779 	}
780 
781 	td->dm_dev.bdev = bdev;
782 	td->dm_dev.dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);
783 	return 0;
784 }
785 
786 /*
787  * Close a table device that we've been using.
788  */
789 static void close_table_device(struct table_device *td, struct mapped_device *md)
790 {
791 	if (!td->dm_dev.bdev)
792 		return;
793 
794 	bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md));
795 	blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL);
796 	put_dax(td->dm_dev.dax_dev);
797 	td->dm_dev.bdev = NULL;
798 	td->dm_dev.dax_dev = NULL;
799 }
800 
801 static struct table_device *find_table_device(struct list_head *l, dev_t dev,
802 					      fmode_t mode) {
803 	struct table_device *td;
804 
805 	list_for_each_entry(td, l, list)
806 		if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode)
807 			return td;
808 
809 	return NULL;
810 }
811 
812 int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode,
813 			struct dm_dev **result) {
814 	int r;
815 	struct table_device *td;
816 
817 	mutex_lock(&md->table_devices_lock);
818 	td = find_table_device(&md->table_devices, dev, mode);
819 	if (!td) {
820 		td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id);
821 		if (!td) {
822 			mutex_unlock(&md->table_devices_lock);
823 			return -ENOMEM;
824 		}
825 
826 		td->dm_dev.mode = mode;
827 		td->dm_dev.bdev = NULL;
828 
829 		if ((r = open_table_device(td, dev, md))) {
830 			mutex_unlock(&md->table_devices_lock);
831 			kfree(td);
832 			return r;
833 		}
834 
835 		format_dev_t(td->dm_dev.name, dev);
836 
837 		refcount_set(&td->count, 1);
838 		list_add(&td->list, &md->table_devices);
839 	} else {
840 		refcount_inc(&td->count);
841 	}
842 	mutex_unlock(&md->table_devices_lock);
843 
844 	*result = &td->dm_dev;
845 	return 0;
846 }
847 EXPORT_SYMBOL_GPL(dm_get_table_device);
848 
849 void dm_put_table_device(struct mapped_device *md, struct dm_dev *d)
850 {
851 	struct table_device *td = container_of(d, struct table_device, dm_dev);
852 
853 	mutex_lock(&md->table_devices_lock);
854 	if (refcount_dec_and_test(&td->count)) {
855 		close_table_device(td, md);
856 		list_del(&td->list);
857 		kfree(td);
858 	}
859 	mutex_unlock(&md->table_devices_lock);
860 }
861 EXPORT_SYMBOL(dm_put_table_device);
862 
863 static void free_table_devices(struct list_head *devices)
864 {
865 	struct list_head *tmp, *next;
866 
867 	list_for_each_safe(tmp, next, devices) {
868 		struct table_device *td = list_entry(tmp, struct table_device, list);
869 
870 		DMWARN("dm_destroy: %s still exists with %d references",
871 		       td->dm_dev.name, refcount_read(&td->count));
872 		kfree(td);
873 	}
874 }
875 
876 /*
877  * Get the geometry associated with a dm device
878  */
879 int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
880 {
881 	*geo = md->geometry;
882 
883 	return 0;
884 }
885 
886 /*
887  * Set the geometry of a device.
888  */
889 int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
890 {
891 	sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
892 
893 	if (geo->start > sz) {
894 		DMWARN("Start sector is beyond the geometry limits.");
895 		return -EINVAL;
896 	}
897 
898 	md->geometry = *geo;
899 
900 	return 0;
901 }
902 
903 static int __noflush_suspending(struct mapped_device *md)
904 {
905 	return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
906 }
907 
908 /*
909  * Decrements the number of outstanding ios that a bio has been
910  * cloned into, completing the original io if necc.
911  */
912 static void dec_pending(struct dm_io *io, blk_status_t error)
913 {
914 	unsigned long flags;
915 	blk_status_t io_error;
916 	struct bio *bio;
917 	struct mapped_device *md = io->md;
918 
919 	/* Push-back supersedes any I/O errors */
920 	if (unlikely(error)) {
921 		spin_lock_irqsave(&io->endio_lock, flags);
922 		if (!(io->status == BLK_STS_DM_REQUEUE && __noflush_suspending(md)))
923 			io->status = error;
924 		spin_unlock_irqrestore(&io->endio_lock, flags);
925 	}
926 
927 	if (atomic_dec_and_test(&io->io_count)) {
928 		if (io->status == BLK_STS_DM_REQUEUE) {
929 			/*
930 			 * Target requested pushing back the I/O.
931 			 */
932 			spin_lock_irqsave(&md->deferred_lock, flags);
933 			if (__noflush_suspending(md))
934 				/* NOTE early return due to BLK_STS_DM_REQUEUE below */
935 				bio_list_add_head(&md->deferred, io->orig_bio);
936 			else
937 				/* noflush suspend was interrupted. */
938 				io->status = BLK_STS_IOERR;
939 			spin_unlock_irqrestore(&md->deferred_lock, flags);
940 		}
941 
942 		io_error = io->status;
943 		bio = io->orig_bio;
944 		end_io_acct(io);
945 		free_io(md, io);
946 
947 		if (io_error == BLK_STS_DM_REQUEUE)
948 			return;
949 
950 		if ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size) {
951 			/*
952 			 * Preflush done for flush with data, reissue
953 			 * without REQ_PREFLUSH.
954 			 */
955 			bio->bi_opf &= ~REQ_PREFLUSH;
956 			queue_io(md, bio);
957 		} else {
958 			/* done with normal IO or empty flush */
959 			if (io_error)
960 				bio->bi_status = io_error;
961 			bio_endio(bio);
962 		}
963 	}
964 }
965 
966 void disable_write_same(struct mapped_device *md)
967 {
968 	struct queue_limits *limits = dm_get_queue_limits(md);
969 
970 	/* device doesn't really support WRITE SAME, disable it */
971 	limits->max_write_same_sectors = 0;
972 }
973 
974 void disable_write_zeroes(struct mapped_device *md)
975 {
976 	struct queue_limits *limits = dm_get_queue_limits(md);
977 
978 	/* device doesn't really support WRITE ZEROES, disable it */
979 	limits->max_write_zeroes_sectors = 0;
980 }
981 
982 static void clone_endio(struct bio *bio)
983 {
984 	blk_status_t error = bio->bi_status;
985 	struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
986 	struct dm_io *io = tio->io;
987 	struct mapped_device *md = tio->io->md;
988 	dm_endio_fn endio = tio->ti->type->end_io;
989 
990 	if (unlikely(error == BLK_STS_TARGET) && md->type != DM_TYPE_NVME_BIO_BASED) {
991 		if (bio_op(bio) == REQ_OP_WRITE_SAME &&
992 		    !bio->bi_disk->queue->limits.max_write_same_sectors)
993 			disable_write_same(md);
994 		if (bio_op(bio) == REQ_OP_WRITE_ZEROES &&
995 		    !bio->bi_disk->queue->limits.max_write_zeroes_sectors)
996 			disable_write_zeroes(md);
997 	}
998 
999 	if (endio) {
1000 		int r = endio(tio->ti, bio, &error);
1001 		switch (r) {
1002 		case DM_ENDIO_REQUEUE:
1003 			error = BLK_STS_DM_REQUEUE;
1004 			/*FALLTHRU*/
1005 		case DM_ENDIO_DONE:
1006 			break;
1007 		case DM_ENDIO_INCOMPLETE:
1008 			/* The target will handle the io */
1009 			return;
1010 		default:
1011 			DMWARN("unimplemented target endio return value: %d", r);
1012 			BUG();
1013 		}
1014 	}
1015 
1016 	free_tio(tio);
1017 	dec_pending(io, error);
1018 }
1019 
1020 /*
1021  * Return maximum size of I/O possible at the supplied sector up to the current
1022  * target boundary.
1023  */
1024 static sector_t max_io_len_target_boundary(sector_t sector, struct dm_target *ti)
1025 {
1026 	sector_t target_offset = dm_target_offset(ti, sector);
1027 
1028 	return ti->len - target_offset;
1029 }
1030 
1031 static sector_t max_io_len(sector_t sector, struct dm_target *ti)
1032 {
1033 	sector_t len = max_io_len_target_boundary(sector, ti);
1034 	sector_t offset, max_len;
1035 
1036 	/*
1037 	 * Does the target need to split even further?
1038 	 */
1039 	if (ti->max_io_len) {
1040 		offset = dm_target_offset(ti, sector);
1041 		if (unlikely(ti->max_io_len & (ti->max_io_len - 1)))
1042 			max_len = sector_div(offset, ti->max_io_len);
1043 		else
1044 			max_len = offset & (ti->max_io_len - 1);
1045 		max_len = ti->max_io_len - max_len;
1046 
1047 		if (len > max_len)
1048 			len = max_len;
1049 	}
1050 
1051 	return len;
1052 }
1053 
1054 int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
1055 {
1056 	if (len > UINT_MAX) {
1057 		DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)",
1058 		      (unsigned long long)len, UINT_MAX);
1059 		ti->error = "Maximum size of target IO is too large";
1060 		return -EINVAL;
1061 	}
1062 
1063 	/*
1064 	 * BIO based queue uses its own splitting. When multipage bvecs
1065 	 * is switched on, size of the incoming bio may be too big to
1066 	 * be handled in some targets, such as crypt.
1067 	 *
1068 	 * When these targets are ready for the big bio, we can remove
1069 	 * the limit.
1070 	 */
1071 	ti->max_io_len = min_t(uint32_t, len, BIO_MAX_PAGES * PAGE_SIZE);
1072 
1073 	return 0;
1074 }
1075 EXPORT_SYMBOL_GPL(dm_set_target_max_io_len);
1076 
1077 static struct dm_target *dm_dax_get_live_target(struct mapped_device *md,
1078 						sector_t sector, int *srcu_idx)
1079 	__acquires(md->io_barrier)
1080 {
1081 	struct dm_table *map;
1082 	struct dm_target *ti;
1083 
1084 	map = dm_get_live_table(md, srcu_idx);
1085 	if (!map)
1086 		return NULL;
1087 
1088 	ti = dm_table_find_target(map, sector);
1089 	if (!dm_target_is_valid(ti))
1090 		return NULL;
1091 
1092 	return ti;
1093 }
1094 
1095 static long dm_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
1096 				 long nr_pages, void **kaddr, pfn_t *pfn)
1097 {
1098 	struct mapped_device *md = dax_get_private(dax_dev);
1099 	sector_t sector = pgoff * PAGE_SECTORS;
1100 	struct dm_target *ti;
1101 	long len, ret = -EIO;
1102 	int srcu_idx;
1103 
1104 	ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1105 
1106 	if (!ti)
1107 		goto out;
1108 	if (!ti->type->direct_access)
1109 		goto out;
1110 	len = max_io_len(sector, ti) / PAGE_SECTORS;
1111 	if (len < 1)
1112 		goto out;
1113 	nr_pages = min(len, nr_pages);
1114 	ret = ti->type->direct_access(ti, pgoff, nr_pages, kaddr, pfn);
1115 
1116  out:
1117 	dm_put_live_table(md, srcu_idx);
1118 
1119 	return ret;
1120 }
1121 
1122 static size_t dm_dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff,
1123 				    void *addr, size_t bytes, struct iov_iter *i)
1124 {
1125 	struct mapped_device *md = dax_get_private(dax_dev);
1126 	sector_t sector = pgoff * PAGE_SECTORS;
1127 	struct dm_target *ti;
1128 	long ret = 0;
1129 	int srcu_idx;
1130 
1131 	ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1132 
1133 	if (!ti)
1134 		goto out;
1135 	if (!ti->type->dax_copy_from_iter) {
1136 		ret = copy_from_iter(addr, bytes, i);
1137 		goto out;
1138 	}
1139 	ret = ti->type->dax_copy_from_iter(ti, pgoff, addr, bytes, i);
1140  out:
1141 	dm_put_live_table(md, srcu_idx);
1142 
1143 	return ret;
1144 }
1145 
1146 static size_t dm_dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff,
1147 		void *addr, size_t bytes, struct iov_iter *i)
1148 {
1149 	struct mapped_device *md = dax_get_private(dax_dev);
1150 	sector_t sector = pgoff * PAGE_SECTORS;
1151 	struct dm_target *ti;
1152 	long ret = 0;
1153 	int srcu_idx;
1154 
1155 	ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1156 
1157 	if (!ti)
1158 		goto out;
1159 	if (!ti->type->dax_copy_to_iter) {
1160 		ret = copy_to_iter(addr, bytes, i);
1161 		goto out;
1162 	}
1163 	ret = ti->type->dax_copy_to_iter(ti, pgoff, addr, bytes, i);
1164  out:
1165 	dm_put_live_table(md, srcu_idx);
1166 
1167 	return ret;
1168 }
1169 
1170 /*
1171  * A target may call dm_accept_partial_bio only from the map routine.  It is
1172  * allowed for all bio types except REQ_PREFLUSH and REQ_OP_ZONE_RESET.
1173  *
1174  * dm_accept_partial_bio informs the dm that the target only wants to process
1175  * additional n_sectors sectors of the bio and the rest of the data should be
1176  * sent in a next bio.
1177  *
1178  * A diagram that explains the arithmetics:
1179  * +--------------------+---------------+-------+
1180  * |         1          |       2       |   3   |
1181  * +--------------------+---------------+-------+
1182  *
1183  * <-------------- *tio->len_ptr --------------->
1184  *                      <------- bi_size ------->
1185  *                      <-- n_sectors -->
1186  *
1187  * Region 1 was already iterated over with bio_advance or similar function.
1188  *	(it may be empty if the target doesn't use bio_advance)
1189  * Region 2 is the remaining bio size that the target wants to process.
1190  *	(it may be empty if region 1 is non-empty, although there is no reason
1191  *	 to make it empty)
1192  * The target requires that region 3 is to be sent in the next bio.
1193  *
1194  * If the target wants to receive multiple copies of the bio (via num_*bios, etc),
1195  * the partially processed part (the sum of regions 1+2) must be the same for all
1196  * copies of the bio.
1197  */
1198 void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors)
1199 {
1200 	struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
1201 	unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT;
1202 	BUG_ON(bio->bi_opf & REQ_PREFLUSH);
1203 	BUG_ON(bi_size > *tio->len_ptr);
1204 	BUG_ON(n_sectors > bi_size);
1205 	*tio->len_ptr -= bi_size - n_sectors;
1206 	bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT;
1207 }
1208 EXPORT_SYMBOL_GPL(dm_accept_partial_bio);
1209 
1210 /*
1211  * The zone descriptors obtained with a zone report indicate
1212  * zone positions within the underlying device of the target. The zone
1213  * descriptors must be remapped to match their position within the dm device.
1214  * The caller target should obtain the zones information using
1215  * blkdev_report_zones() to ensure that remapping for partition offset is
1216  * already handled.
1217  */
1218 void dm_remap_zone_report(struct dm_target *ti, sector_t start,
1219 			  struct blk_zone *zones, unsigned int *nr_zones)
1220 {
1221 #ifdef CONFIG_BLK_DEV_ZONED
1222 	struct blk_zone *zone;
1223 	unsigned int nrz = *nr_zones;
1224 	int i;
1225 
1226 	/*
1227 	 * Remap the start sector and write pointer position of the zones in
1228 	 * the array. Since we may have obtained from the target underlying
1229 	 * device more zones that the target size, also adjust the number
1230 	 * of zones.
1231 	 */
1232 	for (i = 0; i < nrz; i++) {
1233 		zone = zones + i;
1234 		if (zone->start >= start + ti->len) {
1235 			memset(zone, 0, sizeof(struct blk_zone) * (nrz - i));
1236 			break;
1237 		}
1238 
1239 		zone->start = zone->start + ti->begin - start;
1240 		if (zone->type == BLK_ZONE_TYPE_CONVENTIONAL)
1241 			continue;
1242 
1243 		if (zone->cond == BLK_ZONE_COND_FULL)
1244 			zone->wp = zone->start + zone->len;
1245 		else if (zone->cond == BLK_ZONE_COND_EMPTY)
1246 			zone->wp = zone->start;
1247 		else
1248 			zone->wp = zone->wp + ti->begin - start;
1249 	}
1250 
1251 	*nr_zones = i;
1252 #else /* !CONFIG_BLK_DEV_ZONED */
1253 	*nr_zones = 0;
1254 #endif
1255 }
1256 EXPORT_SYMBOL_GPL(dm_remap_zone_report);
1257 
1258 static blk_qc_t __map_bio(struct dm_target_io *tio)
1259 {
1260 	int r;
1261 	sector_t sector;
1262 	struct bio *clone = &tio->clone;
1263 	struct dm_io *io = tio->io;
1264 	struct mapped_device *md = io->md;
1265 	struct dm_target *ti = tio->ti;
1266 	blk_qc_t ret = BLK_QC_T_NONE;
1267 
1268 	clone->bi_end_io = clone_endio;
1269 
1270 	/*
1271 	 * Map the clone.  If r == 0 we don't need to do
1272 	 * anything, the target has assumed ownership of
1273 	 * this io.
1274 	 */
1275 	atomic_inc(&io->io_count);
1276 	sector = clone->bi_iter.bi_sector;
1277 
1278 	r = ti->type->map(ti, clone);
1279 	switch (r) {
1280 	case DM_MAPIO_SUBMITTED:
1281 		break;
1282 	case DM_MAPIO_REMAPPED:
1283 		/* the bio has been remapped so dispatch it */
1284 		trace_block_bio_remap(clone->bi_disk->queue, clone,
1285 				      bio_dev(io->orig_bio), sector);
1286 		if (md->type == DM_TYPE_NVME_BIO_BASED)
1287 			ret = direct_make_request(clone);
1288 		else
1289 			ret = generic_make_request(clone);
1290 		break;
1291 	case DM_MAPIO_KILL:
1292 		free_tio(tio);
1293 		dec_pending(io, BLK_STS_IOERR);
1294 		break;
1295 	case DM_MAPIO_REQUEUE:
1296 		free_tio(tio);
1297 		dec_pending(io, BLK_STS_DM_REQUEUE);
1298 		break;
1299 	default:
1300 		DMWARN("unimplemented target map return value: %d", r);
1301 		BUG();
1302 	}
1303 
1304 	return ret;
1305 }
1306 
1307 static void bio_setup_sector(struct bio *bio, sector_t sector, unsigned len)
1308 {
1309 	bio->bi_iter.bi_sector = sector;
1310 	bio->bi_iter.bi_size = to_bytes(len);
1311 }
1312 
1313 /*
1314  * Creates a bio that consists of range of complete bvecs.
1315  */
1316 static int clone_bio(struct dm_target_io *tio, struct bio *bio,
1317 		     sector_t sector, unsigned len)
1318 {
1319 	struct bio *clone = &tio->clone;
1320 
1321 	__bio_clone_fast(clone, bio);
1322 
1323 	if (unlikely(bio_integrity(bio) != NULL)) {
1324 		int r;
1325 
1326 		if (unlikely(!dm_target_has_integrity(tio->ti->type) &&
1327 			     !dm_target_passes_integrity(tio->ti->type))) {
1328 			DMWARN("%s: the target %s doesn't support integrity data.",
1329 				dm_device_name(tio->io->md),
1330 				tio->ti->type->name);
1331 			return -EIO;
1332 		}
1333 
1334 		r = bio_integrity_clone(clone, bio, GFP_NOIO);
1335 		if (r < 0)
1336 			return r;
1337 	}
1338 
1339 	bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector));
1340 	clone->bi_iter.bi_size = to_bytes(len);
1341 
1342 	if (unlikely(bio_integrity(bio) != NULL))
1343 		bio_integrity_trim(clone);
1344 
1345 	return 0;
1346 }
1347 
1348 static void alloc_multiple_bios(struct bio_list *blist, struct clone_info *ci,
1349 				struct dm_target *ti, unsigned num_bios)
1350 {
1351 	struct dm_target_io *tio;
1352 	int try;
1353 
1354 	if (!num_bios)
1355 		return;
1356 
1357 	if (num_bios == 1) {
1358 		tio = alloc_tio(ci, ti, 0, GFP_NOIO);
1359 		bio_list_add(blist, &tio->clone);
1360 		return;
1361 	}
1362 
1363 	for (try = 0; try < 2; try++) {
1364 		int bio_nr;
1365 		struct bio *bio;
1366 
1367 		if (try)
1368 			mutex_lock(&ci->io->md->table_devices_lock);
1369 		for (bio_nr = 0; bio_nr < num_bios; bio_nr++) {
1370 			tio = alloc_tio(ci, ti, bio_nr, try ? GFP_NOIO : GFP_NOWAIT);
1371 			if (!tio)
1372 				break;
1373 
1374 			bio_list_add(blist, &tio->clone);
1375 		}
1376 		if (try)
1377 			mutex_unlock(&ci->io->md->table_devices_lock);
1378 		if (bio_nr == num_bios)
1379 			return;
1380 
1381 		while ((bio = bio_list_pop(blist))) {
1382 			tio = container_of(bio, struct dm_target_io, clone);
1383 			free_tio(tio);
1384 		}
1385 	}
1386 }
1387 
1388 static blk_qc_t __clone_and_map_simple_bio(struct clone_info *ci,
1389 					   struct dm_target_io *tio, unsigned *len)
1390 {
1391 	struct bio *clone = &tio->clone;
1392 
1393 	tio->len_ptr = len;
1394 
1395 	__bio_clone_fast(clone, ci->bio);
1396 	if (len)
1397 		bio_setup_sector(clone, ci->sector, *len);
1398 
1399 	return __map_bio(tio);
1400 }
1401 
1402 static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti,
1403 				  unsigned num_bios, unsigned *len)
1404 {
1405 	struct bio_list blist = BIO_EMPTY_LIST;
1406 	struct bio *bio;
1407 	struct dm_target_io *tio;
1408 
1409 	alloc_multiple_bios(&blist, ci, ti, num_bios);
1410 
1411 	while ((bio = bio_list_pop(&blist))) {
1412 		tio = container_of(bio, struct dm_target_io, clone);
1413 		(void) __clone_and_map_simple_bio(ci, tio, len);
1414 	}
1415 }
1416 
1417 static int __send_empty_flush(struct clone_info *ci)
1418 {
1419 	unsigned target_nr = 0;
1420 	struct dm_target *ti;
1421 
1422 	/*
1423 	 * Empty flush uses a statically initialized bio, as the base for
1424 	 * cloning.  However, blkg association requires that a bdev is
1425 	 * associated with a gendisk, which doesn't happen until the bdev is
1426 	 * opened.  So, blkg association is done at issue time of the flush
1427 	 * rather than when the device is created in alloc_dev().
1428 	 */
1429 	bio_set_dev(ci->bio, ci->io->md->bdev);
1430 
1431 	BUG_ON(bio_has_data(ci->bio));
1432 	while ((ti = dm_table_get_target(ci->map, target_nr++)))
1433 		__send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL);
1434 
1435 	bio_disassociate_blkg(ci->bio);
1436 
1437 	return 0;
1438 }
1439 
1440 static int __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti,
1441 				    sector_t sector, unsigned *len)
1442 {
1443 	struct bio *bio = ci->bio;
1444 	struct dm_target_io *tio;
1445 	int r;
1446 
1447 	tio = alloc_tio(ci, ti, 0, GFP_NOIO);
1448 	tio->len_ptr = len;
1449 	r = clone_bio(tio, bio, sector, *len);
1450 	if (r < 0) {
1451 		free_tio(tio);
1452 		return r;
1453 	}
1454 	(void) __map_bio(tio);
1455 
1456 	return 0;
1457 }
1458 
1459 typedef unsigned (*get_num_bios_fn)(struct dm_target *ti);
1460 
1461 static unsigned get_num_discard_bios(struct dm_target *ti)
1462 {
1463 	return ti->num_discard_bios;
1464 }
1465 
1466 static unsigned get_num_secure_erase_bios(struct dm_target *ti)
1467 {
1468 	return ti->num_secure_erase_bios;
1469 }
1470 
1471 static unsigned get_num_write_same_bios(struct dm_target *ti)
1472 {
1473 	return ti->num_write_same_bios;
1474 }
1475 
1476 static unsigned get_num_write_zeroes_bios(struct dm_target *ti)
1477 {
1478 	return ti->num_write_zeroes_bios;
1479 }
1480 
1481 typedef bool (*is_split_required_fn)(struct dm_target *ti);
1482 
1483 static bool is_split_required_for_discard(struct dm_target *ti)
1484 {
1485 	return ti->split_discard_bios;
1486 }
1487 
1488 static int __send_changing_extent_only(struct clone_info *ci, struct dm_target *ti,
1489 				       unsigned num_bios, bool is_split_required)
1490 {
1491 	unsigned len;
1492 
1493 	/*
1494 	 * Even though the device advertised support for this type of
1495 	 * request, that does not mean every target supports it, and
1496 	 * reconfiguration might also have changed that since the
1497 	 * check was performed.
1498 	 */
1499 	if (!num_bios)
1500 		return -EOPNOTSUPP;
1501 
1502 	if (!is_split_required)
1503 		len = min((sector_t)ci->sector_count, max_io_len_target_boundary(ci->sector, ti));
1504 	else
1505 		len = min((sector_t)ci->sector_count, max_io_len(ci->sector, ti));
1506 
1507 	__send_duplicate_bios(ci, ti, num_bios, &len);
1508 
1509 	ci->sector += len;
1510 	ci->sector_count -= len;
1511 
1512 	return 0;
1513 }
1514 
1515 static int __send_discard(struct clone_info *ci, struct dm_target *ti)
1516 {
1517 	return __send_changing_extent_only(ci, ti, get_num_discard_bios(ti),
1518 					   is_split_required_for_discard(ti));
1519 }
1520 
1521 static int __send_secure_erase(struct clone_info *ci, struct dm_target *ti)
1522 {
1523 	return __send_changing_extent_only(ci, ti, get_num_secure_erase_bios(ti), false);
1524 }
1525 
1526 static int __send_write_same(struct clone_info *ci, struct dm_target *ti)
1527 {
1528 	return __send_changing_extent_only(ci, ti, get_num_write_same_bios(ti), false);
1529 }
1530 
1531 static int __send_write_zeroes(struct clone_info *ci, struct dm_target *ti)
1532 {
1533 	return __send_changing_extent_only(ci, ti, get_num_write_zeroes_bios(ti), false);
1534 }
1535 
1536 static bool __process_abnormal_io(struct clone_info *ci, struct dm_target *ti,
1537 				  int *result)
1538 {
1539 	struct bio *bio = ci->bio;
1540 
1541 	if (bio_op(bio) == REQ_OP_DISCARD)
1542 		*result = __send_discard(ci, ti);
1543 	else if (bio_op(bio) == REQ_OP_SECURE_ERASE)
1544 		*result = __send_secure_erase(ci, ti);
1545 	else if (bio_op(bio) == REQ_OP_WRITE_SAME)
1546 		*result = __send_write_same(ci, ti);
1547 	else if (bio_op(bio) == REQ_OP_WRITE_ZEROES)
1548 		*result = __send_write_zeroes(ci, ti);
1549 	else
1550 		return false;
1551 
1552 	return true;
1553 }
1554 
1555 /*
1556  * Select the correct strategy for processing a non-flush bio.
1557  */
1558 static int __split_and_process_non_flush(struct clone_info *ci)
1559 {
1560 	struct dm_target *ti;
1561 	unsigned len;
1562 	int r;
1563 
1564 	ti = dm_table_find_target(ci->map, ci->sector);
1565 	if (!dm_target_is_valid(ti))
1566 		return -EIO;
1567 
1568 	if (unlikely(__process_abnormal_io(ci, ti, &r)))
1569 		return r;
1570 
1571 	len = min_t(sector_t, max_io_len(ci->sector, ti), ci->sector_count);
1572 
1573 	r = __clone_and_map_data_bio(ci, ti, ci->sector, &len);
1574 	if (r < 0)
1575 		return r;
1576 
1577 	ci->sector += len;
1578 	ci->sector_count -= len;
1579 
1580 	return 0;
1581 }
1582 
1583 static void init_clone_info(struct clone_info *ci, struct mapped_device *md,
1584 			    struct dm_table *map, struct bio *bio)
1585 {
1586 	ci->map = map;
1587 	ci->io = alloc_io(md, bio);
1588 	ci->sector = bio->bi_iter.bi_sector;
1589 }
1590 
1591 /*
1592  * Entry point to split a bio into clones and submit them to the targets.
1593  */
1594 static blk_qc_t __split_and_process_bio(struct mapped_device *md,
1595 					struct dm_table *map, struct bio *bio)
1596 {
1597 	struct clone_info ci;
1598 	blk_qc_t ret = BLK_QC_T_NONE;
1599 	int error = 0;
1600 
1601 	if (unlikely(!map)) {
1602 		bio_io_error(bio);
1603 		return ret;
1604 	}
1605 
1606 	blk_queue_split(md->queue, &bio);
1607 
1608 	init_clone_info(&ci, md, map, bio);
1609 
1610 	if (bio->bi_opf & REQ_PREFLUSH) {
1611 		struct bio flush_bio;
1612 
1613 		/*
1614 		 * Use an on-stack bio for this, it's safe since we don't
1615 		 * need to reference it after submit. It's just used as
1616 		 * the basis for the clone(s).
1617 		 */
1618 		bio_init(&flush_bio, NULL, 0);
1619 		flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC;
1620 		ci.bio = &flush_bio;
1621 		ci.sector_count = 0;
1622 		error = __send_empty_flush(&ci);
1623 		/* dec_pending submits any data associated with flush */
1624 	} else if (bio_op(bio) == REQ_OP_ZONE_RESET) {
1625 		ci.bio = bio;
1626 		ci.sector_count = 0;
1627 		error = __split_and_process_non_flush(&ci);
1628 	} else {
1629 		ci.bio = bio;
1630 		ci.sector_count = bio_sectors(bio);
1631 		while (ci.sector_count && !error) {
1632 			error = __split_and_process_non_flush(&ci);
1633 			if (current->bio_list && ci.sector_count && !error) {
1634 				/*
1635 				 * Remainder must be passed to generic_make_request()
1636 				 * so that it gets handled *after* bios already submitted
1637 				 * have been completely processed.
1638 				 * We take a clone of the original to store in
1639 				 * ci.io->orig_bio to be used by end_io_acct() and
1640 				 * for dec_pending to use for completion handling.
1641 				 */
1642 				struct bio *b = bio_split(bio, bio_sectors(bio) - ci.sector_count,
1643 							  GFP_NOIO, &md->queue->bio_split);
1644 				ci.io->orig_bio = b;
1645 				bio_chain(b, bio);
1646 				ret = generic_make_request(bio);
1647 				break;
1648 			}
1649 		}
1650 	}
1651 
1652 	/* drop the extra reference count */
1653 	dec_pending(ci.io, errno_to_blk_status(error));
1654 	return ret;
1655 }
1656 
1657 /*
1658  * Optimized variant of __split_and_process_bio that leverages the
1659  * fact that targets that use it do _not_ have a need to split bios.
1660  */
1661 static blk_qc_t __process_bio(struct mapped_device *md,
1662 			      struct dm_table *map, struct bio *bio)
1663 {
1664 	struct clone_info ci;
1665 	blk_qc_t ret = BLK_QC_T_NONE;
1666 	int error = 0;
1667 
1668 	if (unlikely(!map)) {
1669 		bio_io_error(bio);
1670 		return ret;
1671 	}
1672 
1673 	init_clone_info(&ci, md, map, bio);
1674 
1675 	if (bio->bi_opf & REQ_PREFLUSH) {
1676 		struct bio flush_bio;
1677 
1678 		/*
1679 		 * Use an on-stack bio for this, it's safe since we don't
1680 		 * need to reference it after submit. It's just used as
1681 		 * the basis for the clone(s).
1682 		 */
1683 		bio_init(&flush_bio, NULL, 0);
1684 		flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC;
1685 		ci.bio = &flush_bio;
1686 		ci.sector_count = 0;
1687 		error = __send_empty_flush(&ci);
1688 		/* dec_pending submits any data associated with flush */
1689 	} else {
1690 		struct dm_target *ti = md->immutable_target;
1691 		struct dm_target_io *tio;
1692 
1693 		/*
1694 		 * Defend against IO still getting in during teardown
1695 		 * - as was seen for a time with nvme-fcloop
1696 		 */
1697 		if (WARN_ON_ONCE(!ti || !dm_target_is_valid(ti))) {
1698 			error = -EIO;
1699 			goto out;
1700 		}
1701 
1702 		ci.bio = bio;
1703 		ci.sector_count = bio_sectors(bio);
1704 		if (unlikely(__process_abnormal_io(&ci, ti, &error)))
1705 			goto out;
1706 
1707 		tio = alloc_tio(&ci, ti, 0, GFP_NOIO);
1708 		ret = __clone_and_map_simple_bio(&ci, tio, NULL);
1709 	}
1710 out:
1711 	/* drop the extra reference count */
1712 	dec_pending(ci.io, errno_to_blk_status(error));
1713 	return ret;
1714 }
1715 
1716 static blk_qc_t dm_make_request(struct request_queue *q, struct bio *bio)
1717 {
1718 	struct mapped_device *md = q->queuedata;
1719 	blk_qc_t ret = BLK_QC_T_NONE;
1720 	int srcu_idx;
1721 	struct dm_table *map;
1722 
1723 	map = dm_get_live_table(md, &srcu_idx);
1724 
1725 	/* if we're suspended, we have to queue this io for later */
1726 	if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) {
1727 		dm_put_live_table(md, srcu_idx);
1728 
1729 		if (!(bio->bi_opf & REQ_RAHEAD))
1730 			queue_io(md, bio);
1731 		else
1732 			bio_io_error(bio);
1733 		return ret;
1734 	}
1735 
1736 	if (dm_get_md_type(md) == DM_TYPE_NVME_BIO_BASED)
1737 		ret = __process_bio(md, map, bio);
1738 	else
1739 		ret = __split_and_process_bio(md, map, bio);
1740 
1741 	dm_put_live_table(md, srcu_idx);
1742 	return ret;
1743 }
1744 
1745 static int dm_any_congested(void *congested_data, int bdi_bits)
1746 {
1747 	int r = bdi_bits;
1748 	struct mapped_device *md = congested_data;
1749 	struct dm_table *map;
1750 
1751 	if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
1752 		if (dm_request_based(md)) {
1753 			/*
1754 			 * With request-based DM we only need to check the
1755 			 * top-level queue for congestion.
1756 			 */
1757 			r = md->queue->backing_dev_info->wb.state & bdi_bits;
1758 		} else {
1759 			map = dm_get_live_table_fast(md);
1760 			if (map)
1761 				r = dm_table_any_congested(map, bdi_bits);
1762 			dm_put_live_table_fast(md);
1763 		}
1764 	}
1765 
1766 	return r;
1767 }
1768 
1769 /*-----------------------------------------------------------------
1770  * An IDR is used to keep track of allocated minor numbers.
1771  *---------------------------------------------------------------*/
1772 static void free_minor(int minor)
1773 {
1774 	spin_lock(&_minor_lock);
1775 	idr_remove(&_minor_idr, minor);
1776 	spin_unlock(&_minor_lock);
1777 }
1778 
1779 /*
1780  * See if the device with a specific minor # is free.
1781  */
1782 static int specific_minor(int minor)
1783 {
1784 	int r;
1785 
1786 	if (minor >= (1 << MINORBITS))
1787 		return -EINVAL;
1788 
1789 	idr_preload(GFP_KERNEL);
1790 	spin_lock(&_minor_lock);
1791 
1792 	r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT);
1793 
1794 	spin_unlock(&_minor_lock);
1795 	idr_preload_end();
1796 	if (r < 0)
1797 		return r == -ENOSPC ? -EBUSY : r;
1798 	return 0;
1799 }
1800 
1801 static int next_free_minor(int *minor)
1802 {
1803 	int r;
1804 
1805 	idr_preload(GFP_KERNEL);
1806 	spin_lock(&_minor_lock);
1807 
1808 	r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT);
1809 
1810 	spin_unlock(&_minor_lock);
1811 	idr_preload_end();
1812 	if (r < 0)
1813 		return r;
1814 	*minor = r;
1815 	return 0;
1816 }
1817 
1818 static const struct block_device_operations dm_blk_dops;
1819 static const struct dax_operations dm_dax_ops;
1820 
1821 static void dm_wq_work(struct work_struct *work);
1822 
1823 static void dm_init_normal_md_queue(struct mapped_device *md)
1824 {
1825 	/*
1826 	 * Initialize aspects of queue that aren't relevant for blk-mq
1827 	 */
1828 	md->queue->backing_dev_info->congested_fn = dm_any_congested;
1829 }
1830 
1831 static void cleanup_mapped_device(struct mapped_device *md)
1832 {
1833 	if (md->wq)
1834 		destroy_workqueue(md->wq);
1835 	bioset_exit(&md->bs);
1836 	bioset_exit(&md->io_bs);
1837 
1838 	if (md->dax_dev) {
1839 		kill_dax(md->dax_dev);
1840 		put_dax(md->dax_dev);
1841 		md->dax_dev = NULL;
1842 	}
1843 
1844 	if (md->disk) {
1845 		spin_lock(&_minor_lock);
1846 		md->disk->private_data = NULL;
1847 		spin_unlock(&_minor_lock);
1848 		del_gendisk(md->disk);
1849 		put_disk(md->disk);
1850 	}
1851 
1852 	if (md->queue)
1853 		blk_cleanup_queue(md->queue);
1854 
1855 	cleanup_srcu_struct(&md->io_barrier);
1856 
1857 	if (md->bdev) {
1858 		bdput(md->bdev);
1859 		md->bdev = NULL;
1860 	}
1861 
1862 	mutex_destroy(&md->suspend_lock);
1863 	mutex_destroy(&md->type_lock);
1864 	mutex_destroy(&md->table_devices_lock);
1865 
1866 	dm_mq_cleanup_mapped_device(md);
1867 }
1868 
1869 /*
1870  * Allocate and initialise a blank device with a given minor.
1871  */
1872 static struct mapped_device *alloc_dev(int minor)
1873 {
1874 	int r, numa_node_id = dm_get_numa_node();
1875 	struct dax_device *dax_dev = NULL;
1876 	struct mapped_device *md;
1877 	void *old_md;
1878 
1879 	md = kvzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id);
1880 	if (!md) {
1881 		DMWARN("unable to allocate device, out of memory.");
1882 		return NULL;
1883 	}
1884 
1885 	if (!try_module_get(THIS_MODULE))
1886 		goto bad_module_get;
1887 
1888 	/* get a minor number for the dev */
1889 	if (minor == DM_ANY_MINOR)
1890 		r = next_free_minor(&minor);
1891 	else
1892 		r = specific_minor(minor);
1893 	if (r < 0)
1894 		goto bad_minor;
1895 
1896 	r = init_srcu_struct(&md->io_barrier);
1897 	if (r < 0)
1898 		goto bad_io_barrier;
1899 
1900 	md->numa_node_id = numa_node_id;
1901 	md->init_tio_pdu = false;
1902 	md->type = DM_TYPE_NONE;
1903 	mutex_init(&md->suspend_lock);
1904 	mutex_init(&md->type_lock);
1905 	mutex_init(&md->table_devices_lock);
1906 	spin_lock_init(&md->deferred_lock);
1907 	atomic_set(&md->holders, 1);
1908 	atomic_set(&md->open_count, 0);
1909 	atomic_set(&md->event_nr, 0);
1910 	atomic_set(&md->uevent_seq, 0);
1911 	INIT_LIST_HEAD(&md->uevent_list);
1912 	INIT_LIST_HEAD(&md->table_devices);
1913 	spin_lock_init(&md->uevent_lock);
1914 
1915 	md->queue = blk_alloc_queue_node(GFP_KERNEL, numa_node_id);
1916 	if (!md->queue)
1917 		goto bad;
1918 	md->queue->queuedata = md;
1919 	md->queue->backing_dev_info->congested_data = md;
1920 
1921 	md->disk = alloc_disk_node(1, md->numa_node_id);
1922 	if (!md->disk)
1923 		goto bad;
1924 
1925 	init_waitqueue_head(&md->wait);
1926 	INIT_WORK(&md->work, dm_wq_work);
1927 	init_waitqueue_head(&md->eventq);
1928 	init_completion(&md->kobj_holder.completion);
1929 
1930 	md->disk->major = _major;
1931 	md->disk->first_minor = minor;
1932 	md->disk->fops = &dm_blk_dops;
1933 	md->disk->queue = md->queue;
1934 	md->disk->private_data = md;
1935 	sprintf(md->disk->disk_name, "dm-%d", minor);
1936 
1937 	if (IS_ENABLED(CONFIG_DAX_DRIVER)) {
1938 		dax_dev = alloc_dax(md, md->disk->disk_name, &dm_dax_ops);
1939 		if (!dax_dev)
1940 			goto bad;
1941 	}
1942 	md->dax_dev = dax_dev;
1943 
1944 	add_disk_no_queue_reg(md->disk);
1945 	format_dev_t(md->name, MKDEV(_major, minor));
1946 
1947 	md->wq = alloc_workqueue("kdmflush", WQ_MEM_RECLAIM, 0);
1948 	if (!md->wq)
1949 		goto bad;
1950 
1951 	md->bdev = bdget_disk(md->disk, 0);
1952 	if (!md->bdev)
1953 		goto bad;
1954 
1955 	dm_stats_init(&md->stats);
1956 
1957 	/* Populate the mapping, nobody knows we exist yet */
1958 	spin_lock(&_minor_lock);
1959 	old_md = idr_replace(&_minor_idr, md, minor);
1960 	spin_unlock(&_minor_lock);
1961 
1962 	BUG_ON(old_md != MINOR_ALLOCED);
1963 
1964 	return md;
1965 
1966 bad:
1967 	cleanup_mapped_device(md);
1968 bad_io_barrier:
1969 	free_minor(minor);
1970 bad_minor:
1971 	module_put(THIS_MODULE);
1972 bad_module_get:
1973 	kvfree(md);
1974 	return NULL;
1975 }
1976 
1977 static void unlock_fs(struct mapped_device *md);
1978 
1979 static void free_dev(struct mapped_device *md)
1980 {
1981 	int minor = MINOR(disk_devt(md->disk));
1982 
1983 	unlock_fs(md);
1984 
1985 	cleanup_mapped_device(md);
1986 
1987 	free_table_devices(&md->table_devices);
1988 	dm_stats_cleanup(&md->stats);
1989 	free_minor(minor);
1990 
1991 	module_put(THIS_MODULE);
1992 	kvfree(md);
1993 }
1994 
1995 static int __bind_mempools(struct mapped_device *md, struct dm_table *t)
1996 {
1997 	struct dm_md_mempools *p = dm_table_get_md_mempools(t);
1998 	int ret = 0;
1999 
2000 	if (dm_table_bio_based(t)) {
2001 		/*
2002 		 * The md may already have mempools that need changing.
2003 		 * If so, reload bioset because front_pad may have changed
2004 		 * because a different table was loaded.
2005 		 */
2006 		bioset_exit(&md->bs);
2007 		bioset_exit(&md->io_bs);
2008 
2009 	} else if (bioset_initialized(&md->bs)) {
2010 		/*
2011 		 * There's no need to reload with request-based dm
2012 		 * because the size of front_pad doesn't change.
2013 		 * Note for future: If you are to reload bioset,
2014 		 * prep-ed requests in the queue may refer
2015 		 * to bio from the old bioset, so you must walk
2016 		 * through the queue to unprep.
2017 		 */
2018 		goto out;
2019 	}
2020 
2021 	BUG_ON(!p ||
2022 	       bioset_initialized(&md->bs) ||
2023 	       bioset_initialized(&md->io_bs));
2024 
2025 	ret = bioset_init_from_src(&md->bs, &p->bs);
2026 	if (ret)
2027 		goto out;
2028 	ret = bioset_init_from_src(&md->io_bs, &p->io_bs);
2029 	if (ret)
2030 		bioset_exit(&md->bs);
2031 out:
2032 	/* mempool bind completed, no longer need any mempools in the table */
2033 	dm_table_free_md_mempools(t);
2034 	return ret;
2035 }
2036 
2037 /*
2038  * Bind a table to the device.
2039  */
2040 static void event_callback(void *context)
2041 {
2042 	unsigned long flags;
2043 	LIST_HEAD(uevents);
2044 	struct mapped_device *md = (struct mapped_device *) context;
2045 
2046 	spin_lock_irqsave(&md->uevent_lock, flags);
2047 	list_splice_init(&md->uevent_list, &uevents);
2048 	spin_unlock_irqrestore(&md->uevent_lock, flags);
2049 
2050 	dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
2051 
2052 	atomic_inc(&md->event_nr);
2053 	wake_up(&md->eventq);
2054 	dm_issue_global_event();
2055 }
2056 
2057 /*
2058  * Protected by md->suspend_lock obtained by dm_swap_table().
2059  */
2060 static void __set_size(struct mapped_device *md, sector_t size)
2061 {
2062 	lockdep_assert_held(&md->suspend_lock);
2063 
2064 	set_capacity(md->disk, size);
2065 
2066 	i_size_write(md->bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
2067 }
2068 
2069 /*
2070  * Returns old map, which caller must destroy.
2071  */
2072 static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
2073 			       struct queue_limits *limits)
2074 {
2075 	struct dm_table *old_map;
2076 	struct request_queue *q = md->queue;
2077 	bool request_based = dm_table_request_based(t);
2078 	sector_t size;
2079 	int ret;
2080 
2081 	lockdep_assert_held(&md->suspend_lock);
2082 
2083 	size = dm_table_get_size(t);
2084 
2085 	/*
2086 	 * Wipe any geometry if the size of the table changed.
2087 	 */
2088 	if (size != dm_get_size(md))
2089 		memset(&md->geometry, 0, sizeof(md->geometry));
2090 
2091 	__set_size(md, size);
2092 
2093 	dm_table_event_callback(t, event_callback, md);
2094 
2095 	/*
2096 	 * The queue hasn't been stopped yet, if the old table type wasn't
2097 	 * for request-based during suspension.  So stop it to prevent
2098 	 * I/O mapping before resume.
2099 	 * This must be done before setting the queue restrictions,
2100 	 * because request-based dm may be run just after the setting.
2101 	 */
2102 	if (request_based)
2103 		dm_stop_queue(q);
2104 
2105 	if (request_based || md->type == DM_TYPE_NVME_BIO_BASED) {
2106 		/*
2107 		 * Leverage the fact that request-based DM targets and
2108 		 * NVMe bio based targets are immutable singletons
2109 		 * - used to optimize both dm_request_fn and dm_mq_queue_rq;
2110 		 *   and __process_bio.
2111 		 */
2112 		md->immutable_target = dm_table_get_immutable_target(t);
2113 	}
2114 
2115 	ret = __bind_mempools(md, t);
2116 	if (ret) {
2117 		old_map = ERR_PTR(ret);
2118 		goto out;
2119 	}
2120 
2121 	old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2122 	rcu_assign_pointer(md->map, (void *)t);
2123 	md->immutable_target_type = dm_table_get_immutable_target_type(t);
2124 
2125 	dm_table_set_restrictions(t, q, limits);
2126 	if (old_map)
2127 		dm_sync_table(md);
2128 
2129 out:
2130 	return old_map;
2131 }
2132 
2133 /*
2134  * Returns unbound table for the caller to free.
2135  */
2136 static struct dm_table *__unbind(struct mapped_device *md)
2137 {
2138 	struct dm_table *map = rcu_dereference_protected(md->map, 1);
2139 
2140 	if (!map)
2141 		return NULL;
2142 
2143 	dm_table_event_callback(map, NULL, NULL);
2144 	RCU_INIT_POINTER(md->map, NULL);
2145 	dm_sync_table(md);
2146 
2147 	return map;
2148 }
2149 
2150 /*
2151  * Constructor for a new device.
2152  */
2153 int dm_create(int minor, struct mapped_device **result)
2154 {
2155 	int r;
2156 	struct mapped_device *md;
2157 
2158 	md = alloc_dev(minor);
2159 	if (!md)
2160 		return -ENXIO;
2161 
2162 	r = dm_sysfs_init(md);
2163 	if (r) {
2164 		free_dev(md);
2165 		return r;
2166 	}
2167 
2168 	*result = md;
2169 	return 0;
2170 }
2171 
2172 /*
2173  * Functions to manage md->type.
2174  * All are required to hold md->type_lock.
2175  */
2176 void dm_lock_md_type(struct mapped_device *md)
2177 {
2178 	mutex_lock(&md->type_lock);
2179 }
2180 
2181 void dm_unlock_md_type(struct mapped_device *md)
2182 {
2183 	mutex_unlock(&md->type_lock);
2184 }
2185 
2186 void dm_set_md_type(struct mapped_device *md, enum dm_queue_mode type)
2187 {
2188 	BUG_ON(!mutex_is_locked(&md->type_lock));
2189 	md->type = type;
2190 }
2191 
2192 enum dm_queue_mode dm_get_md_type(struct mapped_device *md)
2193 {
2194 	return md->type;
2195 }
2196 
2197 struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
2198 {
2199 	return md->immutable_target_type;
2200 }
2201 
2202 /*
2203  * The queue_limits are only valid as long as you have a reference
2204  * count on 'md'.
2205  */
2206 struct queue_limits *dm_get_queue_limits(struct mapped_device *md)
2207 {
2208 	BUG_ON(!atomic_read(&md->holders));
2209 	return &md->queue->limits;
2210 }
2211 EXPORT_SYMBOL_GPL(dm_get_queue_limits);
2212 
2213 /*
2214  * Setup the DM device's queue based on md's type
2215  */
2216 int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
2217 {
2218 	int r;
2219 	struct queue_limits limits;
2220 	enum dm_queue_mode type = dm_get_md_type(md);
2221 
2222 	switch (type) {
2223 	case DM_TYPE_REQUEST_BASED:
2224 		r = dm_mq_init_request_queue(md, t);
2225 		if (r) {
2226 			DMERR("Cannot initialize queue for request-based dm-mq mapped device");
2227 			return r;
2228 		}
2229 		break;
2230 	case DM_TYPE_BIO_BASED:
2231 	case DM_TYPE_DAX_BIO_BASED:
2232 	case DM_TYPE_NVME_BIO_BASED:
2233 		dm_init_normal_md_queue(md);
2234 		blk_queue_make_request(md->queue, dm_make_request);
2235 		break;
2236 	case DM_TYPE_NONE:
2237 		WARN_ON_ONCE(true);
2238 		break;
2239 	}
2240 
2241 	r = dm_calculate_queue_limits(t, &limits);
2242 	if (r) {
2243 		DMERR("Cannot calculate initial queue limits");
2244 		return r;
2245 	}
2246 	dm_table_set_restrictions(t, md->queue, &limits);
2247 	blk_register_queue(md->disk);
2248 
2249 	return 0;
2250 }
2251 
2252 struct mapped_device *dm_get_md(dev_t dev)
2253 {
2254 	struct mapped_device *md;
2255 	unsigned minor = MINOR(dev);
2256 
2257 	if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
2258 		return NULL;
2259 
2260 	spin_lock(&_minor_lock);
2261 
2262 	md = idr_find(&_minor_idr, minor);
2263 	if (!md || md == MINOR_ALLOCED || (MINOR(disk_devt(dm_disk(md))) != minor) ||
2264 	    test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
2265 		md = NULL;
2266 		goto out;
2267 	}
2268 	dm_get(md);
2269 out:
2270 	spin_unlock(&_minor_lock);
2271 
2272 	return md;
2273 }
2274 EXPORT_SYMBOL_GPL(dm_get_md);
2275 
2276 void *dm_get_mdptr(struct mapped_device *md)
2277 {
2278 	return md->interface_ptr;
2279 }
2280 
2281 void dm_set_mdptr(struct mapped_device *md, void *ptr)
2282 {
2283 	md->interface_ptr = ptr;
2284 }
2285 
2286 void dm_get(struct mapped_device *md)
2287 {
2288 	atomic_inc(&md->holders);
2289 	BUG_ON(test_bit(DMF_FREEING, &md->flags));
2290 }
2291 
2292 int dm_hold(struct mapped_device *md)
2293 {
2294 	spin_lock(&_minor_lock);
2295 	if (test_bit(DMF_FREEING, &md->flags)) {
2296 		spin_unlock(&_minor_lock);
2297 		return -EBUSY;
2298 	}
2299 	dm_get(md);
2300 	spin_unlock(&_minor_lock);
2301 	return 0;
2302 }
2303 EXPORT_SYMBOL_GPL(dm_hold);
2304 
2305 const char *dm_device_name(struct mapped_device *md)
2306 {
2307 	return md->name;
2308 }
2309 EXPORT_SYMBOL_GPL(dm_device_name);
2310 
2311 static void __dm_destroy(struct mapped_device *md, bool wait)
2312 {
2313 	struct dm_table *map;
2314 	int srcu_idx;
2315 
2316 	might_sleep();
2317 
2318 	spin_lock(&_minor_lock);
2319 	idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));
2320 	set_bit(DMF_FREEING, &md->flags);
2321 	spin_unlock(&_minor_lock);
2322 
2323 	blk_set_queue_dying(md->queue);
2324 
2325 	/*
2326 	 * Take suspend_lock so that presuspend and postsuspend methods
2327 	 * do not race with internal suspend.
2328 	 */
2329 	mutex_lock(&md->suspend_lock);
2330 	map = dm_get_live_table(md, &srcu_idx);
2331 	if (!dm_suspended_md(md)) {
2332 		dm_table_presuspend_targets(map);
2333 		dm_table_postsuspend_targets(map);
2334 	}
2335 	/* dm_put_live_table must be before msleep, otherwise deadlock is possible */
2336 	dm_put_live_table(md, srcu_idx);
2337 	mutex_unlock(&md->suspend_lock);
2338 
2339 	/*
2340 	 * Rare, but there may be I/O requests still going to complete,
2341 	 * for example.  Wait for all references to disappear.
2342 	 * No one should increment the reference count of the mapped_device,
2343 	 * after the mapped_device state becomes DMF_FREEING.
2344 	 */
2345 	if (wait)
2346 		while (atomic_read(&md->holders))
2347 			msleep(1);
2348 	else if (atomic_read(&md->holders))
2349 		DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)",
2350 		       dm_device_name(md), atomic_read(&md->holders));
2351 
2352 	dm_sysfs_exit(md);
2353 	dm_table_destroy(__unbind(md));
2354 	free_dev(md);
2355 }
2356 
2357 void dm_destroy(struct mapped_device *md)
2358 {
2359 	__dm_destroy(md, true);
2360 }
2361 
2362 void dm_destroy_immediate(struct mapped_device *md)
2363 {
2364 	__dm_destroy(md, false);
2365 }
2366 
2367 void dm_put(struct mapped_device *md)
2368 {
2369 	atomic_dec(&md->holders);
2370 }
2371 EXPORT_SYMBOL_GPL(dm_put);
2372 
2373 static int dm_wait_for_completion(struct mapped_device *md, long task_state)
2374 {
2375 	int r = 0;
2376 	DEFINE_WAIT(wait);
2377 
2378 	while (1) {
2379 		prepare_to_wait(&md->wait, &wait, task_state);
2380 
2381 		if (!md_in_flight(md))
2382 			break;
2383 
2384 		if (signal_pending_state(task_state, current)) {
2385 			r = -EINTR;
2386 			break;
2387 		}
2388 
2389 		io_schedule();
2390 	}
2391 	finish_wait(&md->wait, &wait);
2392 
2393 	return r;
2394 }
2395 
2396 /*
2397  * Process the deferred bios
2398  */
2399 static void dm_wq_work(struct work_struct *work)
2400 {
2401 	struct mapped_device *md = container_of(work, struct mapped_device,
2402 						work);
2403 	struct bio *c;
2404 	int srcu_idx;
2405 	struct dm_table *map;
2406 
2407 	map = dm_get_live_table(md, &srcu_idx);
2408 
2409 	while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
2410 		spin_lock_irq(&md->deferred_lock);
2411 		c = bio_list_pop(&md->deferred);
2412 		spin_unlock_irq(&md->deferred_lock);
2413 
2414 		if (!c)
2415 			break;
2416 
2417 		if (dm_request_based(md))
2418 			generic_make_request(c);
2419 		else
2420 			__split_and_process_bio(md, map, c);
2421 	}
2422 
2423 	dm_put_live_table(md, srcu_idx);
2424 }
2425 
2426 static void dm_queue_flush(struct mapped_device *md)
2427 {
2428 	clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2429 	smp_mb__after_atomic();
2430 	queue_work(md->wq, &md->work);
2431 }
2432 
2433 /*
2434  * Swap in a new table, returning the old one for the caller to destroy.
2435  */
2436 struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table)
2437 {
2438 	struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL);
2439 	struct queue_limits limits;
2440 	int r;
2441 
2442 	mutex_lock(&md->suspend_lock);
2443 
2444 	/* device must be suspended */
2445 	if (!dm_suspended_md(md))
2446 		goto out;
2447 
2448 	/*
2449 	 * If the new table has no data devices, retain the existing limits.
2450 	 * This helps multipath with queue_if_no_path if all paths disappear,
2451 	 * then new I/O is queued based on these limits, and then some paths
2452 	 * reappear.
2453 	 */
2454 	if (dm_table_has_no_data_devices(table)) {
2455 		live_map = dm_get_live_table_fast(md);
2456 		if (live_map)
2457 			limits = md->queue->limits;
2458 		dm_put_live_table_fast(md);
2459 	}
2460 
2461 	if (!live_map) {
2462 		r = dm_calculate_queue_limits(table, &limits);
2463 		if (r) {
2464 			map = ERR_PTR(r);
2465 			goto out;
2466 		}
2467 	}
2468 
2469 	map = __bind(md, table, &limits);
2470 	dm_issue_global_event();
2471 
2472 out:
2473 	mutex_unlock(&md->suspend_lock);
2474 	return map;
2475 }
2476 
2477 /*
2478  * Functions to lock and unlock any filesystem running on the
2479  * device.
2480  */
2481 static int lock_fs(struct mapped_device *md)
2482 {
2483 	int r;
2484 
2485 	WARN_ON(md->frozen_sb);
2486 
2487 	md->frozen_sb = freeze_bdev(md->bdev);
2488 	if (IS_ERR(md->frozen_sb)) {
2489 		r = PTR_ERR(md->frozen_sb);
2490 		md->frozen_sb = NULL;
2491 		return r;
2492 	}
2493 
2494 	set_bit(DMF_FROZEN, &md->flags);
2495 
2496 	return 0;
2497 }
2498 
2499 static void unlock_fs(struct mapped_device *md)
2500 {
2501 	if (!test_bit(DMF_FROZEN, &md->flags))
2502 		return;
2503 
2504 	thaw_bdev(md->bdev, md->frozen_sb);
2505 	md->frozen_sb = NULL;
2506 	clear_bit(DMF_FROZEN, &md->flags);
2507 }
2508 
2509 /*
2510  * @suspend_flags: DM_SUSPEND_LOCKFS_FLAG and/or DM_SUSPEND_NOFLUSH_FLAG
2511  * @task_state: e.g. TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE
2512  * @dmf_suspended_flag: DMF_SUSPENDED or DMF_SUSPENDED_INTERNALLY
2513  *
2514  * If __dm_suspend returns 0, the device is completely quiescent
2515  * now. There is no request-processing activity. All new requests
2516  * are being added to md->deferred list.
2517  */
2518 static int __dm_suspend(struct mapped_device *md, struct dm_table *map,
2519 			unsigned suspend_flags, long task_state,
2520 			int dmf_suspended_flag)
2521 {
2522 	bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG;
2523 	bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG;
2524 	int r;
2525 
2526 	lockdep_assert_held(&md->suspend_lock);
2527 
2528 	/*
2529 	 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
2530 	 * This flag is cleared before dm_suspend returns.
2531 	 */
2532 	if (noflush)
2533 		set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2534 	else
2535 		pr_debug("%s: suspending with flush\n", dm_device_name(md));
2536 
2537 	/*
2538 	 * This gets reverted if there's an error later and the targets
2539 	 * provide the .presuspend_undo hook.
2540 	 */
2541 	dm_table_presuspend_targets(map);
2542 
2543 	/*
2544 	 * Flush I/O to the device.
2545 	 * Any I/O submitted after lock_fs() may not be flushed.
2546 	 * noflush takes precedence over do_lockfs.
2547 	 * (lock_fs() flushes I/Os and waits for them to complete.)
2548 	 */
2549 	if (!noflush && do_lockfs) {
2550 		r = lock_fs(md);
2551 		if (r) {
2552 			dm_table_presuspend_undo_targets(map);
2553 			return r;
2554 		}
2555 	}
2556 
2557 	/*
2558 	 * Here we must make sure that no processes are submitting requests
2559 	 * to target drivers i.e. no one may be executing
2560 	 * __split_and_process_bio. This is called from dm_request and
2561 	 * dm_wq_work.
2562 	 *
2563 	 * To get all processes out of __split_and_process_bio in dm_request,
2564 	 * we take the write lock. To prevent any process from reentering
2565 	 * __split_and_process_bio from dm_request and quiesce the thread
2566 	 * (dm_wq_work), we set BMF_BLOCK_IO_FOR_SUSPEND and call
2567 	 * flush_workqueue(md->wq).
2568 	 */
2569 	set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2570 	if (map)
2571 		synchronize_srcu(&md->io_barrier);
2572 
2573 	/*
2574 	 * Stop md->queue before flushing md->wq in case request-based
2575 	 * dm defers requests to md->wq from md->queue.
2576 	 */
2577 	if (dm_request_based(md))
2578 		dm_stop_queue(md->queue);
2579 
2580 	flush_workqueue(md->wq);
2581 
2582 	/*
2583 	 * At this point no more requests are entering target request routines.
2584 	 * We call dm_wait_for_completion to wait for all existing requests
2585 	 * to finish.
2586 	 */
2587 	r = dm_wait_for_completion(md, task_state);
2588 	if (!r)
2589 		set_bit(dmf_suspended_flag, &md->flags);
2590 
2591 	if (noflush)
2592 		clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2593 	if (map)
2594 		synchronize_srcu(&md->io_barrier);
2595 
2596 	/* were we interrupted ? */
2597 	if (r < 0) {
2598 		dm_queue_flush(md);
2599 
2600 		if (dm_request_based(md))
2601 			dm_start_queue(md->queue);
2602 
2603 		unlock_fs(md);
2604 		dm_table_presuspend_undo_targets(map);
2605 		/* pushback list is already flushed, so skip flush */
2606 	}
2607 
2608 	return r;
2609 }
2610 
2611 /*
2612  * We need to be able to change a mapping table under a mounted
2613  * filesystem.  For example we might want to move some data in
2614  * the background.  Before the table can be swapped with
2615  * dm_bind_table, dm_suspend must be called to flush any in
2616  * flight bios and ensure that any further io gets deferred.
2617  */
2618 /*
2619  * Suspend mechanism in request-based dm.
2620  *
2621  * 1. Flush all I/Os by lock_fs() if needed.
2622  * 2. Stop dispatching any I/O by stopping the request_queue.
2623  * 3. Wait for all in-flight I/Os to be completed or requeued.
2624  *
2625  * To abort suspend, start the request_queue.
2626  */
2627 int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
2628 {
2629 	struct dm_table *map = NULL;
2630 	int r = 0;
2631 
2632 retry:
2633 	mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2634 
2635 	if (dm_suspended_md(md)) {
2636 		r = -EINVAL;
2637 		goto out_unlock;
2638 	}
2639 
2640 	if (dm_suspended_internally_md(md)) {
2641 		/* already internally suspended, wait for internal resume */
2642 		mutex_unlock(&md->suspend_lock);
2643 		r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2644 		if (r)
2645 			return r;
2646 		goto retry;
2647 	}
2648 
2649 	map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2650 
2651 	r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED);
2652 	if (r)
2653 		goto out_unlock;
2654 
2655 	dm_table_postsuspend_targets(map);
2656 
2657 out_unlock:
2658 	mutex_unlock(&md->suspend_lock);
2659 	return r;
2660 }
2661 
2662 static int __dm_resume(struct mapped_device *md, struct dm_table *map)
2663 {
2664 	if (map) {
2665 		int r = dm_table_resume_targets(map);
2666 		if (r)
2667 			return r;
2668 	}
2669 
2670 	dm_queue_flush(md);
2671 
2672 	/*
2673 	 * Flushing deferred I/Os must be done after targets are resumed
2674 	 * so that mapping of targets can work correctly.
2675 	 * Request-based dm is queueing the deferred I/Os in its request_queue.
2676 	 */
2677 	if (dm_request_based(md))
2678 		dm_start_queue(md->queue);
2679 
2680 	unlock_fs(md);
2681 
2682 	return 0;
2683 }
2684 
2685 int dm_resume(struct mapped_device *md)
2686 {
2687 	int r;
2688 	struct dm_table *map = NULL;
2689 
2690 retry:
2691 	r = -EINVAL;
2692 	mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2693 
2694 	if (!dm_suspended_md(md))
2695 		goto out;
2696 
2697 	if (dm_suspended_internally_md(md)) {
2698 		/* already internally suspended, wait for internal resume */
2699 		mutex_unlock(&md->suspend_lock);
2700 		r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2701 		if (r)
2702 			return r;
2703 		goto retry;
2704 	}
2705 
2706 	map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2707 	if (!map || !dm_table_get_size(map))
2708 		goto out;
2709 
2710 	r = __dm_resume(md, map);
2711 	if (r)
2712 		goto out;
2713 
2714 	clear_bit(DMF_SUSPENDED, &md->flags);
2715 out:
2716 	mutex_unlock(&md->suspend_lock);
2717 
2718 	return r;
2719 }
2720 
2721 /*
2722  * Internal suspend/resume works like userspace-driven suspend. It waits
2723  * until all bios finish and prevents issuing new bios to the target drivers.
2724  * It may be used only from the kernel.
2725  */
2726 
2727 static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags)
2728 {
2729 	struct dm_table *map = NULL;
2730 
2731 	lockdep_assert_held(&md->suspend_lock);
2732 
2733 	if (md->internal_suspend_count++)
2734 		return; /* nested internal suspend */
2735 
2736 	if (dm_suspended_md(md)) {
2737 		set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2738 		return; /* nest suspend */
2739 	}
2740 
2741 	map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2742 
2743 	/*
2744 	 * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is
2745 	 * supported.  Properly supporting a TASK_INTERRUPTIBLE internal suspend
2746 	 * would require changing .presuspend to return an error -- avoid this
2747 	 * until there is a need for more elaborate variants of internal suspend.
2748 	 */
2749 	(void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE,
2750 			    DMF_SUSPENDED_INTERNALLY);
2751 
2752 	dm_table_postsuspend_targets(map);
2753 }
2754 
2755 static void __dm_internal_resume(struct mapped_device *md)
2756 {
2757 	BUG_ON(!md->internal_suspend_count);
2758 
2759 	if (--md->internal_suspend_count)
2760 		return; /* resume from nested internal suspend */
2761 
2762 	if (dm_suspended_md(md))
2763 		goto done; /* resume from nested suspend */
2764 
2765 	/*
2766 	 * NOTE: existing callers don't need to call dm_table_resume_targets
2767 	 * (which may fail -- so best to avoid it for now by passing NULL map)
2768 	 */
2769 	(void) __dm_resume(md, NULL);
2770 
2771 done:
2772 	clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2773 	smp_mb__after_atomic();
2774 	wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY);
2775 }
2776 
2777 void dm_internal_suspend_noflush(struct mapped_device *md)
2778 {
2779 	mutex_lock(&md->suspend_lock);
2780 	__dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG);
2781 	mutex_unlock(&md->suspend_lock);
2782 }
2783 EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush);
2784 
2785 void dm_internal_resume(struct mapped_device *md)
2786 {
2787 	mutex_lock(&md->suspend_lock);
2788 	__dm_internal_resume(md);
2789 	mutex_unlock(&md->suspend_lock);
2790 }
2791 EXPORT_SYMBOL_GPL(dm_internal_resume);
2792 
2793 /*
2794  * Fast variants of internal suspend/resume hold md->suspend_lock,
2795  * which prevents interaction with userspace-driven suspend.
2796  */
2797 
2798 void dm_internal_suspend_fast(struct mapped_device *md)
2799 {
2800 	mutex_lock(&md->suspend_lock);
2801 	if (dm_suspended_md(md) || dm_suspended_internally_md(md))
2802 		return;
2803 
2804 	set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2805 	synchronize_srcu(&md->io_barrier);
2806 	flush_workqueue(md->wq);
2807 	dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
2808 }
2809 EXPORT_SYMBOL_GPL(dm_internal_suspend_fast);
2810 
2811 void dm_internal_resume_fast(struct mapped_device *md)
2812 {
2813 	if (dm_suspended_md(md) || dm_suspended_internally_md(md))
2814 		goto done;
2815 
2816 	dm_queue_flush(md);
2817 
2818 done:
2819 	mutex_unlock(&md->suspend_lock);
2820 }
2821 EXPORT_SYMBOL_GPL(dm_internal_resume_fast);
2822 
2823 /*-----------------------------------------------------------------
2824  * Event notification.
2825  *---------------------------------------------------------------*/
2826 int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action,
2827 		       unsigned cookie)
2828 {
2829 	char udev_cookie[DM_COOKIE_LENGTH];
2830 	char *envp[] = { udev_cookie, NULL };
2831 
2832 	if (!cookie)
2833 		return kobject_uevent(&disk_to_dev(md->disk)->kobj, action);
2834 	else {
2835 		snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u",
2836 			 DM_COOKIE_ENV_VAR_NAME, cookie);
2837 		return kobject_uevent_env(&disk_to_dev(md->disk)->kobj,
2838 					  action, envp);
2839 	}
2840 }
2841 
2842 uint32_t dm_next_uevent_seq(struct mapped_device *md)
2843 {
2844 	return atomic_add_return(1, &md->uevent_seq);
2845 }
2846 
2847 uint32_t dm_get_event_nr(struct mapped_device *md)
2848 {
2849 	return atomic_read(&md->event_nr);
2850 }
2851 
2852 int dm_wait_event(struct mapped_device *md, int event_nr)
2853 {
2854 	return wait_event_interruptible(md->eventq,
2855 			(event_nr != atomic_read(&md->event_nr)));
2856 }
2857 
2858 void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
2859 {
2860 	unsigned long flags;
2861 
2862 	spin_lock_irqsave(&md->uevent_lock, flags);
2863 	list_add(elist, &md->uevent_list);
2864 	spin_unlock_irqrestore(&md->uevent_lock, flags);
2865 }
2866 
2867 /*
2868  * The gendisk is only valid as long as you have a reference
2869  * count on 'md'.
2870  */
2871 struct gendisk *dm_disk(struct mapped_device *md)
2872 {
2873 	return md->disk;
2874 }
2875 EXPORT_SYMBOL_GPL(dm_disk);
2876 
2877 struct kobject *dm_kobject(struct mapped_device *md)
2878 {
2879 	return &md->kobj_holder.kobj;
2880 }
2881 
2882 struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
2883 {
2884 	struct mapped_device *md;
2885 
2886 	md = container_of(kobj, struct mapped_device, kobj_holder.kobj);
2887 
2888 	spin_lock(&_minor_lock);
2889 	if (test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
2890 		md = NULL;
2891 		goto out;
2892 	}
2893 	dm_get(md);
2894 out:
2895 	spin_unlock(&_minor_lock);
2896 
2897 	return md;
2898 }
2899 
2900 int dm_suspended_md(struct mapped_device *md)
2901 {
2902 	return test_bit(DMF_SUSPENDED, &md->flags);
2903 }
2904 
2905 int dm_suspended_internally_md(struct mapped_device *md)
2906 {
2907 	return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2908 }
2909 
2910 int dm_test_deferred_remove_flag(struct mapped_device *md)
2911 {
2912 	return test_bit(DMF_DEFERRED_REMOVE, &md->flags);
2913 }
2914 
2915 int dm_suspended(struct dm_target *ti)
2916 {
2917 	return dm_suspended_md(dm_table_get_md(ti->table));
2918 }
2919 EXPORT_SYMBOL_GPL(dm_suspended);
2920 
2921 int dm_noflush_suspending(struct dm_target *ti)
2922 {
2923 	return __noflush_suspending(dm_table_get_md(ti->table));
2924 }
2925 EXPORT_SYMBOL_GPL(dm_noflush_suspending);
2926 
2927 struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_queue_mode type,
2928 					    unsigned integrity, unsigned per_io_data_size,
2929 					    unsigned min_pool_size)
2930 {
2931 	struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id);
2932 	unsigned int pool_size = 0;
2933 	unsigned int front_pad, io_front_pad;
2934 	int ret;
2935 
2936 	if (!pools)
2937 		return NULL;
2938 
2939 	switch (type) {
2940 	case DM_TYPE_BIO_BASED:
2941 	case DM_TYPE_DAX_BIO_BASED:
2942 	case DM_TYPE_NVME_BIO_BASED:
2943 		pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size);
2944 		front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);
2945 		io_front_pad = roundup(front_pad,  __alignof__(struct dm_io)) + offsetof(struct dm_io, tio);
2946 		ret = bioset_init(&pools->io_bs, pool_size, io_front_pad, 0);
2947 		if (ret)
2948 			goto out;
2949 		if (integrity && bioset_integrity_create(&pools->io_bs, pool_size))
2950 			goto out;
2951 		break;
2952 	case DM_TYPE_REQUEST_BASED:
2953 		pool_size = max(dm_get_reserved_rq_based_ios(), min_pool_size);
2954 		front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
2955 		/* per_io_data_size is used for blk-mq pdu at queue allocation */
2956 		break;
2957 	default:
2958 		BUG();
2959 	}
2960 
2961 	ret = bioset_init(&pools->bs, pool_size, front_pad, 0);
2962 	if (ret)
2963 		goto out;
2964 
2965 	if (integrity && bioset_integrity_create(&pools->bs, pool_size))
2966 		goto out;
2967 
2968 	return pools;
2969 
2970 out:
2971 	dm_free_md_mempools(pools);
2972 
2973 	return NULL;
2974 }
2975 
2976 void dm_free_md_mempools(struct dm_md_mempools *pools)
2977 {
2978 	if (!pools)
2979 		return;
2980 
2981 	bioset_exit(&pools->bs);
2982 	bioset_exit(&pools->io_bs);
2983 
2984 	kfree(pools);
2985 }
2986 
2987 struct dm_pr {
2988 	u64	old_key;
2989 	u64	new_key;
2990 	u32	flags;
2991 	bool	fail_early;
2992 };
2993 
2994 static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn,
2995 		      void *data)
2996 {
2997 	struct mapped_device *md = bdev->bd_disk->private_data;
2998 	struct dm_table *table;
2999 	struct dm_target *ti;
3000 	int ret = -ENOTTY, srcu_idx;
3001 
3002 	table = dm_get_live_table(md, &srcu_idx);
3003 	if (!table || !dm_table_get_size(table))
3004 		goto out;
3005 
3006 	/* We only support devices that have a single target */
3007 	if (dm_table_get_num_targets(table) != 1)
3008 		goto out;
3009 	ti = dm_table_get_target(table, 0);
3010 
3011 	ret = -EINVAL;
3012 	if (!ti->type->iterate_devices)
3013 		goto out;
3014 
3015 	ret = ti->type->iterate_devices(ti, fn, data);
3016 out:
3017 	dm_put_live_table(md, srcu_idx);
3018 	return ret;
3019 }
3020 
3021 /*
3022  * For register / unregister we need to manually call out to every path.
3023  */
3024 static int __dm_pr_register(struct dm_target *ti, struct dm_dev *dev,
3025 			    sector_t start, sector_t len, void *data)
3026 {
3027 	struct dm_pr *pr = data;
3028 	const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
3029 
3030 	if (!ops || !ops->pr_register)
3031 		return -EOPNOTSUPP;
3032 	return ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags);
3033 }
3034 
3035 static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key,
3036 			  u32 flags)
3037 {
3038 	struct dm_pr pr = {
3039 		.old_key	= old_key,
3040 		.new_key	= new_key,
3041 		.flags		= flags,
3042 		.fail_early	= true,
3043 	};
3044 	int ret;
3045 
3046 	ret = dm_call_pr(bdev, __dm_pr_register, &pr);
3047 	if (ret && new_key) {
3048 		/* unregister all paths if we failed to register any path */
3049 		pr.old_key = new_key;
3050 		pr.new_key = 0;
3051 		pr.flags = 0;
3052 		pr.fail_early = false;
3053 		dm_call_pr(bdev, __dm_pr_register, &pr);
3054 	}
3055 
3056 	return ret;
3057 }
3058 
3059 static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type,
3060 			 u32 flags)
3061 {
3062 	struct mapped_device *md = bdev->bd_disk->private_data;
3063 	const struct pr_ops *ops;
3064 	int r, srcu_idx;
3065 
3066 	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
3067 	if (r < 0)
3068 		goto out;
3069 
3070 	ops = bdev->bd_disk->fops->pr_ops;
3071 	if (ops && ops->pr_reserve)
3072 		r = ops->pr_reserve(bdev, key, type, flags);
3073 	else
3074 		r = -EOPNOTSUPP;
3075 out:
3076 	dm_unprepare_ioctl(md, srcu_idx);
3077 	return r;
3078 }
3079 
3080 static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
3081 {
3082 	struct mapped_device *md = bdev->bd_disk->private_data;
3083 	const struct pr_ops *ops;
3084 	int r, srcu_idx;
3085 
3086 	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
3087 	if (r < 0)
3088 		goto out;
3089 
3090 	ops = bdev->bd_disk->fops->pr_ops;
3091 	if (ops && ops->pr_release)
3092 		r = ops->pr_release(bdev, key, type);
3093 	else
3094 		r = -EOPNOTSUPP;
3095 out:
3096 	dm_unprepare_ioctl(md, srcu_idx);
3097 	return r;
3098 }
3099 
3100 static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key,
3101 			 enum pr_type type, bool abort)
3102 {
3103 	struct mapped_device *md = bdev->bd_disk->private_data;
3104 	const struct pr_ops *ops;
3105 	int r, srcu_idx;
3106 
3107 	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
3108 	if (r < 0)
3109 		goto out;
3110 
3111 	ops = bdev->bd_disk->fops->pr_ops;
3112 	if (ops && ops->pr_preempt)
3113 		r = ops->pr_preempt(bdev, old_key, new_key, type, abort);
3114 	else
3115 		r = -EOPNOTSUPP;
3116 out:
3117 	dm_unprepare_ioctl(md, srcu_idx);
3118 	return r;
3119 }
3120 
3121 static int dm_pr_clear(struct block_device *bdev, u64 key)
3122 {
3123 	struct mapped_device *md = bdev->bd_disk->private_data;
3124 	const struct pr_ops *ops;
3125 	int r, srcu_idx;
3126 
3127 	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
3128 	if (r < 0)
3129 		goto out;
3130 
3131 	ops = bdev->bd_disk->fops->pr_ops;
3132 	if (ops && ops->pr_clear)
3133 		r = ops->pr_clear(bdev, key);
3134 	else
3135 		r = -EOPNOTSUPP;
3136 out:
3137 	dm_unprepare_ioctl(md, srcu_idx);
3138 	return r;
3139 }
3140 
3141 static const struct pr_ops dm_pr_ops = {
3142 	.pr_register	= dm_pr_register,
3143 	.pr_reserve	= dm_pr_reserve,
3144 	.pr_release	= dm_pr_release,
3145 	.pr_preempt	= dm_pr_preempt,
3146 	.pr_clear	= dm_pr_clear,
3147 };
3148 
3149 static const struct block_device_operations dm_blk_dops = {
3150 	.open = dm_blk_open,
3151 	.release = dm_blk_close,
3152 	.ioctl = dm_blk_ioctl,
3153 	.getgeo = dm_blk_getgeo,
3154 	.report_zones = dm_blk_report_zones,
3155 	.pr_ops = &dm_pr_ops,
3156 	.owner = THIS_MODULE
3157 };
3158 
3159 static const struct dax_operations dm_dax_ops = {
3160 	.direct_access = dm_dax_direct_access,
3161 	.copy_from_iter = dm_dax_copy_from_iter,
3162 	.copy_to_iter = dm_dax_copy_to_iter,
3163 };
3164 
3165 /*
3166  * module hooks
3167  */
3168 module_init(dm_init);
3169 module_exit(dm_exit);
3170 
3171 module_param(major, uint, 0);
3172 MODULE_PARM_DESC(major, "The major number of the device mapper");
3173 
3174 module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR);
3175 MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools");
3176 
3177 module_param(dm_numa_node, int, S_IRUGO | S_IWUSR);
3178 MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations");
3179 
3180 MODULE_DESCRIPTION(DM_NAME " driver");
3181 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
3182 MODULE_LICENSE("GPL");
3183