xref: /openbmc/linux/drivers/md/dm.c (revision 82003e04)
1 /*
2  * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3  * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
4  *
5  * This file is released under the GPL.
6  */
7 
8 #include "dm-core.h"
9 #include "dm-rq.h"
10 #include "dm-uevent.h"
11 
12 #include <linux/init.h>
13 #include <linux/module.h>
14 #include <linux/mutex.h>
15 #include <linux/blkpg.h>
16 #include <linux/bio.h>
17 #include <linux/mempool.h>
18 #include <linux/slab.h>
19 #include <linux/idr.h>
20 #include <linux/hdreg.h>
21 #include <linux/delay.h>
22 #include <linux/wait.h>
23 #include <linux/pr.h>
24 
25 #define DM_MSG_PREFIX "core"
26 
27 #ifdef CONFIG_PRINTK
28 /*
29  * ratelimit state to be used in DMXXX_LIMIT().
30  */
31 DEFINE_RATELIMIT_STATE(dm_ratelimit_state,
32 		       DEFAULT_RATELIMIT_INTERVAL,
33 		       DEFAULT_RATELIMIT_BURST);
34 EXPORT_SYMBOL(dm_ratelimit_state);
35 #endif
36 
37 /*
38  * Cookies are numeric values sent with CHANGE and REMOVE
39  * uevents while resuming, removing or renaming the device.
40  */
41 #define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE"
42 #define DM_COOKIE_LENGTH 24
43 
44 static const char *_name = DM_NAME;
45 
46 static unsigned int major = 0;
47 static unsigned int _major = 0;
48 
49 static DEFINE_IDR(_minor_idr);
50 
51 static DEFINE_SPINLOCK(_minor_lock);
52 
53 static void do_deferred_remove(struct work_struct *w);
54 
55 static DECLARE_WORK(deferred_remove_work, do_deferred_remove);
56 
57 static struct workqueue_struct *deferred_remove_workqueue;
58 
59 /*
60  * One of these is allocated per bio.
61  */
62 struct dm_io {
63 	struct mapped_device *md;
64 	int error;
65 	atomic_t io_count;
66 	struct bio *bio;
67 	unsigned long start_time;
68 	spinlock_t endio_lock;
69 	struct dm_stats_aux stats_aux;
70 };
71 
72 #define MINOR_ALLOCED ((void *)-1)
73 
74 /*
75  * Bits for the md->flags field.
76  */
77 #define DMF_BLOCK_IO_FOR_SUSPEND 0
78 #define DMF_SUSPENDED 1
79 #define DMF_FROZEN 2
80 #define DMF_FREEING 3
81 #define DMF_DELETING 4
82 #define DMF_NOFLUSH_SUSPENDING 5
83 #define DMF_DEFERRED_REMOVE 6
84 #define DMF_SUSPENDED_INTERNALLY 7
85 
86 #define DM_NUMA_NODE NUMA_NO_NODE
87 static int dm_numa_node = DM_NUMA_NODE;
88 
89 /*
90  * For mempools pre-allocation at the table loading time.
91  */
92 struct dm_md_mempools {
93 	mempool_t *io_pool;
94 	mempool_t *rq_pool;
95 	struct bio_set *bs;
96 };
97 
98 struct table_device {
99 	struct list_head list;
100 	atomic_t count;
101 	struct dm_dev dm_dev;
102 };
103 
104 static struct kmem_cache *_io_cache;
105 static struct kmem_cache *_rq_tio_cache;
106 static struct kmem_cache *_rq_cache;
107 
108 /*
109  * Bio-based DM's mempools' reserved IOs set by the user.
110  */
111 #define RESERVED_BIO_BASED_IOS		16
112 static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS;
113 
114 static int __dm_get_module_param_int(int *module_param, int min, int max)
115 {
116 	int param = ACCESS_ONCE(*module_param);
117 	int modified_param = 0;
118 	bool modified = true;
119 
120 	if (param < min)
121 		modified_param = min;
122 	else if (param > max)
123 		modified_param = max;
124 	else
125 		modified = false;
126 
127 	if (modified) {
128 		(void)cmpxchg(module_param, param, modified_param);
129 		param = modified_param;
130 	}
131 
132 	return param;
133 }
134 
135 unsigned __dm_get_module_param(unsigned *module_param,
136 			       unsigned def, unsigned max)
137 {
138 	unsigned param = ACCESS_ONCE(*module_param);
139 	unsigned modified_param = 0;
140 
141 	if (!param)
142 		modified_param = def;
143 	else if (param > max)
144 		modified_param = max;
145 
146 	if (modified_param) {
147 		(void)cmpxchg(module_param, param, modified_param);
148 		param = modified_param;
149 	}
150 
151 	return param;
152 }
153 
154 unsigned dm_get_reserved_bio_based_ios(void)
155 {
156 	return __dm_get_module_param(&reserved_bio_based_ios,
157 				     RESERVED_BIO_BASED_IOS, DM_RESERVED_MAX_IOS);
158 }
159 EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios);
160 
161 static unsigned dm_get_numa_node(void)
162 {
163 	return __dm_get_module_param_int(&dm_numa_node,
164 					 DM_NUMA_NODE, num_online_nodes() - 1);
165 }
166 
167 static int __init local_init(void)
168 {
169 	int r = -ENOMEM;
170 
171 	/* allocate a slab for the dm_ios */
172 	_io_cache = KMEM_CACHE(dm_io, 0);
173 	if (!_io_cache)
174 		return r;
175 
176 	_rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0);
177 	if (!_rq_tio_cache)
178 		goto out_free_io_cache;
179 
180 	_rq_cache = kmem_cache_create("dm_old_clone_request", sizeof(struct request),
181 				      __alignof__(struct request), 0, NULL);
182 	if (!_rq_cache)
183 		goto out_free_rq_tio_cache;
184 
185 	r = dm_uevent_init();
186 	if (r)
187 		goto out_free_rq_cache;
188 
189 	deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1);
190 	if (!deferred_remove_workqueue) {
191 		r = -ENOMEM;
192 		goto out_uevent_exit;
193 	}
194 
195 	_major = major;
196 	r = register_blkdev(_major, _name);
197 	if (r < 0)
198 		goto out_free_workqueue;
199 
200 	if (!_major)
201 		_major = r;
202 
203 	return 0;
204 
205 out_free_workqueue:
206 	destroy_workqueue(deferred_remove_workqueue);
207 out_uevent_exit:
208 	dm_uevent_exit();
209 out_free_rq_cache:
210 	kmem_cache_destroy(_rq_cache);
211 out_free_rq_tio_cache:
212 	kmem_cache_destroy(_rq_tio_cache);
213 out_free_io_cache:
214 	kmem_cache_destroy(_io_cache);
215 
216 	return r;
217 }
218 
219 static void local_exit(void)
220 {
221 	flush_scheduled_work();
222 	destroy_workqueue(deferred_remove_workqueue);
223 
224 	kmem_cache_destroy(_rq_cache);
225 	kmem_cache_destroy(_rq_tio_cache);
226 	kmem_cache_destroy(_io_cache);
227 	unregister_blkdev(_major, _name);
228 	dm_uevent_exit();
229 
230 	_major = 0;
231 
232 	DMINFO("cleaned up");
233 }
234 
235 static int (*_inits[])(void) __initdata = {
236 	local_init,
237 	dm_target_init,
238 	dm_linear_init,
239 	dm_stripe_init,
240 	dm_io_init,
241 	dm_kcopyd_init,
242 	dm_interface_init,
243 	dm_statistics_init,
244 };
245 
246 static void (*_exits[])(void) = {
247 	local_exit,
248 	dm_target_exit,
249 	dm_linear_exit,
250 	dm_stripe_exit,
251 	dm_io_exit,
252 	dm_kcopyd_exit,
253 	dm_interface_exit,
254 	dm_statistics_exit,
255 };
256 
257 static int __init dm_init(void)
258 {
259 	const int count = ARRAY_SIZE(_inits);
260 
261 	int r, i;
262 
263 	for (i = 0; i < count; i++) {
264 		r = _inits[i]();
265 		if (r)
266 			goto bad;
267 	}
268 
269 	return 0;
270 
271       bad:
272 	while (i--)
273 		_exits[i]();
274 
275 	return r;
276 }
277 
278 static void __exit dm_exit(void)
279 {
280 	int i = ARRAY_SIZE(_exits);
281 
282 	while (i--)
283 		_exits[i]();
284 
285 	/*
286 	 * Should be empty by this point.
287 	 */
288 	idr_destroy(&_minor_idr);
289 }
290 
291 /*
292  * Block device functions
293  */
294 int dm_deleting_md(struct mapped_device *md)
295 {
296 	return test_bit(DMF_DELETING, &md->flags);
297 }
298 
299 static int dm_blk_open(struct block_device *bdev, fmode_t mode)
300 {
301 	struct mapped_device *md;
302 
303 	spin_lock(&_minor_lock);
304 
305 	md = bdev->bd_disk->private_data;
306 	if (!md)
307 		goto out;
308 
309 	if (test_bit(DMF_FREEING, &md->flags) ||
310 	    dm_deleting_md(md)) {
311 		md = NULL;
312 		goto out;
313 	}
314 
315 	dm_get(md);
316 	atomic_inc(&md->open_count);
317 out:
318 	spin_unlock(&_minor_lock);
319 
320 	return md ? 0 : -ENXIO;
321 }
322 
323 static void dm_blk_close(struct gendisk *disk, fmode_t mode)
324 {
325 	struct mapped_device *md;
326 
327 	spin_lock(&_minor_lock);
328 
329 	md = disk->private_data;
330 	if (WARN_ON(!md))
331 		goto out;
332 
333 	if (atomic_dec_and_test(&md->open_count) &&
334 	    (test_bit(DMF_DEFERRED_REMOVE, &md->flags)))
335 		queue_work(deferred_remove_workqueue, &deferred_remove_work);
336 
337 	dm_put(md);
338 out:
339 	spin_unlock(&_minor_lock);
340 }
341 
342 int dm_open_count(struct mapped_device *md)
343 {
344 	return atomic_read(&md->open_count);
345 }
346 
347 /*
348  * Guarantees nothing is using the device before it's deleted.
349  */
350 int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred)
351 {
352 	int r = 0;
353 
354 	spin_lock(&_minor_lock);
355 
356 	if (dm_open_count(md)) {
357 		r = -EBUSY;
358 		if (mark_deferred)
359 			set_bit(DMF_DEFERRED_REMOVE, &md->flags);
360 	} else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags))
361 		r = -EEXIST;
362 	else
363 		set_bit(DMF_DELETING, &md->flags);
364 
365 	spin_unlock(&_minor_lock);
366 
367 	return r;
368 }
369 
370 int dm_cancel_deferred_remove(struct mapped_device *md)
371 {
372 	int r = 0;
373 
374 	spin_lock(&_minor_lock);
375 
376 	if (test_bit(DMF_DELETING, &md->flags))
377 		r = -EBUSY;
378 	else
379 		clear_bit(DMF_DEFERRED_REMOVE, &md->flags);
380 
381 	spin_unlock(&_minor_lock);
382 
383 	return r;
384 }
385 
386 static void do_deferred_remove(struct work_struct *w)
387 {
388 	dm_deferred_remove();
389 }
390 
391 sector_t dm_get_size(struct mapped_device *md)
392 {
393 	return get_capacity(md->disk);
394 }
395 
396 struct request_queue *dm_get_md_queue(struct mapped_device *md)
397 {
398 	return md->queue;
399 }
400 
401 struct dm_stats *dm_get_stats(struct mapped_device *md)
402 {
403 	return &md->stats;
404 }
405 
406 static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
407 {
408 	struct mapped_device *md = bdev->bd_disk->private_data;
409 
410 	return dm_get_geometry(md, geo);
411 }
412 
413 static int dm_grab_bdev_for_ioctl(struct mapped_device *md,
414 				  struct block_device **bdev,
415 				  fmode_t *mode)
416 {
417 	struct dm_target *tgt;
418 	struct dm_table *map;
419 	int srcu_idx, r;
420 
421 retry:
422 	r = -ENOTTY;
423 	map = dm_get_live_table(md, &srcu_idx);
424 	if (!map || !dm_table_get_size(map))
425 		goto out;
426 
427 	/* We only support devices that have a single target */
428 	if (dm_table_get_num_targets(map) != 1)
429 		goto out;
430 
431 	tgt = dm_table_get_target(map, 0);
432 	if (!tgt->type->prepare_ioctl)
433 		goto out;
434 
435 	if (dm_suspended_md(md)) {
436 		r = -EAGAIN;
437 		goto out;
438 	}
439 
440 	r = tgt->type->prepare_ioctl(tgt, bdev, mode);
441 	if (r < 0)
442 		goto out;
443 
444 	bdgrab(*bdev);
445 	dm_put_live_table(md, srcu_idx);
446 	return r;
447 
448 out:
449 	dm_put_live_table(md, srcu_idx);
450 	if (r == -ENOTCONN && !fatal_signal_pending(current)) {
451 		msleep(10);
452 		goto retry;
453 	}
454 	return r;
455 }
456 
457 static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
458 			unsigned int cmd, unsigned long arg)
459 {
460 	struct mapped_device *md = bdev->bd_disk->private_data;
461 	int r;
462 
463 	r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
464 	if (r < 0)
465 		return r;
466 
467 	if (r > 0) {
468 		/*
469 		 * Target determined this ioctl is being issued against
470 		 * a logical partition of the parent bdev; so extra
471 		 * validation is needed.
472 		 */
473 		r = scsi_verify_blk_ioctl(NULL, cmd);
474 		if (r)
475 			goto out;
476 	}
477 
478 	r =  __blkdev_driver_ioctl(bdev, mode, cmd, arg);
479 out:
480 	bdput(bdev);
481 	return r;
482 }
483 
484 static struct dm_io *alloc_io(struct mapped_device *md)
485 {
486 	return mempool_alloc(md->io_pool, GFP_NOIO);
487 }
488 
489 static void free_io(struct mapped_device *md, struct dm_io *io)
490 {
491 	mempool_free(io, md->io_pool);
492 }
493 
494 static void free_tio(struct dm_target_io *tio)
495 {
496 	bio_put(&tio->clone);
497 }
498 
499 int md_in_flight(struct mapped_device *md)
500 {
501 	return atomic_read(&md->pending[READ]) +
502 	       atomic_read(&md->pending[WRITE]);
503 }
504 
505 static void start_io_acct(struct dm_io *io)
506 {
507 	struct mapped_device *md = io->md;
508 	struct bio *bio = io->bio;
509 	int cpu;
510 	int rw = bio_data_dir(bio);
511 
512 	io->start_time = jiffies;
513 
514 	cpu = part_stat_lock();
515 	part_round_stats(cpu, &dm_disk(md)->part0);
516 	part_stat_unlock();
517 	atomic_set(&dm_disk(md)->part0.in_flight[rw],
518 		atomic_inc_return(&md->pending[rw]));
519 
520 	if (unlikely(dm_stats_used(&md->stats)))
521 		dm_stats_account_io(&md->stats, bio_data_dir(bio),
522 				    bio->bi_iter.bi_sector, bio_sectors(bio),
523 				    false, 0, &io->stats_aux);
524 }
525 
526 static void end_io_acct(struct dm_io *io)
527 {
528 	struct mapped_device *md = io->md;
529 	struct bio *bio = io->bio;
530 	unsigned long duration = jiffies - io->start_time;
531 	int pending;
532 	int rw = bio_data_dir(bio);
533 
534 	generic_end_io_acct(rw, &dm_disk(md)->part0, io->start_time);
535 
536 	if (unlikely(dm_stats_used(&md->stats)))
537 		dm_stats_account_io(&md->stats, bio_data_dir(bio),
538 				    bio->bi_iter.bi_sector, bio_sectors(bio),
539 				    true, duration, &io->stats_aux);
540 
541 	/*
542 	 * After this is decremented the bio must not be touched if it is
543 	 * a flush.
544 	 */
545 	pending = atomic_dec_return(&md->pending[rw]);
546 	atomic_set(&dm_disk(md)->part0.in_flight[rw], pending);
547 	pending += atomic_read(&md->pending[rw^0x1]);
548 
549 	/* nudge anyone waiting on suspend queue */
550 	if (!pending)
551 		wake_up(&md->wait);
552 }
553 
554 /*
555  * Add the bio to the list of deferred io.
556  */
557 static void queue_io(struct mapped_device *md, struct bio *bio)
558 {
559 	unsigned long flags;
560 
561 	spin_lock_irqsave(&md->deferred_lock, flags);
562 	bio_list_add(&md->deferred, bio);
563 	spin_unlock_irqrestore(&md->deferred_lock, flags);
564 	queue_work(md->wq, &md->work);
565 }
566 
567 /*
568  * Everyone (including functions in this file), should use this
569  * function to access the md->map field, and make sure they call
570  * dm_put_live_table() when finished.
571  */
572 struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier)
573 {
574 	*srcu_idx = srcu_read_lock(&md->io_barrier);
575 
576 	return srcu_dereference(md->map, &md->io_barrier);
577 }
578 
579 void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier)
580 {
581 	srcu_read_unlock(&md->io_barrier, srcu_idx);
582 }
583 
584 void dm_sync_table(struct mapped_device *md)
585 {
586 	synchronize_srcu(&md->io_barrier);
587 	synchronize_rcu_expedited();
588 }
589 
590 /*
591  * A fast alternative to dm_get_live_table/dm_put_live_table.
592  * The caller must not block between these two functions.
593  */
594 static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU)
595 {
596 	rcu_read_lock();
597 	return rcu_dereference(md->map);
598 }
599 
600 static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU)
601 {
602 	rcu_read_unlock();
603 }
604 
605 /*
606  * Open a table device so we can use it as a map destination.
607  */
608 static int open_table_device(struct table_device *td, dev_t dev,
609 			     struct mapped_device *md)
610 {
611 	static char *_claim_ptr = "I belong to device-mapper";
612 	struct block_device *bdev;
613 
614 	int r;
615 
616 	BUG_ON(td->dm_dev.bdev);
617 
618 	bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _claim_ptr);
619 	if (IS_ERR(bdev))
620 		return PTR_ERR(bdev);
621 
622 	r = bd_link_disk_holder(bdev, dm_disk(md));
623 	if (r) {
624 		blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL);
625 		return r;
626 	}
627 
628 	td->dm_dev.bdev = bdev;
629 	return 0;
630 }
631 
632 /*
633  * Close a table device that we've been using.
634  */
635 static void close_table_device(struct table_device *td, struct mapped_device *md)
636 {
637 	if (!td->dm_dev.bdev)
638 		return;
639 
640 	bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md));
641 	blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL);
642 	td->dm_dev.bdev = NULL;
643 }
644 
645 static struct table_device *find_table_device(struct list_head *l, dev_t dev,
646 					      fmode_t mode) {
647 	struct table_device *td;
648 
649 	list_for_each_entry(td, l, list)
650 		if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode)
651 			return td;
652 
653 	return NULL;
654 }
655 
656 int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode,
657 			struct dm_dev **result) {
658 	int r;
659 	struct table_device *td;
660 
661 	mutex_lock(&md->table_devices_lock);
662 	td = find_table_device(&md->table_devices, dev, mode);
663 	if (!td) {
664 		td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id);
665 		if (!td) {
666 			mutex_unlock(&md->table_devices_lock);
667 			return -ENOMEM;
668 		}
669 
670 		td->dm_dev.mode = mode;
671 		td->dm_dev.bdev = NULL;
672 
673 		if ((r = open_table_device(td, dev, md))) {
674 			mutex_unlock(&md->table_devices_lock);
675 			kfree(td);
676 			return r;
677 		}
678 
679 		format_dev_t(td->dm_dev.name, dev);
680 
681 		atomic_set(&td->count, 0);
682 		list_add(&td->list, &md->table_devices);
683 	}
684 	atomic_inc(&td->count);
685 	mutex_unlock(&md->table_devices_lock);
686 
687 	*result = &td->dm_dev;
688 	return 0;
689 }
690 EXPORT_SYMBOL_GPL(dm_get_table_device);
691 
692 void dm_put_table_device(struct mapped_device *md, struct dm_dev *d)
693 {
694 	struct table_device *td = container_of(d, struct table_device, dm_dev);
695 
696 	mutex_lock(&md->table_devices_lock);
697 	if (atomic_dec_and_test(&td->count)) {
698 		close_table_device(td, md);
699 		list_del(&td->list);
700 		kfree(td);
701 	}
702 	mutex_unlock(&md->table_devices_lock);
703 }
704 EXPORT_SYMBOL(dm_put_table_device);
705 
706 static void free_table_devices(struct list_head *devices)
707 {
708 	struct list_head *tmp, *next;
709 
710 	list_for_each_safe(tmp, next, devices) {
711 		struct table_device *td = list_entry(tmp, struct table_device, list);
712 
713 		DMWARN("dm_destroy: %s still exists with %d references",
714 		       td->dm_dev.name, atomic_read(&td->count));
715 		kfree(td);
716 	}
717 }
718 
719 /*
720  * Get the geometry associated with a dm device
721  */
722 int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
723 {
724 	*geo = md->geometry;
725 
726 	return 0;
727 }
728 
729 /*
730  * Set the geometry of a device.
731  */
732 int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
733 {
734 	sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
735 
736 	if (geo->start > sz) {
737 		DMWARN("Start sector is beyond the geometry limits.");
738 		return -EINVAL;
739 	}
740 
741 	md->geometry = *geo;
742 
743 	return 0;
744 }
745 
746 /*-----------------------------------------------------------------
747  * CRUD START:
748  *   A more elegant soln is in the works that uses the queue
749  *   merge fn, unfortunately there are a couple of changes to
750  *   the block layer that I want to make for this.  So in the
751  *   interests of getting something for people to use I give
752  *   you this clearly demarcated crap.
753  *---------------------------------------------------------------*/
754 
755 static int __noflush_suspending(struct mapped_device *md)
756 {
757 	return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
758 }
759 
760 /*
761  * Decrements the number of outstanding ios that a bio has been
762  * cloned into, completing the original io if necc.
763  */
764 static void dec_pending(struct dm_io *io, int error)
765 {
766 	unsigned long flags;
767 	int io_error;
768 	struct bio *bio;
769 	struct mapped_device *md = io->md;
770 
771 	/* Push-back supersedes any I/O errors */
772 	if (unlikely(error)) {
773 		spin_lock_irqsave(&io->endio_lock, flags);
774 		if (!(io->error > 0 && __noflush_suspending(md)))
775 			io->error = error;
776 		spin_unlock_irqrestore(&io->endio_lock, flags);
777 	}
778 
779 	if (atomic_dec_and_test(&io->io_count)) {
780 		if (io->error == DM_ENDIO_REQUEUE) {
781 			/*
782 			 * Target requested pushing back the I/O.
783 			 */
784 			spin_lock_irqsave(&md->deferred_lock, flags);
785 			if (__noflush_suspending(md))
786 				bio_list_add_head(&md->deferred, io->bio);
787 			else
788 				/* noflush suspend was interrupted. */
789 				io->error = -EIO;
790 			spin_unlock_irqrestore(&md->deferred_lock, flags);
791 		}
792 
793 		io_error = io->error;
794 		bio = io->bio;
795 		end_io_acct(io);
796 		free_io(md, io);
797 
798 		if (io_error == DM_ENDIO_REQUEUE)
799 			return;
800 
801 		if ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size) {
802 			/*
803 			 * Preflush done for flush with data, reissue
804 			 * without REQ_PREFLUSH.
805 			 */
806 			bio->bi_opf &= ~REQ_PREFLUSH;
807 			queue_io(md, bio);
808 		} else {
809 			/* done with normal IO or empty flush */
810 			trace_block_bio_complete(md->queue, bio, io_error);
811 			bio->bi_error = io_error;
812 			bio_endio(bio);
813 		}
814 	}
815 }
816 
817 void disable_write_same(struct mapped_device *md)
818 {
819 	struct queue_limits *limits = dm_get_queue_limits(md);
820 
821 	/* device doesn't really support WRITE SAME, disable it */
822 	limits->max_write_same_sectors = 0;
823 }
824 
825 static void clone_endio(struct bio *bio)
826 {
827 	int error = bio->bi_error;
828 	int r = error;
829 	struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
830 	struct dm_io *io = tio->io;
831 	struct mapped_device *md = tio->io->md;
832 	dm_endio_fn endio = tio->ti->type->end_io;
833 
834 	if (endio) {
835 		r = endio(tio->ti, bio, error);
836 		if (r < 0 || r == DM_ENDIO_REQUEUE)
837 			/*
838 			 * error and requeue request are handled
839 			 * in dec_pending().
840 			 */
841 			error = r;
842 		else if (r == DM_ENDIO_INCOMPLETE)
843 			/* The target will handle the io */
844 			return;
845 		else if (r) {
846 			DMWARN("unimplemented target endio return value: %d", r);
847 			BUG();
848 		}
849 	}
850 
851 	if (unlikely(r == -EREMOTEIO && (bio_op(bio) == REQ_OP_WRITE_SAME) &&
852 		     !bdev_get_queue(bio->bi_bdev)->limits.max_write_same_sectors))
853 		disable_write_same(md);
854 
855 	free_tio(tio);
856 	dec_pending(io, error);
857 }
858 
859 /*
860  * Return maximum size of I/O possible at the supplied sector up to the current
861  * target boundary.
862  */
863 static sector_t max_io_len_target_boundary(sector_t sector, struct dm_target *ti)
864 {
865 	sector_t target_offset = dm_target_offset(ti, sector);
866 
867 	return ti->len - target_offset;
868 }
869 
870 static sector_t max_io_len(sector_t sector, struct dm_target *ti)
871 {
872 	sector_t len = max_io_len_target_boundary(sector, ti);
873 	sector_t offset, max_len;
874 
875 	/*
876 	 * Does the target need to split even further?
877 	 */
878 	if (ti->max_io_len) {
879 		offset = dm_target_offset(ti, sector);
880 		if (unlikely(ti->max_io_len & (ti->max_io_len - 1)))
881 			max_len = sector_div(offset, ti->max_io_len);
882 		else
883 			max_len = offset & (ti->max_io_len - 1);
884 		max_len = ti->max_io_len - max_len;
885 
886 		if (len > max_len)
887 			len = max_len;
888 	}
889 
890 	return len;
891 }
892 
893 int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
894 {
895 	if (len > UINT_MAX) {
896 		DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)",
897 		      (unsigned long long)len, UINT_MAX);
898 		ti->error = "Maximum size of target IO is too large";
899 		return -EINVAL;
900 	}
901 
902 	ti->max_io_len = (uint32_t) len;
903 
904 	return 0;
905 }
906 EXPORT_SYMBOL_GPL(dm_set_target_max_io_len);
907 
908 static long dm_blk_direct_access(struct block_device *bdev, sector_t sector,
909 				 void **kaddr, pfn_t *pfn, long size)
910 {
911 	struct mapped_device *md = bdev->bd_disk->private_data;
912 	struct dm_table *map;
913 	struct dm_target *ti;
914 	int srcu_idx;
915 	long len, ret = -EIO;
916 
917 	map = dm_get_live_table(md, &srcu_idx);
918 	if (!map)
919 		goto out;
920 
921 	ti = dm_table_find_target(map, sector);
922 	if (!dm_target_is_valid(ti))
923 		goto out;
924 
925 	len = max_io_len(sector, ti) << SECTOR_SHIFT;
926 	size = min(len, size);
927 
928 	if (ti->type->direct_access)
929 		ret = ti->type->direct_access(ti, sector, kaddr, pfn, size);
930 out:
931 	dm_put_live_table(md, srcu_idx);
932 	return min(ret, size);
933 }
934 
935 /*
936  * A target may call dm_accept_partial_bio only from the map routine.  It is
937  * allowed for all bio types except REQ_PREFLUSH.
938  *
939  * dm_accept_partial_bio informs the dm that the target only wants to process
940  * additional n_sectors sectors of the bio and the rest of the data should be
941  * sent in a next bio.
942  *
943  * A diagram that explains the arithmetics:
944  * +--------------------+---------------+-------+
945  * |         1          |       2       |   3   |
946  * +--------------------+---------------+-------+
947  *
948  * <-------------- *tio->len_ptr --------------->
949  *                      <------- bi_size ------->
950  *                      <-- n_sectors -->
951  *
952  * Region 1 was already iterated over with bio_advance or similar function.
953  *	(it may be empty if the target doesn't use bio_advance)
954  * Region 2 is the remaining bio size that the target wants to process.
955  *	(it may be empty if region 1 is non-empty, although there is no reason
956  *	 to make it empty)
957  * The target requires that region 3 is to be sent in the next bio.
958  *
959  * If the target wants to receive multiple copies of the bio (via num_*bios, etc),
960  * the partially processed part (the sum of regions 1+2) must be the same for all
961  * copies of the bio.
962  */
963 void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors)
964 {
965 	struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
966 	unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT;
967 	BUG_ON(bio->bi_opf & REQ_PREFLUSH);
968 	BUG_ON(bi_size > *tio->len_ptr);
969 	BUG_ON(n_sectors > bi_size);
970 	*tio->len_ptr -= bi_size - n_sectors;
971 	bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT;
972 }
973 EXPORT_SYMBOL_GPL(dm_accept_partial_bio);
974 
975 static void __map_bio(struct dm_target_io *tio)
976 {
977 	int r;
978 	sector_t sector;
979 	struct bio *clone = &tio->clone;
980 	struct dm_target *ti = tio->ti;
981 
982 	clone->bi_end_io = clone_endio;
983 
984 	/*
985 	 * Map the clone.  If r == 0 we don't need to do
986 	 * anything, the target has assumed ownership of
987 	 * this io.
988 	 */
989 	atomic_inc(&tio->io->io_count);
990 	sector = clone->bi_iter.bi_sector;
991 	r = ti->type->map(ti, clone);
992 	if (r == DM_MAPIO_REMAPPED) {
993 		/* the bio has been remapped so dispatch it */
994 
995 		trace_block_bio_remap(bdev_get_queue(clone->bi_bdev), clone,
996 				      tio->io->bio->bi_bdev->bd_dev, sector);
997 
998 		generic_make_request(clone);
999 	} else if (r < 0 || r == DM_MAPIO_REQUEUE) {
1000 		/* error the io and bail out, or requeue it if needed */
1001 		dec_pending(tio->io, r);
1002 		free_tio(tio);
1003 	} else if (r != DM_MAPIO_SUBMITTED) {
1004 		DMWARN("unimplemented target map return value: %d", r);
1005 		BUG();
1006 	}
1007 }
1008 
1009 struct clone_info {
1010 	struct mapped_device *md;
1011 	struct dm_table *map;
1012 	struct bio *bio;
1013 	struct dm_io *io;
1014 	sector_t sector;
1015 	unsigned sector_count;
1016 };
1017 
1018 static void bio_setup_sector(struct bio *bio, sector_t sector, unsigned len)
1019 {
1020 	bio->bi_iter.bi_sector = sector;
1021 	bio->bi_iter.bi_size = to_bytes(len);
1022 }
1023 
1024 /*
1025  * Creates a bio that consists of range of complete bvecs.
1026  */
1027 static int clone_bio(struct dm_target_io *tio, struct bio *bio,
1028 		     sector_t sector, unsigned len)
1029 {
1030 	struct bio *clone = &tio->clone;
1031 
1032 	__bio_clone_fast(clone, bio);
1033 
1034 	if (bio_integrity(bio)) {
1035 		int r = bio_integrity_clone(clone, bio, GFP_NOIO);
1036 		if (r < 0)
1037 			return r;
1038 	}
1039 
1040 	bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector));
1041 	clone->bi_iter.bi_size = to_bytes(len);
1042 
1043 	if (bio_integrity(bio))
1044 		bio_integrity_trim(clone, 0, len);
1045 
1046 	return 0;
1047 }
1048 
1049 static struct dm_target_io *alloc_tio(struct clone_info *ci,
1050 				      struct dm_target *ti,
1051 				      unsigned target_bio_nr)
1052 {
1053 	struct dm_target_io *tio;
1054 	struct bio *clone;
1055 
1056 	clone = bio_alloc_bioset(GFP_NOIO, 0, ci->md->bs);
1057 	tio = container_of(clone, struct dm_target_io, clone);
1058 
1059 	tio->io = ci->io;
1060 	tio->ti = ti;
1061 	tio->target_bio_nr = target_bio_nr;
1062 
1063 	return tio;
1064 }
1065 
1066 static void __clone_and_map_simple_bio(struct clone_info *ci,
1067 				       struct dm_target *ti,
1068 				       unsigned target_bio_nr, unsigned *len)
1069 {
1070 	struct dm_target_io *tio = alloc_tio(ci, ti, target_bio_nr);
1071 	struct bio *clone = &tio->clone;
1072 
1073 	tio->len_ptr = len;
1074 
1075 	__bio_clone_fast(clone, ci->bio);
1076 	if (len)
1077 		bio_setup_sector(clone, ci->sector, *len);
1078 
1079 	__map_bio(tio);
1080 }
1081 
1082 static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti,
1083 				  unsigned num_bios, unsigned *len)
1084 {
1085 	unsigned target_bio_nr;
1086 
1087 	for (target_bio_nr = 0; target_bio_nr < num_bios; target_bio_nr++)
1088 		__clone_and_map_simple_bio(ci, ti, target_bio_nr, len);
1089 }
1090 
1091 static int __send_empty_flush(struct clone_info *ci)
1092 {
1093 	unsigned target_nr = 0;
1094 	struct dm_target *ti;
1095 
1096 	BUG_ON(bio_has_data(ci->bio));
1097 	while ((ti = dm_table_get_target(ci->map, target_nr++)))
1098 		__send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL);
1099 
1100 	return 0;
1101 }
1102 
1103 static int __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti,
1104 				     sector_t sector, unsigned *len)
1105 {
1106 	struct bio *bio = ci->bio;
1107 	struct dm_target_io *tio;
1108 	unsigned target_bio_nr;
1109 	unsigned num_target_bios = 1;
1110 	int r = 0;
1111 
1112 	/*
1113 	 * Does the target want to receive duplicate copies of the bio?
1114 	 */
1115 	if (bio_data_dir(bio) == WRITE && ti->num_write_bios)
1116 		num_target_bios = ti->num_write_bios(ti, bio);
1117 
1118 	for (target_bio_nr = 0; target_bio_nr < num_target_bios; target_bio_nr++) {
1119 		tio = alloc_tio(ci, ti, target_bio_nr);
1120 		tio->len_ptr = len;
1121 		r = clone_bio(tio, bio, sector, *len);
1122 		if (r < 0) {
1123 			free_tio(tio);
1124 			break;
1125 		}
1126 		__map_bio(tio);
1127 	}
1128 
1129 	return r;
1130 }
1131 
1132 typedef unsigned (*get_num_bios_fn)(struct dm_target *ti);
1133 
1134 static unsigned get_num_discard_bios(struct dm_target *ti)
1135 {
1136 	return ti->num_discard_bios;
1137 }
1138 
1139 static unsigned get_num_write_same_bios(struct dm_target *ti)
1140 {
1141 	return ti->num_write_same_bios;
1142 }
1143 
1144 typedef bool (*is_split_required_fn)(struct dm_target *ti);
1145 
1146 static bool is_split_required_for_discard(struct dm_target *ti)
1147 {
1148 	return ti->split_discard_bios;
1149 }
1150 
1151 static int __send_changing_extent_only(struct clone_info *ci,
1152 				       get_num_bios_fn get_num_bios,
1153 				       is_split_required_fn is_split_required)
1154 {
1155 	struct dm_target *ti;
1156 	unsigned len;
1157 	unsigned num_bios;
1158 
1159 	do {
1160 		ti = dm_table_find_target(ci->map, ci->sector);
1161 		if (!dm_target_is_valid(ti))
1162 			return -EIO;
1163 
1164 		/*
1165 		 * Even though the device advertised support for this type of
1166 		 * request, that does not mean every target supports it, and
1167 		 * reconfiguration might also have changed that since the
1168 		 * check was performed.
1169 		 */
1170 		num_bios = get_num_bios ? get_num_bios(ti) : 0;
1171 		if (!num_bios)
1172 			return -EOPNOTSUPP;
1173 
1174 		if (is_split_required && !is_split_required(ti))
1175 			len = min((sector_t)ci->sector_count, max_io_len_target_boundary(ci->sector, ti));
1176 		else
1177 			len = min((sector_t)ci->sector_count, max_io_len(ci->sector, ti));
1178 
1179 		__send_duplicate_bios(ci, ti, num_bios, &len);
1180 
1181 		ci->sector += len;
1182 	} while (ci->sector_count -= len);
1183 
1184 	return 0;
1185 }
1186 
1187 static int __send_discard(struct clone_info *ci)
1188 {
1189 	return __send_changing_extent_only(ci, get_num_discard_bios,
1190 					   is_split_required_for_discard);
1191 }
1192 
1193 static int __send_write_same(struct clone_info *ci)
1194 {
1195 	return __send_changing_extent_only(ci, get_num_write_same_bios, NULL);
1196 }
1197 
1198 /*
1199  * Select the correct strategy for processing a non-flush bio.
1200  */
1201 static int __split_and_process_non_flush(struct clone_info *ci)
1202 {
1203 	struct bio *bio = ci->bio;
1204 	struct dm_target *ti;
1205 	unsigned len;
1206 	int r;
1207 
1208 	if (unlikely(bio_op(bio) == REQ_OP_DISCARD))
1209 		return __send_discard(ci);
1210 	else if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME))
1211 		return __send_write_same(ci);
1212 
1213 	ti = dm_table_find_target(ci->map, ci->sector);
1214 	if (!dm_target_is_valid(ti))
1215 		return -EIO;
1216 
1217 	len = min_t(sector_t, max_io_len(ci->sector, ti), ci->sector_count);
1218 
1219 	r = __clone_and_map_data_bio(ci, ti, ci->sector, &len);
1220 	if (r < 0)
1221 		return r;
1222 
1223 	ci->sector += len;
1224 	ci->sector_count -= len;
1225 
1226 	return 0;
1227 }
1228 
1229 /*
1230  * Entry point to split a bio into clones and submit them to the targets.
1231  */
1232 static void __split_and_process_bio(struct mapped_device *md,
1233 				    struct dm_table *map, struct bio *bio)
1234 {
1235 	struct clone_info ci;
1236 	int error = 0;
1237 
1238 	if (unlikely(!map)) {
1239 		bio_io_error(bio);
1240 		return;
1241 	}
1242 
1243 	ci.map = map;
1244 	ci.md = md;
1245 	ci.io = alloc_io(md);
1246 	ci.io->error = 0;
1247 	atomic_set(&ci.io->io_count, 1);
1248 	ci.io->bio = bio;
1249 	ci.io->md = md;
1250 	spin_lock_init(&ci.io->endio_lock);
1251 	ci.sector = bio->bi_iter.bi_sector;
1252 
1253 	start_io_acct(ci.io);
1254 
1255 	if (bio->bi_opf & REQ_PREFLUSH) {
1256 		ci.bio = &ci.md->flush_bio;
1257 		ci.sector_count = 0;
1258 		error = __send_empty_flush(&ci);
1259 		/* dec_pending submits any data associated with flush */
1260 	} else {
1261 		ci.bio = bio;
1262 		ci.sector_count = bio_sectors(bio);
1263 		while (ci.sector_count && !error)
1264 			error = __split_and_process_non_flush(&ci);
1265 	}
1266 
1267 	/* drop the extra reference count */
1268 	dec_pending(ci.io, error);
1269 }
1270 /*-----------------------------------------------------------------
1271  * CRUD END
1272  *---------------------------------------------------------------*/
1273 
1274 /*
1275  * The request function that just remaps the bio built up by
1276  * dm_merge_bvec.
1277  */
1278 static blk_qc_t dm_make_request(struct request_queue *q, struct bio *bio)
1279 {
1280 	int rw = bio_data_dir(bio);
1281 	struct mapped_device *md = q->queuedata;
1282 	int srcu_idx;
1283 	struct dm_table *map;
1284 
1285 	map = dm_get_live_table(md, &srcu_idx);
1286 
1287 	generic_start_io_acct(rw, bio_sectors(bio), &dm_disk(md)->part0);
1288 
1289 	/* if we're suspended, we have to queue this io for later */
1290 	if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) {
1291 		dm_put_live_table(md, srcu_idx);
1292 
1293 		if (!(bio->bi_opf & REQ_RAHEAD))
1294 			queue_io(md, bio);
1295 		else
1296 			bio_io_error(bio);
1297 		return BLK_QC_T_NONE;
1298 	}
1299 
1300 	__split_and_process_bio(md, map, bio);
1301 	dm_put_live_table(md, srcu_idx);
1302 	return BLK_QC_T_NONE;
1303 }
1304 
1305 static int dm_any_congested(void *congested_data, int bdi_bits)
1306 {
1307 	int r = bdi_bits;
1308 	struct mapped_device *md = congested_data;
1309 	struct dm_table *map;
1310 
1311 	if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
1312 		if (dm_request_based(md)) {
1313 			/*
1314 			 * With request-based DM we only need to check the
1315 			 * top-level queue for congestion.
1316 			 */
1317 			r = md->queue->backing_dev_info.wb.state & bdi_bits;
1318 		} else {
1319 			map = dm_get_live_table_fast(md);
1320 			if (map)
1321 				r = dm_table_any_congested(map, bdi_bits);
1322 			dm_put_live_table_fast(md);
1323 		}
1324 	}
1325 
1326 	return r;
1327 }
1328 
1329 /*-----------------------------------------------------------------
1330  * An IDR is used to keep track of allocated minor numbers.
1331  *---------------------------------------------------------------*/
1332 static void free_minor(int minor)
1333 {
1334 	spin_lock(&_minor_lock);
1335 	idr_remove(&_minor_idr, minor);
1336 	spin_unlock(&_minor_lock);
1337 }
1338 
1339 /*
1340  * See if the device with a specific minor # is free.
1341  */
1342 static int specific_minor(int minor)
1343 {
1344 	int r;
1345 
1346 	if (minor >= (1 << MINORBITS))
1347 		return -EINVAL;
1348 
1349 	idr_preload(GFP_KERNEL);
1350 	spin_lock(&_minor_lock);
1351 
1352 	r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT);
1353 
1354 	spin_unlock(&_minor_lock);
1355 	idr_preload_end();
1356 	if (r < 0)
1357 		return r == -ENOSPC ? -EBUSY : r;
1358 	return 0;
1359 }
1360 
1361 static int next_free_minor(int *minor)
1362 {
1363 	int r;
1364 
1365 	idr_preload(GFP_KERNEL);
1366 	spin_lock(&_minor_lock);
1367 
1368 	r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT);
1369 
1370 	spin_unlock(&_minor_lock);
1371 	idr_preload_end();
1372 	if (r < 0)
1373 		return r;
1374 	*minor = r;
1375 	return 0;
1376 }
1377 
1378 static const struct block_device_operations dm_blk_dops;
1379 
1380 static void dm_wq_work(struct work_struct *work);
1381 
1382 void dm_init_md_queue(struct mapped_device *md)
1383 {
1384 	/*
1385 	 * Request-based dm devices cannot be stacked on top of bio-based dm
1386 	 * devices.  The type of this dm device may not have been decided yet.
1387 	 * The type is decided at the first table loading time.
1388 	 * To prevent problematic device stacking, clear the queue flag
1389 	 * for request stacking support until then.
1390 	 *
1391 	 * This queue is new, so no concurrency on the queue_flags.
1392 	 */
1393 	queue_flag_clear_unlocked(QUEUE_FLAG_STACKABLE, md->queue);
1394 
1395 	/*
1396 	 * Initialize data that will only be used by a non-blk-mq DM queue
1397 	 * - must do so here (in alloc_dev callchain) before queue is used
1398 	 */
1399 	md->queue->queuedata = md;
1400 	md->queue->backing_dev_info.congested_data = md;
1401 }
1402 
1403 void dm_init_normal_md_queue(struct mapped_device *md)
1404 {
1405 	md->use_blk_mq = false;
1406 	dm_init_md_queue(md);
1407 
1408 	/*
1409 	 * Initialize aspects of queue that aren't relevant for blk-mq
1410 	 */
1411 	md->queue->backing_dev_info.congested_fn = dm_any_congested;
1412 	blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
1413 }
1414 
1415 static void cleanup_mapped_device(struct mapped_device *md)
1416 {
1417 	if (md->wq)
1418 		destroy_workqueue(md->wq);
1419 	if (md->kworker_task)
1420 		kthread_stop(md->kworker_task);
1421 	mempool_destroy(md->io_pool);
1422 	mempool_destroy(md->rq_pool);
1423 	if (md->bs)
1424 		bioset_free(md->bs);
1425 
1426 	if (md->disk) {
1427 		spin_lock(&_minor_lock);
1428 		md->disk->private_data = NULL;
1429 		spin_unlock(&_minor_lock);
1430 		del_gendisk(md->disk);
1431 		put_disk(md->disk);
1432 	}
1433 
1434 	if (md->queue)
1435 		blk_cleanup_queue(md->queue);
1436 
1437 	cleanup_srcu_struct(&md->io_barrier);
1438 
1439 	if (md->bdev) {
1440 		bdput(md->bdev);
1441 		md->bdev = NULL;
1442 	}
1443 
1444 	dm_mq_cleanup_mapped_device(md);
1445 }
1446 
1447 /*
1448  * Allocate and initialise a blank device with a given minor.
1449  */
1450 static struct mapped_device *alloc_dev(int minor)
1451 {
1452 	int r, numa_node_id = dm_get_numa_node();
1453 	struct mapped_device *md;
1454 	void *old_md;
1455 
1456 	md = kzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id);
1457 	if (!md) {
1458 		DMWARN("unable to allocate device, out of memory.");
1459 		return NULL;
1460 	}
1461 
1462 	if (!try_module_get(THIS_MODULE))
1463 		goto bad_module_get;
1464 
1465 	/* get a minor number for the dev */
1466 	if (minor == DM_ANY_MINOR)
1467 		r = next_free_minor(&minor);
1468 	else
1469 		r = specific_minor(minor);
1470 	if (r < 0)
1471 		goto bad_minor;
1472 
1473 	r = init_srcu_struct(&md->io_barrier);
1474 	if (r < 0)
1475 		goto bad_io_barrier;
1476 
1477 	md->numa_node_id = numa_node_id;
1478 	md->use_blk_mq = dm_use_blk_mq_default();
1479 	md->init_tio_pdu = false;
1480 	md->type = DM_TYPE_NONE;
1481 	mutex_init(&md->suspend_lock);
1482 	mutex_init(&md->type_lock);
1483 	mutex_init(&md->table_devices_lock);
1484 	spin_lock_init(&md->deferred_lock);
1485 	atomic_set(&md->holders, 1);
1486 	atomic_set(&md->open_count, 0);
1487 	atomic_set(&md->event_nr, 0);
1488 	atomic_set(&md->uevent_seq, 0);
1489 	INIT_LIST_HEAD(&md->uevent_list);
1490 	INIT_LIST_HEAD(&md->table_devices);
1491 	spin_lock_init(&md->uevent_lock);
1492 
1493 	md->queue = blk_alloc_queue_node(GFP_KERNEL, numa_node_id);
1494 	if (!md->queue)
1495 		goto bad;
1496 
1497 	dm_init_md_queue(md);
1498 
1499 	md->disk = alloc_disk_node(1, numa_node_id);
1500 	if (!md->disk)
1501 		goto bad;
1502 
1503 	atomic_set(&md->pending[0], 0);
1504 	atomic_set(&md->pending[1], 0);
1505 	init_waitqueue_head(&md->wait);
1506 	INIT_WORK(&md->work, dm_wq_work);
1507 	init_waitqueue_head(&md->eventq);
1508 	init_completion(&md->kobj_holder.completion);
1509 	md->kworker_task = NULL;
1510 
1511 	md->disk->major = _major;
1512 	md->disk->first_minor = minor;
1513 	md->disk->fops = &dm_blk_dops;
1514 	md->disk->queue = md->queue;
1515 	md->disk->private_data = md;
1516 	sprintf(md->disk->disk_name, "dm-%d", minor);
1517 	add_disk(md->disk);
1518 	format_dev_t(md->name, MKDEV(_major, minor));
1519 
1520 	md->wq = alloc_workqueue("kdmflush", WQ_MEM_RECLAIM, 0);
1521 	if (!md->wq)
1522 		goto bad;
1523 
1524 	md->bdev = bdget_disk(md->disk, 0);
1525 	if (!md->bdev)
1526 		goto bad;
1527 
1528 	bio_init(&md->flush_bio);
1529 	md->flush_bio.bi_bdev = md->bdev;
1530 	bio_set_op_attrs(&md->flush_bio, REQ_OP_WRITE, WRITE_FLUSH);
1531 
1532 	dm_stats_init(&md->stats);
1533 
1534 	/* Populate the mapping, nobody knows we exist yet */
1535 	spin_lock(&_minor_lock);
1536 	old_md = idr_replace(&_minor_idr, md, minor);
1537 	spin_unlock(&_minor_lock);
1538 
1539 	BUG_ON(old_md != MINOR_ALLOCED);
1540 
1541 	return md;
1542 
1543 bad:
1544 	cleanup_mapped_device(md);
1545 bad_io_barrier:
1546 	free_minor(minor);
1547 bad_minor:
1548 	module_put(THIS_MODULE);
1549 bad_module_get:
1550 	kfree(md);
1551 	return NULL;
1552 }
1553 
1554 static void unlock_fs(struct mapped_device *md);
1555 
1556 static void free_dev(struct mapped_device *md)
1557 {
1558 	int minor = MINOR(disk_devt(md->disk));
1559 
1560 	unlock_fs(md);
1561 
1562 	cleanup_mapped_device(md);
1563 
1564 	free_table_devices(&md->table_devices);
1565 	dm_stats_cleanup(&md->stats);
1566 	free_minor(minor);
1567 
1568 	module_put(THIS_MODULE);
1569 	kfree(md);
1570 }
1571 
1572 static void __bind_mempools(struct mapped_device *md, struct dm_table *t)
1573 {
1574 	struct dm_md_mempools *p = dm_table_get_md_mempools(t);
1575 
1576 	if (md->bs) {
1577 		/* The md already has necessary mempools. */
1578 		if (dm_table_bio_based(t)) {
1579 			/*
1580 			 * Reload bioset because front_pad may have changed
1581 			 * because a different table was loaded.
1582 			 */
1583 			bioset_free(md->bs);
1584 			md->bs = p->bs;
1585 			p->bs = NULL;
1586 		}
1587 		/*
1588 		 * There's no need to reload with request-based dm
1589 		 * because the size of front_pad doesn't change.
1590 		 * Note for future: If you are to reload bioset,
1591 		 * prep-ed requests in the queue may refer
1592 		 * to bio from the old bioset, so you must walk
1593 		 * through the queue to unprep.
1594 		 */
1595 		goto out;
1596 	}
1597 
1598 	BUG_ON(!p || md->io_pool || md->rq_pool || md->bs);
1599 
1600 	md->io_pool = p->io_pool;
1601 	p->io_pool = NULL;
1602 	md->rq_pool = p->rq_pool;
1603 	p->rq_pool = NULL;
1604 	md->bs = p->bs;
1605 	p->bs = NULL;
1606 
1607 out:
1608 	/* mempool bind completed, no longer need any mempools in the table */
1609 	dm_table_free_md_mempools(t);
1610 }
1611 
1612 /*
1613  * Bind a table to the device.
1614  */
1615 static void event_callback(void *context)
1616 {
1617 	unsigned long flags;
1618 	LIST_HEAD(uevents);
1619 	struct mapped_device *md = (struct mapped_device *) context;
1620 
1621 	spin_lock_irqsave(&md->uevent_lock, flags);
1622 	list_splice_init(&md->uevent_list, &uevents);
1623 	spin_unlock_irqrestore(&md->uevent_lock, flags);
1624 
1625 	dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
1626 
1627 	atomic_inc(&md->event_nr);
1628 	wake_up(&md->eventq);
1629 }
1630 
1631 /*
1632  * Protected by md->suspend_lock obtained by dm_swap_table().
1633  */
1634 static void __set_size(struct mapped_device *md, sector_t size)
1635 {
1636 	set_capacity(md->disk, size);
1637 
1638 	i_size_write(md->bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
1639 }
1640 
1641 /*
1642  * Returns old map, which caller must destroy.
1643  */
1644 static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
1645 			       struct queue_limits *limits)
1646 {
1647 	struct dm_table *old_map;
1648 	struct request_queue *q = md->queue;
1649 	sector_t size;
1650 
1651 	lockdep_assert_held(&md->suspend_lock);
1652 
1653 	size = dm_table_get_size(t);
1654 
1655 	/*
1656 	 * Wipe any geometry if the size of the table changed.
1657 	 */
1658 	if (size != dm_get_size(md))
1659 		memset(&md->geometry, 0, sizeof(md->geometry));
1660 
1661 	__set_size(md, size);
1662 
1663 	dm_table_event_callback(t, event_callback, md);
1664 
1665 	/*
1666 	 * The queue hasn't been stopped yet, if the old table type wasn't
1667 	 * for request-based during suspension.  So stop it to prevent
1668 	 * I/O mapping before resume.
1669 	 * This must be done before setting the queue restrictions,
1670 	 * because request-based dm may be run just after the setting.
1671 	 */
1672 	if (dm_table_request_based(t)) {
1673 		dm_stop_queue(q);
1674 		/*
1675 		 * Leverage the fact that request-based DM targets are
1676 		 * immutable singletons and establish md->immutable_target
1677 		 * - used to optimize both dm_request_fn and dm_mq_queue_rq
1678 		 */
1679 		md->immutable_target = dm_table_get_immutable_target(t);
1680 	}
1681 
1682 	__bind_mempools(md, t);
1683 
1684 	old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
1685 	rcu_assign_pointer(md->map, (void *)t);
1686 	md->immutable_target_type = dm_table_get_immutable_target_type(t);
1687 
1688 	dm_table_set_restrictions(t, q, limits);
1689 	if (old_map)
1690 		dm_sync_table(md);
1691 
1692 	return old_map;
1693 }
1694 
1695 /*
1696  * Returns unbound table for the caller to free.
1697  */
1698 static struct dm_table *__unbind(struct mapped_device *md)
1699 {
1700 	struct dm_table *map = rcu_dereference_protected(md->map, 1);
1701 
1702 	if (!map)
1703 		return NULL;
1704 
1705 	dm_table_event_callback(map, NULL, NULL);
1706 	RCU_INIT_POINTER(md->map, NULL);
1707 	dm_sync_table(md);
1708 
1709 	return map;
1710 }
1711 
1712 /*
1713  * Constructor for a new device.
1714  */
1715 int dm_create(int minor, struct mapped_device **result)
1716 {
1717 	struct mapped_device *md;
1718 
1719 	md = alloc_dev(minor);
1720 	if (!md)
1721 		return -ENXIO;
1722 
1723 	dm_sysfs_init(md);
1724 
1725 	*result = md;
1726 	return 0;
1727 }
1728 
1729 /*
1730  * Functions to manage md->type.
1731  * All are required to hold md->type_lock.
1732  */
1733 void dm_lock_md_type(struct mapped_device *md)
1734 {
1735 	mutex_lock(&md->type_lock);
1736 }
1737 
1738 void dm_unlock_md_type(struct mapped_device *md)
1739 {
1740 	mutex_unlock(&md->type_lock);
1741 }
1742 
1743 void dm_set_md_type(struct mapped_device *md, unsigned type)
1744 {
1745 	BUG_ON(!mutex_is_locked(&md->type_lock));
1746 	md->type = type;
1747 }
1748 
1749 unsigned dm_get_md_type(struct mapped_device *md)
1750 {
1751 	return md->type;
1752 }
1753 
1754 struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
1755 {
1756 	return md->immutable_target_type;
1757 }
1758 
1759 /*
1760  * The queue_limits are only valid as long as you have a reference
1761  * count on 'md'.
1762  */
1763 struct queue_limits *dm_get_queue_limits(struct mapped_device *md)
1764 {
1765 	BUG_ON(!atomic_read(&md->holders));
1766 	return &md->queue->limits;
1767 }
1768 EXPORT_SYMBOL_GPL(dm_get_queue_limits);
1769 
1770 /*
1771  * Setup the DM device's queue based on md's type
1772  */
1773 int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
1774 {
1775 	int r;
1776 	unsigned type = dm_get_md_type(md);
1777 
1778 	switch (type) {
1779 	case DM_TYPE_REQUEST_BASED:
1780 		r = dm_old_init_request_queue(md);
1781 		if (r) {
1782 			DMERR("Cannot initialize queue for request-based mapped device");
1783 			return r;
1784 		}
1785 		break;
1786 	case DM_TYPE_MQ_REQUEST_BASED:
1787 		r = dm_mq_init_request_queue(md, t);
1788 		if (r) {
1789 			DMERR("Cannot initialize queue for request-based dm-mq mapped device");
1790 			return r;
1791 		}
1792 		break;
1793 	case DM_TYPE_BIO_BASED:
1794 	case DM_TYPE_DAX_BIO_BASED:
1795 		dm_init_normal_md_queue(md);
1796 		blk_queue_make_request(md->queue, dm_make_request);
1797 		/*
1798 		 * DM handles splitting bios as needed.  Free the bio_split bioset
1799 		 * since it won't be used (saves 1 process per bio-based DM device).
1800 		 */
1801 		bioset_free(md->queue->bio_split);
1802 		md->queue->bio_split = NULL;
1803 
1804 		if (type == DM_TYPE_DAX_BIO_BASED)
1805 			queue_flag_set_unlocked(QUEUE_FLAG_DAX, md->queue);
1806 		break;
1807 	}
1808 
1809 	return 0;
1810 }
1811 
1812 struct mapped_device *dm_get_md(dev_t dev)
1813 {
1814 	struct mapped_device *md;
1815 	unsigned minor = MINOR(dev);
1816 
1817 	if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
1818 		return NULL;
1819 
1820 	spin_lock(&_minor_lock);
1821 
1822 	md = idr_find(&_minor_idr, minor);
1823 	if (md) {
1824 		if ((md == MINOR_ALLOCED ||
1825 		     (MINOR(disk_devt(dm_disk(md))) != minor) ||
1826 		     dm_deleting_md(md) ||
1827 		     test_bit(DMF_FREEING, &md->flags))) {
1828 			md = NULL;
1829 			goto out;
1830 		}
1831 		dm_get(md);
1832 	}
1833 
1834 out:
1835 	spin_unlock(&_minor_lock);
1836 
1837 	return md;
1838 }
1839 EXPORT_SYMBOL_GPL(dm_get_md);
1840 
1841 void *dm_get_mdptr(struct mapped_device *md)
1842 {
1843 	return md->interface_ptr;
1844 }
1845 
1846 void dm_set_mdptr(struct mapped_device *md, void *ptr)
1847 {
1848 	md->interface_ptr = ptr;
1849 }
1850 
1851 void dm_get(struct mapped_device *md)
1852 {
1853 	atomic_inc(&md->holders);
1854 	BUG_ON(test_bit(DMF_FREEING, &md->flags));
1855 }
1856 
1857 int dm_hold(struct mapped_device *md)
1858 {
1859 	spin_lock(&_minor_lock);
1860 	if (test_bit(DMF_FREEING, &md->flags)) {
1861 		spin_unlock(&_minor_lock);
1862 		return -EBUSY;
1863 	}
1864 	dm_get(md);
1865 	spin_unlock(&_minor_lock);
1866 	return 0;
1867 }
1868 EXPORT_SYMBOL_GPL(dm_hold);
1869 
1870 const char *dm_device_name(struct mapped_device *md)
1871 {
1872 	return md->name;
1873 }
1874 EXPORT_SYMBOL_GPL(dm_device_name);
1875 
1876 static void __dm_destroy(struct mapped_device *md, bool wait)
1877 {
1878 	struct request_queue *q = dm_get_md_queue(md);
1879 	struct dm_table *map;
1880 	int srcu_idx;
1881 
1882 	might_sleep();
1883 
1884 	spin_lock(&_minor_lock);
1885 	idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));
1886 	set_bit(DMF_FREEING, &md->flags);
1887 	spin_unlock(&_minor_lock);
1888 
1889 	spin_lock_irq(q->queue_lock);
1890 	queue_flag_set(QUEUE_FLAG_DYING, q);
1891 	spin_unlock_irq(q->queue_lock);
1892 
1893 	if (dm_request_based(md) && md->kworker_task)
1894 		kthread_flush_worker(&md->kworker);
1895 
1896 	/*
1897 	 * Take suspend_lock so that presuspend and postsuspend methods
1898 	 * do not race with internal suspend.
1899 	 */
1900 	mutex_lock(&md->suspend_lock);
1901 	map = dm_get_live_table(md, &srcu_idx);
1902 	if (!dm_suspended_md(md)) {
1903 		dm_table_presuspend_targets(map);
1904 		dm_table_postsuspend_targets(map);
1905 	}
1906 	/* dm_put_live_table must be before msleep, otherwise deadlock is possible */
1907 	dm_put_live_table(md, srcu_idx);
1908 	mutex_unlock(&md->suspend_lock);
1909 
1910 	/*
1911 	 * Rare, but there may be I/O requests still going to complete,
1912 	 * for example.  Wait for all references to disappear.
1913 	 * No one should increment the reference count of the mapped_device,
1914 	 * after the mapped_device state becomes DMF_FREEING.
1915 	 */
1916 	if (wait)
1917 		while (atomic_read(&md->holders))
1918 			msleep(1);
1919 	else if (atomic_read(&md->holders))
1920 		DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)",
1921 		       dm_device_name(md), atomic_read(&md->holders));
1922 
1923 	dm_sysfs_exit(md);
1924 	dm_table_destroy(__unbind(md));
1925 	free_dev(md);
1926 }
1927 
1928 void dm_destroy(struct mapped_device *md)
1929 {
1930 	__dm_destroy(md, true);
1931 }
1932 
1933 void dm_destroy_immediate(struct mapped_device *md)
1934 {
1935 	__dm_destroy(md, false);
1936 }
1937 
1938 void dm_put(struct mapped_device *md)
1939 {
1940 	atomic_dec(&md->holders);
1941 }
1942 EXPORT_SYMBOL_GPL(dm_put);
1943 
1944 static int dm_wait_for_completion(struct mapped_device *md, long task_state)
1945 {
1946 	int r = 0;
1947 	DEFINE_WAIT(wait);
1948 
1949 	while (1) {
1950 		prepare_to_wait(&md->wait, &wait, task_state);
1951 
1952 		if (!md_in_flight(md))
1953 			break;
1954 
1955 		if (signal_pending_state(task_state, current)) {
1956 			r = -EINTR;
1957 			break;
1958 		}
1959 
1960 		io_schedule();
1961 	}
1962 	finish_wait(&md->wait, &wait);
1963 
1964 	return r;
1965 }
1966 
1967 /*
1968  * Process the deferred bios
1969  */
1970 static void dm_wq_work(struct work_struct *work)
1971 {
1972 	struct mapped_device *md = container_of(work, struct mapped_device,
1973 						work);
1974 	struct bio *c;
1975 	int srcu_idx;
1976 	struct dm_table *map;
1977 
1978 	map = dm_get_live_table(md, &srcu_idx);
1979 
1980 	while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
1981 		spin_lock_irq(&md->deferred_lock);
1982 		c = bio_list_pop(&md->deferred);
1983 		spin_unlock_irq(&md->deferred_lock);
1984 
1985 		if (!c)
1986 			break;
1987 
1988 		if (dm_request_based(md))
1989 			generic_make_request(c);
1990 		else
1991 			__split_and_process_bio(md, map, c);
1992 	}
1993 
1994 	dm_put_live_table(md, srcu_idx);
1995 }
1996 
1997 static void dm_queue_flush(struct mapped_device *md)
1998 {
1999 	clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2000 	smp_mb__after_atomic();
2001 	queue_work(md->wq, &md->work);
2002 }
2003 
2004 /*
2005  * Swap in a new table, returning the old one for the caller to destroy.
2006  */
2007 struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table)
2008 {
2009 	struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL);
2010 	struct queue_limits limits;
2011 	int r;
2012 
2013 	mutex_lock(&md->suspend_lock);
2014 
2015 	/* device must be suspended */
2016 	if (!dm_suspended_md(md))
2017 		goto out;
2018 
2019 	/*
2020 	 * If the new table has no data devices, retain the existing limits.
2021 	 * This helps multipath with queue_if_no_path if all paths disappear,
2022 	 * then new I/O is queued based on these limits, and then some paths
2023 	 * reappear.
2024 	 */
2025 	if (dm_table_has_no_data_devices(table)) {
2026 		live_map = dm_get_live_table_fast(md);
2027 		if (live_map)
2028 			limits = md->queue->limits;
2029 		dm_put_live_table_fast(md);
2030 	}
2031 
2032 	if (!live_map) {
2033 		r = dm_calculate_queue_limits(table, &limits);
2034 		if (r) {
2035 			map = ERR_PTR(r);
2036 			goto out;
2037 		}
2038 	}
2039 
2040 	map = __bind(md, table, &limits);
2041 
2042 out:
2043 	mutex_unlock(&md->suspend_lock);
2044 	return map;
2045 }
2046 
2047 /*
2048  * Functions to lock and unlock any filesystem running on the
2049  * device.
2050  */
2051 static int lock_fs(struct mapped_device *md)
2052 {
2053 	int r;
2054 
2055 	WARN_ON(md->frozen_sb);
2056 
2057 	md->frozen_sb = freeze_bdev(md->bdev);
2058 	if (IS_ERR(md->frozen_sb)) {
2059 		r = PTR_ERR(md->frozen_sb);
2060 		md->frozen_sb = NULL;
2061 		return r;
2062 	}
2063 
2064 	set_bit(DMF_FROZEN, &md->flags);
2065 
2066 	return 0;
2067 }
2068 
2069 static void unlock_fs(struct mapped_device *md)
2070 {
2071 	if (!test_bit(DMF_FROZEN, &md->flags))
2072 		return;
2073 
2074 	thaw_bdev(md->bdev, md->frozen_sb);
2075 	md->frozen_sb = NULL;
2076 	clear_bit(DMF_FROZEN, &md->flags);
2077 }
2078 
2079 /*
2080  * @suspend_flags: DM_SUSPEND_LOCKFS_FLAG and/or DM_SUSPEND_NOFLUSH_FLAG
2081  * @task_state: e.g. TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE
2082  * @dmf_suspended_flag: DMF_SUSPENDED or DMF_SUSPENDED_INTERNALLY
2083  *
2084  * If __dm_suspend returns 0, the device is completely quiescent
2085  * now. There is no request-processing activity. All new requests
2086  * are being added to md->deferred list.
2087  *
2088  * Caller must hold md->suspend_lock
2089  */
2090 static int __dm_suspend(struct mapped_device *md, struct dm_table *map,
2091 			unsigned suspend_flags, long task_state,
2092 			int dmf_suspended_flag)
2093 {
2094 	bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG;
2095 	bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG;
2096 	int r;
2097 
2098 	lockdep_assert_held(&md->suspend_lock);
2099 
2100 	/*
2101 	 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
2102 	 * This flag is cleared before dm_suspend returns.
2103 	 */
2104 	if (noflush)
2105 		set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2106 
2107 	/*
2108 	 * This gets reverted if there's an error later and the targets
2109 	 * provide the .presuspend_undo hook.
2110 	 */
2111 	dm_table_presuspend_targets(map);
2112 
2113 	/*
2114 	 * Flush I/O to the device.
2115 	 * Any I/O submitted after lock_fs() may not be flushed.
2116 	 * noflush takes precedence over do_lockfs.
2117 	 * (lock_fs() flushes I/Os and waits for them to complete.)
2118 	 */
2119 	if (!noflush && do_lockfs) {
2120 		r = lock_fs(md);
2121 		if (r) {
2122 			dm_table_presuspend_undo_targets(map);
2123 			return r;
2124 		}
2125 	}
2126 
2127 	/*
2128 	 * Here we must make sure that no processes are submitting requests
2129 	 * to target drivers i.e. no one may be executing
2130 	 * __split_and_process_bio. This is called from dm_request and
2131 	 * dm_wq_work.
2132 	 *
2133 	 * To get all processes out of __split_and_process_bio in dm_request,
2134 	 * we take the write lock. To prevent any process from reentering
2135 	 * __split_and_process_bio from dm_request and quiesce the thread
2136 	 * (dm_wq_work), we set BMF_BLOCK_IO_FOR_SUSPEND and call
2137 	 * flush_workqueue(md->wq).
2138 	 */
2139 	set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2140 	if (map)
2141 		synchronize_srcu(&md->io_barrier);
2142 
2143 	/*
2144 	 * Stop md->queue before flushing md->wq in case request-based
2145 	 * dm defers requests to md->wq from md->queue.
2146 	 */
2147 	if (dm_request_based(md)) {
2148 		dm_stop_queue(md->queue);
2149 		if (md->kworker_task)
2150 			kthread_flush_worker(&md->kworker);
2151 	}
2152 
2153 	flush_workqueue(md->wq);
2154 
2155 	/*
2156 	 * At this point no more requests are entering target request routines.
2157 	 * We call dm_wait_for_completion to wait for all existing requests
2158 	 * to finish.
2159 	 */
2160 	r = dm_wait_for_completion(md, task_state);
2161 	if (!r)
2162 		set_bit(dmf_suspended_flag, &md->flags);
2163 
2164 	if (noflush)
2165 		clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2166 	if (map)
2167 		synchronize_srcu(&md->io_barrier);
2168 
2169 	/* were we interrupted ? */
2170 	if (r < 0) {
2171 		dm_queue_flush(md);
2172 
2173 		if (dm_request_based(md))
2174 			dm_start_queue(md->queue);
2175 
2176 		unlock_fs(md);
2177 		dm_table_presuspend_undo_targets(map);
2178 		/* pushback list is already flushed, so skip flush */
2179 	}
2180 
2181 	return r;
2182 }
2183 
2184 /*
2185  * We need to be able to change a mapping table under a mounted
2186  * filesystem.  For example we might want to move some data in
2187  * the background.  Before the table can be swapped with
2188  * dm_bind_table, dm_suspend must be called to flush any in
2189  * flight bios and ensure that any further io gets deferred.
2190  */
2191 /*
2192  * Suspend mechanism in request-based dm.
2193  *
2194  * 1. Flush all I/Os by lock_fs() if needed.
2195  * 2. Stop dispatching any I/O by stopping the request_queue.
2196  * 3. Wait for all in-flight I/Os to be completed or requeued.
2197  *
2198  * To abort suspend, start the request_queue.
2199  */
2200 int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
2201 {
2202 	struct dm_table *map = NULL;
2203 	int r = 0;
2204 
2205 retry:
2206 	mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2207 
2208 	if (dm_suspended_md(md)) {
2209 		r = -EINVAL;
2210 		goto out_unlock;
2211 	}
2212 
2213 	if (dm_suspended_internally_md(md)) {
2214 		/* already internally suspended, wait for internal resume */
2215 		mutex_unlock(&md->suspend_lock);
2216 		r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2217 		if (r)
2218 			return r;
2219 		goto retry;
2220 	}
2221 
2222 	map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2223 
2224 	r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED);
2225 	if (r)
2226 		goto out_unlock;
2227 
2228 	dm_table_postsuspend_targets(map);
2229 
2230 out_unlock:
2231 	mutex_unlock(&md->suspend_lock);
2232 	return r;
2233 }
2234 
2235 static int __dm_resume(struct mapped_device *md, struct dm_table *map)
2236 {
2237 	if (map) {
2238 		int r = dm_table_resume_targets(map);
2239 		if (r)
2240 			return r;
2241 	}
2242 
2243 	dm_queue_flush(md);
2244 
2245 	/*
2246 	 * Flushing deferred I/Os must be done after targets are resumed
2247 	 * so that mapping of targets can work correctly.
2248 	 * Request-based dm is queueing the deferred I/Os in its request_queue.
2249 	 */
2250 	if (dm_request_based(md))
2251 		dm_start_queue(md->queue);
2252 
2253 	unlock_fs(md);
2254 
2255 	return 0;
2256 }
2257 
2258 int dm_resume(struct mapped_device *md)
2259 {
2260 	int r;
2261 	struct dm_table *map = NULL;
2262 
2263 retry:
2264 	r = -EINVAL;
2265 	mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2266 
2267 	if (!dm_suspended_md(md))
2268 		goto out;
2269 
2270 	if (dm_suspended_internally_md(md)) {
2271 		/* already internally suspended, wait for internal resume */
2272 		mutex_unlock(&md->suspend_lock);
2273 		r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2274 		if (r)
2275 			return r;
2276 		goto retry;
2277 	}
2278 
2279 	map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2280 	if (!map || !dm_table_get_size(map))
2281 		goto out;
2282 
2283 	r = __dm_resume(md, map);
2284 	if (r)
2285 		goto out;
2286 
2287 	clear_bit(DMF_SUSPENDED, &md->flags);
2288 out:
2289 	mutex_unlock(&md->suspend_lock);
2290 
2291 	return r;
2292 }
2293 
2294 /*
2295  * Internal suspend/resume works like userspace-driven suspend. It waits
2296  * until all bios finish and prevents issuing new bios to the target drivers.
2297  * It may be used only from the kernel.
2298  */
2299 
2300 static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags)
2301 {
2302 	struct dm_table *map = NULL;
2303 
2304 	if (md->internal_suspend_count++)
2305 		return; /* nested internal suspend */
2306 
2307 	if (dm_suspended_md(md)) {
2308 		set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2309 		return; /* nest suspend */
2310 	}
2311 
2312 	map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2313 
2314 	/*
2315 	 * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is
2316 	 * supported.  Properly supporting a TASK_INTERRUPTIBLE internal suspend
2317 	 * would require changing .presuspend to return an error -- avoid this
2318 	 * until there is a need for more elaborate variants of internal suspend.
2319 	 */
2320 	(void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE,
2321 			    DMF_SUSPENDED_INTERNALLY);
2322 
2323 	dm_table_postsuspend_targets(map);
2324 }
2325 
2326 static void __dm_internal_resume(struct mapped_device *md)
2327 {
2328 	BUG_ON(!md->internal_suspend_count);
2329 
2330 	if (--md->internal_suspend_count)
2331 		return; /* resume from nested internal suspend */
2332 
2333 	if (dm_suspended_md(md))
2334 		goto done; /* resume from nested suspend */
2335 
2336 	/*
2337 	 * NOTE: existing callers don't need to call dm_table_resume_targets
2338 	 * (which may fail -- so best to avoid it for now by passing NULL map)
2339 	 */
2340 	(void) __dm_resume(md, NULL);
2341 
2342 done:
2343 	clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2344 	smp_mb__after_atomic();
2345 	wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY);
2346 }
2347 
2348 void dm_internal_suspend_noflush(struct mapped_device *md)
2349 {
2350 	mutex_lock(&md->suspend_lock);
2351 	__dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG);
2352 	mutex_unlock(&md->suspend_lock);
2353 }
2354 EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush);
2355 
2356 void dm_internal_resume(struct mapped_device *md)
2357 {
2358 	mutex_lock(&md->suspend_lock);
2359 	__dm_internal_resume(md);
2360 	mutex_unlock(&md->suspend_lock);
2361 }
2362 EXPORT_SYMBOL_GPL(dm_internal_resume);
2363 
2364 /*
2365  * Fast variants of internal suspend/resume hold md->suspend_lock,
2366  * which prevents interaction with userspace-driven suspend.
2367  */
2368 
2369 void dm_internal_suspend_fast(struct mapped_device *md)
2370 {
2371 	mutex_lock(&md->suspend_lock);
2372 	if (dm_suspended_md(md) || dm_suspended_internally_md(md))
2373 		return;
2374 
2375 	set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2376 	synchronize_srcu(&md->io_barrier);
2377 	flush_workqueue(md->wq);
2378 	dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
2379 }
2380 EXPORT_SYMBOL_GPL(dm_internal_suspend_fast);
2381 
2382 void dm_internal_resume_fast(struct mapped_device *md)
2383 {
2384 	if (dm_suspended_md(md) || dm_suspended_internally_md(md))
2385 		goto done;
2386 
2387 	dm_queue_flush(md);
2388 
2389 done:
2390 	mutex_unlock(&md->suspend_lock);
2391 }
2392 EXPORT_SYMBOL_GPL(dm_internal_resume_fast);
2393 
2394 /*-----------------------------------------------------------------
2395  * Event notification.
2396  *---------------------------------------------------------------*/
2397 int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action,
2398 		       unsigned cookie)
2399 {
2400 	char udev_cookie[DM_COOKIE_LENGTH];
2401 	char *envp[] = { udev_cookie, NULL };
2402 
2403 	if (!cookie)
2404 		return kobject_uevent(&disk_to_dev(md->disk)->kobj, action);
2405 	else {
2406 		snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u",
2407 			 DM_COOKIE_ENV_VAR_NAME, cookie);
2408 		return kobject_uevent_env(&disk_to_dev(md->disk)->kobj,
2409 					  action, envp);
2410 	}
2411 }
2412 
2413 uint32_t dm_next_uevent_seq(struct mapped_device *md)
2414 {
2415 	return atomic_add_return(1, &md->uevent_seq);
2416 }
2417 
2418 uint32_t dm_get_event_nr(struct mapped_device *md)
2419 {
2420 	return atomic_read(&md->event_nr);
2421 }
2422 
2423 int dm_wait_event(struct mapped_device *md, int event_nr)
2424 {
2425 	return wait_event_interruptible(md->eventq,
2426 			(event_nr != atomic_read(&md->event_nr)));
2427 }
2428 
2429 void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
2430 {
2431 	unsigned long flags;
2432 
2433 	spin_lock_irqsave(&md->uevent_lock, flags);
2434 	list_add(elist, &md->uevent_list);
2435 	spin_unlock_irqrestore(&md->uevent_lock, flags);
2436 }
2437 
2438 /*
2439  * The gendisk is only valid as long as you have a reference
2440  * count on 'md'.
2441  */
2442 struct gendisk *dm_disk(struct mapped_device *md)
2443 {
2444 	return md->disk;
2445 }
2446 EXPORT_SYMBOL_GPL(dm_disk);
2447 
2448 struct kobject *dm_kobject(struct mapped_device *md)
2449 {
2450 	return &md->kobj_holder.kobj;
2451 }
2452 
2453 struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
2454 {
2455 	struct mapped_device *md;
2456 
2457 	md = container_of(kobj, struct mapped_device, kobj_holder.kobj);
2458 
2459 	if (test_bit(DMF_FREEING, &md->flags) ||
2460 	    dm_deleting_md(md))
2461 		return NULL;
2462 
2463 	dm_get(md);
2464 	return md;
2465 }
2466 
2467 int dm_suspended_md(struct mapped_device *md)
2468 {
2469 	return test_bit(DMF_SUSPENDED, &md->flags);
2470 }
2471 
2472 int dm_suspended_internally_md(struct mapped_device *md)
2473 {
2474 	return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2475 }
2476 
2477 int dm_test_deferred_remove_flag(struct mapped_device *md)
2478 {
2479 	return test_bit(DMF_DEFERRED_REMOVE, &md->flags);
2480 }
2481 
2482 int dm_suspended(struct dm_target *ti)
2483 {
2484 	return dm_suspended_md(dm_table_get_md(ti->table));
2485 }
2486 EXPORT_SYMBOL_GPL(dm_suspended);
2487 
2488 int dm_noflush_suspending(struct dm_target *ti)
2489 {
2490 	return __noflush_suspending(dm_table_get_md(ti->table));
2491 }
2492 EXPORT_SYMBOL_GPL(dm_noflush_suspending);
2493 
2494 struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, unsigned type,
2495 					    unsigned integrity, unsigned per_io_data_size)
2496 {
2497 	struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id);
2498 	struct kmem_cache *cachep = NULL;
2499 	unsigned int pool_size = 0;
2500 	unsigned int front_pad;
2501 
2502 	if (!pools)
2503 		return NULL;
2504 
2505 	switch (type) {
2506 	case DM_TYPE_BIO_BASED:
2507 	case DM_TYPE_DAX_BIO_BASED:
2508 		cachep = _io_cache;
2509 		pool_size = dm_get_reserved_bio_based_ios();
2510 		front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);
2511 		break;
2512 	case DM_TYPE_REQUEST_BASED:
2513 		cachep = _rq_tio_cache;
2514 		pool_size = dm_get_reserved_rq_based_ios();
2515 		pools->rq_pool = mempool_create_slab_pool(pool_size, _rq_cache);
2516 		if (!pools->rq_pool)
2517 			goto out;
2518 		/* fall through to setup remaining rq-based pools */
2519 	case DM_TYPE_MQ_REQUEST_BASED:
2520 		if (!pool_size)
2521 			pool_size = dm_get_reserved_rq_based_ios();
2522 		front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
2523 		/* per_io_data_size is used for blk-mq pdu at queue allocation */
2524 		break;
2525 	default:
2526 		BUG();
2527 	}
2528 
2529 	if (cachep) {
2530 		pools->io_pool = mempool_create_slab_pool(pool_size, cachep);
2531 		if (!pools->io_pool)
2532 			goto out;
2533 	}
2534 
2535 	pools->bs = bioset_create_nobvec(pool_size, front_pad);
2536 	if (!pools->bs)
2537 		goto out;
2538 
2539 	if (integrity && bioset_integrity_create(pools->bs, pool_size))
2540 		goto out;
2541 
2542 	return pools;
2543 
2544 out:
2545 	dm_free_md_mempools(pools);
2546 
2547 	return NULL;
2548 }
2549 
2550 void dm_free_md_mempools(struct dm_md_mempools *pools)
2551 {
2552 	if (!pools)
2553 		return;
2554 
2555 	mempool_destroy(pools->io_pool);
2556 	mempool_destroy(pools->rq_pool);
2557 
2558 	if (pools->bs)
2559 		bioset_free(pools->bs);
2560 
2561 	kfree(pools);
2562 }
2563 
2564 struct dm_pr {
2565 	u64	old_key;
2566 	u64	new_key;
2567 	u32	flags;
2568 	bool	fail_early;
2569 };
2570 
2571 static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn,
2572 		      void *data)
2573 {
2574 	struct mapped_device *md = bdev->bd_disk->private_data;
2575 	struct dm_table *table;
2576 	struct dm_target *ti;
2577 	int ret = -ENOTTY, srcu_idx;
2578 
2579 	table = dm_get_live_table(md, &srcu_idx);
2580 	if (!table || !dm_table_get_size(table))
2581 		goto out;
2582 
2583 	/* We only support devices that have a single target */
2584 	if (dm_table_get_num_targets(table) != 1)
2585 		goto out;
2586 	ti = dm_table_get_target(table, 0);
2587 
2588 	ret = -EINVAL;
2589 	if (!ti->type->iterate_devices)
2590 		goto out;
2591 
2592 	ret = ti->type->iterate_devices(ti, fn, data);
2593 out:
2594 	dm_put_live_table(md, srcu_idx);
2595 	return ret;
2596 }
2597 
2598 /*
2599  * For register / unregister we need to manually call out to every path.
2600  */
2601 static int __dm_pr_register(struct dm_target *ti, struct dm_dev *dev,
2602 			    sector_t start, sector_t len, void *data)
2603 {
2604 	struct dm_pr *pr = data;
2605 	const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
2606 
2607 	if (!ops || !ops->pr_register)
2608 		return -EOPNOTSUPP;
2609 	return ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags);
2610 }
2611 
2612 static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key,
2613 			  u32 flags)
2614 {
2615 	struct dm_pr pr = {
2616 		.old_key	= old_key,
2617 		.new_key	= new_key,
2618 		.flags		= flags,
2619 		.fail_early	= true,
2620 	};
2621 	int ret;
2622 
2623 	ret = dm_call_pr(bdev, __dm_pr_register, &pr);
2624 	if (ret && new_key) {
2625 		/* unregister all paths if we failed to register any path */
2626 		pr.old_key = new_key;
2627 		pr.new_key = 0;
2628 		pr.flags = 0;
2629 		pr.fail_early = false;
2630 		dm_call_pr(bdev, __dm_pr_register, &pr);
2631 	}
2632 
2633 	return ret;
2634 }
2635 
2636 static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type,
2637 			 u32 flags)
2638 {
2639 	struct mapped_device *md = bdev->bd_disk->private_data;
2640 	const struct pr_ops *ops;
2641 	fmode_t mode;
2642 	int r;
2643 
2644 	r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
2645 	if (r < 0)
2646 		return r;
2647 
2648 	ops = bdev->bd_disk->fops->pr_ops;
2649 	if (ops && ops->pr_reserve)
2650 		r = ops->pr_reserve(bdev, key, type, flags);
2651 	else
2652 		r = -EOPNOTSUPP;
2653 
2654 	bdput(bdev);
2655 	return r;
2656 }
2657 
2658 static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
2659 {
2660 	struct mapped_device *md = bdev->bd_disk->private_data;
2661 	const struct pr_ops *ops;
2662 	fmode_t mode;
2663 	int r;
2664 
2665 	r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
2666 	if (r < 0)
2667 		return r;
2668 
2669 	ops = bdev->bd_disk->fops->pr_ops;
2670 	if (ops && ops->pr_release)
2671 		r = ops->pr_release(bdev, key, type);
2672 	else
2673 		r = -EOPNOTSUPP;
2674 
2675 	bdput(bdev);
2676 	return r;
2677 }
2678 
2679 static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key,
2680 			 enum pr_type type, bool abort)
2681 {
2682 	struct mapped_device *md = bdev->bd_disk->private_data;
2683 	const struct pr_ops *ops;
2684 	fmode_t mode;
2685 	int r;
2686 
2687 	r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
2688 	if (r < 0)
2689 		return r;
2690 
2691 	ops = bdev->bd_disk->fops->pr_ops;
2692 	if (ops && ops->pr_preempt)
2693 		r = ops->pr_preempt(bdev, old_key, new_key, type, abort);
2694 	else
2695 		r = -EOPNOTSUPP;
2696 
2697 	bdput(bdev);
2698 	return r;
2699 }
2700 
2701 static int dm_pr_clear(struct block_device *bdev, u64 key)
2702 {
2703 	struct mapped_device *md = bdev->bd_disk->private_data;
2704 	const struct pr_ops *ops;
2705 	fmode_t mode;
2706 	int r;
2707 
2708 	r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
2709 	if (r < 0)
2710 		return r;
2711 
2712 	ops = bdev->bd_disk->fops->pr_ops;
2713 	if (ops && ops->pr_clear)
2714 		r = ops->pr_clear(bdev, key);
2715 	else
2716 		r = -EOPNOTSUPP;
2717 
2718 	bdput(bdev);
2719 	return r;
2720 }
2721 
2722 static const struct pr_ops dm_pr_ops = {
2723 	.pr_register	= dm_pr_register,
2724 	.pr_reserve	= dm_pr_reserve,
2725 	.pr_release	= dm_pr_release,
2726 	.pr_preempt	= dm_pr_preempt,
2727 	.pr_clear	= dm_pr_clear,
2728 };
2729 
2730 static const struct block_device_operations dm_blk_dops = {
2731 	.open = dm_blk_open,
2732 	.release = dm_blk_close,
2733 	.ioctl = dm_blk_ioctl,
2734 	.direct_access = dm_blk_direct_access,
2735 	.getgeo = dm_blk_getgeo,
2736 	.pr_ops = &dm_pr_ops,
2737 	.owner = THIS_MODULE
2738 };
2739 
2740 /*
2741  * module hooks
2742  */
2743 module_init(dm_init);
2744 module_exit(dm_exit);
2745 
2746 module_param(major, uint, 0);
2747 MODULE_PARM_DESC(major, "The major number of the device mapper");
2748 
2749 module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR);
2750 MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools");
2751 
2752 module_param(dm_numa_node, int, S_IRUGO | S_IWUSR);
2753 MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations");
2754 
2755 MODULE_DESCRIPTION(DM_NAME " driver");
2756 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
2757 MODULE_LICENSE("GPL");
2758