xref: /openbmc/linux/drivers/md/dm-table.c (revision 82df5b73)
1 /*
2  * Copyright (C) 2001 Sistina Software (UK) Limited.
3  * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
4  *
5  * This file is released under the GPL.
6  */
7 
8 #include "dm-core.h"
9 
10 #include <linux/module.h>
11 #include <linux/vmalloc.h>
12 #include <linux/blkdev.h>
13 #include <linux/namei.h>
14 #include <linux/ctype.h>
15 #include <linux/string.h>
16 #include <linux/slab.h>
17 #include <linux/interrupt.h>
18 #include <linux/mutex.h>
19 #include <linux/delay.h>
20 #include <linux/atomic.h>
21 #include <linux/blk-mq.h>
22 #include <linux/mount.h>
23 #include <linux/dax.h>
24 
25 #define DM_MSG_PREFIX "table"
26 
27 #define MAX_DEPTH 16
28 #define NODE_SIZE L1_CACHE_BYTES
29 #define KEYS_PER_NODE (NODE_SIZE / sizeof(sector_t))
30 #define CHILDREN_PER_NODE (KEYS_PER_NODE + 1)
31 
32 struct dm_table {
33 	struct mapped_device *md;
34 	enum dm_queue_mode type;
35 
36 	/* btree table */
37 	unsigned int depth;
38 	unsigned int counts[MAX_DEPTH];	/* in nodes */
39 	sector_t *index[MAX_DEPTH];
40 
41 	unsigned int num_targets;
42 	unsigned int num_allocated;
43 	sector_t *highs;
44 	struct dm_target *targets;
45 
46 	struct target_type *immutable_target_type;
47 
48 	bool integrity_supported:1;
49 	bool singleton:1;
50 	unsigned integrity_added:1;
51 
52 	/*
53 	 * Indicates the rw permissions for the new logical
54 	 * device.  This should be a combination of FMODE_READ
55 	 * and FMODE_WRITE.
56 	 */
57 	fmode_t mode;
58 
59 	/* a list of devices used by this table */
60 	struct list_head devices;
61 
62 	/* events get handed up using this callback */
63 	void (*event_fn)(void *);
64 	void *event_context;
65 
66 	struct dm_md_mempools *mempools;
67 
68 	struct list_head target_callbacks;
69 };
70 
71 /*
72  * Similar to ceiling(log_size(n))
73  */
74 static unsigned int int_log(unsigned int n, unsigned int base)
75 {
76 	int result = 0;
77 
78 	while (n > 1) {
79 		n = dm_div_up(n, base);
80 		result++;
81 	}
82 
83 	return result;
84 }
85 
86 /*
87  * Calculate the index of the child node of the n'th node k'th key.
88  */
89 static inline unsigned int get_child(unsigned int n, unsigned int k)
90 {
91 	return (n * CHILDREN_PER_NODE) + k;
92 }
93 
94 /*
95  * Return the n'th node of level l from table t.
96  */
97 static inline sector_t *get_node(struct dm_table *t,
98 				 unsigned int l, unsigned int n)
99 {
100 	return t->index[l] + (n * KEYS_PER_NODE);
101 }
102 
103 /*
104  * Return the highest key that you could lookup from the n'th
105  * node on level l of the btree.
106  */
107 static sector_t high(struct dm_table *t, unsigned int l, unsigned int n)
108 {
109 	for (; l < t->depth - 1; l++)
110 		n = get_child(n, CHILDREN_PER_NODE - 1);
111 
112 	if (n >= t->counts[l])
113 		return (sector_t) - 1;
114 
115 	return get_node(t, l, n)[KEYS_PER_NODE - 1];
116 }
117 
118 /*
119  * Fills in a level of the btree based on the highs of the level
120  * below it.
121  */
122 static int setup_btree_index(unsigned int l, struct dm_table *t)
123 {
124 	unsigned int n, k;
125 	sector_t *node;
126 
127 	for (n = 0U; n < t->counts[l]; n++) {
128 		node = get_node(t, l, n);
129 
130 		for (k = 0U; k < KEYS_PER_NODE; k++)
131 			node[k] = high(t, l + 1, get_child(n, k));
132 	}
133 
134 	return 0;
135 }
136 
137 void *dm_vcalloc(unsigned long nmemb, unsigned long elem_size)
138 {
139 	unsigned long size;
140 	void *addr;
141 
142 	/*
143 	 * Check that we're not going to overflow.
144 	 */
145 	if (nmemb > (ULONG_MAX / elem_size))
146 		return NULL;
147 
148 	size = nmemb * elem_size;
149 	addr = vzalloc(size);
150 
151 	return addr;
152 }
153 EXPORT_SYMBOL(dm_vcalloc);
154 
155 /*
156  * highs, and targets are managed as dynamic arrays during a
157  * table load.
158  */
159 static int alloc_targets(struct dm_table *t, unsigned int num)
160 {
161 	sector_t *n_highs;
162 	struct dm_target *n_targets;
163 
164 	/*
165 	 * Allocate both the target array and offset array at once.
166 	 */
167 	n_highs = (sector_t *) dm_vcalloc(num, sizeof(struct dm_target) +
168 					  sizeof(sector_t));
169 	if (!n_highs)
170 		return -ENOMEM;
171 
172 	n_targets = (struct dm_target *) (n_highs + num);
173 
174 	memset(n_highs, -1, sizeof(*n_highs) * num);
175 	vfree(t->highs);
176 
177 	t->num_allocated = num;
178 	t->highs = n_highs;
179 	t->targets = n_targets;
180 
181 	return 0;
182 }
183 
184 int dm_table_create(struct dm_table **result, fmode_t mode,
185 		    unsigned num_targets, struct mapped_device *md)
186 {
187 	struct dm_table *t = kzalloc(sizeof(*t), GFP_KERNEL);
188 
189 	if (!t)
190 		return -ENOMEM;
191 
192 	INIT_LIST_HEAD(&t->devices);
193 	INIT_LIST_HEAD(&t->target_callbacks);
194 
195 	if (!num_targets)
196 		num_targets = KEYS_PER_NODE;
197 
198 	num_targets = dm_round_up(num_targets, KEYS_PER_NODE);
199 
200 	if (!num_targets) {
201 		kfree(t);
202 		return -ENOMEM;
203 	}
204 
205 	if (alloc_targets(t, num_targets)) {
206 		kfree(t);
207 		return -ENOMEM;
208 	}
209 
210 	t->type = DM_TYPE_NONE;
211 	t->mode = mode;
212 	t->md = md;
213 	*result = t;
214 	return 0;
215 }
216 
217 static void free_devices(struct list_head *devices, struct mapped_device *md)
218 {
219 	struct list_head *tmp, *next;
220 
221 	list_for_each_safe(tmp, next, devices) {
222 		struct dm_dev_internal *dd =
223 		    list_entry(tmp, struct dm_dev_internal, list);
224 		DMWARN("%s: dm_table_destroy: dm_put_device call missing for %s",
225 		       dm_device_name(md), dd->dm_dev->name);
226 		dm_put_table_device(md, dd->dm_dev);
227 		kfree(dd);
228 	}
229 }
230 
231 void dm_table_destroy(struct dm_table *t)
232 {
233 	unsigned int i;
234 
235 	if (!t)
236 		return;
237 
238 	/* free the indexes */
239 	if (t->depth >= 2)
240 		vfree(t->index[t->depth - 2]);
241 
242 	/* free the targets */
243 	for (i = 0; i < t->num_targets; i++) {
244 		struct dm_target *tgt = t->targets + i;
245 
246 		if (tgt->type->dtr)
247 			tgt->type->dtr(tgt);
248 
249 		dm_put_target_type(tgt->type);
250 	}
251 
252 	vfree(t->highs);
253 
254 	/* free the device list */
255 	free_devices(&t->devices, t->md);
256 
257 	dm_free_md_mempools(t->mempools);
258 
259 	kfree(t);
260 }
261 
262 /*
263  * See if we've already got a device in the list.
264  */
265 static struct dm_dev_internal *find_device(struct list_head *l, dev_t dev)
266 {
267 	struct dm_dev_internal *dd;
268 
269 	list_for_each_entry (dd, l, list)
270 		if (dd->dm_dev->bdev->bd_dev == dev)
271 			return dd;
272 
273 	return NULL;
274 }
275 
276 /*
277  * If possible, this checks an area of a destination device is invalid.
278  */
279 static int device_area_is_invalid(struct dm_target *ti, struct dm_dev *dev,
280 				  sector_t start, sector_t len, void *data)
281 {
282 	struct queue_limits *limits = data;
283 	struct block_device *bdev = dev->bdev;
284 	sector_t dev_size =
285 		i_size_read(bdev->bd_inode) >> SECTOR_SHIFT;
286 	unsigned short logical_block_size_sectors =
287 		limits->logical_block_size >> SECTOR_SHIFT;
288 	char b[BDEVNAME_SIZE];
289 
290 	if (!dev_size)
291 		return 0;
292 
293 	if ((start >= dev_size) || (start + len > dev_size)) {
294 		DMWARN("%s: %s too small for target: "
295 		       "start=%llu, len=%llu, dev_size=%llu",
296 		       dm_device_name(ti->table->md), bdevname(bdev, b),
297 		       (unsigned long long)start,
298 		       (unsigned long long)len,
299 		       (unsigned long long)dev_size);
300 		return 1;
301 	}
302 
303 	/*
304 	 * If the target is mapped to zoned block device(s), check
305 	 * that the zones are not partially mapped.
306 	 */
307 	if (bdev_zoned_model(bdev) != BLK_ZONED_NONE) {
308 		unsigned int zone_sectors = bdev_zone_sectors(bdev);
309 
310 		if (start & (zone_sectors - 1)) {
311 			DMWARN("%s: start=%llu not aligned to h/w zone size %u of %s",
312 			       dm_device_name(ti->table->md),
313 			       (unsigned long long)start,
314 			       zone_sectors, bdevname(bdev, b));
315 			return 1;
316 		}
317 
318 		/*
319 		 * Note: The last zone of a zoned block device may be smaller
320 		 * than other zones. So for a target mapping the end of a
321 		 * zoned block device with such a zone, len would not be zone
322 		 * aligned. We do not allow such last smaller zone to be part
323 		 * of the mapping here to ensure that mappings with multiple
324 		 * devices do not end up with a smaller zone in the middle of
325 		 * the sector range.
326 		 */
327 		if (len & (zone_sectors - 1)) {
328 			DMWARN("%s: len=%llu not aligned to h/w zone size %u of %s",
329 			       dm_device_name(ti->table->md),
330 			       (unsigned long long)len,
331 			       zone_sectors, bdevname(bdev, b));
332 			return 1;
333 		}
334 	}
335 
336 	if (logical_block_size_sectors <= 1)
337 		return 0;
338 
339 	if (start & (logical_block_size_sectors - 1)) {
340 		DMWARN("%s: start=%llu not aligned to h/w "
341 		       "logical block size %u of %s",
342 		       dm_device_name(ti->table->md),
343 		       (unsigned long long)start,
344 		       limits->logical_block_size, bdevname(bdev, b));
345 		return 1;
346 	}
347 
348 	if (len & (logical_block_size_sectors - 1)) {
349 		DMWARN("%s: len=%llu not aligned to h/w "
350 		       "logical block size %u of %s",
351 		       dm_device_name(ti->table->md),
352 		       (unsigned long long)len,
353 		       limits->logical_block_size, bdevname(bdev, b));
354 		return 1;
355 	}
356 
357 	return 0;
358 }
359 
360 /*
361  * This upgrades the mode on an already open dm_dev, being
362  * careful to leave things as they were if we fail to reopen the
363  * device and not to touch the existing bdev field in case
364  * it is accessed concurrently inside dm_table_any_congested().
365  */
366 static int upgrade_mode(struct dm_dev_internal *dd, fmode_t new_mode,
367 			struct mapped_device *md)
368 {
369 	int r;
370 	struct dm_dev *old_dev, *new_dev;
371 
372 	old_dev = dd->dm_dev;
373 
374 	r = dm_get_table_device(md, dd->dm_dev->bdev->bd_dev,
375 				dd->dm_dev->mode | new_mode, &new_dev);
376 	if (r)
377 		return r;
378 
379 	dd->dm_dev = new_dev;
380 	dm_put_table_device(md, old_dev);
381 
382 	return 0;
383 }
384 
385 /*
386  * Convert the path to a device
387  */
388 dev_t dm_get_dev_t(const char *path)
389 {
390 	dev_t dev;
391 	struct block_device *bdev;
392 
393 	bdev = lookup_bdev(path);
394 	if (IS_ERR(bdev))
395 		dev = name_to_dev_t(path);
396 	else {
397 		dev = bdev->bd_dev;
398 		bdput(bdev);
399 	}
400 
401 	return dev;
402 }
403 EXPORT_SYMBOL_GPL(dm_get_dev_t);
404 
405 /*
406  * Add a device to the list, or just increment the usage count if
407  * it's already present.
408  */
409 int dm_get_device(struct dm_target *ti, const char *path, fmode_t mode,
410 		  struct dm_dev **result)
411 {
412 	int r;
413 	dev_t dev;
414 	struct dm_dev_internal *dd;
415 	struct dm_table *t = ti->table;
416 
417 	BUG_ON(!t);
418 
419 	dev = dm_get_dev_t(path);
420 	if (!dev)
421 		return -ENODEV;
422 
423 	dd = find_device(&t->devices, dev);
424 	if (!dd) {
425 		dd = kmalloc(sizeof(*dd), GFP_KERNEL);
426 		if (!dd)
427 			return -ENOMEM;
428 
429 		if ((r = dm_get_table_device(t->md, dev, mode, &dd->dm_dev))) {
430 			kfree(dd);
431 			return r;
432 		}
433 
434 		refcount_set(&dd->count, 1);
435 		list_add(&dd->list, &t->devices);
436 		goto out;
437 
438 	} else if (dd->dm_dev->mode != (mode | dd->dm_dev->mode)) {
439 		r = upgrade_mode(dd, mode, t->md);
440 		if (r)
441 			return r;
442 	}
443 	refcount_inc(&dd->count);
444 out:
445 	*result = dd->dm_dev;
446 	return 0;
447 }
448 EXPORT_SYMBOL(dm_get_device);
449 
450 static int dm_set_device_limits(struct dm_target *ti, struct dm_dev *dev,
451 				sector_t start, sector_t len, void *data)
452 {
453 	struct queue_limits *limits = data;
454 	struct block_device *bdev = dev->bdev;
455 	struct request_queue *q = bdev_get_queue(bdev);
456 	char b[BDEVNAME_SIZE];
457 
458 	if (unlikely(!q)) {
459 		DMWARN("%s: Cannot set limits for nonexistent device %s",
460 		       dm_device_name(ti->table->md), bdevname(bdev, b));
461 		return 0;
462 	}
463 
464 	if (bdev_stack_limits(limits, bdev, start) < 0)
465 		DMWARN("%s: adding target device %s caused an alignment inconsistency: "
466 		       "physical_block_size=%u, logical_block_size=%u, "
467 		       "alignment_offset=%u, start=%llu",
468 		       dm_device_name(ti->table->md), bdevname(bdev, b),
469 		       q->limits.physical_block_size,
470 		       q->limits.logical_block_size,
471 		       q->limits.alignment_offset,
472 		       (unsigned long long) start << SECTOR_SHIFT);
473 
474 	limits->zoned = blk_queue_zoned_model(q);
475 
476 	return 0;
477 }
478 
479 /*
480  * Decrement a device's use count and remove it if necessary.
481  */
482 void dm_put_device(struct dm_target *ti, struct dm_dev *d)
483 {
484 	int found = 0;
485 	struct list_head *devices = &ti->table->devices;
486 	struct dm_dev_internal *dd;
487 
488 	list_for_each_entry(dd, devices, list) {
489 		if (dd->dm_dev == d) {
490 			found = 1;
491 			break;
492 		}
493 	}
494 	if (!found) {
495 		DMWARN("%s: device %s not in table devices list",
496 		       dm_device_name(ti->table->md), d->name);
497 		return;
498 	}
499 	if (refcount_dec_and_test(&dd->count)) {
500 		dm_put_table_device(ti->table->md, d);
501 		list_del(&dd->list);
502 		kfree(dd);
503 	}
504 }
505 EXPORT_SYMBOL(dm_put_device);
506 
507 /*
508  * Checks to see if the target joins onto the end of the table.
509  */
510 static int adjoin(struct dm_table *table, struct dm_target *ti)
511 {
512 	struct dm_target *prev;
513 
514 	if (!table->num_targets)
515 		return !ti->begin;
516 
517 	prev = &table->targets[table->num_targets - 1];
518 	return (ti->begin == (prev->begin + prev->len));
519 }
520 
521 /*
522  * Used to dynamically allocate the arg array.
523  *
524  * We do first allocation with GFP_NOIO because dm-mpath and dm-thin must
525  * process messages even if some device is suspended. These messages have a
526  * small fixed number of arguments.
527  *
528  * On the other hand, dm-switch needs to process bulk data using messages and
529  * excessive use of GFP_NOIO could cause trouble.
530  */
531 static char **realloc_argv(unsigned *size, char **old_argv)
532 {
533 	char **argv;
534 	unsigned new_size;
535 	gfp_t gfp;
536 
537 	if (*size) {
538 		new_size = *size * 2;
539 		gfp = GFP_KERNEL;
540 	} else {
541 		new_size = 8;
542 		gfp = GFP_NOIO;
543 	}
544 	argv = kmalloc_array(new_size, sizeof(*argv), gfp);
545 	if (argv && old_argv) {
546 		memcpy(argv, old_argv, *size * sizeof(*argv));
547 		*size = new_size;
548 	}
549 
550 	kfree(old_argv);
551 	return argv;
552 }
553 
554 /*
555  * Destructively splits up the argument list to pass to ctr.
556  */
557 int dm_split_args(int *argc, char ***argvp, char *input)
558 {
559 	char *start, *end = input, *out, **argv = NULL;
560 	unsigned array_size = 0;
561 
562 	*argc = 0;
563 
564 	if (!input) {
565 		*argvp = NULL;
566 		return 0;
567 	}
568 
569 	argv = realloc_argv(&array_size, argv);
570 	if (!argv)
571 		return -ENOMEM;
572 
573 	while (1) {
574 		/* Skip whitespace */
575 		start = skip_spaces(end);
576 
577 		if (!*start)
578 			break;	/* success, we hit the end */
579 
580 		/* 'out' is used to remove any back-quotes */
581 		end = out = start;
582 		while (*end) {
583 			/* Everything apart from '\0' can be quoted */
584 			if (*end == '\\' && *(end + 1)) {
585 				*out++ = *(end + 1);
586 				end += 2;
587 				continue;
588 			}
589 
590 			if (isspace(*end))
591 				break;	/* end of token */
592 
593 			*out++ = *end++;
594 		}
595 
596 		/* have we already filled the array ? */
597 		if ((*argc + 1) > array_size) {
598 			argv = realloc_argv(&array_size, argv);
599 			if (!argv)
600 				return -ENOMEM;
601 		}
602 
603 		/* we know this is whitespace */
604 		if (*end)
605 			end++;
606 
607 		/* terminate the string and put it in the array */
608 		*out = '\0';
609 		argv[*argc] = start;
610 		(*argc)++;
611 	}
612 
613 	*argvp = argv;
614 	return 0;
615 }
616 
617 /*
618  * Impose necessary and sufficient conditions on a devices's table such
619  * that any incoming bio which respects its logical_block_size can be
620  * processed successfully.  If it falls across the boundary between
621  * two or more targets, the size of each piece it gets split into must
622  * be compatible with the logical_block_size of the target processing it.
623  */
624 static int validate_hardware_logical_block_alignment(struct dm_table *table,
625 						 struct queue_limits *limits)
626 {
627 	/*
628 	 * This function uses arithmetic modulo the logical_block_size
629 	 * (in units of 512-byte sectors).
630 	 */
631 	unsigned short device_logical_block_size_sects =
632 		limits->logical_block_size >> SECTOR_SHIFT;
633 
634 	/*
635 	 * Offset of the start of the next table entry, mod logical_block_size.
636 	 */
637 	unsigned short next_target_start = 0;
638 
639 	/*
640 	 * Given an aligned bio that extends beyond the end of a
641 	 * target, how many sectors must the next target handle?
642 	 */
643 	unsigned short remaining = 0;
644 
645 	struct dm_target *uninitialized_var(ti);
646 	struct queue_limits ti_limits;
647 	unsigned i;
648 
649 	/*
650 	 * Check each entry in the table in turn.
651 	 */
652 	for (i = 0; i < dm_table_get_num_targets(table); i++) {
653 		ti = dm_table_get_target(table, i);
654 
655 		blk_set_stacking_limits(&ti_limits);
656 
657 		/* combine all target devices' limits */
658 		if (ti->type->iterate_devices)
659 			ti->type->iterate_devices(ti, dm_set_device_limits,
660 						  &ti_limits);
661 
662 		/*
663 		 * If the remaining sectors fall entirely within this
664 		 * table entry are they compatible with its logical_block_size?
665 		 */
666 		if (remaining < ti->len &&
667 		    remaining & ((ti_limits.logical_block_size >>
668 				  SECTOR_SHIFT) - 1))
669 			break;	/* Error */
670 
671 		next_target_start =
672 		    (unsigned short) ((next_target_start + ti->len) &
673 				      (device_logical_block_size_sects - 1));
674 		remaining = next_target_start ?
675 		    device_logical_block_size_sects - next_target_start : 0;
676 	}
677 
678 	if (remaining) {
679 		DMWARN("%s: table line %u (start sect %llu len %llu) "
680 		       "not aligned to h/w logical block size %u",
681 		       dm_device_name(table->md), i,
682 		       (unsigned long long) ti->begin,
683 		       (unsigned long long) ti->len,
684 		       limits->logical_block_size);
685 		return -EINVAL;
686 	}
687 
688 	return 0;
689 }
690 
691 int dm_table_add_target(struct dm_table *t, const char *type,
692 			sector_t start, sector_t len, char *params)
693 {
694 	int r = -EINVAL, argc;
695 	char **argv;
696 	struct dm_target *tgt;
697 
698 	if (t->singleton) {
699 		DMERR("%s: target type %s must appear alone in table",
700 		      dm_device_name(t->md), t->targets->type->name);
701 		return -EINVAL;
702 	}
703 
704 	BUG_ON(t->num_targets >= t->num_allocated);
705 
706 	tgt = t->targets + t->num_targets;
707 	memset(tgt, 0, sizeof(*tgt));
708 
709 	if (!len) {
710 		DMERR("%s: zero-length target", dm_device_name(t->md));
711 		return -EINVAL;
712 	}
713 
714 	tgt->type = dm_get_target_type(type);
715 	if (!tgt->type) {
716 		DMERR("%s: %s: unknown target type", dm_device_name(t->md), type);
717 		return -EINVAL;
718 	}
719 
720 	if (dm_target_needs_singleton(tgt->type)) {
721 		if (t->num_targets) {
722 			tgt->error = "singleton target type must appear alone in table";
723 			goto bad;
724 		}
725 		t->singleton = true;
726 	}
727 
728 	if (dm_target_always_writeable(tgt->type) && !(t->mode & FMODE_WRITE)) {
729 		tgt->error = "target type may not be included in a read-only table";
730 		goto bad;
731 	}
732 
733 	if (t->immutable_target_type) {
734 		if (t->immutable_target_type != tgt->type) {
735 			tgt->error = "immutable target type cannot be mixed with other target types";
736 			goto bad;
737 		}
738 	} else if (dm_target_is_immutable(tgt->type)) {
739 		if (t->num_targets) {
740 			tgt->error = "immutable target type cannot be mixed with other target types";
741 			goto bad;
742 		}
743 		t->immutable_target_type = tgt->type;
744 	}
745 
746 	if (dm_target_has_integrity(tgt->type))
747 		t->integrity_added = 1;
748 
749 	tgt->table = t;
750 	tgt->begin = start;
751 	tgt->len = len;
752 	tgt->error = "Unknown error";
753 
754 	/*
755 	 * Does this target adjoin the previous one ?
756 	 */
757 	if (!adjoin(t, tgt)) {
758 		tgt->error = "Gap in table";
759 		goto bad;
760 	}
761 
762 	r = dm_split_args(&argc, &argv, params);
763 	if (r) {
764 		tgt->error = "couldn't split parameters (insufficient memory)";
765 		goto bad;
766 	}
767 
768 	r = tgt->type->ctr(tgt, argc, argv);
769 	kfree(argv);
770 	if (r)
771 		goto bad;
772 
773 	t->highs[t->num_targets++] = tgt->begin + tgt->len - 1;
774 
775 	if (!tgt->num_discard_bios && tgt->discards_supported)
776 		DMWARN("%s: %s: ignoring discards_supported because num_discard_bios is zero.",
777 		       dm_device_name(t->md), type);
778 
779 	return 0;
780 
781  bad:
782 	DMERR("%s: %s: %s", dm_device_name(t->md), type, tgt->error);
783 	dm_put_target_type(tgt->type);
784 	return r;
785 }
786 
787 /*
788  * Target argument parsing helpers.
789  */
790 static int validate_next_arg(const struct dm_arg *arg,
791 			     struct dm_arg_set *arg_set,
792 			     unsigned *value, char **error, unsigned grouped)
793 {
794 	const char *arg_str = dm_shift_arg(arg_set);
795 	char dummy;
796 
797 	if (!arg_str ||
798 	    (sscanf(arg_str, "%u%c", value, &dummy) != 1) ||
799 	    (*value < arg->min) ||
800 	    (*value > arg->max) ||
801 	    (grouped && arg_set->argc < *value)) {
802 		*error = arg->error;
803 		return -EINVAL;
804 	}
805 
806 	return 0;
807 }
808 
809 int dm_read_arg(const struct dm_arg *arg, struct dm_arg_set *arg_set,
810 		unsigned *value, char **error)
811 {
812 	return validate_next_arg(arg, arg_set, value, error, 0);
813 }
814 EXPORT_SYMBOL(dm_read_arg);
815 
816 int dm_read_arg_group(const struct dm_arg *arg, struct dm_arg_set *arg_set,
817 		      unsigned *value, char **error)
818 {
819 	return validate_next_arg(arg, arg_set, value, error, 1);
820 }
821 EXPORT_SYMBOL(dm_read_arg_group);
822 
823 const char *dm_shift_arg(struct dm_arg_set *as)
824 {
825 	char *r;
826 
827 	if (as->argc) {
828 		as->argc--;
829 		r = *as->argv;
830 		as->argv++;
831 		return r;
832 	}
833 
834 	return NULL;
835 }
836 EXPORT_SYMBOL(dm_shift_arg);
837 
838 void dm_consume_args(struct dm_arg_set *as, unsigned num_args)
839 {
840 	BUG_ON(as->argc < num_args);
841 	as->argc -= num_args;
842 	as->argv += num_args;
843 }
844 EXPORT_SYMBOL(dm_consume_args);
845 
846 static bool __table_type_bio_based(enum dm_queue_mode table_type)
847 {
848 	return (table_type == DM_TYPE_BIO_BASED ||
849 		table_type == DM_TYPE_DAX_BIO_BASED ||
850 		table_type == DM_TYPE_NVME_BIO_BASED);
851 }
852 
853 static bool __table_type_request_based(enum dm_queue_mode table_type)
854 {
855 	return table_type == DM_TYPE_REQUEST_BASED;
856 }
857 
858 void dm_table_set_type(struct dm_table *t, enum dm_queue_mode type)
859 {
860 	t->type = type;
861 }
862 EXPORT_SYMBOL_GPL(dm_table_set_type);
863 
864 /* validate the dax capability of the target device span */
865 int device_supports_dax(struct dm_target *ti, struct dm_dev *dev,
866 			sector_t start, sector_t len, void *data)
867 {
868 	int blocksize = *(int *) data;
869 
870 	return generic_fsdax_supported(dev->dax_dev, dev->bdev, blocksize,
871 				       start, len);
872 }
873 
874 /* Check devices support synchronous DAX */
875 static int device_dax_synchronous(struct dm_target *ti, struct dm_dev *dev,
876 				  sector_t start, sector_t len, void *data)
877 {
878 	return dev->dax_dev && dax_synchronous(dev->dax_dev);
879 }
880 
881 bool dm_table_supports_dax(struct dm_table *t,
882 			   iterate_devices_callout_fn iterate_fn, int *blocksize)
883 {
884 	struct dm_target *ti;
885 	unsigned i;
886 
887 	/* Ensure that all targets support DAX. */
888 	for (i = 0; i < dm_table_get_num_targets(t); i++) {
889 		ti = dm_table_get_target(t, i);
890 
891 		if (!ti->type->direct_access)
892 			return false;
893 
894 		if (!ti->type->iterate_devices ||
895 		    !ti->type->iterate_devices(ti, iterate_fn, blocksize))
896 			return false;
897 	}
898 
899 	return true;
900 }
901 
902 static bool dm_table_does_not_support_partial_completion(struct dm_table *t);
903 
904 static int device_is_rq_stackable(struct dm_target *ti, struct dm_dev *dev,
905 				  sector_t start, sector_t len, void *data)
906 {
907 	struct block_device *bdev = dev->bdev;
908 	struct request_queue *q = bdev_get_queue(bdev);
909 
910 	/* request-based cannot stack on partitions! */
911 	if (bdev != bdev->bd_contains)
912 		return false;
913 
914 	return queue_is_mq(q);
915 }
916 
917 static int dm_table_determine_type(struct dm_table *t)
918 {
919 	unsigned i;
920 	unsigned bio_based = 0, request_based = 0, hybrid = 0;
921 	struct dm_target *tgt;
922 	struct list_head *devices = dm_table_get_devices(t);
923 	enum dm_queue_mode live_md_type = dm_get_md_type(t->md);
924 	int page_size = PAGE_SIZE;
925 
926 	if (t->type != DM_TYPE_NONE) {
927 		/* target already set the table's type */
928 		if (t->type == DM_TYPE_BIO_BASED) {
929 			/* possibly upgrade to a variant of bio-based */
930 			goto verify_bio_based;
931 		}
932 		BUG_ON(t->type == DM_TYPE_DAX_BIO_BASED);
933 		BUG_ON(t->type == DM_TYPE_NVME_BIO_BASED);
934 		goto verify_rq_based;
935 	}
936 
937 	for (i = 0; i < t->num_targets; i++) {
938 		tgt = t->targets + i;
939 		if (dm_target_hybrid(tgt))
940 			hybrid = 1;
941 		else if (dm_target_request_based(tgt))
942 			request_based = 1;
943 		else
944 			bio_based = 1;
945 
946 		if (bio_based && request_based) {
947 			DMERR("Inconsistent table: different target types"
948 			      " can't be mixed up");
949 			return -EINVAL;
950 		}
951 	}
952 
953 	if (hybrid && !bio_based && !request_based) {
954 		/*
955 		 * The targets can work either way.
956 		 * Determine the type from the live device.
957 		 * Default to bio-based if device is new.
958 		 */
959 		if (__table_type_request_based(live_md_type))
960 			request_based = 1;
961 		else
962 			bio_based = 1;
963 	}
964 
965 	if (bio_based) {
966 verify_bio_based:
967 		/* We must use this table as bio-based */
968 		t->type = DM_TYPE_BIO_BASED;
969 		if (dm_table_supports_dax(t, device_supports_dax, &page_size) ||
970 		    (list_empty(devices) && live_md_type == DM_TYPE_DAX_BIO_BASED)) {
971 			t->type = DM_TYPE_DAX_BIO_BASED;
972 		} else {
973 			/* Check if upgrading to NVMe bio-based is valid or required */
974 			tgt = dm_table_get_immutable_target(t);
975 			if (tgt && !tgt->max_io_len && dm_table_does_not_support_partial_completion(t)) {
976 				t->type = DM_TYPE_NVME_BIO_BASED;
977 				goto verify_rq_based; /* must be stacked directly on NVMe (blk-mq) */
978 			} else if (list_empty(devices) && live_md_type == DM_TYPE_NVME_BIO_BASED) {
979 				t->type = DM_TYPE_NVME_BIO_BASED;
980 			}
981 		}
982 		return 0;
983 	}
984 
985 	BUG_ON(!request_based); /* No targets in this table */
986 
987 	t->type = DM_TYPE_REQUEST_BASED;
988 
989 verify_rq_based:
990 	/*
991 	 * Request-based dm supports only tables that have a single target now.
992 	 * To support multiple targets, request splitting support is needed,
993 	 * and that needs lots of changes in the block-layer.
994 	 * (e.g. request completion process for partial completion.)
995 	 */
996 	if (t->num_targets > 1) {
997 		DMERR("%s DM doesn't support multiple targets",
998 		      t->type == DM_TYPE_NVME_BIO_BASED ? "nvme bio-based" : "request-based");
999 		return -EINVAL;
1000 	}
1001 
1002 	if (list_empty(devices)) {
1003 		int srcu_idx;
1004 		struct dm_table *live_table = dm_get_live_table(t->md, &srcu_idx);
1005 
1006 		/* inherit live table's type */
1007 		if (live_table)
1008 			t->type = live_table->type;
1009 		dm_put_live_table(t->md, srcu_idx);
1010 		return 0;
1011 	}
1012 
1013 	tgt = dm_table_get_immutable_target(t);
1014 	if (!tgt) {
1015 		DMERR("table load rejected: immutable target is required");
1016 		return -EINVAL;
1017 	} else if (tgt->max_io_len) {
1018 		DMERR("table load rejected: immutable target that splits IO is not supported");
1019 		return -EINVAL;
1020 	}
1021 
1022 	/* Non-request-stackable devices can't be used for request-based dm */
1023 	if (!tgt->type->iterate_devices ||
1024 	    !tgt->type->iterate_devices(tgt, device_is_rq_stackable, NULL)) {
1025 		DMERR("table load rejected: including non-request-stackable devices");
1026 		return -EINVAL;
1027 	}
1028 
1029 	return 0;
1030 }
1031 
1032 enum dm_queue_mode dm_table_get_type(struct dm_table *t)
1033 {
1034 	return t->type;
1035 }
1036 
1037 struct target_type *dm_table_get_immutable_target_type(struct dm_table *t)
1038 {
1039 	return t->immutable_target_type;
1040 }
1041 
1042 struct dm_target *dm_table_get_immutable_target(struct dm_table *t)
1043 {
1044 	/* Immutable target is implicitly a singleton */
1045 	if (t->num_targets > 1 ||
1046 	    !dm_target_is_immutable(t->targets[0].type))
1047 		return NULL;
1048 
1049 	return t->targets;
1050 }
1051 
1052 struct dm_target *dm_table_get_wildcard_target(struct dm_table *t)
1053 {
1054 	struct dm_target *ti;
1055 	unsigned i;
1056 
1057 	for (i = 0; i < dm_table_get_num_targets(t); i++) {
1058 		ti = dm_table_get_target(t, i);
1059 		if (dm_target_is_wildcard(ti->type))
1060 			return ti;
1061 	}
1062 
1063 	return NULL;
1064 }
1065 
1066 bool dm_table_bio_based(struct dm_table *t)
1067 {
1068 	return __table_type_bio_based(dm_table_get_type(t));
1069 }
1070 
1071 bool dm_table_request_based(struct dm_table *t)
1072 {
1073 	return __table_type_request_based(dm_table_get_type(t));
1074 }
1075 
1076 static int dm_table_alloc_md_mempools(struct dm_table *t, struct mapped_device *md)
1077 {
1078 	enum dm_queue_mode type = dm_table_get_type(t);
1079 	unsigned per_io_data_size = 0;
1080 	unsigned min_pool_size = 0;
1081 	struct dm_target *ti;
1082 	unsigned i;
1083 
1084 	if (unlikely(type == DM_TYPE_NONE)) {
1085 		DMWARN("no table type is set, can't allocate mempools");
1086 		return -EINVAL;
1087 	}
1088 
1089 	if (__table_type_bio_based(type))
1090 		for (i = 0; i < t->num_targets; i++) {
1091 			ti = t->targets + i;
1092 			per_io_data_size = max(per_io_data_size, ti->per_io_data_size);
1093 			min_pool_size = max(min_pool_size, ti->num_flush_bios);
1094 		}
1095 
1096 	t->mempools = dm_alloc_md_mempools(md, type, t->integrity_supported,
1097 					   per_io_data_size, min_pool_size);
1098 	if (!t->mempools)
1099 		return -ENOMEM;
1100 
1101 	return 0;
1102 }
1103 
1104 void dm_table_free_md_mempools(struct dm_table *t)
1105 {
1106 	dm_free_md_mempools(t->mempools);
1107 	t->mempools = NULL;
1108 }
1109 
1110 struct dm_md_mempools *dm_table_get_md_mempools(struct dm_table *t)
1111 {
1112 	return t->mempools;
1113 }
1114 
1115 static int setup_indexes(struct dm_table *t)
1116 {
1117 	int i;
1118 	unsigned int total = 0;
1119 	sector_t *indexes;
1120 
1121 	/* allocate the space for *all* the indexes */
1122 	for (i = t->depth - 2; i >= 0; i--) {
1123 		t->counts[i] = dm_div_up(t->counts[i + 1], CHILDREN_PER_NODE);
1124 		total += t->counts[i];
1125 	}
1126 
1127 	indexes = (sector_t *) dm_vcalloc(total, (unsigned long) NODE_SIZE);
1128 	if (!indexes)
1129 		return -ENOMEM;
1130 
1131 	/* set up internal nodes, bottom-up */
1132 	for (i = t->depth - 2; i >= 0; i--) {
1133 		t->index[i] = indexes;
1134 		indexes += (KEYS_PER_NODE * t->counts[i]);
1135 		setup_btree_index(i, t);
1136 	}
1137 
1138 	return 0;
1139 }
1140 
1141 /*
1142  * Builds the btree to index the map.
1143  */
1144 static int dm_table_build_index(struct dm_table *t)
1145 {
1146 	int r = 0;
1147 	unsigned int leaf_nodes;
1148 
1149 	/* how many indexes will the btree have ? */
1150 	leaf_nodes = dm_div_up(t->num_targets, KEYS_PER_NODE);
1151 	t->depth = 1 + int_log(leaf_nodes, CHILDREN_PER_NODE);
1152 
1153 	/* leaf layer has already been set up */
1154 	t->counts[t->depth - 1] = leaf_nodes;
1155 	t->index[t->depth - 1] = t->highs;
1156 
1157 	if (t->depth >= 2)
1158 		r = setup_indexes(t);
1159 
1160 	return r;
1161 }
1162 
1163 static bool integrity_profile_exists(struct gendisk *disk)
1164 {
1165 	return !!blk_get_integrity(disk);
1166 }
1167 
1168 /*
1169  * Get a disk whose integrity profile reflects the table's profile.
1170  * Returns NULL if integrity support was inconsistent or unavailable.
1171  */
1172 static struct gendisk * dm_table_get_integrity_disk(struct dm_table *t)
1173 {
1174 	struct list_head *devices = dm_table_get_devices(t);
1175 	struct dm_dev_internal *dd = NULL;
1176 	struct gendisk *prev_disk = NULL, *template_disk = NULL;
1177 	unsigned i;
1178 
1179 	for (i = 0; i < dm_table_get_num_targets(t); i++) {
1180 		struct dm_target *ti = dm_table_get_target(t, i);
1181 		if (!dm_target_passes_integrity(ti->type))
1182 			goto no_integrity;
1183 	}
1184 
1185 	list_for_each_entry(dd, devices, list) {
1186 		template_disk = dd->dm_dev->bdev->bd_disk;
1187 		if (!integrity_profile_exists(template_disk))
1188 			goto no_integrity;
1189 		else if (prev_disk &&
1190 			 blk_integrity_compare(prev_disk, template_disk) < 0)
1191 			goto no_integrity;
1192 		prev_disk = template_disk;
1193 	}
1194 
1195 	return template_disk;
1196 
1197 no_integrity:
1198 	if (prev_disk)
1199 		DMWARN("%s: integrity not set: %s and %s profile mismatch",
1200 		       dm_device_name(t->md),
1201 		       prev_disk->disk_name,
1202 		       template_disk->disk_name);
1203 	return NULL;
1204 }
1205 
1206 /*
1207  * Register the mapped device for blk_integrity support if the
1208  * underlying devices have an integrity profile.  But all devices may
1209  * not have matching profiles (checking all devices isn't reliable
1210  * during table load because this table may use other DM device(s) which
1211  * must be resumed before they will have an initialized integity
1212  * profile).  Consequently, stacked DM devices force a 2 stage integrity
1213  * profile validation: First pass during table load, final pass during
1214  * resume.
1215  */
1216 static int dm_table_register_integrity(struct dm_table *t)
1217 {
1218 	struct mapped_device *md = t->md;
1219 	struct gendisk *template_disk = NULL;
1220 
1221 	/* If target handles integrity itself do not register it here. */
1222 	if (t->integrity_added)
1223 		return 0;
1224 
1225 	template_disk = dm_table_get_integrity_disk(t);
1226 	if (!template_disk)
1227 		return 0;
1228 
1229 	if (!integrity_profile_exists(dm_disk(md))) {
1230 		t->integrity_supported = true;
1231 		/*
1232 		 * Register integrity profile during table load; we can do
1233 		 * this because the final profile must match during resume.
1234 		 */
1235 		blk_integrity_register(dm_disk(md),
1236 				       blk_get_integrity(template_disk));
1237 		return 0;
1238 	}
1239 
1240 	/*
1241 	 * If DM device already has an initialized integrity
1242 	 * profile the new profile should not conflict.
1243 	 */
1244 	if (blk_integrity_compare(dm_disk(md), template_disk) < 0) {
1245 		DMWARN("%s: conflict with existing integrity profile: "
1246 		       "%s profile mismatch",
1247 		       dm_device_name(t->md),
1248 		       template_disk->disk_name);
1249 		return 1;
1250 	}
1251 
1252 	/* Preserve existing integrity profile */
1253 	t->integrity_supported = true;
1254 	return 0;
1255 }
1256 
1257 /*
1258  * Prepares the table for use by building the indices,
1259  * setting the type, and allocating mempools.
1260  */
1261 int dm_table_complete(struct dm_table *t)
1262 {
1263 	int r;
1264 
1265 	r = dm_table_determine_type(t);
1266 	if (r) {
1267 		DMERR("unable to determine table type");
1268 		return r;
1269 	}
1270 
1271 	r = dm_table_build_index(t);
1272 	if (r) {
1273 		DMERR("unable to build btrees");
1274 		return r;
1275 	}
1276 
1277 	r = dm_table_register_integrity(t);
1278 	if (r) {
1279 		DMERR("could not register integrity profile.");
1280 		return r;
1281 	}
1282 
1283 	r = dm_table_alloc_md_mempools(t, t->md);
1284 	if (r)
1285 		DMERR("unable to allocate mempools");
1286 
1287 	return r;
1288 }
1289 
1290 static DEFINE_MUTEX(_event_lock);
1291 void dm_table_event_callback(struct dm_table *t,
1292 			     void (*fn)(void *), void *context)
1293 {
1294 	mutex_lock(&_event_lock);
1295 	t->event_fn = fn;
1296 	t->event_context = context;
1297 	mutex_unlock(&_event_lock);
1298 }
1299 
1300 void dm_table_event(struct dm_table *t)
1301 {
1302 	/*
1303 	 * You can no longer call dm_table_event() from interrupt
1304 	 * context, use a bottom half instead.
1305 	 */
1306 	BUG_ON(in_interrupt());
1307 
1308 	mutex_lock(&_event_lock);
1309 	if (t->event_fn)
1310 		t->event_fn(t->event_context);
1311 	mutex_unlock(&_event_lock);
1312 }
1313 EXPORT_SYMBOL(dm_table_event);
1314 
1315 inline sector_t dm_table_get_size(struct dm_table *t)
1316 {
1317 	return t->num_targets ? (t->highs[t->num_targets - 1] + 1) : 0;
1318 }
1319 EXPORT_SYMBOL(dm_table_get_size);
1320 
1321 struct dm_target *dm_table_get_target(struct dm_table *t, unsigned int index)
1322 {
1323 	if (index >= t->num_targets)
1324 		return NULL;
1325 
1326 	return t->targets + index;
1327 }
1328 
1329 /*
1330  * Search the btree for the correct target.
1331  *
1332  * Caller should check returned pointer for NULL
1333  * to trap I/O beyond end of device.
1334  */
1335 struct dm_target *dm_table_find_target(struct dm_table *t, sector_t sector)
1336 {
1337 	unsigned int l, n = 0, k = 0;
1338 	sector_t *node;
1339 
1340 	if (unlikely(sector >= dm_table_get_size(t)))
1341 		return NULL;
1342 
1343 	for (l = 0; l < t->depth; l++) {
1344 		n = get_child(n, k);
1345 		node = get_node(t, l, n);
1346 
1347 		for (k = 0; k < KEYS_PER_NODE; k++)
1348 			if (node[k] >= sector)
1349 				break;
1350 	}
1351 
1352 	return &t->targets[(KEYS_PER_NODE * n) + k];
1353 }
1354 
1355 static int count_device(struct dm_target *ti, struct dm_dev *dev,
1356 			sector_t start, sector_t len, void *data)
1357 {
1358 	unsigned *num_devices = data;
1359 
1360 	(*num_devices)++;
1361 
1362 	return 0;
1363 }
1364 
1365 /*
1366  * Check whether a table has no data devices attached using each
1367  * target's iterate_devices method.
1368  * Returns false if the result is unknown because a target doesn't
1369  * support iterate_devices.
1370  */
1371 bool dm_table_has_no_data_devices(struct dm_table *table)
1372 {
1373 	struct dm_target *ti;
1374 	unsigned i, num_devices;
1375 
1376 	for (i = 0; i < dm_table_get_num_targets(table); i++) {
1377 		ti = dm_table_get_target(table, i);
1378 
1379 		if (!ti->type->iterate_devices)
1380 			return false;
1381 
1382 		num_devices = 0;
1383 		ti->type->iterate_devices(ti, count_device, &num_devices);
1384 		if (num_devices)
1385 			return false;
1386 	}
1387 
1388 	return true;
1389 }
1390 
1391 static int device_is_zoned_model(struct dm_target *ti, struct dm_dev *dev,
1392 				 sector_t start, sector_t len, void *data)
1393 {
1394 	struct request_queue *q = bdev_get_queue(dev->bdev);
1395 	enum blk_zoned_model *zoned_model = data;
1396 
1397 	return q && blk_queue_zoned_model(q) == *zoned_model;
1398 }
1399 
1400 static bool dm_table_supports_zoned_model(struct dm_table *t,
1401 					  enum blk_zoned_model zoned_model)
1402 {
1403 	struct dm_target *ti;
1404 	unsigned i;
1405 
1406 	for (i = 0; i < dm_table_get_num_targets(t); i++) {
1407 		ti = dm_table_get_target(t, i);
1408 
1409 		if (zoned_model == BLK_ZONED_HM &&
1410 		    !dm_target_supports_zoned_hm(ti->type))
1411 			return false;
1412 
1413 		if (!ti->type->iterate_devices ||
1414 		    !ti->type->iterate_devices(ti, device_is_zoned_model, &zoned_model))
1415 			return false;
1416 	}
1417 
1418 	return true;
1419 }
1420 
1421 static int device_matches_zone_sectors(struct dm_target *ti, struct dm_dev *dev,
1422 				       sector_t start, sector_t len, void *data)
1423 {
1424 	struct request_queue *q = bdev_get_queue(dev->bdev);
1425 	unsigned int *zone_sectors = data;
1426 
1427 	return q && blk_queue_zone_sectors(q) == *zone_sectors;
1428 }
1429 
1430 static bool dm_table_matches_zone_sectors(struct dm_table *t,
1431 					  unsigned int zone_sectors)
1432 {
1433 	struct dm_target *ti;
1434 	unsigned i;
1435 
1436 	for (i = 0; i < dm_table_get_num_targets(t); i++) {
1437 		ti = dm_table_get_target(t, i);
1438 
1439 		if (!ti->type->iterate_devices ||
1440 		    !ti->type->iterate_devices(ti, device_matches_zone_sectors, &zone_sectors))
1441 			return false;
1442 	}
1443 
1444 	return true;
1445 }
1446 
1447 static int validate_hardware_zoned_model(struct dm_table *table,
1448 					 enum blk_zoned_model zoned_model,
1449 					 unsigned int zone_sectors)
1450 {
1451 	if (zoned_model == BLK_ZONED_NONE)
1452 		return 0;
1453 
1454 	if (!dm_table_supports_zoned_model(table, zoned_model)) {
1455 		DMERR("%s: zoned model is not consistent across all devices",
1456 		      dm_device_name(table->md));
1457 		return -EINVAL;
1458 	}
1459 
1460 	/* Check zone size validity and compatibility */
1461 	if (!zone_sectors || !is_power_of_2(zone_sectors))
1462 		return -EINVAL;
1463 
1464 	if (!dm_table_matches_zone_sectors(table, zone_sectors)) {
1465 		DMERR("%s: zone sectors is not consistent across all devices",
1466 		      dm_device_name(table->md));
1467 		return -EINVAL;
1468 	}
1469 
1470 	return 0;
1471 }
1472 
1473 /*
1474  * Establish the new table's queue_limits and validate them.
1475  */
1476 int dm_calculate_queue_limits(struct dm_table *table,
1477 			      struct queue_limits *limits)
1478 {
1479 	struct dm_target *ti;
1480 	struct queue_limits ti_limits;
1481 	unsigned i;
1482 	enum blk_zoned_model zoned_model = BLK_ZONED_NONE;
1483 	unsigned int zone_sectors = 0;
1484 
1485 	blk_set_stacking_limits(limits);
1486 
1487 	for (i = 0; i < dm_table_get_num_targets(table); i++) {
1488 		blk_set_stacking_limits(&ti_limits);
1489 
1490 		ti = dm_table_get_target(table, i);
1491 
1492 		if (!ti->type->iterate_devices)
1493 			goto combine_limits;
1494 
1495 		/*
1496 		 * Combine queue limits of all the devices this target uses.
1497 		 */
1498 		ti->type->iterate_devices(ti, dm_set_device_limits,
1499 					  &ti_limits);
1500 
1501 		if (zoned_model == BLK_ZONED_NONE && ti_limits.zoned != BLK_ZONED_NONE) {
1502 			/*
1503 			 * After stacking all limits, validate all devices
1504 			 * in table support this zoned model and zone sectors.
1505 			 */
1506 			zoned_model = ti_limits.zoned;
1507 			zone_sectors = ti_limits.chunk_sectors;
1508 		}
1509 
1510 		/* Set I/O hints portion of queue limits */
1511 		if (ti->type->io_hints)
1512 			ti->type->io_hints(ti, &ti_limits);
1513 
1514 		/*
1515 		 * Check each device area is consistent with the target's
1516 		 * overall queue limits.
1517 		 */
1518 		if (ti->type->iterate_devices(ti, device_area_is_invalid,
1519 					      &ti_limits))
1520 			return -EINVAL;
1521 
1522 combine_limits:
1523 		/*
1524 		 * Merge this target's queue limits into the overall limits
1525 		 * for the table.
1526 		 */
1527 		if (blk_stack_limits(limits, &ti_limits, 0) < 0)
1528 			DMWARN("%s: adding target device "
1529 			       "(start sect %llu len %llu) "
1530 			       "caused an alignment inconsistency",
1531 			       dm_device_name(table->md),
1532 			       (unsigned long long) ti->begin,
1533 			       (unsigned long long) ti->len);
1534 
1535 		/*
1536 		 * FIXME: this should likely be moved to blk_stack_limits(), would
1537 		 * also eliminate limits->zoned stacking hack in dm_set_device_limits()
1538 		 */
1539 		if (limits->zoned == BLK_ZONED_NONE && ti_limits.zoned != BLK_ZONED_NONE) {
1540 			/*
1541 			 * By default, the stacked limits zoned model is set to
1542 			 * BLK_ZONED_NONE in blk_set_stacking_limits(). Update
1543 			 * this model using the first target model reported
1544 			 * that is not BLK_ZONED_NONE. This will be either the
1545 			 * first target device zoned model or the model reported
1546 			 * by the target .io_hints.
1547 			 */
1548 			limits->zoned = ti_limits.zoned;
1549 		}
1550 	}
1551 
1552 	/*
1553 	 * Verify that the zoned model and zone sectors, as determined before
1554 	 * any .io_hints override, are the same across all devices in the table.
1555 	 * - this is especially relevant if .io_hints is emulating a disk-managed
1556 	 *   zoned model (aka BLK_ZONED_NONE) on host-managed zoned block devices.
1557 	 * BUT...
1558 	 */
1559 	if (limits->zoned != BLK_ZONED_NONE) {
1560 		/*
1561 		 * ...IF the above limits stacking determined a zoned model
1562 		 * validate that all of the table's devices conform to it.
1563 		 */
1564 		zoned_model = limits->zoned;
1565 		zone_sectors = limits->chunk_sectors;
1566 	}
1567 	if (validate_hardware_zoned_model(table, zoned_model, zone_sectors))
1568 		return -EINVAL;
1569 
1570 	return validate_hardware_logical_block_alignment(table, limits);
1571 }
1572 
1573 /*
1574  * Verify that all devices have an integrity profile that matches the
1575  * DM device's registered integrity profile.  If the profiles don't
1576  * match then unregister the DM device's integrity profile.
1577  */
1578 static void dm_table_verify_integrity(struct dm_table *t)
1579 {
1580 	struct gendisk *template_disk = NULL;
1581 
1582 	if (t->integrity_added)
1583 		return;
1584 
1585 	if (t->integrity_supported) {
1586 		/*
1587 		 * Verify that the original integrity profile
1588 		 * matches all the devices in this table.
1589 		 */
1590 		template_disk = dm_table_get_integrity_disk(t);
1591 		if (template_disk &&
1592 		    blk_integrity_compare(dm_disk(t->md), template_disk) >= 0)
1593 			return;
1594 	}
1595 
1596 	if (integrity_profile_exists(dm_disk(t->md))) {
1597 		DMWARN("%s: unable to establish an integrity profile",
1598 		       dm_device_name(t->md));
1599 		blk_integrity_unregister(dm_disk(t->md));
1600 	}
1601 }
1602 
1603 static int device_flush_capable(struct dm_target *ti, struct dm_dev *dev,
1604 				sector_t start, sector_t len, void *data)
1605 {
1606 	unsigned long flush = (unsigned long) data;
1607 	struct request_queue *q = bdev_get_queue(dev->bdev);
1608 
1609 	return q && (q->queue_flags & flush);
1610 }
1611 
1612 static bool dm_table_supports_flush(struct dm_table *t, unsigned long flush)
1613 {
1614 	struct dm_target *ti;
1615 	unsigned i;
1616 
1617 	/*
1618 	 * Require at least one underlying device to support flushes.
1619 	 * t->devices includes internal dm devices such as mirror logs
1620 	 * so we need to use iterate_devices here, which targets
1621 	 * supporting flushes must provide.
1622 	 */
1623 	for (i = 0; i < dm_table_get_num_targets(t); i++) {
1624 		ti = dm_table_get_target(t, i);
1625 
1626 		if (!ti->num_flush_bios)
1627 			continue;
1628 
1629 		if (ti->flush_supported)
1630 			return true;
1631 
1632 		if (ti->type->iterate_devices &&
1633 		    ti->type->iterate_devices(ti, device_flush_capable, (void *) flush))
1634 			return true;
1635 	}
1636 
1637 	return false;
1638 }
1639 
1640 static int device_dax_write_cache_enabled(struct dm_target *ti,
1641 					  struct dm_dev *dev, sector_t start,
1642 					  sector_t len, void *data)
1643 {
1644 	struct dax_device *dax_dev = dev->dax_dev;
1645 
1646 	if (!dax_dev)
1647 		return false;
1648 
1649 	if (dax_write_cache_enabled(dax_dev))
1650 		return true;
1651 	return false;
1652 }
1653 
1654 static int dm_table_supports_dax_write_cache(struct dm_table *t)
1655 {
1656 	struct dm_target *ti;
1657 	unsigned i;
1658 
1659 	for (i = 0; i < dm_table_get_num_targets(t); i++) {
1660 		ti = dm_table_get_target(t, i);
1661 
1662 		if (ti->type->iterate_devices &&
1663 		    ti->type->iterate_devices(ti,
1664 				device_dax_write_cache_enabled, NULL))
1665 			return true;
1666 	}
1667 
1668 	return false;
1669 }
1670 
1671 static int device_is_nonrot(struct dm_target *ti, struct dm_dev *dev,
1672 			    sector_t start, sector_t len, void *data)
1673 {
1674 	struct request_queue *q = bdev_get_queue(dev->bdev);
1675 
1676 	return q && blk_queue_nonrot(q);
1677 }
1678 
1679 static int device_is_not_random(struct dm_target *ti, struct dm_dev *dev,
1680 			     sector_t start, sector_t len, void *data)
1681 {
1682 	struct request_queue *q = bdev_get_queue(dev->bdev);
1683 
1684 	return q && !blk_queue_add_random(q);
1685 }
1686 
1687 static bool dm_table_all_devices_attribute(struct dm_table *t,
1688 					   iterate_devices_callout_fn func)
1689 {
1690 	struct dm_target *ti;
1691 	unsigned i;
1692 
1693 	for (i = 0; i < dm_table_get_num_targets(t); i++) {
1694 		ti = dm_table_get_target(t, i);
1695 
1696 		if (!ti->type->iterate_devices ||
1697 		    !ti->type->iterate_devices(ti, func, NULL))
1698 			return false;
1699 	}
1700 
1701 	return true;
1702 }
1703 
1704 static int device_no_partial_completion(struct dm_target *ti, struct dm_dev *dev,
1705 					sector_t start, sector_t len, void *data)
1706 {
1707 	char b[BDEVNAME_SIZE];
1708 
1709 	/* For now, NVMe devices are the only devices of this class */
1710 	return (strncmp(bdevname(dev->bdev, b), "nvme", 4) == 0);
1711 }
1712 
1713 static bool dm_table_does_not_support_partial_completion(struct dm_table *t)
1714 {
1715 	return dm_table_all_devices_attribute(t, device_no_partial_completion);
1716 }
1717 
1718 static int device_not_write_same_capable(struct dm_target *ti, struct dm_dev *dev,
1719 					 sector_t start, sector_t len, void *data)
1720 {
1721 	struct request_queue *q = bdev_get_queue(dev->bdev);
1722 
1723 	return q && !q->limits.max_write_same_sectors;
1724 }
1725 
1726 static bool dm_table_supports_write_same(struct dm_table *t)
1727 {
1728 	struct dm_target *ti;
1729 	unsigned i;
1730 
1731 	for (i = 0; i < dm_table_get_num_targets(t); i++) {
1732 		ti = dm_table_get_target(t, i);
1733 
1734 		if (!ti->num_write_same_bios)
1735 			return false;
1736 
1737 		if (!ti->type->iterate_devices ||
1738 		    ti->type->iterate_devices(ti, device_not_write_same_capable, NULL))
1739 			return false;
1740 	}
1741 
1742 	return true;
1743 }
1744 
1745 static int device_not_write_zeroes_capable(struct dm_target *ti, struct dm_dev *dev,
1746 					   sector_t start, sector_t len, void *data)
1747 {
1748 	struct request_queue *q = bdev_get_queue(dev->bdev);
1749 
1750 	return q && !q->limits.max_write_zeroes_sectors;
1751 }
1752 
1753 static bool dm_table_supports_write_zeroes(struct dm_table *t)
1754 {
1755 	struct dm_target *ti;
1756 	unsigned i = 0;
1757 
1758 	while (i < dm_table_get_num_targets(t)) {
1759 		ti = dm_table_get_target(t, i++);
1760 
1761 		if (!ti->num_write_zeroes_bios)
1762 			return false;
1763 
1764 		if (!ti->type->iterate_devices ||
1765 		    ti->type->iterate_devices(ti, device_not_write_zeroes_capable, NULL))
1766 			return false;
1767 	}
1768 
1769 	return true;
1770 }
1771 
1772 static int device_not_discard_capable(struct dm_target *ti, struct dm_dev *dev,
1773 				      sector_t start, sector_t len, void *data)
1774 {
1775 	struct request_queue *q = bdev_get_queue(dev->bdev);
1776 
1777 	return q && !blk_queue_discard(q);
1778 }
1779 
1780 static bool dm_table_supports_discards(struct dm_table *t)
1781 {
1782 	struct dm_target *ti;
1783 	unsigned i;
1784 
1785 	for (i = 0; i < dm_table_get_num_targets(t); i++) {
1786 		ti = dm_table_get_target(t, i);
1787 
1788 		if (!ti->num_discard_bios)
1789 			return false;
1790 
1791 		/*
1792 		 * Either the target provides discard support (as implied by setting
1793 		 * 'discards_supported') or it relies on _all_ data devices having
1794 		 * discard support.
1795 		 */
1796 		if (!ti->discards_supported &&
1797 		    (!ti->type->iterate_devices ||
1798 		     ti->type->iterate_devices(ti, device_not_discard_capable, NULL)))
1799 			return false;
1800 	}
1801 
1802 	return true;
1803 }
1804 
1805 static int device_not_secure_erase_capable(struct dm_target *ti,
1806 					   struct dm_dev *dev, sector_t start,
1807 					   sector_t len, void *data)
1808 {
1809 	struct request_queue *q = bdev_get_queue(dev->bdev);
1810 
1811 	return q && !blk_queue_secure_erase(q);
1812 }
1813 
1814 static bool dm_table_supports_secure_erase(struct dm_table *t)
1815 {
1816 	struct dm_target *ti;
1817 	unsigned int i;
1818 
1819 	for (i = 0; i < dm_table_get_num_targets(t); i++) {
1820 		ti = dm_table_get_target(t, i);
1821 
1822 		if (!ti->num_secure_erase_bios)
1823 			return false;
1824 
1825 		if (!ti->type->iterate_devices ||
1826 		    ti->type->iterate_devices(ti, device_not_secure_erase_capable, NULL))
1827 			return false;
1828 	}
1829 
1830 	return true;
1831 }
1832 
1833 static int device_requires_stable_pages(struct dm_target *ti,
1834 					struct dm_dev *dev, sector_t start,
1835 					sector_t len, void *data)
1836 {
1837 	struct request_queue *q = bdev_get_queue(dev->bdev);
1838 
1839 	return q && bdi_cap_stable_pages_required(q->backing_dev_info);
1840 }
1841 
1842 /*
1843  * If any underlying device requires stable pages, a table must require
1844  * them as well.  Only targets that support iterate_devices are considered:
1845  * don't want error, zero, etc to require stable pages.
1846  */
1847 static bool dm_table_requires_stable_pages(struct dm_table *t)
1848 {
1849 	struct dm_target *ti;
1850 	unsigned i;
1851 
1852 	for (i = 0; i < dm_table_get_num_targets(t); i++) {
1853 		ti = dm_table_get_target(t, i);
1854 
1855 		if (ti->type->iterate_devices &&
1856 		    ti->type->iterate_devices(ti, device_requires_stable_pages, NULL))
1857 			return true;
1858 	}
1859 
1860 	return false;
1861 }
1862 
1863 void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q,
1864 			       struct queue_limits *limits)
1865 {
1866 	bool wc = false, fua = false;
1867 	int page_size = PAGE_SIZE;
1868 
1869 	/*
1870 	 * Copy table's limits to the DM device's request_queue
1871 	 */
1872 	q->limits = *limits;
1873 
1874 	if (!dm_table_supports_discards(t)) {
1875 		blk_queue_flag_clear(QUEUE_FLAG_DISCARD, q);
1876 		/* Must also clear discard limits... */
1877 		q->limits.max_discard_sectors = 0;
1878 		q->limits.max_hw_discard_sectors = 0;
1879 		q->limits.discard_granularity = 0;
1880 		q->limits.discard_alignment = 0;
1881 		q->limits.discard_misaligned = 0;
1882 	} else
1883 		blk_queue_flag_set(QUEUE_FLAG_DISCARD, q);
1884 
1885 	if (dm_table_supports_secure_erase(t))
1886 		blk_queue_flag_set(QUEUE_FLAG_SECERASE, q);
1887 
1888 	if (dm_table_supports_flush(t, (1UL << QUEUE_FLAG_WC))) {
1889 		wc = true;
1890 		if (dm_table_supports_flush(t, (1UL << QUEUE_FLAG_FUA)))
1891 			fua = true;
1892 	}
1893 	blk_queue_write_cache(q, wc, fua);
1894 
1895 	if (dm_table_supports_dax(t, device_supports_dax, &page_size)) {
1896 		blk_queue_flag_set(QUEUE_FLAG_DAX, q);
1897 		if (dm_table_supports_dax(t, device_dax_synchronous, NULL))
1898 			set_dax_synchronous(t->md->dax_dev);
1899 	}
1900 	else
1901 		blk_queue_flag_clear(QUEUE_FLAG_DAX, q);
1902 
1903 	if (dm_table_supports_dax_write_cache(t))
1904 		dax_write_cache(t->md->dax_dev, true);
1905 
1906 	/* Ensure that all underlying devices are non-rotational. */
1907 	if (dm_table_all_devices_attribute(t, device_is_nonrot))
1908 		blk_queue_flag_set(QUEUE_FLAG_NONROT, q);
1909 	else
1910 		blk_queue_flag_clear(QUEUE_FLAG_NONROT, q);
1911 
1912 	if (!dm_table_supports_write_same(t))
1913 		q->limits.max_write_same_sectors = 0;
1914 	if (!dm_table_supports_write_zeroes(t))
1915 		q->limits.max_write_zeroes_sectors = 0;
1916 
1917 	dm_table_verify_integrity(t);
1918 
1919 	/*
1920 	 * Some devices don't use blk_integrity but still want stable pages
1921 	 * because they do their own checksumming.
1922 	 */
1923 	if (dm_table_requires_stable_pages(t))
1924 		q->backing_dev_info->capabilities |= BDI_CAP_STABLE_WRITES;
1925 	else
1926 		q->backing_dev_info->capabilities &= ~BDI_CAP_STABLE_WRITES;
1927 
1928 	/*
1929 	 * Determine whether or not this queue's I/O timings contribute
1930 	 * to the entropy pool, Only request-based targets use this.
1931 	 * Clear QUEUE_FLAG_ADD_RANDOM if any underlying device does not
1932 	 * have it set.
1933 	 */
1934 	if (blk_queue_add_random(q) && dm_table_all_devices_attribute(t, device_is_not_random))
1935 		blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, q);
1936 
1937 	/*
1938 	 * For a zoned target, the number of zones should be updated for the
1939 	 * correct value to be exposed in sysfs queue/nr_zones. For a BIO based
1940 	 * target, this is all that is needed.
1941 	 */
1942 #ifdef CONFIG_BLK_DEV_ZONED
1943 	if (blk_queue_is_zoned(q)) {
1944 		WARN_ON_ONCE(queue_is_mq(q));
1945 		q->nr_zones = blkdev_nr_zones(t->md->disk);
1946 	}
1947 #endif
1948 
1949 	/* Allow reads to exceed readahead limits */
1950 	q->backing_dev_info->io_pages = limits->max_sectors >> (PAGE_SHIFT - 9);
1951 }
1952 
1953 unsigned int dm_table_get_num_targets(struct dm_table *t)
1954 {
1955 	return t->num_targets;
1956 }
1957 
1958 struct list_head *dm_table_get_devices(struct dm_table *t)
1959 {
1960 	return &t->devices;
1961 }
1962 
1963 fmode_t dm_table_get_mode(struct dm_table *t)
1964 {
1965 	return t->mode;
1966 }
1967 EXPORT_SYMBOL(dm_table_get_mode);
1968 
1969 enum suspend_mode {
1970 	PRESUSPEND,
1971 	PRESUSPEND_UNDO,
1972 	POSTSUSPEND,
1973 };
1974 
1975 static void suspend_targets(struct dm_table *t, enum suspend_mode mode)
1976 {
1977 	int i = t->num_targets;
1978 	struct dm_target *ti = t->targets;
1979 
1980 	lockdep_assert_held(&t->md->suspend_lock);
1981 
1982 	while (i--) {
1983 		switch (mode) {
1984 		case PRESUSPEND:
1985 			if (ti->type->presuspend)
1986 				ti->type->presuspend(ti);
1987 			break;
1988 		case PRESUSPEND_UNDO:
1989 			if (ti->type->presuspend_undo)
1990 				ti->type->presuspend_undo(ti);
1991 			break;
1992 		case POSTSUSPEND:
1993 			if (ti->type->postsuspend)
1994 				ti->type->postsuspend(ti);
1995 			break;
1996 		}
1997 		ti++;
1998 	}
1999 }
2000 
2001 void dm_table_presuspend_targets(struct dm_table *t)
2002 {
2003 	if (!t)
2004 		return;
2005 
2006 	suspend_targets(t, PRESUSPEND);
2007 }
2008 
2009 void dm_table_presuspend_undo_targets(struct dm_table *t)
2010 {
2011 	if (!t)
2012 		return;
2013 
2014 	suspend_targets(t, PRESUSPEND_UNDO);
2015 }
2016 
2017 void dm_table_postsuspend_targets(struct dm_table *t)
2018 {
2019 	if (!t)
2020 		return;
2021 
2022 	suspend_targets(t, POSTSUSPEND);
2023 }
2024 
2025 int dm_table_resume_targets(struct dm_table *t)
2026 {
2027 	int i, r = 0;
2028 
2029 	lockdep_assert_held(&t->md->suspend_lock);
2030 
2031 	for (i = 0; i < t->num_targets; i++) {
2032 		struct dm_target *ti = t->targets + i;
2033 
2034 		if (!ti->type->preresume)
2035 			continue;
2036 
2037 		r = ti->type->preresume(ti);
2038 		if (r) {
2039 			DMERR("%s: %s: preresume failed, error = %d",
2040 			      dm_device_name(t->md), ti->type->name, r);
2041 			return r;
2042 		}
2043 	}
2044 
2045 	for (i = 0; i < t->num_targets; i++) {
2046 		struct dm_target *ti = t->targets + i;
2047 
2048 		if (ti->type->resume)
2049 			ti->type->resume(ti);
2050 	}
2051 
2052 	return 0;
2053 }
2054 
2055 void dm_table_add_target_callbacks(struct dm_table *t, struct dm_target_callbacks *cb)
2056 {
2057 	list_add(&cb->list, &t->target_callbacks);
2058 }
2059 EXPORT_SYMBOL_GPL(dm_table_add_target_callbacks);
2060 
2061 int dm_table_any_congested(struct dm_table *t, int bdi_bits)
2062 {
2063 	struct dm_dev_internal *dd;
2064 	struct list_head *devices = dm_table_get_devices(t);
2065 	struct dm_target_callbacks *cb;
2066 	int r = 0;
2067 
2068 	list_for_each_entry(dd, devices, list) {
2069 		struct request_queue *q = bdev_get_queue(dd->dm_dev->bdev);
2070 		char b[BDEVNAME_SIZE];
2071 
2072 		if (likely(q))
2073 			r |= bdi_congested(q->backing_dev_info, bdi_bits);
2074 		else
2075 			DMWARN_LIMIT("%s: any_congested: nonexistent device %s",
2076 				     dm_device_name(t->md),
2077 				     bdevname(dd->dm_dev->bdev, b));
2078 	}
2079 
2080 	list_for_each_entry(cb, &t->target_callbacks, list)
2081 		if (cb->congested_fn)
2082 			r |= cb->congested_fn(cb, bdi_bits);
2083 
2084 	return r;
2085 }
2086 
2087 struct mapped_device *dm_table_get_md(struct dm_table *t)
2088 {
2089 	return t->md;
2090 }
2091 EXPORT_SYMBOL(dm_table_get_md);
2092 
2093 const char *dm_table_device_name(struct dm_table *t)
2094 {
2095 	return dm_device_name(t->md);
2096 }
2097 EXPORT_SYMBOL_GPL(dm_table_device_name);
2098 
2099 void dm_table_run_md_queue_async(struct dm_table *t)
2100 {
2101 	struct mapped_device *md;
2102 	struct request_queue *queue;
2103 
2104 	if (!dm_table_request_based(t))
2105 		return;
2106 
2107 	md = dm_table_get_md(t);
2108 	queue = dm_get_md_queue(md);
2109 	if (queue)
2110 		blk_mq_run_hw_queues(queue, true);
2111 }
2112 EXPORT_SYMBOL(dm_table_run_md_queue_async);
2113 
2114