xref: /openbmc/linux/fs/btrfs/volumes.c (revision 81113b04)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2007 Oracle.  All rights reserved.
4  */
5 
6 #include <linux/sched.h>
7 #include <linux/sched/mm.h>
8 #include <linux/bio.h>
9 #include <linux/slab.h>
10 #include <linux/blkdev.h>
11 #include <linux/ratelimit.h>
12 #include <linux/kthread.h>
13 #include <linux/raid/pq.h>
14 #include <linux/semaphore.h>
15 #include <linux/uuid.h>
16 #include <linux/list_sort.h>
17 #include "misc.h"
18 #include "ctree.h"
19 #include "extent_map.h"
20 #include "disk-io.h"
21 #include "transaction.h"
22 #include "print-tree.h"
23 #include "volumes.h"
24 #include "raid56.h"
25 #include "async-thread.h"
26 #include "check-integrity.h"
27 #include "rcu-string.h"
28 #include "dev-replace.h"
29 #include "sysfs.h"
30 #include "tree-checker.h"
31 #include "space-info.h"
32 #include "block-group.h"
33 #include "discard.h"
34 
35 const struct btrfs_raid_attr btrfs_raid_array[BTRFS_NR_RAID_TYPES] = {
36 	[BTRFS_RAID_RAID10] = {
37 		.sub_stripes	= 2,
38 		.dev_stripes	= 1,
39 		.devs_max	= 0,	/* 0 == as many as possible */
40 		.devs_min	= 4,
41 		.tolerated_failures = 1,
42 		.devs_increment	= 2,
43 		.ncopies	= 2,
44 		.nparity        = 0,
45 		.raid_name	= "raid10",
46 		.bg_flag	= BTRFS_BLOCK_GROUP_RAID10,
47 		.mindev_error	= BTRFS_ERROR_DEV_RAID10_MIN_NOT_MET,
48 	},
49 	[BTRFS_RAID_RAID1] = {
50 		.sub_stripes	= 1,
51 		.dev_stripes	= 1,
52 		.devs_max	= 2,
53 		.devs_min	= 2,
54 		.tolerated_failures = 1,
55 		.devs_increment	= 2,
56 		.ncopies	= 2,
57 		.nparity        = 0,
58 		.raid_name	= "raid1",
59 		.bg_flag	= BTRFS_BLOCK_GROUP_RAID1,
60 		.mindev_error	= BTRFS_ERROR_DEV_RAID1_MIN_NOT_MET,
61 	},
62 	[BTRFS_RAID_RAID1C3] = {
63 		.sub_stripes	= 1,
64 		.dev_stripes	= 1,
65 		.devs_max	= 3,
66 		.devs_min	= 3,
67 		.tolerated_failures = 2,
68 		.devs_increment	= 3,
69 		.ncopies	= 3,
70 		.nparity        = 0,
71 		.raid_name	= "raid1c3",
72 		.bg_flag	= BTRFS_BLOCK_GROUP_RAID1C3,
73 		.mindev_error	= BTRFS_ERROR_DEV_RAID1C3_MIN_NOT_MET,
74 	},
75 	[BTRFS_RAID_RAID1C4] = {
76 		.sub_stripes	= 1,
77 		.dev_stripes	= 1,
78 		.devs_max	= 4,
79 		.devs_min	= 4,
80 		.tolerated_failures = 3,
81 		.devs_increment	= 4,
82 		.ncopies	= 4,
83 		.nparity        = 0,
84 		.raid_name	= "raid1c4",
85 		.bg_flag	= BTRFS_BLOCK_GROUP_RAID1C4,
86 		.mindev_error	= BTRFS_ERROR_DEV_RAID1C4_MIN_NOT_MET,
87 	},
88 	[BTRFS_RAID_DUP] = {
89 		.sub_stripes	= 1,
90 		.dev_stripes	= 2,
91 		.devs_max	= 1,
92 		.devs_min	= 1,
93 		.tolerated_failures = 0,
94 		.devs_increment	= 1,
95 		.ncopies	= 2,
96 		.nparity        = 0,
97 		.raid_name	= "dup",
98 		.bg_flag	= BTRFS_BLOCK_GROUP_DUP,
99 		.mindev_error	= 0,
100 	},
101 	[BTRFS_RAID_RAID0] = {
102 		.sub_stripes	= 1,
103 		.dev_stripes	= 1,
104 		.devs_max	= 0,
105 		.devs_min	= 2,
106 		.tolerated_failures = 0,
107 		.devs_increment	= 1,
108 		.ncopies	= 1,
109 		.nparity        = 0,
110 		.raid_name	= "raid0",
111 		.bg_flag	= BTRFS_BLOCK_GROUP_RAID0,
112 		.mindev_error	= 0,
113 	},
114 	[BTRFS_RAID_SINGLE] = {
115 		.sub_stripes	= 1,
116 		.dev_stripes	= 1,
117 		.devs_max	= 1,
118 		.devs_min	= 1,
119 		.tolerated_failures = 0,
120 		.devs_increment	= 1,
121 		.ncopies	= 1,
122 		.nparity        = 0,
123 		.raid_name	= "single",
124 		.bg_flag	= 0,
125 		.mindev_error	= 0,
126 	},
127 	[BTRFS_RAID_RAID5] = {
128 		.sub_stripes	= 1,
129 		.dev_stripes	= 1,
130 		.devs_max	= 0,
131 		.devs_min	= 2,
132 		.tolerated_failures = 1,
133 		.devs_increment	= 1,
134 		.ncopies	= 1,
135 		.nparity        = 1,
136 		.raid_name	= "raid5",
137 		.bg_flag	= BTRFS_BLOCK_GROUP_RAID5,
138 		.mindev_error	= BTRFS_ERROR_DEV_RAID5_MIN_NOT_MET,
139 	},
140 	[BTRFS_RAID_RAID6] = {
141 		.sub_stripes	= 1,
142 		.dev_stripes	= 1,
143 		.devs_max	= 0,
144 		.devs_min	= 3,
145 		.tolerated_failures = 2,
146 		.devs_increment	= 1,
147 		.ncopies	= 1,
148 		.nparity        = 2,
149 		.raid_name	= "raid6",
150 		.bg_flag	= BTRFS_BLOCK_GROUP_RAID6,
151 		.mindev_error	= BTRFS_ERROR_DEV_RAID6_MIN_NOT_MET,
152 	},
153 };
154 
155 const char *btrfs_bg_type_to_raid_name(u64 flags)
156 {
157 	const int index = btrfs_bg_flags_to_raid_index(flags);
158 
159 	if (index >= BTRFS_NR_RAID_TYPES)
160 		return NULL;
161 
162 	return btrfs_raid_array[index].raid_name;
163 }
164 
165 /*
166  * Fill @buf with textual description of @bg_flags, no more than @size_buf
167  * bytes including terminating null byte.
168  */
169 void btrfs_describe_block_groups(u64 bg_flags, char *buf, u32 size_buf)
170 {
171 	int i;
172 	int ret;
173 	char *bp = buf;
174 	u64 flags = bg_flags;
175 	u32 size_bp = size_buf;
176 
177 	if (!flags) {
178 		strcpy(bp, "NONE");
179 		return;
180 	}
181 
182 #define DESCRIBE_FLAG(flag, desc)						\
183 	do {								\
184 		if (flags & (flag)) {					\
185 			ret = snprintf(bp, size_bp, "%s|", (desc));	\
186 			if (ret < 0 || ret >= size_bp)			\
187 				goto out_overflow;			\
188 			size_bp -= ret;					\
189 			bp += ret;					\
190 			flags &= ~(flag);				\
191 		}							\
192 	} while (0)
193 
194 	DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_DATA, "data");
195 	DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_SYSTEM, "system");
196 	DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_METADATA, "metadata");
197 
198 	DESCRIBE_FLAG(BTRFS_AVAIL_ALLOC_BIT_SINGLE, "single");
199 	for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
200 		DESCRIBE_FLAG(btrfs_raid_array[i].bg_flag,
201 			      btrfs_raid_array[i].raid_name);
202 #undef DESCRIBE_FLAG
203 
204 	if (flags) {
205 		ret = snprintf(bp, size_bp, "0x%llx|", flags);
206 		size_bp -= ret;
207 	}
208 
209 	if (size_bp < size_buf)
210 		buf[size_buf - size_bp - 1] = '\0'; /* remove last | */
211 
212 	/*
213 	 * The text is trimmed, it's up to the caller to provide sufficiently
214 	 * large buffer
215 	 */
216 out_overflow:;
217 }
218 
219 static int init_first_rw_device(struct btrfs_trans_handle *trans);
220 static int btrfs_relocate_sys_chunks(struct btrfs_fs_info *fs_info);
221 static void btrfs_dev_stat_print_on_error(struct btrfs_device *dev);
222 static void btrfs_dev_stat_print_on_load(struct btrfs_device *device);
223 static int __btrfs_map_block(struct btrfs_fs_info *fs_info,
224 			     enum btrfs_map_op op,
225 			     u64 logical, u64 *length,
226 			     struct btrfs_bio **bbio_ret,
227 			     int mirror_num, int need_raid_map);
228 
229 /*
230  * Device locking
231  * ==============
232  *
233  * There are several mutexes that protect manipulation of devices and low-level
234  * structures like chunks but not block groups, extents or files
235  *
236  * uuid_mutex (global lock)
237  * ------------------------
238  * protects the fs_uuids list that tracks all per-fs fs_devices, resulting from
239  * the SCAN_DEV ioctl registration or from mount either implicitly (the first
240  * device) or requested by the device= mount option
241  *
242  * the mutex can be very coarse and can cover long-running operations
243  *
244  * protects: updates to fs_devices counters like missing devices, rw devices,
245  * seeding, structure cloning, opening/closing devices at mount/umount time
246  *
247  * global::fs_devs - add, remove, updates to the global list
248  *
249  * does not protect: manipulation of the fs_devices::devices list in general
250  * but in mount context it could be used to exclude list modifications by eg.
251  * scan ioctl
252  *
253  * btrfs_device::name - renames (write side), read is RCU
254  *
255  * fs_devices::device_list_mutex (per-fs, with RCU)
256  * ------------------------------------------------
257  * protects updates to fs_devices::devices, ie. adding and deleting
258  *
259  * simple list traversal with read-only actions can be done with RCU protection
260  *
261  * may be used to exclude some operations from running concurrently without any
262  * modifications to the list (see write_all_supers)
263  *
264  * Is not required at mount and close times, because our device list is
265  * protected by the uuid_mutex at that point.
266  *
267  * balance_mutex
268  * -------------
269  * protects balance structures (status, state) and context accessed from
270  * several places (internally, ioctl)
271  *
272  * chunk_mutex
273  * -----------
274  * protects chunks, adding or removing during allocation, trim or when a new
275  * device is added/removed. Additionally it also protects post_commit_list of
276  * individual devices, since they can be added to the transaction's
277  * post_commit_list only with chunk_mutex held.
278  *
279  * cleaner_mutex
280  * -------------
281  * a big lock that is held by the cleaner thread and prevents running subvolume
282  * cleaning together with relocation or delayed iputs
283  *
284  *
285  * Lock nesting
286  * ============
287  *
288  * uuid_mutex
289  *   device_list_mutex
290  *     chunk_mutex
291  *   balance_mutex
292  *
293  *
294  * Exclusive operations
295  * ====================
296  *
297  * Maintains the exclusivity of the following operations that apply to the
298  * whole filesystem and cannot run in parallel.
299  *
300  * - Balance (*)
301  * - Device add
302  * - Device remove
303  * - Device replace (*)
304  * - Resize
305  *
306  * The device operations (as above) can be in one of the following states:
307  *
308  * - Running state
309  * - Paused state
310  * - Completed state
311  *
312  * Only device operations marked with (*) can go into the Paused state for the
313  * following reasons:
314  *
315  * - ioctl (only Balance can be Paused through ioctl)
316  * - filesystem remounted as read-only
317  * - filesystem unmounted and mounted as read-only
318  * - system power-cycle and filesystem mounted as read-only
319  * - filesystem or device errors leading to forced read-only
320  *
321  * The status of exclusive operation is set and cleared atomically.
322  * During the course of Paused state, fs_info::exclusive_operation remains set.
323  * A device operation in Paused or Running state can be canceled or resumed
324  * either by ioctl (Balance only) or when remounted as read-write.
325  * The exclusive status is cleared when the device operation is canceled or
326  * completed.
327  */
328 
329 DEFINE_MUTEX(uuid_mutex);
330 static LIST_HEAD(fs_uuids);
331 struct list_head * __attribute_const__ btrfs_get_fs_uuids(void)
332 {
333 	return &fs_uuids;
334 }
335 
336 /*
337  * alloc_fs_devices - allocate struct btrfs_fs_devices
338  * @fsid:		if not NULL, copy the UUID to fs_devices::fsid
339  * @metadata_fsid:	if not NULL, copy the UUID to fs_devices::metadata_fsid
340  *
341  * Return a pointer to a new struct btrfs_fs_devices on success, or ERR_PTR().
342  * The returned struct is not linked onto any lists and can be destroyed with
343  * kfree() right away.
344  */
345 static struct btrfs_fs_devices *alloc_fs_devices(const u8 *fsid,
346 						 const u8 *metadata_fsid)
347 {
348 	struct btrfs_fs_devices *fs_devs;
349 
350 	fs_devs = kzalloc(sizeof(*fs_devs), GFP_KERNEL);
351 	if (!fs_devs)
352 		return ERR_PTR(-ENOMEM);
353 
354 	mutex_init(&fs_devs->device_list_mutex);
355 
356 	INIT_LIST_HEAD(&fs_devs->devices);
357 	INIT_LIST_HEAD(&fs_devs->alloc_list);
358 	INIT_LIST_HEAD(&fs_devs->fs_list);
359 	INIT_LIST_HEAD(&fs_devs->seed_list);
360 	if (fsid)
361 		memcpy(fs_devs->fsid, fsid, BTRFS_FSID_SIZE);
362 
363 	if (metadata_fsid)
364 		memcpy(fs_devs->metadata_uuid, metadata_fsid, BTRFS_FSID_SIZE);
365 	else if (fsid)
366 		memcpy(fs_devs->metadata_uuid, fsid, BTRFS_FSID_SIZE);
367 
368 	return fs_devs;
369 }
370 
371 void btrfs_free_device(struct btrfs_device *device)
372 {
373 	WARN_ON(!list_empty(&device->post_commit_list));
374 	rcu_string_free(device->name);
375 	extent_io_tree_release(&device->alloc_state);
376 	bio_put(device->flush_bio);
377 	kfree(device);
378 }
379 
380 static void free_fs_devices(struct btrfs_fs_devices *fs_devices)
381 {
382 	struct btrfs_device *device;
383 	WARN_ON(fs_devices->opened);
384 	while (!list_empty(&fs_devices->devices)) {
385 		device = list_entry(fs_devices->devices.next,
386 				    struct btrfs_device, dev_list);
387 		list_del(&device->dev_list);
388 		btrfs_free_device(device);
389 	}
390 	kfree(fs_devices);
391 }
392 
393 void __exit btrfs_cleanup_fs_uuids(void)
394 {
395 	struct btrfs_fs_devices *fs_devices;
396 
397 	while (!list_empty(&fs_uuids)) {
398 		fs_devices = list_entry(fs_uuids.next,
399 					struct btrfs_fs_devices, fs_list);
400 		list_del(&fs_devices->fs_list);
401 		free_fs_devices(fs_devices);
402 	}
403 }
404 
405 /*
406  * Returns a pointer to a new btrfs_device on success; ERR_PTR() on error.
407  * Returned struct is not linked onto any lists and must be destroyed using
408  * btrfs_free_device.
409  */
410 static struct btrfs_device *__alloc_device(struct btrfs_fs_info *fs_info)
411 {
412 	struct btrfs_device *dev;
413 
414 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
415 	if (!dev)
416 		return ERR_PTR(-ENOMEM);
417 
418 	/*
419 	 * Preallocate a bio that's always going to be used for flushing device
420 	 * barriers and matches the device lifespan
421 	 */
422 	dev->flush_bio = bio_alloc_bioset(GFP_KERNEL, 0, NULL);
423 	if (!dev->flush_bio) {
424 		kfree(dev);
425 		return ERR_PTR(-ENOMEM);
426 	}
427 
428 	INIT_LIST_HEAD(&dev->dev_list);
429 	INIT_LIST_HEAD(&dev->dev_alloc_list);
430 	INIT_LIST_HEAD(&dev->post_commit_list);
431 
432 	atomic_set(&dev->reada_in_flight, 0);
433 	atomic_set(&dev->dev_stats_ccnt, 0);
434 	btrfs_device_data_ordered_init(dev, fs_info);
435 	INIT_RADIX_TREE(&dev->reada_zones, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
436 	INIT_RADIX_TREE(&dev->reada_extents, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
437 	extent_io_tree_init(fs_info, &dev->alloc_state,
438 			    IO_TREE_DEVICE_ALLOC_STATE, NULL);
439 
440 	return dev;
441 }
442 
443 static noinline struct btrfs_fs_devices *find_fsid(
444 		const u8 *fsid, const u8 *metadata_fsid)
445 {
446 	struct btrfs_fs_devices *fs_devices;
447 
448 	ASSERT(fsid);
449 
450 	/* Handle non-split brain cases */
451 	list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
452 		if (metadata_fsid) {
453 			if (memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE) == 0
454 			    && memcmp(metadata_fsid, fs_devices->metadata_uuid,
455 				      BTRFS_FSID_SIZE) == 0)
456 				return fs_devices;
457 		} else {
458 			if (memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE) == 0)
459 				return fs_devices;
460 		}
461 	}
462 	return NULL;
463 }
464 
465 static struct btrfs_fs_devices *find_fsid_with_metadata_uuid(
466 				struct btrfs_super_block *disk_super)
467 {
468 
469 	struct btrfs_fs_devices *fs_devices;
470 
471 	/*
472 	 * Handle scanned device having completed its fsid change but
473 	 * belonging to a fs_devices that was created by first scanning
474 	 * a device which didn't have its fsid/metadata_uuid changed
475 	 * at all and the CHANGING_FSID_V2 flag set.
476 	 */
477 	list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
478 		if (fs_devices->fsid_change &&
479 		    memcmp(disk_super->metadata_uuid, fs_devices->fsid,
480 			   BTRFS_FSID_SIZE) == 0 &&
481 		    memcmp(fs_devices->fsid, fs_devices->metadata_uuid,
482 			   BTRFS_FSID_SIZE) == 0) {
483 			return fs_devices;
484 		}
485 	}
486 	/*
487 	 * Handle scanned device having completed its fsid change but
488 	 * belonging to a fs_devices that was created by a device that
489 	 * has an outdated pair of fsid/metadata_uuid and
490 	 * CHANGING_FSID_V2 flag set.
491 	 */
492 	list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
493 		if (fs_devices->fsid_change &&
494 		    memcmp(fs_devices->metadata_uuid,
495 			   fs_devices->fsid, BTRFS_FSID_SIZE) != 0 &&
496 		    memcmp(disk_super->metadata_uuid, fs_devices->metadata_uuid,
497 			   BTRFS_FSID_SIZE) == 0) {
498 			return fs_devices;
499 		}
500 	}
501 
502 	return find_fsid(disk_super->fsid, disk_super->metadata_uuid);
503 }
504 
505 
506 static int
507 btrfs_get_bdev_and_sb(const char *device_path, fmode_t flags, void *holder,
508 		      int flush, struct block_device **bdev,
509 		      struct btrfs_super_block **disk_super)
510 {
511 	int ret;
512 
513 	*bdev = blkdev_get_by_path(device_path, flags, holder);
514 
515 	if (IS_ERR(*bdev)) {
516 		ret = PTR_ERR(*bdev);
517 		goto error;
518 	}
519 
520 	if (flush)
521 		filemap_write_and_wait((*bdev)->bd_inode->i_mapping);
522 	ret = set_blocksize(*bdev, BTRFS_BDEV_BLOCKSIZE);
523 	if (ret) {
524 		blkdev_put(*bdev, flags);
525 		goto error;
526 	}
527 	invalidate_bdev(*bdev);
528 	*disk_super = btrfs_read_dev_super(*bdev);
529 	if (IS_ERR(*disk_super)) {
530 		ret = PTR_ERR(*disk_super);
531 		blkdev_put(*bdev, flags);
532 		goto error;
533 	}
534 
535 	return 0;
536 
537 error:
538 	*bdev = NULL;
539 	return ret;
540 }
541 
542 static bool device_path_matched(const char *path, struct btrfs_device *device)
543 {
544 	int found;
545 
546 	rcu_read_lock();
547 	found = strcmp(rcu_str_deref(device->name), path);
548 	rcu_read_unlock();
549 
550 	return found == 0;
551 }
552 
553 /*
554  *  Search and remove all stale (devices which are not mounted) devices.
555  *  When both inputs are NULL, it will search and release all stale devices.
556  *  path:	Optional. When provided will it release all unmounted devices
557  *		matching this path only.
558  *  skip_dev:	Optional. Will skip this device when searching for the stale
559  *		devices.
560  *  Return:	0 for success or if @path is NULL.
561  * 		-EBUSY if @path is a mounted device.
562  * 		-ENOENT if @path does not match any device in the list.
563  */
564 static int btrfs_free_stale_devices(const char *path,
565 				     struct btrfs_device *skip_device)
566 {
567 	struct btrfs_fs_devices *fs_devices, *tmp_fs_devices;
568 	struct btrfs_device *device, *tmp_device;
569 	int ret = 0;
570 
571 	if (path)
572 		ret = -ENOENT;
573 
574 	list_for_each_entry_safe(fs_devices, tmp_fs_devices, &fs_uuids, fs_list) {
575 
576 		mutex_lock(&fs_devices->device_list_mutex);
577 		list_for_each_entry_safe(device, tmp_device,
578 					 &fs_devices->devices, dev_list) {
579 			if (skip_device && skip_device == device)
580 				continue;
581 			if (path && !device->name)
582 				continue;
583 			if (path && !device_path_matched(path, device))
584 				continue;
585 			if (fs_devices->opened) {
586 				/* for an already deleted device return 0 */
587 				if (path && ret != 0)
588 					ret = -EBUSY;
589 				break;
590 			}
591 
592 			/* delete the stale device */
593 			fs_devices->num_devices--;
594 			list_del(&device->dev_list);
595 			btrfs_free_device(device);
596 
597 			ret = 0;
598 		}
599 		mutex_unlock(&fs_devices->device_list_mutex);
600 
601 		if (fs_devices->num_devices == 0) {
602 			btrfs_sysfs_remove_fsid(fs_devices);
603 			list_del(&fs_devices->fs_list);
604 			free_fs_devices(fs_devices);
605 		}
606 	}
607 
608 	return ret;
609 }
610 
611 /*
612  * This is only used on mount, and we are protected from competing things
613  * messing with our fs_devices by the uuid_mutex, thus we do not need the
614  * fs_devices->device_list_mutex here.
615  */
616 static int btrfs_open_one_device(struct btrfs_fs_devices *fs_devices,
617 			struct btrfs_device *device, fmode_t flags,
618 			void *holder)
619 {
620 	struct request_queue *q;
621 	struct block_device *bdev;
622 	struct btrfs_super_block *disk_super;
623 	u64 devid;
624 	int ret;
625 
626 	if (device->bdev)
627 		return -EINVAL;
628 	if (!device->name)
629 		return -EINVAL;
630 
631 	ret = btrfs_get_bdev_and_sb(device->name->str, flags, holder, 1,
632 				    &bdev, &disk_super);
633 	if (ret)
634 		return ret;
635 
636 	devid = btrfs_stack_device_id(&disk_super->dev_item);
637 	if (devid != device->devid)
638 		goto error_free_page;
639 
640 	if (memcmp(device->uuid, disk_super->dev_item.uuid, BTRFS_UUID_SIZE))
641 		goto error_free_page;
642 
643 	device->generation = btrfs_super_generation(disk_super);
644 
645 	if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_SEEDING) {
646 		if (btrfs_super_incompat_flags(disk_super) &
647 		    BTRFS_FEATURE_INCOMPAT_METADATA_UUID) {
648 			pr_err(
649 		"BTRFS: Invalid seeding and uuid-changed device detected\n");
650 			goto error_free_page;
651 		}
652 
653 		clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
654 		fs_devices->seeding = true;
655 	} else {
656 		if (bdev_read_only(bdev))
657 			clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
658 		else
659 			set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
660 	}
661 
662 	q = bdev_get_queue(bdev);
663 	if (!blk_queue_nonrot(q))
664 		fs_devices->rotating = true;
665 
666 	device->bdev = bdev;
667 	clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
668 	device->mode = flags;
669 
670 	fs_devices->open_devices++;
671 	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
672 	    device->devid != BTRFS_DEV_REPLACE_DEVID) {
673 		fs_devices->rw_devices++;
674 		list_add_tail(&device->dev_alloc_list, &fs_devices->alloc_list);
675 	}
676 	btrfs_release_disk_super(disk_super);
677 
678 	return 0;
679 
680 error_free_page:
681 	btrfs_release_disk_super(disk_super);
682 	blkdev_put(bdev, flags);
683 
684 	return -EINVAL;
685 }
686 
687 /*
688  * Handle scanned device having its CHANGING_FSID_V2 flag set and the fs_devices
689  * being created with a disk that has already completed its fsid change. Such
690  * disk can belong to an fs which has its FSID changed or to one which doesn't.
691  * Handle both cases here.
692  */
693 static struct btrfs_fs_devices *find_fsid_inprogress(
694 					struct btrfs_super_block *disk_super)
695 {
696 	struct btrfs_fs_devices *fs_devices;
697 
698 	list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
699 		if (memcmp(fs_devices->metadata_uuid, fs_devices->fsid,
700 			   BTRFS_FSID_SIZE) != 0 &&
701 		    memcmp(fs_devices->metadata_uuid, disk_super->fsid,
702 			   BTRFS_FSID_SIZE) == 0 && !fs_devices->fsid_change) {
703 			return fs_devices;
704 		}
705 	}
706 
707 	return find_fsid(disk_super->fsid, NULL);
708 }
709 
710 
711 static struct btrfs_fs_devices *find_fsid_changed(
712 					struct btrfs_super_block *disk_super)
713 {
714 	struct btrfs_fs_devices *fs_devices;
715 
716 	/*
717 	 * Handles the case where scanned device is part of an fs that had
718 	 * multiple successful changes of FSID but curently device didn't
719 	 * observe it. Meaning our fsid will be different than theirs. We need
720 	 * to handle two subcases :
721 	 *  1 - The fs still continues to have different METADATA/FSID uuids.
722 	 *  2 - The fs is switched back to its original FSID (METADATA/FSID
723 	 *  are equal).
724 	 */
725 	list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
726 		/* Changed UUIDs */
727 		if (memcmp(fs_devices->metadata_uuid, fs_devices->fsid,
728 			   BTRFS_FSID_SIZE) != 0 &&
729 		    memcmp(fs_devices->metadata_uuid, disk_super->metadata_uuid,
730 			   BTRFS_FSID_SIZE) == 0 &&
731 		    memcmp(fs_devices->fsid, disk_super->fsid,
732 			   BTRFS_FSID_SIZE) != 0)
733 			return fs_devices;
734 
735 		/* Unchanged UUIDs */
736 		if (memcmp(fs_devices->metadata_uuid, fs_devices->fsid,
737 			   BTRFS_FSID_SIZE) == 0 &&
738 		    memcmp(fs_devices->fsid, disk_super->metadata_uuid,
739 			   BTRFS_FSID_SIZE) == 0)
740 			return fs_devices;
741 	}
742 
743 	return NULL;
744 }
745 
746 static struct btrfs_fs_devices *find_fsid_reverted_metadata(
747 				struct btrfs_super_block *disk_super)
748 {
749 	struct btrfs_fs_devices *fs_devices;
750 
751 	/*
752 	 * Handle the case where the scanned device is part of an fs whose last
753 	 * metadata UUID change reverted it to the original FSID. At the same
754 	 * time * fs_devices was first created by another constitutent device
755 	 * which didn't fully observe the operation. This results in an
756 	 * btrfs_fs_devices created with metadata/fsid different AND
757 	 * btrfs_fs_devices::fsid_change set AND the metadata_uuid of the
758 	 * fs_devices equal to the FSID of the disk.
759 	 */
760 	list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
761 		if (memcmp(fs_devices->fsid, fs_devices->metadata_uuid,
762 			   BTRFS_FSID_SIZE) != 0 &&
763 		    memcmp(fs_devices->metadata_uuid, disk_super->fsid,
764 			   BTRFS_FSID_SIZE) == 0 &&
765 		    fs_devices->fsid_change)
766 			return fs_devices;
767 	}
768 
769 	return NULL;
770 }
771 /*
772  * Add new device to list of registered devices
773  *
774  * Returns:
775  * device pointer which was just added or updated when successful
776  * error pointer when failed
777  */
778 static noinline struct btrfs_device *device_list_add(const char *path,
779 			   struct btrfs_super_block *disk_super,
780 			   bool *new_device_added)
781 {
782 	struct btrfs_device *device;
783 	struct btrfs_fs_devices *fs_devices = NULL;
784 	struct rcu_string *name;
785 	u64 found_transid = btrfs_super_generation(disk_super);
786 	u64 devid = btrfs_stack_device_id(&disk_super->dev_item);
787 	bool has_metadata_uuid = (btrfs_super_incompat_flags(disk_super) &
788 		BTRFS_FEATURE_INCOMPAT_METADATA_UUID);
789 	bool fsid_change_in_progress = (btrfs_super_flags(disk_super) &
790 					BTRFS_SUPER_FLAG_CHANGING_FSID_V2);
791 
792 	if (fsid_change_in_progress) {
793 		if (!has_metadata_uuid)
794 			fs_devices = find_fsid_inprogress(disk_super);
795 		else
796 			fs_devices = find_fsid_changed(disk_super);
797 	} else if (has_metadata_uuid) {
798 		fs_devices = find_fsid_with_metadata_uuid(disk_super);
799 	} else {
800 		fs_devices = find_fsid_reverted_metadata(disk_super);
801 		if (!fs_devices)
802 			fs_devices = find_fsid(disk_super->fsid, NULL);
803 	}
804 
805 
806 	if (!fs_devices) {
807 		if (has_metadata_uuid)
808 			fs_devices = alloc_fs_devices(disk_super->fsid,
809 						      disk_super->metadata_uuid);
810 		else
811 			fs_devices = alloc_fs_devices(disk_super->fsid, NULL);
812 
813 		if (IS_ERR(fs_devices))
814 			return ERR_CAST(fs_devices);
815 
816 		fs_devices->fsid_change = fsid_change_in_progress;
817 
818 		mutex_lock(&fs_devices->device_list_mutex);
819 		list_add(&fs_devices->fs_list, &fs_uuids);
820 
821 		device = NULL;
822 	} else {
823 		mutex_lock(&fs_devices->device_list_mutex);
824 		device = btrfs_find_device(fs_devices, devid,
825 				disk_super->dev_item.uuid, NULL, false);
826 
827 		/*
828 		 * If this disk has been pulled into an fs devices created by
829 		 * a device which had the CHANGING_FSID_V2 flag then replace the
830 		 * metadata_uuid/fsid values of the fs_devices.
831 		 */
832 		if (fs_devices->fsid_change &&
833 		    found_transid > fs_devices->latest_generation) {
834 			memcpy(fs_devices->fsid, disk_super->fsid,
835 					BTRFS_FSID_SIZE);
836 
837 			if (has_metadata_uuid)
838 				memcpy(fs_devices->metadata_uuid,
839 				       disk_super->metadata_uuid,
840 				       BTRFS_FSID_SIZE);
841 			else
842 				memcpy(fs_devices->metadata_uuid,
843 				       disk_super->fsid, BTRFS_FSID_SIZE);
844 
845 			fs_devices->fsid_change = false;
846 		}
847 	}
848 
849 	if (!device) {
850 		if (fs_devices->opened) {
851 			mutex_unlock(&fs_devices->device_list_mutex);
852 			return ERR_PTR(-EBUSY);
853 		}
854 
855 		device = btrfs_alloc_device(NULL, &devid,
856 					    disk_super->dev_item.uuid);
857 		if (IS_ERR(device)) {
858 			mutex_unlock(&fs_devices->device_list_mutex);
859 			/* we can safely leave the fs_devices entry around */
860 			return device;
861 		}
862 
863 		name = rcu_string_strdup(path, GFP_NOFS);
864 		if (!name) {
865 			btrfs_free_device(device);
866 			mutex_unlock(&fs_devices->device_list_mutex);
867 			return ERR_PTR(-ENOMEM);
868 		}
869 		rcu_assign_pointer(device->name, name);
870 
871 		list_add_rcu(&device->dev_list, &fs_devices->devices);
872 		fs_devices->num_devices++;
873 
874 		device->fs_devices = fs_devices;
875 		*new_device_added = true;
876 
877 		if (disk_super->label[0])
878 			pr_info(
879 	"BTRFS: device label %s devid %llu transid %llu %s scanned by %s (%d)\n",
880 				disk_super->label, devid, found_transid, path,
881 				current->comm, task_pid_nr(current));
882 		else
883 			pr_info(
884 	"BTRFS: device fsid %pU devid %llu transid %llu %s scanned by %s (%d)\n",
885 				disk_super->fsid, devid, found_transid, path,
886 				current->comm, task_pid_nr(current));
887 
888 	} else if (!device->name || strcmp(device->name->str, path)) {
889 		/*
890 		 * When FS is already mounted.
891 		 * 1. If you are here and if the device->name is NULL that
892 		 *    means this device was missing at time of FS mount.
893 		 * 2. If you are here and if the device->name is different
894 		 *    from 'path' that means either
895 		 *      a. The same device disappeared and reappeared with
896 		 *         different name. or
897 		 *      b. The missing-disk-which-was-replaced, has
898 		 *         reappeared now.
899 		 *
900 		 * We must allow 1 and 2a above. But 2b would be a spurious
901 		 * and unintentional.
902 		 *
903 		 * Further in case of 1 and 2a above, the disk at 'path'
904 		 * would have missed some transaction when it was away and
905 		 * in case of 2a the stale bdev has to be updated as well.
906 		 * 2b must not be allowed at all time.
907 		 */
908 
909 		/*
910 		 * For now, we do allow update to btrfs_fs_device through the
911 		 * btrfs dev scan cli after FS has been mounted.  We're still
912 		 * tracking a problem where systems fail mount by subvolume id
913 		 * when we reject replacement on a mounted FS.
914 		 */
915 		if (!fs_devices->opened && found_transid < device->generation) {
916 			/*
917 			 * That is if the FS is _not_ mounted and if you
918 			 * are here, that means there is more than one
919 			 * disk with same uuid and devid.We keep the one
920 			 * with larger generation number or the last-in if
921 			 * generation are equal.
922 			 */
923 			mutex_unlock(&fs_devices->device_list_mutex);
924 			return ERR_PTR(-EEXIST);
925 		}
926 
927 		/*
928 		 * We are going to replace the device path for a given devid,
929 		 * make sure it's the same device if the device is mounted
930 		 */
931 		if (device->bdev) {
932 			struct block_device *path_bdev;
933 
934 			path_bdev = lookup_bdev(path);
935 			if (IS_ERR(path_bdev)) {
936 				mutex_unlock(&fs_devices->device_list_mutex);
937 				return ERR_CAST(path_bdev);
938 			}
939 
940 			if (device->bdev != path_bdev) {
941 				bdput(path_bdev);
942 				mutex_unlock(&fs_devices->device_list_mutex);
943 				/*
944 				 * device->fs_info may not be reliable here, so
945 				 * pass in a NULL instead. This avoids a
946 				 * possible use-after-free when the fs_info and
947 				 * fs_info->sb are already torn down.
948 				 */
949 				btrfs_warn_in_rcu(NULL,
950 	"duplicate device %s devid %llu generation %llu scanned by %s (%d)",
951 						  path, devid, found_transid,
952 						  current->comm,
953 						  task_pid_nr(current));
954 				return ERR_PTR(-EEXIST);
955 			}
956 			bdput(path_bdev);
957 			btrfs_info_in_rcu(device->fs_info,
958 	"devid %llu device path %s changed to %s scanned by %s (%d)",
959 					  devid, rcu_str_deref(device->name),
960 					  path, current->comm,
961 					  task_pid_nr(current));
962 		}
963 
964 		name = rcu_string_strdup(path, GFP_NOFS);
965 		if (!name) {
966 			mutex_unlock(&fs_devices->device_list_mutex);
967 			return ERR_PTR(-ENOMEM);
968 		}
969 		rcu_string_free(device->name);
970 		rcu_assign_pointer(device->name, name);
971 		if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) {
972 			fs_devices->missing_devices--;
973 			clear_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state);
974 		}
975 	}
976 
977 	/*
978 	 * Unmount does not free the btrfs_device struct but would zero
979 	 * generation along with most of the other members. So just update
980 	 * it back. We need it to pick the disk with largest generation
981 	 * (as above).
982 	 */
983 	if (!fs_devices->opened) {
984 		device->generation = found_transid;
985 		fs_devices->latest_generation = max_t(u64, found_transid,
986 						fs_devices->latest_generation);
987 	}
988 
989 	fs_devices->total_devices = btrfs_super_num_devices(disk_super);
990 
991 	mutex_unlock(&fs_devices->device_list_mutex);
992 	return device;
993 }
994 
995 static struct btrfs_fs_devices *clone_fs_devices(struct btrfs_fs_devices *orig)
996 {
997 	struct btrfs_fs_devices *fs_devices;
998 	struct btrfs_device *device;
999 	struct btrfs_device *orig_dev;
1000 	int ret = 0;
1001 
1002 	fs_devices = alloc_fs_devices(orig->fsid, NULL);
1003 	if (IS_ERR(fs_devices))
1004 		return fs_devices;
1005 
1006 	mutex_lock(&orig->device_list_mutex);
1007 	fs_devices->total_devices = orig->total_devices;
1008 
1009 	list_for_each_entry(orig_dev, &orig->devices, dev_list) {
1010 		struct rcu_string *name;
1011 
1012 		device = btrfs_alloc_device(NULL, &orig_dev->devid,
1013 					    orig_dev->uuid);
1014 		if (IS_ERR(device)) {
1015 			ret = PTR_ERR(device);
1016 			goto error;
1017 		}
1018 
1019 		/*
1020 		 * This is ok to do without rcu read locked because we hold the
1021 		 * uuid mutex so nothing we touch in here is going to disappear.
1022 		 */
1023 		if (orig_dev->name) {
1024 			name = rcu_string_strdup(orig_dev->name->str,
1025 					GFP_KERNEL);
1026 			if (!name) {
1027 				btrfs_free_device(device);
1028 				ret = -ENOMEM;
1029 				goto error;
1030 			}
1031 			rcu_assign_pointer(device->name, name);
1032 		}
1033 
1034 		list_add(&device->dev_list, &fs_devices->devices);
1035 		device->fs_devices = fs_devices;
1036 		fs_devices->num_devices++;
1037 	}
1038 	mutex_unlock(&orig->device_list_mutex);
1039 	return fs_devices;
1040 error:
1041 	mutex_unlock(&orig->device_list_mutex);
1042 	free_fs_devices(fs_devices);
1043 	return ERR_PTR(ret);
1044 }
1045 
1046 static void __btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices,
1047 				      int step, struct btrfs_device **latest_dev)
1048 {
1049 	struct btrfs_device *device, *next;
1050 
1051 	/* This is the initialized path, it is safe to release the devices. */
1052 	list_for_each_entry_safe(device, next, &fs_devices->devices, dev_list) {
1053 		if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state)) {
1054 			if (!test_bit(BTRFS_DEV_STATE_REPLACE_TGT,
1055 				      &device->dev_state) &&
1056 			    !test_bit(BTRFS_DEV_STATE_MISSING,
1057 				      &device->dev_state) &&
1058 			    (!*latest_dev ||
1059 			     device->generation > (*latest_dev)->generation)) {
1060 				*latest_dev = device;
1061 			}
1062 			continue;
1063 		}
1064 
1065 		/*
1066 		 * We have already validated the presence of BTRFS_DEV_REPLACE_DEVID,
1067 		 * in btrfs_init_dev_replace() so just continue.
1068 		 */
1069 		if (device->devid == BTRFS_DEV_REPLACE_DEVID)
1070 			continue;
1071 
1072 		if (device->bdev) {
1073 			blkdev_put(device->bdev, device->mode);
1074 			device->bdev = NULL;
1075 			fs_devices->open_devices--;
1076 		}
1077 		if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
1078 			list_del_init(&device->dev_alloc_list);
1079 			clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
1080 		}
1081 		list_del_init(&device->dev_list);
1082 		fs_devices->num_devices--;
1083 		btrfs_free_device(device);
1084 	}
1085 
1086 }
1087 
1088 /*
1089  * After we have read the system tree and know devids belonging to this
1090  * filesystem, remove the device which does not belong there.
1091  */
1092 void btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices, int step)
1093 {
1094 	struct btrfs_device *latest_dev = NULL;
1095 	struct btrfs_fs_devices *seed_dev;
1096 
1097 	mutex_lock(&uuid_mutex);
1098 	__btrfs_free_extra_devids(fs_devices, step, &latest_dev);
1099 
1100 	list_for_each_entry(seed_dev, &fs_devices->seed_list, seed_list)
1101 		__btrfs_free_extra_devids(seed_dev, step, &latest_dev);
1102 
1103 	fs_devices->latest_bdev = latest_dev->bdev;
1104 
1105 	mutex_unlock(&uuid_mutex);
1106 }
1107 
1108 static void btrfs_close_bdev(struct btrfs_device *device)
1109 {
1110 	if (!device->bdev)
1111 		return;
1112 
1113 	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
1114 		sync_blockdev(device->bdev);
1115 		invalidate_bdev(device->bdev);
1116 	}
1117 
1118 	blkdev_put(device->bdev, device->mode);
1119 }
1120 
1121 static void btrfs_close_one_device(struct btrfs_device *device)
1122 {
1123 	struct btrfs_fs_devices *fs_devices = device->fs_devices;
1124 
1125 	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
1126 	    device->devid != BTRFS_DEV_REPLACE_DEVID) {
1127 		list_del_init(&device->dev_alloc_list);
1128 		fs_devices->rw_devices--;
1129 	}
1130 
1131 	if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state))
1132 		fs_devices->missing_devices--;
1133 
1134 	btrfs_close_bdev(device);
1135 	if (device->bdev) {
1136 		fs_devices->open_devices--;
1137 		device->bdev = NULL;
1138 	}
1139 	clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
1140 
1141 	device->fs_info = NULL;
1142 	atomic_set(&device->dev_stats_ccnt, 0);
1143 	extent_io_tree_release(&device->alloc_state);
1144 
1145 	/* Verify the device is back in a pristine state  */
1146 	ASSERT(!test_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state));
1147 	ASSERT(!test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state));
1148 	ASSERT(list_empty(&device->dev_alloc_list));
1149 	ASSERT(list_empty(&device->post_commit_list));
1150 	ASSERT(atomic_read(&device->reada_in_flight) == 0);
1151 }
1152 
1153 static void close_fs_devices(struct btrfs_fs_devices *fs_devices)
1154 {
1155 	struct btrfs_device *device, *tmp;
1156 
1157 	lockdep_assert_held(&uuid_mutex);
1158 
1159 	if (--fs_devices->opened > 0)
1160 		return;
1161 
1162 	list_for_each_entry_safe(device, tmp, &fs_devices->devices, dev_list)
1163 		btrfs_close_one_device(device);
1164 
1165 	WARN_ON(fs_devices->open_devices);
1166 	WARN_ON(fs_devices->rw_devices);
1167 	fs_devices->opened = 0;
1168 	fs_devices->seeding = false;
1169 	fs_devices->fs_info = NULL;
1170 }
1171 
1172 void btrfs_close_devices(struct btrfs_fs_devices *fs_devices)
1173 {
1174 	LIST_HEAD(list);
1175 	struct btrfs_fs_devices *tmp;
1176 
1177 	mutex_lock(&uuid_mutex);
1178 	close_fs_devices(fs_devices);
1179 	if (!fs_devices->opened)
1180 		list_splice_init(&fs_devices->seed_list, &list);
1181 
1182 	list_for_each_entry_safe(fs_devices, tmp, &list, seed_list) {
1183 		close_fs_devices(fs_devices);
1184 		list_del(&fs_devices->seed_list);
1185 		free_fs_devices(fs_devices);
1186 	}
1187 	mutex_unlock(&uuid_mutex);
1188 }
1189 
1190 static int open_fs_devices(struct btrfs_fs_devices *fs_devices,
1191 				fmode_t flags, void *holder)
1192 {
1193 	struct btrfs_device *device;
1194 	struct btrfs_device *latest_dev = NULL;
1195 	struct btrfs_device *tmp_device;
1196 
1197 	flags |= FMODE_EXCL;
1198 
1199 	list_for_each_entry_safe(device, tmp_device, &fs_devices->devices,
1200 				 dev_list) {
1201 		int ret;
1202 
1203 		ret = btrfs_open_one_device(fs_devices, device, flags, holder);
1204 		if (ret == 0 &&
1205 		    (!latest_dev || device->generation > latest_dev->generation)) {
1206 			latest_dev = device;
1207 		} else if (ret == -ENODATA) {
1208 			fs_devices->num_devices--;
1209 			list_del(&device->dev_list);
1210 			btrfs_free_device(device);
1211 		}
1212 	}
1213 	if (fs_devices->open_devices == 0)
1214 		return -EINVAL;
1215 
1216 	fs_devices->opened = 1;
1217 	fs_devices->latest_bdev = latest_dev->bdev;
1218 	fs_devices->total_rw_bytes = 0;
1219 	fs_devices->chunk_alloc_policy = BTRFS_CHUNK_ALLOC_REGULAR;
1220 
1221 	return 0;
1222 }
1223 
1224 static int devid_cmp(void *priv, struct list_head *a, struct list_head *b)
1225 {
1226 	struct btrfs_device *dev1, *dev2;
1227 
1228 	dev1 = list_entry(a, struct btrfs_device, dev_list);
1229 	dev2 = list_entry(b, struct btrfs_device, dev_list);
1230 
1231 	if (dev1->devid < dev2->devid)
1232 		return -1;
1233 	else if (dev1->devid > dev2->devid)
1234 		return 1;
1235 	return 0;
1236 }
1237 
1238 int btrfs_open_devices(struct btrfs_fs_devices *fs_devices,
1239 		       fmode_t flags, void *holder)
1240 {
1241 	int ret;
1242 
1243 	lockdep_assert_held(&uuid_mutex);
1244 	/*
1245 	 * The device_list_mutex cannot be taken here in case opening the
1246 	 * underlying device takes further locks like bd_mutex.
1247 	 *
1248 	 * We also don't need the lock here as this is called during mount and
1249 	 * exclusion is provided by uuid_mutex
1250 	 */
1251 
1252 	if (fs_devices->opened) {
1253 		fs_devices->opened++;
1254 		ret = 0;
1255 	} else {
1256 		list_sort(NULL, &fs_devices->devices, devid_cmp);
1257 		ret = open_fs_devices(fs_devices, flags, holder);
1258 	}
1259 
1260 	return ret;
1261 }
1262 
1263 void btrfs_release_disk_super(struct btrfs_super_block *super)
1264 {
1265 	struct page *page = virt_to_page(super);
1266 
1267 	put_page(page);
1268 }
1269 
1270 static struct btrfs_super_block *btrfs_read_disk_super(struct block_device *bdev,
1271 						       u64 bytenr)
1272 {
1273 	struct btrfs_super_block *disk_super;
1274 	struct page *page;
1275 	void *p;
1276 	pgoff_t index;
1277 
1278 	/* make sure our super fits in the device */
1279 	if (bytenr + PAGE_SIZE >= i_size_read(bdev->bd_inode))
1280 		return ERR_PTR(-EINVAL);
1281 
1282 	/* make sure our super fits in the page */
1283 	if (sizeof(*disk_super) > PAGE_SIZE)
1284 		return ERR_PTR(-EINVAL);
1285 
1286 	/* make sure our super doesn't straddle pages on disk */
1287 	index = bytenr >> PAGE_SHIFT;
1288 	if ((bytenr + sizeof(*disk_super) - 1) >> PAGE_SHIFT != index)
1289 		return ERR_PTR(-EINVAL);
1290 
1291 	/* pull in the page with our super */
1292 	page = read_cache_page_gfp(bdev->bd_inode->i_mapping, index, GFP_KERNEL);
1293 
1294 	if (IS_ERR(page))
1295 		return ERR_CAST(page);
1296 
1297 	p = page_address(page);
1298 
1299 	/* align our pointer to the offset of the super block */
1300 	disk_super = p + offset_in_page(bytenr);
1301 
1302 	if (btrfs_super_bytenr(disk_super) != bytenr ||
1303 	    btrfs_super_magic(disk_super) != BTRFS_MAGIC) {
1304 		btrfs_release_disk_super(p);
1305 		return ERR_PTR(-EINVAL);
1306 	}
1307 
1308 	if (disk_super->label[0] && disk_super->label[BTRFS_LABEL_SIZE - 1])
1309 		disk_super->label[BTRFS_LABEL_SIZE - 1] = 0;
1310 
1311 	return disk_super;
1312 }
1313 
1314 int btrfs_forget_devices(const char *path)
1315 {
1316 	int ret;
1317 
1318 	mutex_lock(&uuid_mutex);
1319 	ret = btrfs_free_stale_devices(strlen(path) ? path : NULL, NULL);
1320 	mutex_unlock(&uuid_mutex);
1321 
1322 	return ret;
1323 }
1324 
1325 /*
1326  * Look for a btrfs signature on a device. This may be called out of the mount path
1327  * and we are not allowed to call set_blocksize during the scan. The superblock
1328  * is read via pagecache
1329  */
1330 struct btrfs_device *btrfs_scan_one_device(const char *path, fmode_t flags,
1331 					   void *holder)
1332 {
1333 	struct btrfs_super_block *disk_super;
1334 	bool new_device_added = false;
1335 	struct btrfs_device *device = NULL;
1336 	struct block_device *bdev;
1337 	u64 bytenr;
1338 
1339 	lockdep_assert_held(&uuid_mutex);
1340 
1341 	/*
1342 	 * we would like to check all the supers, but that would make
1343 	 * a btrfs mount succeed after a mkfs from a different FS.
1344 	 * So, we need to add a special mount option to scan for
1345 	 * later supers, using BTRFS_SUPER_MIRROR_MAX instead
1346 	 */
1347 	bytenr = btrfs_sb_offset(0);
1348 	flags |= FMODE_EXCL;
1349 
1350 	bdev = blkdev_get_by_path(path, flags, holder);
1351 	if (IS_ERR(bdev))
1352 		return ERR_CAST(bdev);
1353 
1354 	disk_super = btrfs_read_disk_super(bdev, bytenr);
1355 	if (IS_ERR(disk_super)) {
1356 		device = ERR_CAST(disk_super);
1357 		goto error_bdev_put;
1358 	}
1359 
1360 	device = device_list_add(path, disk_super, &new_device_added);
1361 	if (!IS_ERR(device)) {
1362 		if (new_device_added)
1363 			btrfs_free_stale_devices(path, device);
1364 	}
1365 
1366 	btrfs_release_disk_super(disk_super);
1367 
1368 error_bdev_put:
1369 	blkdev_put(bdev, flags);
1370 
1371 	return device;
1372 }
1373 
1374 /*
1375  * Try to find a chunk that intersects [start, start + len] range and when one
1376  * such is found, record the end of it in *start
1377  */
1378 static bool contains_pending_extent(struct btrfs_device *device, u64 *start,
1379 				    u64 len)
1380 {
1381 	u64 physical_start, physical_end;
1382 
1383 	lockdep_assert_held(&device->fs_info->chunk_mutex);
1384 
1385 	if (!find_first_extent_bit(&device->alloc_state, *start,
1386 				   &physical_start, &physical_end,
1387 				   CHUNK_ALLOCATED, NULL)) {
1388 
1389 		if (in_range(physical_start, *start, len) ||
1390 		    in_range(*start, physical_start,
1391 			     physical_end - physical_start)) {
1392 			*start = physical_end + 1;
1393 			return true;
1394 		}
1395 	}
1396 	return false;
1397 }
1398 
1399 static u64 dev_extent_search_start(struct btrfs_device *device, u64 start)
1400 {
1401 	switch (device->fs_devices->chunk_alloc_policy) {
1402 	case BTRFS_CHUNK_ALLOC_REGULAR:
1403 		/*
1404 		 * We don't want to overwrite the superblock on the drive nor
1405 		 * any area used by the boot loader (grub for example), so we
1406 		 * make sure to start at an offset of at least 1MB.
1407 		 */
1408 		return max_t(u64, start, SZ_1M);
1409 	default:
1410 		BUG();
1411 	}
1412 }
1413 
1414 /**
1415  * dev_extent_hole_check - check if specified hole is suitable for allocation
1416  * @device:	the device which we have the hole
1417  * @hole_start: starting position of the hole
1418  * @hole_size:	the size of the hole
1419  * @num_bytes:	the size of the free space that we need
1420  *
1421  * This function may modify @hole_start and @hole_end to reflect the suitable
1422  * position for allocation. Returns 1 if hole position is updated, 0 otherwise.
1423  */
1424 static bool dev_extent_hole_check(struct btrfs_device *device, u64 *hole_start,
1425 				  u64 *hole_size, u64 num_bytes)
1426 {
1427 	bool changed = false;
1428 	u64 hole_end = *hole_start + *hole_size;
1429 
1430 	/*
1431 	 * Check before we set max_hole_start, otherwise we could end up
1432 	 * sending back this offset anyway.
1433 	 */
1434 	if (contains_pending_extent(device, hole_start, *hole_size)) {
1435 		if (hole_end >= *hole_start)
1436 			*hole_size = hole_end - *hole_start;
1437 		else
1438 			*hole_size = 0;
1439 		changed = true;
1440 	}
1441 
1442 	switch (device->fs_devices->chunk_alloc_policy) {
1443 	case BTRFS_CHUNK_ALLOC_REGULAR:
1444 		/* No extra check */
1445 		break;
1446 	default:
1447 		BUG();
1448 	}
1449 
1450 	return changed;
1451 }
1452 
1453 /*
1454  * find_free_dev_extent_start - find free space in the specified device
1455  * @device:	  the device which we search the free space in
1456  * @num_bytes:	  the size of the free space that we need
1457  * @search_start: the position from which to begin the search
1458  * @start:	  store the start of the free space.
1459  * @len:	  the size of the free space. that we find, or the size
1460  *		  of the max free space if we don't find suitable free space
1461  *
1462  * this uses a pretty simple search, the expectation is that it is
1463  * called very infrequently and that a given device has a small number
1464  * of extents
1465  *
1466  * @start is used to store the start of the free space if we find. But if we
1467  * don't find suitable free space, it will be used to store the start position
1468  * of the max free space.
1469  *
1470  * @len is used to store the size of the free space that we find.
1471  * But if we don't find suitable free space, it is used to store the size of
1472  * the max free space.
1473  *
1474  * NOTE: This function will search *commit* root of device tree, and does extra
1475  * check to ensure dev extents are not double allocated.
1476  * This makes the function safe to allocate dev extents but may not report
1477  * correct usable device space, as device extent freed in current transaction
1478  * is not reported as avaiable.
1479  */
1480 static int find_free_dev_extent_start(struct btrfs_device *device,
1481 				u64 num_bytes, u64 search_start, u64 *start,
1482 				u64 *len)
1483 {
1484 	struct btrfs_fs_info *fs_info = device->fs_info;
1485 	struct btrfs_root *root = fs_info->dev_root;
1486 	struct btrfs_key key;
1487 	struct btrfs_dev_extent *dev_extent;
1488 	struct btrfs_path *path;
1489 	u64 hole_size;
1490 	u64 max_hole_start;
1491 	u64 max_hole_size;
1492 	u64 extent_end;
1493 	u64 search_end = device->total_bytes;
1494 	int ret;
1495 	int slot;
1496 	struct extent_buffer *l;
1497 
1498 	search_start = dev_extent_search_start(device, search_start);
1499 
1500 	path = btrfs_alloc_path();
1501 	if (!path)
1502 		return -ENOMEM;
1503 
1504 	max_hole_start = search_start;
1505 	max_hole_size = 0;
1506 
1507 again:
1508 	if (search_start >= search_end ||
1509 		test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
1510 		ret = -ENOSPC;
1511 		goto out;
1512 	}
1513 
1514 	path->reada = READA_FORWARD;
1515 	path->search_commit_root = 1;
1516 	path->skip_locking = 1;
1517 
1518 	key.objectid = device->devid;
1519 	key.offset = search_start;
1520 	key.type = BTRFS_DEV_EXTENT_KEY;
1521 
1522 	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
1523 	if (ret < 0)
1524 		goto out;
1525 	if (ret > 0) {
1526 		ret = btrfs_previous_item(root, path, key.objectid, key.type);
1527 		if (ret < 0)
1528 			goto out;
1529 	}
1530 
1531 	while (1) {
1532 		l = path->nodes[0];
1533 		slot = path->slots[0];
1534 		if (slot >= btrfs_header_nritems(l)) {
1535 			ret = btrfs_next_leaf(root, path);
1536 			if (ret == 0)
1537 				continue;
1538 			if (ret < 0)
1539 				goto out;
1540 
1541 			break;
1542 		}
1543 		btrfs_item_key_to_cpu(l, &key, slot);
1544 
1545 		if (key.objectid < device->devid)
1546 			goto next;
1547 
1548 		if (key.objectid > device->devid)
1549 			break;
1550 
1551 		if (key.type != BTRFS_DEV_EXTENT_KEY)
1552 			goto next;
1553 
1554 		if (key.offset > search_start) {
1555 			hole_size = key.offset - search_start;
1556 			dev_extent_hole_check(device, &search_start, &hole_size,
1557 					      num_bytes);
1558 
1559 			if (hole_size > max_hole_size) {
1560 				max_hole_start = search_start;
1561 				max_hole_size = hole_size;
1562 			}
1563 
1564 			/*
1565 			 * If this free space is greater than which we need,
1566 			 * it must be the max free space that we have found
1567 			 * until now, so max_hole_start must point to the start
1568 			 * of this free space and the length of this free space
1569 			 * is stored in max_hole_size. Thus, we return
1570 			 * max_hole_start and max_hole_size and go back to the
1571 			 * caller.
1572 			 */
1573 			if (hole_size >= num_bytes) {
1574 				ret = 0;
1575 				goto out;
1576 			}
1577 		}
1578 
1579 		dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
1580 		extent_end = key.offset + btrfs_dev_extent_length(l,
1581 								  dev_extent);
1582 		if (extent_end > search_start)
1583 			search_start = extent_end;
1584 next:
1585 		path->slots[0]++;
1586 		cond_resched();
1587 	}
1588 
1589 	/*
1590 	 * At this point, search_start should be the end of
1591 	 * allocated dev extents, and when shrinking the device,
1592 	 * search_end may be smaller than search_start.
1593 	 */
1594 	if (search_end > search_start) {
1595 		hole_size = search_end - search_start;
1596 		if (dev_extent_hole_check(device, &search_start, &hole_size,
1597 					  num_bytes)) {
1598 			btrfs_release_path(path);
1599 			goto again;
1600 		}
1601 
1602 		if (hole_size > max_hole_size) {
1603 			max_hole_start = search_start;
1604 			max_hole_size = hole_size;
1605 		}
1606 	}
1607 
1608 	/* See above. */
1609 	if (max_hole_size < num_bytes)
1610 		ret = -ENOSPC;
1611 	else
1612 		ret = 0;
1613 
1614 out:
1615 	btrfs_free_path(path);
1616 	*start = max_hole_start;
1617 	if (len)
1618 		*len = max_hole_size;
1619 	return ret;
1620 }
1621 
1622 int find_free_dev_extent(struct btrfs_device *device, u64 num_bytes,
1623 			 u64 *start, u64 *len)
1624 {
1625 	/* FIXME use last free of some kind */
1626 	return find_free_dev_extent_start(device, num_bytes, 0, start, len);
1627 }
1628 
1629 static int btrfs_free_dev_extent(struct btrfs_trans_handle *trans,
1630 			  struct btrfs_device *device,
1631 			  u64 start, u64 *dev_extent_len)
1632 {
1633 	struct btrfs_fs_info *fs_info = device->fs_info;
1634 	struct btrfs_root *root = fs_info->dev_root;
1635 	int ret;
1636 	struct btrfs_path *path;
1637 	struct btrfs_key key;
1638 	struct btrfs_key found_key;
1639 	struct extent_buffer *leaf = NULL;
1640 	struct btrfs_dev_extent *extent = NULL;
1641 
1642 	path = btrfs_alloc_path();
1643 	if (!path)
1644 		return -ENOMEM;
1645 
1646 	key.objectid = device->devid;
1647 	key.offset = start;
1648 	key.type = BTRFS_DEV_EXTENT_KEY;
1649 again:
1650 	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1651 	if (ret > 0) {
1652 		ret = btrfs_previous_item(root, path, key.objectid,
1653 					  BTRFS_DEV_EXTENT_KEY);
1654 		if (ret)
1655 			goto out;
1656 		leaf = path->nodes[0];
1657 		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
1658 		extent = btrfs_item_ptr(leaf, path->slots[0],
1659 					struct btrfs_dev_extent);
1660 		BUG_ON(found_key.offset > start || found_key.offset +
1661 		       btrfs_dev_extent_length(leaf, extent) < start);
1662 		key = found_key;
1663 		btrfs_release_path(path);
1664 		goto again;
1665 	} else if (ret == 0) {
1666 		leaf = path->nodes[0];
1667 		extent = btrfs_item_ptr(leaf, path->slots[0],
1668 					struct btrfs_dev_extent);
1669 	} else {
1670 		btrfs_handle_fs_error(fs_info, ret, "Slot search failed");
1671 		goto out;
1672 	}
1673 
1674 	*dev_extent_len = btrfs_dev_extent_length(leaf, extent);
1675 
1676 	ret = btrfs_del_item(trans, root, path);
1677 	if (ret) {
1678 		btrfs_handle_fs_error(fs_info, ret,
1679 				      "Failed to remove dev extent item");
1680 	} else {
1681 		set_bit(BTRFS_TRANS_HAVE_FREE_BGS, &trans->transaction->flags);
1682 	}
1683 out:
1684 	btrfs_free_path(path);
1685 	return ret;
1686 }
1687 
1688 static int btrfs_alloc_dev_extent(struct btrfs_trans_handle *trans,
1689 				  struct btrfs_device *device,
1690 				  u64 chunk_offset, u64 start, u64 num_bytes)
1691 {
1692 	int ret;
1693 	struct btrfs_path *path;
1694 	struct btrfs_fs_info *fs_info = device->fs_info;
1695 	struct btrfs_root *root = fs_info->dev_root;
1696 	struct btrfs_dev_extent *extent;
1697 	struct extent_buffer *leaf;
1698 	struct btrfs_key key;
1699 
1700 	WARN_ON(!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state));
1701 	WARN_ON(test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state));
1702 	path = btrfs_alloc_path();
1703 	if (!path)
1704 		return -ENOMEM;
1705 
1706 	key.objectid = device->devid;
1707 	key.offset = start;
1708 	key.type = BTRFS_DEV_EXTENT_KEY;
1709 	ret = btrfs_insert_empty_item(trans, root, path, &key,
1710 				      sizeof(*extent));
1711 	if (ret)
1712 		goto out;
1713 
1714 	leaf = path->nodes[0];
1715 	extent = btrfs_item_ptr(leaf, path->slots[0],
1716 				struct btrfs_dev_extent);
1717 	btrfs_set_dev_extent_chunk_tree(leaf, extent,
1718 					BTRFS_CHUNK_TREE_OBJECTID);
1719 	btrfs_set_dev_extent_chunk_objectid(leaf, extent,
1720 					    BTRFS_FIRST_CHUNK_TREE_OBJECTID);
1721 	btrfs_set_dev_extent_chunk_offset(leaf, extent, chunk_offset);
1722 
1723 	btrfs_set_dev_extent_length(leaf, extent, num_bytes);
1724 	btrfs_mark_buffer_dirty(leaf);
1725 out:
1726 	btrfs_free_path(path);
1727 	return ret;
1728 }
1729 
1730 static u64 find_next_chunk(struct btrfs_fs_info *fs_info)
1731 {
1732 	struct extent_map_tree *em_tree;
1733 	struct extent_map *em;
1734 	struct rb_node *n;
1735 	u64 ret = 0;
1736 
1737 	em_tree = &fs_info->mapping_tree;
1738 	read_lock(&em_tree->lock);
1739 	n = rb_last(&em_tree->map.rb_root);
1740 	if (n) {
1741 		em = rb_entry(n, struct extent_map, rb_node);
1742 		ret = em->start + em->len;
1743 	}
1744 	read_unlock(&em_tree->lock);
1745 
1746 	return ret;
1747 }
1748 
1749 static noinline int find_next_devid(struct btrfs_fs_info *fs_info,
1750 				    u64 *devid_ret)
1751 {
1752 	int ret;
1753 	struct btrfs_key key;
1754 	struct btrfs_key found_key;
1755 	struct btrfs_path *path;
1756 
1757 	path = btrfs_alloc_path();
1758 	if (!path)
1759 		return -ENOMEM;
1760 
1761 	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1762 	key.type = BTRFS_DEV_ITEM_KEY;
1763 	key.offset = (u64)-1;
1764 
1765 	ret = btrfs_search_slot(NULL, fs_info->chunk_root, &key, path, 0, 0);
1766 	if (ret < 0)
1767 		goto error;
1768 
1769 	if (ret == 0) {
1770 		/* Corruption */
1771 		btrfs_err(fs_info, "corrupted chunk tree devid -1 matched");
1772 		ret = -EUCLEAN;
1773 		goto error;
1774 	}
1775 
1776 	ret = btrfs_previous_item(fs_info->chunk_root, path,
1777 				  BTRFS_DEV_ITEMS_OBJECTID,
1778 				  BTRFS_DEV_ITEM_KEY);
1779 	if (ret) {
1780 		*devid_ret = 1;
1781 	} else {
1782 		btrfs_item_key_to_cpu(path->nodes[0], &found_key,
1783 				      path->slots[0]);
1784 		*devid_ret = found_key.offset + 1;
1785 	}
1786 	ret = 0;
1787 error:
1788 	btrfs_free_path(path);
1789 	return ret;
1790 }
1791 
1792 /*
1793  * the device information is stored in the chunk root
1794  * the btrfs_device struct should be fully filled in
1795  */
1796 static int btrfs_add_dev_item(struct btrfs_trans_handle *trans,
1797 			    struct btrfs_device *device)
1798 {
1799 	int ret;
1800 	struct btrfs_path *path;
1801 	struct btrfs_dev_item *dev_item;
1802 	struct extent_buffer *leaf;
1803 	struct btrfs_key key;
1804 	unsigned long ptr;
1805 
1806 	path = btrfs_alloc_path();
1807 	if (!path)
1808 		return -ENOMEM;
1809 
1810 	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1811 	key.type = BTRFS_DEV_ITEM_KEY;
1812 	key.offset = device->devid;
1813 
1814 	ret = btrfs_insert_empty_item(trans, trans->fs_info->chunk_root, path,
1815 				      &key, sizeof(*dev_item));
1816 	if (ret)
1817 		goto out;
1818 
1819 	leaf = path->nodes[0];
1820 	dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item);
1821 
1822 	btrfs_set_device_id(leaf, dev_item, device->devid);
1823 	btrfs_set_device_generation(leaf, dev_item, 0);
1824 	btrfs_set_device_type(leaf, dev_item, device->type);
1825 	btrfs_set_device_io_align(leaf, dev_item, device->io_align);
1826 	btrfs_set_device_io_width(leaf, dev_item, device->io_width);
1827 	btrfs_set_device_sector_size(leaf, dev_item, device->sector_size);
1828 	btrfs_set_device_total_bytes(leaf, dev_item,
1829 				     btrfs_device_get_disk_total_bytes(device));
1830 	btrfs_set_device_bytes_used(leaf, dev_item,
1831 				    btrfs_device_get_bytes_used(device));
1832 	btrfs_set_device_group(leaf, dev_item, 0);
1833 	btrfs_set_device_seek_speed(leaf, dev_item, 0);
1834 	btrfs_set_device_bandwidth(leaf, dev_item, 0);
1835 	btrfs_set_device_start_offset(leaf, dev_item, 0);
1836 
1837 	ptr = btrfs_device_uuid(dev_item);
1838 	write_extent_buffer(leaf, device->uuid, ptr, BTRFS_UUID_SIZE);
1839 	ptr = btrfs_device_fsid(dev_item);
1840 	write_extent_buffer(leaf, trans->fs_info->fs_devices->metadata_uuid,
1841 			    ptr, BTRFS_FSID_SIZE);
1842 	btrfs_mark_buffer_dirty(leaf);
1843 
1844 	ret = 0;
1845 out:
1846 	btrfs_free_path(path);
1847 	return ret;
1848 }
1849 
1850 /*
1851  * Function to update ctime/mtime for a given device path.
1852  * Mainly used for ctime/mtime based probe like libblkid.
1853  */
1854 static void update_dev_time(const char *path_name)
1855 {
1856 	struct file *filp;
1857 
1858 	filp = filp_open(path_name, O_RDWR, 0);
1859 	if (IS_ERR(filp))
1860 		return;
1861 	file_update_time(filp);
1862 	filp_close(filp, NULL);
1863 }
1864 
1865 static int btrfs_rm_dev_item(struct btrfs_device *device)
1866 {
1867 	struct btrfs_root *root = device->fs_info->chunk_root;
1868 	int ret;
1869 	struct btrfs_path *path;
1870 	struct btrfs_key key;
1871 	struct btrfs_trans_handle *trans;
1872 
1873 	path = btrfs_alloc_path();
1874 	if (!path)
1875 		return -ENOMEM;
1876 
1877 	trans = btrfs_start_transaction(root, 0);
1878 	if (IS_ERR(trans)) {
1879 		btrfs_free_path(path);
1880 		return PTR_ERR(trans);
1881 	}
1882 	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1883 	key.type = BTRFS_DEV_ITEM_KEY;
1884 	key.offset = device->devid;
1885 
1886 	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1887 	if (ret) {
1888 		if (ret > 0)
1889 			ret = -ENOENT;
1890 		btrfs_abort_transaction(trans, ret);
1891 		btrfs_end_transaction(trans);
1892 		goto out;
1893 	}
1894 
1895 	ret = btrfs_del_item(trans, root, path);
1896 	if (ret) {
1897 		btrfs_abort_transaction(trans, ret);
1898 		btrfs_end_transaction(trans);
1899 	}
1900 
1901 out:
1902 	btrfs_free_path(path);
1903 	if (!ret)
1904 		ret = btrfs_commit_transaction(trans);
1905 	return ret;
1906 }
1907 
1908 /*
1909  * Verify that @num_devices satisfies the RAID profile constraints in the whole
1910  * filesystem. It's up to the caller to adjust that number regarding eg. device
1911  * replace.
1912  */
1913 static int btrfs_check_raid_min_devices(struct btrfs_fs_info *fs_info,
1914 		u64 num_devices)
1915 {
1916 	u64 all_avail;
1917 	unsigned seq;
1918 	int i;
1919 
1920 	do {
1921 		seq = read_seqbegin(&fs_info->profiles_lock);
1922 
1923 		all_avail = fs_info->avail_data_alloc_bits |
1924 			    fs_info->avail_system_alloc_bits |
1925 			    fs_info->avail_metadata_alloc_bits;
1926 	} while (read_seqretry(&fs_info->profiles_lock, seq));
1927 
1928 	for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) {
1929 		if (!(all_avail & btrfs_raid_array[i].bg_flag))
1930 			continue;
1931 
1932 		if (num_devices < btrfs_raid_array[i].devs_min) {
1933 			int ret = btrfs_raid_array[i].mindev_error;
1934 
1935 			if (ret)
1936 				return ret;
1937 		}
1938 	}
1939 
1940 	return 0;
1941 }
1942 
1943 static struct btrfs_device * btrfs_find_next_active_device(
1944 		struct btrfs_fs_devices *fs_devs, struct btrfs_device *device)
1945 {
1946 	struct btrfs_device *next_device;
1947 
1948 	list_for_each_entry(next_device, &fs_devs->devices, dev_list) {
1949 		if (next_device != device &&
1950 		    !test_bit(BTRFS_DEV_STATE_MISSING, &next_device->dev_state)
1951 		    && next_device->bdev)
1952 			return next_device;
1953 	}
1954 
1955 	return NULL;
1956 }
1957 
1958 /*
1959  * Helper function to check if the given device is part of s_bdev / latest_bdev
1960  * and replace it with the provided or the next active device, in the context
1961  * where this function called, there should be always be another device (or
1962  * this_dev) which is active.
1963  */
1964 void __cold btrfs_assign_next_active_device(struct btrfs_device *device,
1965 					    struct btrfs_device *next_device)
1966 {
1967 	struct btrfs_fs_info *fs_info = device->fs_info;
1968 
1969 	if (!next_device)
1970 		next_device = btrfs_find_next_active_device(fs_info->fs_devices,
1971 							    device);
1972 	ASSERT(next_device);
1973 
1974 	if (fs_info->sb->s_bdev &&
1975 			(fs_info->sb->s_bdev == device->bdev))
1976 		fs_info->sb->s_bdev = next_device->bdev;
1977 
1978 	if (fs_info->fs_devices->latest_bdev == device->bdev)
1979 		fs_info->fs_devices->latest_bdev = next_device->bdev;
1980 }
1981 
1982 /*
1983  * Return btrfs_fs_devices::num_devices excluding the device that's being
1984  * currently replaced.
1985  */
1986 static u64 btrfs_num_devices(struct btrfs_fs_info *fs_info)
1987 {
1988 	u64 num_devices = fs_info->fs_devices->num_devices;
1989 
1990 	down_read(&fs_info->dev_replace.rwsem);
1991 	if (btrfs_dev_replace_is_ongoing(&fs_info->dev_replace)) {
1992 		ASSERT(num_devices > 1);
1993 		num_devices--;
1994 	}
1995 	up_read(&fs_info->dev_replace.rwsem);
1996 
1997 	return num_devices;
1998 }
1999 
2000 void btrfs_scratch_superblocks(struct btrfs_fs_info *fs_info,
2001 			       struct block_device *bdev,
2002 			       const char *device_path)
2003 {
2004 	struct btrfs_super_block *disk_super;
2005 	int copy_num;
2006 
2007 	if (!bdev)
2008 		return;
2009 
2010 	for (copy_num = 0; copy_num < BTRFS_SUPER_MIRROR_MAX; copy_num++) {
2011 		struct page *page;
2012 		int ret;
2013 
2014 		disk_super = btrfs_read_dev_one_super(bdev, copy_num);
2015 		if (IS_ERR(disk_super))
2016 			continue;
2017 
2018 		memset(&disk_super->magic, 0, sizeof(disk_super->magic));
2019 
2020 		page = virt_to_page(disk_super);
2021 		set_page_dirty(page);
2022 		lock_page(page);
2023 		/* write_on_page() unlocks the page */
2024 		ret = write_one_page(page);
2025 		if (ret)
2026 			btrfs_warn(fs_info,
2027 				"error clearing superblock number %d (%d)",
2028 				copy_num, ret);
2029 		btrfs_release_disk_super(disk_super);
2030 
2031 	}
2032 
2033 	/* Notify udev that device has changed */
2034 	btrfs_kobject_uevent(bdev, KOBJ_CHANGE);
2035 
2036 	/* Update ctime/mtime for device path for libblkid */
2037 	update_dev_time(device_path);
2038 }
2039 
2040 int btrfs_rm_device(struct btrfs_fs_info *fs_info, const char *device_path,
2041 		    u64 devid)
2042 {
2043 	struct btrfs_device *device;
2044 	struct btrfs_fs_devices *cur_devices;
2045 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2046 	u64 num_devices;
2047 	int ret = 0;
2048 
2049 	mutex_lock(&uuid_mutex);
2050 
2051 	num_devices = btrfs_num_devices(fs_info);
2052 
2053 	ret = btrfs_check_raid_min_devices(fs_info, num_devices - 1);
2054 	if (ret)
2055 		goto out;
2056 
2057 	device = btrfs_find_device_by_devspec(fs_info, devid, device_path);
2058 
2059 	if (IS_ERR(device)) {
2060 		if (PTR_ERR(device) == -ENOENT &&
2061 		    strcmp(device_path, "missing") == 0)
2062 			ret = BTRFS_ERROR_DEV_MISSING_NOT_FOUND;
2063 		else
2064 			ret = PTR_ERR(device);
2065 		goto out;
2066 	}
2067 
2068 	if (btrfs_pinned_by_swapfile(fs_info, device)) {
2069 		btrfs_warn_in_rcu(fs_info,
2070 		  "cannot remove device %s (devid %llu) due to active swapfile",
2071 				  rcu_str_deref(device->name), device->devid);
2072 		ret = -ETXTBSY;
2073 		goto out;
2074 	}
2075 
2076 	if (test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
2077 		ret = BTRFS_ERROR_DEV_TGT_REPLACE;
2078 		goto out;
2079 	}
2080 
2081 	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
2082 	    fs_info->fs_devices->rw_devices == 1) {
2083 		ret = BTRFS_ERROR_DEV_ONLY_WRITABLE;
2084 		goto out;
2085 	}
2086 
2087 	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
2088 		mutex_lock(&fs_info->chunk_mutex);
2089 		list_del_init(&device->dev_alloc_list);
2090 		device->fs_devices->rw_devices--;
2091 		mutex_unlock(&fs_info->chunk_mutex);
2092 	}
2093 
2094 	mutex_unlock(&uuid_mutex);
2095 	ret = btrfs_shrink_device(device, 0);
2096 	if (!ret)
2097 		btrfs_reada_remove_dev(device);
2098 	mutex_lock(&uuid_mutex);
2099 	if (ret)
2100 		goto error_undo;
2101 
2102 	/*
2103 	 * TODO: the superblock still includes this device in its num_devices
2104 	 * counter although write_all_supers() is not locked out. This
2105 	 * could give a filesystem state which requires a degraded mount.
2106 	 */
2107 	ret = btrfs_rm_dev_item(device);
2108 	if (ret)
2109 		goto error_undo;
2110 
2111 	clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
2112 	btrfs_scrub_cancel_dev(device);
2113 
2114 	/*
2115 	 * the device list mutex makes sure that we don't change
2116 	 * the device list while someone else is writing out all
2117 	 * the device supers. Whoever is writing all supers, should
2118 	 * lock the device list mutex before getting the number of
2119 	 * devices in the super block (super_copy). Conversely,
2120 	 * whoever updates the number of devices in the super block
2121 	 * (super_copy) should hold the device list mutex.
2122 	 */
2123 
2124 	/*
2125 	 * In normal cases the cur_devices == fs_devices. But in case
2126 	 * of deleting a seed device, the cur_devices should point to
2127 	 * its own fs_devices listed under the fs_devices->seed.
2128 	 */
2129 	cur_devices = device->fs_devices;
2130 	mutex_lock(&fs_devices->device_list_mutex);
2131 	list_del_rcu(&device->dev_list);
2132 
2133 	cur_devices->num_devices--;
2134 	cur_devices->total_devices--;
2135 	/* Update total_devices of the parent fs_devices if it's seed */
2136 	if (cur_devices != fs_devices)
2137 		fs_devices->total_devices--;
2138 
2139 	if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state))
2140 		cur_devices->missing_devices--;
2141 
2142 	btrfs_assign_next_active_device(device, NULL);
2143 
2144 	if (device->bdev) {
2145 		cur_devices->open_devices--;
2146 		/* remove sysfs entry */
2147 		btrfs_sysfs_remove_device(device);
2148 	}
2149 
2150 	num_devices = btrfs_super_num_devices(fs_info->super_copy) - 1;
2151 	btrfs_set_super_num_devices(fs_info->super_copy, num_devices);
2152 	mutex_unlock(&fs_devices->device_list_mutex);
2153 
2154 	/*
2155 	 * at this point, the device is zero sized and detached from
2156 	 * the devices list.  All that's left is to zero out the old
2157 	 * supers and free the device.
2158 	 */
2159 	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state))
2160 		btrfs_scratch_superblocks(fs_info, device->bdev,
2161 					  device->name->str);
2162 
2163 	btrfs_close_bdev(device);
2164 	synchronize_rcu();
2165 	btrfs_free_device(device);
2166 
2167 	if (cur_devices->open_devices == 0) {
2168 		list_del_init(&cur_devices->seed_list);
2169 		close_fs_devices(cur_devices);
2170 		free_fs_devices(cur_devices);
2171 	}
2172 
2173 out:
2174 	mutex_unlock(&uuid_mutex);
2175 	return ret;
2176 
2177 error_undo:
2178 	btrfs_reada_undo_remove_dev(device);
2179 	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
2180 		mutex_lock(&fs_info->chunk_mutex);
2181 		list_add(&device->dev_alloc_list,
2182 			 &fs_devices->alloc_list);
2183 		device->fs_devices->rw_devices++;
2184 		mutex_unlock(&fs_info->chunk_mutex);
2185 	}
2186 	goto out;
2187 }
2188 
2189 void btrfs_rm_dev_replace_remove_srcdev(struct btrfs_device *srcdev)
2190 {
2191 	struct btrfs_fs_devices *fs_devices;
2192 
2193 	lockdep_assert_held(&srcdev->fs_info->fs_devices->device_list_mutex);
2194 
2195 	/*
2196 	 * in case of fs with no seed, srcdev->fs_devices will point
2197 	 * to fs_devices of fs_info. However when the dev being replaced is
2198 	 * a seed dev it will point to the seed's local fs_devices. In short
2199 	 * srcdev will have its correct fs_devices in both the cases.
2200 	 */
2201 	fs_devices = srcdev->fs_devices;
2202 
2203 	list_del_rcu(&srcdev->dev_list);
2204 	list_del(&srcdev->dev_alloc_list);
2205 	fs_devices->num_devices--;
2206 	if (test_bit(BTRFS_DEV_STATE_MISSING, &srcdev->dev_state))
2207 		fs_devices->missing_devices--;
2208 
2209 	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &srcdev->dev_state))
2210 		fs_devices->rw_devices--;
2211 
2212 	if (srcdev->bdev)
2213 		fs_devices->open_devices--;
2214 }
2215 
2216 void btrfs_rm_dev_replace_free_srcdev(struct btrfs_device *srcdev)
2217 {
2218 	struct btrfs_fs_devices *fs_devices = srcdev->fs_devices;
2219 
2220 	mutex_lock(&uuid_mutex);
2221 
2222 	btrfs_close_bdev(srcdev);
2223 	synchronize_rcu();
2224 	btrfs_free_device(srcdev);
2225 
2226 	/* if this is no devs we rather delete the fs_devices */
2227 	if (!fs_devices->num_devices) {
2228 		/*
2229 		 * On a mounted FS, num_devices can't be zero unless it's a
2230 		 * seed. In case of a seed device being replaced, the replace
2231 		 * target added to the sprout FS, so there will be no more
2232 		 * device left under the seed FS.
2233 		 */
2234 		ASSERT(fs_devices->seeding);
2235 
2236 		list_del_init(&fs_devices->seed_list);
2237 		close_fs_devices(fs_devices);
2238 		free_fs_devices(fs_devices);
2239 	}
2240 	mutex_unlock(&uuid_mutex);
2241 }
2242 
2243 void btrfs_destroy_dev_replace_tgtdev(struct btrfs_device *tgtdev)
2244 {
2245 	struct btrfs_fs_devices *fs_devices = tgtdev->fs_info->fs_devices;
2246 
2247 	mutex_lock(&fs_devices->device_list_mutex);
2248 
2249 	btrfs_sysfs_remove_device(tgtdev);
2250 
2251 	if (tgtdev->bdev)
2252 		fs_devices->open_devices--;
2253 
2254 	fs_devices->num_devices--;
2255 
2256 	btrfs_assign_next_active_device(tgtdev, NULL);
2257 
2258 	list_del_rcu(&tgtdev->dev_list);
2259 
2260 	mutex_unlock(&fs_devices->device_list_mutex);
2261 
2262 	/*
2263 	 * The update_dev_time() with in btrfs_scratch_superblocks()
2264 	 * may lead to a call to btrfs_show_devname() which will try
2265 	 * to hold device_list_mutex. And here this device
2266 	 * is already out of device list, so we don't have to hold
2267 	 * the device_list_mutex lock.
2268 	 */
2269 	btrfs_scratch_superblocks(tgtdev->fs_info, tgtdev->bdev,
2270 				  tgtdev->name->str);
2271 
2272 	btrfs_close_bdev(tgtdev);
2273 	synchronize_rcu();
2274 	btrfs_free_device(tgtdev);
2275 }
2276 
2277 static struct btrfs_device *btrfs_find_device_by_path(
2278 		struct btrfs_fs_info *fs_info, const char *device_path)
2279 {
2280 	int ret = 0;
2281 	struct btrfs_super_block *disk_super;
2282 	u64 devid;
2283 	u8 *dev_uuid;
2284 	struct block_device *bdev;
2285 	struct btrfs_device *device;
2286 
2287 	ret = btrfs_get_bdev_and_sb(device_path, FMODE_READ,
2288 				    fs_info->bdev_holder, 0, &bdev, &disk_super);
2289 	if (ret)
2290 		return ERR_PTR(ret);
2291 
2292 	devid = btrfs_stack_device_id(&disk_super->dev_item);
2293 	dev_uuid = disk_super->dev_item.uuid;
2294 	if (btrfs_fs_incompat(fs_info, METADATA_UUID))
2295 		device = btrfs_find_device(fs_info->fs_devices, devid, dev_uuid,
2296 					   disk_super->metadata_uuid, true);
2297 	else
2298 		device = btrfs_find_device(fs_info->fs_devices, devid, dev_uuid,
2299 					   disk_super->fsid, true);
2300 
2301 	btrfs_release_disk_super(disk_super);
2302 	if (!device)
2303 		device = ERR_PTR(-ENOENT);
2304 	blkdev_put(bdev, FMODE_READ);
2305 	return device;
2306 }
2307 
2308 /*
2309  * Lookup a device given by device id, or the path if the id is 0.
2310  */
2311 struct btrfs_device *btrfs_find_device_by_devspec(
2312 		struct btrfs_fs_info *fs_info, u64 devid,
2313 		const char *device_path)
2314 {
2315 	struct btrfs_device *device;
2316 
2317 	if (devid) {
2318 		device = btrfs_find_device(fs_info->fs_devices, devid, NULL,
2319 					   NULL, true);
2320 		if (!device)
2321 			return ERR_PTR(-ENOENT);
2322 		return device;
2323 	}
2324 
2325 	if (!device_path || !device_path[0])
2326 		return ERR_PTR(-EINVAL);
2327 
2328 	if (strcmp(device_path, "missing") == 0) {
2329 		/* Find first missing device */
2330 		list_for_each_entry(device, &fs_info->fs_devices->devices,
2331 				    dev_list) {
2332 			if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
2333 				     &device->dev_state) && !device->bdev)
2334 				return device;
2335 		}
2336 		return ERR_PTR(-ENOENT);
2337 	}
2338 
2339 	return btrfs_find_device_by_path(fs_info, device_path);
2340 }
2341 
2342 /*
2343  * does all the dirty work required for changing file system's UUID.
2344  */
2345 static int btrfs_prepare_sprout(struct btrfs_fs_info *fs_info)
2346 {
2347 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2348 	struct btrfs_fs_devices *old_devices;
2349 	struct btrfs_fs_devices *seed_devices;
2350 	struct btrfs_super_block *disk_super = fs_info->super_copy;
2351 	struct btrfs_device *device;
2352 	u64 super_flags;
2353 
2354 	lockdep_assert_held(&uuid_mutex);
2355 	if (!fs_devices->seeding)
2356 		return -EINVAL;
2357 
2358 	/*
2359 	 * Private copy of the seed devices, anchored at
2360 	 * fs_info->fs_devices->seed_list
2361 	 */
2362 	seed_devices = alloc_fs_devices(NULL, NULL);
2363 	if (IS_ERR(seed_devices))
2364 		return PTR_ERR(seed_devices);
2365 
2366 	/*
2367 	 * It's necessary to retain a copy of the original seed fs_devices in
2368 	 * fs_uuids so that filesystems which have been seeded can successfully
2369 	 * reference the seed device from open_seed_devices. This also supports
2370 	 * multiple fs seed.
2371 	 */
2372 	old_devices = clone_fs_devices(fs_devices);
2373 	if (IS_ERR(old_devices)) {
2374 		kfree(seed_devices);
2375 		return PTR_ERR(old_devices);
2376 	}
2377 
2378 	list_add(&old_devices->fs_list, &fs_uuids);
2379 
2380 	memcpy(seed_devices, fs_devices, sizeof(*seed_devices));
2381 	seed_devices->opened = 1;
2382 	INIT_LIST_HEAD(&seed_devices->devices);
2383 	INIT_LIST_HEAD(&seed_devices->alloc_list);
2384 	mutex_init(&seed_devices->device_list_mutex);
2385 
2386 	mutex_lock(&fs_devices->device_list_mutex);
2387 	list_splice_init_rcu(&fs_devices->devices, &seed_devices->devices,
2388 			      synchronize_rcu);
2389 	list_for_each_entry(device, &seed_devices->devices, dev_list)
2390 		device->fs_devices = seed_devices;
2391 
2392 	fs_devices->seeding = false;
2393 	fs_devices->num_devices = 0;
2394 	fs_devices->open_devices = 0;
2395 	fs_devices->missing_devices = 0;
2396 	fs_devices->rotating = false;
2397 	list_add(&seed_devices->seed_list, &fs_devices->seed_list);
2398 
2399 	generate_random_uuid(fs_devices->fsid);
2400 	memcpy(fs_devices->metadata_uuid, fs_devices->fsid, BTRFS_FSID_SIZE);
2401 	memcpy(disk_super->fsid, fs_devices->fsid, BTRFS_FSID_SIZE);
2402 	mutex_unlock(&fs_devices->device_list_mutex);
2403 
2404 	super_flags = btrfs_super_flags(disk_super) &
2405 		      ~BTRFS_SUPER_FLAG_SEEDING;
2406 	btrfs_set_super_flags(disk_super, super_flags);
2407 
2408 	return 0;
2409 }
2410 
2411 /*
2412  * Store the expected generation for seed devices in device items.
2413  */
2414 static int btrfs_finish_sprout(struct btrfs_trans_handle *trans)
2415 {
2416 	struct btrfs_fs_info *fs_info = trans->fs_info;
2417 	struct btrfs_root *root = fs_info->chunk_root;
2418 	struct btrfs_path *path;
2419 	struct extent_buffer *leaf;
2420 	struct btrfs_dev_item *dev_item;
2421 	struct btrfs_device *device;
2422 	struct btrfs_key key;
2423 	u8 fs_uuid[BTRFS_FSID_SIZE];
2424 	u8 dev_uuid[BTRFS_UUID_SIZE];
2425 	u64 devid;
2426 	int ret;
2427 
2428 	path = btrfs_alloc_path();
2429 	if (!path)
2430 		return -ENOMEM;
2431 
2432 	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
2433 	key.offset = 0;
2434 	key.type = BTRFS_DEV_ITEM_KEY;
2435 
2436 	while (1) {
2437 		ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2438 		if (ret < 0)
2439 			goto error;
2440 
2441 		leaf = path->nodes[0];
2442 next_slot:
2443 		if (path->slots[0] >= btrfs_header_nritems(leaf)) {
2444 			ret = btrfs_next_leaf(root, path);
2445 			if (ret > 0)
2446 				break;
2447 			if (ret < 0)
2448 				goto error;
2449 			leaf = path->nodes[0];
2450 			btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2451 			btrfs_release_path(path);
2452 			continue;
2453 		}
2454 
2455 		btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2456 		if (key.objectid != BTRFS_DEV_ITEMS_OBJECTID ||
2457 		    key.type != BTRFS_DEV_ITEM_KEY)
2458 			break;
2459 
2460 		dev_item = btrfs_item_ptr(leaf, path->slots[0],
2461 					  struct btrfs_dev_item);
2462 		devid = btrfs_device_id(leaf, dev_item);
2463 		read_extent_buffer(leaf, dev_uuid, btrfs_device_uuid(dev_item),
2464 				   BTRFS_UUID_SIZE);
2465 		read_extent_buffer(leaf, fs_uuid, btrfs_device_fsid(dev_item),
2466 				   BTRFS_FSID_SIZE);
2467 		device = btrfs_find_device(fs_info->fs_devices, devid, dev_uuid,
2468 					   fs_uuid, true);
2469 		BUG_ON(!device); /* Logic error */
2470 
2471 		if (device->fs_devices->seeding) {
2472 			btrfs_set_device_generation(leaf, dev_item,
2473 						    device->generation);
2474 			btrfs_mark_buffer_dirty(leaf);
2475 		}
2476 
2477 		path->slots[0]++;
2478 		goto next_slot;
2479 	}
2480 	ret = 0;
2481 error:
2482 	btrfs_free_path(path);
2483 	return ret;
2484 }
2485 
2486 int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path)
2487 {
2488 	struct btrfs_root *root = fs_info->dev_root;
2489 	struct request_queue *q;
2490 	struct btrfs_trans_handle *trans;
2491 	struct btrfs_device *device;
2492 	struct block_device *bdev;
2493 	struct super_block *sb = fs_info->sb;
2494 	struct rcu_string *name;
2495 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2496 	u64 orig_super_total_bytes;
2497 	u64 orig_super_num_devices;
2498 	int seeding_dev = 0;
2499 	int ret = 0;
2500 	bool locked = false;
2501 
2502 	if (sb_rdonly(sb) && !fs_devices->seeding)
2503 		return -EROFS;
2504 
2505 	bdev = blkdev_get_by_path(device_path, FMODE_WRITE | FMODE_EXCL,
2506 				  fs_info->bdev_holder);
2507 	if (IS_ERR(bdev))
2508 		return PTR_ERR(bdev);
2509 
2510 	if (fs_devices->seeding) {
2511 		seeding_dev = 1;
2512 		down_write(&sb->s_umount);
2513 		mutex_lock(&uuid_mutex);
2514 		locked = true;
2515 	}
2516 
2517 	sync_blockdev(bdev);
2518 
2519 	rcu_read_lock();
2520 	list_for_each_entry_rcu(device, &fs_devices->devices, dev_list) {
2521 		if (device->bdev == bdev) {
2522 			ret = -EEXIST;
2523 			rcu_read_unlock();
2524 			goto error;
2525 		}
2526 	}
2527 	rcu_read_unlock();
2528 
2529 	device = btrfs_alloc_device(fs_info, NULL, NULL);
2530 	if (IS_ERR(device)) {
2531 		/* we can safely leave the fs_devices entry around */
2532 		ret = PTR_ERR(device);
2533 		goto error;
2534 	}
2535 
2536 	name = rcu_string_strdup(device_path, GFP_KERNEL);
2537 	if (!name) {
2538 		ret = -ENOMEM;
2539 		goto error_free_device;
2540 	}
2541 	rcu_assign_pointer(device->name, name);
2542 
2543 	trans = btrfs_start_transaction(root, 0);
2544 	if (IS_ERR(trans)) {
2545 		ret = PTR_ERR(trans);
2546 		goto error_free_device;
2547 	}
2548 
2549 	q = bdev_get_queue(bdev);
2550 	set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
2551 	device->generation = trans->transid;
2552 	device->io_width = fs_info->sectorsize;
2553 	device->io_align = fs_info->sectorsize;
2554 	device->sector_size = fs_info->sectorsize;
2555 	device->total_bytes = round_down(i_size_read(bdev->bd_inode),
2556 					 fs_info->sectorsize);
2557 	device->disk_total_bytes = device->total_bytes;
2558 	device->commit_total_bytes = device->total_bytes;
2559 	device->fs_info = fs_info;
2560 	device->bdev = bdev;
2561 	set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
2562 	clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state);
2563 	device->mode = FMODE_EXCL;
2564 	device->dev_stats_valid = 1;
2565 	set_blocksize(device->bdev, BTRFS_BDEV_BLOCKSIZE);
2566 
2567 	if (seeding_dev) {
2568 		sb->s_flags &= ~SB_RDONLY;
2569 		ret = btrfs_prepare_sprout(fs_info);
2570 		if (ret) {
2571 			btrfs_abort_transaction(trans, ret);
2572 			goto error_trans;
2573 		}
2574 	}
2575 
2576 	device->fs_devices = fs_devices;
2577 
2578 	mutex_lock(&fs_devices->device_list_mutex);
2579 	mutex_lock(&fs_info->chunk_mutex);
2580 	list_add_rcu(&device->dev_list, &fs_devices->devices);
2581 	list_add(&device->dev_alloc_list, &fs_devices->alloc_list);
2582 	fs_devices->num_devices++;
2583 	fs_devices->open_devices++;
2584 	fs_devices->rw_devices++;
2585 	fs_devices->total_devices++;
2586 	fs_devices->total_rw_bytes += device->total_bytes;
2587 
2588 	atomic64_add(device->total_bytes, &fs_info->free_chunk_space);
2589 
2590 	if (!blk_queue_nonrot(q))
2591 		fs_devices->rotating = true;
2592 
2593 	orig_super_total_bytes = btrfs_super_total_bytes(fs_info->super_copy);
2594 	btrfs_set_super_total_bytes(fs_info->super_copy,
2595 		round_down(orig_super_total_bytes + device->total_bytes,
2596 			   fs_info->sectorsize));
2597 
2598 	orig_super_num_devices = btrfs_super_num_devices(fs_info->super_copy);
2599 	btrfs_set_super_num_devices(fs_info->super_copy,
2600 				    orig_super_num_devices + 1);
2601 
2602 	/*
2603 	 * we've got more storage, clear any full flags on the space
2604 	 * infos
2605 	 */
2606 	btrfs_clear_space_info_full(fs_info);
2607 
2608 	mutex_unlock(&fs_info->chunk_mutex);
2609 
2610 	/* Add sysfs device entry */
2611 	btrfs_sysfs_add_device(device);
2612 
2613 	mutex_unlock(&fs_devices->device_list_mutex);
2614 
2615 	if (seeding_dev) {
2616 		mutex_lock(&fs_info->chunk_mutex);
2617 		ret = init_first_rw_device(trans);
2618 		mutex_unlock(&fs_info->chunk_mutex);
2619 		if (ret) {
2620 			btrfs_abort_transaction(trans, ret);
2621 			goto error_sysfs;
2622 		}
2623 	}
2624 
2625 	ret = btrfs_add_dev_item(trans, device);
2626 	if (ret) {
2627 		btrfs_abort_transaction(trans, ret);
2628 		goto error_sysfs;
2629 	}
2630 
2631 	if (seeding_dev) {
2632 		ret = btrfs_finish_sprout(trans);
2633 		if (ret) {
2634 			btrfs_abort_transaction(trans, ret);
2635 			goto error_sysfs;
2636 		}
2637 
2638 		/*
2639 		 * fs_devices now represents the newly sprouted filesystem and
2640 		 * its fsid has been changed by btrfs_prepare_sprout
2641 		 */
2642 		btrfs_sysfs_update_sprout_fsid(fs_devices);
2643 	}
2644 
2645 	ret = btrfs_commit_transaction(trans);
2646 
2647 	if (seeding_dev) {
2648 		mutex_unlock(&uuid_mutex);
2649 		up_write(&sb->s_umount);
2650 		locked = false;
2651 
2652 		if (ret) /* transaction commit */
2653 			return ret;
2654 
2655 		ret = btrfs_relocate_sys_chunks(fs_info);
2656 		if (ret < 0)
2657 			btrfs_handle_fs_error(fs_info, ret,
2658 				    "Failed to relocate sys chunks after device initialization. This can be fixed using the \"btrfs balance\" command.");
2659 		trans = btrfs_attach_transaction(root);
2660 		if (IS_ERR(trans)) {
2661 			if (PTR_ERR(trans) == -ENOENT)
2662 				return 0;
2663 			ret = PTR_ERR(trans);
2664 			trans = NULL;
2665 			goto error_sysfs;
2666 		}
2667 		ret = btrfs_commit_transaction(trans);
2668 	}
2669 
2670 	/*
2671 	 * Now that we have written a new super block to this device, check all
2672 	 * other fs_devices list if device_path alienates any other scanned
2673 	 * device.
2674 	 * We can ignore the return value as it typically returns -EINVAL and
2675 	 * only succeeds if the device was an alien.
2676 	 */
2677 	btrfs_forget_devices(device_path);
2678 
2679 	/* Update ctime/mtime for blkid or udev */
2680 	update_dev_time(device_path);
2681 
2682 	return ret;
2683 
2684 error_sysfs:
2685 	btrfs_sysfs_remove_device(device);
2686 	mutex_lock(&fs_info->fs_devices->device_list_mutex);
2687 	mutex_lock(&fs_info->chunk_mutex);
2688 	list_del_rcu(&device->dev_list);
2689 	list_del(&device->dev_alloc_list);
2690 	fs_info->fs_devices->num_devices--;
2691 	fs_info->fs_devices->open_devices--;
2692 	fs_info->fs_devices->rw_devices--;
2693 	fs_info->fs_devices->total_devices--;
2694 	fs_info->fs_devices->total_rw_bytes -= device->total_bytes;
2695 	atomic64_sub(device->total_bytes, &fs_info->free_chunk_space);
2696 	btrfs_set_super_total_bytes(fs_info->super_copy,
2697 				    orig_super_total_bytes);
2698 	btrfs_set_super_num_devices(fs_info->super_copy,
2699 				    orig_super_num_devices);
2700 	mutex_unlock(&fs_info->chunk_mutex);
2701 	mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2702 error_trans:
2703 	if (seeding_dev)
2704 		sb->s_flags |= SB_RDONLY;
2705 	if (trans)
2706 		btrfs_end_transaction(trans);
2707 error_free_device:
2708 	btrfs_free_device(device);
2709 error:
2710 	blkdev_put(bdev, FMODE_EXCL);
2711 	if (locked) {
2712 		mutex_unlock(&uuid_mutex);
2713 		up_write(&sb->s_umount);
2714 	}
2715 	return ret;
2716 }
2717 
2718 static noinline int btrfs_update_device(struct btrfs_trans_handle *trans,
2719 					struct btrfs_device *device)
2720 {
2721 	int ret;
2722 	struct btrfs_path *path;
2723 	struct btrfs_root *root = device->fs_info->chunk_root;
2724 	struct btrfs_dev_item *dev_item;
2725 	struct extent_buffer *leaf;
2726 	struct btrfs_key key;
2727 
2728 	path = btrfs_alloc_path();
2729 	if (!path)
2730 		return -ENOMEM;
2731 
2732 	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
2733 	key.type = BTRFS_DEV_ITEM_KEY;
2734 	key.offset = device->devid;
2735 
2736 	ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2737 	if (ret < 0)
2738 		goto out;
2739 
2740 	if (ret > 0) {
2741 		ret = -ENOENT;
2742 		goto out;
2743 	}
2744 
2745 	leaf = path->nodes[0];
2746 	dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item);
2747 
2748 	btrfs_set_device_id(leaf, dev_item, device->devid);
2749 	btrfs_set_device_type(leaf, dev_item, device->type);
2750 	btrfs_set_device_io_align(leaf, dev_item, device->io_align);
2751 	btrfs_set_device_io_width(leaf, dev_item, device->io_width);
2752 	btrfs_set_device_sector_size(leaf, dev_item, device->sector_size);
2753 	btrfs_set_device_total_bytes(leaf, dev_item,
2754 				     btrfs_device_get_disk_total_bytes(device));
2755 	btrfs_set_device_bytes_used(leaf, dev_item,
2756 				    btrfs_device_get_bytes_used(device));
2757 	btrfs_mark_buffer_dirty(leaf);
2758 
2759 out:
2760 	btrfs_free_path(path);
2761 	return ret;
2762 }
2763 
2764 int btrfs_grow_device(struct btrfs_trans_handle *trans,
2765 		      struct btrfs_device *device, u64 new_size)
2766 {
2767 	struct btrfs_fs_info *fs_info = device->fs_info;
2768 	struct btrfs_super_block *super_copy = fs_info->super_copy;
2769 	u64 old_total;
2770 	u64 diff;
2771 
2772 	if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state))
2773 		return -EACCES;
2774 
2775 	new_size = round_down(new_size, fs_info->sectorsize);
2776 
2777 	mutex_lock(&fs_info->chunk_mutex);
2778 	old_total = btrfs_super_total_bytes(super_copy);
2779 	diff = round_down(new_size - device->total_bytes, fs_info->sectorsize);
2780 
2781 	if (new_size <= device->total_bytes ||
2782 	    test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
2783 		mutex_unlock(&fs_info->chunk_mutex);
2784 		return -EINVAL;
2785 	}
2786 
2787 	btrfs_set_super_total_bytes(super_copy,
2788 			round_down(old_total + diff, fs_info->sectorsize));
2789 	device->fs_devices->total_rw_bytes += diff;
2790 
2791 	btrfs_device_set_total_bytes(device, new_size);
2792 	btrfs_device_set_disk_total_bytes(device, new_size);
2793 	btrfs_clear_space_info_full(device->fs_info);
2794 	if (list_empty(&device->post_commit_list))
2795 		list_add_tail(&device->post_commit_list,
2796 			      &trans->transaction->dev_update_list);
2797 	mutex_unlock(&fs_info->chunk_mutex);
2798 
2799 	return btrfs_update_device(trans, device);
2800 }
2801 
2802 static int btrfs_free_chunk(struct btrfs_trans_handle *trans, u64 chunk_offset)
2803 {
2804 	struct btrfs_fs_info *fs_info = trans->fs_info;
2805 	struct btrfs_root *root = fs_info->chunk_root;
2806 	int ret;
2807 	struct btrfs_path *path;
2808 	struct btrfs_key key;
2809 
2810 	path = btrfs_alloc_path();
2811 	if (!path)
2812 		return -ENOMEM;
2813 
2814 	key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
2815 	key.offset = chunk_offset;
2816 	key.type = BTRFS_CHUNK_ITEM_KEY;
2817 
2818 	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
2819 	if (ret < 0)
2820 		goto out;
2821 	else if (ret > 0) { /* Logic error or corruption */
2822 		btrfs_handle_fs_error(fs_info, -ENOENT,
2823 				      "Failed lookup while freeing chunk.");
2824 		ret = -ENOENT;
2825 		goto out;
2826 	}
2827 
2828 	ret = btrfs_del_item(trans, root, path);
2829 	if (ret < 0)
2830 		btrfs_handle_fs_error(fs_info, ret,
2831 				      "Failed to delete chunk item.");
2832 out:
2833 	btrfs_free_path(path);
2834 	return ret;
2835 }
2836 
2837 static int btrfs_del_sys_chunk(struct btrfs_fs_info *fs_info, u64 chunk_offset)
2838 {
2839 	struct btrfs_super_block *super_copy = fs_info->super_copy;
2840 	struct btrfs_disk_key *disk_key;
2841 	struct btrfs_chunk *chunk;
2842 	u8 *ptr;
2843 	int ret = 0;
2844 	u32 num_stripes;
2845 	u32 array_size;
2846 	u32 len = 0;
2847 	u32 cur;
2848 	struct btrfs_key key;
2849 
2850 	mutex_lock(&fs_info->chunk_mutex);
2851 	array_size = btrfs_super_sys_array_size(super_copy);
2852 
2853 	ptr = super_copy->sys_chunk_array;
2854 	cur = 0;
2855 
2856 	while (cur < array_size) {
2857 		disk_key = (struct btrfs_disk_key *)ptr;
2858 		btrfs_disk_key_to_cpu(&key, disk_key);
2859 
2860 		len = sizeof(*disk_key);
2861 
2862 		if (key.type == BTRFS_CHUNK_ITEM_KEY) {
2863 			chunk = (struct btrfs_chunk *)(ptr + len);
2864 			num_stripes = btrfs_stack_chunk_num_stripes(chunk);
2865 			len += btrfs_chunk_item_size(num_stripes);
2866 		} else {
2867 			ret = -EIO;
2868 			break;
2869 		}
2870 		if (key.objectid == BTRFS_FIRST_CHUNK_TREE_OBJECTID &&
2871 		    key.offset == chunk_offset) {
2872 			memmove(ptr, ptr + len, array_size - (cur + len));
2873 			array_size -= len;
2874 			btrfs_set_super_sys_array_size(super_copy, array_size);
2875 		} else {
2876 			ptr += len;
2877 			cur += len;
2878 		}
2879 	}
2880 	mutex_unlock(&fs_info->chunk_mutex);
2881 	return ret;
2882 }
2883 
2884 /*
2885  * btrfs_get_chunk_map() - Find the mapping containing the given logical extent.
2886  * @logical: Logical block offset in bytes.
2887  * @length: Length of extent in bytes.
2888  *
2889  * Return: Chunk mapping or ERR_PTR.
2890  */
2891 struct extent_map *btrfs_get_chunk_map(struct btrfs_fs_info *fs_info,
2892 				       u64 logical, u64 length)
2893 {
2894 	struct extent_map_tree *em_tree;
2895 	struct extent_map *em;
2896 
2897 	em_tree = &fs_info->mapping_tree;
2898 	read_lock(&em_tree->lock);
2899 	em = lookup_extent_mapping(em_tree, logical, length);
2900 	read_unlock(&em_tree->lock);
2901 
2902 	if (!em) {
2903 		btrfs_crit(fs_info, "unable to find logical %llu length %llu",
2904 			   logical, length);
2905 		return ERR_PTR(-EINVAL);
2906 	}
2907 
2908 	if (em->start > logical || em->start + em->len < logical) {
2909 		btrfs_crit(fs_info,
2910 			   "found a bad mapping, wanted %llu-%llu, found %llu-%llu",
2911 			   logical, length, em->start, em->start + em->len);
2912 		free_extent_map(em);
2913 		return ERR_PTR(-EINVAL);
2914 	}
2915 
2916 	/* callers are responsible for dropping em's ref. */
2917 	return em;
2918 }
2919 
2920 int btrfs_remove_chunk(struct btrfs_trans_handle *trans, u64 chunk_offset)
2921 {
2922 	struct btrfs_fs_info *fs_info = trans->fs_info;
2923 	struct extent_map *em;
2924 	struct map_lookup *map;
2925 	u64 dev_extent_len = 0;
2926 	int i, ret = 0;
2927 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2928 
2929 	em = btrfs_get_chunk_map(fs_info, chunk_offset, 1);
2930 	if (IS_ERR(em)) {
2931 		/*
2932 		 * This is a logic error, but we don't want to just rely on the
2933 		 * user having built with ASSERT enabled, so if ASSERT doesn't
2934 		 * do anything we still error out.
2935 		 */
2936 		ASSERT(0);
2937 		return PTR_ERR(em);
2938 	}
2939 	map = em->map_lookup;
2940 	mutex_lock(&fs_info->chunk_mutex);
2941 	check_system_chunk(trans, map->type);
2942 	mutex_unlock(&fs_info->chunk_mutex);
2943 
2944 	/*
2945 	 * Take the device list mutex to prevent races with the final phase of
2946 	 * a device replace operation that replaces the device object associated
2947 	 * with map stripes (dev-replace.c:btrfs_dev_replace_finishing()).
2948 	 */
2949 	mutex_lock(&fs_devices->device_list_mutex);
2950 	for (i = 0; i < map->num_stripes; i++) {
2951 		struct btrfs_device *device = map->stripes[i].dev;
2952 		ret = btrfs_free_dev_extent(trans, device,
2953 					    map->stripes[i].physical,
2954 					    &dev_extent_len);
2955 		if (ret) {
2956 			mutex_unlock(&fs_devices->device_list_mutex);
2957 			btrfs_abort_transaction(trans, ret);
2958 			goto out;
2959 		}
2960 
2961 		if (device->bytes_used > 0) {
2962 			mutex_lock(&fs_info->chunk_mutex);
2963 			btrfs_device_set_bytes_used(device,
2964 					device->bytes_used - dev_extent_len);
2965 			atomic64_add(dev_extent_len, &fs_info->free_chunk_space);
2966 			btrfs_clear_space_info_full(fs_info);
2967 			mutex_unlock(&fs_info->chunk_mutex);
2968 		}
2969 
2970 		ret = btrfs_update_device(trans, device);
2971 		if (ret) {
2972 			mutex_unlock(&fs_devices->device_list_mutex);
2973 			btrfs_abort_transaction(trans, ret);
2974 			goto out;
2975 		}
2976 	}
2977 	mutex_unlock(&fs_devices->device_list_mutex);
2978 
2979 	ret = btrfs_free_chunk(trans, chunk_offset);
2980 	if (ret) {
2981 		btrfs_abort_transaction(trans, ret);
2982 		goto out;
2983 	}
2984 
2985 	trace_btrfs_chunk_free(fs_info, map, chunk_offset, em->len);
2986 
2987 	if (map->type & BTRFS_BLOCK_GROUP_SYSTEM) {
2988 		ret = btrfs_del_sys_chunk(fs_info, chunk_offset);
2989 		if (ret) {
2990 			btrfs_abort_transaction(trans, ret);
2991 			goto out;
2992 		}
2993 	}
2994 
2995 	ret = btrfs_remove_block_group(trans, chunk_offset, em);
2996 	if (ret) {
2997 		btrfs_abort_transaction(trans, ret);
2998 		goto out;
2999 	}
3000 
3001 out:
3002 	/* once for us */
3003 	free_extent_map(em);
3004 	return ret;
3005 }
3006 
3007 static int btrfs_relocate_chunk(struct btrfs_fs_info *fs_info, u64 chunk_offset)
3008 {
3009 	struct btrfs_root *root = fs_info->chunk_root;
3010 	struct btrfs_trans_handle *trans;
3011 	struct btrfs_block_group *block_group;
3012 	int ret;
3013 
3014 	/*
3015 	 * Prevent races with automatic removal of unused block groups.
3016 	 * After we relocate and before we remove the chunk with offset
3017 	 * chunk_offset, automatic removal of the block group can kick in,
3018 	 * resulting in a failure when calling btrfs_remove_chunk() below.
3019 	 *
3020 	 * Make sure to acquire this mutex before doing a tree search (dev
3021 	 * or chunk trees) to find chunks. Otherwise the cleaner kthread might
3022 	 * call btrfs_remove_chunk() (through btrfs_delete_unused_bgs()) after
3023 	 * we release the path used to search the chunk/dev tree and before
3024 	 * the current task acquires this mutex and calls us.
3025 	 */
3026 	lockdep_assert_held(&fs_info->delete_unused_bgs_mutex);
3027 
3028 	/* step one, relocate all the extents inside this chunk */
3029 	btrfs_scrub_pause(fs_info);
3030 	ret = btrfs_relocate_block_group(fs_info, chunk_offset);
3031 	btrfs_scrub_continue(fs_info);
3032 	if (ret)
3033 		return ret;
3034 
3035 	block_group = btrfs_lookup_block_group(fs_info, chunk_offset);
3036 	if (!block_group)
3037 		return -ENOENT;
3038 	btrfs_discard_cancel_work(&fs_info->discard_ctl, block_group);
3039 	btrfs_put_block_group(block_group);
3040 
3041 	trans = btrfs_start_trans_remove_block_group(root->fs_info,
3042 						     chunk_offset);
3043 	if (IS_ERR(trans)) {
3044 		ret = PTR_ERR(trans);
3045 		btrfs_handle_fs_error(root->fs_info, ret, NULL);
3046 		return ret;
3047 	}
3048 
3049 	/*
3050 	 * step two, delete the device extents and the
3051 	 * chunk tree entries
3052 	 */
3053 	ret = btrfs_remove_chunk(trans, chunk_offset);
3054 	btrfs_end_transaction(trans);
3055 	return ret;
3056 }
3057 
3058 static int btrfs_relocate_sys_chunks(struct btrfs_fs_info *fs_info)
3059 {
3060 	struct btrfs_root *chunk_root = fs_info->chunk_root;
3061 	struct btrfs_path *path;
3062 	struct extent_buffer *leaf;
3063 	struct btrfs_chunk *chunk;
3064 	struct btrfs_key key;
3065 	struct btrfs_key found_key;
3066 	u64 chunk_type;
3067 	bool retried = false;
3068 	int failed = 0;
3069 	int ret;
3070 
3071 	path = btrfs_alloc_path();
3072 	if (!path)
3073 		return -ENOMEM;
3074 
3075 again:
3076 	key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
3077 	key.offset = (u64)-1;
3078 	key.type = BTRFS_CHUNK_ITEM_KEY;
3079 
3080 	while (1) {
3081 		mutex_lock(&fs_info->delete_unused_bgs_mutex);
3082 		ret = btrfs_search_slot(NULL, chunk_root, &key, path, 0, 0);
3083 		if (ret < 0) {
3084 			mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3085 			goto error;
3086 		}
3087 		BUG_ON(ret == 0); /* Corruption */
3088 
3089 		ret = btrfs_previous_item(chunk_root, path, key.objectid,
3090 					  key.type);
3091 		if (ret)
3092 			mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3093 		if (ret < 0)
3094 			goto error;
3095 		if (ret > 0)
3096 			break;
3097 
3098 		leaf = path->nodes[0];
3099 		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
3100 
3101 		chunk = btrfs_item_ptr(leaf, path->slots[0],
3102 				       struct btrfs_chunk);
3103 		chunk_type = btrfs_chunk_type(leaf, chunk);
3104 		btrfs_release_path(path);
3105 
3106 		if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM) {
3107 			ret = btrfs_relocate_chunk(fs_info, found_key.offset);
3108 			if (ret == -ENOSPC)
3109 				failed++;
3110 			else
3111 				BUG_ON(ret);
3112 		}
3113 		mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3114 
3115 		if (found_key.offset == 0)
3116 			break;
3117 		key.offset = found_key.offset - 1;
3118 	}
3119 	ret = 0;
3120 	if (failed && !retried) {
3121 		failed = 0;
3122 		retried = true;
3123 		goto again;
3124 	} else if (WARN_ON(failed && retried)) {
3125 		ret = -ENOSPC;
3126 	}
3127 error:
3128 	btrfs_free_path(path);
3129 	return ret;
3130 }
3131 
3132 /*
3133  * return 1 : allocate a data chunk successfully,
3134  * return <0: errors during allocating a data chunk,
3135  * return 0 : no need to allocate a data chunk.
3136  */
3137 static int btrfs_may_alloc_data_chunk(struct btrfs_fs_info *fs_info,
3138 				      u64 chunk_offset)
3139 {
3140 	struct btrfs_block_group *cache;
3141 	u64 bytes_used;
3142 	u64 chunk_type;
3143 
3144 	cache = btrfs_lookup_block_group(fs_info, chunk_offset);
3145 	ASSERT(cache);
3146 	chunk_type = cache->flags;
3147 	btrfs_put_block_group(cache);
3148 
3149 	if (!(chunk_type & BTRFS_BLOCK_GROUP_DATA))
3150 		return 0;
3151 
3152 	spin_lock(&fs_info->data_sinfo->lock);
3153 	bytes_used = fs_info->data_sinfo->bytes_used;
3154 	spin_unlock(&fs_info->data_sinfo->lock);
3155 
3156 	if (!bytes_used) {
3157 		struct btrfs_trans_handle *trans;
3158 		int ret;
3159 
3160 		trans =	btrfs_join_transaction(fs_info->tree_root);
3161 		if (IS_ERR(trans))
3162 			return PTR_ERR(trans);
3163 
3164 		ret = btrfs_force_chunk_alloc(trans, BTRFS_BLOCK_GROUP_DATA);
3165 		btrfs_end_transaction(trans);
3166 		if (ret < 0)
3167 			return ret;
3168 		return 1;
3169 	}
3170 
3171 	return 0;
3172 }
3173 
3174 static int insert_balance_item(struct btrfs_fs_info *fs_info,
3175 			       struct btrfs_balance_control *bctl)
3176 {
3177 	struct btrfs_root *root = fs_info->tree_root;
3178 	struct btrfs_trans_handle *trans;
3179 	struct btrfs_balance_item *item;
3180 	struct btrfs_disk_balance_args disk_bargs;
3181 	struct btrfs_path *path;
3182 	struct extent_buffer *leaf;
3183 	struct btrfs_key key;
3184 	int ret, err;
3185 
3186 	path = btrfs_alloc_path();
3187 	if (!path)
3188 		return -ENOMEM;
3189 
3190 	trans = btrfs_start_transaction(root, 0);
3191 	if (IS_ERR(trans)) {
3192 		btrfs_free_path(path);
3193 		return PTR_ERR(trans);
3194 	}
3195 
3196 	key.objectid = BTRFS_BALANCE_OBJECTID;
3197 	key.type = BTRFS_TEMPORARY_ITEM_KEY;
3198 	key.offset = 0;
3199 
3200 	ret = btrfs_insert_empty_item(trans, root, path, &key,
3201 				      sizeof(*item));
3202 	if (ret)
3203 		goto out;
3204 
3205 	leaf = path->nodes[0];
3206 	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_balance_item);
3207 
3208 	memzero_extent_buffer(leaf, (unsigned long)item, sizeof(*item));
3209 
3210 	btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->data);
3211 	btrfs_set_balance_data(leaf, item, &disk_bargs);
3212 	btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->meta);
3213 	btrfs_set_balance_meta(leaf, item, &disk_bargs);
3214 	btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->sys);
3215 	btrfs_set_balance_sys(leaf, item, &disk_bargs);
3216 
3217 	btrfs_set_balance_flags(leaf, item, bctl->flags);
3218 
3219 	btrfs_mark_buffer_dirty(leaf);
3220 out:
3221 	btrfs_free_path(path);
3222 	err = btrfs_commit_transaction(trans);
3223 	if (err && !ret)
3224 		ret = err;
3225 	return ret;
3226 }
3227 
3228 static int del_balance_item(struct btrfs_fs_info *fs_info)
3229 {
3230 	struct btrfs_root *root = fs_info->tree_root;
3231 	struct btrfs_trans_handle *trans;
3232 	struct btrfs_path *path;
3233 	struct btrfs_key key;
3234 	int ret, err;
3235 
3236 	path = btrfs_alloc_path();
3237 	if (!path)
3238 		return -ENOMEM;
3239 
3240 	trans = btrfs_start_transaction_fallback_global_rsv(root, 0);
3241 	if (IS_ERR(trans)) {
3242 		btrfs_free_path(path);
3243 		return PTR_ERR(trans);
3244 	}
3245 
3246 	key.objectid = BTRFS_BALANCE_OBJECTID;
3247 	key.type = BTRFS_TEMPORARY_ITEM_KEY;
3248 	key.offset = 0;
3249 
3250 	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
3251 	if (ret < 0)
3252 		goto out;
3253 	if (ret > 0) {
3254 		ret = -ENOENT;
3255 		goto out;
3256 	}
3257 
3258 	ret = btrfs_del_item(trans, root, path);
3259 out:
3260 	btrfs_free_path(path);
3261 	err = btrfs_commit_transaction(trans);
3262 	if (err && !ret)
3263 		ret = err;
3264 	return ret;
3265 }
3266 
3267 /*
3268  * This is a heuristic used to reduce the number of chunks balanced on
3269  * resume after balance was interrupted.
3270  */
3271 static void update_balance_args(struct btrfs_balance_control *bctl)
3272 {
3273 	/*
3274 	 * Turn on soft mode for chunk types that were being converted.
3275 	 */
3276 	if (bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT)
3277 		bctl->data.flags |= BTRFS_BALANCE_ARGS_SOFT;
3278 	if (bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT)
3279 		bctl->sys.flags |= BTRFS_BALANCE_ARGS_SOFT;
3280 	if (bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT)
3281 		bctl->meta.flags |= BTRFS_BALANCE_ARGS_SOFT;
3282 
3283 	/*
3284 	 * Turn on usage filter if is not already used.  The idea is
3285 	 * that chunks that we have already balanced should be
3286 	 * reasonably full.  Don't do it for chunks that are being
3287 	 * converted - that will keep us from relocating unconverted
3288 	 * (albeit full) chunks.
3289 	 */
3290 	if (!(bctl->data.flags & BTRFS_BALANCE_ARGS_USAGE) &&
3291 	    !(bctl->data.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
3292 	    !(bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT)) {
3293 		bctl->data.flags |= BTRFS_BALANCE_ARGS_USAGE;
3294 		bctl->data.usage = 90;
3295 	}
3296 	if (!(bctl->sys.flags & BTRFS_BALANCE_ARGS_USAGE) &&
3297 	    !(bctl->sys.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
3298 	    !(bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT)) {
3299 		bctl->sys.flags |= BTRFS_BALANCE_ARGS_USAGE;
3300 		bctl->sys.usage = 90;
3301 	}
3302 	if (!(bctl->meta.flags & BTRFS_BALANCE_ARGS_USAGE) &&
3303 	    !(bctl->meta.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
3304 	    !(bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT)) {
3305 		bctl->meta.flags |= BTRFS_BALANCE_ARGS_USAGE;
3306 		bctl->meta.usage = 90;
3307 	}
3308 }
3309 
3310 /*
3311  * Clear the balance status in fs_info and delete the balance item from disk.
3312  */
3313 static void reset_balance_state(struct btrfs_fs_info *fs_info)
3314 {
3315 	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
3316 	int ret;
3317 
3318 	BUG_ON(!fs_info->balance_ctl);
3319 
3320 	spin_lock(&fs_info->balance_lock);
3321 	fs_info->balance_ctl = NULL;
3322 	spin_unlock(&fs_info->balance_lock);
3323 
3324 	kfree(bctl);
3325 	ret = del_balance_item(fs_info);
3326 	if (ret)
3327 		btrfs_handle_fs_error(fs_info, ret, NULL);
3328 }
3329 
3330 /*
3331  * Balance filters.  Return 1 if chunk should be filtered out
3332  * (should not be balanced).
3333  */
3334 static int chunk_profiles_filter(u64 chunk_type,
3335 				 struct btrfs_balance_args *bargs)
3336 {
3337 	chunk_type = chunk_to_extended(chunk_type) &
3338 				BTRFS_EXTENDED_PROFILE_MASK;
3339 
3340 	if (bargs->profiles & chunk_type)
3341 		return 0;
3342 
3343 	return 1;
3344 }
3345 
3346 static int chunk_usage_range_filter(struct btrfs_fs_info *fs_info, u64 chunk_offset,
3347 			      struct btrfs_balance_args *bargs)
3348 {
3349 	struct btrfs_block_group *cache;
3350 	u64 chunk_used;
3351 	u64 user_thresh_min;
3352 	u64 user_thresh_max;
3353 	int ret = 1;
3354 
3355 	cache = btrfs_lookup_block_group(fs_info, chunk_offset);
3356 	chunk_used = cache->used;
3357 
3358 	if (bargs->usage_min == 0)
3359 		user_thresh_min = 0;
3360 	else
3361 		user_thresh_min = div_factor_fine(cache->length,
3362 						  bargs->usage_min);
3363 
3364 	if (bargs->usage_max == 0)
3365 		user_thresh_max = 1;
3366 	else if (bargs->usage_max > 100)
3367 		user_thresh_max = cache->length;
3368 	else
3369 		user_thresh_max = div_factor_fine(cache->length,
3370 						  bargs->usage_max);
3371 
3372 	if (user_thresh_min <= chunk_used && chunk_used < user_thresh_max)
3373 		ret = 0;
3374 
3375 	btrfs_put_block_group(cache);
3376 	return ret;
3377 }
3378 
3379 static int chunk_usage_filter(struct btrfs_fs_info *fs_info,
3380 		u64 chunk_offset, struct btrfs_balance_args *bargs)
3381 {
3382 	struct btrfs_block_group *cache;
3383 	u64 chunk_used, user_thresh;
3384 	int ret = 1;
3385 
3386 	cache = btrfs_lookup_block_group(fs_info, chunk_offset);
3387 	chunk_used = cache->used;
3388 
3389 	if (bargs->usage_min == 0)
3390 		user_thresh = 1;
3391 	else if (bargs->usage > 100)
3392 		user_thresh = cache->length;
3393 	else
3394 		user_thresh = div_factor_fine(cache->length, bargs->usage);
3395 
3396 	if (chunk_used < user_thresh)
3397 		ret = 0;
3398 
3399 	btrfs_put_block_group(cache);
3400 	return ret;
3401 }
3402 
3403 static int chunk_devid_filter(struct extent_buffer *leaf,
3404 			      struct btrfs_chunk *chunk,
3405 			      struct btrfs_balance_args *bargs)
3406 {
3407 	struct btrfs_stripe *stripe;
3408 	int num_stripes = btrfs_chunk_num_stripes(leaf, chunk);
3409 	int i;
3410 
3411 	for (i = 0; i < num_stripes; i++) {
3412 		stripe = btrfs_stripe_nr(chunk, i);
3413 		if (btrfs_stripe_devid(leaf, stripe) == bargs->devid)
3414 			return 0;
3415 	}
3416 
3417 	return 1;
3418 }
3419 
3420 static u64 calc_data_stripes(u64 type, int num_stripes)
3421 {
3422 	const int index = btrfs_bg_flags_to_raid_index(type);
3423 	const int ncopies = btrfs_raid_array[index].ncopies;
3424 	const int nparity = btrfs_raid_array[index].nparity;
3425 
3426 	if (nparity)
3427 		return num_stripes - nparity;
3428 	else
3429 		return num_stripes / ncopies;
3430 }
3431 
3432 /* [pstart, pend) */
3433 static int chunk_drange_filter(struct extent_buffer *leaf,
3434 			       struct btrfs_chunk *chunk,
3435 			       struct btrfs_balance_args *bargs)
3436 {
3437 	struct btrfs_stripe *stripe;
3438 	int num_stripes = btrfs_chunk_num_stripes(leaf, chunk);
3439 	u64 stripe_offset;
3440 	u64 stripe_length;
3441 	u64 type;
3442 	int factor;
3443 	int i;
3444 
3445 	if (!(bargs->flags & BTRFS_BALANCE_ARGS_DEVID))
3446 		return 0;
3447 
3448 	type = btrfs_chunk_type(leaf, chunk);
3449 	factor = calc_data_stripes(type, num_stripes);
3450 
3451 	for (i = 0; i < num_stripes; i++) {
3452 		stripe = btrfs_stripe_nr(chunk, i);
3453 		if (btrfs_stripe_devid(leaf, stripe) != bargs->devid)
3454 			continue;
3455 
3456 		stripe_offset = btrfs_stripe_offset(leaf, stripe);
3457 		stripe_length = btrfs_chunk_length(leaf, chunk);
3458 		stripe_length = div_u64(stripe_length, factor);
3459 
3460 		if (stripe_offset < bargs->pend &&
3461 		    stripe_offset + stripe_length > bargs->pstart)
3462 			return 0;
3463 	}
3464 
3465 	return 1;
3466 }
3467 
3468 /* [vstart, vend) */
3469 static int chunk_vrange_filter(struct extent_buffer *leaf,
3470 			       struct btrfs_chunk *chunk,
3471 			       u64 chunk_offset,
3472 			       struct btrfs_balance_args *bargs)
3473 {
3474 	if (chunk_offset < bargs->vend &&
3475 	    chunk_offset + btrfs_chunk_length(leaf, chunk) > bargs->vstart)
3476 		/* at least part of the chunk is inside this vrange */
3477 		return 0;
3478 
3479 	return 1;
3480 }
3481 
3482 static int chunk_stripes_range_filter(struct extent_buffer *leaf,
3483 			       struct btrfs_chunk *chunk,
3484 			       struct btrfs_balance_args *bargs)
3485 {
3486 	int num_stripes = btrfs_chunk_num_stripes(leaf, chunk);
3487 
3488 	if (bargs->stripes_min <= num_stripes
3489 			&& num_stripes <= bargs->stripes_max)
3490 		return 0;
3491 
3492 	return 1;
3493 }
3494 
3495 static int chunk_soft_convert_filter(u64 chunk_type,
3496 				     struct btrfs_balance_args *bargs)
3497 {
3498 	if (!(bargs->flags & BTRFS_BALANCE_ARGS_CONVERT))
3499 		return 0;
3500 
3501 	chunk_type = chunk_to_extended(chunk_type) &
3502 				BTRFS_EXTENDED_PROFILE_MASK;
3503 
3504 	if (bargs->target == chunk_type)
3505 		return 1;
3506 
3507 	return 0;
3508 }
3509 
3510 static int should_balance_chunk(struct extent_buffer *leaf,
3511 				struct btrfs_chunk *chunk, u64 chunk_offset)
3512 {
3513 	struct btrfs_fs_info *fs_info = leaf->fs_info;
3514 	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
3515 	struct btrfs_balance_args *bargs = NULL;
3516 	u64 chunk_type = btrfs_chunk_type(leaf, chunk);
3517 
3518 	/* type filter */
3519 	if (!((chunk_type & BTRFS_BLOCK_GROUP_TYPE_MASK) &
3520 	      (bctl->flags & BTRFS_BALANCE_TYPE_MASK))) {
3521 		return 0;
3522 	}
3523 
3524 	if (chunk_type & BTRFS_BLOCK_GROUP_DATA)
3525 		bargs = &bctl->data;
3526 	else if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM)
3527 		bargs = &bctl->sys;
3528 	else if (chunk_type & BTRFS_BLOCK_GROUP_METADATA)
3529 		bargs = &bctl->meta;
3530 
3531 	/* profiles filter */
3532 	if ((bargs->flags & BTRFS_BALANCE_ARGS_PROFILES) &&
3533 	    chunk_profiles_filter(chunk_type, bargs)) {
3534 		return 0;
3535 	}
3536 
3537 	/* usage filter */
3538 	if ((bargs->flags & BTRFS_BALANCE_ARGS_USAGE) &&
3539 	    chunk_usage_filter(fs_info, chunk_offset, bargs)) {
3540 		return 0;
3541 	} else if ((bargs->flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
3542 	    chunk_usage_range_filter(fs_info, chunk_offset, bargs)) {
3543 		return 0;
3544 	}
3545 
3546 	/* devid filter */
3547 	if ((bargs->flags & BTRFS_BALANCE_ARGS_DEVID) &&
3548 	    chunk_devid_filter(leaf, chunk, bargs)) {
3549 		return 0;
3550 	}
3551 
3552 	/* drange filter, makes sense only with devid filter */
3553 	if ((bargs->flags & BTRFS_BALANCE_ARGS_DRANGE) &&
3554 	    chunk_drange_filter(leaf, chunk, bargs)) {
3555 		return 0;
3556 	}
3557 
3558 	/* vrange filter */
3559 	if ((bargs->flags & BTRFS_BALANCE_ARGS_VRANGE) &&
3560 	    chunk_vrange_filter(leaf, chunk, chunk_offset, bargs)) {
3561 		return 0;
3562 	}
3563 
3564 	/* stripes filter */
3565 	if ((bargs->flags & BTRFS_BALANCE_ARGS_STRIPES_RANGE) &&
3566 	    chunk_stripes_range_filter(leaf, chunk, bargs)) {
3567 		return 0;
3568 	}
3569 
3570 	/* soft profile changing mode */
3571 	if ((bargs->flags & BTRFS_BALANCE_ARGS_SOFT) &&
3572 	    chunk_soft_convert_filter(chunk_type, bargs)) {
3573 		return 0;
3574 	}
3575 
3576 	/*
3577 	 * limited by count, must be the last filter
3578 	 */
3579 	if ((bargs->flags & BTRFS_BALANCE_ARGS_LIMIT)) {
3580 		if (bargs->limit == 0)
3581 			return 0;
3582 		else
3583 			bargs->limit--;
3584 	} else if ((bargs->flags & BTRFS_BALANCE_ARGS_LIMIT_RANGE)) {
3585 		/*
3586 		 * Same logic as the 'limit' filter; the minimum cannot be
3587 		 * determined here because we do not have the global information
3588 		 * about the count of all chunks that satisfy the filters.
3589 		 */
3590 		if (bargs->limit_max == 0)
3591 			return 0;
3592 		else
3593 			bargs->limit_max--;
3594 	}
3595 
3596 	return 1;
3597 }
3598 
3599 static int __btrfs_balance(struct btrfs_fs_info *fs_info)
3600 {
3601 	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
3602 	struct btrfs_root *chunk_root = fs_info->chunk_root;
3603 	u64 chunk_type;
3604 	struct btrfs_chunk *chunk;
3605 	struct btrfs_path *path = NULL;
3606 	struct btrfs_key key;
3607 	struct btrfs_key found_key;
3608 	struct extent_buffer *leaf;
3609 	int slot;
3610 	int ret;
3611 	int enospc_errors = 0;
3612 	bool counting = true;
3613 	/* The single value limit and min/max limits use the same bytes in the */
3614 	u64 limit_data = bctl->data.limit;
3615 	u64 limit_meta = bctl->meta.limit;
3616 	u64 limit_sys = bctl->sys.limit;
3617 	u32 count_data = 0;
3618 	u32 count_meta = 0;
3619 	u32 count_sys = 0;
3620 	int chunk_reserved = 0;
3621 
3622 	path = btrfs_alloc_path();
3623 	if (!path) {
3624 		ret = -ENOMEM;
3625 		goto error;
3626 	}
3627 
3628 	/* zero out stat counters */
3629 	spin_lock(&fs_info->balance_lock);
3630 	memset(&bctl->stat, 0, sizeof(bctl->stat));
3631 	spin_unlock(&fs_info->balance_lock);
3632 again:
3633 	if (!counting) {
3634 		/*
3635 		 * The single value limit and min/max limits use the same bytes
3636 		 * in the
3637 		 */
3638 		bctl->data.limit = limit_data;
3639 		bctl->meta.limit = limit_meta;
3640 		bctl->sys.limit = limit_sys;
3641 	}
3642 	key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
3643 	key.offset = (u64)-1;
3644 	key.type = BTRFS_CHUNK_ITEM_KEY;
3645 
3646 	while (1) {
3647 		if ((!counting && atomic_read(&fs_info->balance_pause_req)) ||
3648 		    atomic_read(&fs_info->balance_cancel_req)) {
3649 			ret = -ECANCELED;
3650 			goto error;
3651 		}
3652 
3653 		mutex_lock(&fs_info->delete_unused_bgs_mutex);
3654 		ret = btrfs_search_slot(NULL, chunk_root, &key, path, 0, 0);
3655 		if (ret < 0) {
3656 			mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3657 			goto error;
3658 		}
3659 
3660 		/*
3661 		 * this shouldn't happen, it means the last relocate
3662 		 * failed
3663 		 */
3664 		if (ret == 0)
3665 			BUG(); /* FIXME break ? */
3666 
3667 		ret = btrfs_previous_item(chunk_root, path, 0,
3668 					  BTRFS_CHUNK_ITEM_KEY);
3669 		if (ret) {
3670 			mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3671 			ret = 0;
3672 			break;
3673 		}
3674 
3675 		leaf = path->nodes[0];
3676 		slot = path->slots[0];
3677 		btrfs_item_key_to_cpu(leaf, &found_key, slot);
3678 
3679 		if (found_key.objectid != key.objectid) {
3680 			mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3681 			break;
3682 		}
3683 
3684 		chunk = btrfs_item_ptr(leaf, slot, struct btrfs_chunk);
3685 		chunk_type = btrfs_chunk_type(leaf, chunk);
3686 
3687 		if (!counting) {
3688 			spin_lock(&fs_info->balance_lock);
3689 			bctl->stat.considered++;
3690 			spin_unlock(&fs_info->balance_lock);
3691 		}
3692 
3693 		ret = should_balance_chunk(leaf, chunk, found_key.offset);
3694 
3695 		btrfs_release_path(path);
3696 		if (!ret) {
3697 			mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3698 			goto loop;
3699 		}
3700 
3701 		if (counting) {
3702 			mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3703 			spin_lock(&fs_info->balance_lock);
3704 			bctl->stat.expected++;
3705 			spin_unlock(&fs_info->balance_lock);
3706 
3707 			if (chunk_type & BTRFS_BLOCK_GROUP_DATA)
3708 				count_data++;
3709 			else if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM)
3710 				count_sys++;
3711 			else if (chunk_type & BTRFS_BLOCK_GROUP_METADATA)
3712 				count_meta++;
3713 
3714 			goto loop;
3715 		}
3716 
3717 		/*
3718 		 * Apply limit_min filter, no need to check if the LIMITS
3719 		 * filter is used, limit_min is 0 by default
3720 		 */
3721 		if (((chunk_type & BTRFS_BLOCK_GROUP_DATA) &&
3722 					count_data < bctl->data.limit_min)
3723 				|| ((chunk_type & BTRFS_BLOCK_GROUP_METADATA) &&
3724 					count_meta < bctl->meta.limit_min)
3725 				|| ((chunk_type & BTRFS_BLOCK_GROUP_SYSTEM) &&
3726 					count_sys < bctl->sys.limit_min)) {
3727 			mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3728 			goto loop;
3729 		}
3730 
3731 		if (!chunk_reserved) {
3732 			/*
3733 			 * We may be relocating the only data chunk we have,
3734 			 * which could potentially end up with losing data's
3735 			 * raid profile, so lets allocate an empty one in
3736 			 * advance.
3737 			 */
3738 			ret = btrfs_may_alloc_data_chunk(fs_info,
3739 							 found_key.offset);
3740 			if (ret < 0) {
3741 				mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3742 				goto error;
3743 			} else if (ret == 1) {
3744 				chunk_reserved = 1;
3745 			}
3746 		}
3747 
3748 		ret = btrfs_relocate_chunk(fs_info, found_key.offset);
3749 		mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3750 		if (ret == -ENOSPC) {
3751 			enospc_errors++;
3752 		} else if (ret == -ETXTBSY) {
3753 			btrfs_info(fs_info,
3754 	   "skipping relocation of block group %llu due to active swapfile",
3755 				   found_key.offset);
3756 			ret = 0;
3757 		} else if (ret) {
3758 			goto error;
3759 		} else {
3760 			spin_lock(&fs_info->balance_lock);
3761 			bctl->stat.completed++;
3762 			spin_unlock(&fs_info->balance_lock);
3763 		}
3764 loop:
3765 		if (found_key.offset == 0)
3766 			break;
3767 		key.offset = found_key.offset - 1;
3768 	}
3769 
3770 	if (counting) {
3771 		btrfs_release_path(path);
3772 		counting = false;
3773 		goto again;
3774 	}
3775 error:
3776 	btrfs_free_path(path);
3777 	if (enospc_errors) {
3778 		btrfs_info(fs_info, "%d enospc errors during balance",
3779 			   enospc_errors);
3780 		if (!ret)
3781 			ret = -ENOSPC;
3782 	}
3783 
3784 	return ret;
3785 }
3786 
3787 /**
3788  * alloc_profile_is_valid - see if a given profile is valid and reduced
3789  * @flags: profile to validate
3790  * @extended: if true @flags is treated as an extended profile
3791  */
3792 static int alloc_profile_is_valid(u64 flags, int extended)
3793 {
3794 	u64 mask = (extended ? BTRFS_EXTENDED_PROFILE_MASK :
3795 			       BTRFS_BLOCK_GROUP_PROFILE_MASK);
3796 
3797 	flags &= ~BTRFS_BLOCK_GROUP_TYPE_MASK;
3798 
3799 	/* 1) check that all other bits are zeroed */
3800 	if (flags & ~mask)
3801 		return 0;
3802 
3803 	/* 2) see if profile is reduced */
3804 	if (flags == 0)
3805 		return !extended; /* "0" is valid for usual profiles */
3806 
3807 	return has_single_bit_set(flags);
3808 }
3809 
3810 static inline int balance_need_close(struct btrfs_fs_info *fs_info)
3811 {
3812 	/* cancel requested || normal exit path */
3813 	return atomic_read(&fs_info->balance_cancel_req) ||
3814 		(atomic_read(&fs_info->balance_pause_req) == 0 &&
3815 		 atomic_read(&fs_info->balance_cancel_req) == 0);
3816 }
3817 
3818 /*
3819  * Validate target profile against allowed profiles and return true if it's OK.
3820  * Otherwise print the error message and return false.
3821  */
3822 static inline int validate_convert_profile(struct btrfs_fs_info *fs_info,
3823 		const struct btrfs_balance_args *bargs,
3824 		u64 allowed, const char *type)
3825 {
3826 	if (!(bargs->flags & BTRFS_BALANCE_ARGS_CONVERT))
3827 		return true;
3828 
3829 	/* Profile is valid and does not have bits outside of the allowed set */
3830 	if (alloc_profile_is_valid(bargs->target, 1) &&
3831 	    (bargs->target & ~allowed) == 0)
3832 		return true;
3833 
3834 	btrfs_err(fs_info, "balance: invalid convert %s profile %s",
3835 			type, btrfs_bg_type_to_raid_name(bargs->target));
3836 	return false;
3837 }
3838 
3839 /*
3840  * Fill @buf with textual description of balance filter flags @bargs, up to
3841  * @size_buf including the terminating null. The output may be trimmed if it
3842  * does not fit into the provided buffer.
3843  */
3844 static void describe_balance_args(struct btrfs_balance_args *bargs, char *buf,
3845 				 u32 size_buf)
3846 {
3847 	int ret;
3848 	u32 size_bp = size_buf;
3849 	char *bp = buf;
3850 	u64 flags = bargs->flags;
3851 	char tmp_buf[128] = {'\0'};
3852 
3853 	if (!flags)
3854 		return;
3855 
3856 #define CHECK_APPEND_NOARG(a)						\
3857 	do {								\
3858 		ret = snprintf(bp, size_bp, (a));			\
3859 		if (ret < 0 || ret >= size_bp)				\
3860 			goto out_overflow;				\
3861 		size_bp -= ret;						\
3862 		bp += ret;						\
3863 	} while (0)
3864 
3865 #define CHECK_APPEND_1ARG(a, v1)					\
3866 	do {								\
3867 		ret = snprintf(bp, size_bp, (a), (v1));			\
3868 		if (ret < 0 || ret >= size_bp)				\
3869 			goto out_overflow;				\
3870 		size_bp -= ret;						\
3871 		bp += ret;						\
3872 	} while (0)
3873 
3874 #define CHECK_APPEND_2ARG(a, v1, v2)					\
3875 	do {								\
3876 		ret = snprintf(bp, size_bp, (a), (v1), (v2));		\
3877 		if (ret < 0 || ret >= size_bp)				\
3878 			goto out_overflow;				\
3879 		size_bp -= ret;						\
3880 		bp += ret;						\
3881 	} while (0)
3882 
3883 	if (flags & BTRFS_BALANCE_ARGS_CONVERT)
3884 		CHECK_APPEND_1ARG("convert=%s,",
3885 				  btrfs_bg_type_to_raid_name(bargs->target));
3886 
3887 	if (flags & BTRFS_BALANCE_ARGS_SOFT)
3888 		CHECK_APPEND_NOARG("soft,");
3889 
3890 	if (flags & BTRFS_BALANCE_ARGS_PROFILES) {
3891 		btrfs_describe_block_groups(bargs->profiles, tmp_buf,
3892 					    sizeof(tmp_buf));
3893 		CHECK_APPEND_1ARG("profiles=%s,", tmp_buf);
3894 	}
3895 
3896 	if (flags & BTRFS_BALANCE_ARGS_USAGE)
3897 		CHECK_APPEND_1ARG("usage=%llu,", bargs->usage);
3898 
3899 	if (flags & BTRFS_BALANCE_ARGS_USAGE_RANGE)
3900 		CHECK_APPEND_2ARG("usage=%u..%u,",
3901 				  bargs->usage_min, bargs->usage_max);
3902 
3903 	if (flags & BTRFS_BALANCE_ARGS_DEVID)
3904 		CHECK_APPEND_1ARG("devid=%llu,", bargs->devid);
3905 
3906 	if (flags & BTRFS_BALANCE_ARGS_DRANGE)
3907 		CHECK_APPEND_2ARG("drange=%llu..%llu,",
3908 				  bargs->pstart, bargs->pend);
3909 
3910 	if (flags & BTRFS_BALANCE_ARGS_VRANGE)
3911 		CHECK_APPEND_2ARG("vrange=%llu..%llu,",
3912 				  bargs->vstart, bargs->vend);
3913 
3914 	if (flags & BTRFS_BALANCE_ARGS_LIMIT)
3915 		CHECK_APPEND_1ARG("limit=%llu,", bargs->limit);
3916 
3917 	if (flags & BTRFS_BALANCE_ARGS_LIMIT_RANGE)
3918 		CHECK_APPEND_2ARG("limit=%u..%u,",
3919 				bargs->limit_min, bargs->limit_max);
3920 
3921 	if (flags & BTRFS_BALANCE_ARGS_STRIPES_RANGE)
3922 		CHECK_APPEND_2ARG("stripes=%u..%u,",
3923 				  bargs->stripes_min, bargs->stripes_max);
3924 
3925 #undef CHECK_APPEND_2ARG
3926 #undef CHECK_APPEND_1ARG
3927 #undef CHECK_APPEND_NOARG
3928 
3929 out_overflow:
3930 
3931 	if (size_bp < size_buf)
3932 		buf[size_buf - size_bp - 1] = '\0'; /* remove last , */
3933 	else
3934 		buf[0] = '\0';
3935 }
3936 
3937 static void describe_balance_start_or_resume(struct btrfs_fs_info *fs_info)
3938 {
3939 	u32 size_buf = 1024;
3940 	char tmp_buf[192] = {'\0'};
3941 	char *buf;
3942 	char *bp;
3943 	u32 size_bp = size_buf;
3944 	int ret;
3945 	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
3946 
3947 	buf = kzalloc(size_buf, GFP_KERNEL);
3948 	if (!buf)
3949 		return;
3950 
3951 	bp = buf;
3952 
3953 #define CHECK_APPEND_1ARG(a, v1)					\
3954 	do {								\
3955 		ret = snprintf(bp, size_bp, (a), (v1));			\
3956 		if (ret < 0 || ret >= size_bp)				\
3957 			goto out_overflow;				\
3958 		size_bp -= ret;						\
3959 		bp += ret;						\
3960 	} while (0)
3961 
3962 	if (bctl->flags & BTRFS_BALANCE_FORCE)
3963 		CHECK_APPEND_1ARG("%s", "-f ");
3964 
3965 	if (bctl->flags & BTRFS_BALANCE_DATA) {
3966 		describe_balance_args(&bctl->data, tmp_buf, sizeof(tmp_buf));
3967 		CHECK_APPEND_1ARG("-d%s ", tmp_buf);
3968 	}
3969 
3970 	if (bctl->flags & BTRFS_BALANCE_METADATA) {
3971 		describe_balance_args(&bctl->meta, tmp_buf, sizeof(tmp_buf));
3972 		CHECK_APPEND_1ARG("-m%s ", tmp_buf);
3973 	}
3974 
3975 	if (bctl->flags & BTRFS_BALANCE_SYSTEM) {
3976 		describe_balance_args(&bctl->sys, tmp_buf, sizeof(tmp_buf));
3977 		CHECK_APPEND_1ARG("-s%s ", tmp_buf);
3978 	}
3979 
3980 #undef CHECK_APPEND_1ARG
3981 
3982 out_overflow:
3983 
3984 	if (size_bp < size_buf)
3985 		buf[size_buf - size_bp - 1] = '\0'; /* remove last " " */
3986 	btrfs_info(fs_info, "balance: %s %s",
3987 		   (bctl->flags & BTRFS_BALANCE_RESUME) ?
3988 		   "resume" : "start", buf);
3989 
3990 	kfree(buf);
3991 }
3992 
3993 /*
3994  * Should be called with balance mutexe held
3995  */
3996 int btrfs_balance(struct btrfs_fs_info *fs_info,
3997 		  struct btrfs_balance_control *bctl,
3998 		  struct btrfs_ioctl_balance_args *bargs)
3999 {
4000 	u64 meta_target, data_target;
4001 	u64 allowed;
4002 	int mixed = 0;
4003 	int ret;
4004 	u64 num_devices;
4005 	unsigned seq;
4006 	bool reducing_redundancy;
4007 	int i;
4008 
4009 	if (btrfs_fs_closing(fs_info) ||
4010 	    atomic_read(&fs_info->balance_pause_req) ||
4011 	    btrfs_should_cancel_balance(fs_info)) {
4012 		ret = -EINVAL;
4013 		goto out;
4014 	}
4015 
4016 	allowed = btrfs_super_incompat_flags(fs_info->super_copy);
4017 	if (allowed & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS)
4018 		mixed = 1;
4019 
4020 	/*
4021 	 * In case of mixed groups both data and meta should be picked,
4022 	 * and identical options should be given for both of them.
4023 	 */
4024 	allowed = BTRFS_BALANCE_DATA | BTRFS_BALANCE_METADATA;
4025 	if (mixed && (bctl->flags & allowed)) {
4026 		if (!(bctl->flags & BTRFS_BALANCE_DATA) ||
4027 		    !(bctl->flags & BTRFS_BALANCE_METADATA) ||
4028 		    memcmp(&bctl->data, &bctl->meta, sizeof(bctl->data))) {
4029 			btrfs_err(fs_info,
4030 	  "balance: mixed groups data and metadata options must be the same");
4031 			ret = -EINVAL;
4032 			goto out;
4033 		}
4034 	}
4035 
4036 	/*
4037 	 * rw_devices will not change at the moment, device add/delete/replace
4038 	 * are exclusive
4039 	 */
4040 	num_devices = fs_info->fs_devices->rw_devices;
4041 
4042 	/*
4043 	 * SINGLE profile on-disk has no profile bit, but in-memory we have a
4044 	 * special bit for it, to make it easier to distinguish.  Thus we need
4045 	 * to set it manually, or balance would refuse the profile.
4046 	 */
4047 	allowed = BTRFS_AVAIL_ALLOC_BIT_SINGLE;
4048 	for (i = 0; i < ARRAY_SIZE(btrfs_raid_array); i++)
4049 		if (num_devices >= btrfs_raid_array[i].devs_min)
4050 			allowed |= btrfs_raid_array[i].bg_flag;
4051 
4052 	if (!validate_convert_profile(fs_info, &bctl->data, allowed, "data") ||
4053 	    !validate_convert_profile(fs_info, &bctl->meta, allowed, "metadata") ||
4054 	    !validate_convert_profile(fs_info, &bctl->sys,  allowed, "system")) {
4055 		ret = -EINVAL;
4056 		goto out;
4057 	}
4058 
4059 	/*
4060 	 * Allow to reduce metadata or system integrity only if force set for
4061 	 * profiles with redundancy (copies, parity)
4062 	 */
4063 	allowed = 0;
4064 	for (i = 0; i < ARRAY_SIZE(btrfs_raid_array); i++) {
4065 		if (btrfs_raid_array[i].ncopies >= 2 ||
4066 		    btrfs_raid_array[i].tolerated_failures >= 1)
4067 			allowed |= btrfs_raid_array[i].bg_flag;
4068 	}
4069 	do {
4070 		seq = read_seqbegin(&fs_info->profiles_lock);
4071 
4072 		if (((bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) &&
4073 		     (fs_info->avail_system_alloc_bits & allowed) &&
4074 		     !(bctl->sys.target & allowed)) ||
4075 		    ((bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) &&
4076 		     (fs_info->avail_metadata_alloc_bits & allowed) &&
4077 		     !(bctl->meta.target & allowed)))
4078 			reducing_redundancy = true;
4079 		else
4080 			reducing_redundancy = false;
4081 
4082 		/* if we're not converting, the target field is uninitialized */
4083 		meta_target = (bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) ?
4084 			bctl->meta.target : fs_info->avail_metadata_alloc_bits;
4085 		data_target = (bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) ?
4086 			bctl->data.target : fs_info->avail_data_alloc_bits;
4087 	} while (read_seqretry(&fs_info->profiles_lock, seq));
4088 
4089 	if (reducing_redundancy) {
4090 		if (bctl->flags & BTRFS_BALANCE_FORCE) {
4091 			btrfs_info(fs_info,
4092 			   "balance: force reducing metadata redundancy");
4093 		} else {
4094 			btrfs_err(fs_info,
4095 	"balance: reduces metadata redundancy, use --force if you want this");
4096 			ret = -EINVAL;
4097 			goto out;
4098 		}
4099 	}
4100 
4101 	if (btrfs_get_num_tolerated_disk_barrier_failures(meta_target) <
4102 		btrfs_get_num_tolerated_disk_barrier_failures(data_target)) {
4103 		btrfs_warn(fs_info,
4104 	"balance: metadata profile %s has lower redundancy than data profile %s",
4105 				btrfs_bg_type_to_raid_name(meta_target),
4106 				btrfs_bg_type_to_raid_name(data_target));
4107 	}
4108 
4109 	if (fs_info->send_in_progress) {
4110 		btrfs_warn_rl(fs_info,
4111 "cannot run balance while send operations are in progress (%d in progress)",
4112 			      fs_info->send_in_progress);
4113 		ret = -EAGAIN;
4114 		goto out;
4115 	}
4116 
4117 	ret = insert_balance_item(fs_info, bctl);
4118 	if (ret && ret != -EEXIST)
4119 		goto out;
4120 
4121 	if (!(bctl->flags & BTRFS_BALANCE_RESUME)) {
4122 		BUG_ON(ret == -EEXIST);
4123 		BUG_ON(fs_info->balance_ctl);
4124 		spin_lock(&fs_info->balance_lock);
4125 		fs_info->balance_ctl = bctl;
4126 		spin_unlock(&fs_info->balance_lock);
4127 	} else {
4128 		BUG_ON(ret != -EEXIST);
4129 		spin_lock(&fs_info->balance_lock);
4130 		update_balance_args(bctl);
4131 		spin_unlock(&fs_info->balance_lock);
4132 	}
4133 
4134 	ASSERT(!test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4135 	set_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags);
4136 	describe_balance_start_or_resume(fs_info);
4137 	mutex_unlock(&fs_info->balance_mutex);
4138 
4139 	ret = __btrfs_balance(fs_info);
4140 
4141 	mutex_lock(&fs_info->balance_mutex);
4142 	if (ret == -ECANCELED && atomic_read(&fs_info->balance_pause_req))
4143 		btrfs_info(fs_info, "balance: paused");
4144 	/*
4145 	 * Balance can be canceled by:
4146 	 *
4147 	 * - Regular cancel request
4148 	 *   Then ret == -ECANCELED and balance_cancel_req > 0
4149 	 *
4150 	 * - Fatal signal to "btrfs" process
4151 	 *   Either the signal caught by wait_reserve_ticket() and callers
4152 	 *   got -EINTR, or caught by btrfs_should_cancel_balance() and
4153 	 *   got -ECANCELED.
4154 	 *   Either way, in this case balance_cancel_req = 0, and
4155 	 *   ret == -EINTR or ret == -ECANCELED.
4156 	 *
4157 	 * So here we only check the return value to catch canceled balance.
4158 	 */
4159 	else if (ret == -ECANCELED || ret == -EINTR)
4160 		btrfs_info(fs_info, "balance: canceled");
4161 	else
4162 		btrfs_info(fs_info, "balance: ended with status: %d", ret);
4163 
4164 	clear_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags);
4165 
4166 	if (bargs) {
4167 		memset(bargs, 0, sizeof(*bargs));
4168 		btrfs_update_ioctl_balance_args(fs_info, bargs);
4169 	}
4170 
4171 	if ((ret && ret != -ECANCELED && ret != -ENOSPC) ||
4172 	    balance_need_close(fs_info)) {
4173 		reset_balance_state(fs_info);
4174 		btrfs_exclop_finish(fs_info);
4175 	}
4176 
4177 	wake_up(&fs_info->balance_wait_q);
4178 
4179 	return ret;
4180 out:
4181 	if (bctl->flags & BTRFS_BALANCE_RESUME)
4182 		reset_balance_state(fs_info);
4183 	else
4184 		kfree(bctl);
4185 	btrfs_exclop_finish(fs_info);
4186 
4187 	return ret;
4188 }
4189 
4190 static int balance_kthread(void *data)
4191 {
4192 	struct btrfs_fs_info *fs_info = data;
4193 	int ret = 0;
4194 
4195 	mutex_lock(&fs_info->balance_mutex);
4196 	if (fs_info->balance_ctl)
4197 		ret = btrfs_balance(fs_info, fs_info->balance_ctl, NULL);
4198 	mutex_unlock(&fs_info->balance_mutex);
4199 
4200 	return ret;
4201 }
4202 
4203 int btrfs_resume_balance_async(struct btrfs_fs_info *fs_info)
4204 {
4205 	struct task_struct *tsk;
4206 
4207 	mutex_lock(&fs_info->balance_mutex);
4208 	if (!fs_info->balance_ctl) {
4209 		mutex_unlock(&fs_info->balance_mutex);
4210 		return 0;
4211 	}
4212 	mutex_unlock(&fs_info->balance_mutex);
4213 
4214 	if (btrfs_test_opt(fs_info, SKIP_BALANCE)) {
4215 		btrfs_info(fs_info, "balance: resume skipped");
4216 		return 0;
4217 	}
4218 
4219 	/*
4220 	 * A ro->rw remount sequence should continue with the paused balance
4221 	 * regardless of who pauses it, system or the user as of now, so set
4222 	 * the resume flag.
4223 	 */
4224 	spin_lock(&fs_info->balance_lock);
4225 	fs_info->balance_ctl->flags |= BTRFS_BALANCE_RESUME;
4226 	spin_unlock(&fs_info->balance_lock);
4227 
4228 	tsk = kthread_run(balance_kthread, fs_info, "btrfs-balance");
4229 	return PTR_ERR_OR_ZERO(tsk);
4230 }
4231 
4232 int btrfs_recover_balance(struct btrfs_fs_info *fs_info)
4233 {
4234 	struct btrfs_balance_control *bctl;
4235 	struct btrfs_balance_item *item;
4236 	struct btrfs_disk_balance_args disk_bargs;
4237 	struct btrfs_path *path;
4238 	struct extent_buffer *leaf;
4239 	struct btrfs_key key;
4240 	int ret;
4241 
4242 	path = btrfs_alloc_path();
4243 	if (!path)
4244 		return -ENOMEM;
4245 
4246 	key.objectid = BTRFS_BALANCE_OBJECTID;
4247 	key.type = BTRFS_TEMPORARY_ITEM_KEY;
4248 	key.offset = 0;
4249 
4250 	ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
4251 	if (ret < 0)
4252 		goto out;
4253 	if (ret > 0) { /* ret = -ENOENT; */
4254 		ret = 0;
4255 		goto out;
4256 	}
4257 
4258 	bctl = kzalloc(sizeof(*bctl), GFP_NOFS);
4259 	if (!bctl) {
4260 		ret = -ENOMEM;
4261 		goto out;
4262 	}
4263 
4264 	leaf = path->nodes[0];
4265 	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_balance_item);
4266 
4267 	bctl->flags = btrfs_balance_flags(leaf, item);
4268 	bctl->flags |= BTRFS_BALANCE_RESUME;
4269 
4270 	btrfs_balance_data(leaf, item, &disk_bargs);
4271 	btrfs_disk_balance_args_to_cpu(&bctl->data, &disk_bargs);
4272 	btrfs_balance_meta(leaf, item, &disk_bargs);
4273 	btrfs_disk_balance_args_to_cpu(&bctl->meta, &disk_bargs);
4274 	btrfs_balance_sys(leaf, item, &disk_bargs);
4275 	btrfs_disk_balance_args_to_cpu(&bctl->sys, &disk_bargs);
4276 
4277 	/*
4278 	 * This should never happen, as the paused balance state is recovered
4279 	 * during mount without any chance of other exclusive ops to collide.
4280 	 *
4281 	 * This gives the exclusive op status to balance and keeps in paused
4282 	 * state until user intervention (cancel or umount). If the ownership
4283 	 * cannot be assigned, show a message but do not fail. The balance
4284 	 * is in a paused state and must have fs_info::balance_ctl properly
4285 	 * set up.
4286 	 */
4287 	if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_BALANCE))
4288 		btrfs_warn(fs_info,
4289 	"balance: cannot set exclusive op status, resume manually");
4290 
4291 	mutex_lock(&fs_info->balance_mutex);
4292 	BUG_ON(fs_info->balance_ctl);
4293 	spin_lock(&fs_info->balance_lock);
4294 	fs_info->balance_ctl = bctl;
4295 	spin_unlock(&fs_info->balance_lock);
4296 	mutex_unlock(&fs_info->balance_mutex);
4297 out:
4298 	btrfs_free_path(path);
4299 	return ret;
4300 }
4301 
4302 int btrfs_pause_balance(struct btrfs_fs_info *fs_info)
4303 {
4304 	int ret = 0;
4305 
4306 	mutex_lock(&fs_info->balance_mutex);
4307 	if (!fs_info->balance_ctl) {
4308 		mutex_unlock(&fs_info->balance_mutex);
4309 		return -ENOTCONN;
4310 	}
4311 
4312 	if (test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) {
4313 		atomic_inc(&fs_info->balance_pause_req);
4314 		mutex_unlock(&fs_info->balance_mutex);
4315 
4316 		wait_event(fs_info->balance_wait_q,
4317 			   !test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4318 
4319 		mutex_lock(&fs_info->balance_mutex);
4320 		/* we are good with balance_ctl ripped off from under us */
4321 		BUG_ON(test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4322 		atomic_dec(&fs_info->balance_pause_req);
4323 	} else {
4324 		ret = -ENOTCONN;
4325 	}
4326 
4327 	mutex_unlock(&fs_info->balance_mutex);
4328 	return ret;
4329 }
4330 
4331 int btrfs_cancel_balance(struct btrfs_fs_info *fs_info)
4332 {
4333 	mutex_lock(&fs_info->balance_mutex);
4334 	if (!fs_info->balance_ctl) {
4335 		mutex_unlock(&fs_info->balance_mutex);
4336 		return -ENOTCONN;
4337 	}
4338 
4339 	/*
4340 	 * A paused balance with the item stored on disk can be resumed at
4341 	 * mount time if the mount is read-write. Otherwise it's still paused
4342 	 * and we must not allow cancelling as it deletes the item.
4343 	 */
4344 	if (sb_rdonly(fs_info->sb)) {
4345 		mutex_unlock(&fs_info->balance_mutex);
4346 		return -EROFS;
4347 	}
4348 
4349 	atomic_inc(&fs_info->balance_cancel_req);
4350 	/*
4351 	 * if we are running just wait and return, balance item is
4352 	 * deleted in btrfs_balance in this case
4353 	 */
4354 	if (test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) {
4355 		mutex_unlock(&fs_info->balance_mutex);
4356 		wait_event(fs_info->balance_wait_q,
4357 			   !test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4358 		mutex_lock(&fs_info->balance_mutex);
4359 	} else {
4360 		mutex_unlock(&fs_info->balance_mutex);
4361 		/*
4362 		 * Lock released to allow other waiters to continue, we'll
4363 		 * reexamine the status again.
4364 		 */
4365 		mutex_lock(&fs_info->balance_mutex);
4366 
4367 		if (fs_info->balance_ctl) {
4368 			reset_balance_state(fs_info);
4369 			btrfs_exclop_finish(fs_info);
4370 			btrfs_info(fs_info, "balance: canceled");
4371 		}
4372 	}
4373 
4374 	BUG_ON(fs_info->balance_ctl ||
4375 		test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4376 	atomic_dec(&fs_info->balance_cancel_req);
4377 	mutex_unlock(&fs_info->balance_mutex);
4378 	return 0;
4379 }
4380 
4381 int btrfs_uuid_scan_kthread(void *data)
4382 {
4383 	struct btrfs_fs_info *fs_info = data;
4384 	struct btrfs_root *root = fs_info->tree_root;
4385 	struct btrfs_key key;
4386 	struct btrfs_path *path = NULL;
4387 	int ret = 0;
4388 	struct extent_buffer *eb;
4389 	int slot;
4390 	struct btrfs_root_item root_item;
4391 	u32 item_size;
4392 	struct btrfs_trans_handle *trans = NULL;
4393 	bool closing = false;
4394 
4395 	path = btrfs_alloc_path();
4396 	if (!path) {
4397 		ret = -ENOMEM;
4398 		goto out;
4399 	}
4400 
4401 	key.objectid = 0;
4402 	key.type = BTRFS_ROOT_ITEM_KEY;
4403 	key.offset = 0;
4404 
4405 	while (1) {
4406 		if (btrfs_fs_closing(fs_info)) {
4407 			closing = true;
4408 			break;
4409 		}
4410 		ret = btrfs_search_forward(root, &key, path,
4411 				BTRFS_OLDEST_GENERATION);
4412 		if (ret) {
4413 			if (ret > 0)
4414 				ret = 0;
4415 			break;
4416 		}
4417 
4418 		if (key.type != BTRFS_ROOT_ITEM_KEY ||
4419 		    (key.objectid < BTRFS_FIRST_FREE_OBJECTID &&
4420 		     key.objectid != BTRFS_FS_TREE_OBJECTID) ||
4421 		    key.objectid > BTRFS_LAST_FREE_OBJECTID)
4422 			goto skip;
4423 
4424 		eb = path->nodes[0];
4425 		slot = path->slots[0];
4426 		item_size = btrfs_item_size_nr(eb, slot);
4427 		if (item_size < sizeof(root_item))
4428 			goto skip;
4429 
4430 		read_extent_buffer(eb, &root_item,
4431 				   btrfs_item_ptr_offset(eb, slot),
4432 				   (int)sizeof(root_item));
4433 		if (btrfs_root_refs(&root_item) == 0)
4434 			goto skip;
4435 
4436 		if (!btrfs_is_empty_uuid(root_item.uuid) ||
4437 		    !btrfs_is_empty_uuid(root_item.received_uuid)) {
4438 			if (trans)
4439 				goto update_tree;
4440 
4441 			btrfs_release_path(path);
4442 			/*
4443 			 * 1 - subvol uuid item
4444 			 * 1 - received_subvol uuid item
4445 			 */
4446 			trans = btrfs_start_transaction(fs_info->uuid_root, 2);
4447 			if (IS_ERR(trans)) {
4448 				ret = PTR_ERR(trans);
4449 				break;
4450 			}
4451 			continue;
4452 		} else {
4453 			goto skip;
4454 		}
4455 update_tree:
4456 		btrfs_release_path(path);
4457 		if (!btrfs_is_empty_uuid(root_item.uuid)) {
4458 			ret = btrfs_uuid_tree_add(trans, root_item.uuid,
4459 						  BTRFS_UUID_KEY_SUBVOL,
4460 						  key.objectid);
4461 			if (ret < 0) {
4462 				btrfs_warn(fs_info, "uuid_tree_add failed %d",
4463 					ret);
4464 				break;
4465 			}
4466 		}
4467 
4468 		if (!btrfs_is_empty_uuid(root_item.received_uuid)) {
4469 			ret = btrfs_uuid_tree_add(trans,
4470 						  root_item.received_uuid,
4471 						 BTRFS_UUID_KEY_RECEIVED_SUBVOL,
4472 						  key.objectid);
4473 			if (ret < 0) {
4474 				btrfs_warn(fs_info, "uuid_tree_add failed %d",
4475 					ret);
4476 				break;
4477 			}
4478 		}
4479 
4480 skip:
4481 		btrfs_release_path(path);
4482 		if (trans) {
4483 			ret = btrfs_end_transaction(trans);
4484 			trans = NULL;
4485 			if (ret)
4486 				break;
4487 		}
4488 
4489 		if (key.offset < (u64)-1) {
4490 			key.offset++;
4491 		} else if (key.type < BTRFS_ROOT_ITEM_KEY) {
4492 			key.offset = 0;
4493 			key.type = BTRFS_ROOT_ITEM_KEY;
4494 		} else if (key.objectid < (u64)-1) {
4495 			key.offset = 0;
4496 			key.type = BTRFS_ROOT_ITEM_KEY;
4497 			key.objectid++;
4498 		} else {
4499 			break;
4500 		}
4501 		cond_resched();
4502 	}
4503 
4504 out:
4505 	btrfs_free_path(path);
4506 	if (trans && !IS_ERR(trans))
4507 		btrfs_end_transaction(trans);
4508 	if (ret)
4509 		btrfs_warn(fs_info, "btrfs_uuid_scan_kthread failed %d", ret);
4510 	else if (!closing)
4511 		set_bit(BTRFS_FS_UPDATE_UUID_TREE_GEN, &fs_info->flags);
4512 	up(&fs_info->uuid_tree_rescan_sem);
4513 	return 0;
4514 }
4515 
4516 int btrfs_create_uuid_tree(struct btrfs_fs_info *fs_info)
4517 {
4518 	struct btrfs_trans_handle *trans;
4519 	struct btrfs_root *tree_root = fs_info->tree_root;
4520 	struct btrfs_root *uuid_root;
4521 	struct task_struct *task;
4522 	int ret;
4523 
4524 	/*
4525 	 * 1 - root node
4526 	 * 1 - root item
4527 	 */
4528 	trans = btrfs_start_transaction(tree_root, 2);
4529 	if (IS_ERR(trans))
4530 		return PTR_ERR(trans);
4531 
4532 	uuid_root = btrfs_create_tree(trans, BTRFS_UUID_TREE_OBJECTID);
4533 	if (IS_ERR(uuid_root)) {
4534 		ret = PTR_ERR(uuid_root);
4535 		btrfs_abort_transaction(trans, ret);
4536 		btrfs_end_transaction(trans);
4537 		return ret;
4538 	}
4539 
4540 	fs_info->uuid_root = uuid_root;
4541 
4542 	ret = btrfs_commit_transaction(trans);
4543 	if (ret)
4544 		return ret;
4545 
4546 	down(&fs_info->uuid_tree_rescan_sem);
4547 	task = kthread_run(btrfs_uuid_scan_kthread, fs_info, "btrfs-uuid");
4548 	if (IS_ERR(task)) {
4549 		/* fs_info->update_uuid_tree_gen remains 0 in all error case */
4550 		btrfs_warn(fs_info, "failed to start uuid_scan task");
4551 		up(&fs_info->uuid_tree_rescan_sem);
4552 		return PTR_ERR(task);
4553 	}
4554 
4555 	return 0;
4556 }
4557 
4558 /*
4559  * shrinking a device means finding all of the device extents past
4560  * the new size, and then following the back refs to the chunks.
4561  * The chunk relocation code actually frees the device extent
4562  */
4563 int btrfs_shrink_device(struct btrfs_device *device, u64 new_size)
4564 {
4565 	struct btrfs_fs_info *fs_info = device->fs_info;
4566 	struct btrfs_root *root = fs_info->dev_root;
4567 	struct btrfs_trans_handle *trans;
4568 	struct btrfs_dev_extent *dev_extent = NULL;
4569 	struct btrfs_path *path;
4570 	u64 length;
4571 	u64 chunk_offset;
4572 	int ret;
4573 	int slot;
4574 	int failed = 0;
4575 	bool retried = false;
4576 	struct extent_buffer *l;
4577 	struct btrfs_key key;
4578 	struct btrfs_super_block *super_copy = fs_info->super_copy;
4579 	u64 old_total = btrfs_super_total_bytes(super_copy);
4580 	u64 old_size = btrfs_device_get_total_bytes(device);
4581 	u64 diff;
4582 	u64 start;
4583 
4584 	new_size = round_down(new_size, fs_info->sectorsize);
4585 	start = new_size;
4586 	diff = round_down(old_size - new_size, fs_info->sectorsize);
4587 
4588 	if (test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state))
4589 		return -EINVAL;
4590 
4591 	path = btrfs_alloc_path();
4592 	if (!path)
4593 		return -ENOMEM;
4594 
4595 	path->reada = READA_BACK;
4596 
4597 	trans = btrfs_start_transaction(root, 0);
4598 	if (IS_ERR(trans)) {
4599 		btrfs_free_path(path);
4600 		return PTR_ERR(trans);
4601 	}
4602 
4603 	mutex_lock(&fs_info->chunk_mutex);
4604 
4605 	btrfs_device_set_total_bytes(device, new_size);
4606 	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
4607 		device->fs_devices->total_rw_bytes -= diff;
4608 		atomic64_sub(diff, &fs_info->free_chunk_space);
4609 	}
4610 
4611 	/*
4612 	 * Once the device's size has been set to the new size, ensure all
4613 	 * in-memory chunks are synced to disk so that the loop below sees them
4614 	 * and relocates them accordingly.
4615 	 */
4616 	if (contains_pending_extent(device, &start, diff)) {
4617 		mutex_unlock(&fs_info->chunk_mutex);
4618 		ret = btrfs_commit_transaction(trans);
4619 		if (ret)
4620 			goto done;
4621 	} else {
4622 		mutex_unlock(&fs_info->chunk_mutex);
4623 		btrfs_end_transaction(trans);
4624 	}
4625 
4626 again:
4627 	key.objectid = device->devid;
4628 	key.offset = (u64)-1;
4629 	key.type = BTRFS_DEV_EXTENT_KEY;
4630 
4631 	do {
4632 		mutex_lock(&fs_info->delete_unused_bgs_mutex);
4633 		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
4634 		if (ret < 0) {
4635 			mutex_unlock(&fs_info->delete_unused_bgs_mutex);
4636 			goto done;
4637 		}
4638 
4639 		ret = btrfs_previous_item(root, path, 0, key.type);
4640 		if (ret)
4641 			mutex_unlock(&fs_info->delete_unused_bgs_mutex);
4642 		if (ret < 0)
4643 			goto done;
4644 		if (ret) {
4645 			ret = 0;
4646 			btrfs_release_path(path);
4647 			break;
4648 		}
4649 
4650 		l = path->nodes[0];
4651 		slot = path->slots[0];
4652 		btrfs_item_key_to_cpu(l, &key, path->slots[0]);
4653 
4654 		if (key.objectid != device->devid) {
4655 			mutex_unlock(&fs_info->delete_unused_bgs_mutex);
4656 			btrfs_release_path(path);
4657 			break;
4658 		}
4659 
4660 		dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
4661 		length = btrfs_dev_extent_length(l, dev_extent);
4662 
4663 		if (key.offset + length <= new_size) {
4664 			mutex_unlock(&fs_info->delete_unused_bgs_mutex);
4665 			btrfs_release_path(path);
4666 			break;
4667 		}
4668 
4669 		chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);
4670 		btrfs_release_path(path);
4671 
4672 		/*
4673 		 * We may be relocating the only data chunk we have,
4674 		 * which could potentially end up with losing data's
4675 		 * raid profile, so lets allocate an empty one in
4676 		 * advance.
4677 		 */
4678 		ret = btrfs_may_alloc_data_chunk(fs_info, chunk_offset);
4679 		if (ret < 0) {
4680 			mutex_unlock(&fs_info->delete_unused_bgs_mutex);
4681 			goto done;
4682 		}
4683 
4684 		ret = btrfs_relocate_chunk(fs_info, chunk_offset);
4685 		mutex_unlock(&fs_info->delete_unused_bgs_mutex);
4686 		if (ret == -ENOSPC) {
4687 			failed++;
4688 		} else if (ret) {
4689 			if (ret == -ETXTBSY) {
4690 				btrfs_warn(fs_info,
4691 		   "could not shrink block group %llu due to active swapfile",
4692 					   chunk_offset);
4693 			}
4694 			goto done;
4695 		}
4696 	} while (key.offset-- > 0);
4697 
4698 	if (failed && !retried) {
4699 		failed = 0;
4700 		retried = true;
4701 		goto again;
4702 	} else if (failed && retried) {
4703 		ret = -ENOSPC;
4704 		goto done;
4705 	}
4706 
4707 	/* Shrinking succeeded, else we would be at "done". */
4708 	trans = btrfs_start_transaction(root, 0);
4709 	if (IS_ERR(trans)) {
4710 		ret = PTR_ERR(trans);
4711 		goto done;
4712 	}
4713 
4714 	mutex_lock(&fs_info->chunk_mutex);
4715 	/* Clear all state bits beyond the shrunk device size */
4716 	clear_extent_bits(&device->alloc_state, new_size, (u64)-1,
4717 			  CHUNK_STATE_MASK);
4718 
4719 	btrfs_device_set_disk_total_bytes(device, new_size);
4720 	if (list_empty(&device->post_commit_list))
4721 		list_add_tail(&device->post_commit_list,
4722 			      &trans->transaction->dev_update_list);
4723 
4724 	WARN_ON(diff > old_total);
4725 	btrfs_set_super_total_bytes(super_copy,
4726 			round_down(old_total - diff, fs_info->sectorsize));
4727 	mutex_unlock(&fs_info->chunk_mutex);
4728 
4729 	/* Now btrfs_update_device() will change the on-disk size. */
4730 	ret = btrfs_update_device(trans, device);
4731 	if (ret < 0) {
4732 		btrfs_abort_transaction(trans, ret);
4733 		btrfs_end_transaction(trans);
4734 	} else {
4735 		ret = btrfs_commit_transaction(trans);
4736 	}
4737 done:
4738 	btrfs_free_path(path);
4739 	if (ret) {
4740 		mutex_lock(&fs_info->chunk_mutex);
4741 		btrfs_device_set_total_bytes(device, old_size);
4742 		if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state))
4743 			device->fs_devices->total_rw_bytes += diff;
4744 		atomic64_add(diff, &fs_info->free_chunk_space);
4745 		mutex_unlock(&fs_info->chunk_mutex);
4746 	}
4747 	return ret;
4748 }
4749 
4750 static int btrfs_add_system_chunk(struct btrfs_fs_info *fs_info,
4751 			   struct btrfs_key *key,
4752 			   struct btrfs_chunk *chunk, int item_size)
4753 {
4754 	struct btrfs_super_block *super_copy = fs_info->super_copy;
4755 	struct btrfs_disk_key disk_key;
4756 	u32 array_size;
4757 	u8 *ptr;
4758 
4759 	mutex_lock(&fs_info->chunk_mutex);
4760 	array_size = btrfs_super_sys_array_size(super_copy);
4761 	if (array_size + item_size + sizeof(disk_key)
4762 			> BTRFS_SYSTEM_CHUNK_ARRAY_SIZE) {
4763 		mutex_unlock(&fs_info->chunk_mutex);
4764 		return -EFBIG;
4765 	}
4766 
4767 	ptr = super_copy->sys_chunk_array + array_size;
4768 	btrfs_cpu_key_to_disk(&disk_key, key);
4769 	memcpy(ptr, &disk_key, sizeof(disk_key));
4770 	ptr += sizeof(disk_key);
4771 	memcpy(ptr, chunk, item_size);
4772 	item_size += sizeof(disk_key);
4773 	btrfs_set_super_sys_array_size(super_copy, array_size + item_size);
4774 	mutex_unlock(&fs_info->chunk_mutex);
4775 
4776 	return 0;
4777 }
4778 
4779 /*
4780  * sort the devices in descending order by max_avail, total_avail
4781  */
4782 static int btrfs_cmp_device_info(const void *a, const void *b)
4783 {
4784 	const struct btrfs_device_info *di_a = a;
4785 	const struct btrfs_device_info *di_b = b;
4786 
4787 	if (di_a->max_avail > di_b->max_avail)
4788 		return -1;
4789 	if (di_a->max_avail < di_b->max_avail)
4790 		return 1;
4791 	if (di_a->total_avail > di_b->total_avail)
4792 		return -1;
4793 	if (di_a->total_avail < di_b->total_avail)
4794 		return 1;
4795 	return 0;
4796 }
4797 
4798 static void check_raid56_incompat_flag(struct btrfs_fs_info *info, u64 type)
4799 {
4800 	if (!(type & BTRFS_BLOCK_GROUP_RAID56_MASK))
4801 		return;
4802 
4803 	btrfs_set_fs_incompat(info, RAID56);
4804 }
4805 
4806 static void check_raid1c34_incompat_flag(struct btrfs_fs_info *info, u64 type)
4807 {
4808 	if (!(type & (BTRFS_BLOCK_GROUP_RAID1C3 | BTRFS_BLOCK_GROUP_RAID1C4)))
4809 		return;
4810 
4811 	btrfs_set_fs_incompat(info, RAID1C34);
4812 }
4813 
4814 /*
4815  * Structure used internally for __btrfs_alloc_chunk() function.
4816  * Wraps needed parameters.
4817  */
4818 struct alloc_chunk_ctl {
4819 	u64 start;
4820 	u64 type;
4821 	/* Total number of stripes to allocate */
4822 	int num_stripes;
4823 	/* sub_stripes info for map */
4824 	int sub_stripes;
4825 	/* Stripes per device */
4826 	int dev_stripes;
4827 	/* Maximum number of devices to use */
4828 	int devs_max;
4829 	/* Minimum number of devices to use */
4830 	int devs_min;
4831 	/* ndevs has to be a multiple of this */
4832 	int devs_increment;
4833 	/* Number of copies */
4834 	int ncopies;
4835 	/* Number of stripes worth of bytes to store parity information */
4836 	int nparity;
4837 	u64 max_stripe_size;
4838 	u64 max_chunk_size;
4839 	u64 dev_extent_min;
4840 	u64 stripe_size;
4841 	u64 chunk_size;
4842 	int ndevs;
4843 };
4844 
4845 static void init_alloc_chunk_ctl_policy_regular(
4846 				struct btrfs_fs_devices *fs_devices,
4847 				struct alloc_chunk_ctl *ctl)
4848 {
4849 	u64 type = ctl->type;
4850 
4851 	if (type & BTRFS_BLOCK_GROUP_DATA) {
4852 		ctl->max_stripe_size = SZ_1G;
4853 		ctl->max_chunk_size = BTRFS_MAX_DATA_CHUNK_SIZE;
4854 	} else if (type & BTRFS_BLOCK_GROUP_METADATA) {
4855 		/* For larger filesystems, use larger metadata chunks */
4856 		if (fs_devices->total_rw_bytes > 50ULL * SZ_1G)
4857 			ctl->max_stripe_size = SZ_1G;
4858 		else
4859 			ctl->max_stripe_size = SZ_256M;
4860 		ctl->max_chunk_size = ctl->max_stripe_size;
4861 	} else if (type & BTRFS_BLOCK_GROUP_SYSTEM) {
4862 		ctl->max_stripe_size = SZ_32M;
4863 		ctl->max_chunk_size = 2 * ctl->max_stripe_size;
4864 		ctl->devs_max = min_t(int, ctl->devs_max,
4865 				      BTRFS_MAX_DEVS_SYS_CHUNK);
4866 	} else {
4867 		BUG();
4868 	}
4869 
4870 	/* We don't want a chunk larger than 10% of writable space */
4871 	ctl->max_chunk_size = min(div_factor(fs_devices->total_rw_bytes, 1),
4872 				  ctl->max_chunk_size);
4873 	ctl->dev_extent_min = BTRFS_STRIPE_LEN * ctl->dev_stripes;
4874 }
4875 
4876 static void init_alloc_chunk_ctl(struct btrfs_fs_devices *fs_devices,
4877 				 struct alloc_chunk_ctl *ctl)
4878 {
4879 	int index = btrfs_bg_flags_to_raid_index(ctl->type);
4880 
4881 	ctl->sub_stripes = btrfs_raid_array[index].sub_stripes;
4882 	ctl->dev_stripes = btrfs_raid_array[index].dev_stripes;
4883 	ctl->devs_max = btrfs_raid_array[index].devs_max;
4884 	if (!ctl->devs_max)
4885 		ctl->devs_max = BTRFS_MAX_DEVS(fs_devices->fs_info);
4886 	ctl->devs_min = btrfs_raid_array[index].devs_min;
4887 	ctl->devs_increment = btrfs_raid_array[index].devs_increment;
4888 	ctl->ncopies = btrfs_raid_array[index].ncopies;
4889 	ctl->nparity = btrfs_raid_array[index].nparity;
4890 	ctl->ndevs = 0;
4891 
4892 	switch (fs_devices->chunk_alloc_policy) {
4893 	case BTRFS_CHUNK_ALLOC_REGULAR:
4894 		init_alloc_chunk_ctl_policy_regular(fs_devices, ctl);
4895 		break;
4896 	default:
4897 		BUG();
4898 	}
4899 }
4900 
4901 static int gather_device_info(struct btrfs_fs_devices *fs_devices,
4902 			      struct alloc_chunk_ctl *ctl,
4903 			      struct btrfs_device_info *devices_info)
4904 {
4905 	struct btrfs_fs_info *info = fs_devices->fs_info;
4906 	struct btrfs_device *device;
4907 	u64 total_avail;
4908 	u64 dev_extent_want = ctl->max_stripe_size * ctl->dev_stripes;
4909 	int ret;
4910 	int ndevs = 0;
4911 	u64 max_avail;
4912 	u64 dev_offset;
4913 
4914 	/*
4915 	 * in the first pass through the devices list, we gather information
4916 	 * about the available holes on each device.
4917 	 */
4918 	list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) {
4919 		if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
4920 			WARN(1, KERN_ERR
4921 			       "BTRFS: read-only device in alloc_list\n");
4922 			continue;
4923 		}
4924 
4925 		if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
4926 					&device->dev_state) ||
4927 		    test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state))
4928 			continue;
4929 
4930 		if (device->total_bytes > device->bytes_used)
4931 			total_avail = device->total_bytes - device->bytes_used;
4932 		else
4933 			total_avail = 0;
4934 
4935 		/* If there is no space on this device, skip it. */
4936 		if (total_avail < ctl->dev_extent_min)
4937 			continue;
4938 
4939 		ret = find_free_dev_extent(device, dev_extent_want, &dev_offset,
4940 					   &max_avail);
4941 		if (ret && ret != -ENOSPC)
4942 			return ret;
4943 
4944 		if (ret == 0)
4945 			max_avail = dev_extent_want;
4946 
4947 		if (max_avail < ctl->dev_extent_min) {
4948 			if (btrfs_test_opt(info, ENOSPC_DEBUG))
4949 				btrfs_debug(info,
4950 			"%s: devid %llu has no free space, have=%llu want=%llu",
4951 					    __func__, device->devid, max_avail,
4952 					    ctl->dev_extent_min);
4953 			continue;
4954 		}
4955 
4956 		if (ndevs == fs_devices->rw_devices) {
4957 			WARN(1, "%s: found more than %llu devices\n",
4958 			     __func__, fs_devices->rw_devices);
4959 			break;
4960 		}
4961 		devices_info[ndevs].dev_offset = dev_offset;
4962 		devices_info[ndevs].max_avail = max_avail;
4963 		devices_info[ndevs].total_avail = total_avail;
4964 		devices_info[ndevs].dev = device;
4965 		++ndevs;
4966 	}
4967 	ctl->ndevs = ndevs;
4968 
4969 	/*
4970 	 * now sort the devices by hole size / available space
4971 	 */
4972 	sort(devices_info, ndevs, sizeof(struct btrfs_device_info),
4973 	     btrfs_cmp_device_info, NULL);
4974 
4975 	return 0;
4976 }
4977 
4978 static int decide_stripe_size_regular(struct alloc_chunk_ctl *ctl,
4979 				      struct btrfs_device_info *devices_info)
4980 {
4981 	/* Number of stripes that count for block group size */
4982 	int data_stripes;
4983 
4984 	/*
4985 	 * The primary goal is to maximize the number of stripes, so use as
4986 	 * many devices as possible, even if the stripes are not maximum sized.
4987 	 *
4988 	 * The DUP profile stores more than one stripe per device, the
4989 	 * max_avail is the total size so we have to adjust.
4990 	 */
4991 	ctl->stripe_size = div_u64(devices_info[ctl->ndevs - 1].max_avail,
4992 				   ctl->dev_stripes);
4993 	ctl->num_stripes = ctl->ndevs * ctl->dev_stripes;
4994 
4995 	/* This will have to be fixed for RAID1 and RAID10 over more drives */
4996 	data_stripes = (ctl->num_stripes - ctl->nparity) / ctl->ncopies;
4997 
4998 	/*
4999 	 * Use the number of data stripes to figure out how big this chunk is
5000 	 * really going to be in terms of logical address space, and compare
5001 	 * that answer with the max chunk size. If it's higher, we try to
5002 	 * reduce stripe_size.
5003 	 */
5004 	if (ctl->stripe_size * data_stripes > ctl->max_chunk_size) {
5005 		/*
5006 		 * Reduce stripe_size, round it up to a 16MB boundary again and
5007 		 * then use it, unless it ends up being even bigger than the
5008 		 * previous value we had already.
5009 		 */
5010 		ctl->stripe_size = min(round_up(div_u64(ctl->max_chunk_size,
5011 							data_stripes), SZ_16M),
5012 				       ctl->stripe_size);
5013 	}
5014 
5015 	/* Align to BTRFS_STRIPE_LEN */
5016 	ctl->stripe_size = round_down(ctl->stripe_size, BTRFS_STRIPE_LEN);
5017 	ctl->chunk_size = ctl->stripe_size * data_stripes;
5018 
5019 	return 0;
5020 }
5021 
5022 static int decide_stripe_size(struct btrfs_fs_devices *fs_devices,
5023 			      struct alloc_chunk_ctl *ctl,
5024 			      struct btrfs_device_info *devices_info)
5025 {
5026 	struct btrfs_fs_info *info = fs_devices->fs_info;
5027 
5028 	/*
5029 	 * Round down to number of usable stripes, devs_increment can be any
5030 	 * number so we can't use round_down() that requires power of 2, while
5031 	 * rounddown is safe.
5032 	 */
5033 	ctl->ndevs = rounddown(ctl->ndevs, ctl->devs_increment);
5034 
5035 	if (ctl->ndevs < ctl->devs_min) {
5036 		if (btrfs_test_opt(info, ENOSPC_DEBUG)) {
5037 			btrfs_debug(info,
5038 	"%s: not enough devices with free space: have=%d minimum required=%d",
5039 				    __func__, ctl->ndevs, ctl->devs_min);
5040 		}
5041 		return -ENOSPC;
5042 	}
5043 
5044 	ctl->ndevs = min(ctl->ndevs, ctl->devs_max);
5045 
5046 	switch (fs_devices->chunk_alloc_policy) {
5047 	case BTRFS_CHUNK_ALLOC_REGULAR:
5048 		return decide_stripe_size_regular(ctl, devices_info);
5049 	default:
5050 		BUG();
5051 	}
5052 }
5053 
5054 static int create_chunk(struct btrfs_trans_handle *trans,
5055 			struct alloc_chunk_ctl *ctl,
5056 			struct btrfs_device_info *devices_info)
5057 {
5058 	struct btrfs_fs_info *info = trans->fs_info;
5059 	struct map_lookup *map = NULL;
5060 	struct extent_map_tree *em_tree;
5061 	struct extent_map *em;
5062 	u64 start = ctl->start;
5063 	u64 type = ctl->type;
5064 	int ret;
5065 	int i;
5066 	int j;
5067 
5068 	map = kmalloc(map_lookup_size(ctl->num_stripes), GFP_NOFS);
5069 	if (!map)
5070 		return -ENOMEM;
5071 	map->num_stripes = ctl->num_stripes;
5072 
5073 	for (i = 0; i < ctl->ndevs; ++i) {
5074 		for (j = 0; j < ctl->dev_stripes; ++j) {
5075 			int s = i * ctl->dev_stripes + j;
5076 			map->stripes[s].dev = devices_info[i].dev;
5077 			map->stripes[s].physical = devices_info[i].dev_offset +
5078 						   j * ctl->stripe_size;
5079 		}
5080 	}
5081 	map->stripe_len = BTRFS_STRIPE_LEN;
5082 	map->io_align = BTRFS_STRIPE_LEN;
5083 	map->io_width = BTRFS_STRIPE_LEN;
5084 	map->type = type;
5085 	map->sub_stripes = ctl->sub_stripes;
5086 
5087 	trace_btrfs_chunk_alloc(info, map, start, ctl->chunk_size);
5088 
5089 	em = alloc_extent_map();
5090 	if (!em) {
5091 		kfree(map);
5092 		return -ENOMEM;
5093 	}
5094 	set_bit(EXTENT_FLAG_FS_MAPPING, &em->flags);
5095 	em->map_lookup = map;
5096 	em->start = start;
5097 	em->len = ctl->chunk_size;
5098 	em->block_start = 0;
5099 	em->block_len = em->len;
5100 	em->orig_block_len = ctl->stripe_size;
5101 
5102 	em_tree = &info->mapping_tree;
5103 	write_lock(&em_tree->lock);
5104 	ret = add_extent_mapping(em_tree, em, 0);
5105 	if (ret) {
5106 		write_unlock(&em_tree->lock);
5107 		free_extent_map(em);
5108 		return ret;
5109 	}
5110 	write_unlock(&em_tree->lock);
5111 
5112 	ret = btrfs_make_block_group(trans, 0, type, start, ctl->chunk_size);
5113 	if (ret)
5114 		goto error_del_extent;
5115 
5116 	for (i = 0; i < map->num_stripes; i++) {
5117 		struct btrfs_device *dev = map->stripes[i].dev;
5118 
5119 		btrfs_device_set_bytes_used(dev,
5120 					    dev->bytes_used + ctl->stripe_size);
5121 		if (list_empty(&dev->post_commit_list))
5122 			list_add_tail(&dev->post_commit_list,
5123 				      &trans->transaction->dev_update_list);
5124 	}
5125 
5126 	atomic64_sub(ctl->stripe_size * map->num_stripes,
5127 		     &info->free_chunk_space);
5128 
5129 	free_extent_map(em);
5130 	check_raid56_incompat_flag(info, type);
5131 	check_raid1c34_incompat_flag(info, type);
5132 
5133 	return 0;
5134 
5135 error_del_extent:
5136 	write_lock(&em_tree->lock);
5137 	remove_extent_mapping(em_tree, em);
5138 	write_unlock(&em_tree->lock);
5139 
5140 	/* One for our allocation */
5141 	free_extent_map(em);
5142 	/* One for the tree reference */
5143 	free_extent_map(em);
5144 
5145 	return ret;
5146 }
5147 
5148 int btrfs_alloc_chunk(struct btrfs_trans_handle *trans, u64 type)
5149 {
5150 	struct btrfs_fs_info *info = trans->fs_info;
5151 	struct btrfs_fs_devices *fs_devices = info->fs_devices;
5152 	struct btrfs_device_info *devices_info = NULL;
5153 	struct alloc_chunk_ctl ctl;
5154 	int ret;
5155 
5156 	lockdep_assert_held(&info->chunk_mutex);
5157 
5158 	if (!alloc_profile_is_valid(type, 0)) {
5159 		ASSERT(0);
5160 		return -EINVAL;
5161 	}
5162 
5163 	if (list_empty(&fs_devices->alloc_list)) {
5164 		if (btrfs_test_opt(info, ENOSPC_DEBUG))
5165 			btrfs_debug(info, "%s: no writable device", __func__);
5166 		return -ENOSPC;
5167 	}
5168 
5169 	if (!(type & BTRFS_BLOCK_GROUP_TYPE_MASK)) {
5170 		btrfs_err(info, "invalid chunk type 0x%llx requested", type);
5171 		ASSERT(0);
5172 		return -EINVAL;
5173 	}
5174 
5175 	ctl.start = find_next_chunk(info);
5176 	ctl.type = type;
5177 	init_alloc_chunk_ctl(fs_devices, &ctl);
5178 
5179 	devices_info = kcalloc(fs_devices->rw_devices, sizeof(*devices_info),
5180 			       GFP_NOFS);
5181 	if (!devices_info)
5182 		return -ENOMEM;
5183 
5184 	ret = gather_device_info(fs_devices, &ctl, devices_info);
5185 	if (ret < 0)
5186 		goto out;
5187 
5188 	ret = decide_stripe_size(fs_devices, &ctl, devices_info);
5189 	if (ret < 0)
5190 		goto out;
5191 
5192 	ret = create_chunk(trans, &ctl, devices_info);
5193 
5194 out:
5195 	kfree(devices_info);
5196 	return ret;
5197 }
5198 
5199 /*
5200  * Chunk allocation falls into two parts. The first part does work
5201  * that makes the new allocated chunk usable, but does not do any operation
5202  * that modifies the chunk tree. The second part does the work that
5203  * requires modifying the chunk tree. This division is important for the
5204  * bootstrap process of adding storage to a seed btrfs.
5205  */
5206 int btrfs_finish_chunk_alloc(struct btrfs_trans_handle *trans,
5207 			     u64 chunk_offset, u64 chunk_size)
5208 {
5209 	struct btrfs_fs_info *fs_info = trans->fs_info;
5210 	struct btrfs_root *extent_root = fs_info->extent_root;
5211 	struct btrfs_root *chunk_root = fs_info->chunk_root;
5212 	struct btrfs_key key;
5213 	struct btrfs_device *device;
5214 	struct btrfs_chunk *chunk;
5215 	struct btrfs_stripe *stripe;
5216 	struct extent_map *em;
5217 	struct map_lookup *map;
5218 	size_t item_size;
5219 	u64 dev_offset;
5220 	u64 stripe_size;
5221 	int i = 0;
5222 	int ret = 0;
5223 
5224 	em = btrfs_get_chunk_map(fs_info, chunk_offset, chunk_size);
5225 	if (IS_ERR(em))
5226 		return PTR_ERR(em);
5227 
5228 	map = em->map_lookup;
5229 	item_size = btrfs_chunk_item_size(map->num_stripes);
5230 	stripe_size = em->orig_block_len;
5231 
5232 	chunk = kzalloc(item_size, GFP_NOFS);
5233 	if (!chunk) {
5234 		ret = -ENOMEM;
5235 		goto out;
5236 	}
5237 
5238 	/*
5239 	 * Take the device list mutex to prevent races with the final phase of
5240 	 * a device replace operation that replaces the device object associated
5241 	 * with the map's stripes, because the device object's id can change
5242 	 * at any time during that final phase of the device replace operation
5243 	 * (dev-replace.c:btrfs_dev_replace_finishing()).
5244 	 */
5245 	mutex_lock(&fs_info->fs_devices->device_list_mutex);
5246 	for (i = 0; i < map->num_stripes; i++) {
5247 		device = map->stripes[i].dev;
5248 		dev_offset = map->stripes[i].physical;
5249 
5250 		ret = btrfs_update_device(trans, device);
5251 		if (ret)
5252 			break;
5253 		ret = btrfs_alloc_dev_extent(trans, device, chunk_offset,
5254 					     dev_offset, stripe_size);
5255 		if (ret)
5256 			break;
5257 	}
5258 	if (ret) {
5259 		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
5260 		goto out;
5261 	}
5262 
5263 	stripe = &chunk->stripe;
5264 	for (i = 0; i < map->num_stripes; i++) {
5265 		device = map->stripes[i].dev;
5266 		dev_offset = map->stripes[i].physical;
5267 
5268 		btrfs_set_stack_stripe_devid(stripe, device->devid);
5269 		btrfs_set_stack_stripe_offset(stripe, dev_offset);
5270 		memcpy(stripe->dev_uuid, device->uuid, BTRFS_UUID_SIZE);
5271 		stripe++;
5272 	}
5273 	mutex_unlock(&fs_info->fs_devices->device_list_mutex);
5274 
5275 	btrfs_set_stack_chunk_length(chunk, chunk_size);
5276 	btrfs_set_stack_chunk_owner(chunk, extent_root->root_key.objectid);
5277 	btrfs_set_stack_chunk_stripe_len(chunk, map->stripe_len);
5278 	btrfs_set_stack_chunk_type(chunk, map->type);
5279 	btrfs_set_stack_chunk_num_stripes(chunk, map->num_stripes);
5280 	btrfs_set_stack_chunk_io_align(chunk, map->stripe_len);
5281 	btrfs_set_stack_chunk_io_width(chunk, map->stripe_len);
5282 	btrfs_set_stack_chunk_sector_size(chunk, fs_info->sectorsize);
5283 	btrfs_set_stack_chunk_sub_stripes(chunk, map->sub_stripes);
5284 
5285 	key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
5286 	key.type = BTRFS_CHUNK_ITEM_KEY;
5287 	key.offset = chunk_offset;
5288 
5289 	ret = btrfs_insert_item(trans, chunk_root, &key, chunk, item_size);
5290 	if (ret == 0 && map->type & BTRFS_BLOCK_GROUP_SYSTEM) {
5291 		/*
5292 		 * TODO: Cleanup of inserted chunk root in case of
5293 		 * failure.
5294 		 */
5295 		ret = btrfs_add_system_chunk(fs_info, &key, chunk, item_size);
5296 	}
5297 
5298 out:
5299 	kfree(chunk);
5300 	free_extent_map(em);
5301 	return ret;
5302 }
5303 
5304 static noinline int init_first_rw_device(struct btrfs_trans_handle *trans)
5305 {
5306 	struct btrfs_fs_info *fs_info = trans->fs_info;
5307 	u64 alloc_profile;
5308 	int ret;
5309 
5310 	alloc_profile = btrfs_metadata_alloc_profile(fs_info);
5311 	ret = btrfs_alloc_chunk(trans, alloc_profile);
5312 	if (ret)
5313 		return ret;
5314 
5315 	alloc_profile = btrfs_system_alloc_profile(fs_info);
5316 	ret = btrfs_alloc_chunk(trans, alloc_profile);
5317 	return ret;
5318 }
5319 
5320 static inline int btrfs_chunk_max_errors(struct map_lookup *map)
5321 {
5322 	const int index = btrfs_bg_flags_to_raid_index(map->type);
5323 
5324 	return btrfs_raid_array[index].tolerated_failures;
5325 }
5326 
5327 int btrfs_chunk_readonly(struct btrfs_fs_info *fs_info, u64 chunk_offset)
5328 {
5329 	struct extent_map *em;
5330 	struct map_lookup *map;
5331 	int readonly = 0;
5332 	int miss_ndevs = 0;
5333 	int i;
5334 
5335 	em = btrfs_get_chunk_map(fs_info, chunk_offset, 1);
5336 	if (IS_ERR(em))
5337 		return 1;
5338 
5339 	map = em->map_lookup;
5340 	for (i = 0; i < map->num_stripes; i++) {
5341 		if (test_bit(BTRFS_DEV_STATE_MISSING,
5342 					&map->stripes[i].dev->dev_state)) {
5343 			miss_ndevs++;
5344 			continue;
5345 		}
5346 		if (!test_bit(BTRFS_DEV_STATE_WRITEABLE,
5347 					&map->stripes[i].dev->dev_state)) {
5348 			readonly = 1;
5349 			goto end;
5350 		}
5351 	}
5352 
5353 	/*
5354 	 * If the number of missing devices is larger than max errors,
5355 	 * we can not write the data into that chunk successfully, so
5356 	 * set it readonly.
5357 	 */
5358 	if (miss_ndevs > btrfs_chunk_max_errors(map))
5359 		readonly = 1;
5360 end:
5361 	free_extent_map(em);
5362 	return readonly;
5363 }
5364 
5365 void btrfs_mapping_tree_free(struct extent_map_tree *tree)
5366 {
5367 	struct extent_map *em;
5368 
5369 	while (1) {
5370 		write_lock(&tree->lock);
5371 		em = lookup_extent_mapping(tree, 0, (u64)-1);
5372 		if (em)
5373 			remove_extent_mapping(tree, em);
5374 		write_unlock(&tree->lock);
5375 		if (!em)
5376 			break;
5377 		/* once for us */
5378 		free_extent_map(em);
5379 		/* once for the tree */
5380 		free_extent_map(em);
5381 	}
5382 }
5383 
5384 int btrfs_num_copies(struct btrfs_fs_info *fs_info, u64 logical, u64 len)
5385 {
5386 	struct extent_map *em;
5387 	struct map_lookup *map;
5388 	int ret;
5389 
5390 	em = btrfs_get_chunk_map(fs_info, logical, len);
5391 	if (IS_ERR(em))
5392 		/*
5393 		 * We could return errors for these cases, but that could get
5394 		 * ugly and we'd probably do the same thing which is just not do
5395 		 * anything else and exit, so return 1 so the callers don't try
5396 		 * to use other copies.
5397 		 */
5398 		return 1;
5399 
5400 	map = em->map_lookup;
5401 	if (map->type & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1_MASK))
5402 		ret = map->num_stripes;
5403 	else if (map->type & BTRFS_BLOCK_GROUP_RAID10)
5404 		ret = map->sub_stripes;
5405 	else if (map->type & BTRFS_BLOCK_GROUP_RAID5)
5406 		ret = 2;
5407 	else if (map->type & BTRFS_BLOCK_GROUP_RAID6)
5408 		/*
5409 		 * There could be two corrupted data stripes, we need
5410 		 * to loop retry in order to rebuild the correct data.
5411 		 *
5412 		 * Fail a stripe at a time on every retry except the
5413 		 * stripe under reconstruction.
5414 		 */
5415 		ret = map->num_stripes;
5416 	else
5417 		ret = 1;
5418 	free_extent_map(em);
5419 
5420 	down_read(&fs_info->dev_replace.rwsem);
5421 	if (btrfs_dev_replace_is_ongoing(&fs_info->dev_replace) &&
5422 	    fs_info->dev_replace.tgtdev)
5423 		ret++;
5424 	up_read(&fs_info->dev_replace.rwsem);
5425 
5426 	return ret;
5427 }
5428 
5429 unsigned long btrfs_full_stripe_len(struct btrfs_fs_info *fs_info,
5430 				    u64 logical)
5431 {
5432 	struct extent_map *em;
5433 	struct map_lookup *map;
5434 	unsigned long len = fs_info->sectorsize;
5435 
5436 	em = btrfs_get_chunk_map(fs_info, logical, len);
5437 
5438 	if (!WARN_ON(IS_ERR(em))) {
5439 		map = em->map_lookup;
5440 		if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
5441 			len = map->stripe_len * nr_data_stripes(map);
5442 		free_extent_map(em);
5443 	}
5444 	return len;
5445 }
5446 
5447 int btrfs_is_parity_mirror(struct btrfs_fs_info *fs_info, u64 logical, u64 len)
5448 {
5449 	struct extent_map *em;
5450 	struct map_lookup *map;
5451 	int ret = 0;
5452 
5453 	em = btrfs_get_chunk_map(fs_info, logical, len);
5454 
5455 	if(!WARN_ON(IS_ERR(em))) {
5456 		map = em->map_lookup;
5457 		if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
5458 			ret = 1;
5459 		free_extent_map(em);
5460 	}
5461 	return ret;
5462 }
5463 
5464 static int find_live_mirror(struct btrfs_fs_info *fs_info,
5465 			    struct map_lookup *map, int first,
5466 			    int dev_replace_is_ongoing)
5467 {
5468 	int i;
5469 	int num_stripes;
5470 	int preferred_mirror;
5471 	int tolerance;
5472 	struct btrfs_device *srcdev;
5473 
5474 	ASSERT((map->type &
5475 		 (BTRFS_BLOCK_GROUP_RAID1_MASK | BTRFS_BLOCK_GROUP_RAID10)));
5476 
5477 	if (map->type & BTRFS_BLOCK_GROUP_RAID10)
5478 		num_stripes = map->sub_stripes;
5479 	else
5480 		num_stripes = map->num_stripes;
5481 
5482 	preferred_mirror = first + current->pid % num_stripes;
5483 
5484 	if (dev_replace_is_ongoing &&
5485 	    fs_info->dev_replace.cont_reading_from_srcdev_mode ==
5486 	     BTRFS_DEV_REPLACE_ITEM_CONT_READING_FROM_SRCDEV_MODE_AVOID)
5487 		srcdev = fs_info->dev_replace.srcdev;
5488 	else
5489 		srcdev = NULL;
5490 
5491 	/*
5492 	 * try to avoid the drive that is the source drive for a
5493 	 * dev-replace procedure, only choose it if no other non-missing
5494 	 * mirror is available
5495 	 */
5496 	for (tolerance = 0; tolerance < 2; tolerance++) {
5497 		if (map->stripes[preferred_mirror].dev->bdev &&
5498 		    (tolerance || map->stripes[preferred_mirror].dev != srcdev))
5499 			return preferred_mirror;
5500 		for (i = first; i < first + num_stripes; i++) {
5501 			if (map->stripes[i].dev->bdev &&
5502 			    (tolerance || map->stripes[i].dev != srcdev))
5503 				return i;
5504 		}
5505 	}
5506 
5507 	/* we couldn't find one that doesn't fail.  Just return something
5508 	 * and the io error handling code will clean up eventually
5509 	 */
5510 	return preferred_mirror;
5511 }
5512 
5513 /* Bubble-sort the stripe set to put the parity/syndrome stripes last */
5514 static void sort_parity_stripes(struct btrfs_bio *bbio, int num_stripes)
5515 {
5516 	int i;
5517 	int again = 1;
5518 
5519 	while (again) {
5520 		again = 0;
5521 		for (i = 0; i < num_stripes - 1; i++) {
5522 			/* Swap if parity is on a smaller index */
5523 			if (bbio->raid_map[i] > bbio->raid_map[i + 1]) {
5524 				swap(bbio->stripes[i], bbio->stripes[i + 1]);
5525 				swap(bbio->raid_map[i], bbio->raid_map[i + 1]);
5526 				again = 1;
5527 			}
5528 		}
5529 	}
5530 }
5531 
5532 static struct btrfs_bio *alloc_btrfs_bio(int total_stripes, int real_stripes)
5533 {
5534 	struct btrfs_bio *bbio = kzalloc(
5535 		 /* the size of the btrfs_bio */
5536 		sizeof(struct btrfs_bio) +
5537 		/* plus the variable array for the stripes */
5538 		sizeof(struct btrfs_bio_stripe) * (total_stripes) +
5539 		/* plus the variable array for the tgt dev */
5540 		sizeof(int) * (real_stripes) +
5541 		/*
5542 		 * plus the raid_map, which includes both the tgt dev
5543 		 * and the stripes
5544 		 */
5545 		sizeof(u64) * (total_stripes),
5546 		GFP_NOFS|__GFP_NOFAIL);
5547 
5548 	atomic_set(&bbio->error, 0);
5549 	refcount_set(&bbio->refs, 1);
5550 
5551 	bbio->tgtdev_map = (int *)(bbio->stripes + total_stripes);
5552 	bbio->raid_map = (u64 *)(bbio->tgtdev_map + real_stripes);
5553 
5554 	return bbio;
5555 }
5556 
5557 void btrfs_get_bbio(struct btrfs_bio *bbio)
5558 {
5559 	WARN_ON(!refcount_read(&bbio->refs));
5560 	refcount_inc(&bbio->refs);
5561 }
5562 
5563 void btrfs_put_bbio(struct btrfs_bio *bbio)
5564 {
5565 	if (!bbio)
5566 		return;
5567 	if (refcount_dec_and_test(&bbio->refs))
5568 		kfree(bbio);
5569 }
5570 
5571 /* can REQ_OP_DISCARD be sent with other REQ like REQ_OP_WRITE? */
5572 /*
5573  * Please note that, discard won't be sent to target device of device
5574  * replace.
5575  */
5576 static int __btrfs_map_block_for_discard(struct btrfs_fs_info *fs_info,
5577 					 u64 logical, u64 *length_ret,
5578 					 struct btrfs_bio **bbio_ret)
5579 {
5580 	struct extent_map *em;
5581 	struct map_lookup *map;
5582 	struct btrfs_bio *bbio;
5583 	u64 length = *length_ret;
5584 	u64 offset;
5585 	u64 stripe_nr;
5586 	u64 stripe_nr_end;
5587 	u64 stripe_end_offset;
5588 	u64 stripe_cnt;
5589 	u64 stripe_len;
5590 	u64 stripe_offset;
5591 	u64 num_stripes;
5592 	u32 stripe_index;
5593 	u32 factor = 0;
5594 	u32 sub_stripes = 0;
5595 	u64 stripes_per_dev = 0;
5596 	u32 remaining_stripes = 0;
5597 	u32 last_stripe = 0;
5598 	int ret = 0;
5599 	int i;
5600 
5601 	/* discard always return a bbio */
5602 	ASSERT(bbio_ret);
5603 
5604 	em = btrfs_get_chunk_map(fs_info, logical, length);
5605 	if (IS_ERR(em))
5606 		return PTR_ERR(em);
5607 
5608 	map = em->map_lookup;
5609 	/* we don't discard raid56 yet */
5610 	if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
5611 		ret = -EOPNOTSUPP;
5612 		goto out;
5613 	}
5614 
5615 	offset = logical - em->start;
5616 	length = min_t(u64, em->start + em->len - logical, length);
5617 	*length_ret = length;
5618 
5619 	stripe_len = map->stripe_len;
5620 	/*
5621 	 * stripe_nr counts the total number of stripes we have to stride
5622 	 * to get to this block
5623 	 */
5624 	stripe_nr = div64_u64(offset, stripe_len);
5625 
5626 	/* stripe_offset is the offset of this block in its stripe */
5627 	stripe_offset = offset - stripe_nr * stripe_len;
5628 
5629 	stripe_nr_end = round_up(offset + length, map->stripe_len);
5630 	stripe_nr_end = div64_u64(stripe_nr_end, map->stripe_len);
5631 	stripe_cnt = stripe_nr_end - stripe_nr;
5632 	stripe_end_offset = stripe_nr_end * map->stripe_len -
5633 			    (offset + length);
5634 	/*
5635 	 * after this, stripe_nr is the number of stripes on this
5636 	 * device we have to walk to find the data, and stripe_index is
5637 	 * the number of our device in the stripe array
5638 	 */
5639 	num_stripes = 1;
5640 	stripe_index = 0;
5641 	if (map->type & (BTRFS_BLOCK_GROUP_RAID0 |
5642 			 BTRFS_BLOCK_GROUP_RAID10)) {
5643 		if (map->type & BTRFS_BLOCK_GROUP_RAID0)
5644 			sub_stripes = 1;
5645 		else
5646 			sub_stripes = map->sub_stripes;
5647 
5648 		factor = map->num_stripes / sub_stripes;
5649 		num_stripes = min_t(u64, map->num_stripes,
5650 				    sub_stripes * stripe_cnt);
5651 		stripe_nr = div_u64_rem(stripe_nr, factor, &stripe_index);
5652 		stripe_index *= sub_stripes;
5653 		stripes_per_dev = div_u64_rem(stripe_cnt, factor,
5654 					      &remaining_stripes);
5655 		div_u64_rem(stripe_nr_end - 1, factor, &last_stripe);
5656 		last_stripe *= sub_stripes;
5657 	} else if (map->type & (BTRFS_BLOCK_GROUP_RAID1_MASK |
5658 				BTRFS_BLOCK_GROUP_DUP)) {
5659 		num_stripes = map->num_stripes;
5660 	} else {
5661 		stripe_nr = div_u64_rem(stripe_nr, map->num_stripes,
5662 					&stripe_index);
5663 	}
5664 
5665 	bbio = alloc_btrfs_bio(num_stripes, 0);
5666 	if (!bbio) {
5667 		ret = -ENOMEM;
5668 		goto out;
5669 	}
5670 
5671 	for (i = 0; i < num_stripes; i++) {
5672 		bbio->stripes[i].physical =
5673 			map->stripes[stripe_index].physical +
5674 			stripe_offset + stripe_nr * map->stripe_len;
5675 		bbio->stripes[i].dev = map->stripes[stripe_index].dev;
5676 
5677 		if (map->type & (BTRFS_BLOCK_GROUP_RAID0 |
5678 				 BTRFS_BLOCK_GROUP_RAID10)) {
5679 			bbio->stripes[i].length = stripes_per_dev *
5680 				map->stripe_len;
5681 
5682 			if (i / sub_stripes < remaining_stripes)
5683 				bbio->stripes[i].length +=
5684 					map->stripe_len;
5685 
5686 			/*
5687 			 * Special for the first stripe and
5688 			 * the last stripe:
5689 			 *
5690 			 * |-------|...|-------|
5691 			 *     |----------|
5692 			 *    off     end_off
5693 			 */
5694 			if (i < sub_stripes)
5695 				bbio->stripes[i].length -=
5696 					stripe_offset;
5697 
5698 			if (stripe_index >= last_stripe &&
5699 			    stripe_index <= (last_stripe +
5700 					     sub_stripes - 1))
5701 				bbio->stripes[i].length -=
5702 					stripe_end_offset;
5703 
5704 			if (i == sub_stripes - 1)
5705 				stripe_offset = 0;
5706 		} else {
5707 			bbio->stripes[i].length = length;
5708 		}
5709 
5710 		stripe_index++;
5711 		if (stripe_index == map->num_stripes) {
5712 			stripe_index = 0;
5713 			stripe_nr++;
5714 		}
5715 	}
5716 
5717 	*bbio_ret = bbio;
5718 	bbio->map_type = map->type;
5719 	bbio->num_stripes = num_stripes;
5720 out:
5721 	free_extent_map(em);
5722 	return ret;
5723 }
5724 
5725 /*
5726  * In dev-replace case, for repair case (that's the only case where the mirror
5727  * is selected explicitly when calling btrfs_map_block), blocks left of the
5728  * left cursor can also be read from the target drive.
5729  *
5730  * For REQ_GET_READ_MIRRORS, the target drive is added as the last one to the
5731  * array of stripes.
5732  * For READ, it also needs to be supported using the same mirror number.
5733  *
5734  * If the requested block is not left of the left cursor, EIO is returned. This
5735  * can happen because btrfs_num_copies() returns one more in the dev-replace
5736  * case.
5737  */
5738 static int get_extra_mirror_from_replace(struct btrfs_fs_info *fs_info,
5739 					 u64 logical, u64 length,
5740 					 u64 srcdev_devid, int *mirror_num,
5741 					 u64 *physical)
5742 {
5743 	struct btrfs_bio *bbio = NULL;
5744 	int num_stripes;
5745 	int index_srcdev = 0;
5746 	int found = 0;
5747 	u64 physical_of_found = 0;
5748 	int i;
5749 	int ret = 0;
5750 
5751 	ret = __btrfs_map_block(fs_info, BTRFS_MAP_GET_READ_MIRRORS,
5752 				logical, &length, &bbio, 0, 0);
5753 	if (ret) {
5754 		ASSERT(bbio == NULL);
5755 		return ret;
5756 	}
5757 
5758 	num_stripes = bbio->num_stripes;
5759 	if (*mirror_num > num_stripes) {
5760 		/*
5761 		 * BTRFS_MAP_GET_READ_MIRRORS does not contain this mirror,
5762 		 * that means that the requested area is not left of the left
5763 		 * cursor
5764 		 */
5765 		btrfs_put_bbio(bbio);
5766 		return -EIO;
5767 	}
5768 
5769 	/*
5770 	 * process the rest of the function using the mirror_num of the source
5771 	 * drive. Therefore look it up first.  At the end, patch the device
5772 	 * pointer to the one of the target drive.
5773 	 */
5774 	for (i = 0; i < num_stripes; i++) {
5775 		if (bbio->stripes[i].dev->devid != srcdev_devid)
5776 			continue;
5777 
5778 		/*
5779 		 * In case of DUP, in order to keep it simple, only add the
5780 		 * mirror with the lowest physical address
5781 		 */
5782 		if (found &&
5783 		    physical_of_found <= bbio->stripes[i].physical)
5784 			continue;
5785 
5786 		index_srcdev = i;
5787 		found = 1;
5788 		physical_of_found = bbio->stripes[i].physical;
5789 	}
5790 
5791 	btrfs_put_bbio(bbio);
5792 
5793 	ASSERT(found);
5794 	if (!found)
5795 		return -EIO;
5796 
5797 	*mirror_num = index_srcdev + 1;
5798 	*physical = physical_of_found;
5799 	return ret;
5800 }
5801 
5802 static void handle_ops_on_dev_replace(enum btrfs_map_op op,
5803 				      struct btrfs_bio **bbio_ret,
5804 				      struct btrfs_dev_replace *dev_replace,
5805 				      int *num_stripes_ret, int *max_errors_ret)
5806 {
5807 	struct btrfs_bio *bbio = *bbio_ret;
5808 	u64 srcdev_devid = dev_replace->srcdev->devid;
5809 	int tgtdev_indexes = 0;
5810 	int num_stripes = *num_stripes_ret;
5811 	int max_errors = *max_errors_ret;
5812 	int i;
5813 
5814 	if (op == BTRFS_MAP_WRITE) {
5815 		int index_where_to_add;
5816 
5817 		/*
5818 		 * duplicate the write operations while the dev replace
5819 		 * procedure is running. Since the copying of the old disk to
5820 		 * the new disk takes place at run time while the filesystem is
5821 		 * mounted writable, the regular write operations to the old
5822 		 * disk have to be duplicated to go to the new disk as well.
5823 		 *
5824 		 * Note that device->missing is handled by the caller, and that
5825 		 * the write to the old disk is already set up in the stripes
5826 		 * array.
5827 		 */
5828 		index_where_to_add = num_stripes;
5829 		for (i = 0; i < num_stripes; i++) {
5830 			if (bbio->stripes[i].dev->devid == srcdev_devid) {
5831 				/* write to new disk, too */
5832 				struct btrfs_bio_stripe *new =
5833 					bbio->stripes + index_where_to_add;
5834 				struct btrfs_bio_stripe *old =
5835 					bbio->stripes + i;
5836 
5837 				new->physical = old->physical;
5838 				new->length = old->length;
5839 				new->dev = dev_replace->tgtdev;
5840 				bbio->tgtdev_map[i] = index_where_to_add;
5841 				index_where_to_add++;
5842 				max_errors++;
5843 				tgtdev_indexes++;
5844 			}
5845 		}
5846 		num_stripes = index_where_to_add;
5847 	} else if (op == BTRFS_MAP_GET_READ_MIRRORS) {
5848 		int index_srcdev = 0;
5849 		int found = 0;
5850 		u64 physical_of_found = 0;
5851 
5852 		/*
5853 		 * During the dev-replace procedure, the target drive can also
5854 		 * be used to read data in case it is needed to repair a corrupt
5855 		 * block elsewhere. This is possible if the requested area is
5856 		 * left of the left cursor. In this area, the target drive is a
5857 		 * full copy of the source drive.
5858 		 */
5859 		for (i = 0; i < num_stripes; i++) {
5860 			if (bbio->stripes[i].dev->devid == srcdev_devid) {
5861 				/*
5862 				 * In case of DUP, in order to keep it simple,
5863 				 * only add the mirror with the lowest physical
5864 				 * address
5865 				 */
5866 				if (found &&
5867 				    physical_of_found <=
5868 				     bbio->stripes[i].physical)
5869 					continue;
5870 				index_srcdev = i;
5871 				found = 1;
5872 				physical_of_found = bbio->stripes[i].physical;
5873 			}
5874 		}
5875 		if (found) {
5876 			struct btrfs_bio_stripe *tgtdev_stripe =
5877 				bbio->stripes + num_stripes;
5878 
5879 			tgtdev_stripe->physical = physical_of_found;
5880 			tgtdev_stripe->length =
5881 				bbio->stripes[index_srcdev].length;
5882 			tgtdev_stripe->dev = dev_replace->tgtdev;
5883 			bbio->tgtdev_map[index_srcdev] = num_stripes;
5884 
5885 			tgtdev_indexes++;
5886 			num_stripes++;
5887 		}
5888 	}
5889 
5890 	*num_stripes_ret = num_stripes;
5891 	*max_errors_ret = max_errors;
5892 	bbio->num_tgtdevs = tgtdev_indexes;
5893 	*bbio_ret = bbio;
5894 }
5895 
5896 static bool need_full_stripe(enum btrfs_map_op op)
5897 {
5898 	return (op == BTRFS_MAP_WRITE || op == BTRFS_MAP_GET_READ_MIRRORS);
5899 }
5900 
5901 /*
5902  * btrfs_get_io_geometry - calculates the geomery of a particular (address, len)
5903  *		       tuple. This information is used to calculate how big a
5904  *		       particular bio can get before it straddles a stripe.
5905  *
5906  * @fs_info - the filesystem
5907  * @logical - address that we want to figure out the geometry of
5908  * @len	    - the length of IO we are going to perform, starting at @logical
5909  * @op      - type of operation - write or read
5910  * @io_geom - pointer used to return values
5911  *
5912  * Returns < 0 in case a chunk for the given logical address cannot be found,
5913  * usually shouldn't happen unless @logical is corrupted, 0 otherwise.
5914  */
5915 int btrfs_get_io_geometry(struct btrfs_fs_info *fs_info, enum btrfs_map_op op,
5916 			u64 logical, u64 len, struct btrfs_io_geometry *io_geom)
5917 {
5918 	struct extent_map *em;
5919 	struct map_lookup *map;
5920 	u64 offset;
5921 	u64 stripe_offset;
5922 	u64 stripe_nr;
5923 	u64 stripe_len;
5924 	u64 raid56_full_stripe_start = (u64)-1;
5925 	int data_stripes;
5926 	int ret = 0;
5927 
5928 	ASSERT(op != BTRFS_MAP_DISCARD);
5929 
5930 	em = btrfs_get_chunk_map(fs_info, logical, len);
5931 	if (IS_ERR(em))
5932 		return PTR_ERR(em);
5933 
5934 	map = em->map_lookup;
5935 	/* Offset of this logical address in the chunk */
5936 	offset = logical - em->start;
5937 	/* Len of a stripe in a chunk */
5938 	stripe_len = map->stripe_len;
5939 	/* Stripe wher this block falls in */
5940 	stripe_nr = div64_u64(offset, stripe_len);
5941 	/* Offset of stripe in the chunk */
5942 	stripe_offset = stripe_nr * stripe_len;
5943 	if (offset < stripe_offset) {
5944 		btrfs_crit(fs_info,
5945 "stripe math has gone wrong, stripe_offset=%llu offset=%llu start=%llu logical=%llu stripe_len=%llu",
5946 			stripe_offset, offset, em->start, logical, stripe_len);
5947 		ret = -EINVAL;
5948 		goto out;
5949 	}
5950 
5951 	/* stripe_offset is the offset of this block in its stripe */
5952 	stripe_offset = offset - stripe_offset;
5953 	data_stripes = nr_data_stripes(map);
5954 
5955 	if (map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) {
5956 		u64 max_len = stripe_len - stripe_offset;
5957 
5958 		/*
5959 		 * In case of raid56, we need to know the stripe aligned start
5960 		 */
5961 		if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
5962 			unsigned long full_stripe_len = stripe_len * data_stripes;
5963 			raid56_full_stripe_start = offset;
5964 
5965 			/*
5966 			 * Allow a write of a full stripe, but make sure we
5967 			 * don't allow straddling of stripes
5968 			 */
5969 			raid56_full_stripe_start = div64_u64(raid56_full_stripe_start,
5970 					full_stripe_len);
5971 			raid56_full_stripe_start *= full_stripe_len;
5972 
5973 			/*
5974 			 * For writes to RAID[56], allow a full stripeset across
5975 			 * all disks. For other RAID types and for RAID[56]
5976 			 * reads, just allow a single stripe (on a single disk).
5977 			 */
5978 			if (op == BTRFS_MAP_WRITE) {
5979 				max_len = stripe_len * data_stripes -
5980 					  (offset - raid56_full_stripe_start);
5981 			}
5982 		}
5983 		len = min_t(u64, em->len - offset, max_len);
5984 	} else {
5985 		len = em->len - offset;
5986 	}
5987 
5988 	io_geom->len = len;
5989 	io_geom->offset = offset;
5990 	io_geom->stripe_len = stripe_len;
5991 	io_geom->stripe_nr = stripe_nr;
5992 	io_geom->stripe_offset = stripe_offset;
5993 	io_geom->raid56_stripe_offset = raid56_full_stripe_start;
5994 
5995 out:
5996 	/* once for us */
5997 	free_extent_map(em);
5998 	return ret;
5999 }
6000 
6001 static int __btrfs_map_block(struct btrfs_fs_info *fs_info,
6002 			     enum btrfs_map_op op,
6003 			     u64 logical, u64 *length,
6004 			     struct btrfs_bio **bbio_ret,
6005 			     int mirror_num, int need_raid_map)
6006 {
6007 	struct extent_map *em;
6008 	struct map_lookup *map;
6009 	u64 stripe_offset;
6010 	u64 stripe_nr;
6011 	u64 stripe_len;
6012 	u32 stripe_index;
6013 	int data_stripes;
6014 	int i;
6015 	int ret = 0;
6016 	int num_stripes;
6017 	int max_errors = 0;
6018 	int tgtdev_indexes = 0;
6019 	struct btrfs_bio *bbio = NULL;
6020 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
6021 	int dev_replace_is_ongoing = 0;
6022 	int num_alloc_stripes;
6023 	int patch_the_first_stripe_for_dev_replace = 0;
6024 	u64 physical_to_patch_in_first_stripe = 0;
6025 	u64 raid56_full_stripe_start = (u64)-1;
6026 	struct btrfs_io_geometry geom;
6027 
6028 	ASSERT(bbio_ret);
6029 	ASSERT(op != BTRFS_MAP_DISCARD);
6030 
6031 	ret = btrfs_get_io_geometry(fs_info, op, logical, *length, &geom);
6032 	if (ret < 0)
6033 		return ret;
6034 
6035 	em = btrfs_get_chunk_map(fs_info, logical, *length);
6036 	ASSERT(!IS_ERR(em));
6037 	map = em->map_lookup;
6038 
6039 	*length = geom.len;
6040 	stripe_len = geom.stripe_len;
6041 	stripe_nr = geom.stripe_nr;
6042 	stripe_offset = geom.stripe_offset;
6043 	raid56_full_stripe_start = geom.raid56_stripe_offset;
6044 	data_stripes = nr_data_stripes(map);
6045 
6046 	down_read(&dev_replace->rwsem);
6047 	dev_replace_is_ongoing = btrfs_dev_replace_is_ongoing(dev_replace);
6048 	/*
6049 	 * Hold the semaphore for read during the whole operation, write is
6050 	 * requested at commit time but must wait.
6051 	 */
6052 	if (!dev_replace_is_ongoing)
6053 		up_read(&dev_replace->rwsem);
6054 
6055 	if (dev_replace_is_ongoing && mirror_num == map->num_stripes + 1 &&
6056 	    !need_full_stripe(op) && dev_replace->tgtdev != NULL) {
6057 		ret = get_extra_mirror_from_replace(fs_info, logical, *length,
6058 						    dev_replace->srcdev->devid,
6059 						    &mirror_num,
6060 					    &physical_to_patch_in_first_stripe);
6061 		if (ret)
6062 			goto out;
6063 		else
6064 			patch_the_first_stripe_for_dev_replace = 1;
6065 	} else if (mirror_num > map->num_stripes) {
6066 		mirror_num = 0;
6067 	}
6068 
6069 	num_stripes = 1;
6070 	stripe_index = 0;
6071 	if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
6072 		stripe_nr = div_u64_rem(stripe_nr, map->num_stripes,
6073 				&stripe_index);
6074 		if (!need_full_stripe(op))
6075 			mirror_num = 1;
6076 	} else if (map->type & BTRFS_BLOCK_GROUP_RAID1_MASK) {
6077 		if (need_full_stripe(op))
6078 			num_stripes = map->num_stripes;
6079 		else if (mirror_num)
6080 			stripe_index = mirror_num - 1;
6081 		else {
6082 			stripe_index = find_live_mirror(fs_info, map, 0,
6083 					    dev_replace_is_ongoing);
6084 			mirror_num = stripe_index + 1;
6085 		}
6086 
6087 	} else if (map->type & BTRFS_BLOCK_GROUP_DUP) {
6088 		if (need_full_stripe(op)) {
6089 			num_stripes = map->num_stripes;
6090 		} else if (mirror_num) {
6091 			stripe_index = mirror_num - 1;
6092 		} else {
6093 			mirror_num = 1;
6094 		}
6095 
6096 	} else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
6097 		u32 factor = map->num_stripes / map->sub_stripes;
6098 
6099 		stripe_nr = div_u64_rem(stripe_nr, factor, &stripe_index);
6100 		stripe_index *= map->sub_stripes;
6101 
6102 		if (need_full_stripe(op))
6103 			num_stripes = map->sub_stripes;
6104 		else if (mirror_num)
6105 			stripe_index += mirror_num - 1;
6106 		else {
6107 			int old_stripe_index = stripe_index;
6108 			stripe_index = find_live_mirror(fs_info, map,
6109 					      stripe_index,
6110 					      dev_replace_is_ongoing);
6111 			mirror_num = stripe_index - old_stripe_index + 1;
6112 		}
6113 
6114 	} else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
6115 		if (need_raid_map && (need_full_stripe(op) || mirror_num > 1)) {
6116 			/* push stripe_nr back to the start of the full stripe */
6117 			stripe_nr = div64_u64(raid56_full_stripe_start,
6118 					stripe_len * data_stripes);
6119 
6120 			/* RAID[56] write or recovery. Return all stripes */
6121 			num_stripes = map->num_stripes;
6122 			max_errors = nr_parity_stripes(map);
6123 
6124 			*length = map->stripe_len;
6125 			stripe_index = 0;
6126 			stripe_offset = 0;
6127 		} else {
6128 			/*
6129 			 * Mirror #0 or #1 means the original data block.
6130 			 * Mirror #2 is RAID5 parity block.
6131 			 * Mirror #3 is RAID6 Q block.
6132 			 */
6133 			stripe_nr = div_u64_rem(stripe_nr,
6134 					data_stripes, &stripe_index);
6135 			if (mirror_num > 1)
6136 				stripe_index = data_stripes + mirror_num - 2;
6137 
6138 			/* We distribute the parity blocks across stripes */
6139 			div_u64_rem(stripe_nr + stripe_index, map->num_stripes,
6140 					&stripe_index);
6141 			if (!need_full_stripe(op) && mirror_num <= 1)
6142 				mirror_num = 1;
6143 		}
6144 	} else {
6145 		/*
6146 		 * after this, stripe_nr is the number of stripes on this
6147 		 * device we have to walk to find the data, and stripe_index is
6148 		 * the number of our device in the stripe array
6149 		 */
6150 		stripe_nr = div_u64_rem(stripe_nr, map->num_stripes,
6151 				&stripe_index);
6152 		mirror_num = stripe_index + 1;
6153 	}
6154 	if (stripe_index >= map->num_stripes) {
6155 		btrfs_crit(fs_info,
6156 			   "stripe index math went horribly wrong, got stripe_index=%u, num_stripes=%u",
6157 			   stripe_index, map->num_stripes);
6158 		ret = -EINVAL;
6159 		goto out;
6160 	}
6161 
6162 	num_alloc_stripes = num_stripes;
6163 	if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL) {
6164 		if (op == BTRFS_MAP_WRITE)
6165 			num_alloc_stripes <<= 1;
6166 		if (op == BTRFS_MAP_GET_READ_MIRRORS)
6167 			num_alloc_stripes++;
6168 		tgtdev_indexes = num_stripes;
6169 	}
6170 
6171 	bbio = alloc_btrfs_bio(num_alloc_stripes, tgtdev_indexes);
6172 	if (!bbio) {
6173 		ret = -ENOMEM;
6174 		goto out;
6175 	}
6176 
6177 	for (i = 0; i < num_stripes; i++) {
6178 		bbio->stripes[i].physical = map->stripes[stripe_index].physical +
6179 			stripe_offset + stripe_nr * map->stripe_len;
6180 		bbio->stripes[i].dev = map->stripes[stripe_index].dev;
6181 		stripe_index++;
6182 	}
6183 
6184 	/* build raid_map */
6185 	if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK && need_raid_map &&
6186 	    (need_full_stripe(op) || mirror_num > 1)) {
6187 		u64 tmp;
6188 		unsigned rot;
6189 
6190 		/* Work out the disk rotation on this stripe-set */
6191 		div_u64_rem(stripe_nr, num_stripes, &rot);
6192 
6193 		/* Fill in the logical address of each stripe */
6194 		tmp = stripe_nr * data_stripes;
6195 		for (i = 0; i < data_stripes; i++)
6196 			bbio->raid_map[(i+rot) % num_stripes] =
6197 				em->start + (tmp + i) * map->stripe_len;
6198 
6199 		bbio->raid_map[(i+rot) % map->num_stripes] = RAID5_P_STRIPE;
6200 		if (map->type & BTRFS_BLOCK_GROUP_RAID6)
6201 			bbio->raid_map[(i+rot+1) % num_stripes] =
6202 				RAID6_Q_STRIPE;
6203 
6204 		sort_parity_stripes(bbio, num_stripes);
6205 	}
6206 
6207 	if (need_full_stripe(op))
6208 		max_errors = btrfs_chunk_max_errors(map);
6209 
6210 	if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL &&
6211 	    need_full_stripe(op)) {
6212 		handle_ops_on_dev_replace(op, &bbio, dev_replace, &num_stripes,
6213 					  &max_errors);
6214 	}
6215 
6216 	*bbio_ret = bbio;
6217 	bbio->map_type = map->type;
6218 	bbio->num_stripes = num_stripes;
6219 	bbio->max_errors = max_errors;
6220 	bbio->mirror_num = mirror_num;
6221 
6222 	/*
6223 	 * this is the case that REQ_READ && dev_replace_is_ongoing &&
6224 	 * mirror_num == num_stripes + 1 && dev_replace target drive is
6225 	 * available as a mirror
6226 	 */
6227 	if (patch_the_first_stripe_for_dev_replace && num_stripes > 0) {
6228 		WARN_ON(num_stripes > 1);
6229 		bbio->stripes[0].dev = dev_replace->tgtdev;
6230 		bbio->stripes[0].physical = physical_to_patch_in_first_stripe;
6231 		bbio->mirror_num = map->num_stripes + 1;
6232 	}
6233 out:
6234 	if (dev_replace_is_ongoing) {
6235 		lockdep_assert_held(&dev_replace->rwsem);
6236 		/* Unlock and let waiting writers proceed */
6237 		up_read(&dev_replace->rwsem);
6238 	}
6239 	free_extent_map(em);
6240 	return ret;
6241 }
6242 
6243 int btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op,
6244 		      u64 logical, u64 *length,
6245 		      struct btrfs_bio **bbio_ret, int mirror_num)
6246 {
6247 	if (op == BTRFS_MAP_DISCARD)
6248 		return __btrfs_map_block_for_discard(fs_info, logical,
6249 						     length, bbio_ret);
6250 
6251 	return __btrfs_map_block(fs_info, op, logical, length, bbio_ret,
6252 				 mirror_num, 0);
6253 }
6254 
6255 /* For Scrub/replace */
6256 int btrfs_map_sblock(struct btrfs_fs_info *fs_info, enum btrfs_map_op op,
6257 		     u64 logical, u64 *length,
6258 		     struct btrfs_bio **bbio_ret)
6259 {
6260 	return __btrfs_map_block(fs_info, op, logical, length, bbio_ret, 0, 1);
6261 }
6262 
6263 static inline void btrfs_end_bbio(struct btrfs_bio *bbio, struct bio *bio)
6264 {
6265 	bio->bi_private = bbio->private;
6266 	bio->bi_end_io = bbio->end_io;
6267 	bio_endio(bio);
6268 
6269 	btrfs_put_bbio(bbio);
6270 }
6271 
6272 static void btrfs_end_bio(struct bio *bio)
6273 {
6274 	struct btrfs_bio *bbio = bio->bi_private;
6275 	int is_orig_bio = 0;
6276 
6277 	if (bio->bi_status) {
6278 		atomic_inc(&bbio->error);
6279 		if (bio->bi_status == BLK_STS_IOERR ||
6280 		    bio->bi_status == BLK_STS_TARGET) {
6281 			struct btrfs_device *dev = btrfs_io_bio(bio)->device;
6282 
6283 			ASSERT(dev->bdev);
6284 			if (bio_op(bio) == REQ_OP_WRITE)
6285 				btrfs_dev_stat_inc_and_print(dev,
6286 						BTRFS_DEV_STAT_WRITE_ERRS);
6287 			else if (!(bio->bi_opf & REQ_RAHEAD))
6288 				btrfs_dev_stat_inc_and_print(dev,
6289 						BTRFS_DEV_STAT_READ_ERRS);
6290 			if (bio->bi_opf & REQ_PREFLUSH)
6291 				btrfs_dev_stat_inc_and_print(dev,
6292 						BTRFS_DEV_STAT_FLUSH_ERRS);
6293 		}
6294 	}
6295 
6296 	if (bio == bbio->orig_bio)
6297 		is_orig_bio = 1;
6298 
6299 	btrfs_bio_counter_dec(bbio->fs_info);
6300 
6301 	if (atomic_dec_and_test(&bbio->stripes_pending)) {
6302 		if (!is_orig_bio) {
6303 			bio_put(bio);
6304 			bio = bbio->orig_bio;
6305 		}
6306 
6307 		btrfs_io_bio(bio)->mirror_num = bbio->mirror_num;
6308 		/* only send an error to the higher layers if it is
6309 		 * beyond the tolerance of the btrfs bio
6310 		 */
6311 		if (atomic_read(&bbio->error) > bbio->max_errors) {
6312 			bio->bi_status = BLK_STS_IOERR;
6313 		} else {
6314 			/*
6315 			 * this bio is actually up to date, we didn't
6316 			 * go over the max number of errors
6317 			 */
6318 			bio->bi_status = BLK_STS_OK;
6319 		}
6320 
6321 		btrfs_end_bbio(bbio, bio);
6322 	} else if (!is_orig_bio) {
6323 		bio_put(bio);
6324 	}
6325 }
6326 
6327 static void submit_stripe_bio(struct btrfs_bio *bbio, struct bio *bio,
6328 			      u64 physical, struct btrfs_device *dev)
6329 {
6330 	struct btrfs_fs_info *fs_info = bbio->fs_info;
6331 
6332 	bio->bi_private = bbio;
6333 	btrfs_io_bio(bio)->device = dev;
6334 	bio->bi_end_io = btrfs_end_bio;
6335 	bio->bi_iter.bi_sector = physical >> 9;
6336 	btrfs_debug_in_rcu(fs_info,
6337 	"btrfs_map_bio: rw %d 0x%x, sector=%llu, dev=%lu (%s id %llu), size=%u",
6338 		bio_op(bio), bio->bi_opf, (u64)bio->bi_iter.bi_sector,
6339 		(unsigned long)dev->bdev->bd_dev, rcu_str_deref(dev->name),
6340 		dev->devid, bio->bi_iter.bi_size);
6341 	bio_set_dev(bio, dev->bdev);
6342 
6343 	btrfs_bio_counter_inc_noblocked(fs_info);
6344 
6345 	btrfsic_submit_bio(bio);
6346 }
6347 
6348 static void bbio_error(struct btrfs_bio *bbio, struct bio *bio, u64 logical)
6349 {
6350 	atomic_inc(&bbio->error);
6351 	if (atomic_dec_and_test(&bbio->stripes_pending)) {
6352 		/* Should be the original bio. */
6353 		WARN_ON(bio != bbio->orig_bio);
6354 
6355 		btrfs_io_bio(bio)->mirror_num = bbio->mirror_num;
6356 		bio->bi_iter.bi_sector = logical >> 9;
6357 		if (atomic_read(&bbio->error) > bbio->max_errors)
6358 			bio->bi_status = BLK_STS_IOERR;
6359 		else
6360 			bio->bi_status = BLK_STS_OK;
6361 		btrfs_end_bbio(bbio, bio);
6362 	}
6363 }
6364 
6365 blk_status_t btrfs_map_bio(struct btrfs_fs_info *fs_info, struct bio *bio,
6366 			   int mirror_num)
6367 {
6368 	struct btrfs_device *dev;
6369 	struct bio *first_bio = bio;
6370 	u64 logical = (u64)bio->bi_iter.bi_sector << 9;
6371 	u64 length = 0;
6372 	u64 map_length;
6373 	int ret;
6374 	int dev_nr;
6375 	int total_devs;
6376 	struct btrfs_bio *bbio = NULL;
6377 
6378 	length = bio->bi_iter.bi_size;
6379 	map_length = length;
6380 
6381 	btrfs_bio_counter_inc_blocked(fs_info);
6382 	ret = __btrfs_map_block(fs_info, btrfs_op(bio), logical,
6383 				&map_length, &bbio, mirror_num, 1);
6384 	if (ret) {
6385 		btrfs_bio_counter_dec(fs_info);
6386 		return errno_to_blk_status(ret);
6387 	}
6388 
6389 	total_devs = bbio->num_stripes;
6390 	bbio->orig_bio = first_bio;
6391 	bbio->private = first_bio->bi_private;
6392 	bbio->end_io = first_bio->bi_end_io;
6393 	bbio->fs_info = fs_info;
6394 	atomic_set(&bbio->stripes_pending, bbio->num_stripes);
6395 
6396 	if ((bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) &&
6397 	    ((bio_op(bio) == REQ_OP_WRITE) || (mirror_num > 1))) {
6398 		/* In this case, map_length has been set to the length of
6399 		   a single stripe; not the whole write */
6400 		if (bio_op(bio) == REQ_OP_WRITE) {
6401 			ret = raid56_parity_write(fs_info, bio, bbio,
6402 						  map_length);
6403 		} else {
6404 			ret = raid56_parity_recover(fs_info, bio, bbio,
6405 						    map_length, mirror_num, 1);
6406 		}
6407 
6408 		btrfs_bio_counter_dec(fs_info);
6409 		return errno_to_blk_status(ret);
6410 	}
6411 
6412 	if (map_length < length) {
6413 		btrfs_crit(fs_info,
6414 			   "mapping failed logical %llu bio len %llu len %llu",
6415 			   logical, length, map_length);
6416 		BUG();
6417 	}
6418 
6419 	for (dev_nr = 0; dev_nr < total_devs; dev_nr++) {
6420 		dev = bbio->stripes[dev_nr].dev;
6421 		if (!dev || !dev->bdev || test_bit(BTRFS_DEV_STATE_MISSING,
6422 						   &dev->dev_state) ||
6423 		    (bio_op(first_bio) == REQ_OP_WRITE &&
6424 		    !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))) {
6425 			bbio_error(bbio, first_bio, logical);
6426 			continue;
6427 		}
6428 
6429 		if (dev_nr < total_devs - 1)
6430 			bio = btrfs_bio_clone(first_bio);
6431 		else
6432 			bio = first_bio;
6433 
6434 		submit_stripe_bio(bbio, bio, bbio->stripes[dev_nr].physical, dev);
6435 	}
6436 	btrfs_bio_counter_dec(fs_info);
6437 	return BLK_STS_OK;
6438 }
6439 
6440 /*
6441  * Find a device specified by @devid or @uuid in the list of @fs_devices, or
6442  * return NULL.
6443  *
6444  * If devid and uuid are both specified, the match must be exact, otherwise
6445  * only devid is used.
6446  *
6447  * If @seed is true, traverse through the seed devices.
6448  */
6449 struct btrfs_device *btrfs_find_device(struct btrfs_fs_devices *fs_devices,
6450 				       u64 devid, u8 *uuid, u8 *fsid,
6451 				       bool seed)
6452 {
6453 	struct btrfs_device *device;
6454 	struct btrfs_fs_devices *seed_devs;
6455 
6456 	if (!fsid || !memcmp(fs_devices->metadata_uuid, fsid, BTRFS_FSID_SIZE)) {
6457 		list_for_each_entry(device, &fs_devices->devices, dev_list) {
6458 			if (device->devid == devid &&
6459 			    (!uuid || memcmp(device->uuid, uuid,
6460 					     BTRFS_UUID_SIZE) == 0))
6461 				return device;
6462 		}
6463 	}
6464 
6465 	list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) {
6466 		if (!fsid ||
6467 		    !memcmp(seed_devs->metadata_uuid, fsid, BTRFS_FSID_SIZE)) {
6468 			list_for_each_entry(device, &seed_devs->devices,
6469 					    dev_list) {
6470 				if (device->devid == devid &&
6471 				    (!uuid || memcmp(device->uuid, uuid,
6472 						     BTRFS_UUID_SIZE) == 0))
6473 					return device;
6474 			}
6475 		}
6476 	}
6477 
6478 	return NULL;
6479 }
6480 
6481 static struct btrfs_device *add_missing_dev(struct btrfs_fs_devices *fs_devices,
6482 					    u64 devid, u8 *dev_uuid)
6483 {
6484 	struct btrfs_device *device;
6485 	unsigned int nofs_flag;
6486 
6487 	/*
6488 	 * We call this under the chunk_mutex, so we want to use NOFS for this
6489 	 * allocation, however we don't want to change btrfs_alloc_device() to
6490 	 * always do NOFS because we use it in a lot of other GFP_KERNEL safe
6491 	 * places.
6492 	 */
6493 	nofs_flag = memalloc_nofs_save();
6494 	device = btrfs_alloc_device(NULL, &devid, dev_uuid);
6495 	memalloc_nofs_restore(nofs_flag);
6496 	if (IS_ERR(device))
6497 		return device;
6498 
6499 	list_add(&device->dev_list, &fs_devices->devices);
6500 	device->fs_devices = fs_devices;
6501 	fs_devices->num_devices++;
6502 
6503 	set_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state);
6504 	fs_devices->missing_devices++;
6505 
6506 	return device;
6507 }
6508 
6509 /**
6510  * btrfs_alloc_device - allocate struct btrfs_device
6511  * @fs_info:	used only for generating a new devid, can be NULL if
6512  *		devid is provided (i.e. @devid != NULL).
6513  * @devid:	a pointer to devid for this device.  If NULL a new devid
6514  *		is generated.
6515  * @uuid:	a pointer to UUID for this device.  If NULL a new UUID
6516  *		is generated.
6517  *
6518  * Return: a pointer to a new &struct btrfs_device on success; ERR_PTR()
6519  * on error.  Returned struct is not linked onto any lists and must be
6520  * destroyed with btrfs_free_device.
6521  */
6522 struct btrfs_device *btrfs_alloc_device(struct btrfs_fs_info *fs_info,
6523 					const u64 *devid,
6524 					const u8 *uuid)
6525 {
6526 	struct btrfs_device *dev;
6527 	u64 tmp;
6528 
6529 	if (WARN_ON(!devid && !fs_info))
6530 		return ERR_PTR(-EINVAL);
6531 
6532 	dev = __alloc_device(fs_info);
6533 	if (IS_ERR(dev))
6534 		return dev;
6535 
6536 	if (devid)
6537 		tmp = *devid;
6538 	else {
6539 		int ret;
6540 
6541 		ret = find_next_devid(fs_info, &tmp);
6542 		if (ret) {
6543 			btrfs_free_device(dev);
6544 			return ERR_PTR(ret);
6545 		}
6546 	}
6547 	dev->devid = tmp;
6548 
6549 	if (uuid)
6550 		memcpy(dev->uuid, uuid, BTRFS_UUID_SIZE);
6551 	else
6552 		generate_random_uuid(dev->uuid);
6553 
6554 	return dev;
6555 }
6556 
6557 static void btrfs_report_missing_device(struct btrfs_fs_info *fs_info,
6558 					u64 devid, u8 *uuid, bool error)
6559 {
6560 	if (error)
6561 		btrfs_err_rl(fs_info, "devid %llu uuid %pU is missing",
6562 			      devid, uuid);
6563 	else
6564 		btrfs_warn_rl(fs_info, "devid %llu uuid %pU is missing",
6565 			      devid, uuid);
6566 }
6567 
6568 static u64 calc_stripe_length(u64 type, u64 chunk_len, int num_stripes)
6569 {
6570 	int index = btrfs_bg_flags_to_raid_index(type);
6571 	int ncopies = btrfs_raid_array[index].ncopies;
6572 	const int nparity = btrfs_raid_array[index].nparity;
6573 	int data_stripes;
6574 
6575 	if (nparity)
6576 		data_stripes = num_stripes - nparity;
6577 	else
6578 		data_stripes = num_stripes / ncopies;
6579 
6580 	return div_u64(chunk_len, data_stripes);
6581 }
6582 
6583 static int read_one_chunk(struct btrfs_key *key, struct extent_buffer *leaf,
6584 			  struct btrfs_chunk *chunk)
6585 {
6586 	struct btrfs_fs_info *fs_info = leaf->fs_info;
6587 	struct extent_map_tree *map_tree = &fs_info->mapping_tree;
6588 	struct map_lookup *map;
6589 	struct extent_map *em;
6590 	u64 logical;
6591 	u64 length;
6592 	u64 devid;
6593 	u8 uuid[BTRFS_UUID_SIZE];
6594 	int num_stripes;
6595 	int ret;
6596 	int i;
6597 
6598 	logical = key->offset;
6599 	length = btrfs_chunk_length(leaf, chunk);
6600 	num_stripes = btrfs_chunk_num_stripes(leaf, chunk);
6601 
6602 	/*
6603 	 * Only need to verify chunk item if we're reading from sys chunk array,
6604 	 * as chunk item in tree block is already verified by tree-checker.
6605 	 */
6606 	if (leaf->start == BTRFS_SUPER_INFO_OFFSET) {
6607 		ret = btrfs_check_chunk_valid(leaf, chunk, logical);
6608 		if (ret)
6609 			return ret;
6610 	}
6611 
6612 	read_lock(&map_tree->lock);
6613 	em = lookup_extent_mapping(map_tree, logical, 1);
6614 	read_unlock(&map_tree->lock);
6615 
6616 	/* already mapped? */
6617 	if (em && em->start <= logical && em->start + em->len > logical) {
6618 		free_extent_map(em);
6619 		return 0;
6620 	} else if (em) {
6621 		free_extent_map(em);
6622 	}
6623 
6624 	em = alloc_extent_map();
6625 	if (!em)
6626 		return -ENOMEM;
6627 	map = kmalloc(map_lookup_size(num_stripes), GFP_NOFS);
6628 	if (!map) {
6629 		free_extent_map(em);
6630 		return -ENOMEM;
6631 	}
6632 
6633 	set_bit(EXTENT_FLAG_FS_MAPPING, &em->flags);
6634 	em->map_lookup = map;
6635 	em->start = logical;
6636 	em->len = length;
6637 	em->orig_start = 0;
6638 	em->block_start = 0;
6639 	em->block_len = em->len;
6640 
6641 	map->num_stripes = num_stripes;
6642 	map->io_width = btrfs_chunk_io_width(leaf, chunk);
6643 	map->io_align = btrfs_chunk_io_align(leaf, chunk);
6644 	map->stripe_len = btrfs_chunk_stripe_len(leaf, chunk);
6645 	map->type = btrfs_chunk_type(leaf, chunk);
6646 	map->sub_stripes = btrfs_chunk_sub_stripes(leaf, chunk);
6647 	map->verified_stripes = 0;
6648 	em->orig_block_len = calc_stripe_length(map->type, em->len,
6649 						map->num_stripes);
6650 	for (i = 0; i < num_stripes; i++) {
6651 		map->stripes[i].physical =
6652 			btrfs_stripe_offset_nr(leaf, chunk, i);
6653 		devid = btrfs_stripe_devid_nr(leaf, chunk, i);
6654 		read_extent_buffer(leaf, uuid, (unsigned long)
6655 				   btrfs_stripe_dev_uuid_nr(chunk, i),
6656 				   BTRFS_UUID_SIZE);
6657 		map->stripes[i].dev = btrfs_find_device(fs_info->fs_devices,
6658 							devid, uuid, NULL, true);
6659 		if (!map->stripes[i].dev &&
6660 		    !btrfs_test_opt(fs_info, DEGRADED)) {
6661 			free_extent_map(em);
6662 			btrfs_report_missing_device(fs_info, devid, uuid, true);
6663 			return -ENOENT;
6664 		}
6665 		if (!map->stripes[i].dev) {
6666 			map->stripes[i].dev =
6667 				add_missing_dev(fs_info->fs_devices, devid,
6668 						uuid);
6669 			if (IS_ERR(map->stripes[i].dev)) {
6670 				free_extent_map(em);
6671 				btrfs_err(fs_info,
6672 					"failed to init missing dev %llu: %ld",
6673 					devid, PTR_ERR(map->stripes[i].dev));
6674 				return PTR_ERR(map->stripes[i].dev);
6675 			}
6676 			btrfs_report_missing_device(fs_info, devid, uuid, false);
6677 		}
6678 		set_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
6679 				&(map->stripes[i].dev->dev_state));
6680 
6681 	}
6682 
6683 	write_lock(&map_tree->lock);
6684 	ret = add_extent_mapping(map_tree, em, 0);
6685 	write_unlock(&map_tree->lock);
6686 	if (ret < 0) {
6687 		btrfs_err(fs_info,
6688 			  "failed to add chunk map, start=%llu len=%llu: %d",
6689 			  em->start, em->len, ret);
6690 	}
6691 	free_extent_map(em);
6692 
6693 	return ret;
6694 }
6695 
6696 static void fill_device_from_item(struct extent_buffer *leaf,
6697 				 struct btrfs_dev_item *dev_item,
6698 				 struct btrfs_device *device)
6699 {
6700 	unsigned long ptr;
6701 
6702 	device->devid = btrfs_device_id(leaf, dev_item);
6703 	device->disk_total_bytes = btrfs_device_total_bytes(leaf, dev_item);
6704 	device->total_bytes = device->disk_total_bytes;
6705 	device->commit_total_bytes = device->disk_total_bytes;
6706 	device->bytes_used = btrfs_device_bytes_used(leaf, dev_item);
6707 	device->commit_bytes_used = device->bytes_used;
6708 	device->type = btrfs_device_type(leaf, dev_item);
6709 	device->io_align = btrfs_device_io_align(leaf, dev_item);
6710 	device->io_width = btrfs_device_io_width(leaf, dev_item);
6711 	device->sector_size = btrfs_device_sector_size(leaf, dev_item);
6712 	WARN_ON(device->devid == BTRFS_DEV_REPLACE_DEVID);
6713 	clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state);
6714 
6715 	ptr = btrfs_device_uuid(dev_item);
6716 	read_extent_buffer(leaf, device->uuid, ptr, BTRFS_UUID_SIZE);
6717 }
6718 
6719 static struct btrfs_fs_devices *open_seed_devices(struct btrfs_fs_info *fs_info,
6720 						  u8 *fsid)
6721 {
6722 	struct btrfs_fs_devices *fs_devices;
6723 	int ret;
6724 
6725 	lockdep_assert_held(&uuid_mutex);
6726 	ASSERT(fsid);
6727 
6728 	/* This will match only for multi-device seed fs */
6729 	list_for_each_entry(fs_devices, &fs_info->fs_devices->seed_list, seed_list)
6730 		if (!memcmp(fs_devices->fsid, fsid, BTRFS_FSID_SIZE))
6731 			return fs_devices;
6732 
6733 
6734 	fs_devices = find_fsid(fsid, NULL);
6735 	if (!fs_devices) {
6736 		if (!btrfs_test_opt(fs_info, DEGRADED))
6737 			return ERR_PTR(-ENOENT);
6738 
6739 		fs_devices = alloc_fs_devices(fsid, NULL);
6740 		if (IS_ERR(fs_devices))
6741 			return fs_devices;
6742 
6743 		fs_devices->seeding = true;
6744 		fs_devices->opened = 1;
6745 		return fs_devices;
6746 	}
6747 
6748 	/*
6749 	 * Upon first call for a seed fs fsid, just create a private copy of the
6750 	 * respective fs_devices and anchor it at fs_info->fs_devices->seed_list
6751 	 */
6752 	fs_devices = clone_fs_devices(fs_devices);
6753 	if (IS_ERR(fs_devices))
6754 		return fs_devices;
6755 
6756 	ret = open_fs_devices(fs_devices, FMODE_READ, fs_info->bdev_holder);
6757 	if (ret) {
6758 		free_fs_devices(fs_devices);
6759 		return ERR_PTR(ret);
6760 	}
6761 
6762 	if (!fs_devices->seeding) {
6763 		close_fs_devices(fs_devices);
6764 		free_fs_devices(fs_devices);
6765 		return ERR_PTR(-EINVAL);
6766 	}
6767 
6768 	list_add(&fs_devices->seed_list, &fs_info->fs_devices->seed_list);
6769 
6770 	return fs_devices;
6771 }
6772 
6773 static int read_one_dev(struct extent_buffer *leaf,
6774 			struct btrfs_dev_item *dev_item)
6775 {
6776 	struct btrfs_fs_info *fs_info = leaf->fs_info;
6777 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
6778 	struct btrfs_device *device;
6779 	u64 devid;
6780 	int ret;
6781 	u8 fs_uuid[BTRFS_FSID_SIZE];
6782 	u8 dev_uuid[BTRFS_UUID_SIZE];
6783 
6784 	devid = btrfs_device_id(leaf, dev_item);
6785 	read_extent_buffer(leaf, dev_uuid, btrfs_device_uuid(dev_item),
6786 			   BTRFS_UUID_SIZE);
6787 	read_extent_buffer(leaf, fs_uuid, btrfs_device_fsid(dev_item),
6788 			   BTRFS_FSID_SIZE);
6789 
6790 	if (memcmp(fs_uuid, fs_devices->metadata_uuid, BTRFS_FSID_SIZE)) {
6791 		fs_devices = open_seed_devices(fs_info, fs_uuid);
6792 		if (IS_ERR(fs_devices))
6793 			return PTR_ERR(fs_devices);
6794 	}
6795 
6796 	device = btrfs_find_device(fs_info->fs_devices, devid, dev_uuid,
6797 				   fs_uuid, true);
6798 	if (!device) {
6799 		if (!btrfs_test_opt(fs_info, DEGRADED)) {
6800 			btrfs_report_missing_device(fs_info, devid,
6801 							dev_uuid, true);
6802 			return -ENOENT;
6803 		}
6804 
6805 		device = add_missing_dev(fs_devices, devid, dev_uuid);
6806 		if (IS_ERR(device)) {
6807 			btrfs_err(fs_info,
6808 				"failed to add missing dev %llu: %ld",
6809 				devid, PTR_ERR(device));
6810 			return PTR_ERR(device);
6811 		}
6812 		btrfs_report_missing_device(fs_info, devid, dev_uuid, false);
6813 	} else {
6814 		if (!device->bdev) {
6815 			if (!btrfs_test_opt(fs_info, DEGRADED)) {
6816 				btrfs_report_missing_device(fs_info,
6817 						devid, dev_uuid, true);
6818 				return -ENOENT;
6819 			}
6820 			btrfs_report_missing_device(fs_info, devid,
6821 							dev_uuid, false);
6822 		}
6823 
6824 		if (!device->bdev &&
6825 		    !test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) {
6826 			/*
6827 			 * this happens when a device that was properly setup
6828 			 * in the device info lists suddenly goes bad.
6829 			 * device->bdev is NULL, and so we have to set
6830 			 * device->missing to one here
6831 			 */
6832 			device->fs_devices->missing_devices++;
6833 			set_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state);
6834 		}
6835 
6836 		/* Move the device to its own fs_devices */
6837 		if (device->fs_devices != fs_devices) {
6838 			ASSERT(test_bit(BTRFS_DEV_STATE_MISSING,
6839 							&device->dev_state));
6840 
6841 			list_move(&device->dev_list, &fs_devices->devices);
6842 			device->fs_devices->num_devices--;
6843 			fs_devices->num_devices++;
6844 
6845 			device->fs_devices->missing_devices--;
6846 			fs_devices->missing_devices++;
6847 
6848 			device->fs_devices = fs_devices;
6849 		}
6850 	}
6851 
6852 	if (device->fs_devices != fs_info->fs_devices) {
6853 		BUG_ON(test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state));
6854 		if (device->generation !=
6855 		    btrfs_device_generation(leaf, dev_item))
6856 			return -EINVAL;
6857 	}
6858 
6859 	fill_device_from_item(leaf, dev_item, device);
6860 	set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
6861 	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
6862 	   !test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
6863 		device->fs_devices->total_rw_bytes += device->total_bytes;
6864 		atomic64_add(device->total_bytes - device->bytes_used,
6865 				&fs_info->free_chunk_space);
6866 	}
6867 	ret = 0;
6868 	return ret;
6869 }
6870 
6871 int btrfs_read_sys_array(struct btrfs_fs_info *fs_info)
6872 {
6873 	struct btrfs_root *root = fs_info->tree_root;
6874 	struct btrfs_super_block *super_copy = fs_info->super_copy;
6875 	struct extent_buffer *sb;
6876 	struct btrfs_disk_key *disk_key;
6877 	struct btrfs_chunk *chunk;
6878 	u8 *array_ptr;
6879 	unsigned long sb_array_offset;
6880 	int ret = 0;
6881 	u32 num_stripes;
6882 	u32 array_size;
6883 	u32 len = 0;
6884 	u32 cur_offset;
6885 	u64 type;
6886 	struct btrfs_key key;
6887 
6888 	ASSERT(BTRFS_SUPER_INFO_SIZE <= fs_info->nodesize);
6889 	/*
6890 	 * This will create extent buffer of nodesize, superblock size is
6891 	 * fixed to BTRFS_SUPER_INFO_SIZE. If nodesize > sb size, this will
6892 	 * overallocate but we can keep it as-is, only the first page is used.
6893 	 */
6894 	sb = btrfs_find_create_tree_block(fs_info, BTRFS_SUPER_INFO_OFFSET);
6895 	if (IS_ERR(sb))
6896 		return PTR_ERR(sb);
6897 	set_extent_buffer_uptodate(sb);
6898 	btrfs_set_buffer_lockdep_class(root->root_key.objectid, sb, 0);
6899 	/*
6900 	 * The sb extent buffer is artificial and just used to read the system array.
6901 	 * set_extent_buffer_uptodate() call does not properly mark all it's
6902 	 * pages up-to-date when the page is larger: extent does not cover the
6903 	 * whole page and consequently check_page_uptodate does not find all
6904 	 * the page's extents up-to-date (the hole beyond sb),
6905 	 * write_extent_buffer then triggers a WARN_ON.
6906 	 *
6907 	 * Regular short extents go through mark_extent_buffer_dirty/writeback cycle,
6908 	 * but sb spans only this function. Add an explicit SetPageUptodate call
6909 	 * to silence the warning eg. on PowerPC 64.
6910 	 */
6911 	if (PAGE_SIZE > BTRFS_SUPER_INFO_SIZE)
6912 		SetPageUptodate(sb->pages[0]);
6913 
6914 	write_extent_buffer(sb, super_copy, 0, BTRFS_SUPER_INFO_SIZE);
6915 	array_size = btrfs_super_sys_array_size(super_copy);
6916 
6917 	array_ptr = super_copy->sys_chunk_array;
6918 	sb_array_offset = offsetof(struct btrfs_super_block, sys_chunk_array);
6919 	cur_offset = 0;
6920 
6921 	while (cur_offset < array_size) {
6922 		disk_key = (struct btrfs_disk_key *)array_ptr;
6923 		len = sizeof(*disk_key);
6924 		if (cur_offset + len > array_size)
6925 			goto out_short_read;
6926 
6927 		btrfs_disk_key_to_cpu(&key, disk_key);
6928 
6929 		array_ptr += len;
6930 		sb_array_offset += len;
6931 		cur_offset += len;
6932 
6933 		if (key.type != BTRFS_CHUNK_ITEM_KEY) {
6934 			btrfs_err(fs_info,
6935 			    "unexpected item type %u in sys_array at offset %u",
6936 				  (u32)key.type, cur_offset);
6937 			ret = -EIO;
6938 			break;
6939 		}
6940 
6941 		chunk = (struct btrfs_chunk *)sb_array_offset;
6942 		/*
6943 		 * At least one btrfs_chunk with one stripe must be present,
6944 		 * exact stripe count check comes afterwards
6945 		 */
6946 		len = btrfs_chunk_item_size(1);
6947 		if (cur_offset + len > array_size)
6948 			goto out_short_read;
6949 
6950 		num_stripes = btrfs_chunk_num_stripes(sb, chunk);
6951 		if (!num_stripes) {
6952 			btrfs_err(fs_info,
6953 			"invalid number of stripes %u in sys_array at offset %u",
6954 				  num_stripes, cur_offset);
6955 			ret = -EIO;
6956 			break;
6957 		}
6958 
6959 		type = btrfs_chunk_type(sb, chunk);
6960 		if ((type & BTRFS_BLOCK_GROUP_SYSTEM) == 0) {
6961 			btrfs_err(fs_info,
6962 			"invalid chunk type %llu in sys_array at offset %u",
6963 				  type, cur_offset);
6964 			ret = -EIO;
6965 			break;
6966 		}
6967 
6968 		len = btrfs_chunk_item_size(num_stripes);
6969 		if (cur_offset + len > array_size)
6970 			goto out_short_read;
6971 
6972 		ret = read_one_chunk(&key, sb, chunk);
6973 		if (ret)
6974 			break;
6975 
6976 		array_ptr += len;
6977 		sb_array_offset += len;
6978 		cur_offset += len;
6979 	}
6980 	clear_extent_buffer_uptodate(sb);
6981 	free_extent_buffer_stale(sb);
6982 	return ret;
6983 
6984 out_short_read:
6985 	btrfs_err(fs_info, "sys_array too short to read %u bytes at offset %u",
6986 			len, cur_offset);
6987 	clear_extent_buffer_uptodate(sb);
6988 	free_extent_buffer_stale(sb);
6989 	return -EIO;
6990 }
6991 
6992 /*
6993  * Check if all chunks in the fs are OK for read-write degraded mount
6994  *
6995  * If the @failing_dev is specified, it's accounted as missing.
6996  *
6997  * Return true if all chunks meet the minimal RW mount requirements.
6998  * Return false if any chunk doesn't meet the minimal RW mount requirements.
6999  */
7000 bool btrfs_check_rw_degradable(struct btrfs_fs_info *fs_info,
7001 					struct btrfs_device *failing_dev)
7002 {
7003 	struct extent_map_tree *map_tree = &fs_info->mapping_tree;
7004 	struct extent_map *em;
7005 	u64 next_start = 0;
7006 	bool ret = true;
7007 
7008 	read_lock(&map_tree->lock);
7009 	em = lookup_extent_mapping(map_tree, 0, (u64)-1);
7010 	read_unlock(&map_tree->lock);
7011 	/* No chunk at all? Return false anyway */
7012 	if (!em) {
7013 		ret = false;
7014 		goto out;
7015 	}
7016 	while (em) {
7017 		struct map_lookup *map;
7018 		int missing = 0;
7019 		int max_tolerated;
7020 		int i;
7021 
7022 		map = em->map_lookup;
7023 		max_tolerated =
7024 			btrfs_get_num_tolerated_disk_barrier_failures(
7025 					map->type);
7026 		for (i = 0; i < map->num_stripes; i++) {
7027 			struct btrfs_device *dev = map->stripes[i].dev;
7028 
7029 			if (!dev || !dev->bdev ||
7030 			    test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) ||
7031 			    dev->last_flush_error)
7032 				missing++;
7033 			else if (failing_dev && failing_dev == dev)
7034 				missing++;
7035 		}
7036 		if (missing > max_tolerated) {
7037 			if (!failing_dev)
7038 				btrfs_warn(fs_info,
7039 	"chunk %llu missing %d devices, max tolerance is %d for writable mount",
7040 				   em->start, missing, max_tolerated);
7041 			free_extent_map(em);
7042 			ret = false;
7043 			goto out;
7044 		}
7045 		next_start = extent_map_end(em);
7046 		free_extent_map(em);
7047 
7048 		read_lock(&map_tree->lock);
7049 		em = lookup_extent_mapping(map_tree, next_start,
7050 					   (u64)(-1) - next_start);
7051 		read_unlock(&map_tree->lock);
7052 	}
7053 out:
7054 	return ret;
7055 }
7056 
7057 static void readahead_tree_node_children(struct extent_buffer *node)
7058 {
7059 	int i;
7060 	const int nr_items = btrfs_header_nritems(node);
7061 
7062 	for (i = 0; i < nr_items; i++) {
7063 		u64 start;
7064 
7065 		start = btrfs_node_blockptr(node, i);
7066 		readahead_tree_block(node->fs_info, start);
7067 	}
7068 }
7069 
7070 int btrfs_read_chunk_tree(struct btrfs_fs_info *fs_info)
7071 {
7072 	struct btrfs_root *root = fs_info->chunk_root;
7073 	struct btrfs_path *path;
7074 	struct extent_buffer *leaf;
7075 	struct btrfs_key key;
7076 	struct btrfs_key found_key;
7077 	int ret;
7078 	int slot;
7079 	u64 total_dev = 0;
7080 	u64 last_ra_node = 0;
7081 
7082 	path = btrfs_alloc_path();
7083 	if (!path)
7084 		return -ENOMEM;
7085 
7086 	/*
7087 	 * uuid_mutex is needed only if we are mounting a sprout FS
7088 	 * otherwise we don't need it.
7089 	 */
7090 	mutex_lock(&uuid_mutex);
7091 
7092 	/*
7093 	 * It is possible for mount and umount to race in such a way that
7094 	 * we execute this code path, but open_fs_devices failed to clear
7095 	 * total_rw_bytes. We certainly want it cleared before reading the
7096 	 * device items, so clear it here.
7097 	 */
7098 	fs_info->fs_devices->total_rw_bytes = 0;
7099 
7100 	/*
7101 	 * Read all device items, and then all the chunk items. All
7102 	 * device items are found before any chunk item (their object id
7103 	 * is smaller than the lowest possible object id for a chunk
7104 	 * item - BTRFS_FIRST_CHUNK_TREE_OBJECTID).
7105 	 */
7106 	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
7107 	key.offset = 0;
7108 	key.type = 0;
7109 	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
7110 	if (ret < 0)
7111 		goto error;
7112 	while (1) {
7113 		struct extent_buffer *node;
7114 
7115 		leaf = path->nodes[0];
7116 		slot = path->slots[0];
7117 		if (slot >= btrfs_header_nritems(leaf)) {
7118 			ret = btrfs_next_leaf(root, path);
7119 			if (ret == 0)
7120 				continue;
7121 			if (ret < 0)
7122 				goto error;
7123 			break;
7124 		}
7125 		/*
7126 		 * The nodes on level 1 are not locked but we don't need to do
7127 		 * that during mount time as nothing else can access the tree
7128 		 */
7129 		node = path->nodes[1];
7130 		if (node) {
7131 			if (last_ra_node != node->start) {
7132 				readahead_tree_node_children(node);
7133 				last_ra_node = node->start;
7134 			}
7135 		}
7136 		btrfs_item_key_to_cpu(leaf, &found_key, slot);
7137 		if (found_key.type == BTRFS_DEV_ITEM_KEY) {
7138 			struct btrfs_dev_item *dev_item;
7139 			dev_item = btrfs_item_ptr(leaf, slot,
7140 						  struct btrfs_dev_item);
7141 			ret = read_one_dev(leaf, dev_item);
7142 			if (ret)
7143 				goto error;
7144 			total_dev++;
7145 		} else if (found_key.type == BTRFS_CHUNK_ITEM_KEY) {
7146 			struct btrfs_chunk *chunk;
7147 			chunk = btrfs_item_ptr(leaf, slot, struct btrfs_chunk);
7148 			mutex_lock(&fs_info->chunk_mutex);
7149 			ret = read_one_chunk(&found_key, leaf, chunk);
7150 			mutex_unlock(&fs_info->chunk_mutex);
7151 			if (ret)
7152 				goto error;
7153 		}
7154 		path->slots[0]++;
7155 	}
7156 
7157 	/*
7158 	 * After loading chunk tree, we've got all device information,
7159 	 * do another round of validation checks.
7160 	 */
7161 	if (total_dev != fs_info->fs_devices->total_devices) {
7162 		btrfs_err(fs_info,
7163 	   "super_num_devices %llu mismatch with num_devices %llu found here",
7164 			  btrfs_super_num_devices(fs_info->super_copy),
7165 			  total_dev);
7166 		ret = -EINVAL;
7167 		goto error;
7168 	}
7169 	if (btrfs_super_total_bytes(fs_info->super_copy) <
7170 	    fs_info->fs_devices->total_rw_bytes) {
7171 		btrfs_err(fs_info,
7172 	"super_total_bytes %llu mismatch with fs_devices total_rw_bytes %llu",
7173 			  btrfs_super_total_bytes(fs_info->super_copy),
7174 			  fs_info->fs_devices->total_rw_bytes);
7175 		ret = -EINVAL;
7176 		goto error;
7177 	}
7178 	ret = 0;
7179 error:
7180 	mutex_unlock(&uuid_mutex);
7181 
7182 	btrfs_free_path(path);
7183 	return ret;
7184 }
7185 
7186 void btrfs_init_devices_late(struct btrfs_fs_info *fs_info)
7187 {
7188 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs;
7189 	struct btrfs_device *device;
7190 
7191 	fs_devices->fs_info = fs_info;
7192 
7193 	mutex_lock(&fs_devices->device_list_mutex);
7194 	list_for_each_entry(device, &fs_devices->devices, dev_list)
7195 		device->fs_info = fs_info;
7196 
7197 	list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) {
7198 		list_for_each_entry(device, &seed_devs->devices, dev_list)
7199 			device->fs_info = fs_info;
7200 
7201 		seed_devs->fs_info = fs_info;
7202 	}
7203 	mutex_unlock(&fs_devices->device_list_mutex);
7204 }
7205 
7206 static u64 btrfs_dev_stats_value(const struct extent_buffer *eb,
7207 				 const struct btrfs_dev_stats_item *ptr,
7208 				 int index)
7209 {
7210 	u64 val;
7211 
7212 	read_extent_buffer(eb, &val,
7213 			   offsetof(struct btrfs_dev_stats_item, values) +
7214 			    ((unsigned long)ptr) + (index * sizeof(u64)),
7215 			   sizeof(val));
7216 	return val;
7217 }
7218 
7219 static void btrfs_set_dev_stats_value(struct extent_buffer *eb,
7220 				      struct btrfs_dev_stats_item *ptr,
7221 				      int index, u64 val)
7222 {
7223 	write_extent_buffer(eb, &val,
7224 			    offsetof(struct btrfs_dev_stats_item, values) +
7225 			     ((unsigned long)ptr) + (index * sizeof(u64)),
7226 			    sizeof(val));
7227 }
7228 
7229 static int btrfs_device_init_dev_stats(struct btrfs_device *device,
7230 				       struct btrfs_path *path)
7231 {
7232 	struct btrfs_dev_stats_item *ptr;
7233 	struct extent_buffer *eb;
7234 	struct btrfs_key key;
7235 	int item_size;
7236 	int i, ret, slot;
7237 
7238 	key.objectid = BTRFS_DEV_STATS_OBJECTID;
7239 	key.type = BTRFS_PERSISTENT_ITEM_KEY;
7240 	key.offset = device->devid;
7241 	ret = btrfs_search_slot(NULL, device->fs_info->dev_root, &key, path, 0, 0);
7242 	if (ret) {
7243 		for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
7244 			btrfs_dev_stat_set(device, i, 0);
7245 		device->dev_stats_valid = 1;
7246 		btrfs_release_path(path);
7247 		return ret < 0 ? ret : 0;
7248 	}
7249 	slot = path->slots[0];
7250 	eb = path->nodes[0];
7251 	item_size = btrfs_item_size_nr(eb, slot);
7252 
7253 	ptr = btrfs_item_ptr(eb, slot, struct btrfs_dev_stats_item);
7254 
7255 	for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) {
7256 		if (item_size >= (1 + i) * sizeof(__le64))
7257 			btrfs_dev_stat_set(device, i,
7258 					   btrfs_dev_stats_value(eb, ptr, i));
7259 		else
7260 			btrfs_dev_stat_set(device, i, 0);
7261 	}
7262 
7263 	device->dev_stats_valid = 1;
7264 	btrfs_dev_stat_print_on_load(device);
7265 	btrfs_release_path(path);
7266 
7267 	return 0;
7268 }
7269 
7270 int btrfs_init_dev_stats(struct btrfs_fs_info *fs_info)
7271 {
7272 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs;
7273 	struct btrfs_device *device;
7274 	struct btrfs_path *path = NULL;
7275 	int ret = 0;
7276 
7277 	path = btrfs_alloc_path();
7278 	if (!path)
7279 		return -ENOMEM;
7280 
7281 	mutex_lock(&fs_devices->device_list_mutex);
7282 	list_for_each_entry(device, &fs_devices->devices, dev_list) {
7283 		ret = btrfs_device_init_dev_stats(device, path);
7284 		if (ret)
7285 			goto out;
7286 	}
7287 	list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) {
7288 		list_for_each_entry(device, &seed_devs->devices, dev_list) {
7289 			ret = btrfs_device_init_dev_stats(device, path);
7290 			if (ret)
7291 				goto out;
7292 		}
7293 	}
7294 out:
7295 	mutex_unlock(&fs_devices->device_list_mutex);
7296 
7297 	btrfs_free_path(path);
7298 	return ret;
7299 }
7300 
7301 static int update_dev_stat_item(struct btrfs_trans_handle *trans,
7302 				struct btrfs_device *device)
7303 {
7304 	struct btrfs_fs_info *fs_info = trans->fs_info;
7305 	struct btrfs_root *dev_root = fs_info->dev_root;
7306 	struct btrfs_path *path;
7307 	struct btrfs_key key;
7308 	struct extent_buffer *eb;
7309 	struct btrfs_dev_stats_item *ptr;
7310 	int ret;
7311 	int i;
7312 
7313 	key.objectid = BTRFS_DEV_STATS_OBJECTID;
7314 	key.type = BTRFS_PERSISTENT_ITEM_KEY;
7315 	key.offset = device->devid;
7316 
7317 	path = btrfs_alloc_path();
7318 	if (!path)
7319 		return -ENOMEM;
7320 	ret = btrfs_search_slot(trans, dev_root, &key, path, -1, 1);
7321 	if (ret < 0) {
7322 		btrfs_warn_in_rcu(fs_info,
7323 			"error %d while searching for dev_stats item for device %s",
7324 			      ret, rcu_str_deref(device->name));
7325 		goto out;
7326 	}
7327 
7328 	if (ret == 0 &&
7329 	    btrfs_item_size_nr(path->nodes[0], path->slots[0]) < sizeof(*ptr)) {
7330 		/* need to delete old one and insert a new one */
7331 		ret = btrfs_del_item(trans, dev_root, path);
7332 		if (ret != 0) {
7333 			btrfs_warn_in_rcu(fs_info,
7334 				"delete too small dev_stats item for device %s failed %d",
7335 				      rcu_str_deref(device->name), ret);
7336 			goto out;
7337 		}
7338 		ret = 1;
7339 	}
7340 
7341 	if (ret == 1) {
7342 		/* need to insert a new item */
7343 		btrfs_release_path(path);
7344 		ret = btrfs_insert_empty_item(trans, dev_root, path,
7345 					      &key, sizeof(*ptr));
7346 		if (ret < 0) {
7347 			btrfs_warn_in_rcu(fs_info,
7348 				"insert dev_stats item for device %s failed %d",
7349 				rcu_str_deref(device->name), ret);
7350 			goto out;
7351 		}
7352 	}
7353 
7354 	eb = path->nodes[0];
7355 	ptr = btrfs_item_ptr(eb, path->slots[0], struct btrfs_dev_stats_item);
7356 	for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
7357 		btrfs_set_dev_stats_value(eb, ptr, i,
7358 					  btrfs_dev_stat_read(device, i));
7359 	btrfs_mark_buffer_dirty(eb);
7360 
7361 out:
7362 	btrfs_free_path(path);
7363 	return ret;
7364 }
7365 
7366 /*
7367  * called from commit_transaction. Writes all changed device stats to disk.
7368  */
7369 int btrfs_run_dev_stats(struct btrfs_trans_handle *trans)
7370 {
7371 	struct btrfs_fs_info *fs_info = trans->fs_info;
7372 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
7373 	struct btrfs_device *device;
7374 	int stats_cnt;
7375 	int ret = 0;
7376 
7377 	mutex_lock(&fs_devices->device_list_mutex);
7378 	list_for_each_entry(device, &fs_devices->devices, dev_list) {
7379 		stats_cnt = atomic_read(&device->dev_stats_ccnt);
7380 		if (!device->dev_stats_valid || stats_cnt == 0)
7381 			continue;
7382 
7383 
7384 		/*
7385 		 * There is a LOAD-LOAD control dependency between the value of
7386 		 * dev_stats_ccnt and updating the on-disk values which requires
7387 		 * reading the in-memory counters. Such control dependencies
7388 		 * require explicit read memory barriers.
7389 		 *
7390 		 * This memory barriers pairs with smp_mb__before_atomic in
7391 		 * btrfs_dev_stat_inc/btrfs_dev_stat_set and with the full
7392 		 * barrier implied by atomic_xchg in
7393 		 * btrfs_dev_stats_read_and_reset
7394 		 */
7395 		smp_rmb();
7396 
7397 		ret = update_dev_stat_item(trans, device);
7398 		if (!ret)
7399 			atomic_sub(stats_cnt, &device->dev_stats_ccnt);
7400 	}
7401 	mutex_unlock(&fs_devices->device_list_mutex);
7402 
7403 	return ret;
7404 }
7405 
7406 void btrfs_dev_stat_inc_and_print(struct btrfs_device *dev, int index)
7407 {
7408 	btrfs_dev_stat_inc(dev, index);
7409 	btrfs_dev_stat_print_on_error(dev);
7410 }
7411 
7412 static void btrfs_dev_stat_print_on_error(struct btrfs_device *dev)
7413 {
7414 	if (!dev->dev_stats_valid)
7415 		return;
7416 	btrfs_err_rl_in_rcu(dev->fs_info,
7417 		"bdev %s errs: wr %u, rd %u, flush %u, corrupt %u, gen %u",
7418 			   rcu_str_deref(dev->name),
7419 			   btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_WRITE_ERRS),
7420 			   btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_READ_ERRS),
7421 			   btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_FLUSH_ERRS),
7422 			   btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS),
7423 			   btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_GENERATION_ERRS));
7424 }
7425 
7426 static void btrfs_dev_stat_print_on_load(struct btrfs_device *dev)
7427 {
7428 	int i;
7429 
7430 	for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
7431 		if (btrfs_dev_stat_read(dev, i) != 0)
7432 			break;
7433 	if (i == BTRFS_DEV_STAT_VALUES_MAX)
7434 		return; /* all values == 0, suppress message */
7435 
7436 	btrfs_info_in_rcu(dev->fs_info,
7437 		"bdev %s errs: wr %u, rd %u, flush %u, corrupt %u, gen %u",
7438 	       rcu_str_deref(dev->name),
7439 	       btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_WRITE_ERRS),
7440 	       btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_READ_ERRS),
7441 	       btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_FLUSH_ERRS),
7442 	       btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS),
7443 	       btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_GENERATION_ERRS));
7444 }
7445 
7446 int btrfs_get_dev_stats(struct btrfs_fs_info *fs_info,
7447 			struct btrfs_ioctl_get_dev_stats *stats)
7448 {
7449 	struct btrfs_device *dev;
7450 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
7451 	int i;
7452 
7453 	mutex_lock(&fs_devices->device_list_mutex);
7454 	dev = btrfs_find_device(fs_info->fs_devices, stats->devid, NULL, NULL,
7455 				true);
7456 	mutex_unlock(&fs_devices->device_list_mutex);
7457 
7458 	if (!dev) {
7459 		btrfs_warn(fs_info, "get dev_stats failed, device not found");
7460 		return -ENODEV;
7461 	} else if (!dev->dev_stats_valid) {
7462 		btrfs_warn(fs_info, "get dev_stats failed, not yet valid");
7463 		return -ENODEV;
7464 	} else if (stats->flags & BTRFS_DEV_STATS_RESET) {
7465 		for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) {
7466 			if (stats->nr_items > i)
7467 				stats->values[i] =
7468 					btrfs_dev_stat_read_and_reset(dev, i);
7469 			else
7470 				btrfs_dev_stat_set(dev, i, 0);
7471 		}
7472 		btrfs_info(fs_info, "device stats zeroed by %s (%d)",
7473 			   current->comm, task_pid_nr(current));
7474 	} else {
7475 		for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
7476 			if (stats->nr_items > i)
7477 				stats->values[i] = btrfs_dev_stat_read(dev, i);
7478 	}
7479 	if (stats->nr_items > BTRFS_DEV_STAT_VALUES_MAX)
7480 		stats->nr_items = BTRFS_DEV_STAT_VALUES_MAX;
7481 	return 0;
7482 }
7483 
7484 /*
7485  * Update the size and bytes used for each device where it changed.  This is
7486  * delayed since we would otherwise get errors while writing out the
7487  * superblocks.
7488  *
7489  * Must be invoked during transaction commit.
7490  */
7491 void btrfs_commit_device_sizes(struct btrfs_transaction *trans)
7492 {
7493 	struct btrfs_device *curr, *next;
7494 
7495 	ASSERT(trans->state == TRANS_STATE_COMMIT_DOING);
7496 
7497 	if (list_empty(&trans->dev_update_list))
7498 		return;
7499 
7500 	/*
7501 	 * We don't need the device_list_mutex here.  This list is owned by the
7502 	 * transaction and the transaction must complete before the device is
7503 	 * released.
7504 	 */
7505 	mutex_lock(&trans->fs_info->chunk_mutex);
7506 	list_for_each_entry_safe(curr, next, &trans->dev_update_list,
7507 				 post_commit_list) {
7508 		list_del_init(&curr->post_commit_list);
7509 		curr->commit_total_bytes = curr->disk_total_bytes;
7510 		curr->commit_bytes_used = curr->bytes_used;
7511 	}
7512 	mutex_unlock(&trans->fs_info->chunk_mutex);
7513 }
7514 
7515 /*
7516  * Multiplicity factor for simple profiles: DUP, RAID1-like and RAID10.
7517  */
7518 int btrfs_bg_type_to_factor(u64 flags)
7519 {
7520 	const int index = btrfs_bg_flags_to_raid_index(flags);
7521 
7522 	return btrfs_raid_array[index].ncopies;
7523 }
7524 
7525 
7526 
7527 static int verify_one_dev_extent(struct btrfs_fs_info *fs_info,
7528 				 u64 chunk_offset, u64 devid,
7529 				 u64 physical_offset, u64 physical_len)
7530 {
7531 	struct extent_map_tree *em_tree = &fs_info->mapping_tree;
7532 	struct extent_map *em;
7533 	struct map_lookup *map;
7534 	struct btrfs_device *dev;
7535 	u64 stripe_len;
7536 	bool found = false;
7537 	int ret = 0;
7538 	int i;
7539 
7540 	read_lock(&em_tree->lock);
7541 	em = lookup_extent_mapping(em_tree, chunk_offset, 1);
7542 	read_unlock(&em_tree->lock);
7543 
7544 	if (!em) {
7545 		btrfs_err(fs_info,
7546 "dev extent physical offset %llu on devid %llu doesn't have corresponding chunk",
7547 			  physical_offset, devid);
7548 		ret = -EUCLEAN;
7549 		goto out;
7550 	}
7551 
7552 	map = em->map_lookup;
7553 	stripe_len = calc_stripe_length(map->type, em->len, map->num_stripes);
7554 	if (physical_len != stripe_len) {
7555 		btrfs_err(fs_info,
7556 "dev extent physical offset %llu on devid %llu length doesn't match chunk %llu, have %llu expect %llu",
7557 			  physical_offset, devid, em->start, physical_len,
7558 			  stripe_len);
7559 		ret = -EUCLEAN;
7560 		goto out;
7561 	}
7562 
7563 	for (i = 0; i < map->num_stripes; i++) {
7564 		if (map->stripes[i].dev->devid == devid &&
7565 		    map->stripes[i].physical == physical_offset) {
7566 			found = true;
7567 			if (map->verified_stripes >= map->num_stripes) {
7568 				btrfs_err(fs_info,
7569 				"too many dev extents for chunk %llu found",
7570 					  em->start);
7571 				ret = -EUCLEAN;
7572 				goto out;
7573 			}
7574 			map->verified_stripes++;
7575 			break;
7576 		}
7577 	}
7578 	if (!found) {
7579 		btrfs_err(fs_info,
7580 	"dev extent physical offset %llu devid %llu has no corresponding chunk",
7581 			physical_offset, devid);
7582 		ret = -EUCLEAN;
7583 	}
7584 
7585 	/* Make sure no dev extent is beyond device bondary */
7586 	dev = btrfs_find_device(fs_info->fs_devices, devid, NULL, NULL, true);
7587 	if (!dev) {
7588 		btrfs_err(fs_info, "failed to find devid %llu", devid);
7589 		ret = -EUCLEAN;
7590 		goto out;
7591 	}
7592 
7593 	/* It's possible this device is a dummy for seed device */
7594 	if (dev->disk_total_bytes == 0) {
7595 		struct btrfs_fs_devices *devs;
7596 
7597 		devs = list_first_entry(&fs_info->fs_devices->seed_list,
7598 					struct btrfs_fs_devices, seed_list);
7599 		dev = btrfs_find_device(devs, devid, NULL, NULL, false);
7600 		if (!dev) {
7601 			btrfs_err(fs_info, "failed to find seed devid %llu",
7602 				  devid);
7603 			ret = -EUCLEAN;
7604 			goto out;
7605 		}
7606 	}
7607 
7608 	if (physical_offset + physical_len > dev->disk_total_bytes) {
7609 		btrfs_err(fs_info,
7610 "dev extent devid %llu physical offset %llu len %llu is beyond device boundary %llu",
7611 			  devid, physical_offset, physical_len,
7612 			  dev->disk_total_bytes);
7613 		ret = -EUCLEAN;
7614 		goto out;
7615 	}
7616 out:
7617 	free_extent_map(em);
7618 	return ret;
7619 }
7620 
7621 static int verify_chunk_dev_extent_mapping(struct btrfs_fs_info *fs_info)
7622 {
7623 	struct extent_map_tree *em_tree = &fs_info->mapping_tree;
7624 	struct extent_map *em;
7625 	struct rb_node *node;
7626 	int ret = 0;
7627 
7628 	read_lock(&em_tree->lock);
7629 	for (node = rb_first_cached(&em_tree->map); node; node = rb_next(node)) {
7630 		em = rb_entry(node, struct extent_map, rb_node);
7631 		if (em->map_lookup->num_stripes !=
7632 		    em->map_lookup->verified_stripes) {
7633 			btrfs_err(fs_info,
7634 			"chunk %llu has missing dev extent, have %d expect %d",
7635 				  em->start, em->map_lookup->verified_stripes,
7636 				  em->map_lookup->num_stripes);
7637 			ret = -EUCLEAN;
7638 			goto out;
7639 		}
7640 	}
7641 out:
7642 	read_unlock(&em_tree->lock);
7643 	return ret;
7644 }
7645 
7646 /*
7647  * Ensure that all dev extents are mapped to correct chunk, otherwise
7648  * later chunk allocation/free would cause unexpected behavior.
7649  *
7650  * NOTE: This will iterate through the whole device tree, which should be of
7651  * the same size level as the chunk tree.  This slightly increases mount time.
7652  */
7653 int btrfs_verify_dev_extents(struct btrfs_fs_info *fs_info)
7654 {
7655 	struct btrfs_path *path;
7656 	struct btrfs_root *root = fs_info->dev_root;
7657 	struct btrfs_key key;
7658 	u64 prev_devid = 0;
7659 	u64 prev_dev_ext_end = 0;
7660 	int ret = 0;
7661 
7662 	key.objectid = 1;
7663 	key.type = BTRFS_DEV_EXTENT_KEY;
7664 	key.offset = 0;
7665 
7666 	path = btrfs_alloc_path();
7667 	if (!path)
7668 		return -ENOMEM;
7669 
7670 	path->reada = READA_FORWARD;
7671 	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
7672 	if (ret < 0)
7673 		goto out;
7674 
7675 	if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) {
7676 		ret = btrfs_next_item(root, path);
7677 		if (ret < 0)
7678 			goto out;
7679 		/* No dev extents at all? Not good */
7680 		if (ret > 0) {
7681 			ret = -EUCLEAN;
7682 			goto out;
7683 		}
7684 	}
7685 	while (1) {
7686 		struct extent_buffer *leaf = path->nodes[0];
7687 		struct btrfs_dev_extent *dext;
7688 		int slot = path->slots[0];
7689 		u64 chunk_offset;
7690 		u64 physical_offset;
7691 		u64 physical_len;
7692 		u64 devid;
7693 
7694 		btrfs_item_key_to_cpu(leaf, &key, slot);
7695 		if (key.type != BTRFS_DEV_EXTENT_KEY)
7696 			break;
7697 		devid = key.objectid;
7698 		physical_offset = key.offset;
7699 
7700 		dext = btrfs_item_ptr(leaf, slot, struct btrfs_dev_extent);
7701 		chunk_offset = btrfs_dev_extent_chunk_offset(leaf, dext);
7702 		physical_len = btrfs_dev_extent_length(leaf, dext);
7703 
7704 		/* Check if this dev extent overlaps with the previous one */
7705 		if (devid == prev_devid && physical_offset < prev_dev_ext_end) {
7706 			btrfs_err(fs_info,
7707 "dev extent devid %llu physical offset %llu overlap with previous dev extent end %llu",
7708 				  devid, physical_offset, prev_dev_ext_end);
7709 			ret = -EUCLEAN;
7710 			goto out;
7711 		}
7712 
7713 		ret = verify_one_dev_extent(fs_info, chunk_offset, devid,
7714 					    physical_offset, physical_len);
7715 		if (ret < 0)
7716 			goto out;
7717 		prev_devid = devid;
7718 		prev_dev_ext_end = physical_offset + physical_len;
7719 
7720 		ret = btrfs_next_item(root, path);
7721 		if (ret < 0)
7722 			goto out;
7723 		if (ret > 0) {
7724 			ret = 0;
7725 			break;
7726 		}
7727 	}
7728 
7729 	/* Ensure all chunks have corresponding dev extents */
7730 	ret = verify_chunk_dev_extent_mapping(fs_info);
7731 out:
7732 	btrfs_free_path(path);
7733 	return ret;
7734 }
7735 
7736 /*
7737  * Check whether the given block group or device is pinned by any inode being
7738  * used as a swapfile.
7739  */
7740 bool btrfs_pinned_by_swapfile(struct btrfs_fs_info *fs_info, void *ptr)
7741 {
7742 	struct btrfs_swapfile_pin *sp;
7743 	struct rb_node *node;
7744 
7745 	spin_lock(&fs_info->swapfile_pins_lock);
7746 	node = fs_info->swapfile_pins.rb_node;
7747 	while (node) {
7748 		sp = rb_entry(node, struct btrfs_swapfile_pin, node);
7749 		if (ptr < sp->ptr)
7750 			node = node->rb_left;
7751 		else if (ptr > sp->ptr)
7752 			node = node->rb_right;
7753 		else
7754 			break;
7755 	}
7756 	spin_unlock(&fs_info->swapfile_pins_lock);
7757 	return node != NULL;
7758 }
7759