xref: /openbmc/linux/fs/btrfs/volumes.c (revision 465191d6)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2007 Oracle.  All rights reserved.
4  */
5 
6 #include <linux/sched.h>
7 #include <linux/sched/mm.h>
8 #include <linux/bio.h>
9 #include <linux/slab.h>
10 #include <linux/blkdev.h>
11 #include <linux/ratelimit.h>
12 #include <linux/kthread.h>
13 #include <linux/raid/pq.h>
14 #include <linux/semaphore.h>
15 #include <linux/uuid.h>
16 #include <linux/list_sort.h>
17 #include <linux/namei.h>
18 #include "misc.h"
19 #include "ctree.h"
20 #include "extent_map.h"
21 #include "disk-io.h"
22 #include "transaction.h"
23 #include "print-tree.h"
24 #include "volumes.h"
25 #include "raid56.h"
26 #include "async-thread.h"
27 #include "check-integrity.h"
28 #include "rcu-string.h"
29 #include "dev-replace.h"
30 #include "sysfs.h"
31 #include "tree-checker.h"
32 #include "space-info.h"
33 #include "block-group.h"
34 #include "discard.h"
35 #include "zoned.h"
36 
37 #define BTRFS_BLOCK_GROUP_STRIPE_MASK	(BTRFS_BLOCK_GROUP_RAID0 | \
38 					 BTRFS_BLOCK_GROUP_RAID10 | \
39 					 BTRFS_BLOCK_GROUP_RAID56_MASK)
40 
41 const struct btrfs_raid_attr btrfs_raid_array[BTRFS_NR_RAID_TYPES] = {
42 	[BTRFS_RAID_RAID10] = {
43 		.sub_stripes	= 2,
44 		.dev_stripes	= 1,
45 		.devs_max	= 0,	/* 0 == as many as possible */
46 		.devs_min	= 2,
47 		.tolerated_failures = 1,
48 		.devs_increment	= 2,
49 		.ncopies	= 2,
50 		.nparity        = 0,
51 		.raid_name	= "raid10",
52 		.bg_flag	= BTRFS_BLOCK_GROUP_RAID10,
53 		.mindev_error	= BTRFS_ERROR_DEV_RAID10_MIN_NOT_MET,
54 	},
55 	[BTRFS_RAID_RAID1] = {
56 		.sub_stripes	= 1,
57 		.dev_stripes	= 1,
58 		.devs_max	= 2,
59 		.devs_min	= 2,
60 		.tolerated_failures = 1,
61 		.devs_increment	= 2,
62 		.ncopies	= 2,
63 		.nparity        = 0,
64 		.raid_name	= "raid1",
65 		.bg_flag	= BTRFS_BLOCK_GROUP_RAID1,
66 		.mindev_error	= BTRFS_ERROR_DEV_RAID1_MIN_NOT_MET,
67 	},
68 	[BTRFS_RAID_RAID1C3] = {
69 		.sub_stripes	= 1,
70 		.dev_stripes	= 1,
71 		.devs_max	= 3,
72 		.devs_min	= 3,
73 		.tolerated_failures = 2,
74 		.devs_increment	= 3,
75 		.ncopies	= 3,
76 		.nparity        = 0,
77 		.raid_name	= "raid1c3",
78 		.bg_flag	= BTRFS_BLOCK_GROUP_RAID1C3,
79 		.mindev_error	= BTRFS_ERROR_DEV_RAID1C3_MIN_NOT_MET,
80 	},
81 	[BTRFS_RAID_RAID1C4] = {
82 		.sub_stripes	= 1,
83 		.dev_stripes	= 1,
84 		.devs_max	= 4,
85 		.devs_min	= 4,
86 		.tolerated_failures = 3,
87 		.devs_increment	= 4,
88 		.ncopies	= 4,
89 		.nparity        = 0,
90 		.raid_name	= "raid1c4",
91 		.bg_flag	= BTRFS_BLOCK_GROUP_RAID1C4,
92 		.mindev_error	= BTRFS_ERROR_DEV_RAID1C4_MIN_NOT_MET,
93 	},
94 	[BTRFS_RAID_DUP] = {
95 		.sub_stripes	= 1,
96 		.dev_stripes	= 2,
97 		.devs_max	= 1,
98 		.devs_min	= 1,
99 		.tolerated_failures = 0,
100 		.devs_increment	= 1,
101 		.ncopies	= 2,
102 		.nparity        = 0,
103 		.raid_name	= "dup",
104 		.bg_flag	= BTRFS_BLOCK_GROUP_DUP,
105 		.mindev_error	= 0,
106 	},
107 	[BTRFS_RAID_RAID0] = {
108 		.sub_stripes	= 1,
109 		.dev_stripes	= 1,
110 		.devs_max	= 0,
111 		.devs_min	= 1,
112 		.tolerated_failures = 0,
113 		.devs_increment	= 1,
114 		.ncopies	= 1,
115 		.nparity        = 0,
116 		.raid_name	= "raid0",
117 		.bg_flag	= BTRFS_BLOCK_GROUP_RAID0,
118 		.mindev_error	= 0,
119 	},
120 	[BTRFS_RAID_SINGLE] = {
121 		.sub_stripes	= 1,
122 		.dev_stripes	= 1,
123 		.devs_max	= 1,
124 		.devs_min	= 1,
125 		.tolerated_failures = 0,
126 		.devs_increment	= 1,
127 		.ncopies	= 1,
128 		.nparity        = 0,
129 		.raid_name	= "single",
130 		.bg_flag	= 0,
131 		.mindev_error	= 0,
132 	},
133 	[BTRFS_RAID_RAID5] = {
134 		.sub_stripes	= 1,
135 		.dev_stripes	= 1,
136 		.devs_max	= 0,
137 		.devs_min	= 2,
138 		.tolerated_failures = 1,
139 		.devs_increment	= 1,
140 		.ncopies	= 1,
141 		.nparity        = 1,
142 		.raid_name	= "raid5",
143 		.bg_flag	= BTRFS_BLOCK_GROUP_RAID5,
144 		.mindev_error	= BTRFS_ERROR_DEV_RAID5_MIN_NOT_MET,
145 	},
146 	[BTRFS_RAID_RAID6] = {
147 		.sub_stripes	= 1,
148 		.dev_stripes	= 1,
149 		.devs_max	= 0,
150 		.devs_min	= 3,
151 		.tolerated_failures = 2,
152 		.devs_increment	= 1,
153 		.ncopies	= 1,
154 		.nparity        = 2,
155 		.raid_name	= "raid6",
156 		.bg_flag	= BTRFS_BLOCK_GROUP_RAID6,
157 		.mindev_error	= BTRFS_ERROR_DEV_RAID6_MIN_NOT_MET,
158 	},
159 };
160 
161 /*
162  * Convert block group flags (BTRFS_BLOCK_GROUP_*) to btrfs_raid_types, which
163  * can be used as index to access btrfs_raid_array[].
164  */
165 enum btrfs_raid_types __attribute_const__ btrfs_bg_flags_to_raid_index(u64 flags)
166 {
167 	const u64 profile = (flags & BTRFS_BLOCK_GROUP_PROFILE_MASK);
168 
169 	if (!profile)
170 		return BTRFS_RAID_SINGLE;
171 
172 	return BTRFS_BG_FLAG_TO_INDEX(profile);
173 }
174 
175 const char *btrfs_bg_type_to_raid_name(u64 flags)
176 {
177 	const int index = btrfs_bg_flags_to_raid_index(flags);
178 
179 	if (index >= BTRFS_NR_RAID_TYPES)
180 		return NULL;
181 
182 	return btrfs_raid_array[index].raid_name;
183 }
184 
185 /*
186  * Fill @buf with textual description of @bg_flags, no more than @size_buf
187  * bytes including terminating null byte.
188  */
189 void btrfs_describe_block_groups(u64 bg_flags, char *buf, u32 size_buf)
190 {
191 	int i;
192 	int ret;
193 	char *bp = buf;
194 	u64 flags = bg_flags;
195 	u32 size_bp = size_buf;
196 
197 	if (!flags) {
198 		strcpy(bp, "NONE");
199 		return;
200 	}
201 
202 #define DESCRIBE_FLAG(flag, desc)						\
203 	do {								\
204 		if (flags & (flag)) {					\
205 			ret = snprintf(bp, size_bp, "%s|", (desc));	\
206 			if (ret < 0 || ret >= size_bp)			\
207 				goto out_overflow;			\
208 			size_bp -= ret;					\
209 			bp += ret;					\
210 			flags &= ~(flag);				\
211 		}							\
212 	} while (0)
213 
214 	DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_DATA, "data");
215 	DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_SYSTEM, "system");
216 	DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_METADATA, "metadata");
217 
218 	DESCRIBE_FLAG(BTRFS_AVAIL_ALLOC_BIT_SINGLE, "single");
219 	for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
220 		DESCRIBE_FLAG(btrfs_raid_array[i].bg_flag,
221 			      btrfs_raid_array[i].raid_name);
222 #undef DESCRIBE_FLAG
223 
224 	if (flags) {
225 		ret = snprintf(bp, size_bp, "0x%llx|", flags);
226 		size_bp -= ret;
227 	}
228 
229 	if (size_bp < size_buf)
230 		buf[size_buf - size_bp - 1] = '\0'; /* remove last | */
231 
232 	/*
233 	 * The text is trimmed, it's up to the caller to provide sufficiently
234 	 * large buffer
235 	 */
236 out_overflow:;
237 }
238 
239 static int init_first_rw_device(struct btrfs_trans_handle *trans);
240 static int btrfs_relocate_sys_chunks(struct btrfs_fs_info *fs_info);
241 static void btrfs_dev_stat_print_on_error(struct btrfs_device *dev);
242 static void btrfs_dev_stat_print_on_load(struct btrfs_device *device);
243 static int __btrfs_map_block(struct btrfs_fs_info *fs_info,
244 			     enum btrfs_map_op op,
245 			     u64 logical, u64 *length,
246 			     struct btrfs_io_context **bioc_ret,
247 			     int mirror_num, int need_raid_map);
248 
249 /*
250  * Device locking
251  * ==============
252  *
253  * There are several mutexes that protect manipulation of devices and low-level
254  * structures like chunks but not block groups, extents or files
255  *
256  * uuid_mutex (global lock)
257  * ------------------------
258  * protects the fs_uuids list that tracks all per-fs fs_devices, resulting from
259  * the SCAN_DEV ioctl registration or from mount either implicitly (the first
260  * device) or requested by the device= mount option
261  *
262  * the mutex can be very coarse and can cover long-running operations
263  *
264  * protects: updates to fs_devices counters like missing devices, rw devices,
265  * seeding, structure cloning, opening/closing devices at mount/umount time
266  *
267  * global::fs_devs - add, remove, updates to the global list
268  *
269  * does not protect: manipulation of the fs_devices::devices list in general
270  * but in mount context it could be used to exclude list modifications by eg.
271  * scan ioctl
272  *
273  * btrfs_device::name - renames (write side), read is RCU
274  *
275  * fs_devices::device_list_mutex (per-fs, with RCU)
276  * ------------------------------------------------
277  * protects updates to fs_devices::devices, ie. adding and deleting
278  *
279  * simple list traversal with read-only actions can be done with RCU protection
280  *
281  * may be used to exclude some operations from running concurrently without any
282  * modifications to the list (see write_all_supers)
283  *
284  * Is not required at mount and close times, because our device list is
285  * protected by the uuid_mutex at that point.
286  *
287  * balance_mutex
288  * -------------
289  * protects balance structures (status, state) and context accessed from
290  * several places (internally, ioctl)
291  *
292  * chunk_mutex
293  * -----------
294  * protects chunks, adding or removing during allocation, trim or when a new
295  * device is added/removed. Additionally it also protects post_commit_list of
296  * individual devices, since they can be added to the transaction's
297  * post_commit_list only with chunk_mutex held.
298  *
299  * cleaner_mutex
300  * -------------
301  * a big lock that is held by the cleaner thread and prevents running subvolume
302  * cleaning together with relocation or delayed iputs
303  *
304  *
305  * Lock nesting
306  * ============
307  *
308  * uuid_mutex
309  *   device_list_mutex
310  *     chunk_mutex
311  *   balance_mutex
312  *
313  *
314  * Exclusive operations
315  * ====================
316  *
317  * Maintains the exclusivity of the following operations that apply to the
318  * whole filesystem and cannot run in parallel.
319  *
320  * - Balance (*)
321  * - Device add
322  * - Device remove
323  * - Device replace (*)
324  * - Resize
325  *
326  * The device operations (as above) can be in one of the following states:
327  *
328  * - Running state
329  * - Paused state
330  * - Completed state
331  *
332  * Only device operations marked with (*) can go into the Paused state for the
333  * following reasons:
334  *
335  * - ioctl (only Balance can be Paused through ioctl)
336  * - filesystem remounted as read-only
337  * - filesystem unmounted and mounted as read-only
338  * - system power-cycle and filesystem mounted as read-only
339  * - filesystem or device errors leading to forced read-only
340  *
341  * The status of exclusive operation is set and cleared atomically.
342  * During the course of Paused state, fs_info::exclusive_operation remains set.
343  * A device operation in Paused or Running state can be canceled or resumed
344  * either by ioctl (Balance only) or when remounted as read-write.
345  * The exclusive status is cleared when the device operation is canceled or
346  * completed.
347  */
348 
349 DEFINE_MUTEX(uuid_mutex);
350 static LIST_HEAD(fs_uuids);
351 struct list_head * __attribute_const__ btrfs_get_fs_uuids(void)
352 {
353 	return &fs_uuids;
354 }
355 
356 /*
357  * alloc_fs_devices - allocate struct btrfs_fs_devices
358  * @fsid:		if not NULL, copy the UUID to fs_devices::fsid
359  * @metadata_fsid:	if not NULL, copy the UUID to fs_devices::metadata_fsid
360  *
361  * Return a pointer to a new struct btrfs_fs_devices on success, or ERR_PTR().
362  * The returned struct is not linked onto any lists and can be destroyed with
363  * kfree() right away.
364  */
365 static struct btrfs_fs_devices *alloc_fs_devices(const u8 *fsid,
366 						 const u8 *metadata_fsid)
367 {
368 	struct btrfs_fs_devices *fs_devs;
369 
370 	fs_devs = kzalloc(sizeof(*fs_devs), GFP_KERNEL);
371 	if (!fs_devs)
372 		return ERR_PTR(-ENOMEM);
373 
374 	mutex_init(&fs_devs->device_list_mutex);
375 
376 	INIT_LIST_HEAD(&fs_devs->devices);
377 	INIT_LIST_HEAD(&fs_devs->alloc_list);
378 	INIT_LIST_HEAD(&fs_devs->fs_list);
379 	INIT_LIST_HEAD(&fs_devs->seed_list);
380 	if (fsid)
381 		memcpy(fs_devs->fsid, fsid, BTRFS_FSID_SIZE);
382 
383 	if (metadata_fsid)
384 		memcpy(fs_devs->metadata_uuid, metadata_fsid, BTRFS_FSID_SIZE);
385 	else if (fsid)
386 		memcpy(fs_devs->metadata_uuid, fsid, BTRFS_FSID_SIZE);
387 
388 	return fs_devs;
389 }
390 
391 void btrfs_free_device(struct btrfs_device *device)
392 {
393 	WARN_ON(!list_empty(&device->post_commit_list));
394 	rcu_string_free(device->name);
395 	extent_io_tree_release(&device->alloc_state);
396 	btrfs_destroy_dev_zone_info(device);
397 	kfree(device);
398 }
399 
400 static void free_fs_devices(struct btrfs_fs_devices *fs_devices)
401 {
402 	struct btrfs_device *device;
403 	WARN_ON(fs_devices->opened);
404 	while (!list_empty(&fs_devices->devices)) {
405 		device = list_entry(fs_devices->devices.next,
406 				    struct btrfs_device, dev_list);
407 		list_del(&device->dev_list);
408 		btrfs_free_device(device);
409 	}
410 	kfree(fs_devices);
411 }
412 
413 void __exit btrfs_cleanup_fs_uuids(void)
414 {
415 	struct btrfs_fs_devices *fs_devices;
416 
417 	while (!list_empty(&fs_uuids)) {
418 		fs_devices = list_entry(fs_uuids.next,
419 					struct btrfs_fs_devices, fs_list);
420 		list_del(&fs_devices->fs_list);
421 		free_fs_devices(fs_devices);
422 	}
423 }
424 
425 static noinline struct btrfs_fs_devices *find_fsid(
426 		const u8 *fsid, const u8 *metadata_fsid)
427 {
428 	struct btrfs_fs_devices *fs_devices;
429 
430 	ASSERT(fsid);
431 
432 	/* Handle non-split brain cases */
433 	list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
434 		if (metadata_fsid) {
435 			if (memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE) == 0
436 			    && memcmp(metadata_fsid, fs_devices->metadata_uuid,
437 				      BTRFS_FSID_SIZE) == 0)
438 				return fs_devices;
439 		} else {
440 			if (memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE) == 0)
441 				return fs_devices;
442 		}
443 	}
444 	return NULL;
445 }
446 
447 static struct btrfs_fs_devices *find_fsid_with_metadata_uuid(
448 				struct btrfs_super_block *disk_super)
449 {
450 
451 	struct btrfs_fs_devices *fs_devices;
452 
453 	/*
454 	 * Handle scanned device having completed its fsid change but
455 	 * belonging to a fs_devices that was created by first scanning
456 	 * a device which didn't have its fsid/metadata_uuid changed
457 	 * at all and the CHANGING_FSID_V2 flag set.
458 	 */
459 	list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
460 		if (fs_devices->fsid_change &&
461 		    memcmp(disk_super->metadata_uuid, fs_devices->fsid,
462 			   BTRFS_FSID_SIZE) == 0 &&
463 		    memcmp(fs_devices->fsid, fs_devices->metadata_uuid,
464 			   BTRFS_FSID_SIZE) == 0) {
465 			return fs_devices;
466 		}
467 	}
468 	/*
469 	 * Handle scanned device having completed its fsid change but
470 	 * belonging to a fs_devices that was created by a device that
471 	 * has an outdated pair of fsid/metadata_uuid and
472 	 * CHANGING_FSID_V2 flag set.
473 	 */
474 	list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
475 		if (fs_devices->fsid_change &&
476 		    memcmp(fs_devices->metadata_uuid,
477 			   fs_devices->fsid, BTRFS_FSID_SIZE) != 0 &&
478 		    memcmp(disk_super->metadata_uuid, fs_devices->metadata_uuid,
479 			   BTRFS_FSID_SIZE) == 0) {
480 			return fs_devices;
481 		}
482 	}
483 
484 	return find_fsid(disk_super->fsid, disk_super->metadata_uuid);
485 }
486 
487 
488 static int
489 btrfs_get_bdev_and_sb(const char *device_path, fmode_t flags, void *holder,
490 		      int flush, struct block_device **bdev,
491 		      struct btrfs_super_block **disk_super)
492 {
493 	int ret;
494 
495 	*bdev = blkdev_get_by_path(device_path, flags, holder);
496 
497 	if (IS_ERR(*bdev)) {
498 		ret = PTR_ERR(*bdev);
499 		goto error;
500 	}
501 
502 	if (flush)
503 		sync_blockdev(*bdev);
504 	ret = set_blocksize(*bdev, BTRFS_BDEV_BLOCKSIZE);
505 	if (ret) {
506 		blkdev_put(*bdev, flags);
507 		goto error;
508 	}
509 	invalidate_bdev(*bdev);
510 	*disk_super = btrfs_read_dev_super(*bdev);
511 	if (IS_ERR(*disk_super)) {
512 		ret = PTR_ERR(*disk_super);
513 		blkdev_put(*bdev, flags);
514 		goto error;
515 	}
516 
517 	return 0;
518 
519 error:
520 	*bdev = NULL;
521 	return ret;
522 }
523 
524 /**
525  *  Search and remove all stale devices (which are not mounted).
526  *  When both inputs are NULL, it will search and release all stale devices.
527  *
528  *  @devt:	Optional. When provided will it release all unmounted devices
529  *		matching this devt only.
530  *  @skip_device:  Optional. Will skip this device when searching for the stale
531  *		devices.
532  *
533  *  Return:	0 for success or if @devt is 0.
534  *		-EBUSY if @devt is a mounted device.
535  *		-ENOENT if @devt does not match any device in the list.
536  */
537 static int btrfs_free_stale_devices(dev_t devt, struct btrfs_device *skip_device)
538 {
539 	struct btrfs_fs_devices *fs_devices, *tmp_fs_devices;
540 	struct btrfs_device *device, *tmp_device;
541 	int ret = 0;
542 
543 	lockdep_assert_held(&uuid_mutex);
544 
545 	if (devt)
546 		ret = -ENOENT;
547 
548 	list_for_each_entry_safe(fs_devices, tmp_fs_devices, &fs_uuids, fs_list) {
549 
550 		mutex_lock(&fs_devices->device_list_mutex);
551 		list_for_each_entry_safe(device, tmp_device,
552 					 &fs_devices->devices, dev_list) {
553 			if (skip_device && skip_device == device)
554 				continue;
555 			if (devt && devt != device->devt)
556 				continue;
557 			if (fs_devices->opened) {
558 				/* for an already deleted device return 0 */
559 				if (devt && ret != 0)
560 					ret = -EBUSY;
561 				break;
562 			}
563 
564 			/* delete the stale device */
565 			fs_devices->num_devices--;
566 			list_del(&device->dev_list);
567 			btrfs_free_device(device);
568 
569 			ret = 0;
570 		}
571 		mutex_unlock(&fs_devices->device_list_mutex);
572 
573 		if (fs_devices->num_devices == 0) {
574 			btrfs_sysfs_remove_fsid(fs_devices);
575 			list_del(&fs_devices->fs_list);
576 			free_fs_devices(fs_devices);
577 		}
578 	}
579 
580 	return ret;
581 }
582 
583 /*
584  * This is only used on mount, and we are protected from competing things
585  * messing with our fs_devices by the uuid_mutex, thus we do not need the
586  * fs_devices->device_list_mutex here.
587  */
588 static int btrfs_open_one_device(struct btrfs_fs_devices *fs_devices,
589 			struct btrfs_device *device, fmode_t flags,
590 			void *holder)
591 {
592 	struct block_device *bdev;
593 	struct btrfs_super_block *disk_super;
594 	u64 devid;
595 	int ret;
596 
597 	if (device->bdev)
598 		return -EINVAL;
599 	if (!device->name)
600 		return -EINVAL;
601 
602 	ret = btrfs_get_bdev_and_sb(device->name->str, flags, holder, 1,
603 				    &bdev, &disk_super);
604 	if (ret)
605 		return ret;
606 
607 	devid = btrfs_stack_device_id(&disk_super->dev_item);
608 	if (devid != device->devid)
609 		goto error_free_page;
610 
611 	if (memcmp(device->uuid, disk_super->dev_item.uuid, BTRFS_UUID_SIZE))
612 		goto error_free_page;
613 
614 	device->generation = btrfs_super_generation(disk_super);
615 
616 	if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_SEEDING) {
617 		if (btrfs_super_incompat_flags(disk_super) &
618 		    BTRFS_FEATURE_INCOMPAT_METADATA_UUID) {
619 			pr_err(
620 		"BTRFS: Invalid seeding and uuid-changed device detected\n");
621 			goto error_free_page;
622 		}
623 
624 		clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
625 		fs_devices->seeding = true;
626 	} else {
627 		if (bdev_read_only(bdev))
628 			clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
629 		else
630 			set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
631 	}
632 
633 	if (!bdev_nonrot(bdev))
634 		fs_devices->rotating = true;
635 
636 	device->bdev = bdev;
637 	clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
638 	device->mode = flags;
639 
640 	fs_devices->open_devices++;
641 	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
642 	    device->devid != BTRFS_DEV_REPLACE_DEVID) {
643 		fs_devices->rw_devices++;
644 		list_add_tail(&device->dev_alloc_list, &fs_devices->alloc_list);
645 	}
646 	btrfs_release_disk_super(disk_super);
647 
648 	return 0;
649 
650 error_free_page:
651 	btrfs_release_disk_super(disk_super);
652 	blkdev_put(bdev, flags);
653 
654 	return -EINVAL;
655 }
656 
657 /*
658  * Handle scanned device having its CHANGING_FSID_V2 flag set and the fs_devices
659  * being created with a disk that has already completed its fsid change. Such
660  * disk can belong to an fs which has its FSID changed or to one which doesn't.
661  * Handle both cases here.
662  */
663 static struct btrfs_fs_devices *find_fsid_inprogress(
664 					struct btrfs_super_block *disk_super)
665 {
666 	struct btrfs_fs_devices *fs_devices;
667 
668 	list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
669 		if (memcmp(fs_devices->metadata_uuid, fs_devices->fsid,
670 			   BTRFS_FSID_SIZE) != 0 &&
671 		    memcmp(fs_devices->metadata_uuid, disk_super->fsid,
672 			   BTRFS_FSID_SIZE) == 0 && !fs_devices->fsid_change) {
673 			return fs_devices;
674 		}
675 	}
676 
677 	return find_fsid(disk_super->fsid, NULL);
678 }
679 
680 
681 static struct btrfs_fs_devices *find_fsid_changed(
682 					struct btrfs_super_block *disk_super)
683 {
684 	struct btrfs_fs_devices *fs_devices;
685 
686 	/*
687 	 * Handles the case where scanned device is part of an fs that had
688 	 * multiple successful changes of FSID but currently device didn't
689 	 * observe it. Meaning our fsid will be different than theirs. We need
690 	 * to handle two subcases :
691 	 *  1 - The fs still continues to have different METADATA/FSID uuids.
692 	 *  2 - The fs is switched back to its original FSID (METADATA/FSID
693 	 *  are equal).
694 	 */
695 	list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
696 		/* Changed UUIDs */
697 		if (memcmp(fs_devices->metadata_uuid, fs_devices->fsid,
698 			   BTRFS_FSID_SIZE) != 0 &&
699 		    memcmp(fs_devices->metadata_uuid, disk_super->metadata_uuid,
700 			   BTRFS_FSID_SIZE) == 0 &&
701 		    memcmp(fs_devices->fsid, disk_super->fsid,
702 			   BTRFS_FSID_SIZE) != 0)
703 			return fs_devices;
704 
705 		/* Unchanged UUIDs */
706 		if (memcmp(fs_devices->metadata_uuid, fs_devices->fsid,
707 			   BTRFS_FSID_SIZE) == 0 &&
708 		    memcmp(fs_devices->fsid, disk_super->metadata_uuid,
709 			   BTRFS_FSID_SIZE) == 0)
710 			return fs_devices;
711 	}
712 
713 	return NULL;
714 }
715 
716 static struct btrfs_fs_devices *find_fsid_reverted_metadata(
717 				struct btrfs_super_block *disk_super)
718 {
719 	struct btrfs_fs_devices *fs_devices;
720 
721 	/*
722 	 * Handle the case where the scanned device is part of an fs whose last
723 	 * metadata UUID change reverted it to the original FSID. At the same
724 	 * time * fs_devices was first created by another constitutent device
725 	 * which didn't fully observe the operation. This results in an
726 	 * btrfs_fs_devices created with metadata/fsid different AND
727 	 * btrfs_fs_devices::fsid_change set AND the metadata_uuid of the
728 	 * fs_devices equal to the FSID of the disk.
729 	 */
730 	list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
731 		if (memcmp(fs_devices->fsid, fs_devices->metadata_uuid,
732 			   BTRFS_FSID_SIZE) != 0 &&
733 		    memcmp(fs_devices->metadata_uuid, disk_super->fsid,
734 			   BTRFS_FSID_SIZE) == 0 &&
735 		    fs_devices->fsid_change)
736 			return fs_devices;
737 	}
738 
739 	return NULL;
740 }
741 /*
742  * Add new device to list of registered devices
743  *
744  * Returns:
745  * device pointer which was just added or updated when successful
746  * error pointer when failed
747  */
748 static noinline struct btrfs_device *device_list_add(const char *path,
749 			   struct btrfs_super_block *disk_super,
750 			   bool *new_device_added)
751 {
752 	struct btrfs_device *device;
753 	struct btrfs_fs_devices *fs_devices = NULL;
754 	struct rcu_string *name;
755 	u64 found_transid = btrfs_super_generation(disk_super);
756 	u64 devid = btrfs_stack_device_id(&disk_super->dev_item);
757 	dev_t path_devt;
758 	int error;
759 	bool has_metadata_uuid = (btrfs_super_incompat_flags(disk_super) &
760 		BTRFS_FEATURE_INCOMPAT_METADATA_UUID);
761 	bool fsid_change_in_progress = (btrfs_super_flags(disk_super) &
762 					BTRFS_SUPER_FLAG_CHANGING_FSID_V2);
763 
764 	error = lookup_bdev(path, &path_devt);
765 	if (error)
766 		return ERR_PTR(error);
767 
768 	if (fsid_change_in_progress) {
769 		if (!has_metadata_uuid)
770 			fs_devices = find_fsid_inprogress(disk_super);
771 		else
772 			fs_devices = find_fsid_changed(disk_super);
773 	} else if (has_metadata_uuid) {
774 		fs_devices = find_fsid_with_metadata_uuid(disk_super);
775 	} else {
776 		fs_devices = find_fsid_reverted_metadata(disk_super);
777 		if (!fs_devices)
778 			fs_devices = find_fsid(disk_super->fsid, NULL);
779 	}
780 
781 
782 	if (!fs_devices) {
783 		if (has_metadata_uuid)
784 			fs_devices = alloc_fs_devices(disk_super->fsid,
785 						      disk_super->metadata_uuid);
786 		else
787 			fs_devices = alloc_fs_devices(disk_super->fsid, NULL);
788 
789 		if (IS_ERR(fs_devices))
790 			return ERR_CAST(fs_devices);
791 
792 		fs_devices->fsid_change = fsid_change_in_progress;
793 
794 		mutex_lock(&fs_devices->device_list_mutex);
795 		list_add(&fs_devices->fs_list, &fs_uuids);
796 
797 		device = NULL;
798 	} else {
799 		struct btrfs_dev_lookup_args args = {
800 			.devid = devid,
801 			.uuid = disk_super->dev_item.uuid,
802 		};
803 
804 		mutex_lock(&fs_devices->device_list_mutex);
805 		device = btrfs_find_device(fs_devices, &args);
806 
807 		/*
808 		 * If this disk has been pulled into an fs devices created by
809 		 * a device which had the CHANGING_FSID_V2 flag then replace the
810 		 * metadata_uuid/fsid values of the fs_devices.
811 		 */
812 		if (fs_devices->fsid_change &&
813 		    found_transid > fs_devices->latest_generation) {
814 			memcpy(fs_devices->fsid, disk_super->fsid,
815 					BTRFS_FSID_SIZE);
816 
817 			if (has_metadata_uuid)
818 				memcpy(fs_devices->metadata_uuid,
819 				       disk_super->metadata_uuid,
820 				       BTRFS_FSID_SIZE);
821 			else
822 				memcpy(fs_devices->metadata_uuid,
823 				       disk_super->fsid, BTRFS_FSID_SIZE);
824 
825 			fs_devices->fsid_change = false;
826 		}
827 	}
828 
829 	if (!device) {
830 		if (fs_devices->opened) {
831 			mutex_unlock(&fs_devices->device_list_mutex);
832 			return ERR_PTR(-EBUSY);
833 		}
834 
835 		device = btrfs_alloc_device(NULL, &devid,
836 					    disk_super->dev_item.uuid);
837 		if (IS_ERR(device)) {
838 			mutex_unlock(&fs_devices->device_list_mutex);
839 			/* we can safely leave the fs_devices entry around */
840 			return device;
841 		}
842 
843 		name = rcu_string_strdup(path, GFP_NOFS);
844 		if (!name) {
845 			btrfs_free_device(device);
846 			mutex_unlock(&fs_devices->device_list_mutex);
847 			return ERR_PTR(-ENOMEM);
848 		}
849 		rcu_assign_pointer(device->name, name);
850 		device->devt = path_devt;
851 
852 		list_add_rcu(&device->dev_list, &fs_devices->devices);
853 		fs_devices->num_devices++;
854 
855 		device->fs_devices = fs_devices;
856 		*new_device_added = true;
857 
858 		if (disk_super->label[0])
859 			pr_info(
860 	"BTRFS: device label %s devid %llu transid %llu %s scanned by %s (%d)\n",
861 				disk_super->label, devid, found_transid, path,
862 				current->comm, task_pid_nr(current));
863 		else
864 			pr_info(
865 	"BTRFS: device fsid %pU devid %llu transid %llu %s scanned by %s (%d)\n",
866 				disk_super->fsid, devid, found_transid, path,
867 				current->comm, task_pid_nr(current));
868 
869 	} else if (!device->name || strcmp(device->name->str, path)) {
870 		/*
871 		 * When FS is already mounted.
872 		 * 1. If you are here and if the device->name is NULL that
873 		 *    means this device was missing at time of FS mount.
874 		 * 2. If you are here and if the device->name is different
875 		 *    from 'path' that means either
876 		 *      a. The same device disappeared and reappeared with
877 		 *         different name. or
878 		 *      b. The missing-disk-which-was-replaced, has
879 		 *         reappeared now.
880 		 *
881 		 * We must allow 1 and 2a above. But 2b would be a spurious
882 		 * and unintentional.
883 		 *
884 		 * Further in case of 1 and 2a above, the disk at 'path'
885 		 * would have missed some transaction when it was away and
886 		 * in case of 2a the stale bdev has to be updated as well.
887 		 * 2b must not be allowed at all time.
888 		 */
889 
890 		/*
891 		 * For now, we do allow update to btrfs_fs_device through the
892 		 * btrfs dev scan cli after FS has been mounted.  We're still
893 		 * tracking a problem where systems fail mount by subvolume id
894 		 * when we reject replacement on a mounted FS.
895 		 */
896 		if (!fs_devices->opened && found_transid < device->generation) {
897 			/*
898 			 * That is if the FS is _not_ mounted and if you
899 			 * are here, that means there is more than one
900 			 * disk with same uuid and devid.We keep the one
901 			 * with larger generation number or the last-in if
902 			 * generation are equal.
903 			 */
904 			mutex_unlock(&fs_devices->device_list_mutex);
905 			return ERR_PTR(-EEXIST);
906 		}
907 
908 		/*
909 		 * We are going to replace the device path for a given devid,
910 		 * make sure it's the same device if the device is mounted
911 		 *
912 		 * NOTE: the device->fs_info may not be reliable here so pass
913 		 * in a NULL to message helpers instead. This avoids a possible
914 		 * use-after-free when the fs_info and fs_info->sb are already
915 		 * torn down.
916 		 */
917 		if (device->bdev) {
918 			if (device->devt != path_devt) {
919 				mutex_unlock(&fs_devices->device_list_mutex);
920 				btrfs_warn_in_rcu(NULL,
921 	"duplicate device %s devid %llu generation %llu scanned by %s (%d)",
922 						  path, devid, found_transid,
923 						  current->comm,
924 						  task_pid_nr(current));
925 				return ERR_PTR(-EEXIST);
926 			}
927 			btrfs_info_in_rcu(NULL,
928 	"devid %llu device path %s changed to %s scanned by %s (%d)",
929 					  devid, rcu_str_deref(device->name),
930 					  path, current->comm,
931 					  task_pid_nr(current));
932 		}
933 
934 		name = rcu_string_strdup(path, GFP_NOFS);
935 		if (!name) {
936 			mutex_unlock(&fs_devices->device_list_mutex);
937 			return ERR_PTR(-ENOMEM);
938 		}
939 		rcu_string_free(device->name);
940 		rcu_assign_pointer(device->name, name);
941 		if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) {
942 			fs_devices->missing_devices--;
943 			clear_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state);
944 		}
945 		device->devt = path_devt;
946 	}
947 
948 	/*
949 	 * Unmount does not free the btrfs_device struct but would zero
950 	 * generation along with most of the other members. So just update
951 	 * it back. We need it to pick the disk with largest generation
952 	 * (as above).
953 	 */
954 	if (!fs_devices->opened) {
955 		device->generation = found_transid;
956 		fs_devices->latest_generation = max_t(u64, found_transid,
957 						fs_devices->latest_generation);
958 	}
959 
960 	fs_devices->total_devices = btrfs_super_num_devices(disk_super);
961 
962 	mutex_unlock(&fs_devices->device_list_mutex);
963 	return device;
964 }
965 
966 static struct btrfs_fs_devices *clone_fs_devices(struct btrfs_fs_devices *orig)
967 {
968 	struct btrfs_fs_devices *fs_devices;
969 	struct btrfs_device *device;
970 	struct btrfs_device *orig_dev;
971 	int ret = 0;
972 
973 	lockdep_assert_held(&uuid_mutex);
974 
975 	fs_devices = alloc_fs_devices(orig->fsid, NULL);
976 	if (IS_ERR(fs_devices))
977 		return fs_devices;
978 
979 	fs_devices->total_devices = orig->total_devices;
980 
981 	list_for_each_entry(orig_dev, &orig->devices, dev_list) {
982 		struct rcu_string *name;
983 
984 		device = btrfs_alloc_device(NULL, &orig_dev->devid,
985 					    orig_dev->uuid);
986 		if (IS_ERR(device)) {
987 			ret = PTR_ERR(device);
988 			goto error;
989 		}
990 
991 		/*
992 		 * This is ok to do without rcu read locked because we hold the
993 		 * uuid mutex so nothing we touch in here is going to disappear.
994 		 */
995 		if (orig_dev->name) {
996 			name = rcu_string_strdup(orig_dev->name->str,
997 					GFP_KERNEL);
998 			if (!name) {
999 				btrfs_free_device(device);
1000 				ret = -ENOMEM;
1001 				goto error;
1002 			}
1003 			rcu_assign_pointer(device->name, name);
1004 		}
1005 
1006 		list_add(&device->dev_list, &fs_devices->devices);
1007 		device->fs_devices = fs_devices;
1008 		fs_devices->num_devices++;
1009 	}
1010 	return fs_devices;
1011 error:
1012 	free_fs_devices(fs_devices);
1013 	return ERR_PTR(ret);
1014 }
1015 
1016 static void __btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices,
1017 				      struct btrfs_device **latest_dev)
1018 {
1019 	struct btrfs_device *device, *next;
1020 
1021 	/* This is the initialized path, it is safe to release the devices. */
1022 	list_for_each_entry_safe(device, next, &fs_devices->devices, dev_list) {
1023 		if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state)) {
1024 			if (!test_bit(BTRFS_DEV_STATE_REPLACE_TGT,
1025 				      &device->dev_state) &&
1026 			    !test_bit(BTRFS_DEV_STATE_MISSING,
1027 				      &device->dev_state) &&
1028 			    (!*latest_dev ||
1029 			     device->generation > (*latest_dev)->generation)) {
1030 				*latest_dev = device;
1031 			}
1032 			continue;
1033 		}
1034 
1035 		/*
1036 		 * We have already validated the presence of BTRFS_DEV_REPLACE_DEVID,
1037 		 * in btrfs_init_dev_replace() so just continue.
1038 		 */
1039 		if (device->devid == BTRFS_DEV_REPLACE_DEVID)
1040 			continue;
1041 
1042 		if (device->bdev) {
1043 			blkdev_put(device->bdev, device->mode);
1044 			device->bdev = NULL;
1045 			fs_devices->open_devices--;
1046 		}
1047 		if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
1048 			list_del_init(&device->dev_alloc_list);
1049 			clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
1050 			fs_devices->rw_devices--;
1051 		}
1052 		list_del_init(&device->dev_list);
1053 		fs_devices->num_devices--;
1054 		btrfs_free_device(device);
1055 	}
1056 
1057 }
1058 
1059 /*
1060  * After we have read the system tree and know devids belonging to this
1061  * filesystem, remove the device which does not belong there.
1062  */
1063 void btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices)
1064 {
1065 	struct btrfs_device *latest_dev = NULL;
1066 	struct btrfs_fs_devices *seed_dev;
1067 
1068 	mutex_lock(&uuid_mutex);
1069 	__btrfs_free_extra_devids(fs_devices, &latest_dev);
1070 
1071 	list_for_each_entry(seed_dev, &fs_devices->seed_list, seed_list)
1072 		__btrfs_free_extra_devids(seed_dev, &latest_dev);
1073 
1074 	fs_devices->latest_dev = latest_dev;
1075 
1076 	mutex_unlock(&uuid_mutex);
1077 }
1078 
1079 static void btrfs_close_bdev(struct btrfs_device *device)
1080 {
1081 	if (!device->bdev)
1082 		return;
1083 
1084 	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
1085 		sync_blockdev(device->bdev);
1086 		invalidate_bdev(device->bdev);
1087 	}
1088 
1089 	blkdev_put(device->bdev, device->mode);
1090 }
1091 
1092 static void btrfs_close_one_device(struct btrfs_device *device)
1093 {
1094 	struct btrfs_fs_devices *fs_devices = device->fs_devices;
1095 
1096 	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
1097 	    device->devid != BTRFS_DEV_REPLACE_DEVID) {
1098 		list_del_init(&device->dev_alloc_list);
1099 		fs_devices->rw_devices--;
1100 	}
1101 
1102 	if (device->devid == BTRFS_DEV_REPLACE_DEVID)
1103 		clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state);
1104 
1105 	if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) {
1106 		clear_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state);
1107 		fs_devices->missing_devices--;
1108 	}
1109 
1110 	btrfs_close_bdev(device);
1111 	if (device->bdev) {
1112 		fs_devices->open_devices--;
1113 		device->bdev = NULL;
1114 	}
1115 	clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
1116 	btrfs_destroy_dev_zone_info(device);
1117 
1118 	device->fs_info = NULL;
1119 	atomic_set(&device->dev_stats_ccnt, 0);
1120 	extent_io_tree_release(&device->alloc_state);
1121 
1122 	/*
1123 	 * Reset the flush error record. We might have a transient flush error
1124 	 * in this mount, and if so we aborted the current transaction and set
1125 	 * the fs to an error state, guaranteeing no super blocks can be further
1126 	 * committed. However that error might be transient and if we unmount the
1127 	 * filesystem and mount it again, we should allow the mount to succeed
1128 	 * (btrfs_check_rw_degradable() should not fail) - if after mounting the
1129 	 * filesystem again we still get flush errors, then we will again abort
1130 	 * any transaction and set the error state, guaranteeing no commits of
1131 	 * unsafe super blocks.
1132 	 */
1133 	device->last_flush_error = 0;
1134 
1135 	/* Verify the device is back in a pristine state  */
1136 	ASSERT(!test_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state));
1137 	ASSERT(!test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state));
1138 	ASSERT(list_empty(&device->dev_alloc_list));
1139 	ASSERT(list_empty(&device->post_commit_list));
1140 }
1141 
1142 static void close_fs_devices(struct btrfs_fs_devices *fs_devices)
1143 {
1144 	struct btrfs_device *device, *tmp;
1145 
1146 	lockdep_assert_held(&uuid_mutex);
1147 
1148 	if (--fs_devices->opened > 0)
1149 		return;
1150 
1151 	list_for_each_entry_safe(device, tmp, &fs_devices->devices, dev_list)
1152 		btrfs_close_one_device(device);
1153 
1154 	WARN_ON(fs_devices->open_devices);
1155 	WARN_ON(fs_devices->rw_devices);
1156 	fs_devices->opened = 0;
1157 	fs_devices->seeding = false;
1158 	fs_devices->fs_info = NULL;
1159 }
1160 
1161 void btrfs_close_devices(struct btrfs_fs_devices *fs_devices)
1162 {
1163 	LIST_HEAD(list);
1164 	struct btrfs_fs_devices *tmp;
1165 
1166 	mutex_lock(&uuid_mutex);
1167 	close_fs_devices(fs_devices);
1168 	if (!fs_devices->opened)
1169 		list_splice_init(&fs_devices->seed_list, &list);
1170 
1171 	list_for_each_entry_safe(fs_devices, tmp, &list, seed_list) {
1172 		close_fs_devices(fs_devices);
1173 		list_del(&fs_devices->seed_list);
1174 		free_fs_devices(fs_devices);
1175 	}
1176 	mutex_unlock(&uuid_mutex);
1177 }
1178 
1179 static int open_fs_devices(struct btrfs_fs_devices *fs_devices,
1180 				fmode_t flags, void *holder)
1181 {
1182 	struct btrfs_device *device;
1183 	struct btrfs_device *latest_dev = NULL;
1184 	struct btrfs_device *tmp_device;
1185 
1186 	flags |= FMODE_EXCL;
1187 
1188 	list_for_each_entry_safe(device, tmp_device, &fs_devices->devices,
1189 				 dev_list) {
1190 		int ret;
1191 
1192 		ret = btrfs_open_one_device(fs_devices, device, flags, holder);
1193 		if (ret == 0 &&
1194 		    (!latest_dev || device->generation > latest_dev->generation)) {
1195 			latest_dev = device;
1196 		} else if (ret == -ENODATA) {
1197 			fs_devices->num_devices--;
1198 			list_del(&device->dev_list);
1199 			btrfs_free_device(device);
1200 		}
1201 	}
1202 	if (fs_devices->open_devices == 0)
1203 		return -EINVAL;
1204 
1205 	fs_devices->opened = 1;
1206 	fs_devices->latest_dev = latest_dev;
1207 	fs_devices->total_rw_bytes = 0;
1208 	fs_devices->chunk_alloc_policy = BTRFS_CHUNK_ALLOC_REGULAR;
1209 	fs_devices->read_policy = BTRFS_READ_POLICY_PID;
1210 
1211 	return 0;
1212 }
1213 
1214 static int devid_cmp(void *priv, const struct list_head *a,
1215 		     const struct list_head *b)
1216 {
1217 	const struct btrfs_device *dev1, *dev2;
1218 
1219 	dev1 = list_entry(a, struct btrfs_device, dev_list);
1220 	dev2 = list_entry(b, struct btrfs_device, dev_list);
1221 
1222 	if (dev1->devid < dev2->devid)
1223 		return -1;
1224 	else if (dev1->devid > dev2->devid)
1225 		return 1;
1226 	return 0;
1227 }
1228 
1229 int btrfs_open_devices(struct btrfs_fs_devices *fs_devices,
1230 		       fmode_t flags, void *holder)
1231 {
1232 	int ret;
1233 
1234 	lockdep_assert_held(&uuid_mutex);
1235 	/*
1236 	 * The device_list_mutex cannot be taken here in case opening the
1237 	 * underlying device takes further locks like open_mutex.
1238 	 *
1239 	 * We also don't need the lock here as this is called during mount and
1240 	 * exclusion is provided by uuid_mutex
1241 	 */
1242 
1243 	if (fs_devices->opened) {
1244 		fs_devices->opened++;
1245 		ret = 0;
1246 	} else {
1247 		list_sort(NULL, &fs_devices->devices, devid_cmp);
1248 		ret = open_fs_devices(fs_devices, flags, holder);
1249 	}
1250 
1251 	return ret;
1252 }
1253 
1254 void btrfs_release_disk_super(struct btrfs_super_block *super)
1255 {
1256 	struct page *page = virt_to_page(super);
1257 
1258 	put_page(page);
1259 }
1260 
1261 static struct btrfs_super_block *btrfs_read_disk_super(struct block_device *bdev,
1262 						       u64 bytenr, u64 bytenr_orig)
1263 {
1264 	struct btrfs_super_block *disk_super;
1265 	struct page *page;
1266 	void *p;
1267 	pgoff_t index;
1268 
1269 	/* make sure our super fits in the device */
1270 	if (bytenr + PAGE_SIZE >= bdev_nr_bytes(bdev))
1271 		return ERR_PTR(-EINVAL);
1272 
1273 	/* make sure our super fits in the page */
1274 	if (sizeof(*disk_super) > PAGE_SIZE)
1275 		return ERR_PTR(-EINVAL);
1276 
1277 	/* make sure our super doesn't straddle pages on disk */
1278 	index = bytenr >> PAGE_SHIFT;
1279 	if ((bytenr + sizeof(*disk_super) - 1) >> PAGE_SHIFT != index)
1280 		return ERR_PTR(-EINVAL);
1281 
1282 	/* pull in the page with our super */
1283 	page = read_cache_page_gfp(bdev->bd_inode->i_mapping, index, GFP_KERNEL);
1284 
1285 	if (IS_ERR(page))
1286 		return ERR_CAST(page);
1287 
1288 	p = page_address(page);
1289 
1290 	/* align our pointer to the offset of the super block */
1291 	disk_super = p + offset_in_page(bytenr);
1292 
1293 	if (btrfs_super_bytenr(disk_super) != bytenr_orig ||
1294 	    btrfs_super_magic(disk_super) != BTRFS_MAGIC) {
1295 		btrfs_release_disk_super(p);
1296 		return ERR_PTR(-EINVAL);
1297 	}
1298 
1299 	if (disk_super->label[0] && disk_super->label[BTRFS_LABEL_SIZE - 1])
1300 		disk_super->label[BTRFS_LABEL_SIZE - 1] = 0;
1301 
1302 	return disk_super;
1303 }
1304 
1305 int btrfs_forget_devices(dev_t devt)
1306 {
1307 	int ret;
1308 
1309 	mutex_lock(&uuid_mutex);
1310 	ret = btrfs_free_stale_devices(devt, NULL);
1311 	mutex_unlock(&uuid_mutex);
1312 
1313 	return ret;
1314 }
1315 
1316 /*
1317  * Look for a btrfs signature on a device. This may be called out of the mount path
1318  * and we are not allowed to call set_blocksize during the scan. The superblock
1319  * is read via pagecache
1320  */
1321 struct btrfs_device *btrfs_scan_one_device(const char *path, fmode_t flags,
1322 					   void *holder)
1323 {
1324 	struct btrfs_super_block *disk_super;
1325 	bool new_device_added = false;
1326 	struct btrfs_device *device = NULL;
1327 	struct block_device *bdev;
1328 	u64 bytenr, bytenr_orig;
1329 	int ret;
1330 
1331 	lockdep_assert_held(&uuid_mutex);
1332 
1333 	/*
1334 	 * we would like to check all the supers, but that would make
1335 	 * a btrfs mount succeed after a mkfs from a different FS.
1336 	 * So, we need to add a special mount option to scan for
1337 	 * later supers, using BTRFS_SUPER_MIRROR_MAX instead
1338 	 */
1339 	flags |= FMODE_EXCL;
1340 
1341 	bdev = blkdev_get_by_path(path, flags, holder);
1342 	if (IS_ERR(bdev))
1343 		return ERR_CAST(bdev);
1344 
1345 	bytenr_orig = btrfs_sb_offset(0);
1346 	ret = btrfs_sb_log_location_bdev(bdev, 0, READ, &bytenr);
1347 	if (ret) {
1348 		device = ERR_PTR(ret);
1349 		goto error_bdev_put;
1350 	}
1351 
1352 	disk_super = btrfs_read_disk_super(bdev, bytenr, bytenr_orig);
1353 	if (IS_ERR(disk_super)) {
1354 		device = ERR_CAST(disk_super);
1355 		goto error_bdev_put;
1356 	}
1357 
1358 	device = device_list_add(path, disk_super, &new_device_added);
1359 	if (!IS_ERR(device) && new_device_added)
1360 		btrfs_free_stale_devices(device->devt, device);
1361 
1362 	btrfs_release_disk_super(disk_super);
1363 
1364 error_bdev_put:
1365 	blkdev_put(bdev, flags);
1366 
1367 	return device;
1368 }
1369 
1370 /*
1371  * Try to find a chunk that intersects [start, start + len] range and when one
1372  * such is found, record the end of it in *start
1373  */
1374 static bool contains_pending_extent(struct btrfs_device *device, u64 *start,
1375 				    u64 len)
1376 {
1377 	u64 physical_start, physical_end;
1378 
1379 	lockdep_assert_held(&device->fs_info->chunk_mutex);
1380 
1381 	if (!find_first_extent_bit(&device->alloc_state, *start,
1382 				   &physical_start, &physical_end,
1383 				   CHUNK_ALLOCATED, NULL)) {
1384 
1385 		if (in_range(physical_start, *start, len) ||
1386 		    in_range(*start, physical_start,
1387 			     physical_end - physical_start)) {
1388 			*start = physical_end + 1;
1389 			return true;
1390 		}
1391 	}
1392 	return false;
1393 }
1394 
1395 static u64 dev_extent_search_start(struct btrfs_device *device, u64 start)
1396 {
1397 	switch (device->fs_devices->chunk_alloc_policy) {
1398 	case BTRFS_CHUNK_ALLOC_REGULAR:
1399 		/*
1400 		 * We don't want to overwrite the superblock on the drive nor
1401 		 * any area used by the boot loader (grub for example), so we
1402 		 * make sure to start at an offset of at least 1MB.
1403 		 */
1404 		return max_t(u64, start, SZ_1M);
1405 	case BTRFS_CHUNK_ALLOC_ZONED:
1406 		/*
1407 		 * We don't care about the starting region like regular
1408 		 * allocator, because we anyway use/reserve the first two zones
1409 		 * for superblock logging.
1410 		 */
1411 		return ALIGN(start, device->zone_info->zone_size);
1412 	default:
1413 		BUG();
1414 	}
1415 }
1416 
1417 static bool dev_extent_hole_check_zoned(struct btrfs_device *device,
1418 					u64 *hole_start, u64 *hole_size,
1419 					u64 num_bytes)
1420 {
1421 	u64 zone_size = device->zone_info->zone_size;
1422 	u64 pos;
1423 	int ret;
1424 	bool changed = false;
1425 
1426 	ASSERT(IS_ALIGNED(*hole_start, zone_size));
1427 
1428 	while (*hole_size > 0) {
1429 		pos = btrfs_find_allocatable_zones(device, *hole_start,
1430 						   *hole_start + *hole_size,
1431 						   num_bytes);
1432 		if (pos != *hole_start) {
1433 			*hole_size = *hole_start + *hole_size - pos;
1434 			*hole_start = pos;
1435 			changed = true;
1436 			if (*hole_size < num_bytes)
1437 				break;
1438 		}
1439 
1440 		ret = btrfs_ensure_empty_zones(device, pos, num_bytes);
1441 
1442 		/* Range is ensured to be empty */
1443 		if (!ret)
1444 			return changed;
1445 
1446 		/* Given hole range was invalid (outside of device) */
1447 		if (ret == -ERANGE) {
1448 			*hole_start += *hole_size;
1449 			*hole_size = 0;
1450 			return true;
1451 		}
1452 
1453 		*hole_start += zone_size;
1454 		*hole_size -= zone_size;
1455 		changed = true;
1456 	}
1457 
1458 	return changed;
1459 }
1460 
1461 /**
1462  * dev_extent_hole_check - check if specified hole is suitable for allocation
1463  * @device:	the device which we have the hole
1464  * @hole_start: starting position of the hole
1465  * @hole_size:	the size of the hole
1466  * @num_bytes:	the size of the free space that we need
1467  *
1468  * This function may modify @hole_start and @hole_size to reflect the suitable
1469  * position for allocation. Returns 1 if hole position is updated, 0 otherwise.
1470  */
1471 static bool dev_extent_hole_check(struct btrfs_device *device, u64 *hole_start,
1472 				  u64 *hole_size, u64 num_bytes)
1473 {
1474 	bool changed = false;
1475 	u64 hole_end = *hole_start + *hole_size;
1476 
1477 	for (;;) {
1478 		/*
1479 		 * Check before we set max_hole_start, otherwise we could end up
1480 		 * sending back this offset anyway.
1481 		 */
1482 		if (contains_pending_extent(device, hole_start, *hole_size)) {
1483 			if (hole_end >= *hole_start)
1484 				*hole_size = hole_end - *hole_start;
1485 			else
1486 				*hole_size = 0;
1487 			changed = true;
1488 		}
1489 
1490 		switch (device->fs_devices->chunk_alloc_policy) {
1491 		case BTRFS_CHUNK_ALLOC_REGULAR:
1492 			/* No extra check */
1493 			break;
1494 		case BTRFS_CHUNK_ALLOC_ZONED:
1495 			if (dev_extent_hole_check_zoned(device, hole_start,
1496 							hole_size, num_bytes)) {
1497 				changed = true;
1498 				/*
1499 				 * The changed hole can contain pending extent.
1500 				 * Loop again to check that.
1501 				 */
1502 				continue;
1503 			}
1504 			break;
1505 		default:
1506 			BUG();
1507 		}
1508 
1509 		break;
1510 	}
1511 
1512 	return changed;
1513 }
1514 
1515 /*
1516  * find_free_dev_extent_start - find free space in the specified device
1517  * @device:	  the device which we search the free space in
1518  * @num_bytes:	  the size of the free space that we need
1519  * @search_start: the position from which to begin the search
1520  * @start:	  store the start of the free space.
1521  * @len:	  the size of the free space. that we find, or the size
1522  *		  of the max free space if we don't find suitable free space
1523  *
1524  * this uses a pretty simple search, the expectation is that it is
1525  * called very infrequently and that a given device has a small number
1526  * of extents
1527  *
1528  * @start is used to store the start of the free space if we find. But if we
1529  * don't find suitable free space, it will be used to store the start position
1530  * of the max free space.
1531  *
1532  * @len is used to store the size of the free space that we find.
1533  * But if we don't find suitable free space, it is used to store the size of
1534  * the max free space.
1535  *
1536  * NOTE: This function will search *commit* root of device tree, and does extra
1537  * check to ensure dev extents are not double allocated.
1538  * This makes the function safe to allocate dev extents but may not report
1539  * correct usable device space, as device extent freed in current transaction
1540  * is not reported as available.
1541  */
1542 static int find_free_dev_extent_start(struct btrfs_device *device,
1543 				u64 num_bytes, u64 search_start, u64 *start,
1544 				u64 *len)
1545 {
1546 	struct btrfs_fs_info *fs_info = device->fs_info;
1547 	struct btrfs_root *root = fs_info->dev_root;
1548 	struct btrfs_key key;
1549 	struct btrfs_dev_extent *dev_extent;
1550 	struct btrfs_path *path;
1551 	u64 hole_size;
1552 	u64 max_hole_start;
1553 	u64 max_hole_size;
1554 	u64 extent_end;
1555 	u64 search_end = device->total_bytes;
1556 	int ret;
1557 	int slot;
1558 	struct extent_buffer *l;
1559 
1560 	search_start = dev_extent_search_start(device, search_start);
1561 
1562 	WARN_ON(device->zone_info &&
1563 		!IS_ALIGNED(num_bytes, device->zone_info->zone_size));
1564 
1565 	path = btrfs_alloc_path();
1566 	if (!path)
1567 		return -ENOMEM;
1568 
1569 	max_hole_start = search_start;
1570 	max_hole_size = 0;
1571 
1572 again:
1573 	if (search_start >= search_end ||
1574 		test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
1575 		ret = -ENOSPC;
1576 		goto out;
1577 	}
1578 
1579 	path->reada = READA_FORWARD;
1580 	path->search_commit_root = 1;
1581 	path->skip_locking = 1;
1582 
1583 	key.objectid = device->devid;
1584 	key.offset = search_start;
1585 	key.type = BTRFS_DEV_EXTENT_KEY;
1586 
1587 	ret = btrfs_search_backwards(root, &key, path);
1588 	if (ret < 0)
1589 		goto out;
1590 
1591 	while (1) {
1592 		l = path->nodes[0];
1593 		slot = path->slots[0];
1594 		if (slot >= btrfs_header_nritems(l)) {
1595 			ret = btrfs_next_leaf(root, path);
1596 			if (ret == 0)
1597 				continue;
1598 			if (ret < 0)
1599 				goto out;
1600 
1601 			break;
1602 		}
1603 		btrfs_item_key_to_cpu(l, &key, slot);
1604 
1605 		if (key.objectid < device->devid)
1606 			goto next;
1607 
1608 		if (key.objectid > device->devid)
1609 			break;
1610 
1611 		if (key.type != BTRFS_DEV_EXTENT_KEY)
1612 			goto next;
1613 
1614 		if (key.offset > search_start) {
1615 			hole_size = key.offset - search_start;
1616 			dev_extent_hole_check(device, &search_start, &hole_size,
1617 					      num_bytes);
1618 
1619 			if (hole_size > max_hole_size) {
1620 				max_hole_start = search_start;
1621 				max_hole_size = hole_size;
1622 			}
1623 
1624 			/*
1625 			 * If this free space is greater than which we need,
1626 			 * it must be the max free space that we have found
1627 			 * until now, so max_hole_start must point to the start
1628 			 * of this free space and the length of this free space
1629 			 * is stored in max_hole_size. Thus, we return
1630 			 * max_hole_start and max_hole_size and go back to the
1631 			 * caller.
1632 			 */
1633 			if (hole_size >= num_bytes) {
1634 				ret = 0;
1635 				goto out;
1636 			}
1637 		}
1638 
1639 		dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
1640 		extent_end = key.offset + btrfs_dev_extent_length(l,
1641 								  dev_extent);
1642 		if (extent_end > search_start)
1643 			search_start = extent_end;
1644 next:
1645 		path->slots[0]++;
1646 		cond_resched();
1647 	}
1648 
1649 	/*
1650 	 * At this point, search_start should be the end of
1651 	 * allocated dev extents, and when shrinking the device,
1652 	 * search_end may be smaller than search_start.
1653 	 */
1654 	if (search_end > search_start) {
1655 		hole_size = search_end - search_start;
1656 		if (dev_extent_hole_check(device, &search_start, &hole_size,
1657 					  num_bytes)) {
1658 			btrfs_release_path(path);
1659 			goto again;
1660 		}
1661 
1662 		if (hole_size > max_hole_size) {
1663 			max_hole_start = search_start;
1664 			max_hole_size = hole_size;
1665 		}
1666 	}
1667 
1668 	/* See above. */
1669 	if (max_hole_size < num_bytes)
1670 		ret = -ENOSPC;
1671 	else
1672 		ret = 0;
1673 
1674 out:
1675 	btrfs_free_path(path);
1676 	*start = max_hole_start;
1677 	if (len)
1678 		*len = max_hole_size;
1679 	return ret;
1680 }
1681 
1682 int find_free_dev_extent(struct btrfs_device *device, u64 num_bytes,
1683 			 u64 *start, u64 *len)
1684 {
1685 	/* FIXME use last free of some kind */
1686 	return find_free_dev_extent_start(device, num_bytes, 0, start, len);
1687 }
1688 
1689 static int btrfs_free_dev_extent(struct btrfs_trans_handle *trans,
1690 			  struct btrfs_device *device,
1691 			  u64 start, u64 *dev_extent_len)
1692 {
1693 	struct btrfs_fs_info *fs_info = device->fs_info;
1694 	struct btrfs_root *root = fs_info->dev_root;
1695 	int ret;
1696 	struct btrfs_path *path;
1697 	struct btrfs_key key;
1698 	struct btrfs_key found_key;
1699 	struct extent_buffer *leaf = NULL;
1700 	struct btrfs_dev_extent *extent = NULL;
1701 
1702 	path = btrfs_alloc_path();
1703 	if (!path)
1704 		return -ENOMEM;
1705 
1706 	key.objectid = device->devid;
1707 	key.offset = start;
1708 	key.type = BTRFS_DEV_EXTENT_KEY;
1709 again:
1710 	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1711 	if (ret > 0) {
1712 		ret = btrfs_previous_item(root, path, key.objectid,
1713 					  BTRFS_DEV_EXTENT_KEY);
1714 		if (ret)
1715 			goto out;
1716 		leaf = path->nodes[0];
1717 		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
1718 		extent = btrfs_item_ptr(leaf, path->slots[0],
1719 					struct btrfs_dev_extent);
1720 		BUG_ON(found_key.offset > start || found_key.offset +
1721 		       btrfs_dev_extent_length(leaf, extent) < start);
1722 		key = found_key;
1723 		btrfs_release_path(path);
1724 		goto again;
1725 	} else if (ret == 0) {
1726 		leaf = path->nodes[0];
1727 		extent = btrfs_item_ptr(leaf, path->slots[0],
1728 					struct btrfs_dev_extent);
1729 	} else {
1730 		goto out;
1731 	}
1732 
1733 	*dev_extent_len = btrfs_dev_extent_length(leaf, extent);
1734 
1735 	ret = btrfs_del_item(trans, root, path);
1736 	if (ret == 0)
1737 		set_bit(BTRFS_TRANS_HAVE_FREE_BGS, &trans->transaction->flags);
1738 out:
1739 	btrfs_free_path(path);
1740 	return ret;
1741 }
1742 
1743 static u64 find_next_chunk(struct btrfs_fs_info *fs_info)
1744 {
1745 	struct extent_map_tree *em_tree;
1746 	struct extent_map *em;
1747 	struct rb_node *n;
1748 	u64 ret = 0;
1749 
1750 	em_tree = &fs_info->mapping_tree;
1751 	read_lock(&em_tree->lock);
1752 	n = rb_last(&em_tree->map.rb_root);
1753 	if (n) {
1754 		em = rb_entry(n, struct extent_map, rb_node);
1755 		ret = em->start + em->len;
1756 	}
1757 	read_unlock(&em_tree->lock);
1758 
1759 	return ret;
1760 }
1761 
1762 static noinline int find_next_devid(struct btrfs_fs_info *fs_info,
1763 				    u64 *devid_ret)
1764 {
1765 	int ret;
1766 	struct btrfs_key key;
1767 	struct btrfs_key found_key;
1768 	struct btrfs_path *path;
1769 
1770 	path = btrfs_alloc_path();
1771 	if (!path)
1772 		return -ENOMEM;
1773 
1774 	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1775 	key.type = BTRFS_DEV_ITEM_KEY;
1776 	key.offset = (u64)-1;
1777 
1778 	ret = btrfs_search_slot(NULL, fs_info->chunk_root, &key, path, 0, 0);
1779 	if (ret < 0)
1780 		goto error;
1781 
1782 	if (ret == 0) {
1783 		/* Corruption */
1784 		btrfs_err(fs_info, "corrupted chunk tree devid -1 matched");
1785 		ret = -EUCLEAN;
1786 		goto error;
1787 	}
1788 
1789 	ret = btrfs_previous_item(fs_info->chunk_root, path,
1790 				  BTRFS_DEV_ITEMS_OBJECTID,
1791 				  BTRFS_DEV_ITEM_KEY);
1792 	if (ret) {
1793 		*devid_ret = 1;
1794 	} else {
1795 		btrfs_item_key_to_cpu(path->nodes[0], &found_key,
1796 				      path->slots[0]);
1797 		*devid_ret = found_key.offset + 1;
1798 	}
1799 	ret = 0;
1800 error:
1801 	btrfs_free_path(path);
1802 	return ret;
1803 }
1804 
1805 /*
1806  * the device information is stored in the chunk root
1807  * the btrfs_device struct should be fully filled in
1808  */
1809 static int btrfs_add_dev_item(struct btrfs_trans_handle *trans,
1810 			    struct btrfs_device *device)
1811 {
1812 	int ret;
1813 	struct btrfs_path *path;
1814 	struct btrfs_dev_item *dev_item;
1815 	struct extent_buffer *leaf;
1816 	struct btrfs_key key;
1817 	unsigned long ptr;
1818 
1819 	path = btrfs_alloc_path();
1820 	if (!path)
1821 		return -ENOMEM;
1822 
1823 	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1824 	key.type = BTRFS_DEV_ITEM_KEY;
1825 	key.offset = device->devid;
1826 
1827 	btrfs_reserve_chunk_metadata(trans, true);
1828 	ret = btrfs_insert_empty_item(trans, trans->fs_info->chunk_root, path,
1829 				      &key, sizeof(*dev_item));
1830 	btrfs_trans_release_chunk_metadata(trans);
1831 	if (ret)
1832 		goto out;
1833 
1834 	leaf = path->nodes[0];
1835 	dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item);
1836 
1837 	btrfs_set_device_id(leaf, dev_item, device->devid);
1838 	btrfs_set_device_generation(leaf, dev_item, 0);
1839 	btrfs_set_device_type(leaf, dev_item, device->type);
1840 	btrfs_set_device_io_align(leaf, dev_item, device->io_align);
1841 	btrfs_set_device_io_width(leaf, dev_item, device->io_width);
1842 	btrfs_set_device_sector_size(leaf, dev_item, device->sector_size);
1843 	btrfs_set_device_total_bytes(leaf, dev_item,
1844 				     btrfs_device_get_disk_total_bytes(device));
1845 	btrfs_set_device_bytes_used(leaf, dev_item,
1846 				    btrfs_device_get_bytes_used(device));
1847 	btrfs_set_device_group(leaf, dev_item, 0);
1848 	btrfs_set_device_seek_speed(leaf, dev_item, 0);
1849 	btrfs_set_device_bandwidth(leaf, dev_item, 0);
1850 	btrfs_set_device_start_offset(leaf, dev_item, 0);
1851 
1852 	ptr = btrfs_device_uuid(dev_item);
1853 	write_extent_buffer(leaf, device->uuid, ptr, BTRFS_UUID_SIZE);
1854 	ptr = btrfs_device_fsid(dev_item);
1855 	write_extent_buffer(leaf, trans->fs_info->fs_devices->metadata_uuid,
1856 			    ptr, BTRFS_FSID_SIZE);
1857 	btrfs_mark_buffer_dirty(leaf);
1858 
1859 	ret = 0;
1860 out:
1861 	btrfs_free_path(path);
1862 	return ret;
1863 }
1864 
1865 /*
1866  * Function to update ctime/mtime for a given device path.
1867  * Mainly used for ctime/mtime based probe like libblkid.
1868  *
1869  * We don't care about errors here, this is just to be kind to userspace.
1870  */
1871 static void update_dev_time(const char *device_path)
1872 {
1873 	struct path path;
1874 	struct timespec64 now;
1875 	int ret;
1876 
1877 	ret = kern_path(device_path, LOOKUP_FOLLOW, &path);
1878 	if (ret)
1879 		return;
1880 
1881 	now = current_time(d_inode(path.dentry));
1882 	inode_update_time(d_inode(path.dentry), &now, S_MTIME | S_CTIME);
1883 	path_put(&path);
1884 }
1885 
1886 static int btrfs_rm_dev_item(struct btrfs_trans_handle *trans,
1887 			     struct btrfs_device *device)
1888 {
1889 	struct btrfs_root *root = device->fs_info->chunk_root;
1890 	int ret;
1891 	struct btrfs_path *path;
1892 	struct btrfs_key key;
1893 
1894 	path = btrfs_alloc_path();
1895 	if (!path)
1896 		return -ENOMEM;
1897 
1898 	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1899 	key.type = BTRFS_DEV_ITEM_KEY;
1900 	key.offset = device->devid;
1901 
1902 	btrfs_reserve_chunk_metadata(trans, false);
1903 	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1904 	btrfs_trans_release_chunk_metadata(trans);
1905 	if (ret) {
1906 		if (ret > 0)
1907 			ret = -ENOENT;
1908 		goto out;
1909 	}
1910 
1911 	ret = btrfs_del_item(trans, root, path);
1912 out:
1913 	btrfs_free_path(path);
1914 	return ret;
1915 }
1916 
1917 /*
1918  * Verify that @num_devices satisfies the RAID profile constraints in the whole
1919  * filesystem. It's up to the caller to adjust that number regarding eg. device
1920  * replace.
1921  */
1922 static int btrfs_check_raid_min_devices(struct btrfs_fs_info *fs_info,
1923 		u64 num_devices)
1924 {
1925 	u64 all_avail;
1926 	unsigned seq;
1927 	int i;
1928 
1929 	do {
1930 		seq = read_seqbegin(&fs_info->profiles_lock);
1931 
1932 		all_avail = fs_info->avail_data_alloc_bits |
1933 			    fs_info->avail_system_alloc_bits |
1934 			    fs_info->avail_metadata_alloc_bits;
1935 	} while (read_seqretry(&fs_info->profiles_lock, seq));
1936 
1937 	for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) {
1938 		if (!(all_avail & btrfs_raid_array[i].bg_flag))
1939 			continue;
1940 
1941 		if (num_devices < btrfs_raid_array[i].devs_min)
1942 			return btrfs_raid_array[i].mindev_error;
1943 	}
1944 
1945 	return 0;
1946 }
1947 
1948 static struct btrfs_device * btrfs_find_next_active_device(
1949 		struct btrfs_fs_devices *fs_devs, struct btrfs_device *device)
1950 {
1951 	struct btrfs_device *next_device;
1952 
1953 	list_for_each_entry(next_device, &fs_devs->devices, dev_list) {
1954 		if (next_device != device &&
1955 		    !test_bit(BTRFS_DEV_STATE_MISSING, &next_device->dev_state)
1956 		    && next_device->bdev)
1957 			return next_device;
1958 	}
1959 
1960 	return NULL;
1961 }
1962 
1963 /*
1964  * Helper function to check if the given device is part of s_bdev / latest_dev
1965  * and replace it with the provided or the next active device, in the context
1966  * where this function called, there should be always be another device (or
1967  * this_dev) which is active.
1968  */
1969 void __cold btrfs_assign_next_active_device(struct btrfs_device *device,
1970 					    struct btrfs_device *next_device)
1971 {
1972 	struct btrfs_fs_info *fs_info = device->fs_info;
1973 
1974 	if (!next_device)
1975 		next_device = btrfs_find_next_active_device(fs_info->fs_devices,
1976 							    device);
1977 	ASSERT(next_device);
1978 
1979 	if (fs_info->sb->s_bdev &&
1980 			(fs_info->sb->s_bdev == device->bdev))
1981 		fs_info->sb->s_bdev = next_device->bdev;
1982 
1983 	if (fs_info->fs_devices->latest_dev->bdev == device->bdev)
1984 		fs_info->fs_devices->latest_dev = next_device;
1985 }
1986 
1987 /*
1988  * Return btrfs_fs_devices::num_devices excluding the device that's being
1989  * currently replaced.
1990  */
1991 static u64 btrfs_num_devices(struct btrfs_fs_info *fs_info)
1992 {
1993 	u64 num_devices = fs_info->fs_devices->num_devices;
1994 
1995 	down_read(&fs_info->dev_replace.rwsem);
1996 	if (btrfs_dev_replace_is_ongoing(&fs_info->dev_replace)) {
1997 		ASSERT(num_devices > 1);
1998 		num_devices--;
1999 	}
2000 	up_read(&fs_info->dev_replace.rwsem);
2001 
2002 	return num_devices;
2003 }
2004 
2005 void btrfs_scratch_superblocks(struct btrfs_fs_info *fs_info,
2006 			       struct block_device *bdev,
2007 			       const char *device_path)
2008 {
2009 	struct btrfs_super_block *disk_super;
2010 	int copy_num;
2011 
2012 	if (!bdev)
2013 		return;
2014 
2015 	for (copy_num = 0; copy_num < BTRFS_SUPER_MIRROR_MAX; copy_num++) {
2016 		struct page *page;
2017 		int ret;
2018 
2019 		disk_super = btrfs_read_dev_one_super(bdev, copy_num);
2020 		if (IS_ERR(disk_super))
2021 			continue;
2022 
2023 		if (bdev_is_zoned(bdev)) {
2024 			btrfs_reset_sb_log_zones(bdev, copy_num);
2025 			continue;
2026 		}
2027 
2028 		memset(&disk_super->magic, 0, sizeof(disk_super->magic));
2029 
2030 		page = virt_to_page(disk_super);
2031 		set_page_dirty(page);
2032 		lock_page(page);
2033 		/* write_on_page() unlocks the page */
2034 		ret = write_one_page(page);
2035 		if (ret)
2036 			btrfs_warn(fs_info,
2037 				"error clearing superblock number %d (%d)",
2038 				copy_num, ret);
2039 		btrfs_release_disk_super(disk_super);
2040 
2041 	}
2042 
2043 	/* Notify udev that device has changed */
2044 	btrfs_kobject_uevent(bdev, KOBJ_CHANGE);
2045 
2046 	/* Update ctime/mtime for device path for libblkid */
2047 	update_dev_time(device_path);
2048 }
2049 
2050 int btrfs_rm_device(struct btrfs_fs_info *fs_info,
2051 		    struct btrfs_dev_lookup_args *args,
2052 		    struct block_device **bdev, fmode_t *mode)
2053 {
2054 	struct btrfs_trans_handle *trans;
2055 	struct btrfs_device *device;
2056 	struct btrfs_fs_devices *cur_devices;
2057 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2058 	u64 num_devices;
2059 	int ret = 0;
2060 
2061 	if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) {
2062 		btrfs_err(fs_info, "device remove not supported on extent tree v2 yet");
2063 		return -EINVAL;
2064 	}
2065 
2066 	/*
2067 	 * The device list in fs_devices is accessed without locks (neither
2068 	 * uuid_mutex nor device_list_mutex) as it won't change on a mounted
2069 	 * filesystem and another device rm cannot run.
2070 	 */
2071 	num_devices = btrfs_num_devices(fs_info);
2072 
2073 	ret = btrfs_check_raid_min_devices(fs_info, num_devices - 1);
2074 	if (ret)
2075 		return ret;
2076 
2077 	device = btrfs_find_device(fs_info->fs_devices, args);
2078 	if (!device) {
2079 		if (args->missing)
2080 			ret = BTRFS_ERROR_DEV_MISSING_NOT_FOUND;
2081 		else
2082 			ret = -ENOENT;
2083 		return ret;
2084 	}
2085 
2086 	if (btrfs_pinned_by_swapfile(fs_info, device)) {
2087 		btrfs_warn_in_rcu(fs_info,
2088 		  "cannot remove device %s (devid %llu) due to active swapfile",
2089 				  rcu_str_deref(device->name), device->devid);
2090 		return -ETXTBSY;
2091 	}
2092 
2093 	if (test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state))
2094 		return BTRFS_ERROR_DEV_TGT_REPLACE;
2095 
2096 	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
2097 	    fs_info->fs_devices->rw_devices == 1)
2098 		return BTRFS_ERROR_DEV_ONLY_WRITABLE;
2099 
2100 	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
2101 		mutex_lock(&fs_info->chunk_mutex);
2102 		list_del_init(&device->dev_alloc_list);
2103 		device->fs_devices->rw_devices--;
2104 		mutex_unlock(&fs_info->chunk_mutex);
2105 	}
2106 
2107 	ret = btrfs_shrink_device(device, 0);
2108 	if (ret)
2109 		goto error_undo;
2110 
2111 	trans = btrfs_start_transaction(fs_info->chunk_root, 0);
2112 	if (IS_ERR(trans)) {
2113 		ret = PTR_ERR(trans);
2114 		goto error_undo;
2115 	}
2116 
2117 	ret = btrfs_rm_dev_item(trans, device);
2118 	if (ret) {
2119 		/* Any error in dev item removal is critical */
2120 		btrfs_crit(fs_info,
2121 			   "failed to remove device item for devid %llu: %d",
2122 			   device->devid, ret);
2123 		btrfs_abort_transaction(trans, ret);
2124 		btrfs_end_transaction(trans);
2125 		return ret;
2126 	}
2127 
2128 	clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
2129 	btrfs_scrub_cancel_dev(device);
2130 
2131 	/*
2132 	 * the device list mutex makes sure that we don't change
2133 	 * the device list while someone else is writing out all
2134 	 * the device supers. Whoever is writing all supers, should
2135 	 * lock the device list mutex before getting the number of
2136 	 * devices in the super block (super_copy). Conversely,
2137 	 * whoever updates the number of devices in the super block
2138 	 * (super_copy) should hold the device list mutex.
2139 	 */
2140 
2141 	/*
2142 	 * In normal cases the cur_devices == fs_devices. But in case
2143 	 * of deleting a seed device, the cur_devices should point to
2144 	 * its own fs_devices listed under the fs_devices->seed_list.
2145 	 */
2146 	cur_devices = device->fs_devices;
2147 	mutex_lock(&fs_devices->device_list_mutex);
2148 	list_del_rcu(&device->dev_list);
2149 
2150 	cur_devices->num_devices--;
2151 	cur_devices->total_devices--;
2152 	/* Update total_devices of the parent fs_devices if it's seed */
2153 	if (cur_devices != fs_devices)
2154 		fs_devices->total_devices--;
2155 
2156 	if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state))
2157 		cur_devices->missing_devices--;
2158 
2159 	btrfs_assign_next_active_device(device, NULL);
2160 
2161 	if (device->bdev) {
2162 		cur_devices->open_devices--;
2163 		/* remove sysfs entry */
2164 		btrfs_sysfs_remove_device(device);
2165 	}
2166 
2167 	num_devices = btrfs_super_num_devices(fs_info->super_copy) - 1;
2168 	btrfs_set_super_num_devices(fs_info->super_copy, num_devices);
2169 	mutex_unlock(&fs_devices->device_list_mutex);
2170 
2171 	/*
2172 	 * At this point, the device is zero sized and detached from the
2173 	 * devices list.  All that's left is to zero out the old supers and
2174 	 * free the device.
2175 	 *
2176 	 * We cannot call btrfs_close_bdev() here because we're holding the sb
2177 	 * write lock, and blkdev_put() will pull in the ->open_mutex on the
2178 	 * block device and it's dependencies.  Instead just flush the device
2179 	 * and let the caller do the final blkdev_put.
2180 	 */
2181 	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
2182 		btrfs_scratch_superblocks(fs_info, device->bdev,
2183 					  device->name->str);
2184 		if (device->bdev) {
2185 			sync_blockdev(device->bdev);
2186 			invalidate_bdev(device->bdev);
2187 		}
2188 	}
2189 
2190 	*bdev = device->bdev;
2191 	*mode = device->mode;
2192 	synchronize_rcu();
2193 	btrfs_free_device(device);
2194 
2195 	/*
2196 	 * This can happen if cur_devices is the private seed devices list.  We
2197 	 * cannot call close_fs_devices() here because it expects the uuid_mutex
2198 	 * to be held, but in fact we don't need that for the private
2199 	 * seed_devices, we can simply decrement cur_devices->opened and then
2200 	 * remove it from our list and free the fs_devices.
2201 	 */
2202 	if (cur_devices->num_devices == 0) {
2203 		list_del_init(&cur_devices->seed_list);
2204 		ASSERT(cur_devices->opened == 1);
2205 		cur_devices->opened--;
2206 		free_fs_devices(cur_devices);
2207 	}
2208 
2209 	ret = btrfs_commit_transaction(trans);
2210 
2211 	return ret;
2212 
2213 error_undo:
2214 	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
2215 		mutex_lock(&fs_info->chunk_mutex);
2216 		list_add(&device->dev_alloc_list,
2217 			 &fs_devices->alloc_list);
2218 		device->fs_devices->rw_devices++;
2219 		mutex_unlock(&fs_info->chunk_mutex);
2220 	}
2221 	return ret;
2222 }
2223 
2224 void btrfs_rm_dev_replace_remove_srcdev(struct btrfs_device *srcdev)
2225 {
2226 	struct btrfs_fs_devices *fs_devices;
2227 
2228 	lockdep_assert_held(&srcdev->fs_info->fs_devices->device_list_mutex);
2229 
2230 	/*
2231 	 * in case of fs with no seed, srcdev->fs_devices will point
2232 	 * to fs_devices of fs_info. However when the dev being replaced is
2233 	 * a seed dev it will point to the seed's local fs_devices. In short
2234 	 * srcdev will have its correct fs_devices in both the cases.
2235 	 */
2236 	fs_devices = srcdev->fs_devices;
2237 
2238 	list_del_rcu(&srcdev->dev_list);
2239 	list_del(&srcdev->dev_alloc_list);
2240 	fs_devices->num_devices--;
2241 	if (test_bit(BTRFS_DEV_STATE_MISSING, &srcdev->dev_state))
2242 		fs_devices->missing_devices--;
2243 
2244 	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &srcdev->dev_state))
2245 		fs_devices->rw_devices--;
2246 
2247 	if (srcdev->bdev)
2248 		fs_devices->open_devices--;
2249 }
2250 
2251 void btrfs_rm_dev_replace_free_srcdev(struct btrfs_device *srcdev)
2252 {
2253 	struct btrfs_fs_devices *fs_devices = srcdev->fs_devices;
2254 
2255 	mutex_lock(&uuid_mutex);
2256 
2257 	btrfs_close_bdev(srcdev);
2258 	synchronize_rcu();
2259 	btrfs_free_device(srcdev);
2260 
2261 	/* if this is no devs we rather delete the fs_devices */
2262 	if (!fs_devices->num_devices) {
2263 		/*
2264 		 * On a mounted FS, num_devices can't be zero unless it's a
2265 		 * seed. In case of a seed device being replaced, the replace
2266 		 * target added to the sprout FS, so there will be no more
2267 		 * device left under the seed FS.
2268 		 */
2269 		ASSERT(fs_devices->seeding);
2270 
2271 		list_del_init(&fs_devices->seed_list);
2272 		close_fs_devices(fs_devices);
2273 		free_fs_devices(fs_devices);
2274 	}
2275 	mutex_unlock(&uuid_mutex);
2276 }
2277 
2278 void btrfs_destroy_dev_replace_tgtdev(struct btrfs_device *tgtdev)
2279 {
2280 	struct btrfs_fs_devices *fs_devices = tgtdev->fs_info->fs_devices;
2281 
2282 	mutex_lock(&fs_devices->device_list_mutex);
2283 
2284 	btrfs_sysfs_remove_device(tgtdev);
2285 
2286 	if (tgtdev->bdev)
2287 		fs_devices->open_devices--;
2288 
2289 	fs_devices->num_devices--;
2290 
2291 	btrfs_assign_next_active_device(tgtdev, NULL);
2292 
2293 	list_del_rcu(&tgtdev->dev_list);
2294 
2295 	mutex_unlock(&fs_devices->device_list_mutex);
2296 
2297 	btrfs_scratch_superblocks(tgtdev->fs_info, tgtdev->bdev,
2298 				  tgtdev->name->str);
2299 
2300 	btrfs_close_bdev(tgtdev);
2301 	synchronize_rcu();
2302 	btrfs_free_device(tgtdev);
2303 }
2304 
2305 /**
2306  * Populate args from device at path
2307  *
2308  * @fs_info:	the filesystem
2309  * @args:	the args to populate
2310  * @path:	the path to the device
2311  *
2312  * This will read the super block of the device at @path and populate @args with
2313  * the devid, fsid, and uuid.  This is meant to be used for ioctls that need to
2314  * lookup a device to operate on, but need to do it before we take any locks.
2315  * This properly handles the special case of "missing" that a user may pass in,
2316  * and does some basic sanity checks.  The caller must make sure that @path is
2317  * properly NUL terminated before calling in, and must call
2318  * btrfs_put_dev_args_from_path() in order to free up the temporary fsid and
2319  * uuid buffers.
2320  *
2321  * Return: 0 for success, -errno for failure
2322  */
2323 int btrfs_get_dev_args_from_path(struct btrfs_fs_info *fs_info,
2324 				 struct btrfs_dev_lookup_args *args,
2325 				 const char *path)
2326 {
2327 	struct btrfs_super_block *disk_super;
2328 	struct block_device *bdev;
2329 	int ret;
2330 
2331 	if (!path || !path[0])
2332 		return -EINVAL;
2333 	if (!strcmp(path, "missing")) {
2334 		args->missing = true;
2335 		return 0;
2336 	}
2337 
2338 	args->uuid = kzalloc(BTRFS_UUID_SIZE, GFP_KERNEL);
2339 	args->fsid = kzalloc(BTRFS_FSID_SIZE, GFP_KERNEL);
2340 	if (!args->uuid || !args->fsid) {
2341 		btrfs_put_dev_args_from_path(args);
2342 		return -ENOMEM;
2343 	}
2344 
2345 	ret = btrfs_get_bdev_and_sb(path, FMODE_READ, fs_info->bdev_holder, 0,
2346 				    &bdev, &disk_super);
2347 	if (ret)
2348 		return ret;
2349 	args->devid = btrfs_stack_device_id(&disk_super->dev_item);
2350 	memcpy(args->uuid, disk_super->dev_item.uuid, BTRFS_UUID_SIZE);
2351 	if (btrfs_fs_incompat(fs_info, METADATA_UUID))
2352 		memcpy(args->fsid, disk_super->metadata_uuid, BTRFS_FSID_SIZE);
2353 	else
2354 		memcpy(args->fsid, disk_super->fsid, BTRFS_FSID_SIZE);
2355 	btrfs_release_disk_super(disk_super);
2356 	blkdev_put(bdev, FMODE_READ);
2357 	return 0;
2358 }
2359 
2360 /*
2361  * Only use this jointly with btrfs_get_dev_args_from_path() because we will
2362  * allocate our ->uuid and ->fsid pointers, everybody else uses local variables
2363  * that don't need to be freed.
2364  */
2365 void btrfs_put_dev_args_from_path(struct btrfs_dev_lookup_args *args)
2366 {
2367 	kfree(args->uuid);
2368 	kfree(args->fsid);
2369 	args->uuid = NULL;
2370 	args->fsid = NULL;
2371 }
2372 
2373 struct btrfs_device *btrfs_find_device_by_devspec(
2374 		struct btrfs_fs_info *fs_info, u64 devid,
2375 		const char *device_path)
2376 {
2377 	BTRFS_DEV_LOOKUP_ARGS(args);
2378 	struct btrfs_device *device;
2379 	int ret;
2380 
2381 	if (devid) {
2382 		args.devid = devid;
2383 		device = btrfs_find_device(fs_info->fs_devices, &args);
2384 		if (!device)
2385 			return ERR_PTR(-ENOENT);
2386 		return device;
2387 	}
2388 
2389 	ret = btrfs_get_dev_args_from_path(fs_info, &args, device_path);
2390 	if (ret)
2391 		return ERR_PTR(ret);
2392 	device = btrfs_find_device(fs_info->fs_devices, &args);
2393 	btrfs_put_dev_args_from_path(&args);
2394 	if (!device)
2395 		return ERR_PTR(-ENOENT);
2396 	return device;
2397 }
2398 
2399 static struct btrfs_fs_devices *btrfs_init_sprout(struct btrfs_fs_info *fs_info)
2400 {
2401 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2402 	struct btrfs_fs_devices *old_devices;
2403 	struct btrfs_fs_devices *seed_devices;
2404 
2405 	lockdep_assert_held(&uuid_mutex);
2406 	if (!fs_devices->seeding)
2407 		return ERR_PTR(-EINVAL);
2408 
2409 	/*
2410 	 * Private copy of the seed devices, anchored at
2411 	 * fs_info->fs_devices->seed_list
2412 	 */
2413 	seed_devices = alloc_fs_devices(NULL, NULL);
2414 	if (IS_ERR(seed_devices))
2415 		return seed_devices;
2416 
2417 	/*
2418 	 * It's necessary to retain a copy of the original seed fs_devices in
2419 	 * fs_uuids so that filesystems which have been seeded can successfully
2420 	 * reference the seed device from open_seed_devices. This also supports
2421 	 * multiple fs seed.
2422 	 */
2423 	old_devices = clone_fs_devices(fs_devices);
2424 	if (IS_ERR(old_devices)) {
2425 		kfree(seed_devices);
2426 		return old_devices;
2427 	}
2428 
2429 	list_add(&old_devices->fs_list, &fs_uuids);
2430 
2431 	memcpy(seed_devices, fs_devices, sizeof(*seed_devices));
2432 	seed_devices->opened = 1;
2433 	INIT_LIST_HEAD(&seed_devices->devices);
2434 	INIT_LIST_HEAD(&seed_devices->alloc_list);
2435 	mutex_init(&seed_devices->device_list_mutex);
2436 
2437 	return seed_devices;
2438 }
2439 
2440 /*
2441  * Splice seed devices into the sprout fs_devices.
2442  * Generate a new fsid for the sprouted read-write filesystem.
2443  */
2444 static void btrfs_setup_sprout(struct btrfs_fs_info *fs_info,
2445 			       struct btrfs_fs_devices *seed_devices)
2446 {
2447 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2448 	struct btrfs_super_block *disk_super = fs_info->super_copy;
2449 	struct btrfs_device *device;
2450 	u64 super_flags;
2451 
2452 	/*
2453 	 * We are updating the fsid, the thread leading to device_list_add()
2454 	 * could race, so uuid_mutex is needed.
2455 	 */
2456 	lockdep_assert_held(&uuid_mutex);
2457 
2458 	/*
2459 	 * The threads listed below may traverse dev_list but can do that without
2460 	 * device_list_mutex:
2461 	 * - All device ops and balance - as we are in btrfs_exclop_start.
2462 	 * - Various dev_list readers - are using RCU.
2463 	 * - btrfs_ioctl_fitrim() - is using RCU.
2464 	 *
2465 	 * For-read threads as below are using device_list_mutex:
2466 	 * - Readonly scrub btrfs_scrub_dev()
2467 	 * - Readonly scrub btrfs_scrub_progress()
2468 	 * - btrfs_get_dev_stats()
2469 	 */
2470 	lockdep_assert_held(&fs_devices->device_list_mutex);
2471 
2472 	list_splice_init_rcu(&fs_devices->devices, &seed_devices->devices,
2473 			      synchronize_rcu);
2474 	list_for_each_entry(device, &seed_devices->devices, dev_list)
2475 		device->fs_devices = seed_devices;
2476 
2477 	fs_devices->seeding = false;
2478 	fs_devices->num_devices = 0;
2479 	fs_devices->open_devices = 0;
2480 	fs_devices->missing_devices = 0;
2481 	fs_devices->rotating = false;
2482 	list_add(&seed_devices->seed_list, &fs_devices->seed_list);
2483 
2484 	generate_random_uuid(fs_devices->fsid);
2485 	memcpy(fs_devices->metadata_uuid, fs_devices->fsid, BTRFS_FSID_SIZE);
2486 	memcpy(disk_super->fsid, fs_devices->fsid, BTRFS_FSID_SIZE);
2487 
2488 	super_flags = btrfs_super_flags(disk_super) &
2489 		      ~BTRFS_SUPER_FLAG_SEEDING;
2490 	btrfs_set_super_flags(disk_super, super_flags);
2491 }
2492 
2493 /*
2494  * Store the expected generation for seed devices in device items.
2495  */
2496 static int btrfs_finish_sprout(struct btrfs_trans_handle *trans)
2497 {
2498 	BTRFS_DEV_LOOKUP_ARGS(args);
2499 	struct btrfs_fs_info *fs_info = trans->fs_info;
2500 	struct btrfs_root *root = fs_info->chunk_root;
2501 	struct btrfs_path *path;
2502 	struct extent_buffer *leaf;
2503 	struct btrfs_dev_item *dev_item;
2504 	struct btrfs_device *device;
2505 	struct btrfs_key key;
2506 	u8 fs_uuid[BTRFS_FSID_SIZE];
2507 	u8 dev_uuid[BTRFS_UUID_SIZE];
2508 	int ret;
2509 
2510 	path = btrfs_alloc_path();
2511 	if (!path)
2512 		return -ENOMEM;
2513 
2514 	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
2515 	key.offset = 0;
2516 	key.type = BTRFS_DEV_ITEM_KEY;
2517 
2518 	while (1) {
2519 		btrfs_reserve_chunk_metadata(trans, false);
2520 		ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2521 		btrfs_trans_release_chunk_metadata(trans);
2522 		if (ret < 0)
2523 			goto error;
2524 
2525 		leaf = path->nodes[0];
2526 next_slot:
2527 		if (path->slots[0] >= btrfs_header_nritems(leaf)) {
2528 			ret = btrfs_next_leaf(root, path);
2529 			if (ret > 0)
2530 				break;
2531 			if (ret < 0)
2532 				goto error;
2533 			leaf = path->nodes[0];
2534 			btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2535 			btrfs_release_path(path);
2536 			continue;
2537 		}
2538 
2539 		btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2540 		if (key.objectid != BTRFS_DEV_ITEMS_OBJECTID ||
2541 		    key.type != BTRFS_DEV_ITEM_KEY)
2542 			break;
2543 
2544 		dev_item = btrfs_item_ptr(leaf, path->slots[0],
2545 					  struct btrfs_dev_item);
2546 		args.devid = btrfs_device_id(leaf, dev_item);
2547 		read_extent_buffer(leaf, dev_uuid, btrfs_device_uuid(dev_item),
2548 				   BTRFS_UUID_SIZE);
2549 		read_extent_buffer(leaf, fs_uuid, btrfs_device_fsid(dev_item),
2550 				   BTRFS_FSID_SIZE);
2551 		args.uuid = dev_uuid;
2552 		args.fsid = fs_uuid;
2553 		device = btrfs_find_device(fs_info->fs_devices, &args);
2554 		BUG_ON(!device); /* Logic error */
2555 
2556 		if (device->fs_devices->seeding) {
2557 			btrfs_set_device_generation(leaf, dev_item,
2558 						    device->generation);
2559 			btrfs_mark_buffer_dirty(leaf);
2560 		}
2561 
2562 		path->slots[0]++;
2563 		goto next_slot;
2564 	}
2565 	ret = 0;
2566 error:
2567 	btrfs_free_path(path);
2568 	return ret;
2569 }
2570 
2571 int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path)
2572 {
2573 	struct btrfs_root *root = fs_info->dev_root;
2574 	struct btrfs_trans_handle *trans;
2575 	struct btrfs_device *device;
2576 	struct block_device *bdev;
2577 	struct super_block *sb = fs_info->sb;
2578 	struct rcu_string *name;
2579 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2580 	struct btrfs_fs_devices *seed_devices;
2581 	u64 orig_super_total_bytes;
2582 	u64 orig_super_num_devices;
2583 	int ret = 0;
2584 	bool seeding_dev = false;
2585 	bool locked = false;
2586 
2587 	if (sb_rdonly(sb) && !fs_devices->seeding)
2588 		return -EROFS;
2589 
2590 	bdev = blkdev_get_by_path(device_path, FMODE_WRITE | FMODE_EXCL,
2591 				  fs_info->bdev_holder);
2592 	if (IS_ERR(bdev))
2593 		return PTR_ERR(bdev);
2594 
2595 	if (!btrfs_check_device_zone_type(fs_info, bdev)) {
2596 		ret = -EINVAL;
2597 		goto error;
2598 	}
2599 
2600 	if (fs_devices->seeding) {
2601 		seeding_dev = true;
2602 		down_write(&sb->s_umount);
2603 		mutex_lock(&uuid_mutex);
2604 		locked = true;
2605 	}
2606 
2607 	sync_blockdev(bdev);
2608 
2609 	rcu_read_lock();
2610 	list_for_each_entry_rcu(device, &fs_devices->devices, dev_list) {
2611 		if (device->bdev == bdev) {
2612 			ret = -EEXIST;
2613 			rcu_read_unlock();
2614 			goto error;
2615 		}
2616 	}
2617 	rcu_read_unlock();
2618 
2619 	device = btrfs_alloc_device(fs_info, NULL, NULL);
2620 	if (IS_ERR(device)) {
2621 		/* we can safely leave the fs_devices entry around */
2622 		ret = PTR_ERR(device);
2623 		goto error;
2624 	}
2625 
2626 	name = rcu_string_strdup(device_path, GFP_KERNEL);
2627 	if (!name) {
2628 		ret = -ENOMEM;
2629 		goto error_free_device;
2630 	}
2631 	rcu_assign_pointer(device->name, name);
2632 
2633 	device->fs_info = fs_info;
2634 	device->bdev = bdev;
2635 	ret = lookup_bdev(device_path, &device->devt);
2636 	if (ret)
2637 		goto error_free_device;
2638 
2639 	ret = btrfs_get_dev_zone_info(device, false);
2640 	if (ret)
2641 		goto error_free_device;
2642 
2643 	trans = btrfs_start_transaction(root, 0);
2644 	if (IS_ERR(trans)) {
2645 		ret = PTR_ERR(trans);
2646 		goto error_free_zone;
2647 	}
2648 
2649 	set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
2650 	device->generation = trans->transid;
2651 	device->io_width = fs_info->sectorsize;
2652 	device->io_align = fs_info->sectorsize;
2653 	device->sector_size = fs_info->sectorsize;
2654 	device->total_bytes =
2655 		round_down(bdev_nr_bytes(bdev), fs_info->sectorsize);
2656 	device->disk_total_bytes = device->total_bytes;
2657 	device->commit_total_bytes = device->total_bytes;
2658 	set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
2659 	clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state);
2660 	device->mode = FMODE_EXCL;
2661 	device->dev_stats_valid = 1;
2662 	set_blocksize(device->bdev, BTRFS_BDEV_BLOCKSIZE);
2663 
2664 	if (seeding_dev) {
2665 		btrfs_clear_sb_rdonly(sb);
2666 
2667 		/* GFP_KERNEL allocation must not be under device_list_mutex */
2668 		seed_devices = btrfs_init_sprout(fs_info);
2669 		if (IS_ERR(seed_devices)) {
2670 			ret = PTR_ERR(seed_devices);
2671 			btrfs_abort_transaction(trans, ret);
2672 			goto error_trans;
2673 		}
2674 	}
2675 
2676 	mutex_lock(&fs_devices->device_list_mutex);
2677 	if (seeding_dev) {
2678 		btrfs_setup_sprout(fs_info, seed_devices);
2679 		btrfs_assign_next_active_device(fs_info->fs_devices->latest_dev,
2680 						device);
2681 	}
2682 
2683 	device->fs_devices = fs_devices;
2684 
2685 	mutex_lock(&fs_info->chunk_mutex);
2686 	list_add_rcu(&device->dev_list, &fs_devices->devices);
2687 	list_add(&device->dev_alloc_list, &fs_devices->alloc_list);
2688 	fs_devices->num_devices++;
2689 	fs_devices->open_devices++;
2690 	fs_devices->rw_devices++;
2691 	fs_devices->total_devices++;
2692 	fs_devices->total_rw_bytes += device->total_bytes;
2693 
2694 	atomic64_add(device->total_bytes, &fs_info->free_chunk_space);
2695 
2696 	if (!bdev_nonrot(bdev))
2697 		fs_devices->rotating = true;
2698 
2699 	orig_super_total_bytes = btrfs_super_total_bytes(fs_info->super_copy);
2700 	btrfs_set_super_total_bytes(fs_info->super_copy,
2701 		round_down(orig_super_total_bytes + device->total_bytes,
2702 			   fs_info->sectorsize));
2703 
2704 	orig_super_num_devices = btrfs_super_num_devices(fs_info->super_copy);
2705 	btrfs_set_super_num_devices(fs_info->super_copy,
2706 				    orig_super_num_devices + 1);
2707 
2708 	/*
2709 	 * we've got more storage, clear any full flags on the space
2710 	 * infos
2711 	 */
2712 	btrfs_clear_space_info_full(fs_info);
2713 
2714 	mutex_unlock(&fs_info->chunk_mutex);
2715 
2716 	/* Add sysfs device entry */
2717 	btrfs_sysfs_add_device(device);
2718 
2719 	mutex_unlock(&fs_devices->device_list_mutex);
2720 
2721 	if (seeding_dev) {
2722 		mutex_lock(&fs_info->chunk_mutex);
2723 		ret = init_first_rw_device(trans);
2724 		mutex_unlock(&fs_info->chunk_mutex);
2725 		if (ret) {
2726 			btrfs_abort_transaction(trans, ret);
2727 			goto error_sysfs;
2728 		}
2729 	}
2730 
2731 	ret = btrfs_add_dev_item(trans, device);
2732 	if (ret) {
2733 		btrfs_abort_transaction(trans, ret);
2734 		goto error_sysfs;
2735 	}
2736 
2737 	if (seeding_dev) {
2738 		ret = btrfs_finish_sprout(trans);
2739 		if (ret) {
2740 			btrfs_abort_transaction(trans, ret);
2741 			goto error_sysfs;
2742 		}
2743 
2744 		/*
2745 		 * fs_devices now represents the newly sprouted filesystem and
2746 		 * its fsid has been changed by btrfs_sprout_splice().
2747 		 */
2748 		btrfs_sysfs_update_sprout_fsid(fs_devices);
2749 	}
2750 
2751 	ret = btrfs_commit_transaction(trans);
2752 
2753 	if (seeding_dev) {
2754 		mutex_unlock(&uuid_mutex);
2755 		up_write(&sb->s_umount);
2756 		locked = false;
2757 
2758 		if (ret) /* transaction commit */
2759 			return ret;
2760 
2761 		ret = btrfs_relocate_sys_chunks(fs_info);
2762 		if (ret < 0)
2763 			btrfs_handle_fs_error(fs_info, ret,
2764 				    "Failed to relocate sys chunks after device initialization. This can be fixed using the \"btrfs balance\" command.");
2765 		trans = btrfs_attach_transaction(root);
2766 		if (IS_ERR(trans)) {
2767 			if (PTR_ERR(trans) == -ENOENT)
2768 				return 0;
2769 			ret = PTR_ERR(trans);
2770 			trans = NULL;
2771 			goto error_sysfs;
2772 		}
2773 		ret = btrfs_commit_transaction(trans);
2774 	}
2775 
2776 	/*
2777 	 * Now that we have written a new super block to this device, check all
2778 	 * other fs_devices list if device_path alienates any other scanned
2779 	 * device.
2780 	 * We can ignore the return value as it typically returns -EINVAL and
2781 	 * only succeeds if the device was an alien.
2782 	 */
2783 	btrfs_forget_devices(device->devt);
2784 
2785 	/* Update ctime/mtime for blkid or udev */
2786 	update_dev_time(device_path);
2787 
2788 	return ret;
2789 
2790 error_sysfs:
2791 	btrfs_sysfs_remove_device(device);
2792 	mutex_lock(&fs_info->fs_devices->device_list_mutex);
2793 	mutex_lock(&fs_info->chunk_mutex);
2794 	list_del_rcu(&device->dev_list);
2795 	list_del(&device->dev_alloc_list);
2796 	fs_info->fs_devices->num_devices--;
2797 	fs_info->fs_devices->open_devices--;
2798 	fs_info->fs_devices->rw_devices--;
2799 	fs_info->fs_devices->total_devices--;
2800 	fs_info->fs_devices->total_rw_bytes -= device->total_bytes;
2801 	atomic64_sub(device->total_bytes, &fs_info->free_chunk_space);
2802 	btrfs_set_super_total_bytes(fs_info->super_copy,
2803 				    orig_super_total_bytes);
2804 	btrfs_set_super_num_devices(fs_info->super_copy,
2805 				    orig_super_num_devices);
2806 	mutex_unlock(&fs_info->chunk_mutex);
2807 	mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2808 error_trans:
2809 	if (seeding_dev)
2810 		btrfs_set_sb_rdonly(sb);
2811 	if (trans)
2812 		btrfs_end_transaction(trans);
2813 error_free_zone:
2814 	btrfs_destroy_dev_zone_info(device);
2815 error_free_device:
2816 	btrfs_free_device(device);
2817 error:
2818 	blkdev_put(bdev, FMODE_EXCL);
2819 	if (locked) {
2820 		mutex_unlock(&uuid_mutex);
2821 		up_write(&sb->s_umount);
2822 	}
2823 	return ret;
2824 }
2825 
2826 static noinline int btrfs_update_device(struct btrfs_trans_handle *trans,
2827 					struct btrfs_device *device)
2828 {
2829 	int ret;
2830 	struct btrfs_path *path;
2831 	struct btrfs_root *root = device->fs_info->chunk_root;
2832 	struct btrfs_dev_item *dev_item;
2833 	struct extent_buffer *leaf;
2834 	struct btrfs_key key;
2835 
2836 	path = btrfs_alloc_path();
2837 	if (!path)
2838 		return -ENOMEM;
2839 
2840 	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
2841 	key.type = BTRFS_DEV_ITEM_KEY;
2842 	key.offset = device->devid;
2843 
2844 	ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2845 	if (ret < 0)
2846 		goto out;
2847 
2848 	if (ret > 0) {
2849 		ret = -ENOENT;
2850 		goto out;
2851 	}
2852 
2853 	leaf = path->nodes[0];
2854 	dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item);
2855 
2856 	btrfs_set_device_id(leaf, dev_item, device->devid);
2857 	btrfs_set_device_type(leaf, dev_item, device->type);
2858 	btrfs_set_device_io_align(leaf, dev_item, device->io_align);
2859 	btrfs_set_device_io_width(leaf, dev_item, device->io_width);
2860 	btrfs_set_device_sector_size(leaf, dev_item, device->sector_size);
2861 	btrfs_set_device_total_bytes(leaf, dev_item,
2862 				     btrfs_device_get_disk_total_bytes(device));
2863 	btrfs_set_device_bytes_used(leaf, dev_item,
2864 				    btrfs_device_get_bytes_used(device));
2865 	btrfs_mark_buffer_dirty(leaf);
2866 
2867 out:
2868 	btrfs_free_path(path);
2869 	return ret;
2870 }
2871 
2872 int btrfs_grow_device(struct btrfs_trans_handle *trans,
2873 		      struct btrfs_device *device, u64 new_size)
2874 {
2875 	struct btrfs_fs_info *fs_info = device->fs_info;
2876 	struct btrfs_super_block *super_copy = fs_info->super_copy;
2877 	u64 old_total;
2878 	u64 diff;
2879 	int ret;
2880 
2881 	if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state))
2882 		return -EACCES;
2883 
2884 	new_size = round_down(new_size, fs_info->sectorsize);
2885 
2886 	mutex_lock(&fs_info->chunk_mutex);
2887 	old_total = btrfs_super_total_bytes(super_copy);
2888 	diff = round_down(new_size - device->total_bytes, fs_info->sectorsize);
2889 
2890 	if (new_size <= device->total_bytes ||
2891 	    test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
2892 		mutex_unlock(&fs_info->chunk_mutex);
2893 		return -EINVAL;
2894 	}
2895 
2896 	btrfs_set_super_total_bytes(super_copy,
2897 			round_down(old_total + diff, fs_info->sectorsize));
2898 	device->fs_devices->total_rw_bytes += diff;
2899 
2900 	btrfs_device_set_total_bytes(device, new_size);
2901 	btrfs_device_set_disk_total_bytes(device, new_size);
2902 	btrfs_clear_space_info_full(device->fs_info);
2903 	if (list_empty(&device->post_commit_list))
2904 		list_add_tail(&device->post_commit_list,
2905 			      &trans->transaction->dev_update_list);
2906 	mutex_unlock(&fs_info->chunk_mutex);
2907 
2908 	btrfs_reserve_chunk_metadata(trans, false);
2909 	ret = btrfs_update_device(trans, device);
2910 	btrfs_trans_release_chunk_metadata(trans);
2911 
2912 	return ret;
2913 }
2914 
2915 static int btrfs_free_chunk(struct btrfs_trans_handle *trans, u64 chunk_offset)
2916 {
2917 	struct btrfs_fs_info *fs_info = trans->fs_info;
2918 	struct btrfs_root *root = fs_info->chunk_root;
2919 	int ret;
2920 	struct btrfs_path *path;
2921 	struct btrfs_key key;
2922 
2923 	path = btrfs_alloc_path();
2924 	if (!path)
2925 		return -ENOMEM;
2926 
2927 	key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
2928 	key.offset = chunk_offset;
2929 	key.type = BTRFS_CHUNK_ITEM_KEY;
2930 
2931 	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
2932 	if (ret < 0)
2933 		goto out;
2934 	else if (ret > 0) { /* Logic error or corruption */
2935 		btrfs_handle_fs_error(fs_info, -ENOENT,
2936 				      "Failed lookup while freeing chunk.");
2937 		ret = -ENOENT;
2938 		goto out;
2939 	}
2940 
2941 	ret = btrfs_del_item(trans, root, path);
2942 	if (ret < 0)
2943 		btrfs_handle_fs_error(fs_info, ret,
2944 				      "Failed to delete chunk item.");
2945 out:
2946 	btrfs_free_path(path);
2947 	return ret;
2948 }
2949 
2950 static int btrfs_del_sys_chunk(struct btrfs_fs_info *fs_info, u64 chunk_offset)
2951 {
2952 	struct btrfs_super_block *super_copy = fs_info->super_copy;
2953 	struct btrfs_disk_key *disk_key;
2954 	struct btrfs_chunk *chunk;
2955 	u8 *ptr;
2956 	int ret = 0;
2957 	u32 num_stripes;
2958 	u32 array_size;
2959 	u32 len = 0;
2960 	u32 cur;
2961 	struct btrfs_key key;
2962 
2963 	lockdep_assert_held(&fs_info->chunk_mutex);
2964 	array_size = btrfs_super_sys_array_size(super_copy);
2965 
2966 	ptr = super_copy->sys_chunk_array;
2967 	cur = 0;
2968 
2969 	while (cur < array_size) {
2970 		disk_key = (struct btrfs_disk_key *)ptr;
2971 		btrfs_disk_key_to_cpu(&key, disk_key);
2972 
2973 		len = sizeof(*disk_key);
2974 
2975 		if (key.type == BTRFS_CHUNK_ITEM_KEY) {
2976 			chunk = (struct btrfs_chunk *)(ptr + len);
2977 			num_stripes = btrfs_stack_chunk_num_stripes(chunk);
2978 			len += btrfs_chunk_item_size(num_stripes);
2979 		} else {
2980 			ret = -EIO;
2981 			break;
2982 		}
2983 		if (key.objectid == BTRFS_FIRST_CHUNK_TREE_OBJECTID &&
2984 		    key.offset == chunk_offset) {
2985 			memmove(ptr, ptr + len, array_size - (cur + len));
2986 			array_size -= len;
2987 			btrfs_set_super_sys_array_size(super_copy, array_size);
2988 		} else {
2989 			ptr += len;
2990 			cur += len;
2991 		}
2992 	}
2993 	return ret;
2994 }
2995 
2996 /*
2997  * btrfs_get_chunk_map() - Find the mapping containing the given logical extent.
2998  * @logical: Logical block offset in bytes.
2999  * @length: Length of extent in bytes.
3000  *
3001  * Return: Chunk mapping or ERR_PTR.
3002  */
3003 struct extent_map *btrfs_get_chunk_map(struct btrfs_fs_info *fs_info,
3004 				       u64 logical, u64 length)
3005 {
3006 	struct extent_map_tree *em_tree;
3007 	struct extent_map *em;
3008 
3009 	em_tree = &fs_info->mapping_tree;
3010 	read_lock(&em_tree->lock);
3011 	em = lookup_extent_mapping(em_tree, logical, length);
3012 	read_unlock(&em_tree->lock);
3013 
3014 	if (!em) {
3015 		btrfs_crit(fs_info, "unable to find logical %llu length %llu",
3016 			   logical, length);
3017 		return ERR_PTR(-EINVAL);
3018 	}
3019 
3020 	if (em->start > logical || em->start + em->len < logical) {
3021 		btrfs_crit(fs_info,
3022 			   "found a bad mapping, wanted %llu-%llu, found %llu-%llu",
3023 			   logical, length, em->start, em->start + em->len);
3024 		free_extent_map(em);
3025 		return ERR_PTR(-EINVAL);
3026 	}
3027 
3028 	/* callers are responsible for dropping em's ref. */
3029 	return em;
3030 }
3031 
3032 static int remove_chunk_item(struct btrfs_trans_handle *trans,
3033 			     struct map_lookup *map, u64 chunk_offset)
3034 {
3035 	int i;
3036 
3037 	/*
3038 	 * Removing chunk items and updating the device items in the chunks btree
3039 	 * requires holding the chunk_mutex.
3040 	 * See the comment at btrfs_chunk_alloc() for the details.
3041 	 */
3042 	lockdep_assert_held(&trans->fs_info->chunk_mutex);
3043 
3044 	for (i = 0; i < map->num_stripes; i++) {
3045 		int ret;
3046 
3047 		ret = btrfs_update_device(trans, map->stripes[i].dev);
3048 		if (ret)
3049 			return ret;
3050 	}
3051 
3052 	return btrfs_free_chunk(trans, chunk_offset);
3053 }
3054 
3055 int btrfs_remove_chunk(struct btrfs_trans_handle *trans, u64 chunk_offset)
3056 {
3057 	struct btrfs_fs_info *fs_info = trans->fs_info;
3058 	struct extent_map *em;
3059 	struct map_lookup *map;
3060 	u64 dev_extent_len = 0;
3061 	int i, ret = 0;
3062 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
3063 
3064 	em = btrfs_get_chunk_map(fs_info, chunk_offset, 1);
3065 	if (IS_ERR(em)) {
3066 		/*
3067 		 * This is a logic error, but we don't want to just rely on the
3068 		 * user having built with ASSERT enabled, so if ASSERT doesn't
3069 		 * do anything we still error out.
3070 		 */
3071 		ASSERT(0);
3072 		return PTR_ERR(em);
3073 	}
3074 	map = em->map_lookup;
3075 
3076 	/*
3077 	 * First delete the device extent items from the devices btree.
3078 	 * We take the device_list_mutex to avoid racing with the finishing phase
3079 	 * of a device replace operation. See the comment below before acquiring
3080 	 * fs_info->chunk_mutex. Note that here we do not acquire the chunk_mutex
3081 	 * because that can result in a deadlock when deleting the device extent
3082 	 * items from the devices btree - COWing an extent buffer from the btree
3083 	 * may result in allocating a new metadata chunk, which would attempt to
3084 	 * lock again fs_info->chunk_mutex.
3085 	 */
3086 	mutex_lock(&fs_devices->device_list_mutex);
3087 	for (i = 0; i < map->num_stripes; i++) {
3088 		struct btrfs_device *device = map->stripes[i].dev;
3089 		ret = btrfs_free_dev_extent(trans, device,
3090 					    map->stripes[i].physical,
3091 					    &dev_extent_len);
3092 		if (ret) {
3093 			mutex_unlock(&fs_devices->device_list_mutex);
3094 			btrfs_abort_transaction(trans, ret);
3095 			goto out;
3096 		}
3097 
3098 		if (device->bytes_used > 0) {
3099 			mutex_lock(&fs_info->chunk_mutex);
3100 			btrfs_device_set_bytes_used(device,
3101 					device->bytes_used - dev_extent_len);
3102 			atomic64_add(dev_extent_len, &fs_info->free_chunk_space);
3103 			btrfs_clear_space_info_full(fs_info);
3104 			mutex_unlock(&fs_info->chunk_mutex);
3105 		}
3106 	}
3107 	mutex_unlock(&fs_devices->device_list_mutex);
3108 
3109 	/*
3110 	 * We acquire fs_info->chunk_mutex for 2 reasons:
3111 	 *
3112 	 * 1) Just like with the first phase of the chunk allocation, we must
3113 	 *    reserve system space, do all chunk btree updates and deletions, and
3114 	 *    update the system chunk array in the superblock while holding this
3115 	 *    mutex. This is for similar reasons as explained on the comment at
3116 	 *    the top of btrfs_chunk_alloc();
3117 	 *
3118 	 * 2) Prevent races with the final phase of a device replace operation
3119 	 *    that replaces the device object associated with the map's stripes,
3120 	 *    because the device object's id can change at any time during that
3121 	 *    final phase of the device replace operation
3122 	 *    (dev-replace.c:btrfs_dev_replace_finishing()), so we could grab the
3123 	 *    replaced device and then see it with an ID of
3124 	 *    BTRFS_DEV_REPLACE_DEVID, which would cause a failure when updating
3125 	 *    the device item, which does not exists on the chunk btree.
3126 	 *    The finishing phase of device replace acquires both the
3127 	 *    device_list_mutex and the chunk_mutex, in that order, so we are
3128 	 *    safe by just acquiring the chunk_mutex.
3129 	 */
3130 	trans->removing_chunk = true;
3131 	mutex_lock(&fs_info->chunk_mutex);
3132 
3133 	check_system_chunk(trans, map->type);
3134 
3135 	ret = remove_chunk_item(trans, map, chunk_offset);
3136 	/*
3137 	 * Normally we should not get -ENOSPC since we reserved space before
3138 	 * through the call to check_system_chunk().
3139 	 *
3140 	 * Despite our system space_info having enough free space, we may not
3141 	 * be able to allocate extents from its block groups, because all have
3142 	 * an incompatible profile, which will force us to allocate a new system
3143 	 * block group with the right profile, or right after we called
3144 	 * check_system_space() above, a scrub turned the only system block group
3145 	 * with enough free space into RO mode.
3146 	 * This is explained with more detail at do_chunk_alloc().
3147 	 *
3148 	 * So if we get -ENOSPC, allocate a new system chunk and retry once.
3149 	 */
3150 	if (ret == -ENOSPC) {
3151 		const u64 sys_flags = btrfs_system_alloc_profile(fs_info);
3152 		struct btrfs_block_group *sys_bg;
3153 
3154 		sys_bg = btrfs_create_chunk(trans, sys_flags);
3155 		if (IS_ERR(sys_bg)) {
3156 			ret = PTR_ERR(sys_bg);
3157 			btrfs_abort_transaction(trans, ret);
3158 			goto out;
3159 		}
3160 
3161 		ret = btrfs_chunk_alloc_add_chunk_item(trans, sys_bg);
3162 		if (ret) {
3163 			btrfs_abort_transaction(trans, ret);
3164 			goto out;
3165 		}
3166 
3167 		ret = remove_chunk_item(trans, map, chunk_offset);
3168 		if (ret) {
3169 			btrfs_abort_transaction(trans, ret);
3170 			goto out;
3171 		}
3172 	} else if (ret) {
3173 		btrfs_abort_transaction(trans, ret);
3174 		goto out;
3175 	}
3176 
3177 	trace_btrfs_chunk_free(fs_info, map, chunk_offset, em->len);
3178 
3179 	if (map->type & BTRFS_BLOCK_GROUP_SYSTEM) {
3180 		ret = btrfs_del_sys_chunk(fs_info, chunk_offset);
3181 		if (ret) {
3182 			btrfs_abort_transaction(trans, ret);
3183 			goto out;
3184 		}
3185 	}
3186 
3187 	mutex_unlock(&fs_info->chunk_mutex);
3188 	trans->removing_chunk = false;
3189 
3190 	/*
3191 	 * We are done with chunk btree updates and deletions, so release the
3192 	 * system space we previously reserved (with check_system_chunk()).
3193 	 */
3194 	btrfs_trans_release_chunk_metadata(trans);
3195 
3196 	ret = btrfs_remove_block_group(trans, chunk_offset, em);
3197 	if (ret) {
3198 		btrfs_abort_transaction(trans, ret);
3199 		goto out;
3200 	}
3201 
3202 out:
3203 	if (trans->removing_chunk) {
3204 		mutex_unlock(&fs_info->chunk_mutex);
3205 		trans->removing_chunk = false;
3206 	}
3207 	/* once for us */
3208 	free_extent_map(em);
3209 	return ret;
3210 }
3211 
3212 int btrfs_relocate_chunk(struct btrfs_fs_info *fs_info, u64 chunk_offset)
3213 {
3214 	struct btrfs_root *root = fs_info->chunk_root;
3215 	struct btrfs_trans_handle *trans;
3216 	struct btrfs_block_group *block_group;
3217 	u64 length;
3218 	int ret;
3219 
3220 	if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) {
3221 		btrfs_err(fs_info,
3222 			  "relocate: not supported on extent tree v2 yet");
3223 		return -EINVAL;
3224 	}
3225 
3226 	/*
3227 	 * Prevent races with automatic removal of unused block groups.
3228 	 * After we relocate and before we remove the chunk with offset
3229 	 * chunk_offset, automatic removal of the block group can kick in,
3230 	 * resulting in a failure when calling btrfs_remove_chunk() below.
3231 	 *
3232 	 * Make sure to acquire this mutex before doing a tree search (dev
3233 	 * or chunk trees) to find chunks. Otherwise the cleaner kthread might
3234 	 * call btrfs_remove_chunk() (through btrfs_delete_unused_bgs()) after
3235 	 * we release the path used to search the chunk/dev tree and before
3236 	 * the current task acquires this mutex and calls us.
3237 	 */
3238 	lockdep_assert_held(&fs_info->reclaim_bgs_lock);
3239 
3240 	/* step one, relocate all the extents inside this chunk */
3241 	btrfs_scrub_pause(fs_info);
3242 	ret = btrfs_relocate_block_group(fs_info, chunk_offset);
3243 	btrfs_scrub_continue(fs_info);
3244 	if (ret)
3245 		return ret;
3246 
3247 	block_group = btrfs_lookup_block_group(fs_info, chunk_offset);
3248 	if (!block_group)
3249 		return -ENOENT;
3250 	btrfs_discard_cancel_work(&fs_info->discard_ctl, block_group);
3251 	length = block_group->length;
3252 	btrfs_put_block_group(block_group);
3253 
3254 	/*
3255 	 * On a zoned file system, discard the whole block group, this will
3256 	 * trigger a REQ_OP_ZONE_RESET operation on the device zone. If
3257 	 * resetting the zone fails, don't treat it as a fatal problem from the
3258 	 * filesystem's point of view.
3259 	 */
3260 	if (btrfs_is_zoned(fs_info)) {
3261 		ret = btrfs_discard_extent(fs_info, chunk_offset, length, NULL);
3262 		if (ret)
3263 			btrfs_info(fs_info,
3264 				"failed to reset zone %llu after relocation",
3265 				chunk_offset);
3266 	}
3267 
3268 	trans = btrfs_start_trans_remove_block_group(root->fs_info,
3269 						     chunk_offset);
3270 	if (IS_ERR(trans)) {
3271 		ret = PTR_ERR(trans);
3272 		btrfs_handle_fs_error(root->fs_info, ret, NULL);
3273 		return ret;
3274 	}
3275 
3276 	/*
3277 	 * step two, delete the device extents and the
3278 	 * chunk tree entries
3279 	 */
3280 	ret = btrfs_remove_chunk(trans, chunk_offset);
3281 	btrfs_end_transaction(trans);
3282 	return ret;
3283 }
3284 
3285 static int btrfs_relocate_sys_chunks(struct btrfs_fs_info *fs_info)
3286 {
3287 	struct btrfs_root *chunk_root = fs_info->chunk_root;
3288 	struct btrfs_path *path;
3289 	struct extent_buffer *leaf;
3290 	struct btrfs_chunk *chunk;
3291 	struct btrfs_key key;
3292 	struct btrfs_key found_key;
3293 	u64 chunk_type;
3294 	bool retried = false;
3295 	int failed = 0;
3296 	int ret;
3297 
3298 	path = btrfs_alloc_path();
3299 	if (!path)
3300 		return -ENOMEM;
3301 
3302 again:
3303 	key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
3304 	key.offset = (u64)-1;
3305 	key.type = BTRFS_CHUNK_ITEM_KEY;
3306 
3307 	while (1) {
3308 		mutex_lock(&fs_info->reclaim_bgs_lock);
3309 		ret = btrfs_search_slot(NULL, chunk_root, &key, path, 0, 0);
3310 		if (ret < 0) {
3311 			mutex_unlock(&fs_info->reclaim_bgs_lock);
3312 			goto error;
3313 		}
3314 		BUG_ON(ret == 0); /* Corruption */
3315 
3316 		ret = btrfs_previous_item(chunk_root, path, key.objectid,
3317 					  key.type);
3318 		if (ret)
3319 			mutex_unlock(&fs_info->reclaim_bgs_lock);
3320 		if (ret < 0)
3321 			goto error;
3322 		if (ret > 0)
3323 			break;
3324 
3325 		leaf = path->nodes[0];
3326 		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
3327 
3328 		chunk = btrfs_item_ptr(leaf, path->slots[0],
3329 				       struct btrfs_chunk);
3330 		chunk_type = btrfs_chunk_type(leaf, chunk);
3331 		btrfs_release_path(path);
3332 
3333 		if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM) {
3334 			ret = btrfs_relocate_chunk(fs_info, found_key.offset);
3335 			if (ret == -ENOSPC)
3336 				failed++;
3337 			else
3338 				BUG_ON(ret);
3339 		}
3340 		mutex_unlock(&fs_info->reclaim_bgs_lock);
3341 
3342 		if (found_key.offset == 0)
3343 			break;
3344 		key.offset = found_key.offset - 1;
3345 	}
3346 	ret = 0;
3347 	if (failed && !retried) {
3348 		failed = 0;
3349 		retried = true;
3350 		goto again;
3351 	} else if (WARN_ON(failed && retried)) {
3352 		ret = -ENOSPC;
3353 	}
3354 error:
3355 	btrfs_free_path(path);
3356 	return ret;
3357 }
3358 
3359 /*
3360  * return 1 : allocate a data chunk successfully,
3361  * return <0: errors during allocating a data chunk,
3362  * return 0 : no need to allocate a data chunk.
3363  */
3364 static int btrfs_may_alloc_data_chunk(struct btrfs_fs_info *fs_info,
3365 				      u64 chunk_offset)
3366 {
3367 	struct btrfs_block_group *cache;
3368 	u64 bytes_used;
3369 	u64 chunk_type;
3370 
3371 	cache = btrfs_lookup_block_group(fs_info, chunk_offset);
3372 	ASSERT(cache);
3373 	chunk_type = cache->flags;
3374 	btrfs_put_block_group(cache);
3375 
3376 	if (!(chunk_type & BTRFS_BLOCK_GROUP_DATA))
3377 		return 0;
3378 
3379 	spin_lock(&fs_info->data_sinfo->lock);
3380 	bytes_used = fs_info->data_sinfo->bytes_used;
3381 	spin_unlock(&fs_info->data_sinfo->lock);
3382 
3383 	if (!bytes_used) {
3384 		struct btrfs_trans_handle *trans;
3385 		int ret;
3386 
3387 		trans =	btrfs_join_transaction(fs_info->tree_root);
3388 		if (IS_ERR(trans))
3389 			return PTR_ERR(trans);
3390 
3391 		ret = btrfs_force_chunk_alloc(trans, BTRFS_BLOCK_GROUP_DATA);
3392 		btrfs_end_transaction(trans);
3393 		if (ret < 0)
3394 			return ret;
3395 		return 1;
3396 	}
3397 
3398 	return 0;
3399 }
3400 
3401 static int insert_balance_item(struct btrfs_fs_info *fs_info,
3402 			       struct btrfs_balance_control *bctl)
3403 {
3404 	struct btrfs_root *root = fs_info->tree_root;
3405 	struct btrfs_trans_handle *trans;
3406 	struct btrfs_balance_item *item;
3407 	struct btrfs_disk_balance_args disk_bargs;
3408 	struct btrfs_path *path;
3409 	struct extent_buffer *leaf;
3410 	struct btrfs_key key;
3411 	int ret, err;
3412 
3413 	path = btrfs_alloc_path();
3414 	if (!path)
3415 		return -ENOMEM;
3416 
3417 	trans = btrfs_start_transaction(root, 0);
3418 	if (IS_ERR(trans)) {
3419 		btrfs_free_path(path);
3420 		return PTR_ERR(trans);
3421 	}
3422 
3423 	key.objectid = BTRFS_BALANCE_OBJECTID;
3424 	key.type = BTRFS_TEMPORARY_ITEM_KEY;
3425 	key.offset = 0;
3426 
3427 	ret = btrfs_insert_empty_item(trans, root, path, &key,
3428 				      sizeof(*item));
3429 	if (ret)
3430 		goto out;
3431 
3432 	leaf = path->nodes[0];
3433 	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_balance_item);
3434 
3435 	memzero_extent_buffer(leaf, (unsigned long)item, sizeof(*item));
3436 
3437 	btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->data);
3438 	btrfs_set_balance_data(leaf, item, &disk_bargs);
3439 	btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->meta);
3440 	btrfs_set_balance_meta(leaf, item, &disk_bargs);
3441 	btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->sys);
3442 	btrfs_set_balance_sys(leaf, item, &disk_bargs);
3443 
3444 	btrfs_set_balance_flags(leaf, item, bctl->flags);
3445 
3446 	btrfs_mark_buffer_dirty(leaf);
3447 out:
3448 	btrfs_free_path(path);
3449 	err = btrfs_commit_transaction(trans);
3450 	if (err && !ret)
3451 		ret = err;
3452 	return ret;
3453 }
3454 
3455 static int del_balance_item(struct btrfs_fs_info *fs_info)
3456 {
3457 	struct btrfs_root *root = fs_info->tree_root;
3458 	struct btrfs_trans_handle *trans;
3459 	struct btrfs_path *path;
3460 	struct btrfs_key key;
3461 	int ret, err;
3462 
3463 	path = btrfs_alloc_path();
3464 	if (!path)
3465 		return -ENOMEM;
3466 
3467 	trans = btrfs_start_transaction_fallback_global_rsv(root, 0);
3468 	if (IS_ERR(trans)) {
3469 		btrfs_free_path(path);
3470 		return PTR_ERR(trans);
3471 	}
3472 
3473 	key.objectid = BTRFS_BALANCE_OBJECTID;
3474 	key.type = BTRFS_TEMPORARY_ITEM_KEY;
3475 	key.offset = 0;
3476 
3477 	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
3478 	if (ret < 0)
3479 		goto out;
3480 	if (ret > 0) {
3481 		ret = -ENOENT;
3482 		goto out;
3483 	}
3484 
3485 	ret = btrfs_del_item(trans, root, path);
3486 out:
3487 	btrfs_free_path(path);
3488 	err = btrfs_commit_transaction(trans);
3489 	if (err && !ret)
3490 		ret = err;
3491 	return ret;
3492 }
3493 
3494 /*
3495  * This is a heuristic used to reduce the number of chunks balanced on
3496  * resume after balance was interrupted.
3497  */
3498 static void update_balance_args(struct btrfs_balance_control *bctl)
3499 {
3500 	/*
3501 	 * Turn on soft mode for chunk types that were being converted.
3502 	 */
3503 	if (bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT)
3504 		bctl->data.flags |= BTRFS_BALANCE_ARGS_SOFT;
3505 	if (bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT)
3506 		bctl->sys.flags |= BTRFS_BALANCE_ARGS_SOFT;
3507 	if (bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT)
3508 		bctl->meta.flags |= BTRFS_BALANCE_ARGS_SOFT;
3509 
3510 	/*
3511 	 * Turn on usage filter if is not already used.  The idea is
3512 	 * that chunks that we have already balanced should be
3513 	 * reasonably full.  Don't do it for chunks that are being
3514 	 * converted - that will keep us from relocating unconverted
3515 	 * (albeit full) chunks.
3516 	 */
3517 	if (!(bctl->data.flags & BTRFS_BALANCE_ARGS_USAGE) &&
3518 	    !(bctl->data.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
3519 	    !(bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT)) {
3520 		bctl->data.flags |= BTRFS_BALANCE_ARGS_USAGE;
3521 		bctl->data.usage = 90;
3522 	}
3523 	if (!(bctl->sys.flags & BTRFS_BALANCE_ARGS_USAGE) &&
3524 	    !(bctl->sys.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
3525 	    !(bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT)) {
3526 		bctl->sys.flags |= BTRFS_BALANCE_ARGS_USAGE;
3527 		bctl->sys.usage = 90;
3528 	}
3529 	if (!(bctl->meta.flags & BTRFS_BALANCE_ARGS_USAGE) &&
3530 	    !(bctl->meta.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
3531 	    !(bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT)) {
3532 		bctl->meta.flags |= BTRFS_BALANCE_ARGS_USAGE;
3533 		bctl->meta.usage = 90;
3534 	}
3535 }
3536 
3537 /*
3538  * Clear the balance status in fs_info and delete the balance item from disk.
3539  */
3540 static void reset_balance_state(struct btrfs_fs_info *fs_info)
3541 {
3542 	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
3543 	int ret;
3544 
3545 	BUG_ON(!fs_info->balance_ctl);
3546 
3547 	spin_lock(&fs_info->balance_lock);
3548 	fs_info->balance_ctl = NULL;
3549 	spin_unlock(&fs_info->balance_lock);
3550 
3551 	kfree(bctl);
3552 	ret = del_balance_item(fs_info);
3553 	if (ret)
3554 		btrfs_handle_fs_error(fs_info, ret, NULL);
3555 }
3556 
3557 /*
3558  * Balance filters.  Return 1 if chunk should be filtered out
3559  * (should not be balanced).
3560  */
3561 static int chunk_profiles_filter(u64 chunk_type,
3562 				 struct btrfs_balance_args *bargs)
3563 {
3564 	chunk_type = chunk_to_extended(chunk_type) &
3565 				BTRFS_EXTENDED_PROFILE_MASK;
3566 
3567 	if (bargs->profiles & chunk_type)
3568 		return 0;
3569 
3570 	return 1;
3571 }
3572 
3573 static int chunk_usage_range_filter(struct btrfs_fs_info *fs_info, u64 chunk_offset,
3574 			      struct btrfs_balance_args *bargs)
3575 {
3576 	struct btrfs_block_group *cache;
3577 	u64 chunk_used;
3578 	u64 user_thresh_min;
3579 	u64 user_thresh_max;
3580 	int ret = 1;
3581 
3582 	cache = btrfs_lookup_block_group(fs_info, chunk_offset);
3583 	chunk_used = cache->used;
3584 
3585 	if (bargs->usage_min == 0)
3586 		user_thresh_min = 0;
3587 	else
3588 		user_thresh_min = div_factor_fine(cache->length,
3589 						  bargs->usage_min);
3590 
3591 	if (bargs->usage_max == 0)
3592 		user_thresh_max = 1;
3593 	else if (bargs->usage_max > 100)
3594 		user_thresh_max = cache->length;
3595 	else
3596 		user_thresh_max = div_factor_fine(cache->length,
3597 						  bargs->usage_max);
3598 
3599 	if (user_thresh_min <= chunk_used && chunk_used < user_thresh_max)
3600 		ret = 0;
3601 
3602 	btrfs_put_block_group(cache);
3603 	return ret;
3604 }
3605 
3606 static int chunk_usage_filter(struct btrfs_fs_info *fs_info,
3607 		u64 chunk_offset, struct btrfs_balance_args *bargs)
3608 {
3609 	struct btrfs_block_group *cache;
3610 	u64 chunk_used, user_thresh;
3611 	int ret = 1;
3612 
3613 	cache = btrfs_lookup_block_group(fs_info, chunk_offset);
3614 	chunk_used = cache->used;
3615 
3616 	if (bargs->usage_min == 0)
3617 		user_thresh = 1;
3618 	else if (bargs->usage > 100)
3619 		user_thresh = cache->length;
3620 	else
3621 		user_thresh = div_factor_fine(cache->length, bargs->usage);
3622 
3623 	if (chunk_used < user_thresh)
3624 		ret = 0;
3625 
3626 	btrfs_put_block_group(cache);
3627 	return ret;
3628 }
3629 
3630 static int chunk_devid_filter(struct extent_buffer *leaf,
3631 			      struct btrfs_chunk *chunk,
3632 			      struct btrfs_balance_args *bargs)
3633 {
3634 	struct btrfs_stripe *stripe;
3635 	int num_stripes = btrfs_chunk_num_stripes(leaf, chunk);
3636 	int i;
3637 
3638 	for (i = 0; i < num_stripes; i++) {
3639 		stripe = btrfs_stripe_nr(chunk, i);
3640 		if (btrfs_stripe_devid(leaf, stripe) == bargs->devid)
3641 			return 0;
3642 	}
3643 
3644 	return 1;
3645 }
3646 
3647 static u64 calc_data_stripes(u64 type, int num_stripes)
3648 {
3649 	const int index = btrfs_bg_flags_to_raid_index(type);
3650 	const int ncopies = btrfs_raid_array[index].ncopies;
3651 	const int nparity = btrfs_raid_array[index].nparity;
3652 
3653 	return (num_stripes - nparity) / ncopies;
3654 }
3655 
3656 /* [pstart, pend) */
3657 static int chunk_drange_filter(struct extent_buffer *leaf,
3658 			       struct btrfs_chunk *chunk,
3659 			       struct btrfs_balance_args *bargs)
3660 {
3661 	struct btrfs_stripe *stripe;
3662 	int num_stripes = btrfs_chunk_num_stripes(leaf, chunk);
3663 	u64 stripe_offset;
3664 	u64 stripe_length;
3665 	u64 type;
3666 	int factor;
3667 	int i;
3668 
3669 	if (!(bargs->flags & BTRFS_BALANCE_ARGS_DEVID))
3670 		return 0;
3671 
3672 	type = btrfs_chunk_type(leaf, chunk);
3673 	factor = calc_data_stripes(type, num_stripes);
3674 
3675 	for (i = 0; i < num_stripes; i++) {
3676 		stripe = btrfs_stripe_nr(chunk, i);
3677 		if (btrfs_stripe_devid(leaf, stripe) != bargs->devid)
3678 			continue;
3679 
3680 		stripe_offset = btrfs_stripe_offset(leaf, stripe);
3681 		stripe_length = btrfs_chunk_length(leaf, chunk);
3682 		stripe_length = div_u64(stripe_length, factor);
3683 
3684 		if (stripe_offset < bargs->pend &&
3685 		    stripe_offset + stripe_length > bargs->pstart)
3686 			return 0;
3687 	}
3688 
3689 	return 1;
3690 }
3691 
3692 /* [vstart, vend) */
3693 static int chunk_vrange_filter(struct extent_buffer *leaf,
3694 			       struct btrfs_chunk *chunk,
3695 			       u64 chunk_offset,
3696 			       struct btrfs_balance_args *bargs)
3697 {
3698 	if (chunk_offset < bargs->vend &&
3699 	    chunk_offset + btrfs_chunk_length(leaf, chunk) > bargs->vstart)
3700 		/* at least part of the chunk is inside this vrange */
3701 		return 0;
3702 
3703 	return 1;
3704 }
3705 
3706 static int chunk_stripes_range_filter(struct extent_buffer *leaf,
3707 			       struct btrfs_chunk *chunk,
3708 			       struct btrfs_balance_args *bargs)
3709 {
3710 	int num_stripes = btrfs_chunk_num_stripes(leaf, chunk);
3711 
3712 	if (bargs->stripes_min <= num_stripes
3713 			&& num_stripes <= bargs->stripes_max)
3714 		return 0;
3715 
3716 	return 1;
3717 }
3718 
3719 static int chunk_soft_convert_filter(u64 chunk_type,
3720 				     struct btrfs_balance_args *bargs)
3721 {
3722 	if (!(bargs->flags & BTRFS_BALANCE_ARGS_CONVERT))
3723 		return 0;
3724 
3725 	chunk_type = chunk_to_extended(chunk_type) &
3726 				BTRFS_EXTENDED_PROFILE_MASK;
3727 
3728 	if (bargs->target == chunk_type)
3729 		return 1;
3730 
3731 	return 0;
3732 }
3733 
3734 static int should_balance_chunk(struct extent_buffer *leaf,
3735 				struct btrfs_chunk *chunk, u64 chunk_offset)
3736 {
3737 	struct btrfs_fs_info *fs_info = leaf->fs_info;
3738 	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
3739 	struct btrfs_balance_args *bargs = NULL;
3740 	u64 chunk_type = btrfs_chunk_type(leaf, chunk);
3741 
3742 	/* type filter */
3743 	if (!((chunk_type & BTRFS_BLOCK_GROUP_TYPE_MASK) &
3744 	      (bctl->flags & BTRFS_BALANCE_TYPE_MASK))) {
3745 		return 0;
3746 	}
3747 
3748 	if (chunk_type & BTRFS_BLOCK_GROUP_DATA)
3749 		bargs = &bctl->data;
3750 	else if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM)
3751 		bargs = &bctl->sys;
3752 	else if (chunk_type & BTRFS_BLOCK_GROUP_METADATA)
3753 		bargs = &bctl->meta;
3754 
3755 	/* profiles filter */
3756 	if ((bargs->flags & BTRFS_BALANCE_ARGS_PROFILES) &&
3757 	    chunk_profiles_filter(chunk_type, bargs)) {
3758 		return 0;
3759 	}
3760 
3761 	/* usage filter */
3762 	if ((bargs->flags & BTRFS_BALANCE_ARGS_USAGE) &&
3763 	    chunk_usage_filter(fs_info, chunk_offset, bargs)) {
3764 		return 0;
3765 	} else if ((bargs->flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
3766 	    chunk_usage_range_filter(fs_info, chunk_offset, bargs)) {
3767 		return 0;
3768 	}
3769 
3770 	/* devid filter */
3771 	if ((bargs->flags & BTRFS_BALANCE_ARGS_DEVID) &&
3772 	    chunk_devid_filter(leaf, chunk, bargs)) {
3773 		return 0;
3774 	}
3775 
3776 	/* drange filter, makes sense only with devid filter */
3777 	if ((bargs->flags & BTRFS_BALANCE_ARGS_DRANGE) &&
3778 	    chunk_drange_filter(leaf, chunk, bargs)) {
3779 		return 0;
3780 	}
3781 
3782 	/* vrange filter */
3783 	if ((bargs->flags & BTRFS_BALANCE_ARGS_VRANGE) &&
3784 	    chunk_vrange_filter(leaf, chunk, chunk_offset, bargs)) {
3785 		return 0;
3786 	}
3787 
3788 	/* stripes filter */
3789 	if ((bargs->flags & BTRFS_BALANCE_ARGS_STRIPES_RANGE) &&
3790 	    chunk_stripes_range_filter(leaf, chunk, bargs)) {
3791 		return 0;
3792 	}
3793 
3794 	/* soft profile changing mode */
3795 	if ((bargs->flags & BTRFS_BALANCE_ARGS_SOFT) &&
3796 	    chunk_soft_convert_filter(chunk_type, bargs)) {
3797 		return 0;
3798 	}
3799 
3800 	/*
3801 	 * limited by count, must be the last filter
3802 	 */
3803 	if ((bargs->flags & BTRFS_BALANCE_ARGS_LIMIT)) {
3804 		if (bargs->limit == 0)
3805 			return 0;
3806 		else
3807 			bargs->limit--;
3808 	} else if ((bargs->flags & BTRFS_BALANCE_ARGS_LIMIT_RANGE)) {
3809 		/*
3810 		 * Same logic as the 'limit' filter; the minimum cannot be
3811 		 * determined here because we do not have the global information
3812 		 * about the count of all chunks that satisfy the filters.
3813 		 */
3814 		if (bargs->limit_max == 0)
3815 			return 0;
3816 		else
3817 			bargs->limit_max--;
3818 	}
3819 
3820 	return 1;
3821 }
3822 
3823 static int __btrfs_balance(struct btrfs_fs_info *fs_info)
3824 {
3825 	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
3826 	struct btrfs_root *chunk_root = fs_info->chunk_root;
3827 	u64 chunk_type;
3828 	struct btrfs_chunk *chunk;
3829 	struct btrfs_path *path = NULL;
3830 	struct btrfs_key key;
3831 	struct btrfs_key found_key;
3832 	struct extent_buffer *leaf;
3833 	int slot;
3834 	int ret;
3835 	int enospc_errors = 0;
3836 	bool counting = true;
3837 	/* The single value limit and min/max limits use the same bytes in the */
3838 	u64 limit_data = bctl->data.limit;
3839 	u64 limit_meta = bctl->meta.limit;
3840 	u64 limit_sys = bctl->sys.limit;
3841 	u32 count_data = 0;
3842 	u32 count_meta = 0;
3843 	u32 count_sys = 0;
3844 	int chunk_reserved = 0;
3845 
3846 	path = btrfs_alloc_path();
3847 	if (!path) {
3848 		ret = -ENOMEM;
3849 		goto error;
3850 	}
3851 
3852 	/* zero out stat counters */
3853 	spin_lock(&fs_info->balance_lock);
3854 	memset(&bctl->stat, 0, sizeof(bctl->stat));
3855 	spin_unlock(&fs_info->balance_lock);
3856 again:
3857 	if (!counting) {
3858 		/*
3859 		 * The single value limit and min/max limits use the same bytes
3860 		 * in the
3861 		 */
3862 		bctl->data.limit = limit_data;
3863 		bctl->meta.limit = limit_meta;
3864 		bctl->sys.limit = limit_sys;
3865 	}
3866 	key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
3867 	key.offset = (u64)-1;
3868 	key.type = BTRFS_CHUNK_ITEM_KEY;
3869 
3870 	while (1) {
3871 		if ((!counting && atomic_read(&fs_info->balance_pause_req)) ||
3872 		    atomic_read(&fs_info->balance_cancel_req)) {
3873 			ret = -ECANCELED;
3874 			goto error;
3875 		}
3876 
3877 		mutex_lock(&fs_info->reclaim_bgs_lock);
3878 		ret = btrfs_search_slot(NULL, chunk_root, &key, path, 0, 0);
3879 		if (ret < 0) {
3880 			mutex_unlock(&fs_info->reclaim_bgs_lock);
3881 			goto error;
3882 		}
3883 
3884 		/*
3885 		 * this shouldn't happen, it means the last relocate
3886 		 * failed
3887 		 */
3888 		if (ret == 0)
3889 			BUG(); /* FIXME break ? */
3890 
3891 		ret = btrfs_previous_item(chunk_root, path, 0,
3892 					  BTRFS_CHUNK_ITEM_KEY);
3893 		if (ret) {
3894 			mutex_unlock(&fs_info->reclaim_bgs_lock);
3895 			ret = 0;
3896 			break;
3897 		}
3898 
3899 		leaf = path->nodes[0];
3900 		slot = path->slots[0];
3901 		btrfs_item_key_to_cpu(leaf, &found_key, slot);
3902 
3903 		if (found_key.objectid != key.objectid) {
3904 			mutex_unlock(&fs_info->reclaim_bgs_lock);
3905 			break;
3906 		}
3907 
3908 		chunk = btrfs_item_ptr(leaf, slot, struct btrfs_chunk);
3909 		chunk_type = btrfs_chunk_type(leaf, chunk);
3910 
3911 		if (!counting) {
3912 			spin_lock(&fs_info->balance_lock);
3913 			bctl->stat.considered++;
3914 			spin_unlock(&fs_info->balance_lock);
3915 		}
3916 
3917 		ret = should_balance_chunk(leaf, chunk, found_key.offset);
3918 
3919 		btrfs_release_path(path);
3920 		if (!ret) {
3921 			mutex_unlock(&fs_info->reclaim_bgs_lock);
3922 			goto loop;
3923 		}
3924 
3925 		if (counting) {
3926 			mutex_unlock(&fs_info->reclaim_bgs_lock);
3927 			spin_lock(&fs_info->balance_lock);
3928 			bctl->stat.expected++;
3929 			spin_unlock(&fs_info->balance_lock);
3930 
3931 			if (chunk_type & BTRFS_BLOCK_GROUP_DATA)
3932 				count_data++;
3933 			else if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM)
3934 				count_sys++;
3935 			else if (chunk_type & BTRFS_BLOCK_GROUP_METADATA)
3936 				count_meta++;
3937 
3938 			goto loop;
3939 		}
3940 
3941 		/*
3942 		 * Apply limit_min filter, no need to check if the LIMITS
3943 		 * filter is used, limit_min is 0 by default
3944 		 */
3945 		if (((chunk_type & BTRFS_BLOCK_GROUP_DATA) &&
3946 					count_data < bctl->data.limit_min)
3947 				|| ((chunk_type & BTRFS_BLOCK_GROUP_METADATA) &&
3948 					count_meta < bctl->meta.limit_min)
3949 				|| ((chunk_type & BTRFS_BLOCK_GROUP_SYSTEM) &&
3950 					count_sys < bctl->sys.limit_min)) {
3951 			mutex_unlock(&fs_info->reclaim_bgs_lock);
3952 			goto loop;
3953 		}
3954 
3955 		if (!chunk_reserved) {
3956 			/*
3957 			 * We may be relocating the only data chunk we have,
3958 			 * which could potentially end up with losing data's
3959 			 * raid profile, so lets allocate an empty one in
3960 			 * advance.
3961 			 */
3962 			ret = btrfs_may_alloc_data_chunk(fs_info,
3963 							 found_key.offset);
3964 			if (ret < 0) {
3965 				mutex_unlock(&fs_info->reclaim_bgs_lock);
3966 				goto error;
3967 			} else if (ret == 1) {
3968 				chunk_reserved = 1;
3969 			}
3970 		}
3971 
3972 		ret = btrfs_relocate_chunk(fs_info, found_key.offset);
3973 		mutex_unlock(&fs_info->reclaim_bgs_lock);
3974 		if (ret == -ENOSPC) {
3975 			enospc_errors++;
3976 		} else if (ret == -ETXTBSY) {
3977 			btrfs_info(fs_info,
3978 	   "skipping relocation of block group %llu due to active swapfile",
3979 				   found_key.offset);
3980 			ret = 0;
3981 		} else if (ret) {
3982 			goto error;
3983 		} else {
3984 			spin_lock(&fs_info->balance_lock);
3985 			bctl->stat.completed++;
3986 			spin_unlock(&fs_info->balance_lock);
3987 		}
3988 loop:
3989 		if (found_key.offset == 0)
3990 			break;
3991 		key.offset = found_key.offset - 1;
3992 	}
3993 
3994 	if (counting) {
3995 		btrfs_release_path(path);
3996 		counting = false;
3997 		goto again;
3998 	}
3999 error:
4000 	btrfs_free_path(path);
4001 	if (enospc_errors) {
4002 		btrfs_info(fs_info, "%d enospc errors during balance",
4003 			   enospc_errors);
4004 		if (!ret)
4005 			ret = -ENOSPC;
4006 	}
4007 
4008 	return ret;
4009 }
4010 
4011 /**
4012  * alloc_profile_is_valid - see if a given profile is valid and reduced
4013  * @flags: profile to validate
4014  * @extended: if true @flags is treated as an extended profile
4015  */
4016 static int alloc_profile_is_valid(u64 flags, int extended)
4017 {
4018 	u64 mask = (extended ? BTRFS_EXTENDED_PROFILE_MASK :
4019 			       BTRFS_BLOCK_GROUP_PROFILE_MASK);
4020 
4021 	flags &= ~BTRFS_BLOCK_GROUP_TYPE_MASK;
4022 
4023 	/* 1) check that all other bits are zeroed */
4024 	if (flags & ~mask)
4025 		return 0;
4026 
4027 	/* 2) see if profile is reduced */
4028 	if (flags == 0)
4029 		return !extended; /* "0" is valid for usual profiles */
4030 
4031 	return has_single_bit_set(flags);
4032 }
4033 
4034 static inline int balance_need_close(struct btrfs_fs_info *fs_info)
4035 {
4036 	/* cancel requested || normal exit path */
4037 	return atomic_read(&fs_info->balance_cancel_req) ||
4038 		(atomic_read(&fs_info->balance_pause_req) == 0 &&
4039 		 atomic_read(&fs_info->balance_cancel_req) == 0);
4040 }
4041 
4042 /*
4043  * Validate target profile against allowed profiles and return true if it's OK.
4044  * Otherwise print the error message and return false.
4045  */
4046 static inline int validate_convert_profile(struct btrfs_fs_info *fs_info,
4047 		const struct btrfs_balance_args *bargs,
4048 		u64 allowed, const char *type)
4049 {
4050 	if (!(bargs->flags & BTRFS_BALANCE_ARGS_CONVERT))
4051 		return true;
4052 
4053 	/* Profile is valid and does not have bits outside of the allowed set */
4054 	if (alloc_profile_is_valid(bargs->target, 1) &&
4055 	    (bargs->target & ~allowed) == 0)
4056 		return true;
4057 
4058 	btrfs_err(fs_info, "balance: invalid convert %s profile %s",
4059 			type, btrfs_bg_type_to_raid_name(bargs->target));
4060 	return false;
4061 }
4062 
4063 /*
4064  * Fill @buf with textual description of balance filter flags @bargs, up to
4065  * @size_buf including the terminating null. The output may be trimmed if it
4066  * does not fit into the provided buffer.
4067  */
4068 static void describe_balance_args(struct btrfs_balance_args *bargs, char *buf,
4069 				 u32 size_buf)
4070 {
4071 	int ret;
4072 	u32 size_bp = size_buf;
4073 	char *bp = buf;
4074 	u64 flags = bargs->flags;
4075 	char tmp_buf[128] = {'\0'};
4076 
4077 	if (!flags)
4078 		return;
4079 
4080 #define CHECK_APPEND_NOARG(a)						\
4081 	do {								\
4082 		ret = snprintf(bp, size_bp, (a));			\
4083 		if (ret < 0 || ret >= size_bp)				\
4084 			goto out_overflow;				\
4085 		size_bp -= ret;						\
4086 		bp += ret;						\
4087 	} while (0)
4088 
4089 #define CHECK_APPEND_1ARG(a, v1)					\
4090 	do {								\
4091 		ret = snprintf(bp, size_bp, (a), (v1));			\
4092 		if (ret < 0 || ret >= size_bp)				\
4093 			goto out_overflow;				\
4094 		size_bp -= ret;						\
4095 		bp += ret;						\
4096 	} while (0)
4097 
4098 #define CHECK_APPEND_2ARG(a, v1, v2)					\
4099 	do {								\
4100 		ret = snprintf(bp, size_bp, (a), (v1), (v2));		\
4101 		if (ret < 0 || ret >= size_bp)				\
4102 			goto out_overflow;				\
4103 		size_bp -= ret;						\
4104 		bp += ret;						\
4105 	} while (0)
4106 
4107 	if (flags & BTRFS_BALANCE_ARGS_CONVERT)
4108 		CHECK_APPEND_1ARG("convert=%s,",
4109 				  btrfs_bg_type_to_raid_name(bargs->target));
4110 
4111 	if (flags & BTRFS_BALANCE_ARGS_SOFT)
4112 		CHECK_APPEND_NOARG("soft,");
4113 
4114 	if (flags & BTRFS_BALANCE_ARGS_PROFILES) {
4115 		btrfs_describe_block_groups(bargs->profiles, tmp_buf,
4116 					    sizeof(tmp_buf));
4117 		CHECK_APPEND_1ARG("profiles=%s,", tmp_buf);
4118 	}
4119 
4120 	if (flags & BTRFS_BALANCE_ARGS_USAGE)
4121 		CHECK_APPEND_1ARG("usage=%llu,", bargs->usage);
4122 
4123 	if (flags & BTRFS_BALANCE_ARGS_USAGE_RANGE)
4124 		CHECK_APPEND_2ARG("usage=%u..%u,",
4125 				  bargs->usage_min, bargs->usage_max);
4126 
4127 	if (flags & BTRFS_BALANCE_ARGS_DEVID)
4128 		CHECK_APPEND_1ARG("devid=%llu,", bargs->devid);
4129 
4130 	if (flags & BTRFS_BALANCE_ARGS_DRANGE)
4131 		CHECK_APPEND_2ARG("drange=%llu..%llu,",
4132 				  bargs->pstart, bargs->pend);
4133 
4134 	if (flags & BTRFS_BALANCE_ARGS_VRANGE)
4135 		CHECK_APPEND_2ARG("vrange=%llu..%llu,",
4136 				  bargs->vstart, bargs->vend);
4137 
4138 	if (flags & BTRFS_BALANCE_ARGS_LIMIT)
4139 		CHECK_APPEND_1ARG("limit=%llu,", bargs->limit);
4140 
4141 	if (flags & BTRFS_BALANCE_ARGS_LIMIT_RANGE)
4142 		CHECK_APPEND_2ARG("limit=%u..%u,",
4143 				bargs->limit_min, bargs->limit_max);
4144 
4145 	if (flags & BTRFS_BALANCE_ARGS_STRIPES_RANGE)
4146 		CHECK_APPEND_2ARG("stripes=%u..%u,",
4147 				  bargs->stripes_min, bargs->stripes_max);
4148 
4149 #undef CHECK_APPEND_2ARG
4150 #undef CHECK_APPEND_1ARG
4151 #undef CHECK_APPEND_NOARG
4152 
4153 out_overflow:
4154 
4155 	if (size_bp < size_buf)
4156 		buf[size_buf - size_bp - 1] = '\0'; /* remove last , */
4157 	else
4158 		buf[0] = '\0';
4159 }
4160 
4161 static void describe_balance_start_or_resume(struct btrfs_fs_info *fs_info)
4162 {
4163 	u32 size_buf = 1024;
4164 	char tmp_buf[192] = {'\0'};
4165 	char *buf;
4166 	char *bp;
4167 	u32 size_bp = size_buf;
4168 	int ret;
4169 	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
4170 
4171 	buf = kzalloc(size_buf, GFP_KERNEL);
4172 	if (!buf)
4173 		return;
4174 
4175 	bp = buf;
4176 
4177 #define CHECK_APPEND_1ARG(a, v1)					\
4178 	do {								\
4179 		ret = snprintf(bp, size_bp, (a), (v1));			\
4180 		if (ret < 0 || ret >= size_bp)				\
4181 			goto out_overflow;				\
4182 		size_bp -= ret;						\
4183 		bp += ret;						\
4184 	} while (0)
4185 
4186 	if (bctl->flags & BTRFS_BALANCE_FORCE)
4187 		CHECK_APPEND_1ARG("%s", "-f ");
4188 
4189 	if (bctl->flags & BTRFS_BALANCE_DATA) {
4190 		describe_balance_args(&bctl->data, tmp_buf, sizeof(tmp_buf));
4191 		CHECK_APPEND_1ARG("-d%s ", tmp_buf);
4192 	}
4193 
4194 	if (bctl->flags & BTRFS_BALANCE_METADATA) {
4195 		describe_balance_args(&bctl->meta, tmp_buf, sizeof(tmp_buf));
4196 		CHECK_APPEND_1ARG("-m%s ", tmp_buf);
4197 	}
4198 
4199 	if (bctl->flags & BTRFS_BALANCE_SYSTEM) {
4200 		describe_balance_args(&bctl->sys, tmp_buf, sizeof(tmp_buf));
4201 		CHECK_APPEND_1ARG("-s%s ", tmp_buf);
4202 	}
4203 
4204 #undef CHECK_APPEND_1ARG
4205 
4206 out_overflow:
4207 
4208 	if (size_bp < size_buf)
4209 		buf[size_buf - size_bp - 1] = '\0'; /* remove last " " */
4210 	btrfs_info(fs_info, "balance: %s %s",
4211 		   (bctl->flags & BTRFS_BALANCE_RESUME) ?
4212 		   "resume" : "start", buf);
4213 
4214 	kfree(buf);
4215 }
4216 
4217 /*
4218  * Should be called with balance mutexe held
4219  */
4220 int btrfs_balance(struct btrfs_fs_info *fs_info,
4221 		  struct btrfs_balance_control *bctl,
4222 		  struct btrfs_ioctl_balance_args *bargs)
4223 {
4224 	u64 meta_target, data_target;
4225 	u64 allowed;
4226 	int mixed = 0;
4227 	int ret;
4228 	u64 num_devices;
4229 	unsigned seq;
4230 	bool reducing_redundancy;
4231 	int i;
4232 
4233 	if (btrfs_fs_closing(fs_info) ||
4234 	    atomic_read(&fs_info->balance_pause_req) ||
4235 	    btrfs_should_cancel_balance(fs_info)) {
4236 		ret = -EINVAL;
4237 		goto out;
4238 	}
4239 
4240 	allowed = btrfs_super_incompat_flags(fs_info->super_copy);
4241 	if (allowed & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS)
4242 		mixed = 1;
4243 
4244 	/*
4245 	 * In case of mixed groups both data and meta should be picked,
4246 	 * and identical options should be given for both of them.
4247 	 */
4248 	allowed = BTRFS_BALANCE_DATA | BTRFS_BALANCE_METADATA;
4249 	if (mixed && (bctl->flags & allowed)) {
4250 		if (!(bctl->flags & BTRFS_BALANCE_DATA) ||
4251 		    !(bctl->flags & BTRFS_BALANCE_METADATA) ||
4252 		    memcmp(&bctl->data, &bctl->meta, sizeof(bctl->data))) {
4253 			btrfs_err(fs_info,
4254 	  "balance: mixed groups data and metadata options must be the same");
4255 			ret = -EINVAL;
4256 			goto out;
4257 		}
4258 	}
4259 
4260 	/*
4261 	 * rw_devices will not change at the moment, device add/delete/replace
4262 	 * are exclusive
4263 	 */
4264 	num_devices = fs_info->fs_devices->rw_devices;
4265 
4266 	/*
4267 	 * SINGLE profile on-disk has no profile bit, but in-memory we have a
4268 	 * special bit for it, to make it easier to distinguish.  Thus we need
4269 	 * to set it manually, or balance would refuse the profile.
4270 	 */
4271 	allowed = BTRFS_AVAIL_ALLOC_BIT_SINGLE;
4272 	for (i = 0; i < ARRAY_SIZE(btrfs_raid_array); i++)
4273 		if (num_devices >= btrfs_raid_array[i].devs_min)
4274 			allowed |= btrfs_raid_array[i].bg_flag;
4275 
4276 	if (!validate_convert_profile(fs_info, &bctl->data, allowed, "data") ||
4277 	    !validate_convert_profile(fs_info, &bctl->meta, allowed, "metadata") ||
4278 	    !validate_convert_profile(fs_info, &bctl->sys,  allowed, "system")) {
4279 		ret = -EINVAL;
4280 		goto out;
4281 	}
4282 
4283 	/*
4284 	 * Allow to reduce metadata or system integrity only if force set for
4285 	 * profiles with redundancy (copies, parity)
4286 	 */
4287 	allowed = 0;
4288 	for (i = 0; i < ARRAY_SIZE(btrfs_raid_array); i++) {
4289 		if (btrfs_raid_array[i].ncopies >= 2 ||
4290 		    btrfs_raid_array[i].tolerated_failures >= 1)
4291 			allowed |= btrfs_raid_array[i].bg_flag;
4292 	}
4293 	do {
4294 		seq = read_seqbegin(&fs_info->profiles_lock);
4295 
4296 		if (((bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) &&
4297 		     (fs_info->avail_system_alloc_bits & allowed) &&
4298 		     !(bctl->sys.target & allowed)) ||
4299 		    ((bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) &&
4300 		     (fs_info->avail_metadata_alloc_bits & allowed) &&
4301 		     !(bctl->meta.target & allowed)))
4302 			reducing_redundancy = true;
4303 		else
4304 			reducing_redundancy = false;
4305 
4306 		/* if we're not converting, the target field is uninitialized */
4307 		meta_target = (bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) ?
4308 			bctl->meta.target : fs_info->avail_metadata_alloc_bits;
4309 		data_target = (bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) ?
4310 			bctl->data.target : fs_info->avail_data_alloc_bits;
4311 	} while (read_seqretry(&fs_info->profiles_lock, seq));
4312 
4313 	if (reducing_redundancy) {
4314 		if (bctl->flags & BTRFS_BALANCE_FORCE) {
4315 			btrfs_info(fs_info,
4316 			   "balance: force reducing metadata redundancy");
4317 		} else {
4318 			btrfs_err(fs_info,
4319 	"balance: reduces metadata redundancy, use --force if you want this");
4320 			ret = -EINVAL;
4321 			goto out;
4322 		}
4323 	}
4324 
4325 	if (btrfs_get_num_tolerated_disk_barrier_failures(meta_target) <
4326 		btrfs_get_num_tolerated_disk_barrier_failures(data_target)) {
4327 		btrfs_warn(fs_info,
4328 	"balance: metadata profile %s has lower redundancy than data profile %s",
4329 				btrfs_bg_type_to_raid_name(meta_target),
4330 				btrfs_bg_type_to_raid_name(data_target));
4331 	}
4332 
4333 	ret = insert_balance_item(fs_info, bctl);
4334 	if (ret && ret != -EEXIST)
4335 		goto out;
4336 
4337 	if (!(bctl->flags & BTRFS_BALANCE_RESUME)) {
4338 		BUG_ON(ret == -EEXIST);
4339 		BUG_ON(fs_info->balance_ctl);
4340 		spin_lock(&fs_info->balance_lock);
4341 		fs_info->balance_ctl = bctl;
4342 		spin_unlock(&fs_info->balance_lock);
4343 	} else {
4344 		BUG_ON(ret != -EEXIST);
4345 		spin_lock(&fs_info->balance_lock);
4346 		update_balance_args(bctl);
4347 		spin_unlock(&fs_info->balance_lock);
4348 	}
4349 
4350 	ASSERT(!test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4351 	set_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags);
4352 	describe_balance_start_or_resume(fs_info);
4353 	mutex_unlock(&fs_info->balance_mutex);
4354 
4355 	ret = __btrfs_balance(fs_info);
4356 
4357 	mutex_lock(&fs_info->balance_mutex);
4358 	if (ret == -ECANCELED && atomic_read(&fs_info->balance_pause_req)) {
4359 		btrfs_info(fs_info, "balance: paused");
4360 		btrfs_exclop_balance(fs_info, BTRFS_EXCLOP_BALANCE_PAUSED);
4361 	}
4362 	/*
4363 	 * Balance can be canceled by:
4364 	 *
4365 	 * - Regular cancel request
4366 	 *   Then ret == -ECANCELED and balance_cancel_req > 0
4367 	 *
4368 	 * - Fatal signal to "btrfs" process
4369 	 *   Either the signal caught by wait_reserve_ticket() and callers
4370 	 *   got -EINTR, or caught by btrfs_should_cancel_balance() and
4371 	 *   got -ECANCELED.
4372 	 *   Either way, in this case balance_cancel_req = 0, and
4373 	 *   ret == -EINTR or ret == -ECANCELED.
4374 	 *
4375 	 * So here we only check the return value to catch canceled balance.
4376 	 */
4377 	else if (ret == -ECANCELED || ret == -EINTR)
4378 		btrfs_info(fs_info, "balance: canceled");
4379 	else
4380 		btrfs_info(fs_info, "balance: ended with status: %d", ret);
4381 
4382 	clear_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags);
4383 
4384 	if (bargs) {
4385 		memset(bargs, 0, sizeof(*bargs));
4386 		btrfs_update_ioctl_balance_args(fs_info, bargs);
4387 	}
4388 
4389 	if ((ret && ret != -ECANCELED && ret != -ENOSPC) ||
4390 	    balance_need_close(fs_info)) {
4391 		reset_balance_state(fs_info);
4392 		btrfs_exclop_finish(fs_info);
4393 	}
4394 
4395 	wake_up(&fs_info->balance_wait_q);
4396 
4397 	return ret;
4398 out:
4399 	if (bctl->flags & BTRFS_BALANCE_RESUME)
4400 		reset_balance_state(fs_info);
4401 	else
4402 		kfree(bctl);
4403 	btrfs_exclop_finish(fs_info);
4404 
4405 	return ret;
4406 }
4407 
4408 static int balance_kthread(void *data)
4409 {
4410 	struct btrfs_fs_info *fs_info = data;
4411 	int ret = 0;
4412 
4413 	sb_start_write(fs_info->sb);
4414 	mutex_lock(&fs_info->balance_mutex);
4415 	if (fs_info->balance_ctl)
4416 		ret = btrfs_balance(fs_info, fs_info->balance_ctl, NULL);
4417 	mutex_unlock(&fs_info->balance_mutex);
4418 	sb_end_write(fs_info->sb);
4419 
4420 	return ret;
4421 }
4422 
4423 int btrfs_resume_balance_async(struct btrfs_fs_info *fs_info)
4424 {
4425 	struct task_struct *tsk;
4426 
4427 	mutex_lock(&fs_info->balance_mutex);
4428 	if (!fs_info->balance_ctl) {
4429 		mutex_unlock(&fs_info->balance_mutex);
4430 		return 0;
4431 	}
4432 	mutex_unlock(&fs_info->balance_mutex);
4433 
4434 	if (btrfs_test_opt(fs_info, SKIP_BALANCE)) {
4435 		btrfs_info(fs_info, "balance: resume skipped");
4436 		return 0;
4437 	}
4438 
4439 	spin_lock(&fs_info->super_lock);
4440 	ASSERT(fs_info->exclusive_operation == BTRFS_EXCLOP_BALANCE_PAUSED);
4441 	fs_info->exclusive_operation = BTRFS_EXCLOP_BALANCE;
4442 	spin_unlock(&fs_info->super_lock);
4443 	/*
4444 	 * A ro->rw remount sequence should continue with the paused balance
4445 	 * regardless of who pauses it, system or the user as of now, so set
4446 	 * the resume flag.
4447 	 */
4448 	spin_lock(&fs_info->balance_lock);
4449 	fs_info->balance_ctl->flags |= BTRFS_BALANCE_RESUME;
4450 	spin_unlock(&fs_info->balance_lock);
4451 
4452 	tsk = kthread_run(balance_kthread, fs_info, "btrfs-balance");
4453 	return PTR_ERR_OR_ZERO(tsk);
4454 }
4455 
4456 int btrfs_recover_balance(struct btrfs_fs_info *fs_info)
4457 {
4458 	struct btrfs_balance_control *bctl;
4459 	struct btrfs_balance_item *item;
4460 	struct btrfs_disk_balance_args disk_bargs;
4461 	struct btrfs_path *path;
4462 	struct extent_buffer *leaf;
4463 	struct btrfs_key key;
4464 	int ret;
4465 
4466 	path = btrfs_alloc_path();
4467 	if (!path)
4468 		return -ENOMEM;
4469 
4470 	key.objectid = BTRFS_BALANCE_OBJECTID;
4471 	key.type = BTRFS_TEMPORARY_ITEM_KEY;
4472 	key.offset = 0;
4473 
4474 	ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
4475 	if (ret < 0)
4476 		goto out;
4477 	if (ret > 0) { /* ret = -ENOENT; */
4478 		ret = 0;
4479 		goto out;
4480 	}
4481 
4482 	bctl = kzalloc(sizeof(*bctl), GFP_NOFS);
4483 	if (!bctl) {
4484 		ret = -ENOMEM;
4485 		goto out;
4486 	}
4487 
4488 	leaf = path->nodes[0];
4489 	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_balance_item);
4490 
4491 	bctl->flags = btrfs_balance_flags(leaf, item);
4492 	bctl->flags |= BTRFS_BALANCE_RESUME;
4493 
4494 	btrfs_balance_data(leaf, item, &disk_bargs);
4495 	btrfs_disk_balance_args_to_cpu(&bctl->data, &disk_bargs);
4496 	btrfs_balance_meta(leaf, item, &disk_bargs);
4497 	btrfs_disk_balance_args_to_cpu(&bctl->meta, &disk_bargs);
4498 	btrfs_balance_sys(leaf, item, &disk_bargs);
4499 	btrfs_disk_balance_args_to_cpu(&bctl->sys, &disk_bargs);
4500 
4501 	/*
4502 	 * This should never happen, as the paused balance state is recovered
4503 	 * during mount without any chance of other exclusive ops to collide.
4504 	 *
4505 	 * This gives the exclusive op status to balance and keeps in paused
4506 	 * state until user intervention (cancel or umount). If the ownership
4507 	 * cannot be assigned, show a message but do not fail. The balance
4508 	 * is in a paused state and must have fs_info::balance_ctl properly
4509 	 * set up.
4510 	 */
4511 	if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_BALANCE_PAUSED))
4512 		btrfs_warn(fs_info,
4513 	"balance: cannot set exclusive op status, resume manually");
4514 
4515 	btrfs_release_path(path);
4516 
4517 	mutex_lock(&fs_info->balance_mutex);
4518 	BUG_ON(fs_info->balance_ctl);
4519 	spin_lock(&fs_info->balance_lock);
4520 	fs_info->balance_ctl = bctl;
4521 	spin_unlock(&fs_info->balance_lock);
4522 	mutex_unlock(&fs_info->balance_mutex);
4523 out:
4524 	btrfs_free_path(path);
4525 	return ret;
4526 }
4527 
4528 int btrfs_pause_balance(struct btrfs_fs_info *fs_info)
4529 {
4530 	int ret = 0;
4531 
4532 	mutex_lock(&fs_info->balance_mutex);
4533 	if (!fs_info->balance_ctl) {
4534 		mutex_unlock(&fs_info->balance_mutex);
4535 		return -ENOTCONN;
4536 	}
4537 
4538 	if (test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) {
4539 		atomic_inc(&fs_info->balance_pause_req);
4540 		mutex_unlock(&fs_info->balance_mutex);
4541 
4542 		wait_event(fs_info->balance_wait_q,
4543 			   !test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4544 
4545 		mutex_lock(&fs_info->balance_mutex);
4546 		/* we are good with balance_ctl ripped off from under us */
4547 		BUG_ON(test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4548 		atomic_dec(&fs_info->balance_pause_req);
4549 	} else {
4550 		ret = -ENOTCONN;
4551 	}
4552 
4553 	mutex_unlock(&fs_info->balance_mutex);
4554 	return ret;
4555 }
4556 
4557 int btrfs_cancel_balance(struct btrfs_fs_info *fs_info)
4558 {
4559 	mutex_lock(&fs_info->balance_mutex);
4560 	if (!fs_info->balance_ctl) {
4561 		mutex_unlock(&fs_info->balance_mutex);
4562 		return -ENOTCONN;
4563 	}
4564 
4565 	/*
4566 	 * A paused balance with the item stored on disk can be resumed at
4567 	 * mount time if the mount is read-write. Otherwise it's still paused
4568 	 * and we must not allow cancelling as it deletes the item.
4569 	 */
4570 	if (sb_rdonly(fs_info->sb)) {
4571 		mutex_unlock(&fs_info->balance_mutex);
4572 		return -EROFS;
4573 	}
4574 
4575 	atomic_inc(&fs_info->balance_cancel_req);
4576 	/*
4577 	 * if we are running just wait and return, balance item is
4578 	 * deleted in btrfs_balance in this case
4579 	 */
4580 	if (test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) {
4581 		mutex_unlock(&fs_info->balance_mutex);
4582 		wait_event(fs_info->balance_wait_q,
4583 			   !test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4584 		mutex_lock(&fs_info->balance_mutex);
4585 	} else {
4586 		mutex_unlock(&fs_info->balance_mutex);
4587 		/*
4588 		 * Lock released to allow other waiters to continue, we'll
4589 		 * reexamine the status again.
4590 		 */
4591 		mutex_lock(&fs_info->balance_mutex);
4592 
4593 		if (fs_info->balance_ctl) {
4594 			reset_balance_state(fs_info);
4595 			btrfs_exclop_finish(fs_info);
4596 			btrfs_info(fs_info, "balance: canceled");
4597 		}
4598 	}
4599 
4600 	BUG_ON(fs_info->balance_ctl ||
4601 		test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4602 	atomic_dec(&fs_info->balance_cancel_req);
4603 	mutex_unlock(&fs_info->balance_mutex);
4604 	return 0;
4605 }
4606 
4607 int btrfs_uuid_scan_kthread(void *data)
4608 {
4609 	struct btrfs_fs_info *fs_info = data;
4610 	struct btrfs_root *root = fs_info->tree_root;
4611 	struct btrfs_key key;
4612 	struct btrfs_path *path = NULL;
4613 	int ret = 0;
4614 	struct extent_buffer *eb;
4615 	int slot;
4616 	struct btrfs_root_item root_item;
4617 	u32 item_size;
4618 	struct btrfs_trans_handle *trans = NULL;
4619 	bool closing = false;
4620 
4621 	path = btrfs_alloc_path();
4622 	if (!path) {
4623 		ret = -ENOMEM;
4624 		goto out;
4625 	}
4626 
4627 	key.objectid = 0;
4628 	key.type = BTRFS_ROOT_ITEM_KEY;
4629 	key.offset = 0;
4630 
4631 	while (1) {
4632 		if (btrfs_fs_closing(fs_info)) {
4633 			closing = true;
4634 			break;
4635 		}
4636 		ret = btrfs_search_forward(root, &key, path,
4637 				BTRFS_OLDEST_GENERATION);
4638 		if (ret) {
4639 			if (ret > 0)
4640 				ret = 0;
4641 			break;
4642 		}
4643 
4644 		if (key.type != BTRFS_ROOT_ITEM_KEY ||
4645 		    (key.objectid < BTRFS_FIRST_FREE_OBJECTID &&
4646 		     key.objectid != BTRFS_FS_TREE_OBJECTID) ||
4647 		    key.objectid > BTRFS_LAST_FREE_OBJECTID)
4648 			goto skip;
4649 
4650 		eb = path->nodes[0];
4651 		slot = path->slots[0];
4652 		item_size = btrfs_item_size(eb, slot);
4653 		if (item_size < sizeof(root_item))
4654 			goto skip;
4655 
4656 		read_extent_buffer(eb, &root_item,
4657 				   btrfs_item_ptr_offset(eb, slot),
4658 				   (int)sizeof(root_item));
4659 		if (btrfs_root_refs(&root_item) == 0)
4660 			goto skip;
4661 
4662 		if (!btrfs_is_empty_uuid(root_item.uuid) ||
4663 		    !btrfs_is_empty_uuid(root_item.received_uuid)) {
4664 			if (trans)
4665 				goto update_tree;
4666 
4667 			btrfs_release_path(path);
4668 			/*
4669 			 * 1 - subvol uuid item
4670 			 * 1 - received_subvol uuid item
4671 			 */
4672 			trans = btrfs_start_transaction(fs_info->uuid_root, 2);
4673 			if (IS_ERR(trans)) {
4674 				ret = PTR_ERR(trans);
4675 				break;
4676 			}
4677 			continue;
4678 		} else {
4679 			goto skip;
4680 		}
4681 update_tree:
4682 		btrfs_release_path(path);
4683 		if (!btrfs_is_empty_uuid(root_item.uuid)) {
4684 			ret = btrfs_uuid_tree_add(trans, root_item.uuid,
4685 						  BTRFS_UUID_KEY_SUBVOL,
4686 						  key.objectid);
4687 			if (ret < 0) {
4688 				btrfs_warn(fs_info, "uuid_tree_add failed %d",
4689 					ret);
4690 				break;
4691 			}
4692 		}
4693 
4694 		if (!btrfs_is_empty_uuid(root_item.received_uuid)) {
4695 			ret = btrfs_uuid_tree_add(trans,
4696 						  root_item.received_uuid,
4697 						 BTRFS_UUID_KEY_RECEIVED_SUBVOL,
4698 						  key.objectid);
4699 			if (ret < 0) {
4700 				btrfs_warn(fs_info, "uuid_tree_add failed %d",
4701 					ret);
4702 				break;
4703 			}
4704 		}
4705 
4706 skip:
4707 		btrfs_release_path(path);
4708 		if (trans) {
4709 			ret = btrfs_end_transaction(trans);
4710 			trans = NULL;
4711 			if (ret)
4712 				break;
4713 		}
4714 
4715 		if (key.offset < (u64)-1) {
4716 			key.offset++;
4717 		} else if (key.type < BTRFS_ROOT_ITEM_KEY) {
4718 			key.offset = 0;
4719 			key.type = BTRFS_ROOT_ITEM_KEY;
4720 		} else if (key.objectid < (u64)-1) {
4721 			key.offset = 0;
4722 			key.type = BTRFS_ROOT_ITEM_KEY;
4723 			key.objectid++;
4724 		} else {
4725 			break;
4726 		}
4727 		cond_resched();
4728 	}
4729 
4730 out:
4731 	btrfs_free_path(path);
4732 	if (trans && !IS_ERR(trans))
4733 		btrfs_end_transaction(trans);
4734 	if (ret)
4735 		btrfs_warn(fs_info, "btrfs_uuid_scan_kthread failed %d", ret);
4736 	else if (!closing)
4737 		set_bit(BTRFS_FS_UPDATE_UUID_TREE_GEN, &fs_info->flags);
4738 	up(&fs_info->uuid_tree_rescan_sem);
4739 	return 0;
4740 }
4741 
4742 int btrfs_create_uuid_tree(struct btrfs_fs_info *fs_info)
4743 {
4744 	struct btrfs_trans_handle *trans;
4745 	struct btrfs_root *tree_root = fs_info->tree_root;
4746 	struct btrfs_root *uuid_root;
4747 	struct task_struct *task;
4748 	int ret;
4749 
4750 	/*
4751 	 * 1 - root node
4752 	 * 1 - root item
4753 	 */
4754 	trans = btrfs_start_transaction(tree_root, 2);
4755 	if (IS_ERR(trans))
4756 		return PTR_ERR(trans);
4757 
4758 	uuid_root = btrfs_create_tree(trans, BTRFS_UUID_TREE_OBJECTID);
4759 	if (IS_ERR(uuid_root)) {
4760 		ret = PTR_ERR(uuid_root);
4761 		btrfs_abort_transaction(trans, ret);
4762 		btrfs_end_transaction(trans);
4763 		return ret;
4764 	}
4765 
4766 	fs_info->uuid_root = uuid_root;
4767 
4768 	ret = btrfs_commit_transaction(trans);
4769 	if (ret)
4770 		return ret;
4771 
4772 	down(&fs_info->uuid_tree_rescan_sem);
4773 	task = kthread_run(btrfs_uuid_scan_kthread, fs_info, "btrfs-uuid");
4774 	if (IS_ERR(task)) {
4775 		/* fs_info->update_uuid_tree_gen remains 0 in all error case */
4776 		btrfs_warn(fs_info, "failed to start uuid_scan task");
4777 		up(&fs_info->uuid_tree_rescan_sem);
4778 		return PTR_ERR(task);
4779 	}
4780 
4781 	return 0;
4782 }
4783 
4784 /*
4785  * shrinking a device means finding all of the device extents past
4786  * the new size, and then following the back refs to the chunks.
4787  * The chunk relocation code actually frees the device extent
4788  */
4789 int btrfs_shrink_device(struct btrfs_device *device, u64 new_size)
4790 {
4791 	struct btrfs_fs_info *fs_info = device->fs_info;
4792 	struct btrfs_root *root = fs_info->dev_root;
4793 	struct btrfs_trans_handle *trans;
4794 	struct btrfs_dev_extent *dev_extent = NULL;
4795 	struct btrfs_path *path;
4796 	u64 length;
4797 	u64 chunk_offset;
4798 	int ret;
4799 	int slot;
4800 	int failed = 0;
4801 	bool retried = false;
4802 	struct extent_buffer *l;
4803 	struct btrfs_key key;
4804 	struct btrfs_super_block *super_copy = fs_info->super_copy;
4805 	u64 old_total = btrfs_super_total_bytes(super_copy);
4806 	u64 old_size = btrfs_device_get_total_bytes(device);
4807 	u64 diff;
4808 	u64 start;
4809 
4810 	new_size = round_down(new_size, fs_info->sectorsize);
4811 	start = new_size;
4812 	diff = round_down(old_size - new_size, fs_info->sectorsize);
4813 
4814 	if (test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state))
4815 		return -EINVAL;
4816 
4817 	path = btrfs_alloc_path();
4818 	if (!path)
4819 		return -ENOMEM;
4820 
4821 	path->reada = READA_BACK;
4822 
4823 	trans = btrfs_start_transaction(root, 0);
4824 	if (IS_ERR(trans)) {
4825 		btrfs_free_path(path);
4826 		return PTR_ERR(trans);
4827 	}
4828 
4829 	mutex_lock(&fs_info->chunk_mutex);
4830 
4831 	btrfs_device_set_total_bytes(device, new_size);
4832 	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
4833 		device->fs_devices->total_rw_bytes -= diff;
4834 		atomic64_sub(diff, &fs_info->free_chunk_space);
4835 	}
4836 
4837 	/*
4838 	 * Once the device's size has been set to the new size, ensure all
4839 	 * in-memory chunks are synced to disk so that the loop below sees them
4840 	 * and relocates them accordingly.
4841 	 */
4842 	if (contains_pending_extent(device, &start, diff)) {
4843 		mutex_unlock(&fs_info->chunk_mutex);
4844 		ret = btrfs_commit_transaction(trans);
4845 		if (ret)
4846 			goto done;
4847 	} else {
4848 		mutex_unlock(&fs_info->chunk_mutex);
4849 		btrfs_end_transaction(trans);
4850 	}
4851 
4852 again:
4853 	key.objectid = device->devid;
4854 	key.offset = (u64)-1;
4855 	key.type = BTRFS_DEV_EXTENT_KEY;
4856 
4857 	do {
4858 		mutex_lock(&fs_info->reclaim_bgs_lock);
4859 		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
4860 		if (ret < 0) {
4861 			mutex_unlock(&fs_info->reclaim_bgs_lock);
4862 			goto done;
4863 		}
4864 
4865 		ret = btrfs_previous_item(root, path, 0, key.type);
4866 		if (ret) {
4867 			mutex_unlock(&fs_info->reclaim_bgs_lock);
4868 			if (ret < 0)
4869 				goto done;
4870 			ret = 0;
4871 			btrfs_release_path(path);
4872 			break;
4873 		}
4874 
4875 		l = path->nodes[0];
4876 		slot = path->slots[0];
4877 		btrfs_item_key_to_cpu(l, &key, path->slots[0]);
4878 
4879 		if (key.objectid != device->devid) {
4880 			mutex_unlock(&fs_info->reclaim_bgs_lock);
4881 			btrfs_release_path(path);
4882 			break;
4883 		}
4884 
4885 		dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
4886 		length = btrfs_dev_extent_length(l, dev_extent);
4887 
4888 		if (key.offset + length <= new_size) {
4889 			mutex_unlock(&fs_info->reclaim_bgs_lock);
4890 			btrfs_release_path(path);
4891 			break;
4892 		}
4893 
4894 		chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);
4895 		btrfs_release_path(path);
4896 
4897 		/*
4898 		 * We may be relocating the only data chunk we have,
4899 		 * which could potentially end up with losing data's
4900 		 * raid profile, so lets allocate an empty one in
4901 		 * advance.
4902 		 */
4903 		ret = btrfs_may_alloc_data_chunk(fs_info, chunk_offset);
4904 		if (ret < 0) {
4905 			mutex_unlock(&fs_info->reclaim_bgs_lock);
4906 			goto done;
4907 		}
4908 
4909 		ret = btrfs_relocate_chunk(fs_info, chunk_offset);
4910 		mutex_unlock(&fs_info->reclaim_bgs_lock);
4911 		if (ret == -ENOSPC) {
4912 			failed++;
4913 		} else if (ret) {
4914 			if (ret == -ETXTBSY) {
4915 				btrfs_warn(fs_info,
4916 		   "could not shrink block group %llu due to active swapfile",
4917 					   chunk_offset);
4918 			}
4919 			goto done;
4920 		}
4921 	} while (key.offset-- > 0);
4922 
4923 	if (failed && !retried) {
4924 		failed = 0;
4925 		retried = true;
4926 		goto again;
4927 	} else if (failed && retried) {
4928 		ret = -ENOSPC;
4929 		goto done;
4930 	}
4931 
4932 	/* Shrinking succeeded, else we would be at "done". */
4933 	trans = btrfs_start_transaction(root, 0);
4934 	if (IS_ERR(trans)) {
4935 		ret = PTR_ERR(trans);
4936 		goto done;
4937 	}
4938 
4939 	mutex_lock(&fs_info->chunk_mutex);
4940 	/* Clear all state bits beyond the shrunk device size */
4941 	clear_extent_bits(&device->alloc_state, new_size, (u64)-1,
4942 			  CHUNK_STATE_MASK);
4943 
4944 	btrfs_device_set_disk_total_bytes(device, new_size);
4945 	if (list_empty(&device->post_commit_list))
4946 		list_add_tail(&device->post_commit_list,
4947 			      &trans->transaction->dev_update_list);
4948 
4949 	WARN_ON(diff > old_total);
4950 	btrfs_set_super_total_bytes(super_copy,
4951 			round_down(old_total - diff, fs_info->sectorsize));
4952 	mutex_unlock(&fs_info->chunk_mutex);
4953 
4954 	btrfs_reserve_chunk_metadata(trans, false);
4955 	/* Now btrfs_update_device() will change the on-disk size. */
4956 	ret = btrfs_update_device(trans, device);
4957 	btrfs_trans_release_chunk_metadata(trans);
4958 	if (ret < 0) {
4959 		btrfs_abort_transaction(trans, ret);
4960 		btrfs_end_transaction(trans);
4961 	} else {
4962 		ret = btrfs_commit_transaction(trans);
4963 	}
4964 done:
4965 	btrfs_free_path(path);
4966 	if (ret) {
4967 		mutex_lock(&fs_info->chunk_mutex);
4968 		btrfs_device_set_total_bytes(device, old_size);
4969 		if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state))
4970 			device->fs_devices->total_rw_bytes += diff;
4971 		atomic64_add(diff, &fs_info->free_chunk_space);
4972 		mutex_unlock(&fs_info->chunk_mutex);
4973 	}
4974 	return ret;
4975 }
4976 
4977 static int btrfs_add_system_chunk(struct btrfs_fs_info *fs_info,
4978 			   struct btrfs_key *key,
4979 			   struct btrfs_chunk *chunk, int item_size)
4980 {
4981 	struct btrfs_super_block *super_copy = fs_info->super_copy;
4982 	struct btrfs_disk_key disk_key;
4983 	u32 array_size;
4984 	u8 *ptr;
4985 
4986 	lockdep_assert_held(&fs_info->chunk_mutex);
4987 
4988 	array_size = btrfs_super_sys_array_size(super_copy);
4989 	if (array_size + item_size + sizeof(disk_key)
4990 			> BTRFS_SYSTEM_CHUNK_ARRAY_SIZE)
4991 		return -EFBIG;
4992 
4993 	ptr = super_copy->sys_chunk_array + array_size;
4994 	btrfs_cpu_key_to_disk(&disk_key, key);
4995 	memcpy(ptr, &disk_key, sizeof(disk_key));
4996 	ptr += sizeof(disk_key);
4997 	memcpy(ptr, chunk, item_size);
4998 	item_size += sizeof(disk_key);
4999 	btrfs_set_super_sys_array_size(super_copy, array_size + item_size);
5000 
5001 	return 0;
5002 }
5003 
5004 /*
5005  * sort the devices in descending order by max_avail, total_avail
5006  */
5007 static int btrfs_cmp_device_info(const void *a, const void *b)
5008 {
5009 	const struct btrfs_device_info *di_a = a;
5010 	const struct btrfs_device_info *di_b = b;
5011 
5012 	if (di_a->max_avail > di_b->max_avail)
5013 		return -1;
5014 	if (di_a->max_avail < di_b->max_avail)
5015 		return 1;
5016 	if (di_a->total_avail > di_b->total_avail)
5017 		return -1;
5018 	if (di_a->total_avail < di_b->total_avail)
5019 		return 1;
5020 	return 0;
5021 }
5022 
5023 static void check_raid56_incompat_flag(struct btrfs_fs_info *info, u64 type)
5024 {
5025 	if (!(type & BTRFS_BLOCK_GROUP_RAID56_MASK))
5026 		return;
5027 
5028 	btrfs_set_fs_incompat(info, RAID56);
5029 }
5030 
5031 static void check_raid1c34_incompat_flag(struct btrfs_fs_info *info, u64 type)
5032 {
5033 	if (!(type & (BTRFS_BLOCK_GROUP_RAID1C3 | BTRFS_BLOCK_GROUP_RAID1C4)))
5034 		return;
5035 
5036 	btrfs_set_fs_incompat(info, RAID1C34);
5037 }
5038 
5039 /*
5040  * Structure used internally for btrfs_create_chunk() function.
5041  * Wraps needed parameters.
5042  */
5043 struct alloc_chunk_ctl {
5044 	u64 start;
5045 	u64 type;
5046 	/* Total number of stripes to allocate */
5047 	int num_stripes;
5048 	/* sub_stripes info for map */
5049 	int sub_stripes;
5050 	/* Stripes per device */
5051 	int dev_stripes;
5052 	/* Maximum number of devices to use */
5053 	int devs_max;
5054 	/* Minimum number of devices to use */
5055 	int devs_min;
5056 	/* ndevs has to be a multiple of this */
5057 	int devs_increment;
5058 	/* Number of copies */
5059 	int ncopies;
5060 	/* Number of stripes worth of bytes to store parity information */
5061 	int nparity;
5062 	u64 max_stripe_size;
5063 	u64 max_chunk_size;
5064 	u64 dev_extent_min;
5065 	u64 stripe_size;
5066 	u64 chunk_size;
5067 	int ndevs;
5068 };
5069 
5070 static void init_alloc_chunk_ctl_policy_regular(
5071 				struct btrfs_fs_devices *fs_devices,
5072 				struct alloc_chunk_ctl *ctl)
5073 {
5074 	u64 type = ctl->type;
5075 
5076 	if (type & BTRFS_BLOCK_GROUP_DATA) {
5077 		ctl->max_stripe_size = SZ_1G;
5078 		ctl->max_chunk_size = BTRFS_MAX_DATA_CHUNK_SIZE;
5079 	} else if (type & BTRFS_BLOCK_GROUP_METADATA) {
5080 		/* For larger filesystems, use larger metadata chunks */
5081 		if (fs_devices->total_rw_bytes > 50ULL * SZ_1G)
5082 			ctl->max_stripe_size = SZ_1G;
5083 		else
5084 			ctl->max_stripe_size = SZ_256M;
5085 		ctl->max_chunk_size = ctl->max_stripe_size;
5086 	} else if (type & BTRFS_BLOCK_GROUP_SYSTEM) {
5087 		ctl->max_stripe_size = SZ_32M;
5088 		ctl->max_chunk_size = 2 * ctl->max_stripe_size;
5089 		ctl->devs_max = min_t(int, ctl->devs_max,
5090 				      BTRFS_MAX_DEVS_SYS_CHUNK);
5091 	} else {
5092 		BUG();
5093 	}
5094 
5095 	/* We don't want a chunk larger than 10% of writable space */
5096 	ctl->max_chunk_size = min(div_factor(fs_devices->total_rw_bytes, 1),
5097 				  ctl->max_chunk_size);
5098 	ctl->dev_extent_min = BTRFS_STRIPE_LEN * ctl->dev_stripes;
5099 }
5100 
5101 static void init_alloc_chunk_ctl_policy_zoned(
5102 				      struct btrfs_fs_devices *fs_devices,
5103 				      struct alloc_chunk_ctl *ctl)
5104 {
5105 	u64 zone_size = fs_devices->fs_info->zone_size;
5106 	u64 limit;
5107 	int min_num_stripes = ctl->devs_min * ctl->dev_stripes;
5108 	int min_data_stripes = (min_num_stripes - ctl->nparity) / ctl->ncopies;
5109 	u64 min_chunk_size = min_data_stripes * zone_size;
5110 	u64 type = ctl->type;
5111 
5112 	ctl->max_stripe_size = zone_size;
5113 	if (type & BTRFS_BLOCK_GROUP_DATA) {
5114 		ctl->max_chunk_size = round_down(BTRFS_MAX_DATA_CHUNK_SIZE,
5115 						 zone_size);
5116 	} else if (type & BTRFS_BLOCK_GROUP_METADATA) {
5117 		ctl->max_chunk_size = ctl->max_stripe_size;
5118 	} else if (type & BTRFS_BLOCK_GROUP_SYSTEM) {
5119 		ctl->max_chunk_size = 2 * ctl->max_stripe_size;
5120 		ctl->devs_max = min_t(int, ctl->devs_max,
5121 				      BTRFS_MAX_DEVS_SYS_CHUNK);
5122 	} else {
5123 		BUG();
5124 	}
5125 
5126 	/* We don't want a chunk larger than 10% of writable space */
5127 	limit = max(round_down(div_factor(fs_devices->total_rw_bytes, 1),
5128 			       zone_size),
5129 		    min_chunk_size);
5130 	ctl->max_chunk_size = min(limit, ctl->max_chunk_size);
5131 	ctl->dev_extent_min = zone_size * ctl->dev_stripes;
5132 }
5133 
5134 static void init_alloc_chunk_ctl(struct btrfs_fs_devices *fs_devices,
5135 				 struct alloc_chunk_ctl *ctl)
5136 {
5137 	int index = btrfs_bg_flags_to_raid_index(ctl->type);
5138 
5139 	ctl->sub_stripes = btrfs_raid_array[index].sub_stripes;
5140 	ctl->dev_stripes = btrfs_raid_array[index].dev_stripes;
5141 	ctl->devs_max = btrfs_raid_array[index].devs_max;
5142 	if (!ctl->devs_max)
5143 		ctl->devs_max = BTRFS_MAX_DEVS(fs_devices->fs_info);
5144 	ctl->devs_min = btrfs_raid_array[index].devs_min;
5145 	ctl->devs_increment = btrfs_raid_array[index].devs_increment;
5146 	ctl->ncopies = btrfs_raid_array[index].ncopies;
5147 	ctl->nparity = btrfs_raid_array[index].nparity;
5148 	ctl->ndevs = 0;
5149 
5150 	switch (fs_devices->chunk_alloc_policy) {
5151 	case BTRFS_CHUNK_ALLOC_REGULAR:
5152 		init_alloc_chunk_ctl_policy_regular(fs_devices, ctl);
5153 		break;
5154 	case BTRFS_CHUNK_ALLOC_ZONED:
5155 		init_alloc_chunk_ctl_policy_zoned(fs_devices, ctl);
5156 		break;
5157 	default:
5158 		BUG();
5159 	}
5160 }
5161 
5162 static int gather_device_info(struct btrfs_fs_devices *fs_devices,
5163 			      struct alloc_chunk_ctl *ctl,
5164 			      struct btrfs_device_info *devices_info)
5165 {
5166 	struct btrfs_fs_info *info = fs_devices->fs_info;
5167 	struct btrfs_device *device;
5168 	u64 total_avail;
5169 	u64 dev_extent_want = ctl->max_stripe_size * ctl->dev_stripes;
5170 	int ret;
5171 	int ndevs = 0;
5172 	u64 max_avail;
5173 	u64 dev_offset;
5174 
5175 	/*
5176 	 * in the first pass through the devices list, we gather information
5177 	 * about the available holes on each device.
5178 	 */
5179 	list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) {
5180 		if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
5181 			WARN(1, KERN_ERR
5182 			       "BTRFS: read-only device in alloc_list\n");
5183 			continue;
5184 		}
5185 
5186 		if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
5187 					&device->dev_state) ||
5188 		    test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state))
5189 			continue;
5190 
5191 		if (device->total_bytes > device->bytes_used)
5192 			total_avail = device->total_bytes - device->bytes_used;
5193 		else
5194 			total_avail = 0;
5195 
5196 		/* If there is no space on this device, skip it. */
5197 		if (total_avail < ctl->dev_extent_min)
5198 			continue;
5199 
5200 		ret = find_free_dev_extent(device, dev_extent_want, &dev_offset,
5201 					   &max_avail);
5202 		if (ret && ret != -ENOSPC)
5203 			return ret;
5204 
5205 		if (ret == 0)
5206 			max_avail = dev_extent_want;
5207 
5208 		if (max_avail < ctl->dev_extent_min) {
5209 			if (btrfs_test_opt(info, ENOSPC_DEBUG))
5210 				btrfs_debug(info,
5211 			"%s: devid %llu has no free space, have=%llu want=%llu",
5212 					    __func__, device->devid, max_avail,
5213 					    ctl->dev_extent_min);
5214 			continue;
5215 		}
5216 
5217 		if (ndevs == fs_devices->rw_devices) {
5218 			WARN(1, "%s: found more than %llu devices\n",
5219 			     __func__, fs_devices->rw_devices);
5220 			break;
5221 		}
5222 		devices_info[ndevs].dev_offset = dev_offset;
5223 		devices_info[ndevs].max_avail = max_avail;
5224 		devices_info[ndevs].total_avail = total_avail;
5225 		devices_info[ndevs].dev = device;
5226 		++ndevs;
5227 	}
5228 	ctl->ndevs = ndevs;
5229 
5230 	/*
5231 	 * now sort the devices by hole size / available space
5232 	 */
5233 	sort(devices_info, ndevs, sizeof(struct btrfs_device_info),
5234 	     btrfs_cmp_device_info, NULL);
5235 
5236 	return 0;
5237 }
5238 
5239 static int decide_stripe_size_regular(struct alloc_chunk_ctl *ctl,
5240 				      struct btrfs_device_info *devices_info)
5241 {
5242 	/* Number of stripes that count for block group size */
5243 	int data_stripes;
5244 
5245 	/*
5246 	 * The primary goal is to maximize the number of stripes, so use as
5247 	 * many devices as possible, even if the stripes are not maximum sized.
5248 	 *
5249 	 * The DUP profile stores more than one stripe per device, the
5250 	 * max_avail is the total size so we have to adjust.
5251 	 */
5252 	ctl->stripe_size = div_u64(devices_info[ctl->ndevs - 1].max_avail,
5253 				   ctl->dev_stripes);
5254 	ctl->num_stripes = ctl->ndevs * ctl->dev_stripes;
5255 
5256 	/* This will have to be fixed for RAID1 and RAID10 over more drives */
5257 	data_stripes = (ctl->num_stripes - ctl->nparity) / ctl->ncopies;
5258 
5259 	/*
5260 	 * Use the number of data stripes to figure out how big this chunk is
5261 	 * really going to be in terms of logical address space, and compare
5262 	 * that answer with the max chunk size. If it's higher, we try to
5263 	 * reduce stripe_size.
5264 	 */
5265 	if (ctl->stripe_size * data_stripes > ctl->max_chunk_size) {
5266 		/*
5267 		 * Reduce stripe_size, round it up to a 16MB boundary again and
5268 		 * then use it, unless it ends up being even bigger than the
5269 		 * previous value we had already.
5270 		 */
5271 		ctl->stripe_size = min(round_up(div_u64(ctl->max_chunk_size,
5272 							data_stripes), SZ_16M),
5273 				       ctl->stripe_size);
5274 	}
5275 
5276 	/* Align to BTRFS_STRIPE_LEN */
5277 	ctl->stripe_size = round_down(ctl->stripe_size, BTRFS_STRIPE_LEN);
5278 	ctl->chunk_size = ctl->stripe_size * data_stripes;
5279 
5280 	return 0;
5281 }
5282 
5283 static int decide_stripe_size_zoned(struct alloc_chunk_ctl *ctl,
5284 				    struct btrfs_device_info *devices_info)
5285 {
5286 	u64 zone_size = devices_info[0].dev->zone_info->zone_size;
5287 	/* Number of stripes that count for block group size */
5288 	int data_stripes;
5289 
5290 	/*
5291 	 * It should hold because:
5292 	 *    dev_extent_min == dev_extent_want == zone_size * dev_stripes
5293 	 */
5294 	ASSERT(devices_info[ctl->ndevs - 1].max_avail == ctl->dev_extent_min);
5295 
5296 	ctl->stripe_size = zone_size;
5297 	ctl->num_stripes = ctl->ndevs * ctl->dev_stripes;
5298 	data_stripes = (ctl->num_stripes - ctl->nparity) / ctl->ncopies;
5299 
5300 	/* stripe_size is fixed in zoned filesysmte. Reduce ndevs instead. */
5301 	if (ctl->stripe_size * data_stripes > ctl->max_chunk_size) {
5302 		ctl->ndevs = div_u64(div_u64(ctl->max_chunk_size * ctl->ncopies,
5303 					     ctl->stripe_size) + ctl->nparity,
5304 				     ctl->dev_stripes);
5305 		ctl->num_stripes = ctl->ndevs * ctl->dev_stripes;
5306 		data_stripes = (ctl->num_stripes - ctl->nparity) / ctl->ncopies;
5307 		ASSERT(ctl->stripe_size * data_stripes <= ctl->max_chunk_size);
5308 	}
5309 
5310 	ctl->chunk_size = ctl->stripe_size * data_stripes;
5311 
5312 	return 0;
5313 }
5314 
5315 static int decide_stripe_size(struct btrfs_fs_devices *fs_devices,
5316 			      struct alloc_chunk_ctl *ctl,
5317 			      struct btrfs_device_info *devices_info)
5318 {
5319 	struct btrfs_fs_info *info = fs_devices->fs_info;
5320 
5321 	/*
5322 	 * Round down to number of usable stripes, devs_increment can be any
5323 	 * number so we can't use round_down() that requires power of 2, while
5324 	 * rounddown is safe.
5325 	 */
5326 	ctl->ndevs = rounddown(ctl->ndevs, ctl->devs_increment);
5327 
5328 	if (ctl->ndevs < ctl->devs_min) {
5329 		if (btrfs_test_opt(info, ENOSPC_DEBUG)) {
5330 			btrfs_debug(info,
5331 	"%s: not enough devices with free space: have=%d minimum required=%d",
5332 				    __func__, ctl->ndevs, ctl->devs_min);
5333 		}
5334 		return -ENOSPC;
5335 	}
5336 
5337 	ctl->ndevs = min(ctl->ndevs, ctl->devs_max);
5338 
5339 	switch (fs_devices->chunk_alloc_policy) {
5340 	case BTRFS_CHUNK_ALLOC_REGULAR:
5341 		return decide_stripe_size_regular(ctl, devices_info);
5342 	case BTRFS_CHUNK_ALLOC_ZONED:
5343 		return decide_stripe_size_zoned(ctl, devices_info);
5344 	default:
5345 		BUG();
5346 	}
5347 }
5348 
5349 static struct btrfs_block_group *create_chunk(struct btrfs_trans_handle *trans,
5350 			struct alloc_chunk_ctl *ctl,
5351 			struct btrfs_device_info *devices_info)
5352 {
5353 	struct btrfs_fs_info *info = trans->fs_info;
5354 	struct map_lookup *map = NULL;
5355 	struct extent_map_tree *em_tree;
5356 	struct btrfs_block_group *block_group;
5357 	struct extent_map *em;
5358 	u64 start = ctl->start;
5359 	u64 type = ctl->type;
5360 	int ret;
5361 	int i;
5362 	int j;
5363 
5364 	map = kmalloc(map_lookup_size(ctl->num_stripes), GFP_NOFS);
5365 	if (!map)
5366 		return ERR_PTR(-ENOMEM);
5367 	map->num_stripes = ctl->num_stripes;
5368 
5369 	for (i = 0; i < ctl->ndevs; ++i) {
5370 		for (j = 0; j < ctl->dev_stripes; ++j) {
5371 			int s = i * ctl->dev_stripes + j;
5372 			map->stripes[s].dev = devices_info[i].dev;
5373 			map->stripes[s].physical = devices_info[i].dev_offset +
5374 						   j * ctl->stripe_size;
5375 		}
5376 	}
5377 	map->stripe_len = BTRFS_STRIPE_LEN;
5378 	map->io_align = BTRFS_STRIPE_LEN;
5379 	map->io_width = BTRFS_STRIPE_LEN;
5380 	map->type = type;
5381 	map->sub_stripes = ctl->sub_stripes;
5382 
5383 	trace_btrfs_chunk_alloc(info, map, start, ctl->chunk_size);
5384 
5385 	em = alloc_extent_map();
5386 	if (!em) {
5387 		kfree(map);
5388 		return ERR_PTR(-ENOMEM);
5389 	}
5390 	set_bit(EXTENT_FLAG_FS_MAPPING, &em->flags);
5391 	em->map_lookup = map;
5392 	em->start = start;
5393 	em->len = ctl->chunk_size;
5394 	em->block_start = 0;
5395 	em->block_len = em->len;
5396 	em->orig_block_len = ctl->stripe_size;
5397 
5398 	em_tree = &info->mapping_tree;
5399 	write_lock(&em_tree->lock);
5400 	ret = add_extent_mapping(em_tree, em, 0);
5401 	if (ret) {
5402 		write_unlock(&em_tree->lock);
5403 		free_extent_map(em);
5404 		return ERR_PTR(ret);
5405 	}
5406 	write_unlock(&em_tree->lock);
5407 
5408 	block_group = btrfs_make_block_group(trans, 0, type, start, ctl->chunk_size);
5409 	if (IS_ERR(block_group))
5410 		goto error_del_extent;
5411 
5412 	for (i = 0; i < map->num_stripes; i++) {
5413 		struct btrfs_device *dev = map->stripes[i].dev;
5414 
5415 		btrfs_device_set_bytes_used(dev,
5416 					    dev->bytes_used + ctl->stripe_size);
5417 		if (list_empty(&dev->post_commit_list))
5418 			list_add_tail(&dev->post_commit_list,
5419 				      &trans->transaction->dev_update_list);
5420 	}
5421 
5422 	atomic64_sub(ctl->stripe_size * map->num_stripes,
5423 		     &info->free_chunk_space);
5424 
5425 	free_extent_map(em);
5426 	check_raid56_incompat_flag(info, type);
5427 	check_raid1c34_incompat_flag(info, type);
5428 
5429 	return block_group;
5430 
5431 error_del_extent:
5432 	write_lock(&em_tree->lock);
5433 	remove_extent_mapping(em_tree, em);
5434 	write_unlock(&em_tree->lock);
5435 
5436 	/* One for our allocation */
5437 	free_extent_map(em);
5438 	/* One for the tree reference */
5439 	free_extent_map(em);
5440 
5441 	return block_group;
5442 }
5443 
5444 struct btrfs_block_group *btrfs_create_chunk(struct btrfs_trans_handle *trans,
5445 					    u64 type)
5446 {
5447 	struct btrfs_fs_info *info = trans->fs_info;
5448 	struct btrfs_fs_devices *fs_devices = info->fs_devices;
5449 	struct btrfs_device_info *devices_info = NULL;
5450 	struct alloc_chunk_ctl ctl;
5451 	struct btrfs_block_group *block_group;
5452 	int ret;
5453 
5454 	lockdep_assert_held(&info->chunk_mutex);
5455 
5456 	if (!alloc_profile_is_valid(type, 0)) {
5457 		ASSERT(0);
5458 		return ERR_PTR(-EINVAL);
5459 	}
5460 
5461 	if (list_empty(&fs_devices->alloc_list)) {
5462 		if (btrfs_test_opt(info, ENOSPC_DEBUG))
5463 			btrfs_debug(info, "%s: no writable device", __func__);
5464 		return ERR_PTR(-ENOSPC);
5465 	}
5466 
5467 	if (!(type & BTRFS_BLOCK_GROUP_TYPE_MASK)) {
5468 		btrfs_err(info, "invalid chunk type 0x%llx requested", type);
5469 		ASSERT(0);
5470 		return ERR_PTR(-EINVAL);
5471 	}
5472 
5473 	ctl.start = find_next_chunk(info);
5474 	ctl.type = type;
5475 	init_alloc_chunk_ctl(fs_devices, &ctl);
5476 
5477 	devices_info = kcalloc(fs_devices->rw_devices, sizeof(*devices_info),
5478 			       GFP_NOFS);
5479 	if (!devices_info)
5480 		return ERR_PTR(-ENOMEM);
5481 
5482 	ret = gather_device_info(fs_devices, &ctl, devices_info);
5483 	if (ret < 0) {
5484 		block_group = ERR_PTR(ret);
5485 		goto out;
5486 	}
5487 
5488 	ret = decide_stripe_size(fs_devices, &ctl, devices_info);
5489 	if (ret < 0) {
5490 		block_group = ERR_PTR(ret);
5491 		goto out;
5492 	}
5493 
5494 	block_group = create_chunk(trans, &ctl, devices_info);
5495 
5496 out:
5497 	kfree(devices_info);
5498 	return block_group;
5499 }
5500 
5501 /*
5502  * This function, btrfs_chunk_alloc_add_chunk_item(), typically belongs to the
5503  * phase 1 of chunk allocation. It belongs to phase 2 only when allocating system
5504  * chunks.
5505  *
5506  * See the comment at btrfs_chunk_alloc() for details about the chunk allocation
5507  * phases.
5508  */
5509 int btrfs_chunk_alloc_add_chunk_item(struct btrfs_trans_handle *trans,
5510 				     struct btrfs_block_group *bg)
5511 {
5512 	struct btrfs_fs_info *fs_info = trans->fs_info;
5513 	struct btrfs_root *chunk_root = fs_info->chunk_root;
5514 	struct btrfs_key key;
5515 	struct btrfs_chunk *chunk;
5516 	struct btrfs_stripe *stripe;
5517 	struct extent_map *em;
5518 	struct map_lookup *map;
5519 	size_t item_size;
5520 	int i;
5521 	int ret;
5522 
5523 	/*
5524 	 * We take the chunk_mutex for 2 reasons:
5525 	 *
5526 	 * 1) Updates and insertions in the chunk btree must be done while holding
5527 	 *    the chunk_mutex, as well as updating the system chunk array in the
5528 	 *    superblock. See the comment on top of btrfs_chunk_alloc() for the
5529 	 *    details;
5530 	 *
5531 	 * 2) To prevent races with the final phase of a device replace operation
5532 	 *    that replaces the device object associated with the map's stripes,
5533 	 *    because the device object's id can change at any time during that
5534 	 *    final phase of the device replace operation
5535 	 *    (dev-replace.c:btrfs_dev_replace_finishing()), so we could grab the
5536 	 *    replaced device and then see it with an ID of BTRFS_DEV_REPLACE_DEVID,
5537 	 *    which would cause a failure when updating the device item, which does
5538 	 *    not exists, or persisting a stripe of the chunk item with such ID.
5539 	 *    Here we can't use the device_list_mutex because our caller already
5540 	 *    has locked the chunk_mutex, and the final phase of device replace
5541 	 *    acquires both mutexes - first the device_list_mutex and then the
5542 	 *    chunk_mutex. Using any of those two mutexes protects us from a
5543 	 *    concurrent device replace.
5544 	 */
5545 	lockdep_assert_held(&fs_info->chunk_mutex);
5546 
5547 	em = btrfs_get_chunk_map(fs_info, bg->start, bg->length);
5548 	if (IS_ERR(em)) {
5549 		ret = PTR_ERR(em);
5550 		btrfs_abort_transaction(trans, ret);
5551 		return ret;
5552 	}
5553 
5554 	map = em->map_lookup;
5555 	item_size = btrfs_chunk_item_size(map->num_stripes);
5556 
5557 	chunk = kzalloc(item_size, GFP_NOFS);
5558 	if (!chunk) {
5559 		ret = -ENOMEM;
5560 		btrfs_abort_transaction(trans, ret);
5561 		goto out;
5562 	}
5563 
5564 	for (i = 0; i < map->num_stripes; i++) {
5565 		struct btrfs_device *device = map->stripes[i].dev;
5566 
5567 		ret = btrfs_update_device(trans, device);
5568 		if (ret)
5569 			goto out;
5570 	}
5571 
5572 	stripe = &chunk->stripe;
5573 	for (i = 0; i < map->num_stripes; i++) {
5574 		struct btrfs_device *device = map->stripes[i].dev;
5575 		const u64 dev_offset = map->stripes[i].physical;
5576 
5577 		btrfs_set_stack_stripe_devid(stripe, device->devid);
5578 		btrfs_set_stack_stripe_offset(stripe, dev_offset);
5579 		memcpy(stripe->dev_uuid, device->uuid, BTRFS_UUID_SIZE);
5580 		stripe++;
5581 	}
5582 
5583 	btrfs_set_stack_chunk_length(chunk, bg->length);
5584 	btrfs_set_stack_chunk_owner(chunk, BTRFS_EXTENT_TREE_OBJECTID);
5585 	btrfs_set_stack_chunk_stripe_len(chunk, map->stripe_len);
5586 	btrfs_set_stack_chunk_type(chunk, map->type);
5587 	btrfs_set_stack_chunk_num_stripes(chunk, map->num_stripes);
5588 	btrfs_set_stack_chunk_io_align(chunk, map->stripe_len);
5589 	btrfs_set_stack_chunk_io_width(chunk, map->stripe_len);
5590 	btrfs_set_stack_chunk_sector_size(chunk, fs_info->sectorsize);
5591 	btrfs_set_stack_chunk_sub_stripes(chunk, map->sub_stripes);
5592 
5593 	key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
5594 	key.type = BTRFS_CHUNK_ITEM_KEY;
5595 	key.offset = bg->start;
5596 
5597 	ret = btrfs_insert_item(trans, chunk_root, &key, chunk, item_size);
5598 	if (ret)
5599 		goto out;
5600 
5601 	bg->chunk_item_inserted = 1;
5602 
5603 	if (map->type & BTRFS_BLOCK_GROUP_SYSTEM) {
5604 		ret = btrfs_add_system_chunk(fs_info, &key, chunk, item_size);
5605 		if (ret)
5606 			goto out;
5607 	}
5608 
5609 out:
5610 	kfree(chunk);
5611 	free_extent_map(em);
5612 	return ret;
5613 }
5614 
5615 static noinline int init_first_rw_device(struct btrfs_trans_handle *trans)
5616 {
5617 	struct btrfs_fs_info *fs_info = trans->fs_info;
5618 	u64 alloc_profile;
5619 	struct btrfs_block_group *meta_bg;
5620 	struct btrfs_block_group *sys_bg;
5621 
5622 	/*
5623 	 * When adding a new device for sprouting, the seed device is read-only
5624 	 * so we must first allocate a metadata and a system chunk. But before
5625 	 * adding the block group items to the extent, device and chunk btrees,
5626 	 * we must first:
5627 	 *
5628 	 * 1) Create both chunks without doing any changes to the btrees, as
5629 	 *    otherwise we would get -ENOSPC since the block groups from the
5630 	 *    seed device are read-only;
5631 	 *
5632 	 * 2) Add the device item for the new sprout device - finishing the setup
5633 	 *    of a new block group requires updating the device item in the chunk
5634 	 *    btree, so it must exist when we attempt to do it. The previous step
5635 	 *    ensures this does not fail with -ENOSPC.
5636 	 *
5637 	 * After that we can add the block group items to their btrees:
5638 	 * update existing device item in the chunk btree, add a new block group
5639 	 * item to the extent btree, add a new chunk item to the chunk btree and
5640 	 * finally add the new device extent items to the devices btree.
5641 	 */
5642 
5643 	alloc_profile = btrfs_metadata_alloc_profile(fs_info);
5644 	meta_bg = btrfs_create_chunk(trans, alloc_profile);
5645 	if (IS_ERR(meta_bg))
5646 		return PTR_ERR(meta_bg);
5647 
5648 	alloc_profile = btrfs_system_alloc_profile(fs_info);
5649 	sys_bg = btrfs_create_chunk(trans, alloc_profile);
5650 	if (IS_ERR(sys_bg))
5651 		return PTR_ERR(sys_bg);
5652 
5653 	return 0;
5654 }
5655 
5656 static inline int btrfs_chunk_max_errors(struct map_lookup *map)
5657 {
5658 	const int index = btrfs_bg_flags_to_raid_index(map->type);
5659 
5660 	return btrfs_raid_array[index].tolerated_failures;
5661 }
5662 
5663 bool btrfs_chunk_writeable(struct btrfs_fs_info *fs_info, u64 chunk_offset)
5664 {
5665 	struct extent_map *em;
5666 	struct map_lookup *map;
5667 	int miss_ndevs = 0;
5668 	int i;
5669 	bool ret = true;
5670 
5671 	em = btrfs_get_chunk_map(fs_info, chunk_offset, 1);
5672 	if (IS_ERR(em))
5673 		return false;
5674 
5675 	map = em->map_lookup;
5676 	for (i = 0; i < map->num_stripes; i++) {
5677 		if (test_bit(BTRFS_DEV_STATE_MISSING,
5678 					&map->stripes[i].dev->dev_state)) {
5679 			miss_ndevs++;
5680 			continue;
5681 		}
5682 		if (!test_bit(BTRFS_DEV_STATE_WRITEABLE,
5683 					&map->stripes[i].dev->dev_state)) {
5684 			ret = false;
5685 			goto end;
5686 		}
5687 	}
5688 
5689 	/*
5690 	 * If the number of missing devices is larger than max errors, we can
5691 	 * not write the data into that chunk successfully.
5692 	 */
5693 	if (miss_ndevs > btrfs_chunk_max_errors(map))
5694 		ret = false;
5695 end:
5696 	free_extent_map(em);
5697 	return ret;
5698 }
5699 
5700 void btrfs_mapping_tree_free(struct extent_map_tree *tree)
5701 {
5702 	struct extent_map *em;
5703 
5704 	while (1) {
5705 		write_lock(&tree->lock);
5706 		em = lookup_extent_mapping(tree, 0, (u64)-1);
5707 		if (em)
5708 			remove_extent_mapping(tree, em);
5709 		write_unlock(&tree->lock);
5710 		if (!em)
5711 			break;
5712 		/* once for us */
5713 		free_extent_map(em);
5714 		/* once for the tree */
5715 		free_extent_map(em);
5716 	}
5717 }
5718 
5719 int btrfs_num_copies(struct btrfs_fs_info *fs_info, u64 logical, u64 len)
5720 {
5721 	struct extent_map *em;
5722 	struct map_lookup *map;
5723 	int ret;
5724 
5725 	em = btrfs_get_chunk_map(fs_info, logical, len);
5726 	if (IS_ERR(em))
5727 		/*
5728 		 * We could return errors for these cases, but that could get
5729 		 * ugly and we'd probably do the same thing which is just not do
5730 		 * anything else and exit, so return 1 so the callers don't try
5731 		 * to use other copies.
5732 		 */
5733 		return 1;
5734 
5735 	map = em->map_lookup;
5736 	if (map->type & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1_MASK))
5737 		ret = map->num_stripes;
5738 	else if (map->type & BTRFS_BLOCK_GROUP_RAID10)
5739 		ret = map->sub_stripes;
5740 	else if (map->type & BTRFS_BLOCK_GROUP_RAID5)
5741 		ret = 2;
5742 	else if (map->type & BTRFS_BLOCK_GROUP_RAID6)
5743 		/*
5744 		 * There could be two corrupted data stripes, we need
5745 		 * to loop retry in order to rebuild the correct data.
5746 		 *
5747 		 * Fail a stripe at a time on every retry except the
5748 		 * stripe under reconstruction.
5749 		 */
5750 		ret = map->num_stripes;
5751 	else
5752 		ret = 1;
5753 	free_extent_map(em);
5754 
5755 	down_read(&fs_info->dev_replace.rwsem);
5756 	if (btrfs_dev_replace_is_ongoing(&fs_info->dev_replace) &&
5757 	    fs_info->dev_replace.tgtdev)
5758 		ret++;
5759 	up_read(&fs_info->dev_replace.rwsem);
5760 
5761 	return ret;
5762 }
5763 
5764 unsigned long btrfs_full_stripe_len(struct btrfs_fs_info *fs_info,
5765 				    u64 logical)
5766 {
5767 	struct extent_map *em;
5768 	struct map_lookup *map;
5769 	unsigned long len = fs_info->sectorsize;
5770 
5771 	em = btrfs_get_chunk_map(fs_info, logical, len);
5772 
5773 	if (!WARN_ON(IS_ERR(em))) {
5774 		map = em->map_lookup;
5775 		if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
5776 			len = map->stripe_len * nr_data_stripes(map);
5777 		free_extent_map(em);
5778 	}
5779 	return len;
5780 }
5781 
5782 int btrfs_is_parity_mirror(struct btrfs_fs_info *fs_info, u64 logical, u64 len)
5783 {
5784 	struct extent_map *em;
5785 	struct map_lookup *map;
5786 	int ret = 0;
5787 
5788 	em = btrfs_get_chunk_map(fs_info, logical, len);
5789 
5790 	if(!WARN_ON(IS_ERR(em))) {
5791 		map = em->map_lookup;
5792 		if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
5793 			ret = 1;
5794 		free_extent_map(em);
5795 	}
5796 	return ret;
5797 }
5798 
5799 static int find_live_mirror(struct btrfs_fs_info *fs_info,
5800 			    struct map_lookup *map, int first,
5801 			    int dev_replace_is_ongoing)
5802 {
5803 	int i;
5804 	int num_stripes;
5805 	int preferred_mirror;
5806 	int tolerance;
5807 	struct btrfs_device *srcdev;
5808 
5809 	ASSERT((map->type &
5810 		 (BTRFS_BLOCK_GROUP_RAID1_MASK | BTRFS_BLOCK_GROUP_RAID10)));
5811 
5812 	if (map->type & BTRFS_BLOCK_GROUP_RAID10)
5813 		num_stripes = map->sub_stripes;
5814 	else
5815 		num_stripes = map->num_stripes;
5816 
5817 	switch (fs_info->fs_devices->read_policy) {
5818 	default:
5819 		/* Shouldn't happen, just warn and use pid instead of failing */
5820 		btrfs_warn_rl(fs_info,
5821 			      "unknown read_policy type %u, reset to pid",
5822 			      fs_info->fs_devices->read_policy);
5823 		fs_info->fs_devices->read_policy = BTRFS_READ_POLICY_PID;
5824 		fallthrough;
5825 	case BTRFS_READ_POLICY_PID:
5826 		preferred_mirror = first + (current->pid % num_stripes);
5827 		break;
5828 	}
5829 
5830 	if (dev_replace_is_ongoing &&
5831 	    fs_info->dev_replace.cont_reading_from_srcdev_mode ==
5832 	     BTRFS_DEV_REPLACE_ITEM_CONT_READING_FROM_SRCDEV_MODE_AVOID)
5833 		srcdev = fs_info->dev_replace.srcdev;
5834 	else
5835 		srcdev = NULL;
5836 
5837 	/*
5838 	 * try to avoid the drive that is the source drive for a
5839 	 * dev-replace procedure, only choose it if no other non-missing
5840 	 * mirror is available
5841 	 */
5842 	for (tolerance = 0; tolerance < 2; tolerance++) {
5843 		if (map->stripes[preferred_mirror].dev->bdev &&
5844 		    (tolerance || map->stripes[preferred_mirror].dev != srcdev))
5845 			return preferred_mirror;
5846 		for (i = first; i < first + num_stripes; i++) {
5847 			if (map->stripes[i].dev->bdev &&
5848 			    (tolerance || map->stripes[i].dev != srcdev))
5849 				return i;
5850 		}
5851 	}
5852 
5853 	/* we couldn't find one that doesn't fail.  Just return something
5854 	 * and the io error handling code will clean up eventually
5855 	 */
5856 	return preferred_mirror;
5857 }
5858 
5859 /* Bubble-sort the stripe set to put the parity/syndrome stripes last */
5860 static void sort_parity_stripes(struct btrfs_io_context *bioc, int num_stripes)
5861 {
5862 	int i;
5863 	int again = 1;
5864 
5865 	while (again) {
5866 		again = 0;
5867 		for (i = 0; i < num_stripes - 1; i++) {
5868 			/* Swap if parity is on a smaller index */
5869 			if (bioc->raid_map[i] > bioc->raid_map[i + 1]) {
5870 				swap(bioc->stripes[i], bioc->stripes[i + 1]);
5871 				swap(bioc->raid_map[i], bioc->raid_map[i + 1]);
5872 				again = 1;
5873 			}
5874 		}
5875 	}
5876 }
5877 
5878 static struct btrfs_io_context *alloc_btrfs_io_context(struct btrfs_fs_info *fs_info,
5879 						       int total_stripes,
5880 						       int real_stripes)
5881 {
5882 	struct btrfs_io_context *bioc = kzalloc(
5883 		 /* The size of btrfs_io_context */
5884 		sizeof(struct btrfs_io_context) +
5885 		/* Plus the variable array for the stripes */
5886 		sizeof(struct btrfs_io_stripe) * (total_stripes) +
5887 		/* Plus the variable array for the tgt dev */
5888 		sizeof(int) * (real_stripes) +
5889 		/*
5890 		 * Plus the raid_map, which includes both the tgt dev
5891 		 * and the stripes.
5892 		 */
5893 		sizeof(u64) * (total_stripes),
5894 		GFP_NOFS|__GFP_NOFAIL);
5895 
5896 	atomic_set(&bioc->error, 0);
5897 	refcount_set(&bioc->refs, 1);
5898 
5899 	bioc->fs_info = fs_info;
5900 	bioc->tgtdev_map = (int *)(bioc->stripes + total_stripes);
5901 	bioc->raid_map = (u64 *)(bioc->tgtdev_map + real_stripes);
5902 
5903 	return bioc;
5904 }
5905 
5906 void btrfs_get_bioc(struct btrfs_io_context *bioc)
5907 {
5908 	WARN_ON(!refcount_read(&bioc->refs));
5909 	refcount_inc(&bioc->refs);
5910 }
5911 
5912 void btrfs_put_bioc(struct btrfs_io_context *bioc)
5913 {
5914 	if (!bioc)
5915 		return;
5916 	if (refcount_dec_and_test(&bioc->refs))
5917 		kfree(bioc);
5918 }
5919 
5920 /* can REQ_OP_DISCARD be sent with other REQ like REQ_OP_WRITE? */
5921 /*
5922  * Please note that, discard won't be sent to target device of device
5923  * replace.
5924  */
5925 static int __btrfs_map_block_for_discard(struct btrfs_fs_info *fs_info,
5926 					 u64 logical, u64 *length_ret,
5927 					 struct btrfs_io_context **bioc_ret)
5928 {
5929 	struct extent_map *em;
5930 	struct map_lookup *map;
5931 	struct btrfs_io_context *bioc;
5932 	u64 length = *length_ret;
5933 	u64 offset;
5934 	u64 stripe_nr;
5935 	u64 stripe_nr_end;
5936 	u64 stripe_end_offset;
5937 	u64 stripe_cnt;
5938 	u64 stripe_len;
5939 	u64 stripe_offset;
5940 	u64 num_stripes;
5941 	u32 stripe_index;
5942 	u32 factor = 0;
5943 	u32 sub_stripes = 0;
5944 	u64 stripes_per_dev = 0;
5945 	u32 remaining_stripes = 0;
5946 	u32 last_stripe = 0;
5947 	int ret = 0;
5948 	int i;
5949 
5950 	/* Discard always returns a bioc. */
5951 	ASSERT(bioc_ret);
5952 
5953 	em = btrfs_get_chunk_map(fs_info, logical, length);
5954 	if (IS_ERR(em))
5955 		return PTR_ERR(em);
5956 
5957 	map = em->map_lookup;
5958 	/* we don't discard raid56 yet */
5959 	if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
5960 		ret = -EOPNOTSUPP;
5961 		goto out;
5962 	}
5963 
5964 	offset = logical - em->start;
5965 	length = min_t(u64, em->start + em->len - logical, length);
5966 	*length_ret = length;
5967 
5968 	stripe_len = map->stripe_len;
5969 	/*
5970 	 * stripe_nr counts the total number of stripes we have to stride
5971 	 * to get to this block
5972 	 */
5973 	stripe_nr = div64_u64(offset, stripe_len);
5974 
5975 	/* stripe_offset is the offset of this block in its stripe */
5976 	stripe_offset = offset - stripe_nr * stripe_len;
5977 
5978 	stripe_nr_end = round_up(offset + length, map->stripe_len);
5979 	stripe_nr_end = div64_u64(stripe_nr_end, map->stripe_len);
5980 	stripe_cnt = stripe_nr_end - stripe_nr;
5981 	stripe_end_offset = stripe_nr_end * map->stripe_len -
5982 			    (offset + length);
5983 	/*
5984 	 * after this, stripe_nr is the number of stripes on this
5985 	 * device we have to walk to find the data, and stripe_index is
5986 	 * the number of our device in the stripe array
5987 	 */
5988 	num_stripes = 1;
5989 	stripe_index = 0;
5990 	if (map->type & (BTRFS_BLOCK_GROUP_RAID0 |
5991 			 BTRFS_BLOCK_GROUP_RAID10)) {
5992 		if (map->type & BTRFS_BLOCK_GROUP_RAID0)
5993 			sub_stripes = 1;
5994 		else
5995 			sub_stripes = map->sub_stripes;
5996 
5997 		factor = map->num_stripes / sub_stripes;
5998 		num_stripes = min_t(u64, map->num_stripes,
5999 				    sub_stripes * stripe_cnt);
6000 		stripe_nr = div_u64_rem(stripe_nr, factor, &stripe_index);
6001 		stripe_index *= sub_stripes;
6002 		stripes_per_dev = div_u64_rem(stripe_cnt, factor,
6003 					      &remaining_stripes);
6004 		div_u64_rem(stripe_nr_end - 1, factor, &last_stripe);
6005 		last_stripe *= sub_stripes;
6006 	} else if (map->type & (BTRFS_BLOCK_GROUP_RAID1_MASK |
6007 				BTRFS_BLOCK_GROUP_DUP)) {
6008 		num_stripes = map->num_stripes;
6009 	} else {
6010 		stripe_nr = div_u64_rem(stripe_nr, map->num_stripes,
6011 					&stripe_index);
6012 	}
6013 
6014 	bioc = alloc_btrfs_io_context(fs_info, num_stripes, 0);
6015 	if (!bioc) {
6016 		ret = -ENOMEM;
6017 		goto out;
6018 	}
6019 
6020 	for (i = 0; i < num_stripes; i++) {
6021 		bioc->stripes[i].physical =
6022 			map->stripes[stripe_index].physical +
6023 			stripe_offset + stripe_nr * map->stripe_len;
6024 		bioc->stripes[i].dev = map->stripes[stripe_index].dev;
6025 
6026 		if (map->type & (BTRFS_BLOCK_GROUP_RAID0 |
6027 				 BTRFS_BLOCK_GROUP_RAID10)) {
6028 			bioc->stripes[i].length = stripes_per_dev *
6029 				map->stripe_len;
6030 
6031 			if (i / sub_stripes < remaining_stripes)
6032 				bioc->stripes[i].length += map->stripe_len;
6033 
6034 			/*
6035 			 * Special for the first stripe and
6036 			 * the last stripe:
6037 			 *
6038 			 * |-------|...|-------|
6039 			 *     |----------|
6040 			 *    off     end_off
6041 			 */
6042 			if (i < sub_stripes)
6043 				bioc->stripes[i].length -= stripe_offset;
6044 
6045 			if (stripe_index >= last_stripe &&
6046 			    stripe_index <= (last_stripe +
6047 					     sub_stripes - 1))
6048 				bioc->stripes[i].length -= stripe_end_offset;
6049 
6050 			if (i == sub_stripes - 1)
6051 				stripe_offset = 0;
6052 		} else {
6053 			bioc->stripes[i].length = length;
6054 		}
6055 
6056 		stripe_index++;
6057 		if (stripe_index == map->num_stripes) {
6058 			stripe_index = 0;
6059 			stripe_nr++;
6060 		}
6061 	}
6062 
6063 	*bioc_ret = bioc;
6064 	bioc->map_type = map->type;
6065 	bioc->num_stripes = num_stripes;
6066 out:
6067 	free_extent_map(em);
6068 	return ret;
6069 }
6070 
6071 /*
6072  * In dev-replace case, for repair case (that's the only case where the mirror
6073  * is selected explicitly when calling btrfs_map_block), blocks left of the
6074  * left cursor can also be read from the target drive.
6075  *
6076  * For REQ_GET_READ_MIRRORS, the target drive is added as the last one to the
6077  * array of stripes.
6078  * For READ, it also needs to be supported using the same mirror number.
6079  *
6080  * If the requested block is not left of the left cursor, EIO is returned. This
6081  * can happen because btrfs_num_copies() returns one more in the dev-replace
6082  * case.
6083  */
6084 static int get_extra_mirror_from_replace(struct btrfs_fs_info *fs_info,
6085 					 u64 logical, u64 length,
6086 					 u64 srcdev_devid, int *mirror_num,
6087 					 u64 *physical)
6088 {
6089 	struct btrfs_io_context *bioc = NULL;
6090 	int num_stripes;
6091 	int index_srcdev = 0;
6092 	int found = 0;
6093 	u64 physical_of_found = 0;
6094 	int i;
6095 	int ret = 0;
6096 
6097 	ret = __btrfs_map_block(fs_info, BTRFS_MAP_GET_READ_MIRRORS,
6098 				logical, &length, &bioc, 0, 0);
6099 	if (ret) {
6100 		ASSERT(bioc == NULL);
6101 		return ret;
6102 	}
6103 
6104 	num_stripes = bioc->num_stripes;
6105 	if (*mirror_num > num_stripes) {
6106 		/*
6107 		 * BTRFS_MAP_GET_READ_MIRRORS does not contain this mirror,
6108 		 * that means that the requested area is not left of the left
6109 		 * cursor
6110 		 */
6111 		btrfs_put_bioc(bioc);
6112 		return -EIO;
6113 	}
6114 
6115 	/*
6116 	 * process the rest of the function using the mirror_num of the source
6117 	 * drive. Therefore look it up first.  At the end, patch the device
6118 	 * pointer to the one of the target drive.
6119 	 */
6120 	for (i = 0; i < num_stripes; i++) {
6121 		if (bioc->stripes[i].dev->devid != srcdev_devid)
6122 			continue;
6123 
6124 		/*
6125 		 * In case of DUP, in order to keep it simple, only add the
6126 		 * mirror with the lowest physical address
6127 		 */
6128 		if (found &&
6129 		    physical_of_found <= bioc->stripes[i].physical)
6130 			continue;
6131 
6132 		index_srcdev = i;
6133 		found = 1;
6134 		physical_of_found = bioc->stripes[i].physical;
6135 	}
6136 
6137 	btrfs_put_bioc(bioc);
6138 
6139 	ASSERT(found);
6140 	if (!found)
6141 		return -EIO;
6142 
6143 	*mirror_num = index_srcdev + 1;
6144 	*physical = physical_of_found;
6145 	return ret;
6146 }
6147 
6148 static bool is_block_group_to_copy(struct btrfs_fs_info *fs_info, u64 logical)
6149 {
6150 	struct btrfs_block_group *cache;
6151 	bool ret;
6152 
6153 	/* Non zoned filesystem does not use "to_copy" flag */
6154 	if (!btrfs_is_zoned(fs_info))
6155 		return false;
6156 
6157 	cache = btrfs_lookup_block_group(fs_info, logical);
6158 
6159 	spin_lock(&cache->lock);
6160 	ret = cache->to_copy;
6161 	spin_unlock(&cache->lock);
6162 
6163 	btrfs_put_block_group(cache);
6164 	return ret;
6165 }
6166 
6167 static void handle_ops_on_dev_replace(enum btrfs_map_op op,
6168 				      struct btrfs_io_context **bioc_ret,
6169 				      struct btrfs_dev_replace *dev_replace,
6170 				      u64 logical,
6171 				      int *num_stripes_ret, int *max_errors_ret)
6172 {
6173 	struct btrfs_io_context *bioc = *bioc_ret;
6174 	u64 srcdev_devid = dev_replace->srcdev->devid;
6175 	int tgtdev_indexes = 0;
6176 	int num_stripes = *num_stripes_ret;
6177 	int max_errors = *max_errors_ret;
6178 	int i;
6179 
6180 	if (op == BTRFS_MAP_WRITE) {
6181 		int index_where_to_add;
6182 
6183 		/*
6184 		 * A block group which have "to_copy" set will eventually
6185 		 * copied by dev-replace process. We can avoid cloning IO here.
6186 		 */
6187 		if (is_block_group_to_copy(dev_replace->srcdev->fs_info, logical))
6188 			return;
6189 
6190 		/*
6191 		 * duplicate the write operations while the dev replace
6192 		 * procedure is running. Since the copying of the old disk to
6193 		 * the new disk takes place at run time while the filesystem is
6194 		 * mounted writable, the regular write operations to the old
6195 		 * disk have to be duplicated to go to the new disk as well.
6196 		 *
6197 		 * Note that device->missing is handled by the caller, and that
6198 		 * the write to the old disk is already set up in the stripes
6199 		 * array.
6200 		 */
6201 		index_where_to_add = num_stripes;
6202 		for (i = 0; i < num_stripes; i++) {
6203 			if (bioc->stripes[i].dev->devid == srcdev_devid) {
6204 				/* write to new disk, too */
6205 				struct btrfs_io_stripe *new =
6206 					bioc->stripes + index_where_to_add;
6207 				struct btrfs_io_stripe *old =
6208 					bioc->stripes + i;
6209 
6210 				new->physical = old->physical;
6211 				new->length = old->length;
6212 				new->dev = dev_replace->tgtdev;
6213 				bioc->tgtdev_map[i] = index_where_to_add;
6214 				index_where_to_add++;
6215 				max_errors++;
6216 				tgtdev_indexes++;
6217 			}
6218 		}
6219 		num_stripes = index_where_to_add;
6220 	} else if (op == BTRFS_MAP_GET_READ_MIRRORS) {
6221 		int index_srcdev = 0;
6222 		int found = 0;
6223 		u64 physical_of_found = 0;
6224 
6225 		/*
6226 		 * During the dev-replace procedure, the target drive can also
6227 		 * be used to read data in case it is needed to repair a corrupt
6228 		 * block elsewhere. This is possible if the requested area is
6229 		 * left of the left cursor. In this area, the target drive is a
6230 		 * full copy of the source drive.
6231 		 */
6232 		for (i = 0; i < num_stripes; i++) {
6233 			if (bioc->stripes[i].dev->devid == srcdev_devid) {
6234 				/*
6235 				 * In case of DUP, in order to keep it simple,
6236 				 * only add the mirror with the lowest physical
6237 				 * address
6238 				 */
6239 				if (found &&
6240 				    physical_of_found <= bioc->stripes[i].physical)
6241 					continue;
6242 				index_srcdev = i;
6243 				found = 1;
6244 				physical_of_found = bioc->stripes[i].physical;
6245 			}
6246 		}
6247 		if (found) {
6248 			struct btrfs_io_stripe *tgtdev_stripe =
6249 				bioc->stripes + num_stripes;
6250 
6251 			tgtdev_stripe->physical = physical_of_found;
6252 			tgtdev_stripe->length =
6253 				bioc->stripes[index_srcdev].length;
6254 			tgtdev_stripe->dev = dev_replace->tgtdev;
6255 			bioc->tgtdev_map[index_srcdev] = num_stripes;
6256 
6257 			tgtdev_indexes++;
6258 			num_stripes++;
6259 		}
6260 	}
6261 
6262 	*num_stripes_ret = num_stripes;
6263 	*max_errors_ret = max_errors;
6264 	bioc->num_tgtdevs = tgtdev_indexes;
6265 	*bioc_ret = bioc;
6266 }
6267 
6268 static bool need_full_stripe(enum btrfs_map_op op)
6269 {
6270 	return (op == BTRFS_MAP_WRITE || op == BTRFS_MAP_GET_READ_MIRRORS);
6271 }
6272 
6273 /*
6274  * Calculate the geometry of a particular (address, len) tuple. This
6275  * information is used to calculate how big a particular bio can get before it
6276  * straddles a stripe.
6277  *
6278  * @fs_info: the filesystem
6279  * @em:      mapping containing the logical extent
6280  * @op:      type of operation - write or read
6281  * @logical: address that we want to figure out the geometry of
6282  * @io_geom: pointer used to return values
6283  *
6284  * Returns < 0 in case a chunk for the given logical address cannot be found,
6285  * usually shouldn't happen unless @logical is corrupted, 0 otherwise.
6286  */
6287 int btrfs_get_io_geometry(struct btrfs_fs_info *fs_info, struct extent_map *em,
6288 			  enum btrfs_map_op op, u64 logical,
6289 			  struct btrfs_io_geometry *io_geom)
6290 {
6291 	struct map_lookup *map;
6292 	u64 len;
6293 	u64 offset;
6294 	u64 stripe_offset;
6295 	u64 stripe_nr;
6296 	u32 stripe_len;
6297 	u64 raid56_full_stripe_start = (u64)-1;
6298 	int data_stripes;
6299 
6300 	ASSERT(op != BTRFS_MAP_DISCARD);
6301 
6302 	map = em->map_lookup;
6303 	/* Offset of this logical address in the chunk */
6304 	offset = logical - em->start;
6305 	/* Len of a stripe in a chunk */
6306 	stripe_len = map->stripe_len;
6307 	/*
6308 	 * Stripe_nr is where this block falls in
6309 	 * stripe_offset is the offset of this block in its stripe.
6310 	 */
6311 	stripe_nr = div64_u64_rem(offset, stripe_len, &stripe_offset);
6312 	ASSERT(stripe_offset < U32_MAX);
6313 
6314 	data_stripes = nr_data_stripes(map);
6315 
6316 	/* Only stripe based profiles needs to check against stripe length. */
6317 	if (map->type & BTRFS_BLOCK_GROUP_STRIPE_MASK) {
6318 		u64 max_len = stripe_len - stripe_offset;
6319 
6320 		/*
6321 		 * In case of raid56, we need to know the stripe aligned start
6322 		 */
6323 		if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
6324 			unsigned long full_stripe_len = stripe_len * data_stripes;
6325 			raid56_full_stripe_start = offset;
6326 
6327 			/*
6328 			 * Allow a write of a full stripe, but make sure we
6329 			 * don't allow straddling of stripes
6330 			 */
6331 			raid56_full_stripe_start = div64_u64(raid56_full_stripe_start,
6332 					full_stripe_len);
6333 			raid56_full_stripe_start *= full_stripe_len;
6334 
6335 			/*
6336 			 * For writes to RAID[56], allow a full stripeset across
6337 			 * all disks. For other RAID types and for RAID[56]
6338 			 * reads, just allow a single stripe (on a single disk).
6339 			 */
6340 			if (op == BTRFS_MAP_WRITE) {
6341 				max_len = stripe_len * data_stripes -
6342 					  (offset - raid56_full_stripe_start);
6343 			}
6344 		}
6345 		len = min_t(u64, em->len - offset, max_len);
6346 	} else {
6347 		len = em->len - offset;
6348 	}
6349 
6350 	io_geom->len = len;
6351 	io_geom->offset = offset;
6352 	io_geom->stripe_len = stripe_len;
6353 	io_geom->stripe_nr = stripe_nr;
6354 	io_geom->stripe_offset = stripe_offset;
6355 	io_geom->raid56_stripe_offset = raid56_full_stripe_start;
6356 
6357 	return 0;
6358 }
6359 
6360 static int __btrfs_map_block(struct btrfs_fs_info *fs_info,
6361 			     enum btrfs_map_op op,
6362 			     u64 logical, u64 *length,
6363 			     struct btrfs_io_context **bioc_ret,
6364 			     int mirror_num, int need_raid_map)
6365 {
6366 	struct extent_map *em;
6367 	struct map_lookup *map;
6368 	u64 stripe_offset;
6369 	u64 stripe_nr;
6370 	u64 stripe_len;
6371 	u32 stripe_index;
6372 	int data_stripes;
6373 	int i;
6374 	int ret = 0;
6375 	int num_stripes;
6376 	int max_errors = 0;
6377 	int tgtdev_indexes = 0;
6378 	struct btrfs_io_context *bioc = NULL;
6379 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
6380 	int dev_replace_is_ongoing = 0;
6381 	int num_alloc_stripes;
6382 	int patch_the_first_stripe_for_dev_replace = 0;
6383 	u64 physical_to_patch_in_first_stripe = 0;
6384 	u64 raid56_full_stripe_start = (u64)-1;
6385 	struct btrfs_io_geometry geom;
6386 
6387 	ASSERT(bioc_ret);
6388 	ASSERT(op != BTRFS_MAP_DISCARD);
6389 
6390 	em = btrfs_get_chunk_map(fs_info, logical, *length);
6391 	ASSERT(!IS_ERR(em));
6392 
6393 	ret = btrfs_get_io_geometry(fs_info, em, op, logical, &geom);
6394 	if (ret < 0)
6395 		return ret;
6396 
6397 	map = em->map_lookup;
6398 
6399 	*length = geom.len;
6400 	stripe_len = geom.stripe_len;
6401 	stripe_nr = geom.stripe_nr;
6402 	stripe_offset = geom.stripe_offset;
6403 	raid56_full_stripe_start = geom.raid56_stripe_offset;
6404 	data_stripes = nr_data_stripes(map);
6405 
6406 	down_read(&dev_replace->rwsem);
6407 	dev_replace_is_ongoing = btrfs_dev_replace_is_ongoing(dev_replace);
6408 	/*
6409 	 * Hold the semaphore for read during the whole operation, write is
6410 	 * requested at commit time but must wait.
6411 	 */
6412 	if (!dev_replace_is_ongoing)
6413 		up_read(&dev_replace->rwsem);
6414 
6415 	if (dev_replace_is_ongoing && mirror_num == map->num_stripes + 1 &&
6416 	    !need_full_stripe(op) && dev_replace->tgtdev != NULL) {
6417 		ret = get_extra_mirror_from_replace(fs_info, logical, *length,
6418 						    dev_replace->srcdev->devid,
6419 						    &mirror_num,
6420 					    &physical_to_patch_in_first_stripe);
6421 		if (ret)
6422 			goto out;
6423 		else
6424 			patch_the_first_stripe_for_dev_replace = 1;
6425 	} else if (mirror_num > map->num_stripes) {
6426 		mirror_num = 0;
6427 	}
6428 
6429 	num_stripes = 1;
6430 	stripe_index = 0;
6431 	if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
6432 		stripe_nr = div_u64_rem(stripe_nr, map->num_stripes,
6433 				&stripe_index);
6434 		if (!need_full_stripe(op))
6435 			mirror_num = 1;
6436 	} else if (map->type & BTRFS_BLOCK_GROUP_RAID1_MASK) {
6437 		if (need_full_stripe(op))
6438 			num_stripes = map->num_stripes;
6439 		else if (mirror_num)
6440 			stripe_index = mirror_num - 1;
6441 		else {
6442 			stripe_index = find_live_mirror(fs_info, map, 0,
6443 					    dev_replace_is_ongoing);
6444 			mirror_num = stripe_index + 1;
6445 		}
6446 
6447 	} else if (map->type & BTRFS_BLOCK_GROUP_DUP) {
6448 		if (need_full_stripe(op)) {
6449 			num_stripes = map->num_stripes;
6450 		} else if (mirror_num) {
6451 			stripe_index = mirror_num - 1;
6452 		} else {
6453 			mirror_num = 1;
6454 		}
6455 
6456 	} else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
6457 		u32 factor = map->num_stripes / map->sub_stripes;
6458 
6459 		stripe_nr = div_u64_rem(stripe_nr, factor, &stripe_index);
6460 		stripe_index *= map->sub_stripes;
6461 
6462 		if (need_full_stripe(op))
6463 			num_stripes = map->sub_stripes;
6464 		else if (mirror_num)
6465 			stripe_index += mirror_num - 1;
6466 		else {
6467 			int old_stripe_index = stripe_index;
6468 			stripe_index = find_live_mirror(fs_info, map,
6469 					      stripe_index,
6470 					      dev_replace_is_ongoing);
6471 			mirror_num = stripe_index - old_stripe_index + 1;
6472 		}
6473 
6474 	} else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
6475 		if (need_raid_map && (need_full_stripe(op) || mirror_num > 1)) {
6476 			/* push stripe_nr back to the start of the full stripe */
6477 			stripe_nr = div64_u64(raid56_full_stripe_start,
6478 					stripe_len * data_stripes);
6479 
6480 			/* RAID[56] write or recovery. Return all stripes */
6481 			num_stripes = map->num_stripes;
6482 			max_errors = nr_parity_stripes(map);
6483 
6484 			*length = map->stripe_len;
6485 			stripe_index = 0;
6486 			stripe_offset = 0;
6487 		} else {
6488 			/*
6489 			 * Mirror #0 or #1 means the original data block.
6490 			 * Mirror #2 is RAID5 parity block.
6491 			 * Mirror #3 is RAID6 Q block.
6492 			 */
6493 			stripe_nr = div_u64_rem(stripe_nr,
6494 					data_stripes, &stripe_index);
6495 			if (mirror_num > 1)
6496 				stripe_index = data_stripes + mirror_num - 2;
6497 
6498 			/* We distribute the parity blocks across stripes */
6499 			div_u64_rem(stripe_nr + stripe_index, map->num_stripes,
6500 					&stripe_index);
6501 			if (!need_full_stripe(op) && mirror_num <= 1)
6502 				mirror_num = 1;
6503 		}
6504 	} else {
6505 		/*
6506 		 * after this, stripe_nr is the number of stripes on this
6507 		 * device we have to walk to find the data, and stripe_index is
6508 		 * the number of our device in the stripe array
6509 		 */
6510 		stripe_nr = div_u64_rem(stripe_nr, map->num_stripes,
6511 				&stripe_index);
6512 		mirror_num = stripe_index + 1;
6513 	}
6514 	if (stripe_index >= map->num_stripes) {
6515 		btrfs_crit(fs_info,
6516 			   "stripe index math went horribly wrong, got stripe_index=%u, num_stripes=%u",
6517 			   stripe_index, map->num_stripes);
6518 		ret = -EINVAL;
6519 		goto out;
6520 	}
6521 
6522 	num_alloc_stripes = num_stripes;
6523 	if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL) {
6524 		if (op == BTRFS_MAP_WRITE)
6525 			num_alloc_stripes <<= 1;
6526 		if (op == BTRFS_MAP_GET_READ_MIRRORS)
6527 			num_alloc_stripes++;
6528 		tgtdev_indexes = num_stripes;
6529 	}
6530 
6531 	bioc = alloc_btrfs_io_context(fs_info, num_alloc_stripes, tgtdev_indexes);
6532 	if (!bioc) {
6533 		ret = -ENOMEM;
6534 		goto out;
6535 	}
6536 
6537 	for (i = 0; i < num_stripes; i++) {
6538 		bioc->stripes[i].physical = map->stripes[stripe_index].physical +
6539 			stripe_offset + stripe_nr * map->stripe_len;
6540 		bioc->stripes[i].dev = map->stripes[stripe_index].dev;
6541 		stripe_index++;
6542 	}
6543 
6544 	/* Build raid_map */
6545 	if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK && need_raid_map &&
6546 	    (need_full_stripe(op) || mirror_num > 1)) {
6547 		u64 tmp;
6548 		unsigned rot;
6549 
6550 		/* Work out the disk rotation on this stripe-set */
6551 		div_u64_rem(stripe_nr, num_stripes, &rot);
6552 
6553 		/* Fill in the logical address of each stripe */
6554 		tmp = stripe_nr * data_stripes;
6555 		for (i = 0; i < data_stripes; i++)
6556 			bioc->raid_map[(i + rot) % num_stripes] =
6557 				em->start + (tmp + i) * map->stripe_len;
6558 
6559 		bioc->raid_map[(i + rot) % map->num_stripes] = RAID5_P_STRIPE;
6560 		if (map->type & BTRFS_BLOCK_GROUP_RAID6)
6561 			bioc->raid_map[(i + rot + 1) % num_stripes] =
6562 				RAID6_Q_STRIPE;
6563 
6564 		sort_parity_stripes(bioc, num_stripes);
6565 	}
6566 
6567 	if (need_full_stripe(op))
6568 		max_errors = btrfs_chunk_max_errors(map);
6569 
6570 	if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL &&
6571 	    need_full_stripe(op)) {
6572 		handle_ops_on_dev_replace(op, &bioc, dev_replace, logical,
6573 					  &num_stripes, &max_errors);
6574 	}
6575 
6576 	*bioc_ret = bioc;
6577 	bioc->map_type = map->type;
6578 	bioc->num_stripes = num_stripes;
6579 	bioc->max_errors = max_errors;
6580 	bioc->mirror_num = mirror_num;
6581 
6582 	/*
6583 	 * this is the case that REQ_READ && dev_replace_is_ongoing &&
6584 	 * mirror_num == num_stripes + 1 && dev_replace target drive is
6585 	 * available as a mirror
6586 	 */
6587 	if (patch_the_first_stripe_for_dev_replace && num_stripes > 0) {
6588 		WARN_ON(num_stripes > 1);
6589 		bioc->stripes[0].dev = dev_replace->tgtdev;
6590 		bioc->stripes[0].physical = physical_to_patch_in_first_stripe;
6591 		bioc->mirror_num = map->num_stripes + 1;
6592 	}
6593 out:
6594 	if (dev_replace_is_ongoing) {
6595 		lockdep_assert_held(&dev_replace->rwsem);
6596 		/* Unlock and let waiting writers proceed */
6597 		up_read(&dev_replace->rwsem);
6598 	}
6599 	free_extent_map(em);
6600 	return ret;
6601 }
6602 
6603 int btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op,
6604 		      u64 logical, u64 *length,
6605 		      struct btrfs_io_context **bioc_ret, int mirror_num)
6606 {
6607 	if (op == BTRFS_MAP_DISCARD)
6608 		return __btrfs_map_block_for_discard(fs_info, logical,
6609 						     length, bioc_ret);
6610 
6611 	return __btrfs_map_block(fs_info, op, logical, length, bioc_ret,
6612 				 mirror_num, 0);
6613 }
6614 
6615 /* For Scrub/replace */
6616 int btrfs_map_sblock(struct btrfs_fs_info *fs_info, enum btrfs_map_op op,
6617 		     u64 logical, u64 *length,
6618 		     struct btrfs_io_context **bioc_ret)
6619 {
6620 	return __btrfs_map_block(fs_info, op, logical, length, bioc_ret, 0, 1);
6621 }
6622 
6623 static inline void btrfs_end_bioc(struct btrfs_io_context *bioc, struct bio *bio)
6624 {
6625 	bio->bi_private = bioc->private;
6626 	bio->bi_end_io = bioc->end_io;
6627 	bio_endio(bio);
6628 
6629 	btrfs_put_bioc(bioc);
6630 }
6631 
6632 static void btrfs_end_bio(struct bio *bio)
6633 {
6634 	struct btrfs_io_context *bioc = bio->bi_private;
6635 	int is_orig_bio = 0;
6636 
6637 	if (bio->bi_status) {
6638 		atomic_inc(&bioc->error);
6639 		if (bio->bi_status == BLK_STS_IOERR ||
6640 		    bio->bi_status == BLK_STS_TARGET) {
6641 			struct btrfs_device *dev = btrfs_bio(bio)->device;
6642 
6643 			ASSERT(dev->bdev);
6644 			if (btrfs_op(bio) == BTRFS_MAP_WRITE)
6645 				btrfs_dev_stat_inc_and_print(dev,
6646 						BTRFS_DEV_STAT_WRITE_ERRS);
6647 			else if (!(bio->bi_opf & REQ_RAHEAD))
6648 				btrfs_dev_stat_inc_and_print(dev,
6649 						BTRFS_DEV_STAT_READ_ERRS);
6650 			if (bio->bi_opf & REQ_PREFLUSH)
6651 				btrfs_dev_stat_inc_and_print(dev,
6652 						BTRFS_DEV_STAT_FLUSH_ERRS);
6653 		}
6654 	}
6655 
6656 	if (bio == bioc->orig_bio)
6657 		is_orig_bio = 1;
6658 
6659 	btrfs_bio_counter_dec(bioc->fs_info);
6660 
6661 	if (atomic_dec_and_test(&bioc->stripes_pending)) {
6662 		if (!is_orig_bio) {
6663 			bio_put(bio);
6664 			bio = bioc->orig_bio;
6665 		}
6666 
6667 		btrfs_bio(bio)->mirror_num = bioc->mirror_num;
6668 		/* only send an error to the higher layers if it is
6669 		 * beyond the tolerance of the btrfs bio
6670 		 */
6671 		if (atomic_read(&bioc->error) > bioc->max_errors) {
6672 			bio->bi_status = BLK_STS_IOERR;
6673 		} else {
6674 			/*
6675 			 * this bio is actually up to date, we didn't
6676 			 * go over the max number of errors
6677 			 */
6678 			bio->bi_status = BLK_STS_OK;
6679 		}
6680 
6681 		btrfs_end_bioc(bioc, bio);
6682 	} else if (!is_orig_bio) {
6683 		bio_put(bio);
6684 	}
6685 }
6686 
6687 static void submit_stripe_bio(struct btrfs_io_context *bioc, struct bio *bio,
6688 			      u64 physical, struct btrfs_device *dev)
6689 {
6690 	struct btrfs_fs_info *fs_info = bioc->fs_info;
6691 
6692 	bio->bi_private = bioc;
6693 	btrfs_bio(bio)->device = dev;
6694 	bio->bi_end_io = btrfs_end_bio;
6695 	bio->bi_iter.bi_sector = physical >> 9;
6696 	/*
6697 	 * For zone append writing, bi_sector must point the beginning of the
6698 	 * zone
6699 	 */
6700 	if (bio_op(bio) == REQ_OP_ZONE_APPEND) {
6701 		if (btrfs_dev_is_sequential(dev, physical)) {
6702 			u64 zone_start = round_down(physical, fs_info->zone_size);
6703 
6704 			bio->bi_iter.bi_sector = zone_start >> SECTOR_SHIFT;
6705 		} else {
6706 			bio->bi_opf &= ~REQ_OP_ZONE_APPEND;
6707 			bio->bi_opf |= REQ_OP_WRITE;
6708 		}
6709 	}
6710 	btrfs_debug_in_rcu(fs_info,
6711 	"btrfs_map_bio: rw %d 0x%x, sector=%llu, dev=%lu (%s id %llu), size=%u",
6712 		bio_op(bio), bio->bi_opf, bio->bi_iter.bi_sector,
6713 		(unsigned long)dev->bdev->bd_dev, rcu_str_deref(dev->name),
6714 		dev->devid, bio->bi_iter.bi_size);
6715 
6716 	btrfs_bio_counter_inc_noblocked(fs_info);
6717 
6718 	btrfsic_check_bio(bio);
6719 	submit_bio(bio);
6720 }
6721 
6722 static void bioc_error(struct btrfs_io_context *bioc, struct bio *bio, u64 logical)
6723 {
6724 	atomic_inc(&bioc->error);
6725 	if (atomic_dec_and_test(&bioc->stripes_pending)) {
6726 		/* Should be the original bio. */
6727 		WARN_ON(bio != bioc->orig_bio);
6728 
6729 		btrfs_bio(bio)->mirror_num = bioc->mirror_num;
6730 		bio->bi_iter.bi_sector = logical >> 9;
6731 		if (atomic_read(&bioc->error) > bioc->max_errors)
6732 			bio->bi_status = BLK_STS_IOERR;
6733 		else
6734 			bio->bi_status = BLK_STS_OK;
6735 		btrfs_end_bioc(bioc, bio);
6736 	}
6737 }
6738 
6739 blk_status_t btrfs_map_bio(struct btrfs_fs_info *fs_info, struct bio *bio,
6740 			   int mirror_num)
6741 {
6742 	struct btrfs_device *dev;
6743 	struct bio *first_bio = bio;
6744 	u64 logical = bio->bi_iter.bi_sector << 9;
6745 	u64 length = 0;
6746 	u64 map_length;
6747 	int ret;
6748 	int dev_nr;
6749 	int total_devs;
6750 	struct btrfs_io_context *bioc = NULL;
6751 
6752 	length = bio->bi_iter.bi_size;
6753 	map_length = length;
6754 
6755 	btrfs_bio_counter_inc_blocked(fs_info);
6756 	ret = __btrfs_map_block(fs_info, btrfs_op(bio), logical,
6757 				&map_length, &bioc, mirror_num, 1);
6758 	if (ret) {
6759 		btrfs_bio_counter_dec(fs_info);
6760 		return errno_to_blk_status(ret);
6761 	}
6762 
6763 	total_devs = bioc->num_stripes;
6764 	bioc->orig_bio = first_bio;
6765 	bioc->private = first_bio->bi_private;
6766 	bioc->end_io = first_bio->bi_end_io;
6767 	atomic_set(&bioc->stripes_pending, bioc->num_stripes);
6768 
6769 	if ((bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) &&
6770 	    ((btrfs_op(bio) == BTRFS_MAP_WRITE) || (mirror_num > 1))) {
6771 		/* In this case, map_length has been set to the length of
6772 		   a single stripe; not the whole write */
6773 		if (btrfs_op(bio) == BTRFS_MAP_WRITE) {
6774 			ret = raid56_parity_write(bio, bioc, map_length);
6775 		} else {
6776 			ret = raid56_parity_recover(bio, bioc, map_length,
6777 						    mirror_num, 1);
6778 		}
6779 
6780 		btrfs_bio_counter_dec(fs_info);
6781 		return errno_to_blk_status(ret);
6782 	}
6783 
6784 	if (map_length < length) {
6785 		btrfs_crit(fs_info,
6786 			   "mapping failed logical %llu bio len %llu len %llu",
6787 			   logical, length, map_length);
6788 		BUG();
6789 	}
6790 
6791 	for (dev_nr = 0; dev_nr < total_devs; dev_nr++) {
6792 		dev = bioc->stripes[dev_nr].dev;
6793 		if (!dev || !dev->bdev || test_bit(BTRFS_DEV_STATE_MISSING,
6794 						   &dev->dev_state) ||
6795 		    (btrfs_op(first_bio) == BTRFS_MAP_WRITE &&
6796 		    !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))) {
6797 			bioc_error(bioc, first_bio, logical);
6798 			continue;
6799 		}
6800 
6801 		if (dev_nr < total_devs - 1) {
6802 			bio = btrfs_bio_clone(dev->bdev, first_bio);
6803 		} else {
6804 			bio = first_bio;
6805 			bio_set_dev(bio, dev->bdev);
6806 		}
6807 
6808 		submit_stripe_bio(bioc, bio, bioc->stripes[dev_nr].physical, dev);
6809 	}
6810 	btrfs_bio_counter_dec(fs_info);
6811 	return BLK_STS_OK;
6812 }
6813 
6814 static bool dev_args_match_fs_devices(const struct btrfs_dev_lookup_args *args,
6815 				      const struct btrfs_fs_devices *fs_devices)
6816 {
6817 	if (args->fsid == NULL)
6818 		return true;
6819 	if (memcmp(fs_devices->metadata_uuid, args->fsid, BTRFS_FSID_SIZE) == 0)
6820 		return true;
6821 	return false;
6822 }
6823 
6824 static bool dev_args_match_device(const struct btrfs_dev_lookup_args *args,
6825 				  const struct btrfs_device *device)
6826 {
6827 	ASSERT((args->devid != (u64)-1) || args->missing);
6828 
6829 	if ((args->devid != (u64)-1) && device->devid != args->devid)
6830 		return false;
6831 	if (args->uuid && memcmp(device->uuid, args->uuid, BTRFS_UUID_SIZE) != 0)
6832 		return false;
6833 	if (!args->missing)
6834 		return true;
6835 	if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state) &&
6836 	    !device->bdev)
6837 		return true;
6838 	return false;
6839 }
6840 
6841 /*
6842  * Find a device specified by @devid or @uuid in the list of @fs_devices, or
6843  * return NULL.
6844  *
6845  * If devid and uuid are both specified, the match must be exact, otherwise
6846  * only devid is used.
6847  */
6848 struct btrfs_device *btrfs_find_device(const struct btrfs_fs_devices *fs_devices,
6849 				       const struct btrfs_dev_lookup_args *args)
6850 {
6851 	struct btrfs_device *device;
6852 	struct btrfs_fs_devices *seed_devs;
6853 
6854 	if (dev_args_match_fs_devices(args, fs_devices)) {
6855 		list_for_each_entry(device, &fs_devices->devices, dev_list) {
6856 			if (dev_args_match_device(args, device))
6857 				return device;
6858 		}
6859 	}
6860 
6861 	list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) {
6862 		if (!dev_args_match_fs_devices(args, seed_devs))
6863 			continue;
6864 		list_for_each_entry(device, &seed_devs->devices, dev_list) {
6865 			if (dev_args_match_device(args, device))
6866 				return device;
6867 		}
6868 	}
6869 
6870 	return NULL;
6871 }
6872 
6873 static struct btrfs_device *add_missing_dev(struct btrfs_fs_devices *fs_devices,
6874 					    u64 devid, u8 *dev_uuid)
6875 {
6876 	struct btrfs_device *device;
6877 	unsigned int nofs_flag;
6878 
6879 	/*
6880 	 * We call this under the chunk_mutex, so we want to use NOFS for this
6881 	 * allocation, however we don't want to change btrfs_alloc_device() to
6882 	 * always do NOFS because we use it in a lot of other GFP_KERNEL safe
6883 	 * places.
6884 	 */
6885 	nofs_flag = memalloc_nofs_save();
6886 	device = btrfs_alloc_device(NULL, &devid, dev_uuid);
6887 	memalloc_nofs_restore(nofs_flag);
6888 	if (IS_ERR(device))
6889 		return device;
6890 
6891 	list_add(&device->dev_list, &fs_devices->devices);
6892 	device->fs_devices = fs_devices;
6893 	fs_devices->num_devices++;
6894 
6895 	set_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state);
6896 	fs_devices->missing_devices++;
6897 
6898 	return device;
6899 }
6900 
6901 /**
6902  * btrfs_alloc_device - allocate struct btrfs_device
6903  * @fs_info:	used only for generating a new devid, can be NULL if
6904  *		devid is provided (i.e. @devid != NULL).
6905  * @devid:	a pointer to devid for this device.  If NULL a new devid
6906  *		is generated.
6907  * @uuid:	a pointer to UUID for this device.  If NULL a new UUID
6908  *		is generated.
6909  *
6910  * Return: a pointer to a new &struct btrfs_device on success; ERR_PTR()
6911  * on error.  Returned struct is not linked onto any lists and must be
6912  * destroyed with btrfs_free_device.
6913  */
6914 struct btrfs_device *btrfs_alloc_device(struct btrfs_fs_info *fs_info,
6915 					const u64 *devid,
6916 					const u8 *uuid)
6917 {
6918 	struct btrfs_device *dev;
6919 	u64 tmp;
6920 
6921 	if (WARN_ON(!devid && !fs_info))
6922 		return ERR_PTR(-EINVAL);
6923 
6924 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
6925 	if (!dev)
6926 		return ERR_PTR(-ENOMEM);
6927 
6928 	INIT_LIST_HEAD(&dev->dev_list);
6929 	INIT_LIST_HEAD(&dev->dev_alloc_list);
6930 	INIT_LIST_HEAD(&dev->post_commit_list);
6931 
6932 	atomic_set(&dev->dev_stats_ccnt, 0);
6933 	btrfs_device_data_ordered_init(dev);
6934 	extent_io_tree_init(fs_info, &dev->alloc_state,
6935 			    IO_TREE_DEVICE_ALLOC_STATE, NULL);
6936 
6937 	if (devid)
6938 		tmp = *devid;
6939 	else {
6940 		int ret;
6941 
6942 		ret = find_next_devid(fs_info, &tmp);
6943 		if (ret) {
6944 			btrfs_free_device(dev);
6945 			return ERR_PTR(ret);
6946 		}
6947 	}
6948 	dev->devid = tmp;
6949 
6950 	if (uuid)
6951 		memcpy(dev->uuid, uuid, BTRFS_UUID_SIZE);
6952 	else
6953 		generate_random_uuid(dev->uuid);
6954 
6955 	return dev;
6956 }
6957 
6958 static void btrfs_report_missing_device(struct btrfs_fs_info *fs_info,
6959 					u64 devid, u8 *uuid, bool error)
6960 {
6961 	if (error)
6962 		btrfs_err_rl(fs_info, "devid %llu uuid %pU is missing",
6963 			      devid, uuid);
6964 	else
6965 		btrfs_warn_rl(fs_info, "devid %llu uuid %pU is missing",
6966 			      devid, uuid);
6967 }
6968 
6969 static u64 calc_stripe_length(u64 type, u64 chunk_len, int num_stripes)
6970 {
6971 	const int data_stripes = calc_data_stripes(type, num_stripes);
6972 
6973 	return div_u64(chunk_len, data_stripes);
6974 }
6975 
6976 #if BITS_PER_LONG == 32
6977 /*
6978  * Due to page cache limit, metadata beyond BTRFS_32BIT_MAX_FILE_SIZE
6979  * can't be accessed on 32bit systems.
6980  *
6981  * This function do mount time check to reject the fs if it already has
6982  * metadata chunk beyond that limit.
6983  */
6984 static int check_32bit_meta_chunk(struct btrfs_fs_info *fs_info,
6985 				  u64 logical, u64 length, u64 type)
6986 {
6987 	if (!(type & BTRFS_BLOCK_GROUP_METADATA))
6988 		return 0;
6989 
6990 	if (logical + length < MAX_LFS_FILESIZE)
6991 		return 0;
6992 
6993 	btrfs_err_32bit_limit(fs_info);
6994 	return -EOVERFLOW;
6995 }
6996 
6997 /*
6998  * This is to give early warning for any metadata chunk reaching
6999  * BTRFS_32BIT_EARLY_WARN_THRESHOLD.
7000  * Although we can still access the metadata, it's not going to be possible
7001  * once the limit is reached.
7002  */
7003 static void warn_32bit_meta_chunk(struct btrfs_fs_info *fs_info,
7004 				  u64 logical, u64 length, u64 type)
7005 {
7006 	if (!(type & BTRFS_BLOCK_GROUP_METADATA))
7007 		return;
7008 
7009 	if (logical + length < BTRFS_32BIT_EARLY_WARN_THRESHOLD)
7010 		return;
7011 
7012 	btrfs_warn_32bit_limit(fs_info);
7013 }
7014 #endif
7015 
7016 static struct btrfs_device *handle_missing_device(struct btrfs_fs_info *fs_info,
7017 						  u64 devid, u8 *uuid)
7018 {
7019 	struct btrfs_device *dev;
7020 
7021 	if (!btrfs_test_opt(fs_info, DEGRADED)) {
7022 		btrfs_report_missing_device(fs_info, devid, uuid, true);
7023 		return ERR_PTR(-ENOENT);
7024 	}
7025 
7026 	dev = add_missing_dev(fs_info->fs_devices, devid, uuid);
7027 	if (IS_ERR(dev)) {
7028 		btrfs_err(fs_info, "failed to init missing device %llu: %ld",
7029 			  devid, PTR_ERR(dev));
7030 		return dev;
7031 	}
7032 	btrfs_report_missing_device(fs_info, devid, uuid, false);
7033 
7034 	return dev;
7035 }
7036 
7037 static int read_one_chunk(struct btrfs_key *key, struct extent_buffer *leaf,
7038 			  struct btrfs_chunk *chunk)
7039 {
7040 	BTRFS_DEV_LOOKUP_ARGS(args);
7041 	struct btrfs_fs_info *fs_info = leaf->fs_info;
7042 	struct extent_map_tree *map_tree = &fs_info->mapping_tree;
7043 	struct map_lookup *map;
7044 	struct extent_map *em;
7045 	u64 logical;
7046 	u64 length;
7047 	u64 devid;
7048 	u64 type;
7049 	u8 uuid[BTRFS_UUID_SIZE];
7050 	int num_stripes;
7051 	int ret;
7052 	int i;
7053 
7054 	logical = key->offset;
7055 	length = btrfs_chunk_length(leaf, chunk);
7056 	type = btrfs_chunk_type(leaf, chunk);
7057 	num_stripes = btrfs_chunk_num_stripes(leaf, chunk);
7058 
7059 #if BITS_PER_LONG == 32
7060 	ret = check_32bit_meta_chunk(fs_info, logical, length, type);
7061 	if (ret < 0)
7062 		return ret;
7063 	warn_32bit_meta_chunk(fs_info, logical, length, type);
7064 #endif
7065 
7066 	/*
7067 	 * Only need to verify chunk item if we're reading from sys chunk array,
7068 	 * as chunk item in tree block is already verified by tree-checker.
7069 	 */
7070 	if (leaf->start == BTRFS_SUPER_INFO_OFFSET) {
7071 		ret = btrfs_check_chunk_valid(leaf, chunk, logical);
7072 		if (ret)
7073 			return ret;
7074 	}
7075 
7076 	read_lock(&map_tree->lock);
7077 	em = lookup_extent_mapping(map_tree, logical, 1);
7078 	read_unlock(&map_tree->lock);
7079 
7080 	/* already mapped? */
7081 	if (em && em->start <= logical && em->start + em->len > logical) {
7082 		free_extent_map(em);
7083 		return 0;
7084 	} else if (em) {
7085 		free_extent_map(em);
7086 	}
7087 
7088 	em = alloc_extent_map();
7089 	if (!em)
7090 		return -ENOMEM;
7091 	map = kmalloc(map_lookup_size(num_stripes), GFP_NOFS);
7092 	if (!map) {
7093 		free_extent_map(em);
7094 		return -ENOMEM;
7095 	}
7096 
7097 	set_bit(EXTENT_FLAG_FS_MAPPING, &em->flags);
7098 	em->map_lookup = map;
7099 	em->start = logical;
7100 	em->len = length;
7101 	em->orig_start = 0;
7102 	em->block_start = 0;
7103 	em->block_len = em->len;
7104 
7105 	map->num_stripes = num_stripes;
7106 	map->io_width = btrfs_chunk_io_width(leaf, chunk);
7107 	map->io_align = btrfs_chunk_io_align(leaf, chunk);
7108 	map->stripe_len = btrfs_chunk_stripe_len(leaf, chunk);
7109 	map->type = type;
7110 	map->sub_stripes = btrfs_chunk_sub_stripes(leaf, chunk);
7111 	map->verified_stripes = 0;
7112 	em->orig_block_len = calc_stripe_length(type, em->len,
7113 						map->num_stripes);
7114 	for (i = 0; i < num_stripes; i++) {
7115 		map->stripes[i].physical =
7116 			btrfs_stripe_offset_nr(leaf, chunk, i);
7117 		devid = btrfs_stripe_devid_nr(leaf, chunk, i);
7118 		args.devid = devid;
7119 		read_extent_buffer(leaf, uuid, (unsigned long)
7120 				   btrfs_stripe_dev_uuid_nr(chunk, i),
7121 				   BTRFS_UUID_SIZE);
7122 		args.uuid = uuid;
7123 		map->stripes[i].dev = btrfs_find_device(fs_info->fs_devices, &args);
7124 		if (!map->stripes[i].dev) {
7125 			map->stripes[i].dev = handle_missing_device(fs_info,
7126 								    devid, uuid);
7127 			if (IS_ERR(map->stripes[i].dev)) {
7128 				free_extent_map(em);
7129 				return PTR_ERR(map->stripes[i].dev);
7130 			}
7131 		}
7132 
7133 		set_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
7134 				&(map->stripes[i].dev->dev_state));
7135 	}
7136 
7137 	write_lock(&map_tree->lock);
7138 	ret = add_extent_mapping(map_tree, em, 0);
7139 	write_unlock(&map_tree->lock);
7140 	if (ret < 0) {
7141 		btrfs_err(fs_info,
7142 			  "failed to add chunk map, start=%llu len=%llu: %d",
7143 			  em->start, em->len, ret);
7144 	}
7145 	free_extent_map(em);
7146 
7147 	return ret;
7148 }
7149 
7150 static void fill_device_from_item(struct extent_buffer *leaf,
7151 				 struct btrfs_dev_item *dev_item,
7152 				 struct btrfs_device *device)
7153 {
7154 	unsigned long ptr;
7155 
7156 	device->devid = btrfs_device_id(leaf, dev_item);
7157 	device->disk_total_bytes = btrfs_device_total_bytes(leaf, dev_item);
7158 	device->total_bytes = device->disk_total_bytes;
7159 	device->commit_total_bytes = device->disk_total_bytes;
7160 	device->bytes_used = btrfs_device_bytes_used(leaf, dev_item);
7161 	device->commit_bytes_used = device->bytes_used;
7162 	device->type = btrfs_device_type(leaf, dev_item);
7163 	device->io_align = btrfs_device_io_align(leaf, dev_item);
7164 	device->io_width = btrfs_device_io_width(leaf, dev_item);
7165 	device->sector_size = btrfs_device_sector_size(leaf, dev_item);
7166 	WARN_ON(device->devid == BTRFS_DEV_REPLACE_DEVID);
7167 	clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state);
7168 
7169 	ptr = btrfs_device_uuid(dev_item);
7170 	read_extent_buffer(leaf, device->uuid, ptr, BTRFS_UUID_SIZE);
7171 }
7172 
7173 static struct btrfs_fs_devices *open_seed_devices(struct btrfs_fs_info *fs_info,
7174 						  u8 *fsid)
7175 {
7176 	struct btrfs_fs_devices *fs_devices;
7177 	int ret;
7178 
7179 	lockdep_assert_held(&uuid_mutex);
7180 	ASSERT(fsid);
7181 
7182 	/* This will match only for multi-device seed fs */
7183 	list_for_each_entry(fs_devices, &fs_info->fs_devices->seed_list, seed_list)
7184 		if (!memcmp(fs_devices->fsid, fsid, BTRFS_FSID_SIZE))
7185 			return fs_devices;
7186 
7187 
7188 	fs_devices = find_fsid(fsid, NULL);
7189 	if (!fs_devices) {
7190 		if (!btrfs_test_opt(fs_info, DEGRADED))
7191 			return ERR_PTR(-ENOENT);
7192 
7193 		fs_devices = alloc_fs_devices(fsid, NULL);
7194 		if (IS_ERR(fs_devices))
7195 			return fs_devices;
7196 
7197 		fs_devices->seeding = true;
7198 		fs_devices->opened = 1;
7199 		return fs_devices;
7200 	}
7201 
7202 	/*
7203 	 * Upon first call for a seed fs fsid, just create a private copy of the
7204 	 * respective fs_devices and anchor it at fs_info->fs_devices->seed_list
7205 	 */
7206 	fs_devices = clone_fs_devices(fs_devices);
7207 	if (IS_ERR(fs_devices))
7208 		return fs_devices;
7209 
7210 	ret = open_fs_devices(fs_devices, FMODE_READ, fs_info->bdev_holder);
7211 	if (ret) {
7212 		free_fs_devices(fs_devices);
7213 		return ERR_PTR(ret);
7214 	}
7215 
7216 	if (!fs_devices->seeding) {
7217 		close_fs_devices(fs_devices);
7218 		free_fs_devices(fs_devices);
7219 		return ERR_PTR(-EINVAL);
7220 	}
7221 
7222 	list_add(&fs_devices->seed_list, &fs_info->fs_devices->seed_list);
7223 
7224 	return fs_devices;
7225 }
7226 
7227 static int read_one_dev(struct extent_buffer *leaf,
7228 			struct btrfs_dev_item *dev_item)
7229 {
7230 	BTRFS_DEV_LOOKUP_ARGS(args);
7231 	struct btrfs_fs_info *fs_info = leaf->fs_info;
7232 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
7233 	struct btrfs_device *device;
7234 	u64 devid;
7235 	int ret;
7236 	u8 fs_uuid[BTRFS_FSID_SIZE];
7237 	u8 dev_uuid[BTRFS_UUID_SIZE];
7238 
7239 	devid = args.devid = btrfs_device_id(leaf, dev_item);
7240 	read_extent_buffer(leaf, dev_uuid, btrfs_device_uuid(dev_item),
7241 			   BTRFS_UUID_SIZE);
7242 	read_extent_buffer(leaf, fs_uuid, btrfs_device_fsid(dev_item),
7243 			   BTRFS_FSID_SIZE);
7244 	args.uuid = dev_uuid;
7245 	args.fsid = fs_uuid;
7246 
7247 	if (memcmp(fs_uuid, fs_devices->metadata_uuid, BTRFS_FSID_SIZE)) {
7248 		fs_devices = open_seed_devices(fs_info, fs_uuid);
7249 		if (IS_ERR(fs_devices))
7250 			return PTR_ERR(fs_devices);
7251 	}
7252 
7253 	device = btrfs_find_device(fs_info->fs_devices, &args);
7254 	if (!device) {
7255 		if (!btrfs_test_opt(fs_info, DEGRADED)) {
7256 			btrfs_report_missing_device(fs_info, devid,
7257 							dev_uuid, true);
7258 			return -ENOENT;
7259 		}
7260 
7261 		device = add_missing_dev(fs_devices, devid, dev_uuid);
7262 		if (IS_ERR(device)) {
7263 			btrfs_err(fs_info,
7264 				"failed to add missing dev %llu: %ld",
7265 				devid, PTR_ERR(device));
7266 			return PTR_ERR(device);
7267 		}
7268 		btrfs_report_missing_device(fs_info, devid, dev_uuid, false);
7269 	} else {
7270 		if (!device->bdev) {
7271 			if (!btrfs_test_opt(fs_info, DEGRADED)) {
7272 				btrfs_report_missing_device(fs_info,
7273 						devid, dev_uuid, true);
7274 				return -ENOENT;
7275 			}
7276 			btrfs_report_missing_device(fs_info, devid,
7277 							dev_uuid, false);
7278 		}
7279 
7280 		if (!device->bdev &&
7281 		    !test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) {
7282 			/*
7283 			 * this happens when a device that was properly setup
7284 			 * in the device info lists suddenly goes bad.
7285 			 * device->bdev is NULL, and so we have to set
7286 			 * device->missing to one here
7287 			 */
7288 			device->fs_devices->missing_devices++;
7289 			set_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state);
7290 		}
7291 
7292 		/* Move the device to its own fs_devices */
7293 		if (device->fs_devices != fs_devices) {
7294 			ASSERT(test_bit(BTRFS_DEV_STATE_MISSING,
7295 							&device->dev_state));
7296 
7297 			list_move(&device->dev_list, &fs_devices->devices);
7298 			device->fs_devices->num_devices--;
7299 			fs_devices->num_devices++;
7300 
7301 			device->fs_devices->missing_devices--;
7302 			fs_devices->missing_devices++;
7303 
7304 			device->fs_devices = fs_devices;
7305 		}
7306 	}
7307 
7308 	if (device->fs_devices != fs_info->fs_devices) {
7309 		BUG_ON(test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state));
7310 		if (device->generation !=
7311 		    btrfs_device_generation(leaf, dev_item))
7312 			return -EINVAL;
7313 	}
7314 
7315 	fill_device_from_item(leaf, dev_item, device);
7316 	if (device->bdev) {
7317 		u64 max_total_bytes = bdev_nr_bytes(device->bdev);
7318 
7319 		if (device->total_bytes > max_total_bytes) {
7320 			btrfs_err(fs_info,
7321 			"device total_bytes should be at most %llu but found %llu",
7322 				  max_total_bytes, device->total_bytes);
7323 			return -EINVAL;
7324 		}
7325 	}
7326 	set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
7327 	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
7328 	   !test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
7329 		device->fs_devices->total_rw_bytes += device->total_bytes;
7330 		atomic64_add(device->total_bytes - device->bytes_used,
7331 				&fs_info->free_chunk_space);
7332 	}
7333 	ret = 0;
7334 	return ret;
7335 }
7336 
7337 int btrfs_read_sys_array(struct btrfs_fs_info *fs_info)
7338 {
7339 	struct btrfs_super_block *super_copy = fs_info->super_copy;
7340 	struct extent_buffer *sb;
7341 	struct btrfs_disk_key *disk_key;
7342 	struct btrfs_chunk *chunk;
7343 	u8 *array_ptr;
7344 	unsigned long sb_array_offset;
7345 	int ret = 0;
7346 	u32 num_stripes;
7347 	u32 array_size;
7348 	u32 len = 0;
7349 	u32 cur_offset;
7350 	u64 type;
7351 	struct btrfs_key key;
7352 
7353 	ASSERT(BTRFS_SUPER_INFO_SIZE <= fs_info->nodesize);
7354 
7355 	/*
7356 	 * We allocated a dummy extent, just to use extent buffer accessors.
7357 	 * There will be unused space after BTRFS_SUPER_INFO_SIZE, but
7358 	 * that's fine, we will not go beyond system chunk array anyway.
7359 	 */
7360 	sb = alloc_dummy_extent_buffer(fs_info, BTRFS_SUPER_INFO_OFFSET);
7361 	if (!sb)
7362 		return -ENOMEM;
7363 	set_extent_buffer_uptodate(sb);
7364 
7365 	write_extent_buffer(sb, super_copy, 0, BTRFS_SUPER_INFO_SIZE);
7366 	array_size = btrfs_super_sys_array_size(super_copy);
7367 
7368 	array_ptr = super_copy->sys_chunk_array;
7369 	sb_array_offset = offsetof(struct btrfs_super_block, sys_chunk_array);
7370 	cur_offset = 0;
7371 
7372 	while (cur_offset < array_size) {
7373 		disk_key = (struct btrfs_disk_key *)array_ptr;
7374 		len = sizeof(*disk_key);
7375 		if (cur_offset + len > array_size)
7376 			goto out_short_read;
7377 
7378 		btrfs_disk_key_to_cpu(&key, disk_key);
7379 
7380 		array_ptr += len;
7381 		sb_array_offset += len;
7382 		cur_offset += len;
7383 
7384 		if (key.type != BTRFS_CHUNK_ITEM_KEY) {
7385 			btrfs_err(fs_info,
7386 			    "unexpected item type %u in sys_array at offset %u",
7387 				  (u32)key.type, cur_offset);
7388 			ret = -EIO;
7389 			break;
7390 		}
7391 
7392 		chunk = (struct btrfs_chunk *)sb_array_offset;
7393 		/*
7394 		 * At least one btrfs_chunk with one stripe must be present,
7395 		 * exact stripe count check comes afterwards
7396 		 */
7397 		len = btrfs_chunk_item_size(1);
7398 		if (cur_offset + len > array_size)
7399 			goto out_short_read;
7400 
7401 		num_stripes = btrfs_chunk_num_stripes(sb, chunk);
7402 		if (!num_stripes) {
7403 			btrfs_err(fs_info,
7404 			"invalid number of stripes %u in sys_array at offset %u",
7405 				  num_stripes, cur_offset);
7406 			ret = -EIO;
7407 			break;
7408 		}
7409 
7410 		type = btrfs_chunk_type(sb, chunk);
7411 		if ((type & BTRFS_BLOCK_GROUP_SYSTEM) == 0) {
7412 			btrfs_err(fs_info,
7413 			"invalid chunk type %llu in sys_array at offset %u",
7414 				  type, cur_offset);
7415 			ret = -EIO;
7416 			break;
7417 		}
7418 
7419 		len = btrfs_chunk_item_size(num_stripes);
7420 		if (cur_offset + len > array_size)
7421 			goto out_short_read;
7422 
7423 		ret = read_one_chunk(&key, sb, chunk);
7424 		if (ret)
7425 			break;
7426 
7427 		array_ptr += len;
7428 		sb_array_offset += len;
7429 		cur_offset += len;
7430 	}
7431 	clear_extent_buffer_uptodate(sb);
7432 	free_extent_buffer_stale(sb);
7433 	return ret;
7434 
7435 out_short_read:
7436 	btrfs_err(fs_info, "sys_array too short to read %u bytes at offset %u",
7437 			len, cur_offset);
7438 	clear_extent_buffer_uptodate(sb);
7439 	free_extent_buffer_stale(sb);
7440 	return -EIO;
7441 }
7442 
7443 /*
7444  * Check if all chunks in the fs are OK for read-write degraded mount
7445  *
7446  * If the @failing_dev is specified, it's accounted as missing.
7447  *
7448  * Return true if all chunks meet the minimal RW mount requirements.
7449  * Return false if any chunk doesn't meet the minimal RW mount requirements.
7450  */
7451 bool btrfs_check_rw_degradable(struct btrfs_fs_info *fs_info,
7452 					struct btrfs_device *failing_dev)
7453 {
7454 	struct extent_map_tree *map_tree = &fs_info->mapping_tree;
7455 	struct extent_map *em;
7456 	u64 next_start = 0;
7457 	bool ret = true;
7458 
7459 	read_lock(&map_tree->lock);
7460 	em = lookup_extent_mapping(map_tree, 0, (u64)-1);
7461 	read_unlock(&map_tree->lock);
7462 	/* No chunk at all? Return false anyway */
7463 	if (!em) {
7464 		ret = false;
7465 		goto out;
7466 	}
7467 	while (em) {
7468 		struct map_lookup *map;
7469 		int missing = 0;
7470 		int max_tolerated;
7471 		int i;
7472 
7473 		map = em->map_lookup;
7474 		max_tolerated =
7475 			btrfs_get_num_tolerated_disk_barrier_failures(
7476 					map->type);
7477 		for (i = 0; i < map->num_stripes; i++) {
7478 			struct btrfs_device *dev = map->stripes[i].dev;
7479 
7480 			if (!dev || !dev->bdev ||
7481 			    test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) ||
7482 			    dev->last_flush_error)
7483 				missing++;
7484 			else if (failing_dev && failing_dev == dev)
7485 				missing++;
7486 		}
7487 		if (missing > max_tolerated) {
7488 			if (!failing_dev)
7489 				btrfs_warn(fs_info,
7490 	"chunk %llu missing %d devices, max tolerance is %d for writable mount",
7491 				   em->start, missing, max_tolerated);
7492 			free_extent_map(em);
7493 			ret = false;
7494 			goto out;
7495 		}
7496 		next_start = extent_map_end(em);
7497 		free_extent_map(em);
7498 
7499 		read_lock(&map_tree->lock);
7500 		em = lookup_extent_mapping(map_tree, next_start,
7501 					   (u64)(-1) - next_start);
7502 		read_unlock(&map_tree->lock);
7503 	}
7504 out:
7505 	return ret;
7506 }
7507 
7508 static void readahead_tree_node_children(struct extent_buffer *node)
7509 {
7510 	int i;
7511 	const int nr_items = btrfs_header_nritems(node);
7512 
7513 	for (i = 0; i < nr_items; i++)
7514 		btrfs_readahead_node_child(node, i);
7515 }
7516 
7517 int btrfs_read_chunk_tree(struct btrfs_fs_info *fs_info)
7518 {
7519 	struct btrfs_root *root = fs_info->chunk_root;
7520 	struct btrfs_path *path;
7521 	struct extent_buffer *leaf;
7522 	struct btrfs_key key;
7523 	struct btrfs_key found_key;
7524 	int ret;
7525 	int slot;
7526 	int iter_ret = 0;
7527 	u64 total_dev = 0;
7528 	u64 last_ra_node = 0;
7529 
7530 	path = btrfs_alloc_path();
7531 	if (!path)
7532 		return -ENOMEM;
7533 
7534 	/*
7535 	 * uuid_mutex is needed only if we are mounting a sprout FS
7536 	 * otherwise we don't need it.
7537 	 */
7538 	mutex_lock(&uuid_mutex);
7539 
7540 	/*
7541 	 * It is possible for mount and umount to race in such a way that
7542 	 * we execute this code path, but open_fs_devices failed to clear
7543 	 * total_rw_bytes. We certainly want it cleared before reading the
7544 	 * device items, so clear it here.
7545 	 */
7546 	fs_info->fs_devices->total_rw_bytes = 0;
7547 
7548 	/*
7549 	 * Lockdep complains about possible circular locking dependency between
7550 	 * a disk's open_mutex (struct gendisk.open_mutex), the rw semaphores
7551 	 * used for freeze procection of a fs (struct super_block.s_writers),
7552 	 * which we take when starting a transaction, and extent buffers of the
7553 	 * chunk tree if we call read_one_dev() while holding a lock on an
7554 	 * extent buffer of the chunk tree. Since we are mounting the filesystem
7555 	 * and at this point there can't be any concurrent task modifying the
7556 	 * chunk tree, to keep it simple, just skip locking on the chunk tree.
7557 	 */
7558 	ASSERT(!test_bit(BTRFS_FS_OPEN, &fs_info->flags));
7559 	path->skip_locking = 1;
7560 
7561 	/*
7562 	 * Read all device items, and then all the chunk items. All
7563 	 * device items are found before any chunk item (their object id
7564 	 * is smaller than the lowest possible object id for a chunk
7565 	 * item - BTRFS_FIRST_CHUNK_TREE_OBJECTID).
7566 	 */
7567 	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
7568 	key.offset = 0;
7569 	key.type = 0;
7570 	btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
7571 		struct extent_buffer *node = path->nodes[1];
7572 
7573 		leaf = path->nodes[0];
7574 		slot = path->slots[0];
7575 
7576 		if (node) {
7577 			if (last_ra_node != node->start) {
7578 				readahead_tree_node_children(node);
7579 				last_ra_node = node->start;
7580 			}
7581 		}
7582 		if (found_key.type == BTRFS_DEV_ITEM_KEY) {
7583 			struct btrfs_dev_item *dev_item;
7584 			dev_item = btrfs_item_ptr(leaf, slot,
7585 						  struct btrfs_dev_item);
7586 			ret = read_one_dev(leaf, dev_item);
7587 			if (ret)
7588 				goto error;
7589 			total_dev++;
7590 		} else if (found_key.type == BTRFS_CHUNK_ITEM_KEY) {
7591 			struct btrfs_chunk *chunk;
7592 
7593 			/*
7594 			 * We are only called at mount time, so no need to take
7595 			 * fs_info->chunk_mutex. Plus, to avoid lockdep warnings,
7596 			 * we always lock first fs_info->chunk_mutex before
7597 			 * acquiring any locks on the chunk tree. This is a
7598 			 * requirement for chunk allocation, see the comment on
7599 			 * top of btrfs_chunk_alloc() for details.
7600 			 */
7601 			chunk = btrfs_item_ptr(leaf, slot, struct btrfs_chunk);
7602 			ret = read_one_chunk(&found_key, leaf, chunk);
7603 			if (ret)
7604 				goto error;
7605 		}
7606 	}
7607 	/* Catch error found during iteration */
7608 	if (iter_ret < 0) {
7609 		ret = iter_ret;
7610 		goto error;
7611 	}
7612 
7613 	/*
7614 	 * After loading chunk tree, we've got all device information,
7615 	 * do another round of validation checks.
7616 	 */
7617 	if (total_dev != fs_info->fs_devices->total_devices) {
7618 		btrfs_warn(fs_info,
7619 "super block num_devices %llu mismatch with DEV_ITEM count %llu, will be repaired on next transaction commit",
7620 			  btrfs_super_num_devices(fs_info->super_copy),
7621 			  total_dev);
7622 		fs_info->fs_devices->total_devices = total_dev;
7623 		btrfs_set_super_num_devices(fs_info->super_copy, total_dev);
7624 	}
7625 	if (btrfs_super_total_bytes(fs_info->super_copy) <
7626 	    fs_info->fs_devices->total_rw_bytes) {
7627 		btrfs_err(fs_info,
7628 	"super_total_bytes %llu mismatch with fs_devices total_rw_bytes %llu",
7629 			  btrfs_super_total_bytes(fs_info->super_copy),
7630 			  fs_info->fs_devices->total_rw_bytes);
7631 		ret = -EINVAL;
7632 		goto error;
7633 	}
7634 	ret = 0;
7635 error:
7636 	mutex_unlock(&uuid_mutex);
7637 
7638 	btrfs_free_path(path);
7639 	return ret;
7640 }
7641 
7642 void btrfs_init_devices_late(struct btrfs_fs_info *fs_info)
7643 {
7644 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs;
7645 	struct btrfs_device *device;
7646 
7647 	fs_devices->fs_info = fs_info;
7648 
7649 	mutex_lock(&fs_devices->device_list_mutex);
7650 	list_for_each_entry(device, &fs_devices->devices, dev_list)
7651 		device->fs_info = fs_info;
7652 
7653 	list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) {
7654 		list_for_each_entry(device, &seed_devs->devices, dev_list)
7655 			device->fs_info = fs_info;
7656 
7657 		seed_devs->fs_info = fs_info;
7658 	}
7659 	mutex_unlock(&fs_devices->device_list_mutex);
7660 }
7661 
7662 static u64 btrfs_dev_stats_value(const struct extent_buffer *eb,
7663 				 const struct btrfs_dev_stats_item *ptr,
7664 				 int index)
7665 {
7666 	u64 val;
7667 
7668 	read_extent_buffer(eb, &val,
7669 			   offsetof(struct btrfs_dev_stats_item, values) +
7670 			    ((unsigned long)ptr) + (index * sizeof(u64)),
7671 			   sizeof(val));
7672 	return val;
7673 }
7674 
7675 static void btrfs_set_dev_stats_value(struct extent_buffer *eb,
7676 				      struct btrfs_dev_stats_item *ptr,
7677 				      int index, u64 val)
7678 {
7679 	write_extent_buffer(eb, &val,
7680 			    offsetof(struct btrfs_dev_stats_item, values) +
7681 			     ((unsigned long)ptr) + (index * sizeof(u64)),
7682 			    sizeof(val));
7683 }
7684 
7685 static int btrfs_device_init_dev_stats(struct btrfs_device *device,
7686 				       struct btrfs_path *path)
7687 {
7688 	struct btrfs_dev_stats_item *ptr;
7689 	struct extent_buffer *eb;
7690 	struct btrfs_key key;
7691 	int item_size;
7692 	int i, ret, slot;
7693 
7694 	if (!device->fs_info->dev_root)
7695 		return 0;
7696 
7697 	key.objectid = BTRFS_DEV_STATS_OBJECTID;
7698 	key.type = BTRFS_PERSISTENT_ITEM_KEY;
7699 	key.offset = device->devid;
7700 	ret = btrfs_search_slot(NULL, device->fs_info->dev_root, &key, path, 0, 0);
7701 	if (ret) {
7702 		for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
7703 			btrfs_dev_stat_set(device, i, 0);
7704 		device->dev_stats_valid = 1;
7705 		btrfs_release_path(path);
7706 		return ret < 0 ? ret : 0;
7707 	}
7708 	slot = path->slots[0];
7709 	eb = path->nodes[0];
7710 	item_size = btrfs_item_size(eb, slot);
7711 
7712 	ptr = btrfs_item_ptr(eb, slot, struct btrfs_dev_stats_item);
7713 
7714 	for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) {
7715 		if (item_size >= (1 + i) * sizeof(__le64))
7716 			btrfs_dev_stat_set(device, i,
7717 					   btrfs_dev_stats_value(eb, ptr, i));
7718 		else
7719 			btrfs_dev_stat_set(device, i, 0);
7720 	}
7721 
7722 	device->dev_stats_valid = 1;
7723 	btrfs_dev_stat_print_on_load(device);
7724 	btrfs_release_path(path);
7725 
7726 	return 0;
7727 }
7728 
7729 int btrfs_init_dev_stats(struct btrfs_fs_info *fs_info)
7730 {
7731 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs;
7732 	struct btrfs_device *device;
7733 	struct btrfs_path *path = NULL;
7734 	int ret = 0;
7735 
7736 	path = btrfs_alloc_path();
7737 	if (!path)
7738 		return -ENOMEM;
7739 
7740 	mutex_lock(&fs_devices->device_list_mutex);
7741 	list_for_each_entry(device, &fs_devices->devices, dev_list) {
7742 		ret = btrfs_device_init_dev_stats(device, path);
7743 		if (ret)
7744 			goto out;
7745 	}
7746 	list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) {
7747 		list_for_each_entry(device, &seed_devs->devices, dev_list) {
7748 			ret = btrfs_device_init_dev_stats(device, path);
7749 			if (ret)
7750 				goto out;
7751 		}
7752 	}
7753 out:
7754 	mutex_unlock(&fs_devices->device_list_mutex);
7755 
7756 	btrfs_free_path(path);
7757 	return ret;
7758 }
7759 
7760 static int update_dev_stat_item(struct btrfs_trans_handle *trans,
7761 				struct btrfs_device *device)
7762 {
7763 	struct btrfs_fs_info *fs_info = trans->fs_info;
7764 	struct btrfs_root *dev_root = fs_info->dev_root;
7765 	struct btrfs_path *path;
7766 	struct btrfs_key key;
7767 	struct extent_buffer *eb;
7768 	struct btrfs_dev_stats_item *ptr;
7769 	int ret;
7770 	int i;
7771 
7772 	key.objectid = BTRFS_DEV_STATS_OBJECTID;
7773 	key.type = BTRFS_PERSISTENT_ITEM_KEY;
7774 	key.offset = device->devid;
7775 
7776 	path = btrfs_alloc_path();
7777 	if (!path)
7778 		return -ENOMEM;
7779 	ret = btrfs_search_slot(trans, dev_root, &key, path, -1, 1);
7780 	if (ret < 0) {
7781 		btrfs_warn_in_rcu(fs_info,
7782 			"error %d while searching for dev_stats item for device %s",
7783 			      ret, rcu_str_deref(device->name));
7784 		goto out;
7785 	}
7786 
7787 	if (ret == 0 &&
7788 	    btrfs_item_size(path->nodes[0], path->slots[0]) < sizeof(*ptr)) {
7789 		/* need to delete old one and insert a new one */
7790 		ret = btrfs_del_item(trans, dev_root, path);
7791 		if (ret != 0) {
7792 			btrfs_warn_in_rcu(fs_info,
7793 				"delete too small dev_stats item for device %s failed %d",
7794 				      rcu_str_deref(device->name), ret);
7795 			goto out;
7796 		}
7797 		ret = 1;
7798 	}
7799 
7800 	if (ret == 1) {
7801 		/* need to insert a new item */
7802 		btrfs_release_path(path);
7803 		ret = btrfs_insert_empty_item(trans, dev_root, path,
7804 					      &key, sizeof(*ptr));
7805 		if (ret < 0) {
7806 			btrfs_warn_in_rcu(fs_info,
7807 				"insert dev_stats item for device %s failed %d",
7808 				rcu_str_deref(device->name), ret);
7809 			goto out;
7810 		}
7811 	}
7812 
7813 	eb = path->nodes[0];
7814 	ptr = btrfs_item_ptr(eb, path->slots[0], struct btrfs_dev_stats_item);
7815 	for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
7816 		btrfs_set_dev_stats_value(eb, ptr, i,
7817 					  btrfs_dev_stat_read(device, i));
7818 	btrfs_mark_buffer_dirty(eb);
7819 
7820 out:
7821 	btrfs_free_path(path);
7822 	return ret;
7823 }
7824 
7825 /*
7826  * called from commit_transaction. Writes all changed device stats to disk.
7827  */
7828 int btrfs_run_dev_stats(struct btrfs_trans_handle *trans)
7829 {
7830 	struct btrfs_fs_info *fs_info = trans->fs_info;
7831 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
7832 	struct btrfs_device *device;
7833 	int stats_cnt;
7834 	int ret = 0;
7835 
7836 	mutex_lock(&fs_devices->device_list_mutex);
7837 	list_for_each_entry(device, &fs_devices->devices, dev_list) {
7838 		stats_cnt = atomic_read(&device->dev_stats_ccnt);
7839 		if (!device->dev_stats_valid || stats_cnt == 0)
7840 			continue;
7841 
7842 
7843 		/*
7844 		 * There is a LOAD-LOAD control dependency between the value of
7845 		 * dev_stats_ccnt and updating the on-disk values which requires
7846 		 * reading the in-memory counters. Such control dependencies
7847 		 * require explicit read memory barriers.
7848 		 *
7849 		 * This memory barriers pairs with smp_mb__before_atomic in
7850 		 * btrfs_dev_stat_inc/btrfs_dev_stat_set and with the full
7851 		 * barrier implied by atomic_xchg in
7852 		 * btrfs_dev_stats_read_and_reset
7853 		 */
7854 		smp_rmb();
7855 
7856 		ret = update_dev_stat_item(trans, device);
7857 		if (!ret)
7858 			atomic_sub(stats_cnt, &device->dev_stats_ccnt);
7859 	}
7860 	mutex_unlock(&fs_devices->device_list_mutex);
7861 
7862 	return ret;
7863 }
7864 
7865 void btrfs_dev_stat_inc_and_print(struct btrfs_device *dev, int index)
7866 {
7867 	btrfs_dev_stat_inc(dev, index);
7868 	btrfs_dev_stat_print_on_error(dev);
7869 }
7870 
7871 static void btrfs_dev_stat_print_on_error(struct btrfs_device *dev)
7872 {
7873 	if (!dev->dev_stats_valid)
7874 		return;
7875 	btrfs_err_rl_in_rcu(dev->fs_info,
7876 		"bdev %s errs: wr %u, rd %u, flush %u, corrupt %u, gen %u",
7877 			   rcu_str_deref(dev->name),
7878 			   btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_WRITE_ERRS),
7879 			   btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_READ_ERRS),
7880 			   btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_FLUSH_ERRS),
7881 			   btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS),
7882 			   btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_GENERATION_ERRS));
7883 }
7884 
7885 static void btrfs_dev_stat_print_on_load(struct btrfs_device *dev)
7886 {
7887 	int i;
7888 
7889 	for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
7890 		if (btrfs_dev_stat_read(dev, i) != 0)
7891 			break;
7892 	if (i == BTRFS_DEV_STAT_VALUES_MAX)
7893 		return; /* all values == 0, suppress message */
7894 
7895 	btrfs_info_in_rcu(dev->fs_info,
7896 		"bdev %s errs: wr %u, rd %u, flush %u, corrupt %u, gen %u",
7897 	       rcu_str_deref(dev->name),
7898 	       btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_WRITE_ERRS),
7899 	       btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_READ_ERRS),
7900 	       btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_FLUSH_ERRS),
7901 	       btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS),
7902 	       btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_GENERATION_ERRS));
7903 }
7904 
7905 int btrfs_get_dev_stats(struct btrfs_fs_info *fs_info,
7906 			struct btrfs_ioctl_get_dev_stats *stats)
7907 {
7908 	BTRFS_DEV_LOOKUP_ARGS(args);
7909 	struct btrfs_device *dev;
7910 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
7911 	int i;
7912 
7913 	mutex_lock(&fs_devices->device_list_mutex);
7914 	args.devid = stats->devid;
7915 	dev = btrfs_find_device(fs_info->fs_devices, &args);
7916 	mutex_unlock(&fs_devices->device_list_mutex);
7917 
7918 	if (!dev) {
7919 		btrfs_warn(fs_info, "get dev_stats failed, device not found");
7920 		return -ENODEV;
7921 	} else if (!dev->dev_stats_valid) {
7922 		btrfs_warn(fs_info, "get dev_stats failed, not yet valid");
7923 		return -ENODEV;
7924 	} else if (stats->flags & BTRFS_DEV_STATS_RESET) {
7925 		for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) {
7926 			if (stats->nr_items > i)
7927 				stats->values[i] =
7928 					btrfs_dev_stat_read_and_reset(dev, i);
7929 			else
7930 				btrfs_dev_stat_set(dev, i, 0);
7931 		}
7932 		btrfs_info(fs_info, "device stats zeroed by %s (%d)",
7933 			   current->comm, task_pid_nr(current));
7934 	} else {
7935 		for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
7936 			if (stats->nr_items > i)
7937 				stats->values[i] = btrfs_dev_stat_read(dev, i);
7938 	}
7939 	if (stats->nr_items > BTRFS_DEV_STAT_VALUES_MAX)
7940 		stats->nr_items = BTRFS_DEV_STAT_VALUES_MAX;
7941 	return 0;
7942 }
7943 
7944 /*
7945  * Update the size and bytes used for each device where it changed.  This is
7946  * delayed since we would otherwise get errors while writing out the
7947  * superblocks.
7948  *
7949  * Must be invoked during transaction commit.
7950  */
7951 void btrfs_commit_device_sizes(struct btrfs_transaction *trans)
7952 {
7953 	struct btrfs_device *curr, *next;
7954 
7955 	ASSERT(trans->state == TRANS_STATE_COMMIT_DOING);
7956 
7957 	if (list_empty(&trans->dev_update_list))
7958 		return;
7959 
7960 	/*
7961 	 * We don't need the device_list_mutex here.  This list is owned by the
7962 	 * transaction and the transaction must complete before the device is
7963 	 * released.
7964 	 */
7965 	mutex_lock(&trans->fs_info->chunk_mutex);
7966 	list_for_each_entry_safe(curr, next, &trans->dev_update_list,
7967 				 post_commit_list) {
7968 		list_del_init(&curr->post_commit_list);
7969 		curr->commit_total_bytes = curr->disk_total_bytes;
7970 		curr->commit_bytes_used = curr->bytes_used;
7971 	}
7972 	mutex_unlock(&trans->fs_info->chunk_mutex);
7973 }
7974 
7975 /*
7976  * Multiplicity factor for simple profiles: DUP, RAID1-like and RAID10.
7977  */
7978 int btrfs_bg_type_to_factor(u64 flags)
7979 {
7980 	const int index = btrfs_bg_flags_to_raid_index(flags);
7981 
7982 	return btrfs_raid_array[index].ncopies;
7983 }
7984 
7985 
7986 
7987 static int verify_one_dev_extent(struct btrfs_fs_info *fs_info,
7988 				 u64 chunk_offset, u64 devid,
7989 				 u64 physical_offset, u64 physical_len)
7990 {
7991 	struct btrfs_dev_lookup_args args = { .devid = devid };
7992 	struct extent_map_tree *em_tree = &fs_info->mapping_tree;
7993 	struct extent_map *em;
7994 	struct map_lookup *map;
7995 	struct btrfs_device *dev;
7996 	u64 stripe_len;
7997 	bool found = false;
7998 	int ret = 0;
7999 	int i;
8000 
8001 	read_lock(&em_tree->lock);
8002 	em = lookup_extent_mapping(em_tree, chunk_offset, 1);
8003 	read_unlock(&em_tree->lock);
8004 
8005 	if (!em) {
8006 		btrfs_err(fs_info,
8007 "dev extent physical offset %llu on devid %llu doesn't have corresponding chunk",
8008 			  physical_offset, devid);
8009 		ret = -EUCLEAN;
8010 		goto out;
8011 	}
8012 
8013 	map = em->map_lookup;
8014 	stripe_len = calc_stripe_length(map->type, em->len, map->num_stripes);
8015 	if (physical_len != stripe_len) {
8016 		btrfs_err(fs_info,
8017 "dev extent physical offset %llu on devid %llu length doesn't match chunk %llu, have %llu expect %llu",
8018 			  physical_offset, devid, em->start, physical_len,
8019 			  stripe_len);
8020 		ret = -EUCLEAN;
8021 		goto out;
8022 	}
8023 
8024 	for (i = 0; i < map->num_stripes; i++) {
8025 		if (map->stripes[i].dev->devid == devid &&
8026 		    map->stripes[i].physical == physical_offset) {
8027 			found = true;
8028 			if (map->verified_stripes >= map->num_stripes) {
8029 				btrfs_err(fs_info,
8030 				"too many dev extents for chunk %llu found",
8031 					  em->start);
8032 				ret = -EUCLEAN;
8033 				goto out;
8034 			}
8035 			map->verified_stripes++;
8036 			break;
8037 		}
8038 	}
8039 	if (!found) {
8040 		btrfs_err(fs_info,
8041 	"dev extent physical offset %llu devid %llu has no corresponding chunk",
8042 			physical_offset, devid);
8043 		ret = -EUCLEAN;
8044 	}
8045 
8046 	/* Make sure no dev extent is beyond device boundary */
8047 	dev = btrfs_find_device(fs_info->fs_devices, &args);
8048 	if (!dev) {
8049 		btrfs_err(fs_info, "failed to find devid %llu", devid);
8050 		ret = -EUCLEAN;
8051 		goto out;
8052 	}
8053 
8054 	if (physical_offset + physical_len > dev->disk_total_bytes) {
8055 		btrfs_err(fs_info,
8056 "dev extent devid %llu physical offset %llu len %llu is beyond device boundary %llu",
8057 			  devid, physical_offset, physical_len,
8058 			  dev->disk_total_bytes);
8059 		ret = -EUCLEAN;
8060 		goto out;
8061 	}
8062 
8063 	if (dev->zone_info) {
8064 		u64 zone_size = dev->zone_info->zone_size;
8065 
8066 		if (!IS_ALIGNED(physical_offset, zone_size) ||
8067 		    !IS_ALIGNED(physical_len, zone_size)) {
8068 			btrfs_err(fs_info,
8069 "zoned: dev extent devid %llu physical offset %llu len %llu is not aligned to device zone",
8070 				  devid, physical_offset, physical_len);
8071 			ret = -EUCLEAN;
8072 			goto out;
8073 		}
8074 	}
8075 
8076 out:
8077 	free_extent_map(em);
8078 	return ret;
8079 }
8080 
8081 static int verify_chunk_dev_extent_mapping(struct btrfs_fs_info *fs_info)
8082 {
8083 	struct extent_map_tree *em_tree = &fs_info->mapping_tree;
8084 	struct extent_map *em;
8085 	struct rb_node *node;
8086 	int ret = 0;
8087 
8088 	read_lock(&em_tree->lock);
8089 	for (node = rb_first_cached(&em_tree->map); node; node = rb_next(node)) {
8090 		em = rb_entry(node, struct extent_map, rb_node);
8091 		if (em->map_lookup->num_stripes !=
8092 		    em->map_lookup->verified_stripes) {
8093 			btrfs_err(fs_info,
8094 			"chunk %llu has missing dev extent, have %d expect %d",
8095 				  em->start, em->map_lookup->verified_stripes,
8096 				  em->map_lookup->num_stripes);
8097 			ret = -EUCLEAN;
8098 			goto out;
8099 		}
8100 	}
8101 out:
8102 	read_unlock(&em_tree->lock);
8103 	return ret;
8104 }
8105 
8106 /*
8107  * Ensure that all dev extents are mapped to correct chunk, otherwise
8108  * later chunk allocation/free would cause unexpected behavior.
8109  *
8110  * NOTE: This will iterate through the whole device tree, which should be of
8111  * the same size level as the chunk tree.  This slightly increases mount time.
8112  */
8113 int btrfs_verify_dev_extents(struct btrfs_fs_info *fs_info)
8114 {
8115 	struct btrfs_path *path;
8116 	struct btrfs_root *root = fs_info->dev_root;
8117 	struct btrfs_key key;
8118 	u64 prev_devid = 0;
8119 	u64 prev_dev_ext_end = 0;
8120 	int ret = 0;
8121 
8122 	/*
8123 	 * We don't have a dev_root because we mounted with ignorebadroots and
8124 	 * failed to load the root, so we want to skip the verification in this
8125 	 * case for sure.
8126 	 *
8127 	 * However if the dev root is fine, but the tree itself is corrupted
8128 	 * we'd still fail to mount.  This verification is only to make sure
8129 	 * writes can happen safely, so instead just bypass this check
8130 	 * completely in the case of IGNOREBADROOTS.
8131 	 */
8132 	if (btrfs_test_opt(fs_info, IGNOREBADROOTS))
8133 		return 0;
8134 
8135 	key.objectid = 1;
8136 	key.type = BTRFS_DEV_EXTENT_KEY;
8137 	key.offset = 0;
8138 
8139 	path = btrfs_alloc_path();
8140 	if (!path)
8141 		return -ENOMEM;
8142 
8143 	path->reada = READA_FORWARD;
8144 	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
8145 	if (ret < 0)
8146 		goto out;
8147 
8148 	if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) {
8149 		ret = btrfs_next_leaf(root, path);
8150 		if (ret < 0)
8151 			goto out;
8152 		/* No dev extents at all? Not good */
8153 		if (ret > 0) {
8154 			ret = -EUCLEAN;
8155 			goto out;
8156 		}
8157 	}
8158 	while (1) {
8159 		struct extent_buffer *leaf = path->nodes[0];
8160 		struct btrfs_dev_extent *dext;
8161 		int slot = path->slots[0];
8162 		u64 chunk_offset;
8163 		u64 physical_offset;
8164 		u64 physical_len;
8165 		u64 devid;
8166 
8167 		btrfs_item_key_to_cpu(leaf, &key, slot);
8168 		if (key.type != BTRFS_DEV_EXTENT_KEY)
8169 			break;
8170 		devid = key.objectid;
8171 		physical_offset = key.offset;
8172 
8173 		dext = btrfs_item_ptr(leaf, slot, struct btrfs_dev_extent);
8174 		chunk_offset = btrfs_dev_extent_chunk_offset(leaf, dext);
8175 		physical_len = btrfs_dev_extent_length(leaf, dext);
8176 
8177 		/* Check if this dev extent overlaps with the previous one */
8178 		if (devid == prev_devid && physical_offset < prev_dev_ext_end) {
8179 			btrfs_err(fs_info,
8180 "dev extent devid %llu physical offset %llu overlap with previous dev extent end %llu",
8181 				  devid, physical_offset, prev_dev_ext_end);
8182 			ret = -EUCLEAN;
8183 			goto out;
8184 		}
8185 
8186 		ret = verify_one_dev_extent(fs_info, chunk_offset, devid,
8187 					    physical_offset, physical_len);
8188 		if (ret < 0)
8189 			goto out;
8190 		prev_devid = devid;
8191 		prev_dev_ext_end = physical_offset + physical_len;
8192 
8193 		ret = btrfs_next_item(root, path);
8194 		if (ret < 0)
8195 			goto out;
8196 		if (ret > 0) {
8197 			ret = 0;
8198 			break;
8199 		}
8200 	}
8201 
8202 	/* Ensure all chunks have corresponding dev extents */
8203 	ret = verify_chunk_dev_extent_mapping(fs_info);
8204 out:
8205 	btrfs_free_path(path);
8206 	return ret;
8207 }
8208 
8209 /*
8210  * Check whether the given block group or device is pinned by any inode being
8211  * used as a swapfile.
8212  */
8213 bool btrfs_pinned_by_swapfile(struct btrfs_fs_info *fs_info, void *ptr)
8214 {
8215 	struct btrfs_swapfile_pin *sp;
8216 	struct rb_node *node;
8217 
8218 	spin_lock(&fs_info->swapfile_pins_lock);
8219 	node = fs_info->swapfile_pins.rb_node;
8220 	while (node) {
8221 		sp = rb_entry(node, struct btrfs_swapfile_pin, node);
8222 		if (ptr < sp->ptr)
8223 			node = node->rb_left;
8224 		else if (ptr > sp->ptr)
8225 			node = node->rb_right;
8226 		else
8227 			break;
8228 	}
8229 	spin_unlock(&fs_info->swapfile_pins_lock);
8230 	return node != NULL;
8231 }
8232 
8233 static int relocating_repair_kthread(void *data)
8234 {
8235 	struct btrfs_block_group *cache = data;
8236 	struct btrfs_fs_info *fs_info = cache->fs_info;
8237 	u64 target;
8238 	int ret = 0;
8239 
8240 	target = cache->start;
8241 	btrfs_put_block_group(cache);
8242 
8243 	sb_start_write(fs_info->sb);
8244 	if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_BALANCE)) {
8245 		btrfs_info(fs_info,
8246 			   "zoned: skip relocating block group %llu to repair: EBUSY",
8247 			   target);
8248 		sb_end_write(fs_info->sb);
8249 		return -EBUSY;
8250 	}
8251 
8252 	mutex_lock(&fs_info->reclaim_bgs_lock);
8253 
8254 	/* Ensure block group still exists */
8255 	cache = btrfs_lookup_block_group(fs_info, target);
8256 	if (!cache)
8257 		goto out;
8258 
8259 	if (!cache->relocating_repair)
8260 		goto out;
8261 
8262 	ret = btrfs_may_alloc_data_chunk(fs_info, target);
8263 	if (ret < 0)
8264 		goto out;
8265 
8266 	btrfs_info(fs_info,
8267 		   "zoned: relocating block group %llu to repair IO failure",
8268 		   target);
8269 	ret = btrfs_relocate_chunk(fs_info, target);
8270 
8271 out:
8272 	if (cache)
8273 		btrfs_put_block_group(cache);
8274 	mutex_unlock(&fs_info->reclaim_bgs_lock);
8275 	btrfs_exclop_finish(fs_info);
8276 	sb_end_write(fs_info->sb);
8277 
8278 	return ret;
8279 }
8280 
8281 bool btrfs_repair_one_zone(struct btrfs_fs_info *fs_info, u64 logical)
8282 {
8283 	struct btrfs_block_group *cache;
8284 
8285 	if (!btrfs_is_zoned(fs_info))
8286 		return false;
8287 
8288 	/* Do not attempt to repair in degraded state */
8289 	if (btrfs_test_opt(fs_info, DEGRADED))
8290 		return true;
8291 
8292 	cache = btrfs_lookup_block_group(fs_info, logical);
8293 	if (!cache)
8294 		return true;
8295 
8296 	spin_lock(&cache->lock);
8297 	if (cache->relocating_repair) {
8298 		spin_unlock(&cache->lock);
8299 		btrfs_put_block_group(cache);
8300 		return true;
8301 	}
8302 	cache->relocating_repair = 1;
8303 	spin_unlock(&cache->lock);
8304 
8305 	kthread_run(relocating_repair_kthread, cache,
8306 		    "btrfs-relocating-repair");
8307 
8308 	return true;
8309 }
8310