xref: /openbmc/linux/fs/btrfs/dev-replace.c (revision ecd25094)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) STRATO AG 2012.  All rights reserved.
4  */
5 
6 #include <linux/sched.h>
7 #include <linux/bio.h>
8 #include <linux/slab.h>
9 #include <linux/blkdev.h>
10 #include <linux/kthread.h>
11 #include <linux/math64.h>
12 #include "misc.h"
13 #include "ctree.h"
14 #include "extent_map.h"
15 #include "disk-io.h"
16 #include "transaction.h"
17 #include "print-tree.h"
18 #include "volumes.h"
19 #include "async-thread.h"
20 #include "check-integrity.h"
21 #include "rcu-string.h"
22 #include "dev-replace.h"
23 #include "sysfs.h"
24 
25 static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
26 				       int scrub_ret);
27 static void btrfs_dev_replace_update_device_in_mapping_tree(
28 						struct btrfs_fs_info *fs_info,
29 						struct btrfs_device *srcdev,
30 						struct btrfs_device *tgtdev);
31 static int btrfs_dev_replace_kthread(void *data);
32 
33 int btrfs_init_dev_replace(struct btrfs_fs_info *fs_info)
34 {
35 	struct btrfs_key key;
36 	struct btrfs_root *dev_root = fs_info->dev_root;
37 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
38 	struct extent_buffer *eb;
39 	int slot;
40 	int ret = 0;
41 	struct btrfs_path *path = NULL;
42 	int item_size;
43 	struct btrfs_dev_replace_item *ptr;
44 	u64 src_devid;
45 
46 	path = btrfs_alloc_path();
47 	if (!path) {
48 		ret = -ENOMEM;
49 		goto out;
50 	}
51 
52 	key.objectid = 0;
53 	key.type = BTRFS_DEV_REPLACE_KEY;
54 	key.offset = 0;
55 	ret = btrfs_search_slot(NULL, dev_root, &key, path, 0, 0);
56 	if (ret) {
57 no_valid_dev_replace_entry_found:
58 		ret = 0;
59 		dev_replace->replace_state =
60 			BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED;
61 		dev_replace->cont_reading_from_srcdev_mode =
62 		    BTRFS_DEV_REPLACE_ITEM_CONT_READING_FROM_SRCDEV_MODE_ALWAYS;
63 		dev_replace->time_started = 0;
64 		dev_replace->time_stopped = 0;
65 		atomic64_set(&dev_replace->num_write_errors, 0);
66 		atomic64_set(&dev_replace->num_uncorrectable_read_errors, 0);
67 		dev_replace->cursor_left = 0;
68 		dev_replace->committed_cursor_left = 0;
69 		dev_replace->cursor_left_last_write_of_item = 0;
70 		dev_replace->cursor_right = 0;
71 		dev_replace->srcdev = NULL;
72 		dev_replace->tgtdev = NULL;
73 		dev_replace->is_valid = 0;
74 		dev_replace->item_needs_writeback = 0;
75 		goto out;
76 	}
77 	slot = path->slots[0];
78 	eb = path->nodes[0];
79 	item_size = btrfs_item_size_nr(eb, slot);
80 	ptr = btrfs_item_ptr(eb, slot, struct btrfs_dev_replace_item);
81 
82 	if (item_size != sizeof(struct btrfs_dev_replace_item)) {
83 		btrfs_warn(fs_info,
84 			"dev_replace entry found has unexpected size, ignore entry");
85 		goto no_valid_dev_replace_entry_found;
86 	}
87 
88 	src_devid = btrfs_dev_replace_src_devid(eb, ptr);
89 	dev_replace->cont_reading_from_srcdev_mode =
90 		btrfs_dev_replace_cont_reading_from_srcdev_mode(eb, ptr);
91 	dev_replace->replace_state = btrfs_dev_replace_replace_state(eb, ptr);
92 	dev_replace->time_started = btrfs_dev_replace_time_started(eb, ptr);
93 	dev_replace->time_stopped =
94 		btrfs_dev_replace_time_stopped(eb, ptr);
95 	atomic64_set(&dev_replace->num_write_errors,
96 		     btrfs_dev_replace_num_write_errors(eb, ptr));
97 	atomic64_set(&dev_replace->num_uncorrectable_read_errors,
98 		     btrfs_dev_replace_num_uncorrectable_read_errors(eb, ptr));
99 	dev_replace->cursor_left = btrfs_dev_replace_cursor_left(eb, ptr);
100 	dev_replace->committed_cursor_left = dev_replace->cursor_left;
101 	dev_replace->cursor_left_last_write_of_item = dev_replace->cursor_left;
102 	dev_replace->cursor_right = btrfs_dev_replace_cursor_right(eb, ptr);
103 	dev_replace->is_valid = 1;
104 
105 	dev_replace->item_needs_writeback = 0;
106 	switch (dev_replace->replace_state) {
107 	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
108 	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
109 	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
110 		dev_replace->srcdev = NULL;
111 		dev_replace->tgtdev = NULL;
112 		break;
113 	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
114 	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
115 		dev_replace->srcdev = btrfs_find_device(fs_info->fs_devices,
116 						src_devid, NULL, NULL, true);
117 		dev_replace->tgtdev = btrfs_find_device(fs_info->fs_devices,
118 							BTRFS_DEV_REPLACE_DEVID,
119 							NULL, NULL, true);
120 		/*
121 		 * allow 'btrfs dev replace_cancel' if src/tgt device is
122 		 * missing
123 		 */
124 		if (!dev_replace->srcdev &&
125 		    !btrfs_test_opt(fs_info, DEGRADED)) {
126 			ret = -EIO;
127 			btrfs_warn(fs_info,
128 			   "cannot mount because device replace operation is ongoing and");
129 			btrfs_warn(fs_info,
130 			   "srcdev (devid %llu) is missing, need to run 'btrfs dev scan'?",
131 			   src_devid);
132 		}
133 		if (!dev_replace->tgtdev &&
134 		    !btrfs_test_opt(fs_info, DEGRADED)) {
135 			ret = -EIO;
136 			btrfs_warn(fs_info,
137 			   "cannot mount because device replace operation is ongoing and");
138 			btrfs_warn(fs_info,
139 			   "tgtdev (devid %llu) is missing, need to run 'btrfs dev scan'?",
140 				BTRFS_DEV_REPLACE_DEVID);
141 		}
142 		if (dev_replace->tgtdev) {
143 			if (dev_replace->srcdev) {
144 				dev_replace->tgtdev->total_bytes =
145 					dev_replace->srcdev->total_bytes;
146 				dev_replace->tgtdev->disk_total_bytes =
147 					dev_replace->srcdev->disk_total_bytes;
148 				dev_replace->tgtdev->commit_total_bytes =
149 					dev_replace->srcdev->commit_total_bytes;
150 				dev_replace->tgtdev->bytes_used =
151 					dev_replace->srcdev->bytes_used;
152 				dev_replace->tgtdev->commit_bytes_used =
153 					dev_replace->srcdev->commit_bytes_used;
154 			}
155 			set_bit(BTRFS_DEV_STATE_REPLACE_TGT,
156 				&dev_replace->tgtdev->dev_state);
157 
158 			WARN_ON(fs_info->fs_devices->rw_devices == 0);
159 			dev_replace->tgtdev->io_width = fs_info->sectorsize;
160 			dev_replace->tgtdev->io_align = fs_info->sectorsize;
161 			dev_replace->tgtdev->sector_size = fs_info->sectorsize;
162 			dev_replace->tgtdev->fs_info = fs_info;
163 			set_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
164 				&dev_replace->tgtdev->dev_state);
165 		}
166 		break;
167 	}
168 
169 out:
170 	btrfs_free_path(path);
171 	return ret;
172 }
173 
174 /*
175  * Initialize a new device for device replace target from a given source dev
176  * and path.
177  *
178  * Return 0 and new device in @device_out, otherwise return < 0
179  */
180 static int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info,
181 				  const char *device_path,
182 				  struct btrfs_device *srcdev,
183 				  struct btrfs_device **device_out)
184 {
185 	struct btrfs_device *device;
186 	struct block_device *bdev;
187 	struct list_head *devices;
188 	struct rcu_string *name;
189 	u64 devid = BTRFS_DEV_REPLACE_DEVID;
190 	int ret = 0;
191 
192 	*device_out = NULL;
193 	if (fs_info->fs_devices->seeding) {
194 		btrfs_err(fs_info, "the filesystem is a seed filesystem!");
195 		return -EINVAL;
196 	}
197 
198 	bdev = blkdev_get_by_path(device_path, FMODE_WRITE | FMODE_EXCL,
199 				  fs_info->bdev_holder);
200 	if (IS_ERR(bdev)) {
201 		btrfs_err(fs_info, "target device %s is invalid!", device_path);
202 		return PTR_ERR(bdev);
203 	}
204 
205 	sync_blockdev(bdev);
206 
207 	devices = &fs_info->fs_devices->devices;
208 	list_for_each_entry(device, devices, dev_list) {
209 		if (device->bdev == bdev) {
210 			btrfs_err(fs_info,
211 				  "target device is in the filesystem!");
212 			ret = -EEXIST;
213 			goto error;
214 		}
215 	}
216 
217 
218 	if (i_size_read(bdev->bd_inode) <
219 	    btrfs_device_get_total_bytes(srcdev)) {
220 		btrfs_err(fs_info,
221 			  "target device is smaller than source device!");
222 		ret = -EINVAL;
223 		goto error;
224 	}
225 
226 
227 	device = btrfs_alloc_device(NULL, &devid, NULL);
228 	if (IS_ERR(device)) {
229 		ret = PTR_ERR(device);
230 		goto error;
231 	}
232 
233 	name = rcu_string_strdup(device_path, GFP_KERNEL);
234 	if (!name) {
235 		btrfs_free_device(device);
236 		ret = -ENOMEM;
237 		goto error;
238 	}
239 	rcu_assign_pointer(device->name, name);
240 
241 	set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
242 	device->generation = 0;
243 	device->io_width = fs_info->sectorsize;
244 	device->io_align = fs_info->sectorsize;
245 	device->sector_size = fs_info->sectorsize;
246 	device->total_bytes = btrfs_device_get_total_bytes(srcdev);
247 	device->disk_total_bytes = btrfs_device_get_disk_total_bytes(srcdev);
248 	device->bytes_used = btrfs_device_get_bytes_used(srcdev);
249 	device->commit_total_bytes = srcdev->commit_total_bytes;
250 	device->commit_bytes_used = device->bytes_used;
251 	device->fs_info = fs_info;
252 	device->bdev = bdev;
253 	set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
254 	set_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state);
255 	device->mode = FMODE_EXCL;
256 	device->dev_stats_valid = 1;
257 	set_blocksize(device->bdev, BTRFS_BDEV_BLOCKSIZE);
258 	device->fs_devices = fs_info->fs_devices;
259 
260 	mutex_lock(&fs_info->fs_devices->device_list_mutex);
261 	list_add(&device->dev_list, &fs_info->fs_devices->devices);
262 	fs_info->fs_devices->num_devices++;
263 	fs_info->fs_devices->open_devices++;
264 	mutex_unlock(&fs_info->fs_devices->device_list_mutex);
265 
266 	*device_out = device;
267 	return 0;
268 
269 error:
270 	blkdev_put(bdev, FMODE_EXCL);
271 	return ret;
272 }
273 
274 /*
275  * called from commit_transaction. Writes changed device replace state to
276  * disk.
277  */
278 int btrfs_run_dev_replace(struct btrfs_trans_handle *trans)
279 {
280 	struct btrfs_fs_info *fs_info = trans->fs_info;
281 	int ret;
282 	struct btrfs_root *dev_root = fs_info->dev_root;
283 	struct btrfs_path *path;
284 	struct btrfs_key key;
285 	struct extent_buffer *eb;
286 	struct btrfs_dev_replace_item *ptr;
287 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
288 
289 	down_read(&dev_replace->rwsem);
290 	if (!dev_replace->is_valid ||
291 	    !dev_replace->item_needs_writeback) {
292 		up_read(&dev_replace->rwsem);
293 		return 0;
294 	}
295 	up_read(&dev_replace->rwsem);
296 
297 	key.objectid = 0;
298 	key.type = BTRFS_DEV_REPLACE_KEY;
299 	key.offset = 0;
300 
301 	path = btrfs_alloc_path();
302 	if (!path) {
303 		ret = -ENOMEM;
304 		goto out;
305 	}
306 	ret = btrfs_search_slot(trans, dev_root, &key, path, -1, 1);
307 	if (ret < 0) {
308 		btrfs_warn(fs_info,
309 			   "error %d while searching for dev_replace item!",
310 			   ret);
311 		goto out;
312 	}
313 
314 	if (ret == 0 &&
315 	    btrfs_item_size_nr(path->nodes[0], path->slots[0]) < sizeof(*ptr)) {
316 		/*
317 		 * need to delete old one and insert a new one.
318 		 * Since no attempt is made to recover any old state, if the
319 		 * dev_replace state is 'running', the data on the target
320 		 * drive is lost.
321 		 * It would be possible to recover the state: just make sure
322 		 * that the beginning of the item is never changed and always
323 		 * contains all the essential information. Then read this
324 		 * minimal set of information and use it as a base for the
325 		 * new state.
326 		 */
327 		ret = btrfs_del_item(trans, dev_root, path);
328 		if (ret != 0) {
329 			btrfs_warn(fs_info,
330 				   "delete too small dev_replace item failed %d!",
331 				   ret);
332 			goto out;
333 		}
334 		ret = 1;
335 	}
336 
337 	if (ret == 1) {
338 		/* need to insert a new item */
339 		btrfs_release_path(path);
340 		ret = btrfs_insert_empty_item(trans, dev_root, path,
341 					      &key, sizeof(*ptr));
342 		if (ret < 0) {
343 			btrfs_warn(fs_info,
344 				   "insert dev_replace item failed %d!", ret);
345 			goto out;
346 		}
347 	}
348 
349 	eb = path->nodes[0];
350 	ptr = btrfs_item_ptr(eb, path->slots[0],
351 			     struct btrfs_dev_replace_item);
352 
353 	down_write(&dev_replace->rwsem);
354 	if (dev_replace->srcdev)
355 		btrfs_set_dev_replace_src_devid(eb, ptr,
356 			dev_replace->srcdev->devid);
357 	else
358 		btrfs_set_dev_replace_src_devid(eb, ptr, (u64)-1);
359 	btrfs_set_dev_replace_cont_reading_from_srcdev_mode(eb, ptr,
360 		dev_replace->cont_reading_from_srcdev_mode);
361 	btrfs_set_dev_replace_replace_state(eb, ptr,
362 		dev_replace->replace_state);
363 	btrfs_set_dev_replace_time_started(eb, ptr, dev_replace->time_started);
364 	btrfs_set_dev_replace_time_stopped(eb, ptr, dev_replace->time_stopped);
365 	btrfs_set_dev_replace_num_write_errors(eb, ptr,
366 		atomic64_read(&dev_replace->num_write_errors));
367 	btrfs_set_dev_replace_num_uncorrectable_read_errors(eb, ptr,
368 		atomic64_read(&dev_replace->num_uncorrectable_read_errors));
369 	dev_replace->cursor_left_last_write_of_item =
370 		dev_replace->cursor_left;
371 	btrfs_set_dev_replace_cursor_left(eb, ptr,
372 		dev_replace->cursor_left_last_write_of_item);
373 	btrfs_set_dev_replace_cursor_right(eb, ptr,
374 		dev_replace->cursor_right);
375 	dev_replace->item_needs_writeback = 0;
376 	up_write(&dev_replace->rwsem);
377 
378 	btrfs_mark_buffer_dirty(eb);
379 
380 out:
381 	btrfs_free_path(path);
382 
383 	return ret;
384 }
385 
386 static char* btrfs_dev_name(struct btrfs_device *device)
387 {
388 	if (!device || test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state))
389 		return "<missing disk>";
390 	else
391 		return rcu_str_deref(device->name);
392 }
393 
394 static int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info,
395 		const char *tgtdev_name, u64 srcdevid, const char *srcdev_name,
396 		int read_src)
397 {
398 	struct btrfs_root *root = fs_info->dev_root;
399 	struct btrfs_trans_handle *trans;
400 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
401 	int ret;
402 	struct btrfs_device *tgt_device = NULL;
403 	struct btrfs_device *src_device = NULL;
404 
405 	src_device = btrfs_find_device_by_devspec(fs_info, srcdevid,
406 						  srcdev_name);
407 	if (IS_ERR(src_device))
408 		return PTR_ERR(src_device);
409 
410 	if (btrfs_pinned_by_swapfile(fs_info, src_device)) {
411 		btrfs_warn_in_rcu(fs_info,
412 	  "cannot replace device %s (devid %llu) due to active swapfile",
413 			btrfs_dev_name(src_device), src_device->devid);
414 		return -ETXTBSY;
415 	}
416 
417 	/*
418 	 * Here we commit the transaction to make sure commit_total_bytes
419 	 * of all the devices are updated.
420 	 */
421 	trans = btrfs_attach_transaction(root);
422 	if (!IS_ERR(trans)) {
423 		ret = btrfs_commit_transaction(trans);
424 		if (ret)
425 			return ret;
426 	} else if (PTR_ERR(trans) != -ENOENT) {
427 		return PTR_ERR(trans);
428 	}
429 
430 	ret = btrfs_init_dev_replace_tgtdev(fs_info, tgtdev_name,
431 					    src_device, &tgt_device);
432 	if (ret)
433 		return ret;
434 
435 	down_write(&dev_replace->rwsem);
436 	switch (dev_replace->replace_state) {
437 	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
438 	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
439 	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
440 		break;
441 	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
442 	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
443 		ASSERT(0);
444 		ret = BTRFS_IOCTL_DEV_REPLACE_RESULT_ALREADY_STARTED;
445 		up_write(&dev_replace->rwsem);
446 		goto leave;
447 	}
448 
449 	dev_replace->cont_reading_from_srcdev_mode = read_src;
450 	dev_replace->srcdev = src_device;
451 	dev_replace->tgtdev = tgt_device;
452 
453 	btrfs_info_in_rcu(fs_info,
454 		      "dev_replace from %s (devid %llu) to %s started",
455 		      btrfs_dev_name(src_device),
456 		      src_device->devid,
457 		      rcu_str_deref(tgt_device->name));
458 
459 	/*
460 	 * from now on, the writes to the srcdev are all duplicated to
461 	 * go to the tgtdev as well (refer to btrfs_map_block()).
462 	 */
463 	dev_replace->replace_state = BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED;
464 	dev_replace->time_started = ktime_get_real_seconds();
465 	dev_replace->cursor_left = 0;
466 	dev_replace->committed_cursor_left = 0;
467 	dev_replace->cursor_left_last_write_of_item = 0;
468 	dev_replace->cursor_right = 0;
469 	dev_replace->is_valid = 1;
470 	dev_replace->item_needs_writeback = 1;
471 	atomic64_set(&dev_replace->num_write_errors, 0);
472 	atomic64_set(&dev_replace->num_uncorrectable_read_errors, 0);
473 	up_write(&dev_replace->rwsem);
474 
475 	ret = btrfs_sysfs_add_device_link(tgt_device->fs_devices, tgt_device);
476 	if (ret)
477 		btrfs_err(fs_info, "kobj add dev failed %d", ret);
478 
479 	btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, (u64)-1);
480 
481 	/* Commit dev_replace state and reserve 1 item for it. */
482 	trans = btrfs_start_transaction(root, 1);
483 	if (IS_ERR(trans)) {
484 		ret = PTR_ERR(trans);
485 		down_write(&dev_replace->rwsem);
486 		dev_replace->replace_state =
487 			BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED;
488 		dev_replace->srcdev = NULL;
489 		dev_replace->tgtdev = NULL;
490 		up_write(&dev_replace->rwsem);
491 		goto leave;
492 	}
493 
494 	ret = btrfs_commit_transaction(trans);
495 	WARN_ON(ret);
496 
497 	/* the disk copy procedure reuses the scrub code */
498 	ret = btrfs_scrub_dev(fs_info, src_device->devid, 0,
499 			      btrfs_device_get_total_bytes(src_device),
500 			      &dev_replace->scrub_progress, 0, 1);
501 
502 	ret = btrfs_dev_replace_finishing(fs_info, ret);
503 	if (ret == -EINPROGRESS) {
504 		ret = BTRFS_IOCTL_DEV_REPLACE_RESULT_SCRUB_INPROGRESS;
505 	} else if (ret != -ECANCELED) {
506 		WARN_ON(ret);
507 	}
508 
509 	return ret;
510 
511 leave:
512 	btrfs_destroy_dev_replace_tgtdev(tgt_device);
513 	return ret;
514 }
515 
516 int btrfs_dev_replace_by_ioctl(struct btrfs_fs_info *fs_info,
517 			    struct btrfs_ioctl_dev_replace_args *args)
518 {
519 	int ret;
520 
521 	switch (args->start.cont_reading_from_srcdev_mode) {
522 	case BTRFS_IOCTL_DEV_REPLACE_CONT_READING_FROM_SRCDEV_MODE_ALWAYS:
523 	case BTRFS_IOCTL_DEV_REPLACE_CONT_READING_FROM_SRCDEV_MODE_AVOID:
524 		break;
525 	default:
526 		return -EINVAL;
527 	}
528 
529 	if ((args->start.srcdevid == 0 && args->start.srcdev_name[0] == '\0') ||
530 	    args->start.tgtdev_name[0] == '\0')
531 		return -EINVAL;
532 
533 	ret = btrfs_dev_replace_start(fs_info, args->start.tgtdev_name,
534 					args->start.srcdevid,
535 					args->start.srcdev_name,
536 					args->start.cont_reading_from_srcdev_mode);
537 	args->result = ret;
538 	/* don't warn if EINPROGRESS, someone else might be running scrub */
539 	if (ret == BTRFS_IOCTL_DEV_REPLACE_RESULT_SCRUB_INPROGRESS ||
540 	    ret == BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR)
541 		return 0;
542 
543 	return ret;
544 }
545 
546 /*
547  * blocked until all in-flight bios operations are finished.
548  */
549 static void btrfs_rm_dev_replace_blocked(struct btrfs_fs_info *fs_info)
550 {
551 	set_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state);
552 	wait_event(fs_info->dev_replace.replace_wait, !percpu_counter_sum(
553 		   &fs_info->dev_replace.bio_counter));
554 }
555 
556 /*
557  * we have removed target device, it is safe to allow new bios request.
558  */
559 static void btrfs_rm_dev_replace_unblocked(struct btrfs_fs_info *fs_info)
560 {
561 	clear_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state);
562 	wake_up(&fs_info->dev_replace.replace_wait);
563 }
564 
565 static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
566 				       int scrub_ret)
567 {
568 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
569 	struct btrfs_device *tgt_device;
570 	struct btrfs_device *src_device;
571 	struct btrfs_root *root = fs_info->tree_root;
572 	u8 uuid_tmp[BTRFS_UUID_SIZE];
573 	struct btrfs_trans_handle *trans;
574 	int ret = 0;
575 
576 	/* don't allow cancel or unmount to disturb the finishing procedure */
577 	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);
578 
579 	down_read(&dev_replace->rwsem);
580 	/* was the operation canceled, or is it finished? */
581 	if (dev_replace->replace_state !=
582 	    BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED) {
583 		up_read(&dev_replace->rwsem);
584 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
585 		return 0;
586 	}
587 
588 	tgt_device = dev_replace->tgtdev;
589 	src_device = dev_replace->srcdev;
590 	up_read(&dev_replace->rwsem);
591 
592 	/*
593 	 * flush all outstanding I/O and inode extent mappings before the
594 	 * copy operation is declared as being finished
595 	 */
596 	ret = btrfs_start_delalloc_roots(fs_info, -1);
597 	if (ret) {
598 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
599 		return ret;
600 	}
601 	btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, (u64)-1);
602 
603 	/*
604 	 * We have to use this loop approach because at this point src_device
605 	 * has to be available for transaction commit to complete, yet new
606 	 * chunks shouldn't be allocated on the device.
607 	 */
608 	while (1) {
609 		trans = btrfs_start_transaction(root, 0);
610 		if (IS_ERR(trans)) {
611 			mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
612 			return PTR_ERR(trans);
613 		}
614 		ret = btrfs_commit_transaction(trans);
615 		WARN_ON(ret);
616 
617 		/* Prevent write_all_supers() during the finishing procedure */
618 		mutex_lock(&fs_info->fs_devices->device_list_mutex);
619 		/* Prevent new chunks being allocated on the source device */
620 		mutex_lock(&fs_info->chunk_mutex);
621 
622 		if (!list_empty(&src_device->post_commit_list)) {
623 			mutex_unlock(&fs_info->fs_devices->device_list_mutex);
624 			mutex_unlock(&fs_info->chunk_mutex);
625 		} else {
626 			break;
627 		}
628 	}
629 
630 	down_write(&dev_replace->rwsem);
631 	dev_replace->replace_state =
632 		scrub_ret ? BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED
633 			  : BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED;
634 	dev_replace->tgtdev = NULL;
635 	dev_replace->srcdev = NULL;
636 	dev_replace->time_stopped = ktime_get_real_seconds();
637 	dev_replace->item_needs_writeback = 1;
638 
639 	/* replace old device with new one in mapping tree */
640 	if (!scrub_ret) {
641 		btrfs_dev_replace_update_device_in_mapping_tree(fs_info,
642 								src_device,
643 								tgt_device);
644 	} else {
645 		if (scrub_ret != -ECANCELED)
646 			btrfs_err_in_rcu(fs_info,
647 				 "btrfs_scrub_dev(%s, %llu, %s) failed %d",
648 				 btrfs_dev_name(src_device),
649 				 src_device->devid,
650 				 rcu_str_deref(tgt_device->name), scrub_ret);
651 		up_write(&dev_replace->rwsem);
652 		mutex_unlock(&fs_info->chunk_mutex);
653 		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
654 		btrfs_rm_dev_replace_blocked(fs_info);
655 		if (tgt_device)
656 			btrfs_destroy_dev_replace_tgtdev(tgt_device);
657 		btrfs_rm_dev_replace_unblocked(fs_info);
658 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
659 
660 		return scrub_ret;
661 	}
662 
663 	btrfs_info_in_rcu(fs_info,
664 			  "dev_replace from %s (devid %llu) to %s finished",
665 			  btrfs_dev_name(src_device),
666 			  src_device->devid,
667 			  rcu_str_deref(tgt_device->name));
668 	clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &tgt_device->dev_state);
669 	tgt_device->devid = src_device->devid;
670 	src_device->devid = BTRFS_DEV_REPLACE_DEVID;
671 	memcpy(uuid_tmp, tgt_device->uuid, sizeof(uuid_tmp));
672 	memcpy(tgt_device->uuid, src_device->uuid, sizeof(tgt_device->uuid));
673 	memcpy(src_device->uuid, uuid_tmp, sizeof(src_device->uuid));
674 	btrfs_device_set_total_bytes(tgt_device, src_device->total_bytes);
675 	btrfs_device_set_disk_total_bytes(tgt_device,
676 					  src_device->disk_total_bytes);
677 	btrfs_device_set_bytes_used(tgt_device, src_device->bytes_used);
678 	tgt_device->commit_bytes_used = src_device->bytes_used;
679 
680 	btrfs_assign_next_active_device(src_device, tgt_device);
681 
682 	list_add(&tgt_device->dev_alloc_list, &fs_info->fs_devices->alloc_list);
683 	fs_info->fs_devices->rw_devices++;
684 
685 	up_write(&dev_replace->rwsem);
686 	btrfs_rm_dev_replace_blocked(fs_info);
687 
688 	btrfs_rm_dev_replace_remove_srcdev(src_device);
689 
690 	btrfs_rm_dev_replace_unblocked(fs_info);
691 
692 	/*
693 	 * Increment dev_stats_ccnt so that btrfs_run_dev_stats() will
694 	 * update on-disk dev stats value during commit transaction
695 	 */
696 	atomic_inc(&tgt_device->dev_stats_ccnt);
697 
698 	/*
699 	 * this is again a consistent state where no dev_replace procedure
700 	 * is running, the target device is part of the filesystem, the
701 	 * source device is not part of the filesystem anymore and its 1st
702 	 * superblock is scratched out so that it is no longer marked to
703 	 * belong to this filesystem.
704 	 */
705 	mutex_unlock(&fs_info->chunk_mutex);
706 	mutex_unlock(&fs_info->fs_devices->device_list_mutex);
707 
708 	/* replace the sysfs entry */
709 	btrfs_sysfs_rm_device_link(fs_info->fs_devices, src_device);
710 	btrfs_rm_dev_replace_free_srcdev(src_device);
711 
712 	/* write back the superblocks */
713 	trans = btrfs_start_transaction(root, 0);
714 	if (!IS_ERR(trans))
715 		btrfs_commit_transaction(trans);
716 
717 	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
718 
719 	return 0;
720 }
721 
722 static void btrfs_dev_replace_update_device_in_mapping_tree(
723 						struct btrfs_fs_info *fs_info,
724 						struct btrfs_device *srcdev,
725 						struct btrfs_device *tgtdev)
726 {
727 	struct extent_map_tree *em_tree = &fs_info->mapping_tree;
728 	struct extent_map *em;
729 	struct map_lookup *map;
730 	u64 start = 0;
731 	int i;
732 
733 	write_lock(&em_tree->lock);
734 	do {
735 		em = lookup_extent_mapping(em_tree, start, (u64)-1);
736 		if (!em)
737 			break;
738 		map = em->map_lookup;
739 		for (i = 0; i < map->num_stripes; i++)
740 			if (srcdev == map->stripes[i].dev)
741 				map->stripes[i].dev = tgtdev;
742 		start = em->start + em->len;
743 		free_extent_map(em);
744 	} while (start);
745 	write_unlock(&em_tree->lock);
746 }
747 
748 /*
749  * Read progress of device replace status according to the state and last
750  * stored position. The value format is the same as for
751  * btrfs_dev_replace::progress_1000
752  */
753 static u64 btrfs_dev_replace_progress(struct btrfs_fs_info *fs_info)
754 {
755 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
756 	u64 ret = 0;
757 
758 	switch (dev_replace->replace_state) {
759 	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
760 	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
761 		ret = 0;
762 		break;
763 	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
764 		ret = 1000;
765 		break;
766 	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
767 	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
768 		ret = div64_u64(dev_replace->cursor_left,
769 				div_u64(btrfs_device_get_total_bytes(
770 						dev_replace->srcdev), 1000));
771 		break;
772 	}
773 
774 	return ret;
775 }
776 
777 void btrfs_dev_replace_status(struct btrfs_fs_info *fs_info,
778 			      struct btrfs_ioctl_dev_replace_args *args)
779 {
780 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
781 
782 	down_read(&dev_replace->rwsem);
783 	/* even if !dev_replace_is_valid, the values are good enough for
784 	 * the replace_status ioctl */
785 	args->result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR;
786 	args->status.replace_state = dev_replace->replace_state;
787 	args->status.time_started = dev_replace->time_started;
788 	args->status.time_stopped = dev_replace->time_stopped;
789 	args->status.num_write_errors =
790 		atomic64_read(&dev_replace->num_write_errors);
791 	args->status.num_uncorrectable_read_errors =
792 		atomic64_read(&dev_replace->num_uncorrectable_read_errors);
793 	args->status.progress_1000 = btrfs_dev_replace_progress(fs_info);
794 	up_read(&dev_replace->rwsem);
795 }
796 
797 int btrfs_dev_replace_cancel(struct btrfs_fs_info *fs_info)
798 {
799 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
800 	struct btrfs_device *tgt_device = NULL;
801 	struct btrfs_device *src_device = NULL;
802 	struct btrfs_trans_handle *trans;
803 	struct btrfs_root *root = fs_info->tree_root;
804 	int result;
805 	int ret;
806 
807 	if (sb_rdonly(fs_info->sb))
808 		return -EROFS;
809 
810 	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);
811 	down_write(&dev_replace->rwsem);
812 	switch (dev_replace->replace_state) {
813 	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
814 	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
815 	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
816 		result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NOT_STARTED;
817 		up_write(&dev_replace->rwsem);
818 		break;
819 	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
820 		tgt_device = dev_replace->tgtdev;
821 		src_device = dev_replace->srcdev;
822 		up_write(&dev_replace->rwsem);
823 		ret = btrfs_scrub_cancel(fs_info);
824 		if (ret < 0) {
825 			result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NOT_STARTED;
826 		} else {
827 			result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR;
828 			/*
829 			 * btrfs_dev_replace_finishing() will handle the
830 			 * cleanup part
831 			 */
832 			btrfs_info_in_rcu(fs_info,
833 				"dev_replace from %s (devid %llu) to %s canceled",
834 				btrfs_dev_name(src_device), src_device->devid,
835 				btrfs_dev_name(tgt_device));
836 		}
837 		break;
838 	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
839 		/*
840 		 * Scrub doing the replace isn't running so we need to do the
841 		 * cleanup step of btrfs_dev_replace_finishing() here
842 		 */
843 		result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR;
844 		tgt_device = dev_replace->tgtdev;
845 		src_device = dev_replace->srcdev;
846 		dev_replace->tgtdev = NULL;
847 		dev_replace->srcdev = NULL;
848 		dev_replace->replace_state =
849 				BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED;
850 		dev_replace->time_stopped = ktime_get_real_seconds();
851 		dev_replace->item_needs_writeback = 1;
852 
853 		up_write(&dev_replace->rwsem);
854 
855 		/* Scrub for replace must not be running in suspended state */
856 		ret = btrfs_scrub_cancel(fs_info);
857 		ASSERT(ret != -ENOTCONN);
858 
859 		trans = btrfs_start_transaction(root, 0);
860 		if (IS_ERR(trans)) {
861 			mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
862 			return PTR_ERR(trans);
863 		}
864 		ret = btrfs_commit_transaction(trans);
865 		WARN_ON(ret);
866 
867 		btrfs_info_in_rcu(fs_info,
868 		"suspended dev_replace from %s (devid %llu) to %s canceled",
869 			btrfs_dev_name(src_device), src_device->devid,
870 			btrfs_dev_name(tgt_device));
871 
872 		if (tgt_device)
873 			btrfs_destroy_dev_replace_tgtdev(tgt_device);
874 		break;
875 	default:
876 		up_write(&dev_replace->rwsem);
877 		result = -EINVAL;
878 	}
879 
880 	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
881 	return result;
882 }
883 
884 void btrfs_dev_replace_suspend_for_unmount(struct btrfs_fs_info *fs_info)
885 {
886 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
887 
888 	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);
889 	down_write(&dev_replace->rwsem);
890 
891 	switch (dev_replace->replace_state) {
892 	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
893 	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
894 	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
895 	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
896 		break;
897 	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
898 		dev_replace->replace_state =
899 			BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED;
900 		dev_replace->time_stopped = ktime_get_real_seconds();
901 		dev_replace->item_needs_writeback = 1;
902 		btrfs_info(fs_info, "suspending dev_replace for unmount");
903 		break;
904 	}
905 
906 	up_write(&dev_replace->rwsem);
907 	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
908 }
909 
910 /* resume dev_replace procedure that was interrupted by unmount */
911 int btrfs_resume_dev_replace_async(struct btrfs_fs_info *fs_info)
912 {
913 	struct task_struct *task;
914 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
915 
916 	down_write(&dev_replace->rwsem);
917 
918 	switch (dev_replace->replace_state) {
919 	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
920 	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
921 	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
922 		up_write(&dev_replace->rwsem);
923 		return 0;
924 	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
925 		break;
926 	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
927 		dev_replace->replace_state =
928 			BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED;
929 		break;
930 	}
931 	if (!dev_replace->tgtdev || !dev_replace->tgtdev->bdev) {
932 		btrfs_info(fs_info,
933 			   "cannot continue dev_replace, tgtdev is missing");
934 		btrfs_info(fs_info,
935 			   "you may cancel the operation after 'mount -o degraded'");
936 		dev_replace->replace_state =
937 					BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED;
938 		up_write(&dev_replace->rwsem);
939 		return 0;
940 	}
941 	up_write(&dev_replace->rwsem);
942 
943 	/*
944 	 * This could collide with a paused balance, but the exclusive op logic
945 	 * should never allow both to start and pause. We don't want to allow
946 	 * dev-replace to start anyway.
947 	 */
948 	if (test_and_set_bit(BTRFS_FS_EXCL_OP, &fs_info->flags)) {
949 		down_write(&dev_replace->rwsem);
950 		dev_replace->replace_state =
951 					BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED;
952 		up_write(&dev_replace->rwsem);
953 		btrfs_info(fs_info,
954 		"cannot resume dev-replace, other exclusive operation running");
955 		return 0;
956 	}
957 
958 	task = kthread_run(btrfs_dev_replace_kthread, fs_info, "btrfs-devrepl");
959 	return PTR_ERR_OR_ZERO(task);
960 }
961 
962 static int btrfs_dev_replace_kthread(void *data)
963 {
964 	struct btrfs_fs_info *fs_info = data;
965 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
966 	u64 progress;
967 	int ret;
968 
969 	progress = btrfs_dev_replace_progress(fs_info);
970 	progress = div_u64(progress, 10);
971 	btrfs_info_in_rcu(fs_info,
972 		"continuing dev_replace from %s (devid %llu) to target %s @%u%%",
973 		btrfs_dev_name(dev_replace->srcdev),
974 		dev_replace->srcdev->devid,
975 		btrfs_dev_name(dev_replace->tgtdev),
976 		(unsigned int)progress);
977 
978 	ret = btrfs_scrub_dev(fs_info, dev_replace->srcdev->devid,
979 			      dev_replace->committed_cursor_left,
980 			      btrfs_device_get_total_bytes(dev_replace->srcdev),
981 			      &dev_replace->scrub_progress, 0, 1);
982 	ret = btrfs_dev_replace_finishing(fs_info, ret);
983 	WARN_ON(ret && ret != -ECANCELED);
984 
985 	clear_bit(BTRFS_FS_EXCL_OP, &fs_info->flags);
986 	return 0;
987 }
988 
989 int btrfs_dev_replace_is_ongoing(struct btrfs_dev_replace *dev_replace)
990 {
991 	if (!dev_replace->is_valid)
992 		return 0;
993 
994 	switch (dev_replace->replace_state) {
995 	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
996 	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
997 	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
998 		return 0;
999 	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
1000 	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
1001 		/*
1002 		 * return true even if tgtdev is missing (this is
1003 		 * something that can happen if the dev_replace
1004 		 * procedure is suspended by an umount and then
1005 		 * the tgtdev is missing (or "btrfs dev scan") was
1006 		 * not called and the filesystem is remounted
1007 		 * in degraded state. This does not stop the
1008 		 * dev_replace procedure. It needs to be canceled
1009 		 * manually if the cancellation is wanted.
1010 		 */
1011 		break;
1012 	}
1013 	return 1;
1014 }
1015 
1016 void btrfs_bio_counter_inc_noblocked(struct btrfs_fs_info *fs_info)
1017 {
1018 	percpu_counter_inc(&fs_info->dev_replace.bio_counter);
1019 }
1020 
1021 void btrfs_bio_counter_sub(struct btrfs_fs_info *fs_info, s64 amount)
1022 {
1023 	percpu_counter_sub(&fs_info->dev_replace.bio_counter, amount);
1024 	cond_wake_up_nomb(&fs_info->dev_replace.replace_wait);
1025 }
1026 
1027 void btrfs_bio_counter_inc_blocked(struct btrfs_fs_info *fs_info)
1028 {
1029 	while (1) {
1030 		percpu_counter_inc(&fs_info->dev_replace.bio_counter);
1031 		if (likely(!test_bit(BTRFS_FS_STATE_DEV_REPLACING,
1032 				     &fs_info->fs_state)))
1033 			break;
1034 
1035 		btrfs_bio_counter_dec(fs_info);
1036 		wait_event(fs_info->dev_replace.replace_wait,
1037 			   !test_bit(BTRFS_FS_STATE_DEV_REPLACING,
1038 				     &fs_info->fs_state));
1039 	}
1040 }
1041