xref: /openbmc/linux/fs/btrfs/extent_io.c (revision 738f6ba1)
1 // SPDX-License-Identifier: GPL-2.0
2 
3 #include <linux/bitops.h>
4 #include <linux/slab.h>
5 #include <linux/bio.h>
6 #include <linux/mm.h>
7 #include <linux/pagemap.h>
8 #include <linux/page-flags.h>
9 #include <linux/spinlock.h>
10 #include <linux/blkdev.h>
11 #include <linux/swap.h>
12 #include <linux/writeback.h>
13 #include <linux/pagevec.h>
14 #include <linux/prefetch.h>
15 #include <linux/cleancache.h>
16 #include "extent_io.h"
17 #include "extent-io-tree.h"
18 #include "extent_map.h"
19 #include "ctree.h"
20 #include "btrfs_inode.h"
21 #include "volumes.h"
22 #include "check-integrity.h"
23 #include "locking.h"
24 #include "rcu-string.h"
25 #include "backref.h"
26 #include "disk-io.h"
27 #include "subpage.h"
28 #include "zoned.h"
29 #include "block-group.h"
30 
31 static struct kmem_cache *extent_state_cache;
32 static struct kmem_cache *extent_buffer_cache;
33 static struct bio_set btrfs_bioset;
34 
35 static inline bool extent_state_in_tree(const struct extent_state *state)
36 {
37 	return !RB_EMPTY_NODE(&state->rb_node);
38 }
39 
40 #ifdef CONFIG_BTRFS_DEBUG
41 static LIST_HEAD(states);
42 static DEFINE_SPINLOCK(leak_lock);
43 
44 static inline void btrfs_leak_debug_add(spinlock_t *lock,
45 					struct list_head *new,
46 					struct list_head *head)
47 {
48 	unsigned long flags;
49 
50 	spin_lock_irqsave(lock, flags);
51 	list_add(new, head);
52 	spin_unlock_irqrestore(lock, flags);
53 }
54 
55 static inline void btrfs_leak_debug_del(spinlock_t *lock,
56 					struct list_head *entry)
57 {
58 	unsigned long flags;
59 
60 	spin_lock_irqsave(lock, flags);
61 	list_del(entry);
62 	spin_unlock_irqrestore(lock, flags);
63 }
64 
65 void btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info *fs_info)
66 {
67 	struct extent_buffer *eb;
68 	unsigned long flags;
69 
70 	/*
71 	 * If we didn't get into open_ctree our allocated_ebs will not be
72 	 * initialized, so just skip this.
73 	 */
74 	if (!fs_info->allocated_ebs.next)
75 		return;
76 
77 	spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
78 	while (!list_empty(&fs_info->allocated_ebs)) {
79 		eb = list_first_entry(&fs_info->allocated_ebs,
80 				      struct extent_buffer, leak_list);
81 		pr_err(
82 	"BTRFS: buffer leak start %llu len %lu refs %d bflags %lu owner %llu\n",
83 		       eb->start, eb->len, atomic_read(&eb->refs), eb->bflags,
84 		       btrfs_header_owner(eb));
85 		list_del(&eb->leak_list);
86 		kmem_cache_free(extent_buffer_cache, eb);
87 	}
88 	spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags);
89 }
90 
91 static inline void btrfs_extent_state_leak_debug_check(void)
92 {
93 	struct extent_state *state;
94 
95 	while (!list_empty(&states)) {
96 		state = list_entry(states.next, struct extent_state, leak_list);
97 		pr_err("BTRFS: state leak: start %llu end %llu state %u in tree %d refs %d\n",
98 		       state->start, state->end, state->state,
99 		       extent_state_in_tree(state),
100 		       refcount_read(&state->refs));
101 		list_del(&state->leak_list);
102 		kmem_cache_free(extent_state_cache, state);
103 	}
104 }
105 
106 #define btrfs_debug_check_extent_io_range(tree, start, end)		\
107 	__btrfs_debug_check_extent_io_range(__func__, (tree), (start), (end))
108 static inline void __btrfs_debug_check_extent_io_range(const char *caller,
109 		struct extent_io_tree *tree, u64 start, u64 end)
110 {
111 	struct inode *inode = tree->private_data;
112 	u64 isize;
113 
114 	if (!inode || !is_data_inode(inode))
115 		return;
116 
117 	isize = i_size_read(inode);
118 	if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) {
119 		btrfs_debug_rl(BTRFS_I(inode)->root->fs_info,
120 		    "%s: ino %llu isize %llu odd range [%llu,%llu]",
121 			caller, btrfs_ino(BTRFS_I(inode)), isize, start, end);
122 	}
123 }
124 #else
125 #define btrfs_leak_debug_add(lock, new, head)	do {} while (0)
126 #define btrfs_leak_debug_del(lock, entry)	do {} while (0)
127 #define btrfs_extent_state_leak_debug_check()	do {} while (0)
128 #define btrfs_debug_check_extent_io_range(c, s, e)	do {} while (0)
129 #endif
130 
131 struct tree_entry {
132 	u64 start;
133 	u64 end;
134 	struct rb_node rb_node;
135 };
136 
137 struct extent_page_data {
138 	struct bio *bio;
139 	/* tells writepage not to lock the state bits for this range
140 	 * it still does the unlocking
141 	 */
142 	unsigned int extent_locked:1;
143 
144 	/* tells the submit_bio code to use REQ_SYNC */
145 	unsigned int sync_io:1;
146 };
147 
148 static int add_extent_changeset(struct extent_state *state, u32 bits,
149 				 struct extent_changeset *changeset,
150 				 int set)
151 {
152 	int ret;
153 
154 	if (!changeset)
155 		return 0;
156 	if (set && (state->state & bits) == bits)
157 		return 0;
158 	if (!set && (state->state & bits) == 0)
159 		return 0;
160 	changeset->bytes_changed += state->end - state->start + 1;
161 	ret = ulist_add(&changeset->range_changed, state->start, state->end,
162 			GFP_ATOMIC);
163 	return ret;
164 }
165 
166 int __must_check submit_one_bio(struct bio *bio, int mirror_num,
167 				unsigned long bio_flags)
168 {
169 	blk_status_t ret = 0;
170 	struct extent_io_tree *tree = bio->bi_private;
171 
172 	bio->bi_private = NULL;
173 
174 	if (is_data_inode(tree->private_data))
175 		ret = btrfs_submit_data_bio(tree->private_data, bio, mirror_num,
176 					    bio_flags);
177 	else
178 		ret = btrfs_submit_metadata_bio(tree->private_data, bio,
179 						mirror_num, bio_flags);
180 
181 	return blk_status_to_errno(ret);
182 }
183 
184 /* Cleanup unsubmitted bios */
185 static void end_write_bio(struct extent_page_data *epd, int ret)
186 {
187 	if (epd->bio) {
188 		epd->bio->bi_status = errno_to_blk_status(ret);
189 		bio_endio(epd->bio);
190 		epd->bio = NULL;
191 	}
192 }
193 
194 /*
195  * Submit bio from extent page data via submit_one_bio
196  *
197  * Return 0 if everything is OK.
198  * Return <0 for error.
199  */
200 static int __must_check flush_write_bio(struct extent_page_data *epd)
201 {
202 	int ret = 0;
203 
204 	if (epd->bio) {
205 		ret = submit_one_bio(epd->bio, 0, 0);
206 		/*
207 		 * Clean up of epd->bio is handled by its endio function.
208 		 * And endio is either triggered by successful bio execution
209 		 * or the error handler of submit bio hook.
210 		 * So at this point, no matter what happened, we don't need
211 		 * to clean up epd->bio.
212 		 */
213 		epd->bio = NULL;
214 	}
215 	return ret;
216 }
217 
218 int __init extent_state_cache_init(void)
219 {
220 	extent_state_cache = kmem_cache_create("btrfs_extent_state",
221 			sizeof(struct extent_state), 0,
222 			SLAB_MEM_SPREAD, NULL);
223 	if (!extent_state_cache)
224 		return -ENOMEM;
225 	return 0;
226 }
227 
228 int __init extent_io_init(void)
229 {
230 	extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
231 			sizeof(struct extent_buffer), 0,
232 			SLAB_MEM_SPREAD, NULL);
233 	if (!extent_buffer_cache)
234 		return -ENOMEM;
235 
236 	if (bioset_init(&btrfs_bioset, BIO_POOL_SIZE,
237 			offsetof(struct btrfs_io_bio, bio),
238 			BIOSET_NEED_BVECS))
239 		goto free_buffer_cache;
240 
241 	if (bioset_integrity_create(&btrfs_bioset, BIO_POOL_SIZE))
242 		goto free_bioset;
243 
244 	return 0;
245 
246 free_bioset:
247 	bioset_exit(&btrfs_bioset);
248 
249 free_buffer_cache:
250 	kmem_cache_destroy(extent_buffer_cache);
251 	extent_buffer_cache = NULL;
252 	return -ENOMEM;
253 }
254 
255 void __cold extent_state_cache_exit(void)
256 {
257 	btrfs_extent_state_leak_debug_check();
258 	kmem_cache_destroy(extent_state_cache);
259 }
260 
261 void __cold extent_io_exit(void)
262 {
263 	/*
264 	 * Make sure all delayed rcu free are flushed before we
265 	 * destroy caches.
266 	 */
267 	rcu_barrier();
268 	kmem_cache_destroy(extent_buffer_cache);
269 	bioset_exit(&btrfs_bioset);
270 }
271 
272 /*
273  * For the file_extent_tree, we want to hold the inode lock when we lookup and
274  * update the disk_i_size, but lockdep will complain because our io_tree we hold
275  * the tree lock and get the inode lock when setting delalloc.  These two things
276  * are unrelated, so make a class for the file_extent_tree so we don't get the
277  * two locking patterns mixed up.
278  */
279 static struct lock_class_key file_extent_tree_class;
280 
281 void extent_io_tree_init(struct btrfs_fs_info *fs_info,
282 			 struct extent_io_tree *tree, unsigned int owner,
283 			 void *private_data)
284 {
285 	tree->fs_info = fs_info;
286 	tree->state = RB_ROOT;
287 	tree->dirty_bytes = 0;
288 	spin_lock_init(&tree->lock);
289 	tree->private_data = private_data;
290 	tree->owner = owner;
291 	if (owner == IO_TREE_INODE_FILE_EXTENT)
292 		lockdep_set_class(&tree->lock, &file_extent_tree_class);
293 }
294 
295 void extent_io_tree_release(struct extent_io_tree *tree)
296 {
297 	spin_lock(&tree->lock);
298 	/*
299 	 * Do a single barrier for the waitqueue_active check here, the state
300 	 * of the waitqueue should not change once extent_io_tree_release is
301 	 * called.
302 	 */
303 	smp_mb();
304 	while (!RB_EMPTY_ROOT(&tree->state)) {
305 		struct rb_node *node;
306 		struct extent_state *state;
307 
308 		node = rb_first(&tree->state);
309 		state = rb_entry(node, struct extent_state, rb_node);
310 		rb_erase(&state->rb_node, &tree->state);
311 		RB_CLEAR_NODE(&state->rb_node);
312 		/*
313 		 * btree io trees aren't supposed to have tasks waiting for
314 		 * changes in the flags of extent states ever.
315 		 */
316 		ASSERT(!waitqueue_active(&state->wq));
317 		free_extent_state(state);
318 
319 		cond_resched_lock(&tree->lock);
320 	}
321 	spin_unlock(&tree->lock);
322 }
323 
324 static struct extent_state *alloc_extent_state(gfp_t mask)
325 {
326 	struct extent_state *state;
327 
328 	/*
329 	 * The given mask might be not appropriate for the slab allocator,
330 	 * drop the unsupported bits
331 	 */
332 	mask &= ~(__GFP_DMA32|__GFP_HIGHMEM);
333 	state = kmem_cache_alloc(extent_state_cache, mask);
334 	if (!state)
335 		return state;
336 	state->state = 0;
337 	state->failrec = NULL;
338 	RB_CLEAR_NODE(&state->rb_node);
339 	btrfs_leak_debug_add(&leak_lock, &state->leak_list, &states);
340 	refcount_set(&state->refs, 1);
341 	init_waitqueue_head(&state->wq);
342 	trace_alloc_extent_state(state, mask, _RET_IP_);
343 	return state;
344 }
345 
346 void free_extent_state(struct extent_state *state)
347 {
348 	if (!state)
349 		return;
350 	if (refcount_dec_and_test(&state->refs)) {
351 		WARN_ON(extent_state_in_tree(state));
352 		btrfs_leak_debug_del(&leak_lock, &state->leak_list);
353 		trace_free_extent_state(state, _RET_IP_);
354 		kmem_cache_free(extent_state_cache, state);
355 	}
356 }
357 
358 static struct rb_node *tree_insert(struct rb_root *root,
359 				   struct rb_node *search_start,
360 				   u64 offset,
361 				   struct rb_node *node,
362 				   struct rb_node ***p_in,
363 				   struct rb_node **parent_in)
364 {
365 	struct rb_node **p;
366 	struct rb_node *parent = NULL;
367 	struct tree_entry *entry;
368 
369 	if (p_in && parent_in) {
370 		p = *p_in;
371 		parent = *parent_in;
372 		goto do_insert;
373 	}
374 
375 	p = search_start ? &search_start : &root->rb_node;
376 	while (*p) {
377 		parent = *p;
378 		entry = rb_entry(parent, struct tree_entry, rb_node);
379 
380 		if (offset < entry->start)
381 			p = &(*p)->rb_left;
382 		else if (offset > entry->end)
383 			p = &(*p)->rb_right;
384 		else
385 			return parent;
386 	}
387 
388 do_insert:
389 	rb_link_node(node, parent, p);
390 	rb_insert_color(node, root);
391 	return NULL;
392 }
393 
394 /**
395  * Search @tree for an entry that contains @offset. Such entry would have
396  * entry->start <= offset && entry->end >= offset.
397  *
398  * @tree:       the tree to search
399  * @offset:     offset that should fall within an entry in @tree
400  * @next_ret:   pointer to the first entry whose range ends after @offset
401  * @prev_ret:   pointer to the first entry whose range begins before @offset
402  * @p_ret:      pointer where new node should be anchored (used when inserting an
403  *	        entry in the tree)
404  * @parent_ret: points to entry which would have been the parent of the entry,
405  *               containing @offset
406  *
407  * This function returns a pointer to the entry that contains @offset byte
408  * address. If no such entry exists, then NULL is returned and the other
409  * pointer arguments to the function are filled, otherwise the found entry is
410  * returned and other pointers are left untouched.
411  */
412 static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
413 				      struct rb_node **next_ret,
414 				      struct rb_node **prev_ret,
415 				      struct rb_node ***p_ret,
416 				      struct rb_node **parent_ret)
417 {
418 	struct rb_root *root = &tree->state;
419 	struct rb_node **n = &root->rb_node;
420 	struct rb_node *prev = NULL;
421 	struct rb_node *orig_prev = NULL;
422 	struct tree_entry *entry;
423 	struct tree_entry *prev_entry = NULL;
424 
425 	while (*n) {
426 		prev = *n;
427 		entry = rb_entry(prev, struct tree_entry, rb_node);
428 		prev_entry = entry;
429 
430 		if (offset < entry->start)
431 			n = &(*n)->rb_left;
432 		else if (offset > entry->end)
433 			n = &(*n)->rb_right;
434 		else
435 			return *n;
436 	}
437 
438 	if (p_ret)
439 		*p_ret = n;
440 	if (parent_ret)
441 		*parent_ret = prev;
442 
443 	if (next_ret) {
444 		orig_prev = prev;
445 		while (prev && offset > prev_entry->end) {
446 			prev = rb_next(prev);
447 			prev_entry = rb_entry(prev, struct tree_entry, rb_node);
448 		}
449 		*next_ret = prev;
450 		prev = orig_prev;
451 	}
452 
453 	if (prev_ret) {
454 		prev_entry = rb_entry(prev, struct tree_entry, rb_node);
455 		while (prev && offset < prev_entry->start) {
456 			prev = rb_prev(prev);
457 			prev_entry = rb_entry(prev, struct tree_entry, rb_node);
458 		}
459 		*prev_ret = prev;
460 	}
461 	return NULL;
462 }
463 
464 static inline struct rb_node *
465 tree_search_for_insert(struct extent_io_tree *tree,
466 		       u64 offset,
467 		       struct rb_node ***p_ret,
468 		       struct rb_node **parent_ret)
469 {
470 	struct rb_node *next= NULL;
471 	struct rb_node *ret;
472 
473 	ret = __etree_search(tree, offset, &next, NULL, p_ret, parent_ret);
474 	if (!ret)
475 		return next;
476 	return ret;
477 }
478 
479 static inline struct rb_node *tree_search(struct extent_io_tree *tree,
480 					  u64 offset)
481 {
482 	return tree_search_for_insert(tree, offset, NULL, NULL);
483 }
484 
485 /*
486  * utility function to look for merge candidates inside a given range.
487  * Any extents with matching state are merged together into a single
488  * extent in the tree.  Extents with EXTENT_IO in their state field
489  * are not merged because the end_io handlers need to be able to do
490  * operations on them without sleeping (or doing allocations/splits).
491  *
492  * This should be called with the tree lock held.
493  */
494 static void merge_state(struct extent_io_tree *tree,
495 		        struct extent_state *state)
496 {
497 	struct extent_state *other;
498 	struct rb_node *other_node;
499 
500 	if (state->state & (EXTENT_LOCKED | EXTENT_BOUNDARY))
501 		return;
502 
503 	other_node = rb_prev(&state->rb_node);
504 	if (other_node) {
505 		other = rb_entry(other_node, struct extent_state, rb_node);
506 		if (other->end == state->start - 1 &&
507 		    other->state == state->state) {
508 			if (tree->private_data &&
509 			    is_data_inode(tree->private_data))
510 				btrfs_merge_delalloc_extent(tree->private_data,
511 							    state, other);
512 			state->start = other->start;
513 			rb_erase(&other->rb_node, &tree->state);
514 			RB_CLEAR_NODE(&other->rb_node);
515 			free_extent_state(other);
516 		}
517 	}
518 	other_node = rb_next(&state->rb_node);
519 	if (other_node) {
520 		other = rb_entry(other_node, struct extent_state, rb_node);
521 		if (other->start == state->end + 1 &&
522 		    other->state == state->state) {
523 			if (tree->private_data &&
524 			    is_data_inode(tree->private_data))
525 				btrfs_merge_delalloc_extent(tree->private_data,
526 							    state, other);
527 			state->end = other->end;
528 			rb_erase(&other->rb_node, &tree->state);
529 			RB_CLEAR_NODE(&other->rb_node);
530 			free_extent_state(other);
531 		}
532 	}
533 }
534 
535 static void set_state_bits(struct extent_io_tree *tree,
536 			   struct extent_state *state, u32 *bits,
537 			   struct extent_changeset *changeset);
538 
539 /*
540  * insert an extent_state struct into the tree.  'bits' are set on the
541  * struct before it is inserted.
542  *
543  * This may return -EEXIST if the extent is already there, in which case the
544  * state struct is freed.
545  *
546  * The tree lock is not taken internally.  This is a utility function and
547  * probably isn't what you want to call (see set/clear_extent_bit).
548  */
549 static int insert_state(struct extent_io_tree *tree,
550 			struct extent_state *state, u64 start, u64 end,
551 			struct rb_node ***p,
552 			struct rb_node **parent,
553 			u32 *bits, struct extent_changeset *changeset)
554 {
555 	struct rb_node *node;
556 
557 	if (end < start) {
558 		btrfs_err(tree->fs_info,
559 			"insert state: end < start %llu %llu", end, start);
560 		WARN_ON(1);
561 	}
562 	state->start = start;
563 	state->end = end;
564 
565 	set_state_bits(tree, state, bits, changeset);
566 
567 	node = tree_insert(&tree->state, NULL, end, &state->rb_node, p, parent);
568 	if (node) {
569 		struct extent_state *found;
570 		found = rb_entry(node, struct extent_state, rb_node);
571 		btrfs_err(tree->fs_info,
572 		       "found node %llu %llu on insert of %llu %llu",
573 		       found->start, found->end, start, end);
574 		return -EEXIST;
575 	}
576 	merge_state(tree, state);
577 	return 0;
578 }
579 
580 /*
581  * split a given extent state struct in two, inserting the preallocated
582  * struct 'prealloc' as the newly created second half.  'split' indicates an
583  * offset inside 'orig' where it should be split.
584  *
585  * Before calling,
586  * the tree has 'orig' at [orig->start, orig->end].  After calling, there
587  * are two extent state structs in the tree:
588  * prealloc: [orig->start, split - 1]
589  * orig: [ split, orig->end ]
590  *
591  * The tree locks are not taken by this function. They need to be held
592  * by the caller.
593  */
594 static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
595 		       struct extent_state *prealloc, u64 split)
596 {
597 	struct rb_node *node;
598 
599 	if (tree->private_data && is_data_inode(tree->private_data))
600 		btrfs_split_delalloc_extent(tree->private_data, orig, split);
601 
602 	prealloc->start = orig->start;
603 	prealloc->end = split - 1;
604 	prealloc->state = orig->state;
605 	orig->start = split;
606 
607 	node = tree_insert(&tree->state, &orig->rb_node, prealloc->end,
608 			   &prealloc->rb_node, NULL, NULL);
609 	if (node) {
610 		free_extent_state(prealloc);
611 		return -EEXIST;
612 	}
613 	return 0;
614 }
615 
616 static struct extent_state *next_state(struct extent_state *state)
617 {
618 	struct rb_node *next = rb_next(&state->rb_node);
619 	if (next)
620 		return rb_entry(next, struct extent_state, rb_node);
621 	else
622 		return NULL;
623 }
624 
625 /*
626  * utility function to clear some bits in an extent state struct.
627  * it will optionally wake up anyone waiting on this state (wake == 1).
628  *
629  * If no bits are set on the state struct after clearing things, the
630  * struct is freed and removed from the tree
631  */
632 static struct extent_state *clear_state_bit(struct extent_io_tree *tree,
633 					    struct extent_state *state,
634 					    u32 *bits, int wake,
635 					    struct extent_changeset *changeset)
636 {
637 	struct extent_state *next;
638 	u32 bits_to_clear = *bits & ~EXTENT_CTLBITS;
639 	int ret;
640 
641 	if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
642 		u64 range = state->end - state->start + 1;
643 		WARN_ON(range > tree->dirty_bytes);
644 		tree->dirty_bytes -= range;
645 	}
646 
647 	if (tree->private_data && is_data_inode(tree->private_data))
648 		btrfs_clear_delalloc_extent(tree->private_data, state, bits);
649 
650 	ret = add_extent_changeset(state, bits_to_clear, changeset, 0);
651 	BUG_ON(ret < 0);
652 	state->state &= ~bits_to_clear;
653 	if (wake)
654 		wake_up(&state->wq);
655 	if (state->state == 0) {
656 		next = next_state(state);
657 		if (extent_state_in_tree(state)) {
658 			rb_erase(&state->rb_node, &tree->state);
659 			RB_CLEAR_NODE(&state->rb_node);
660 			free_extent_state(state);
661 		} else {
662 			WARN_ON(1);
663 		}
664 	} else {
665 		merge_state(tree, state);
666 		next = next_state(state);
667 	}
668 	return next;
669 }
670 
671 static struct extent_state *
672 alloc_extent_state_atomic(struct extent_state *prealloc)
673 {
674 	if (!prealloc)
675 		prealloc = alloc_extent_state(GFP_ATOMIC);
676 
677 	return prealloc;
678 }
679 
680 static void extent_io_tree_panic(struct extent_io_tree *tree, int err)
681 {
682 	btrfs_panic(tree->fs_info, err,
683 	"locking error: extent tree was modified by another thread while locked");
684 }
685 
686 /*
687  * clear some bits on a range in the tree.  This may require splitting
688  * or inserting elements in the tree, so the gfp mask is used to
689  * indicate which allocations or sleeping are allowed.
690  *
691  * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
692  * the given range from the tree regardless of state (ie for truncate).
693  *
694  * the range [start, end] is inclusive.
695  *
696  * This takes the tree lock, and returns 0 on success and < 0 on error.
697  */
698 int __clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
699 		       u32 bits, int wake, int delete,
700 		       struct extent_state **cached_state,
701 		       gfp_t mask, struct extent_changeset *changeset)
702 {
703 	struct extent_state *state;
704 	struct extent_state *cached;
705 	struct extent_state *prealloc = NULL;
706 	struct rb_node *node;
707 	u64 last_end;
708 	int err;
709 	int clear = 0;
710 
711 	btrfs_debug_check_extent_io_range(tree, start, end);
712 	trace_btrfs_clear_extent_bit(tree, start, end - start + 1, bits);
713 
714 	if (bits & EXTENT_DELALLOC)
715 		bits |= EXTENT_NORESERVE;
716 
717 	if (delete)
718 		bits |= ~EXTENT_CTLBITS;
719 
720 	if (bits & (EXTENT_LOCKED | EXTENT_BOUNDARY))
721 		clear = 1;
722 again:
723 	if (!prealloc && gfpflags_allow_blocking(mask)) {
724 		/*
725 		 * Don't care for allocation failure here because we might end
726 		 * up not needing the pre-allocated extent state at all, which
727 		 * is the case if we only have in the tree extent states that
728 		 * cover our input range and don't cover too any other range.
729 		 * If we end up needing a new extent state we allocate it later.
730 		 */
731 		prealloc = alloc_extent_state(mask);
732 	}
733 
734 	spin_lock(&tree->lock);
735 	if (cached_state) {
736 		cached = *cached_state;
737 
738 		if (clear) {
739 			*cached_state = NULL;
740 			cached_state = NULL;
741 		}
742 
743 		if (cached && extent_state_in_tree(cached) &&
744 		    cached->start <= start && cached->end > start) {
745 			if (clear)
746 				refcount_dec(&cached->refs);
747 			state = cached;
748 			goto hit_next;
749 		}
750 		if (clear)
751 			free_extent_state(cached);
752 	}
753 	/*
754 	 * this search will find the extents that end after
755 	 * our range starts
756 	 */
757 	node = tree_search(tree, start);
758 	if (!node)
759 		goto out;
760 	state = rb_entry(node, struct extent_state, rb_node);
761 hit_next:
762 	if (state->start > end)
763 		goto out;
764 	WARN_ON(state->end < start);
765 	last_end = state->end;
766 
767 	/* the state doesn't have the wanted bits, go ahead */
768 	if (!(state->state & bits)) {
769 		state = next_state(state);
770 		goto next;
771 	}
772 
773 	/*
774 	 *     | ---- desired range ---- |
775 	 *  | state | or
776 	 *  | ------------- state -------------- |
777 	 *
778 	 * We need to split the extent we found, and may flip
779 	 * bits on second half.
780 	 *
781 	 * If the extent we found extends past our range, we
782 	 * just split and search again.  It'll get split again
783 	 * the next time though.
784 	 *
785 	 * If the extent we found is inside our range, we clear
786 	 * the desired bit on it.
787 	 */
788 
789 	if (state->start < start) {
790 		prealloc = alloc_extent_state_atomic(prealloc);
791 		BUG_ON(!prealloc);
792 		err = split_state(tree, state, prealloc, start);
793 		if (err)
794 			extent_io_tree_panic(tree, err);
795 
796 		prealloc = NULL;
797 		if (err)
798 			goto out;
799 		if (state->end <= end) {
800 			state = clear_state_bit(tree, state, &bits, wake,
801 						changeset);
802 			goto next;
803 		}
804 		goto search_again;
805 	}
806 	/*
807 	 * | ---- desired range ---- |
808 	 *                        | state |
809 	 * We need to split the extent, and clear the bit
810 	 * on the first half
811 	 */
812 	if (state->start <= end && state->end > end) {
813 		prealloc = alloc_extent_state_atomic(prealloc);
814 		BUG_ON(!prealloc);
815 		err = split_state(tree, state, prealloc, end + 1);
816 		if (err)
817 			extent_io_tree_panic(tree, err);
818 
819 		if (wake)
820 			wake_up(&state->wq);
821 
822 		clear_state_bit(tree, prealloc, &bits, wake, changeset);
823 
824 		prealloc = NULL;
825 		goto out;
826 	}
827 
828 	state = clear_state_bit(tree, state, &bits, wake, changeset);
829 next:
830 	if (last_end == (u64)-1)
831 		goto out;
832 	start = last_end + 1;
833 	if (start <= end && state && !need_resched())
834 		goto hit_next;
835 
836 search_again:
837 	if (start > end)
838 		goto out;
839 	spin_unlock(&tree->lock);
840 	if (gfpflags_allow_blocking(mask))
841 		cond_resched();
842 	goto again;
843 
844 out:
845 	spin_unlock(&tree->lock);
846 	if (prealloc)
847 		free_extent_state(prealloc);
848 
849 	return 0;
850 
851 }
852 
853 static void wait_on_state(struct extent_io_tree *tree,
854 			  struct extent_state *state)
855 		__releases(tree->lock)
856 		__acquires(tree->lock)
857 {
858 	DEFINE_WAIT(wait);
859 	prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
860 	spin_unlock(&tree->lock);
861 	schedule();
862 	spin_lock(&tree->lock);
863 	finish_wait(&state->wq, &wait);
864 }
865 
866 /*
867  * waits for one or more bits to clear on a range in the state tree.
868  * The range [start, end] is inclusive.
869  * The tree lock is taken by this function
870  */
871 static void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
872 			    u32 bits)
873 {
874 	struct extent_state *state;
875 	struct rb_node *node;
876 
877 	btrfs_debug_check_extent_io_range(tree, start, end);
878 
879 	spin_lock(&tree->lock);
880 again:
881 	while (1) {
882 		/*
883 		 * this search will find all the extents that end after
884 		 * our range starts
885 		 */
886 		node = tree_search(tree, start);
887 process_node:
888 		if (!node)
889 			break;
890 
891 		state = rb_entry(node, struct extent_state, rb_node);
892 
893 		if (state->start > end)
894 			goto out;
895 
896 		if (state->state & bits) {
897 			start = state->start;
898 			refcount_inc(&state->refs);
899 			wait_on_state(tree, state);
900 			free_extent_state(state);
901 			goto again;
902 		}
903 		start = state->end + 1;
904 
905 		if (start > end)
906 			break;
907 
908 		if (!cond_resched_lock(&tree->lock)) {
909 			node = rb_next(node);
910 			goto process_node;
911 		}
912 	}
913 out:
914 	spin_unlock(&tree->lock);
915 }
916 
917 static void set_state_bits(struct extent_io_tree *tree,
918 			   struct extent_state *state,
919 			   u32 *bits, struct extent_changeset *changeset)
920 {
921 	u32 bits_to_set = *bits & ~EXTENT_CTLBITS;
922 	int ret;
923 
924 	if (tree->private_data && is_data_inode(tree->private_data))
925 		btrfs_set_delalloc_extent(tree->private_data, state, bits);
926 
927 	if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
928 		u64 range = state->end - state->start + 1;
929 		tree->dirty_bytes += range;
930 	}
931 	ret = add_extent_changeset(state, bits_to_set, changeset, 1);
932 	BUG_ON(ret < 0);
933 	state->state |= bits_to_set;
934 }
935 
936 static void cache_state_if_flags(struct extent_state *state,
937 				 struct extent_state **cached_ptr,
938 				 unsigned flags)
939 {
940 	if (cached_ptr && !(*cached_ptr)) {
941 		if (!flags || (state->state & flags)) {
942 			*cached_ptr = state;
943 			refcount_inc(&state->refs);
944 		}
945 	}
946 }
947 
948 static void cache_state(struct extent_state *state,
949 			struct extent_state **cached_ptr)
950 {
951 	return cache_state_if_flags(state, cached_ptr,
952 				    EXTENT_LOCKED | EXTENT_BOUNDARY);
953 }
954 
955 /*
956  * set some bits on a range in the tree.  This may require allocations or
957  * sleeping, so the gfp mask is used to indicate what is allowed.
958  *
959  * If any of the exclusive bits are set, this will fail with -EEXIST if some
960  * part of the range already has the desired bits set.  The start of the
961  * existing range is returned in failed_start in this case.
962  *
963  * [start, end] is inclusive This takes the tree lock.
964  */
965 int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, u32 bits,
966 		   u32 exclusive_bits, u64 *failed_start,
967 		   struct extent_state **cached_state, gfp_t mask,
968 		   struct extent_changeset *changeset)
969 {
970 	struct extent_state *state;
971 	struct extent_state *prealloc = NULL;
972 	struct rb_node *node;
973 	struct rb_node **p;
974 	struct rb_node *parent;
975 	int err = 0;
976 	u64 last_start;
977 	u64 last_end;
978 
979 	btrfs_debug_check_extent_io_range(tree, start, end);
980 	trace_btrfs_set_extent_bit(tree, start, end - start + 1, bits);
981 
982 	if (exclusive_bits)
983 		ASSERT(failed_start);
984 	else
985 		ASSERT(failed_start == NULL);
986 again:
987 	if (!prealloc && gfpflags_allow_blocking(mask)) {
988 		/*
989 		 * Don't care for allocation failure here because we might end
990 		 * up not needing the pre-allocated extent state at all, which
991 		 * is the case if we only have in the tree extent states that
992 		 * cover our input range and don't cover too any other range.
993 		 * If we end up needing a new extent state we allocate it later.
994 		 */
995 		prealloc = alloc_extent_state(mask);
996 	}
997 
998 	spin_lock(&tree->lock);
999 	if (cached_state && *cached_state) {
1000 		state = *cached_state;
1001 		if (state->start <= start && state->end > start &&
1002 		    extent_state_in_tree(state)) {
1003 			node = &state->rb_node;
1004 			goto hit_next;
1005 		}
1006 	}
1007 	/*
1008 	 * this search will find all the extents that end after
1009 	 * our range starts.
1010 	 */
1011 	node = tree_search_for_insert(tree, start, &p, &parent);
1012 	if (!node) {
1013 		prealloc = alloc_extent_state_atomic(prealloc);
1014 		BUG_ON(!prealloc);
1015 		err = insert_state(tree, prealloc, start, end,
1016 				   &p, &parent, &bits, changeset);
1017 		if (err)
1018 			extent_io_tree_panic(tree, err);
1019 
1020 		cache_state(prealloc, cached_state);
1021 		prealloc = NULL;
1022 		goto out;
1023 	}
1024 	state = rb_entry(node, struct extent_state, rb_node);
1025 hit_next:
1026 	last_start = state->start;
1027 	last_end = state->end;
1028 
1029 	/*
1030 	 * | ---- desired range ---- |
1031 	 * | state |
1032 	 *
1033 	 * Just lock what we found and keep going
1034 	 */
1035 	if (state->start == start && state->end <= end) {
1036 		if (state->state & exclusive_bits) {
1037 			*failed_start = state->start;
1038 			err = -EEXIST;
1039 			goto out;
1040 		}
1041 
1042 		set_state_bits(tree, state, &bits, changeset);
1043 		cache_state(state, cached_state);
1044 		merge_state(tree, state);
1045 		if (last_end == (u64)-1)
1046 			goto out;
1047 		start = last_end + 1;
1048 		state = next_state(state);
1049 		if (start < end && state && state->start == start &&
1050 		    !need_resched())
1051 			goto hit_next;
1052 		goto search_again;
1053 	}
1054 
1055 	/*
1056 	 *     | ---- desired range ---- |
1057 	 * | state |
1058 	 *   or
1059 	 * | ------------- state -------------- |
1060 	 *
1061 	 * We need to split the extent we found, and may flip bits on
1062 	 * second half.
1063 	 *
1064 	 * If the extent we found extends past our
1065 	 * range, we just split and search again.  It'll get split
1066 	 * again the next time though.
1067 	 *
1068 	 * If the extent we found is inside our range, we set the
1069 	 * desired bit on it.
1070 	 */
1071 	if (state->start < start) {
1072 		if (state->state & exclusive_bits) {
1073 			*failed_start = start;
1074 			err = -EEXIST;
1075 			goto out;
1076 		}
1077 
1078 		/*
1079 		 * If this extent already has all the bits we want set, then
1080 		 * skip it, not necessary to split it or do anything with it.
1081 		 */
1082 		if ((state->state & bits) == bits) {
1083 			start = state->end + 1;
1084 			cache_state(state, cached_state);
1085 			goto search_again;
1086 		}
1087 
1088 		prealloc = alloc_extent_state_atomic(prealloc);
1089 		BUG_ON(!prealloc);
1090 		err = split_state(tree, state, prealloc, start);
1091 		if (err)
1092 			extent_io_tree_panic(tree, err);
1093 
1094 		prealloc = NULL;
1095 		if (err)
1096 			goto out;
1097 		if (state->end <= end) {
1098 			set_state_bits(tree, state, &bits, changeset);
1099 			cache_state(state, cached_state);
1100 			merge_state(tree, state);
1101 			if (last_end == (u64)-1)
1102 				goto out;
1103 			start = last_end + 1;
1104 			state = next_state(state);
1105 			if (start < end && state && state->start == start &&
1106 			    !need_resched())
1107 				goto hit_next;
1108 		}
1109 		goto search_again;
1110 	}
1111 	/*
1112 	 * | ---- desired range ---- |
1113 	 *     | state | or               | state |
1114 	 *
1115 	 * There's a hole, we need to insert something in it and
1116 	 * ignore the extent we found.
1117 	 */
1118 	if (state->start > start) {
1119 		u64 this_end;
1120 		if (end < last_start)
1121 			this_end = end;
1122 		else
1123 			this_end = last_start - 1;
1124 
1125 		prealloc = alloc_extent_state_atomic(prealloc);
1126 		BUG_ON(!prealloc);
1127 
1128 		/*
1129 		 * Avoid to free 'prealloc' if it can be merged with
1130 		 * the later extent.
1131 		 */
1132 		err = insert_state(tree, prealloc, start, this_end,
1133 				   NULL, NULL, &bits, changeset);
1134 		if (err)
1135 			extent_io_tree_panic(tree, err);
1136 
1137 		cache_state(prealloc, cached_state);
1138 		prealloc = NULL;
1139 		start = this_end + 1;
1140 		goto search_again;
1141 	}
1142 	/*
1143 	 * | ---- desired range ---- |
1144 	 *                        | state |
1145 	 * We need to split the extent, and set the bit
1146 	 * on the first half
1147 	 */
1148 	if (state->start <= end && state->end > end) {
1149 		if (state->state & exclusive_bits) {
1150 			*failed_start = start;
1151 			err = -EEXIST;
1152 			goto out;
1153 		}
1154 
1155 		prealloc = alloc_extent_state_atomic(prealloc);
1156 		BUG_ON(!prealloc);
1157 		err = split_state(tree, state, prealloc, end + 1);
1158 		if (err)
1159 			extent_io_tree_panic(tree, err);
1160 
1161 		set_state_bits(tree, prealloc, &bits, changeset);
1162 		cache_state(prealloc, cached_state);
1163 		merge_state(tree, prealloc);
1164 		prealloc = NULL;
1165 		goto out;
1166 	}
1167 
1168 search_again:
1169 	if (start > end)
1170 		goto out;
1171 	spin_unlock(&tree->lock);
1172 	if (gfpflags_allow_blocking(mask))
1173 		cond_resched();
1174 	goto again;
1175 
1176 out:
1177 	spin_unlock(&tree->lock);
1178 	if (prealloc)
1179 		free_extent_state(prealloc);
1180 
1181 	return err;
1182 
1183 }
1184 
1185 /**
1186  * convert_extent_bit - convert all bits in a given range from one bit to
1187  * 			another
1188  * @tree:	the io tree to search
1189  * @start:	the start offset in bytes
1190  * @end:	the end offset in bytes (inclusive)
1191  * @bits:	the bits to set in this range
1192  * @clear_bits:	the bits to clear in this range
1193  * @cached_state:	state that we're going to cache
1194  *
1195  * This will go through and set bits for the given range.  If any states exist
1196  * already in this range they are set with the given bit and cleared of the
1197  * clear_bits.  This is only meant to be used by things that are mergeable, ie
1198  * converting from say DELALLOC to DIRTY.  This is not meant to be used with
1199  * boundary bits like LOCK.
1200  *
1201  * All allocations are done with GFP_NOFS.
1202  */
1203 int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1204 		       u32 bits, u32 clear_bits,
1205 		       struct extent_state **cached_state)
1206 {
1207 	struct extent_state *state;
1208 	struct extent_state *prealloc = NULL;
1209 	struct rb_node *node;
1210 	struct rb_node **p;
1211 	struct rb_node *parent;
1212 	int err = 0;
1213 	u64 last_start;
1214 	u64 last_end;
1215 	bool first_iteration = true;
1216 
1217 	btrfs_debug_check_extent_io_range(tree, start, end);
1218 	trace_btrfs_convert_extent_bit(tree, start, end - start + 1, bits,
1219 				       clear_bits);
1220 
1221 again:
1222 	if (!prealloc) {
1223 		/*
1224 		 * Best effort, don't worry if extent state allocation fails
1225 		 * here for the first iteration. We might have a cached state
1226 		 * that matches exactly the target range, in which case no
1227 		 * extent state allocations are needed. We'll only know this
1228 		 * after locking the tree.
1229 		 */
1230 		prealloc = alloc_extent_state(GFP_NOFS);
1231 		if (!prealloc && !first_iteration)
1232 			return -ENOMEM;
1233 	}
1234 
1235 	spin_lock(&tree->lock);
1236 	if (cached_state && *cached_state) {
1237 		state = *cached_state;
1238 		if (state->start <= start && state->end > start &&
1239 		    extent_state_in_tree(state)) {
1240 			node = &state->rb_node;
1241 			goto hit_next;
1242 		}
1243 	}
1244 
1245 	/*
1246 	 * this search will find all the extents that end after
1247 	 * our range starts.
1248 	 */
1249 	node = tree_search_for_insert(tree, start, &p, &parent);
1250 	if (!node) {
1251 		prealloc = alloc_extent_state_atomic(prealloc);
1252 		if (!prealloc) {
1253 			err = -ENOMEM;
1254 			goto out;
1255 		}
1256 		err = insert_state(tree, prealloc, start, end,
1257 				   &p, &parent, &bits, NULL);
1258 		if (err)
1259 			extent_io_tree_panic(tree, err);
1260 		cache_state(prealloc, cached_state);
1261 		prealloc = NULL;
1262 		goto out;
1263 	}
1264 	state = rb_entry(node, struct extent_state, rb_node);
1265 hit_next:
1266 	last_start = state->start;
1267 	last_end = state->end;
1268 
1269 	/*
1270 	 * | ---- desired range ---- |
1271 	 * | state |
1272 	 *
1273 	 * Just lock what we found and keep going
1274 	 */
1275 	if (state->start == start && state->end <= end) {
1276 		set_state_bits(tree, state, &bits, NULL);
1277 		cache_state(state, cached_state);
1278 		state = clear_state_bit(tree, state, &clear_bits, 0, NULL);
1279 		if (last_end == (u64)-1)
1280 			goto out;
1281 		start = last_end + 1;
1282 		if (start < end && state && state->start == start &&
1283 		    !need_resched())
1284 			goto hit_next;
1285 		goto search_again;
1286 	}
1287 
1288 	/*
1289 	 *     | ---- desired range ---- |
1290 	 * | state |
1291 	 *   or
1292 	 * | ------------- state -------------- |
1293 	 *
1294 	 * We need to split the extent we found, and may flip bits on
1295 	 * second half.
1296 	 *
1297 	 * If the extent we found extends past our
1298 	 * range, we just split and search again.  It'll get split
1299 	 * again the next time though.
1300 	 *
1301 	 * If the extent we found is inside our range, we set the
1302 	 * desired bit on it.
1303 	 */
1304 	if (state->start < start) {
1305 		prealloc = alloc_extent_state_atomic(prealloc);
1306 		if (!prealloc) {
1307 			err = -ENOMEM;
1308 			goto out;
1309 		}
1310 		err = split_state(tree, state, prealloc, start);
1311 		if (err)
1312 			extent_io_tree_panic(tree, err);
1313 		prealloc = NULL;
1314 		if (err)
1315 			goto out;
1316 		if (state->end <= end) {
1317 			set_state_bits(tree, state, &bits, NULL);
1318 			cache_state(state, cached_state);
1319 			state = clear_state_bit(tree, state, &clear_bits, 0,
1320 						NULL);
1321 			if (last_end == (u64)-1)
1322 				goto out;
1323 			start = last_end + 1;
1324 			if (start < end && state && state->start == start &&
1325 			    !need_resched())
1326 				goto hit_next;
1327 		}
1328 		goto search_again;
1329 	}
1330 	/*
1331 	 * | ---- desired range ---- |
1332 	 *     | state | or               | state |
1333 	 *
1334 	 * There's a hole, we need to insert something in it and
1335 	 * ignore the extent we found.
1336 	 */
1337 	if (state->start > start) {
1338 		u64 this_end;
1339 		if (end < last_start)
1340 			this_end = end;
1341 		else
1342 			this_end = last_start - 1;
1343 
1344 		prealloc = alloc_extent_state_atomic(prealloc);
1345 		if (!prealloc) {
1346 			err = -ENOMEM;
1347 			goto out;
1348 		}
1349 
1350 		/*
1351 		 * Avoid to free 'prealloc' if it can be merged with
1352 		 * the later extent.
1353 		 */
1354 		err = insert_state(tree, prealloc, start, this_end,
1355 				   NULL, NULL, &bits, NULL);
1356 		if (err)
1357 			extent_io_tree_panic(tree, err);
1358 		cache_state(prealloc, cached_state);
1359 		prealloc = NULL;
1360 		start = this_end + 1;
1361 		goto search_again;
1362 	}
1363 	/*
1364 	 * | ---- desired range ---- |
1365 	 *                        | state |
1366 	 * We need to split the extent, and set the bit
1367 	 * on the first half
1368 	 */
1369 	if (state->start <= end && state->end > end) {
1370 		prealloc = alloc_extent_state_atomic(prealloc);
1371 		if (!prealloc) {
1372 			err = -ENOMEM;
1373 			goto out;
1374 		}
1375 
1376 		err = split_state(tree, state, prealloc, end + 1);
1377 		if (err)
1378 			extent_io_tree_panic(tree, err);
1379 
1380 		set_state_bits(tree, prealloc, &bits, NULL);
1381 		cache_state(prealloc, cached_state);
1382 		clear_state_bit(tree, prealloc, &clear_bits, 0, NULL);
1383 		prealloc = NULL;
1384 		goto out;
1385 	}
1386 
1387 search_again:
1388 	if (start > end)
1389 		goto out;
1390 	spin_unlock(&tree->lock);
1391 	cond_resched();
1392 	first_iteration = false;
1393 	goto again;
1394 
1395 out:
1396 	spin_unlock(&tree->lock);
1397 	if (prealloc)
1398 		free_extent_state(prealloc);
1399 
1400 	return err;
1401 }
1402 
1403 /* wrappers around set/clear extent bit */
1404 int set_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1405 			   u32 bits, struct extent_changeset *changeset)
1406 {
1407 	/*
1408 	 * We don't support EXTENT_LOCKED yet, as current changeset will
1409 	 * record any bits changed, so for EXTENT_LOCKED case, it will
1410 	 * either fail with -EEXIST or changeset will record the whole
1411 	 * range.
1412 	 */
1413 	BUG_ON(bits & EXTENT_LOCKED);
1414 
1415 	return set_extent_bit(tree, start, end, bits, 0, NULL, NULL, GFP_NOFS,
1416 			      changeset);
1417 }
1418 
1419 int set_extent_bits_nowait(struct extent_io_tree *tree, u64 start, u64 end,
1420 			   u32 bits)
1421 {
1422 	return set_extent_bit(tree, start, end, bits, 0, NULL, NULL,
1423 			      GFP_NOWAIT, NULL);
1424 }
1425 
1426 int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1427 		     u32 bits, int wake, int delete,
1428 		     struct extent_state **cached)
1429 {
1430 	return __clear_extent_bit(tree, start, end, bits, wake, delete,
1431 				  cached, GFP_NOFS, NULL);
1432 }
1433 
1434 int clear_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1435 		u32 bits, struct extent_changeset *changeset)
1436 {
1437 	/*
1438 	 * Don't support EXTENT_LOCKED case, same reason as
1439 	 * set_record_extent_bits().
1440 	 */
1441 	BUG_ON(bits & EXTENT_LOCKED);
1442 
1443 	return __clear_extent_bit(tree, start, end, bits, 0, 0, NULL, GFP_NOFS,
1444 				  changeset);
1445 }
1446 
1447 /*
1448  * either insert or lock state struct between start and end use mask to tell
1449  * us if waiting is desired.
1450  */
1451 int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1452 		     struct extent_state **cached_state)
1453 {
1454 	int err;
1455 	u64 failed_start;
1456 
1457 	while (1) {
1458 		err = set_extent_bit(tree, start, end, EXTENT_LOCKED,
1459 				     EXTENT_LOCKED, &failed_start,
1460 				     cached_state, GFP_NOFS, NULL);
1461 		if (err == -EEXIST) {
1462 			wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
1463 			start = failed_start;
1464 		} else
1465 			break;
1466 		WARN_ON(start > end);
1467 	}
1468 	return err;
1469 }
1470 
1471 int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
1472 {
1473 	int err;
1474 	u64 failed_start;
1475 
1476 	err = set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
1477 			     &failed_start, NULL, GFP_NOFS, NULL);
1478 	if (err == -EEXIST) {
1479 		if (failed_start > start)
1480 			clear_extent_bit(tree, start, failed_start - 1,
1481 					 EXTENT_LOCKED, 1, 0, NULL);
1482 		return 0;
1483 	}
1484 	return 1;
1485 }
1486 
1487 void extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
1488 {
1489 	unsigned long index = start >> PAGE_SHIFT;
1490 	unsigned long end_index = end >> PAGE_SHIFT;
1491 	struct page *page;
1492 
1493 	while (index <= end_index) {
1494 		page = find_get_page(inode->i_mapping, index);
1495 		BUG_ON(!page); /* Pages should be in the extent_io_tree */
1496 		clear_page_dirty_for_io(page);
1497 		put_page(page);
1498 		index++;
1499 	}
1500 }
1501 
1502 void extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end)
1503 {
1504 	unsigned long index = start >> PAGE_SHIFT;
1505 	unsigned long end_index = end >> PAGE_SHIFT;
1506 	struct page *page;
1507 
1508 	while (index <= end_index) {
1509 		page = find_get_page(inode->i_mapping, index);
1510 		BUG_ON(!page); /* Pages should be in the extent_io_tree */
1511 		__set_page_dirty_nobuffers(page);
1512 		account_page_redirty(page);
1513 		put_page(page);
1514 		index++;
1515 	}
1516 }
1517 
1518 /* find the first state struct with 'bits' set after 'start', and
1519  * return it.  tree->lock must be held.  NULL will returned if
1520  * nothing was found after 'start'
1521  */
1522 static struct extent_state *
1523 find_first_extent_bit_state(struct extent_io_tree *tree, u64 start, u32 bits)
1524 {
1525 	struct rb_node *node;
1526 	struct extent_state *state;
1527 
1528 	/*
1529 	 * this search will find all the extents that end after
1530 	 * our range starts.
1531 	 */
1532 	node = tree_search(tree, start);
1533 	if (!node)
1534 		goto out;
1535 
1536 	while (1) {
1537 		state = rb_entry(node, struct extent_state, rb_node);
1538 		if (state->end >= start && (state->state & bits))
1539 			return state;
1540 
1541 		node = rb_next(node);
1542 		if (!node)
1543 			break;
1544 	}
1545 out:
1546 	return NULL;
1547 }
1548 
1549 /*
1550  * Find the first offset in the io tree with one or more @bits set.
1551  *
1552  * Note: If there are multiple bits set in @bits, any of them will match.
1553  *
1554  * Return 0 if we find something, and update @start_ret and @end_ret.
1555  * Return 1 if we found nothing.
1556  */
1557 int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
1558 			  u64 *start_ret, u64 *end_ret, u32 bits,
1559 			  struct extent_state **cached_state)
1560 {
1561 	struct extent_state *state;
1562 	int ret = 1;
1563 
1564 	spin_lock(&tree->lock);
1565 	if (cached_state && *cached_state) {
1566 		state = *cached_state;
1567 		if (state->end == start - 1 && extent_state_in_tree(state)) {
1568 			while ((state = next_state(state)) != NULL) {
1569 				if (state->state & bits)
1570 					goto got_it;
1571 			}
1572 			free_extent_state(*cached_state);
1573 			*cached_state = NULL;
1574 			goto out;
1575 		}
1576 		free_extent_state(*cached_state);
1577 		*cached_state = NULL;
1578 	}
1579 
1580 	state = find_first_extent_bit_state(tree, start, bits);
1581 got_it:
1582 	if (state) {
1583 		cache_state_if_flags(state, cached_state, 0);
1584 		*start_ret = state->start;
1585 		*end_ret = state->end;
1586 		ret = 0;
1587 	}
1588 out:
1589 	spin_unlock(&tree->lock);
1590 	return ret;
1591 }
1592 
1593 /**
1594  * Find a contiguous area of bits
1595  *
1596  * @tree:      io tree to check
1597  * @start:     offset to start the search from
1598  * @start_ret: the first offset we found with the bits set
1599  * @end_ret:   the final contiguous range of the bits that were set
1600  * @bits:      bits to look for
1601  *
1602  * set_extent_bit and clear_extent_bit can temporarily split contiguous ranges
1603  * to set bits appropriately, and then merge them again.  During this time it
1604  * will drop the tree->lock, so use this helper if you want to find the actual
1605  * contiguous area for given bits.  We will search to the first bit we find, and
1606  * then walk down the tree until we find a non-contiguous area.  The area
1607  * returned will be the full contiguous area with the bits set.
1608  */
1609 int find_contiguous_extent_bit(struct extent_io_tree *tree, u64 start,
1610 			       u64 *start_ret, u64 *end_ret, u32 bits)
1611 {
1612 	struct extent_state *state;
1613 	int ret = 1;
1614 
1615 	spin_lock(&tree->lock);
1616 	state = find_first_extent_bit_state(tree, start, bits);
1617 	if (state) {
1618 		*start_ret = state->start;
1619 		*end_ret = state->end;
1620 		while ((state = next_state(state)) != NULL) {
1621 			if (state->start > (*end_ret + 1))
1622 				break;
1623 			*end_ret = state->end;
1624 		}
1625 		ret = 0;
1626 	}
1627 	spin_unlock(&tree->lock);
1628 	return ret;
1629 }
1630 
1631 /**
1632  * Find the first range that has @bits not set. This range could start before
1633  * @start.
1634  *
1635  * @tree:      the tree to search
1636  * @start:     offset at/after which the found extent should start
1637  * @start_ret: records the beginning of the range
1638  * @end_ret:   records the end of the range (inclusive)
1639  * @bits:      the set of bits which must be unset
1640  *
1641  * Since unallocated range is also considered one which doesn't have the bits
1642  * set it's possible that @end_ret contains -1, this happens in case the range
1643  * spans (last_range_end, end of device]. In this case it's up to the caller to
1644  * trim @end_ret to the appropriate size.
1645  */
1646 void find_first_clear_extent_bit(struct extent_io_tree *tree, u64 start,
1647 				 u64 *start_ret, u64 *end_ret, u32 bits)
1648 {
1649 	struct extent_state *state;
1650 	struct rb_node *node, *prev = NULL, *next;
1651 
1652 	spin_lock(&tree->lock);
1653 
1654 	/* Find first extent with bits cleared */
1655 	while (1) {
1656 		node = __etree_search(tree, start, &next, &prev, NULL, NULL);
1657 		if (!node && !next && !prev) {
1658 			/*
1659 			 * Tree is completely empty, send full range and let
1660 			 * caller deal with it
1661 			 */
1662 			*start_ret = 0;
1663 			*end_ret = -1;
1664 			goto out;
1665 		} else if (!node && !next) {
1666 			/*
1667 			 * We are past the last allocated chunk, set start at
1668 			 * the end of the last extent.
1669 			 */
1670 			state = rb_entry(prev, struct extent_state, rb_node);
1671 			*start_ret = state->end + 1;
1672 			*end_ret = -1;
1673 			goto out;
1674 		} else if (!node) {
1675 			node = next;
1676 		}
1677 		/*
1678 		 * At this point 'node' either contains 'start' or start is
1679 		 * before 'node'
1680 		 */
1681 		state = rb_entry(node, struct extent_state, rb_node);
1682 
1683 		if (in_range(start, state->start, state->end - state->start + 1)) {
1684 			if (state->state & bits) {
1685 				/*
1686 				 * |--range with bits sets--|
1687 				 *    |
1688 				 *    start
1689 				 */
1690 				start = state->end + 1;
1691 			} else {
1692 				/*
1693 				 * 'start' falls within a range that doesn't
1694 				 * have the bits set, so take its start as
1695 				 * the beginning of the desired range
1696 				 *
1697 				 * |--range with bits cleared----|
1698 				 *      |
1699 				 *      start
1700 				 */
1701 				*start_ret = state->start;
1702 				break;
1703 			}
1704 		} else {
1705 			/*
1706 			 * |---prev range---|---hole/unset---|---node range---|
1707 			 *                          |
1708 			 *                        start
1709 			 *
1710 			 *                        or
1711 			 *
1712 			 * |---hole/unset--||--first node--|
1713 			 * 0   |
1714 			 *    start
1715 			 */
1716 			if (prev) {
1717 				state = rb_entry(prev, struct extent_state,
1718 						 rb_node);
1719 				*start_ret = state->end + 1;
1720 			} else {
1721 				*start_ret = 0;
1722 			}
1723 			break;
1724 		}
1725 	}
1726 
1727 	/*
1728 	 * Find the longest stretch from start until an entry which has the
1729 	 * bits set
1730 	 */
1731 	while (1) {
1732 		state = rb_entry(node, struct extent_state, rb_node);
1733 		if (state->end >= start && !(state->state & bits)) {
1734 			*end_ret = state->end;
1735 		} else {
1736 			*end_ret = state->start - 1;
1737 			break;
1738 		}
1739 
1740 		node = rb_next(node);
1741 		if (!node)
1742 			break;
1743 	}
1744 out:
1745 	spin_unlock(&tree->lock);
1746 }
1747 
1748 /*
1749  * find a contiguous range of bytes in the file marked as delalloc, not
1750  * more than 'max_bytes'.  start and end are used to return the range,
1751  *
1752  * true is returned if we find something, false if nothing was in the tree
1753  */
1754 bool btrfs_find_delalloc_range(struct extent_io_tree *tree, u64 *start,
1755 			       u64 *end, u64 max_bytes,
1756 			       struct extent_state **cached_state)
1757 {
1758 	struct rb_node *node;
1759 	struct extent_state *state;
1760 	u64 cur_start = *start;
1761 	bool found = false;
1762 	u64 total_bytes = 0;
1763 
1764 	spin_lock(&tree->lock);
1765 
1766 	/*
1767 	 * this search will find all the extents that end after
1768 	 * our range starts.
1769 	 */
1770 	node = tree_search(tree, cur_start);
1771 	if (!node) {
1772 		*end = (u64)-1;
1773 		goto out;
1774 	}
1775 
1776 	while (1) {
1777 		state = rb_entry(node, struct extent_state, rb_node);
1778 		if (found && (state->start != cur_start ||
1779 			      (state->state & EXTENT_BOUNDARY))) {
1780 			goto out;
1781 		}
1782 		if (!(state->state & EXTENT_DELALLOC)) {
1783 			if (!found)
1784 				*end = state->end;
1785 			goto out;
1786 		}
1787 		if (!found) {
1788 			*start = state->start;
1789 			*cached_state = state;
1790 			refcount_inc(&state->refs);
1791 		}
1792 		found = true;
1793 		*end = state->end;
1794 		cur_start = state->end + 1;
1795 		node = rb_next(node);
1796 		total_bytes += state->end - state->start + 1;
1797 		if (total_bytes >= max_bytes)
1798 			break;
1799 		if (!node)
1800 			break;
1801 	}
1802 out:
1803 	spin_unlock(&tree->lock);
1804 	return found;
1805 }
1806 
1807 static int __process_pages_contig(struct address_space *mapping,
1808 				  struct page *locked_page,
1809 				  pgoff_t start_index, pgoff_t end_index,
1810 				  unsigned long page_ops, pgoff_t *index_ret);
1811 
1812 static noinline void __unlock_for_delalloc(struct inode *inode,
1813 					   struct page *locked_page,
1814 					   u64 start, u64 end)
1815 {
1816 	unsigned long index = start >> PAGE_SHIFT;
1817 	unsigned long end_index = end >> PAGE_SHIFT;
1818 
1819 	ASSERT(locked_page);
1820 	if (index == locked_page->index && end_index == index)
1821 		return;
1822 
1823 	__process_pages_contig(inode->i_mapping, locked_page, index, end_index,
1824 			       PAGE_UNLOCK, NULL);
1825 }
1826 
1827 static noinline int lock_delalloc_pages(struct inode *inode,
1828 					struct page *locked_page,
1829 					u64 delalloc_start,
1830 					u64 delalloc_end)
1831 {
1832 	unsigned long index = delalloc_start >> PAGE_SHIFT;
1833 	unsigned long index_ret = index;
1834 	unsigned long end_index = delalloc_end >> PAGE_SHIFT;
1835 	int ret;
1836 
1837 	ASSERT(locked_page);
1838 	if (index == locked_page->index && index == end_index)
1839 		return 0;
1840 
1841 	ret = __process_pages_contig(inode->i_mapping, locked_page, index,
1842 				     end_index, PAGE_LOCK, &index_ret);
1843 	if (ret == -EAGAIN)
1844 		__unlock_for_delalloc(inode, locked_page, delalloc_start,
1845 				      (u64)index_ret << PAGE_SHIFT);
1846 	return ret;
1847 }
1848 
1849 /*
1850  * Find and lock a contiguous range of bytes in the file marked as delalloc, no
1851  * more than @max_bytes.  @Start and @end are used to return the range,
1852  *
1853  * Return: true if we find something
1854  *         false if nothing was in the tree
1855  */
1856 EXPORT_FOR_TESTS
1857 noinline_for_stack bool find_lock_delalloc_range(struct inode *inode,
1858 				    struct page *locked_page, u64 *start,
1859 				    u64 *end)
1860 {
1861 	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
1862 	u64 max_bytes = BTRFS_MAX_EXTENT_SIZE;
1863 	u64 delalloc_start;
1864 	u64 delalloc_end;
1865 	bool found;
1866 	struct extent_state *cached_state = NULL;
1867 	int ret;
1868 	int loops = 0;
1869 
1870 again:
1871 	/* step one, find a bunch of delalloc bytes starting at start */
1872 	delalloc_start = *start;
1873 	delalloc_end = 0;
1874 	found = btrfs_find_delalloc_range(tree, &delalloc_start, &delalloc_end,
1875 					  max_bytes, &cached_state);
1876 	if (!found || delalloc_end <= *start) {
1877 		*start = delalloc_start;
1878 		*end = delalloc_end;
1879 		free_extent_state(cached_state);
1880 		return false;
1881 	}
1882 
1883 	/*
1884 	 * start comes from the offset of locked_page.  We have to lock
1885 	 * pages in order, so we can't process delalloc bytes before
1886 	 * locked_page
1887 	 */
1888 	if (delalloc_start < *start)
1889 		delalloc_start = *start;
1890 
1891 	/*
1892 	 * make sure to limit the number of pages we try to lock down
1893 	 */
1894 	if (delalloc_end + 1 - delalloc_start > max_bytes)
1895 		delalloc_end = delalloc_start + max_bytes - 1;
1896 
1897 	/* step two, lock all the pages after the page that has start */
1898 	ret = lock_delalloc_pages(inode, locked_page,
1899 				  delalloc_start, delalloc_end);
1900 	ASSERT(!ret || ret == -EAGAIN);
1901 	if (ret == -EAGAIN) {
1902 		/* some of the pages are gone, lets avoid looping by
1903 		 * shortening the size of the delalloc range we're searching
1904 		 */
1905 		free_extent_state(cached_state);
1906 		cached_state = NULL;
1907 		if (!loops) {
1908 			max_bytes = PAGE_SIZE;
1909 			loops = 1;
1910 			goto again;
1911 		} else {
1912 			found = false;
1913 			goto out_failed;
1914 		}
1915 	}
1916 
1917 	/* step three, lock the state bits for the whole range */
1918 	lock_extent_bits(tree, delalloc_start, delalloc_end, &cached_state);
1919 
1920 	/* then test to make sure it is all still delalloc */
1921 	ret = test_range_bit(tree, delalloc_start, delalloc_end,
1922 			     EXTENT_DELALLOC, 1, cached_state);
1923 	if (!ret) {
1924 		unlock_extent_cached(tree, delalloc_start, delalloc_end,
1925 				     &cached_state);
1926 		__unlock_for_delalloc(inode, locked_page,
1927 			      delalloc_start, delalloc_end);
1928 		cond_resched();
1929 		goto again;
1930 	}
1931 	free_extent_state(cached_state);
1932 	*start = delalloc_start;
1933 	*end = delalloc_end;
1934 out_failed:
1935 	return found;
1936 }
1937 
1938 static int __process_pages_contig(struct address_space *mapping,
1939 				  struct page *locked_page,
1940 				  pgoff_t start_index, pgoff_t end_index,
1941 				  unsigned long page_ops, pgoff_t *index_ret)
1942 {
1943 	unsigned long nr_pages = end_index - start_index + 1;
1944 	unsigned long pages_processed = 0;
1945 	pgoff_t index = start_index;
1946 	struct page *pages[16];
1947 	unsigned ret;
1948 	int err = 0;
1949 	int i;
1950 
1951 	if (page_ops & PAGE_LOCK) {
1952 		ASSERT(page_ops == PAGE_LOCK);
1953 		ASSERT(index_ret && *index_ret == start_index);
1954 	}
1955 
1956 	if ((page_ops & PAGE_SET_ERROR) && nr_pages > 0)
1957 		mapping_set_error(mapping, -EIO);
1958 
1959 	while (nr_pages > 0) {
1960 		ret = find_get_pages_contig(mapping, index,
1961 				     min_t(unsigned long,
1962 				     nr_pages, ARRAY_SIZE(pages)), pages);
1963 		if (ret == 0) {
1964 			/*
1965 			 * Only if we're going to lock these pages,
1966 			 * can we find nothing at @index.
1967 			 */
1968 			ASSERT(page_ops & PAGE_LOCK);
1969 			err = -EAGAIN;
1970 			goto out;
1971 		}
1972 
1973 		for (i = 0; i < ret; i++) {
1974 			if (page_ops & PAGE_SET_PRIVATE2)
1975 				SetPagePrivate2(pages[i]);
1976 
1977 			if (locked_page && pages[i] == locked_page) {
1978 				put_page(pages[i]);
1979 				pages_processed++;
1980 				continue;
1981 			}
1982 			if (page_ops & PAGE_START_WRITEBACK) {
1983 				clear_page_dirty_for_io(pages[i]);
1984 				set_page_writeback(pages[i]);
1985 			}
1986 			if (page_ops & PAGE_SET_ERROR)
1987 				SetPageError(pages[i]);
1988 			if (page_ops & PAGE_END_WRITEBACK)
1989 				end_page_writeback(pages[i]);
1990 			if (page_ops & PAGE_UNLOCK)
1991 				unlock_page(pages[i]);
1992 			if (page_ops & PAGE_LOCK) {
1993 				lock_page(pages[i]);
1994 				if (!PageDirty(pages[i]) ||
1995 				    pages[i]->mapping != mapping) {
1996 					unlock_page(pages[i]);
1997 					for (; i < ret; i++)
1998 						put_page(pages[i]);
1999 					err = -EAGAIN;
2000 					goto out;
2001 				}
2002 			}
2003 			put_page(pages[i]);
2004 			pages_processed++;
2005 		}
2006 		nr_pages -= ret;
2007 		index += ret;
2008 		cond_resched();
2009 	}
2010 out:
2011 	if (err && index_ret)
2012 		*index_ret = start_index + pages_processed - 1;
2013 	return err;
2014 }
2015 
2016 void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end,
2017 				  struct page *locked_page,
2018 				  u32 clear_bits, unsigned long page_ops)
2019 {
2020 	clear_extent_bit(&inode->io_tree, start, end, clear_bits, 1, 0, NULL);
2021 
2022 	__process_pages_contig(inode->vfs_inode.i_mapping, locked_page,
2023 			       start >> PAGE_SHIFT, end >> PAGE_SHIFT,
2024 			       page_ops, NULL);
2025 }
2026 
2027 /*
2028  * count the number of bytes in the tree that have a given bit(s)
2029  * set.  This can be fairly slow, except for EXTENT_DIRTY which is
2030  * cached.  The total number found is returned.
2031  */
2032 u64 count_range_bits(struct extent_io_tree *tree,
2033 		     u64 *start, u64 search_end, u64 max_bytes,
2034 		     u32 bits, int contig)
2035 {
2036 	struct rb_node *node;
2037 	struct extent_state *state;
2038 	u64 cur_start = *start;
2039 	u64 total_bytes = 0;
2040 	u64 last = 0;
2041 	int found = 0;
2042 
2043 	if (WARN_ON(search_end <= cur_start))
2044 		return 0;
2045 
2046 	spin_lock(&tree->lock);
2047 	if (cur_start == 0 && bits == EXTENT_DIRTY) {
2048 		total_bytes = tree->dirty_bytes;
2049 		goto out;
2050 	}
2051 	/*
2052 	 * this search will find all the extents that end after
2053 	 * our range starts.
2054 	 */
2055 	node = tree_search(tree, cur_start);
2056 	if (!node)
2057 		goto out;
2058 
2059 	while (1) {
2060 		state = rb_entry(node, struct extent_state, rb_node);
2061 		if (state->start > search_end)
2062 			break;
2063 		if (contig && found && state->start > last + 1)
2064 			break;
2065 		if (state->end >= cur_start && (state->state & bits) == bits) {
2066 			total_bytes += min(search_end, state->end) + 1 -
2067 				       max(cur_start, state->start);
2068 			if (total_bytes >= max_bytes)
2069 				break;
2070 			if (!found) {
2071 				*start = max(cur_start, state->start);
2072 				found = 1;
2073 			}
2074 			last = state->end;
2075 		} else if (contig && found) {
2076 			break;
2077 		}
2078 		node = rb_next(node);
2079 		if (!node)
2080 			break;
2081 	}
2082 out:
2083 	spin_unlock(&tree->lock);
2084 	return total_bytes;
2085 }
2086 
2087 /*
2088  * set the private field for a given byte offset in the tree.  If there isn't
2089  * an extent_state there already, this does nothing.
2090  */
2091 int set_state_failrec(struct extent_io_tree *tree, u64 start,
2092 		      struct io_failure_record *failrec)
2093 {
2094 	struct rb_node *node;
2095 	struct extent_state *state;
2096 	int ret = 0;
2097 
2098 	spin_lock(&tree->lock);
2099 	/*
2100 	 * this search will find all the extents that end after
2101 	 * our range starts.
2102 	 */
2103 	node = tree_search(tree, start);
2104 	if (!node) {
2105 		ret = -ENOENT;
2106 		goto out;
2107 	}
2108 	state = rb_entry(node, struct extent_state, rb_node);
2109 	if (state->start != start) {
2110 		ret = -ENOENT;
2111 		goto out;
2112 	}
2113 	state->failrec = failrec;
2114 out:
2115 	spin_unlock(&tree->lock);
2116 	return ret;
2117 }
2118 
2119 struct io_failure_record *get_state_failrec(struct extent_io_tree *tree, u64 start)
2120 {
2121 	struct rb_node *node;
2122 	struct extent_state *state;
2123 	struct io_failure_record *failrec;
2124 
2125 	spin_lock(&tree->lock);
2126 	/*
2127 	 * this search will find all the extents that end after
2128 	 * our range starts.
2129 	 */
2130 	node = tree_search(tree, start);
2131 	if (!node) {
2132 		failrec = ERR_PTR(-ENOENT);
2133 		goto out;
2134 	}
2135 	state = rb_entry(node, struct extent_state, rb_node);
2136 	if (state->start != start) {
2137 		failrec = ERR_PTR(-ENOENT);
2138 		goto out;
2139 	}
2140 
2141 	failrec = state->failrec;
2142 out:
2143 	spin_unlock(&tree->lock);
2144 	return failrec;
2145 }
2146 
2147 /*
2148  * searches a range in the state tree for a given mask.
2149  * If 'filled' == 1, this returns 1 only if every extent in the tree
2150  * has the bits set.  Otherwise, 1 is returned if any bit in the
2151  * range is found set.
2152  */
2153 int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
2154 		   u32 bits, int filled, struct extent_state *cached)
2155 {
2156 	struct extent_state *state = NULL;
2157 	struct rb_node *node;
2158 	int bitset = 0;
2159 
2160 	spin_lock(&tree->lock);
2161 	if (cached && extent_state_in_tree(cached) && cached->start <= start &&
2162 	    cached->end > start)
2163 		node = &cached->rb_node;
2164 	else
2165 		node = tree_search(tree, start);
2166 	while (node && start <= end) {
2167 		state = rb_entry(node, struct extent_state, rb_node);
2168 
2169 		if (filled && state->start > start) {
2170 			bitset = 0;
2171 			break;
2172 		}
2173 
2174 		if (state->start > end)
2175 			break;
2176 
2177 		if (state->state & bits) {
2178 			bitset = 1;
2179 			if (!filled)
2180 				break;
2181 		} else if (filled) {
2182 			bitset = 0;
2183 			break;
2184 		}
2185 
2186 		if (state->end == (u64)-1)
2187 			break;
2188 
2189 		start = state->end + 1;
2190 		if (start > end)
2191 			break;
2192 		node = rb_next(node);
2193 		if (!node) {
2194 			if (filled)
2195 				bitset = 0;
2196 			break;
2197 		}
2198 	}
2199 	spin_unlock(&tree->lock);
2200 	return bitset;
2201 }
2202 
2203 /*
2204  * helper function to set a given page up to date if all the
2205  * extents in the tree for that page are up to date
2206  */
2207 static void check_page_uptodate(struct extent_io_tree *tree, struct page *page)
2208 {
2209 	u64 start = page_offset(page);
2210 	u64 end = start + PAGE_SIZE - 1;
2211 	if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
2212 		SetPageUptodate(page);
2213 }
2214 
2215 int free_io_failure(struct extent_io_tree *failure_tree,
2216 		    struct extent_io_tree *io_tree,
2217 		    struct io_failure_record *rec)
2218 {
2219 	int ret;
2220 	int err = 0;
2221 
2222 	set_state_failrec(failure_tree, rec->start, NULL);
2223 	ret = clear_extent_bits(failure_tree, rec->start,
2224 				rec->start + rec->len - 1,
2225 				EXTENT_LOCKED | EXTENT_DIRTY);
2226 	if (ret)
2227 		err = ret;
2228 
2229 	ret = clear_extent_bits(io_tree, rec->start,
2230 				rec->start + rec->len - 1,
2231 				EXTENT_DAMAGED);
2232 	if (ret && !err)
2233 		err = ret;
2234 
2235 	kfree(rec);
2236 	return err;
2237 }
2238 
2239 /*
2240  * this bypasses the standard btrfs submit functions deliberately, as
2241  * the standard behavior is to write all copies in a raid setup. here we only
2242  * want to write the one bad copy. so we do the mapping for ourselves and issue
2243  * submit_bio directly.
2244  * to avoid any synchronization issues, wait for the data after writing, which
2245  * actually prevents the read that triggered the error from finishing.
2246  * currently, there can be no more than two copies of every data bit. thus,
2247  * exactly one rewrite is required.
2248  */
2249 int repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start,
2250 		      u64 length, u64 logical, struct page *page,
2251 		      unsigned int pg_offset, int mirror_num)
2252 {
2253 	struct bio *bio;
2254 	struct btrfs_device *dev;
2255 	u64 map_length = 0;
2256 	u64 sector;
2257 	struct btrfs_bio *bbio = NULL;
2258 	int ret;
2259 
2260 	ASSERT(!(fs_info->sb->s_flags & SB_RDONLY));
2261 	BUG_ON(!mirror_num);
2262 
2263 	if (btrfs_is_zoned(fs_info))
2264 		return btrfs_repair_one_zone(fs_info, logical);
2265 
2266 	bio = btrfs_io_bio_alloc(1);
2267 	bio->bi_iter.bi_size = 0;
2268 	map_length = length;
2269 
2270 	/*
2271 	 * Avoid races with device replace and make sure our bbio has devices
2272 	 * associated to its stripes that don't go away while we are doing the
2273 	 * read repair operation.
2274 	 */
2275 	btrfs_bio_counter_inc_blocked(fs_info);
2276 	if (btrfs_is_parity_mirror(fs_info, logical, length)) {
2277 		/*
2278 		 * Note that we don't use BTRFS_MAP_WRITE because it's supposed
2279 		 * to update all raid stripes, but here we just want to correct
2280 		 * bad stripe, thus BTRFS_MAP_READ is abused to only get the bad
2281 		 * stripe's dev and sector.
2282 		 */
2283 		ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, logical,
2284 				      &map_length, &bbio, 0);
2285 		if (ret) {
2286 			btrfs_bio_counter_dec(fs_info);
2287 			bio_put(bio);
2288 			return -EIO;
2289 		}
2290 		ASSERT(bbio->mirror_num == 1);
2291 	} else {
2292 		ret = btrfs_map_block(fs_info, BTRFS_MAP_WRITE, logical,
2293 				      &map_length, &bbio, mirror_num);
2294 		if (ret) {
2295 			btrfs_bio_counter_dec(fs_info);
2296 			bio_put(bio);
2297 			return -EIO;
2298 		}
2299 		BUG_ON(mirror_num != bbio->mirror_num);
2300 	}
2301 
2302 	sector = bbio->stripes[bbio->mirror_num - 1].physical >> 9;
2303 	bio->bi_iter.bi_sector = sector;
2304 	dev = bbio->stripes[bbio->mirror_num - 1].dev;
2305 	btrfs_put_bbio(bbio);
2306 	if (!dev || !dev->bdev ||
2307 	    !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) {
2308 		btrfs_bio_counter_dec(fs_info);
2309 		bio_put(bio);
2310 		return -EIO;
2311 	}
2312 	bio_set_dev(bio, dev->bdev);
2313 	bio->bi_opf = REQ_OP_WRITE | REQ_SYNC;
2314 	bio_add_page(bio, page, length, pg_offset);
2315 
2316 	if (btrfsic_submit_bio_wait(bio)) {
2317 		/* try to remap that extent elsewhere? */
2318 		btrfs_bio_counter_dec(fs_info);
2319 		bio_put(bio);
2320 		btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
2321 		return -EIO;
2322 	}
2323 
2324 	btrfs_info_rl_in_rcu(fs_info,
2325 		"read error corrected: ino %llu off %llu (dev %s sector %llu)",
2326 				  ino, start,
2327 				  rcu_str_deref(dev->name), sector);
2328 	btrfs_bio_counter_dec(fs_info);
2329 	bio_put(bio);
2330 	return 0;
2331 }
2332 
2333 int btrfs_repair_eb_io_failure(const struct extent_buffer *eb, int mirror_num)
2334 {
2335 	struct btrfs_fs_info *fs_info = eb->fs_info;
2336 	u64 start = eb->start;
2337 	int i, num_pages = num_extent_pages(eb);
2338 	int ret = 0;
2339 
2340 	if (sb_rdonly(fs_info->sb))
2341 		return -EROFS;
2342 
2343 	for (i = 0; i < num_pages; i++) {
2344 		struct page *p = eb->pages[i];
2345 
2346 		ret = repair_io_failure(fs_info, 0, start, PAGE_SIZE, start, p,
2347 					start - page_offset(p), mirror_num);
2348 		if (ret)
2349 			break;
2350 		start += PAGE_SIZE;
2351 	}
2352 
2353 	return ret;
2354 }
2355 
2356 /*
2357  * each time an IO finishes, we do a fast check in the IO failure tree
2358  * to see if we need to process or clean up an io_failure_record
2359  */
2360 int clean_io_failure(struct btrfs_fs_info *fs_info,
2361 		     struct extent_io_tree *failure_tree,
2362 		     struct extent_io_tree *io_tree, u64 start,
2363 		     struct page *page, u64 ino, unsigned int pg_offset)
2364 {
2365 	u64 private;
2366 	struct io_failure_record *failrec;
2367 	struct extent_state *state;
2368 	int num_copies;
2369 	int ret;
2370 
2371 	private = 0;
2372 	ret = count_range_bits(failure_tree, &private, (u64)-1, 1,
2373 			       EXTENT_DIRTY, 0);
2374 	if (!ret)
2375 		return 0;
2376 
2377 	failrec = get_state_failrec(failure_tree, start);
2378 	if (IS_ERR(failrec))
2379 		return 0;
2380 
2381 	BUG_ON(!failrec->this_mirror);
2382 
2383 	if (failrec->in_validation) {
2384 		/* there was no real error, just free the record */
2385 		btrfs_debug(fs_info,
2386 			"clean_io_failure: freeing dummy error at %llu",
2387 			failrec->start);
2388 		goto out;
2389 	}
2390 	if (sb_rdonly(fs_info->sb))
2391 		goto out;
2392 
2393 	spin_lock(&io_tree->lock);
2394 	state = find_first_extent_bit_state(io_tree,
2395 					    failrec->start,
2396 					    EXTENT_LOCKED);
2397 	spin_unlock(&io_tree->lock);
2398 
2399 	if (state && state->start <= failrec->start &&
2400 	    state->end >= failrec->start + failrec->len - 1) {
2401 		num_copies = btrfs_num_copies(fs_info, failrec->logical,
2402 					      failrec->len);
2403 		if (num_copies > 1)  {
2404 			repair_io_failure(fs_info, ino, start, failrec->len,
2405 					  failrec->logical, page, pg_offset,
2406 					  failrec->failed_mirror);
2407 		}
2408 	}
2409 
2410 out:
2411 	free_io_failure(failure_tree, io_tree, failrec);
2412 
2413 	return 0;
2414 }
2415 
2416 /*
2417  * Can be called when
2418  * - hold extent lock
2419  * - under ordered extent
2420  * - the inode is freeing
2421  */
2422 void btrfs_free_io_failure_record(struct btrfs_inode *inode, u64 start, u64 end)
2423 {
2424 	struct extent_io_tree *failure_tree = &inode->io_failure_tree;
2425 	struct io_failure_record *failrec;
2426 	struct extent_state *state, *next;
2427 
2428 	if (RB_EMPTY_ROOT(&failure_tree->state))
2429 		return;
2430 
2431 	spin_lock(&failure_tree->lock);
2432 	state = find_first_extent_bit_state(failure_tree, start, EXTENT_DIRTY);
2433 	while (state) {
2434 		if (state->start > end)
2435 			break;
2436 
2437 		ASSERT(state->end <= end);
2438 
2439 		next = next_state(state);
2440 
2441 		failrec = state->failrec;
2442 		free_extent_state(state);
2443 		kfree(failrec);
2444 
2445 		state = next;
2446 	}
2447 	spin_unlock(&failure_tree->lock);
2448 }
2449 
2450 static struct io_failure_record *btrfs_get_io_failure_record(struct inode *inode,
2451 							     u64 start, u64 end)
2452 {
2453 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2454 	struct io_failure_record *failrec;
2455 	struct extent_map *em;
2456 	struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2457 	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2458 	struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
2459 	int ret;
2460 	u64 logical;
2461 
2462 	failrec = get_state_failrec(failure_tree, start);
2463 	if (!IS_ERR(failrec)) {
2464 		btrfs_debug(fs_info,
2465 			"Get IO Failure Record: (found) logical=%llu, start=%llu, len=%llu, validation=%d",
2466 			failrec->logical, failrec->start, failrec->len,
2467 			failrec->in_validation);
2468 		/*
2469 		 * when data can be on disk more than twice, add to failrec here
2470 		 * (e.g. with a list for failed_mirror) to make
2471 		 * clean_io_failure() clean all those errors at once.
2472 		 */
2473 
2474 		return failrec;
2475 	}
2476 
2477 	failrec = kzalloc(sizeof(*failrec), GFP_NOFS);
2478 	if (!failrec)
2479 		return ERR_PTR(-ENOMEM);
2480 
2481 	failrec->start = start;
2482 	failrec->len = end - start + 1;
2483 	failrec->this_mirror = 0;
2484 	failrec->bio_flags = 0;
2485 	failrec->in_validation = 0;
2486 
2487 	read_lock(&em_tree->lock);
2488 	em = lookup_extent_mapping(em_tree, start, failrec->len);
2489 	if (!em) {
2490 		read_unlock(&em_tree->lock);
2491 		kfree(failrec);
2492 		return ERR_PTR(-EIO);
2493 	}
2494 
2495 	if (em->start > start || em->start + em->len <= start) {
2496 		free_extent_map(em);
2497 		em = NULL;
2498 	}
2499 	read_unlock(&em_tree->lock);
2500 	if (!em) {
2501 		kfree(failrec);
2502 		return ERR_PTR(-EIO);
2503 	}
2504 
2505 	logical = start - em->start;
2506 	logical = em->block_start + logical;
2507 	if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2508 		logical = em->block_start;
2509 		failrec->bio_flags = EXTENT_BIO_COMPRESSED;
2510 		extent_set_compress_type(&failrec->bio_flags, em->compress_type);
2511 	}
2512 
2513 	btrfs_debug(fs_info,
2514 		    "Get IO Failure Record: (new) logical=%llu, start=%llu, len=%llu",
2515 		    logical, start, failrec->len);
2516 
2517 	failrec->logical = logical;
2518 	free_extent_map(em);
2519 
2520 	/* Set the bits in the private failure tree */
2521 	ret = set_extent_bits(failure_tree, start, end,
2522 			      EXTENT_LOCKED | EXTENT_DIRTY);
2523 	if (ret >= 0) {
2524 		ret = set_state_failrec(failure_tree, start, failrec);
2525 		/* Set the bits in the inode's tree */
2526 		ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED);
2527 	} else if (ret < 0) {
2528 		kfree(failrec);
2529 		return ERR_PTR(ret);
2530 	}
2531 
2532 	return failrec;
2533 }
2534 
2535 static bool btrfs_check_repairable(struct inode *inode, bool needs_validation,
2536 				   struct io_failure_record *failrec,
2537 				   int failed_mirror)
2538 {
2539 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2540 	int num_copies;
2541 
2542 	num_copies = btrfs_num_copies(fs_info, failrec->logical, failrec->len);
2543 	if (num_copies == 1) {
2544 		/*
2545 		 * we only have a single copy of the data, so don't bother with
2546 		 * all the retry and error correction code that follows. no
2547 		 * matter what the error is, it is very likely to persist.
2548 		 */
2549 		btrfs_debug(fs_info,
2550 			"Check Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d",
2551 			num_copies, failrec->this_mirror, failed_mirror);
2552 		return false;
2553 	}
2554 
2555 	/*
2556 	 * there are two premises:
2557 	 *	a) deliver good data to the caller
2558 	 *	b) correct the bad sectors on disk
2559 	 */
2560 	if (needs_validation) {
2561 		/*
2562 		 * to fulfill b), we need to know the exact failing sectors, as
2563 		 * we don't want to rewrite any more than the failed ones. thus,
2564 		 * we need separate read requests for the failed bio
2565 		 *
2566 		 * if the following BUG_ON triggers, our validation request got
2567 		 * merged. we need separate requests for our algorithm to work.
2568 		 */
2569 		BUG_ON(failrec->in_validation);
2570 		failrec->in_validation = 1;
2571 		failrec->this_mirror = failed_mirror;
2572 	} else {
2573 		/*
2574 		 * we're ready to fulfill a) and b) alongside. get a good copy
2575 		 * of the failed sector and if we succeed, we have setup
2576 		 * everything for repair_io_failure to do the rest for us.
2577 		 */
2578 		if (failrec->in_validation) {
2579 			BUG_ON(failrec->this_mirror != failed_mirror);
2580 			failrec->in_validation = 0;
2581 			failrec->this_mirror = 0;
2582 		}
2583 		failrec->failed_mirror = failed_mirror;
2584 		failrec->this_mirror++;
2585 		if (failrec->this_mirror == failed_mirror)
2586 			failrec->this_mirror++;
2587 	}
2588 
2589 	if (failrec->this_mirror > num_copies) {
2590 		btrfs_debug(fs_info,
2591 			"Check Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d",
2592 			num_copies, failrec->this_mirror, failed_mirror);
2593 		return false;
2594 	}
2595 
2596 	return true;
2597 }
2598 
2599 static bool btrfs_io_needs_validation(struct inode *inode, struct bio *bio)
2600 {
2601 	u64 len = 0;
2602 	const u32 blocksize = inode->i_sb->s_blocksize;
2603 
2604 	/*
2605 	 * If bi_status is BLK_STS_OK, then this was a checksum error, not an
2606 	 * I/O error. In this case, we already know exactly which sector was
2607 	 * bad, so we don't need to validate.
2608 	 */
2609 	if (bio->bi_status == BLK_STS_OK)
2610 		return false;
2611 
2612 	/*
2613 	 * We need to validate each sector individually if the failed I/O was
2614 	 * for multiple sectors.
2615 	 *
2616 	 * There are a few possible bios that can end up here:
2617 	 * 1. A buffered read bio, which is not cloned.
2618 	 * 2. A direct I/O read bio, which is cloned.
2619 	 * 3. A (buffered or direct) repair bio, which is not cloned.
2620 	 *
2621 	 * For cloned bios (case 2), we can get the size from
2622 	 * btrfs_io_bio->iter; for non-cloned bios (cases 1 and 3), we can get
2623 	 * it from the bvecs.
2624 	 */
2625 	if (bio_flagged(bio, BIO_CLONED)) {
2626 		if (btrfs_io_bio(bio)->iter.bi_size > blocksize)
2627 			return true;
2628 	} else {
2629 		struct bio_vec *bvec;
2630 		int i;
2631 
2632 		bio_for_each_bvec_all(bvec, bio, i) {
2633 			len += bvec->bv_len;
2634 			if (len > blocksize)
2635 				return true;
2636 		}
2637 	}
2638 	return false;
2639 }
2640 
2641 blk_status_t btrfs_submit_read_repair(struct inode *inode,
2642 				      struct bio *failed_bio, u32 bio_offset,
2643 				      struct page *page, unsigned int pgoff,
2644 				      u64 start, u64 end, int failed_mirror,
2645 				      submit_bio_hook_t *submit_bio_hook)
2646 {
2647 	struct io_failure_record *failrec;
2648 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2649 	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2650 	struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2651 	struct btrfs_io_bio *failed_io_bio = btrfs_io_bio(failed_bio);
2652 	const int icsum = bio_offset >> fs_info->sectorsize_bits;
2653 	bool need_validation;
2654 	struct bio *repair_bio;
2655 	struct btrfs_io_bio *repair_io_bio;
2656 	blk_status_t status;
2657 
2658 	btrfs_debug(fs_info,
2659 		   "repair read error: read error at %llu", start);
2660 
2661 	BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);
2662 
2663 	failrec = btrfs_get_io_failure_record(inode, start, end);
2664 	if (IS_ERR(failrec))
2665 		return errno_to_blk_status(PTR_ERR(failrec));
2666 
2667 	need_validation = btrfs_io_needs_validation(inode, failed_bio);
2668 
2669 	if (!btrfs_check_repairable(inode, need_validation, failrec,
2670 				    failed_mirror)) {
2671 		free_io_failure(failure_tree, tree, failrec);
2672 		return BLK_STS_IOERR;
2673 	}
2674 
2675 	repair_bio = btrfs_io_bio_alloc(1);
2676 	repair_io_bio = btrfs_io_bio(repair_bio);
2677 	repair_bio->bi_opf = REQ_OP_READ;
2678 	if (need_validation)
2679 		repair_bio->bi_opf |= REQ_FAILFAST_DEV;
2680 	repair_bio->bi_end_io = failed_bio->bi_end_io;
2681 	repair_bio->bi_iter.bi_sector = failrec->logical >> 9;
2682 	repair_bio->bi_private = failed_bio->bi_private;
2683 
2684 	if (failed_io_bio->csum) {
2685 		const u32 csum_size = fs_info->csum_size;
2686 
2687 		repair_io_bio->csum = repair_io_bio->csum_inline;
2688 		memcpy(repair_io_bio->csum,
2689 		       failed_io_bio->csum + csum_size * icsum, csum_size);
2690 	}
2691 
2692 	bio_add_page(repair_bio, page, failrec->len, pgoff);
2693 	repair_io_bio->logical = failrec->start;
2694 	repair_io_bio->iter = repair_bio->bi_iter;
2695 
2696 	btrfs_debug(btrfs_sb(inode->i_sb),
2697 "repair read error: submitting new read to mirror %d, in_validation=%d",
2698 		    failrec->this_mirror, failrec->in_validation);
2699 
2700 	status = submit_bio_hook(inode, repair_bio, failrec->this_mirror,
2701 				 failrec->bio_flags);
2702 	if (status) {
2703 		free_io_failure(failure_tree, tree, failrec);
2704 		bio_put(repair_bio);
2705 	}
2706 	return status;
2707 }
2708 
2709 /* lots and lots of room for performance fixes in the end_bio funcs */
2710 
2711 void end_extent_writepage(struct page *page, int err, u64 start, u64 end)
2712 {
2713 	int uptodate = (err == 0);
2714 	int ret = 0;
2715 
2716 	btrfs_writepage_endio_finish_ordered(page, start, end, uptodate);
2717 
2718 	if (!uptodate) {
2719 		ClearPageUptodate(page);
2720 		SetPageError(page);
2721 		ret = err < 0 ? err : -EIO;
2722 		mapping_set_error(page->mapping, ret);
2723 	}
2724 }
2725 
2726 /*
2727  * after a writepage IO is done, we need to:
2728  * clear the uptodate bits on error
2729  * clear the writeback bits in the extent tree for this IO
2730  * end_page_writeback if the page has no more pending IO
2731  *
2732  * Scheduling is not allowed, so the extent state tree is expected
2733  * to have one and only one object corresponding to this IO.
2734  */
2735 static void end_bio_extent_writepage(struct bio *bio)
2736 {
2737 	int error = blk_status_to_errno(bio->bi_status);
2738 	struct bio_vec *bvec;
2739 	u64 start;
2740 	u64 end;
2741 	struct bvec_iter_all iter_all;
2742 	bool first_bvec = true;
2743 
2744 	ASSERT(!bio_flagged(bio, BIO_CLONED));
2745 	bio_for_each_segment_all(bvec, bio, iter_all) {
2746 		struct page *page = bvec->bv_page;
2747 		struct inode *inode = page->mapping->host;
2748 		struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2749 
2750 		/* We always issue full-page reads, but if some block
2751 		 * in a page fails to read, blk_update_request() will
2752 		 * advance bv_offset and adjust bv_len to compensate.
2753 		 * Print a warning for nonzero offsets, and an error
2754 		 * if they don't add up to a full page.  */
2755 		if (bvec->bv_offset || bvec->bv_len != PAGE_SIZE) {
2756 			if (bvec->bv_offset + bvec->bv_len != PAGE_SIZE)
2757 				btrfs_err(fs_info,
2758 				   "partial page write in btrfs with offset %u and length %u",
2759 					bvec->bv_offset, bvec->bv_len);
2760 			else
2761 				btrfs_info(fs_info,
2762 				   "incomplete page write in btrfs with offset %u and length %u",
2763 					bvec->bv_offset, bvec->bv_len);
2764 		}
2765 
2766 		start = page_offset(page);
2767 		end = start + bvec->bv_offset + bvec->bv_len - 1;
2768 
2769 		if (first_bvec) {
2770 			btrfs_record_physical_zoned(inode, start, bio);
2771 			first_bvec = false;
2772 		}
2773 
2774 		end_extent_writepage(page, error, start, end);
2775 		end_page_writeback(page);
2776 	}
2777 
2778 	bio_put(bio);
2779 }
2780 
2781 /*
2782  * Record previously processed extent range
2783  *
2784  * For endio_readpage_release_extent() to handle a full extent range, reducing
2785  * the extent io operations.
2786  */
2787 struct processed_extent {
2788 	struct btrfs_inode *inode;
2789 	/* Start of the range in @inode */
2790 	u64 start;
2791 	/* End of the range in @inode */
2792 	u64 end;
2793 	bool uptodate;
2794 };
2795 
2796 /*
2797  * Try to release processed extent range
2798  *
2799  * May not release the extent range right now if the current range is
2800  * contiguous to processed extent.
2801  *
2802  * Will release processed extent when any of @inode, @uptodate, the range is
2803  * no longer contiguous to the processed range.
2804  *
2805  * Passing @inode == NULL will force processed extent to be released.
2806  */
2807 static void endio_readpage_release_extent(struct processed_extent *processed,
2808 			      struct btrfs_inode *inode, u64 start, u64 end,
2809 			      bool uptodate)
2810 {
2811 	struct extent_state *cached = NULL;
2812 	struct extent_io_tree *tree;
2813 
2814 	/* The first extent, initialize @processed */
2815 	if (!processed->inode)
2816 		goto update;
2817 
2818 	/*
2819 	 * Contiguous to processed extent, just uptodate the end.
2820 	 *
2821 	 * Several things to notice:
2822 	 *
2823 	 * - bio can be merged as long as on-disk bytenr is contiguous
2824 	 *   This means we can have page belonging to other inodes, thus need to
2825 	 *   check if the inode still matches.
2826 	 * - bvec can contain range beyond current page for multi-page bvec
2827 	 *   Thus we need to do processed->end + 1 >= start check
2828 	 */
2829 	if (processed->inode == inode && processed->uptodate == uptodate &&
2830 	    processed->end + 1 >= start && end >= processed->end) {
2831 		processed->end = end;
2832 		return;
2833 	}
2834 
2835 	tree = &processed->inode->io_tree;
2836 	/*
2837 	 * Now we don't have range contiguous to the processed range, release
2838 	 * the processed range now.
2839 	 */
2840 	if (processed->uptodate && tree->track_uptodate)
2841 		set_extent_uptodate(tree, processed->start, processed->end,
2842 				    &cached, GFP_ATOMIC);
2843 	unlock_extent_cached_atomic(tree, processed->start, processed->end,
2844 				    &cached);
2845 
2846 update:
2847 	/* Update processed to current range */
2848 	processed->inode = inode;
2849 	processed->start = start;
2850 	processed->end = end;
2851 	processed->uptodate = uptodate;
2852 }
2853 
2854 static void begin_page_read(struct btrfs_fs_info *fs_info, struct page *page)
2855 {
2856 	ASSERT(PageLocked(page));
2857 	if (fs_info->sectorsize == PAGE_SIZE)
2858 		return;
2859 
2860 	ASSERT(PagePrivate(page));
2861 	btrfs_subpage_start_reader(fs_info, page, page_offset(page), PAGE_SIZE);
2862 }
2863 
2864 static void end_page_read(struct page *page, bool uptodate, u64 start, u32 len)
2865 {
2866 	struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb);
2867 
2868 	ASSERT(page_offset(page) <= start &&
2869 		start + len <= page_offset(page) + PAGE_SIZE);
2870 
2871 	if (uptodate) {
2872 		btrfs_page_set_uptodate(fs_info, page, start, len);
2873 	} else {
2874 		btrfs_page_clear_uptodate(fs_info, page, start, len);
2875 		btrfs_page_set_error(fs_info, page, start, len);
2876 	}
2877 
2878 	if (fs_info->sectorsize == PAGE_SIZE)
2879 		unlock_page(page);
2880 	else if (is_data_inode(page->mapping->host))
2881 		/*
2882 		 * For subpage data, unlock the page if we're the last reader.
2883 		 * For subpage metadata, page lock is not utilized for read.
2884 		 */
2885 		btrfs_subpage_end_reader(fs_info, page, start, len);
2886 }
2887 
2888 /*
2889  * after a readpage IO is done, we need to:
2890  * clear the uptodate bits on error
2891  * set the uptodate bits if things worked
2892  * set the page up to date if all extents in the tree are uptodate
2893  * clear the lock bit in the extent tree
2894  * unlock the page if there are no other extents locked for it
2895  *
2896  * Scheduling is not allowed, so the extent state tree is expected
2897  * to have one and only one object corresponding to this IO.
2898  */
2899 static void end_bio_extent_readpage(struct bio *bio)
2900 {
2901 	struct bio_vec *bvec;
2902 	int uptodate = !bio->bi_status;
2903 	struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
2904 	struct extent_io_tree *tree, *failure_tree;
2905 	struct processed_extent processed = { 0 };
2906 	/*
2907 	 * The offset to the beginning of a bio, since one bio can never be
2908 	 * larger than UINT_MAX, u32 here is enough.
2909 	 */
2910 	u32 bio_offset = 0;
2911 	int mirror;
2912 	int ret;
2913 	struct bvec_iter_all iter_all;
2914 
2915 	ASSERT(!bio_flagged(bio, BIO_CLONED));
2916 	bio_for_each_segment_all(bvec, bio, iter_all) {
2917 		struct page *page = bvec->bv_page;
2918 		struct inode *inode = page->mapping->host;
2919 		struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2920 		const u32 sectorsize = fs_info->sectorsize;
2921 		u64 start;
2922 		u64 end;
2923 		u32 len;
2924 
2925 		btrfs_debug(fs_info,
2926 			"end_bio_extent_readpage: bi_sector=%llu, err=%d, mirror=%u",
2927 			bio->bi_iter.bi_sector, bio->bi_status,
2928 			io_bio->mirror_num);
2929 		tree = &BTRFS_I(inode)->io_tree;
2930 		failure_tree = &BTRFS_I(inode)->io_failure_tree;
2931 
2932 		/*
2933 		 * We always issue full-sector reads, but if some block in a
2934 		 * page fails to read, blk_update_request() will advance
2935 		 * bv_offset and adjust bv_len to compensate.  Print a warning
2936 		 * for unaligned offsets, and an error if they don't add up to
2937 		 * a full sector.
2938 		 */
2939 		if (!IS_ALIGNED(bvec->bv_offset, sectorsize))
2940 			btrfs_err(fs_info,
2941 		"partial page read in btrfs with offset %u and length %u",
2942 				  bvec->bv_offset, bvec->bv_len);
2943 		else if (!IS_ALIGNED(bvec->bv_offset + bvec->bv_len,
2944 				     sectorsize))
2945 			btrfs_info(fs_info,
2946 		"incomplete page read with offset %u and length %u",
2947 				   bvec->bv_offset, bvec->bv_len);
2948 
2949 		start = page_offset(page) + bvec->bv_offset;
2950 		end = start + bvec->bv_len - 1;
2951 		len = bvec->bv_len;
2952 
2953 		mirror = io_bio->mirror_num;
2954 		if (likely(uptodate)) {
2955 			if (is_data_inode(inode))
2956 				ret = btrfs_verify_data_csum(io_bio,
2957 						bio_offset, page, start, end,
2958 						mirror);
2959 			else
2960 				ret = btrfs_validate_metadata_buffer(io_bio,
2961 					page, start, end, mirror);
2962 			if (ret)
2963 				uptodate = 0;
2964 			else
2965 				clean_io_failure(BTRFS_I(inode)->root->fs_info,
2966 						 failure_tree, tree, start,
2967 						 page,
2968 						 btrfs_ino(BTRFS_I(inode)), 0);
2969 		}
2970 
2971 		if (likely(uptodate))
2972 			goto readpage_ok;
2973 
2974 		if (is_data_inode(inode)) {
2975 
2976 			/*
2977 			 * The generic bio_readpage_error handles errors the
2978 			 * following way: If possible, new read requests are
2979 			 * created and submitted and will end up in
2980 			 * end_bio_extent_readpage as well (if we're lucky,
2981 			 * not in the !uptodate case). In that case it returns
2982 			 * 0 and we just go on with the next page in our bio.
2983 			 * If it can't handle the error it will return -EIO and
2984 			 * we remain responsible for that page.
2985 			 */
2986 			if (!btrfs_submit_read_repair(inode, bio, bio_offset,
2987 						page,
2988 						start - page_offset(page),
2989 						start, end, mirror,
2990 						btrfs_submit_data_bio)) {
2991 				uptodate = !bio->bi_status;
2992 				ASSERT(bio_offset + len > bio_offset);
2993 				bio_offset += len;
2994 				continue;
2995 			}
2996 		} else {
2997 			struct extent_buffer *eb;
2998 
2999 			eb = (struct extent_buffer *)page->private;
3000 			set_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
3001 			eb->read_mirror = mirror;
3002 			atomic_dec(&eb->io_pages);
3003 			if (test_and_clear_bit(EXTENT_BUFFER_READAHEAD,
3004 					       &eb->bflags))
3005 				btree_readahead_hook(eb, -EIO);
3006 		}
3007 readpage_ok:
3008 		if (likely(uptodate)) {
3009 			loff_t i_size = i_size_read(inode);
3010 			pgoff_t end_index = i_size >> PAGE_SHIFT;
3011 
3012 			/*
3013 			 * Zero out the remaining part if this range straddles
3014 			 * i_size.
3015 			 *
3016 			 * Here we should only zero the range inside the bvec,
3017 			 * not touch anything else.
3018 			 *
3019 			 * NOTE: i_size is exclusive while end is inclusive.
3020 			 */
3021 			if (page->index == end_index && i_size <= end) {
3022 				u32 zero_start = max(offset_in_page(i_size),
3023 						     offset_in_page(end));
3024 
3025 				zero_user_segment(page, zero_start,
3026 						  offset_in_page(end) + 1);
3027 			}
3028 		}
3029 		ASSERT(bio_offset + len > bio_offset);
3030 		bio_offset += len;
3031 
3032 		/* Update page status and unlock */
3033 		end_page_read(page, uptodate, start, len);
3034 		endio_readpage_release_extent(&processed, BTRFS_I(inode),
3035 					      start, end, uptodate);
3036 	}
3037 	/* Release the last extent */
3038 	endio_readpage_release_extent(&processed, NULL, 0, 0, false);
3039 	btrfs_io_bio_free_csum(io_bio);
3040 	bio_put(bio);
3041 }
3042 
3043 /*
3044  * Initialize the members up to but not including 'bio'. Use after allocating a
3045  * new bio by bio_alloc_bioset as it does not initialize the bytes outside of
3046  * 'bio' because use of __GFP_ZERO is not supported.
3047  */
3048 static inline void btrfs_io_bio_init(struct btrfs_io_bio *btrfs_bio)
3049 {
3050 	memset(btrfs_bio, 0, offsetof(struct btrfs_io_bio, bio));
3051 }
3052 
3053 /*
3054  * The following helpers allocate a bio. As it's backed by a bioset, it'll
3055  * never fail.  We're returning a bio right now but you can call btrfs_io_bio
3056  * for the appropriate container_of magic
3057  */
3058 struct bio *btrfs_bio_alloc(u64 first_byte)
3059 {
3060 	struct bio *bio;
3061 
3062 	bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_PAGES, &btrfs_bioset);
3063 	bio->bi_iter.bi_sector = first_byte >> 9;
3064 	btrfs_io_bio_init(btrfs_io_bio(bio));
3065 	return bio;
3066 }
3067 
3068 struct bio *btrfs_bio_clone(struct bio *bio)
3069 {
3070 	struct btrfs_io_bio *btrfs_bio;
3071 	struct bio *new;
3072 
3073 	/* Bio allocation backed by a bioset does not fail */
3074 	new = bio_clone_fast(bio, GFP_NOFS, &btrfs_bioset);
3075 	btrfs_bio = btrfs_io_bio(new);
3076 	btrfs_io_bio_init(btrfs_bio);
3077 	btrfs_bio->iter = bio->bi_iter;
3078 	return new;
3079 }
3080 
3081 struct bio *btrfs_io_bio_alloc(unsigned int nr_iovecs)
3082 {
3083 	struct bio *bio;
3084 
3085 	/* Bio allocation backed by a bioset does not fail */
3086 	bio = bio_alloc_bioset(GFP_NOFS, nr_iovecs, &btrfs_bioset);
3087 	btrfs_io_bio_init(btrfs_io_bio(bio));
3088 	return bio;
3089 }
3090 
3091 struct bio *btrfs_bio_clone_partial(struct bio *orig, int offset, int size)
3092 {
3093 	struct bio *bio;
3094 	struct btrfs_io_bio *btrfs_bio;
3095 
3096 	/* this will never fail when it's backed by a bioset */
3097 	bio = bio_clone_fast(orig, GFP_NOFS, &btrfs_bioset);
3098 	ASSERT(bio);
3099 
3100 	btrfs_bio = btrfs_io_bio(bio);
3101 	btrfs_io_bio_init(btrfs_bio);
3102 
3103 	bio_trim(bio, offset >> 9, size >> 9);
3104 	btrfs_bio->iter = bio->bi_iter;
3105 	return bio;
3106 }
3107 
3108 /**
3109  * Attempt to add a page to bio
3110  *
3111  * @bio:	destination bio
3112  * @page:	page to add to the bio
3113  * @disk_bytenr:  offset of the new bio or to check whether we are adding
3114  *                a contiguous page to the previous one
3115  * @pg_offset:	starting offset in the page
3116  * @size:	portion of page that we want to write
3117  * @prev_bio_flags:  flags of previous bio to see if we can merge the current one
3118  * @bio_flags:	flags of the current bio to see if we can merge them
3119  * @return:	true if page was added, false otherwise
3120  *
3121  * Attempt to add a page to bio considering stripe alignment etc.
3122  *
3123  * Return true if successfully page added. Otherwise, return false.
3124  */
3125 static bool btrfs_bio_add_page(struct bio *bio, struct page *page,
3126 			       u64 disk_bytenr, unsigned int size,
3127 			       unsigned int pg_offset,
3128 			       unsigned long prev_bio_flags,
3129 			       unsigned long bio_flags)
3130 {
3131 	const sector_t sector = disk_bytenr >> SECTOR_SHIFT;
3132 	bool contig;
3133 	int ret;
3134 
3135 	if (prev_bio_flags != bio_flags)
3136 		return false;
3137 
3138 	if (prev_bio_flags & EXTENT_BIO_COMPRESSED)
3139 		contig = bio->bi_iter.bi_sector == sector;
3140 	else
3141 		contig = bio_end_sector(bio) == sector;
3142 	if (!contig)
3143 		return false;
3144 
3145 	if (btrfs_bio_fits_in_stripe(page, size, bio, bio_flags))
3146 		return false;
3147 
3148 	if (bio_op(bio) == REQ_OP_ZONE_APPEND) {
3149 		struct page *first_page = bio_first_bvec_all(bio)->bv_page;
3150 
3151 		if (!btrfs_bio_fits_in_ordered_extent(first_page, bio, size))
3152 			return false;
3153 		ret = bio_add_zone_append_page(bio, page, size, pg_offset);
3154 	} else {
3155 		ret = bio_add_page(bio, page, size, pg_offset);
3156 	}
3157 
3158 	return ret == size;
3159 }
3160 
3161 /*
3162  * @opf:	bio REQ_OP_* and REQ_* flags as one value
3163  * @wbc:	optional writeback control for io accounting
3164  * @page:	page to add to the bio
3165  * @disk_bytenr: logical bytenr where the write will be
3166  * @size:	portion of page that we want to write to
3167  * @pg_offset:	offset of the new bio or to check whether we are adding
3168  *              a contiguous page to the previous one
3169  * @bio_ret:	must be valid pointer, newly allocated bio will be stored there
3170  * @end_io_func:     end_io callback for new bio
3171  * @mirror_num:	     desired mirror to read/write
3172  * @prev_bio_flags:  flags of previous bio to see if we can merge the current one
3173  * @bio_flags:	flags of the current bio to see if we can merge them
3174  */
3175 static int submit_extent_page(unsigned int opf,
3176 			      struct writeback_control *wbc,
3177 			      struct page *page, u64 disk_bytenr,
3178 			      size_t size, unsigned long pg_offset,
3179 			      struct bio **bio_ret,
3180 			      bio_end_io_t end_io_func,
3181 			      int mirror_num,
3182 			      unsigned long prev_bio_flags,
3183 			      unsigned long bio_flags,
3184 			      bool force_bio_submit)
3185 {
3186 	int ret = 0;
3187 	struct bio *bio;
3188 	size_t io_size = min_t(size_t, size, PAGE_SIZE);
3189 	struct btrfs_inode *inode = BTRFS_I(page->mapping->host);
3190 	struct extent_io_tree *tree = &inode->io_tree;
3191 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
3192 
3193 	ASSERT(bio_ret);
3194 
3195 	if (*bio_ret) {
3196 		bio = *bio_ret;
3197 		if (force_bio_submit ||
3198 		    !btrfs_bio_add_page(bio, page, disk_bytenr, io_size,
3199 					pg_offset, prev_bio_flags, bio_flags)) {
3200 			ret = submit_one_bio(bio, mirror_num, prev_bio_flags);
3201 			if (ret < 0) {
3202 				*bio_ret = NULL;
3203 				return ret;
3204 			}
3205 			bio = NULL;
3206 		} else {
3207 			if (wbc)
3208 				wbc_account_cgroup_owner(wbc, page, io_size);
3209 			return 0;
3210 		}
3211 	}
3212 
3213 	bio = btrfs_bio_alloc(disk_bytenr);
3214 	bio_add_page(bio, page, io_size, pg_offset);
3215 	bio->bi_end_io = end_io_func;
3216 	bio->bi_private = tree;
3217 	bio->bi_write_hint = page->mapping->host->i_write_hint;
3218 	bio->bi_opf = opf;
3219 	if (wbc) {
3220 		struct block_device *bdev;
3221 
3222 		bdev = fs_info->fs_devices->latest_bdev;
3223 		bio_set_dev(bio, bdev);
3224 		wbc_init_bio(wbc, bio);
3225 		wbc_account_cgroup_owner(wbc, page, io_size);
3226 	}
3227 	if (btrfs_is_zoned(fs_info) && bio_op(bio) == REQ_OP_ZONE_APPEND) {
3228 		struct extent_map *em;
3229 		struct map_lookup *map;
3230 
3231 		em = btrfs_get_chunk_map(fs_info, disk_bytenr, io_size);
3232 		if (IS_ERR(em))
3233 			return PTR_ERR(em);
3234 
3235 		map = em->map_lookup;
3236 		/* We only support single profile for now */
3237 		ASSERT(map->num_stripes == 1);
3238 		btrfs_io_bio(bio)->device = map->stripes[0].dev;
3239 
3240 		free_extent_map(em);
3241 	}
3242 
3243 	*bio_ret = bio;
3244 
3245 	return ret;
3246 }
3247 
3248 static int attach_extent_buffer_page(struct extent_buffer *eb,
3249 				     struct page *page,
3250 				     struct btrfs_subpage *prealloc)
3251 {
3252 	struct btrfs_fs_info *fs_info = eb->fs_info;
3253 	int ret = 0;
3254 
3255 	/*
3256 	 * If the page is mapped to btree inode, we should hold the private
3257 	 * lock to prevent race.
3258 	 * For cloned or dummy extent buffers, their pages are not mapped and
3259 	 * will not race with any other ebs.
3260 	 */
3261 	if (page->mapping)
3262 		lockdep_assert_held(&page->mapping->private_lock);
3263 
3264 	if (fs_info->sectorsize == PAGE_SIZE) {
3265 		if (!PagePrivate(page))
3266 			attach_page_private(page, eb);
3267 		else
3268 			WARN_ON(page->private != (unsigned long)eb);
3269 		return 0;
3270 	}
3271 
3272 	/* Already mapped, just free prealloc */
3273 	if (PagePrivate(page)) {
3274 		btrfs_free_subpage(prealloc);
3275 		return 0;
3276 	}
3277 
3278 	if (prealloc)
3279 		/* Has preallocated memory for subpage */
3280 		attach_page_private(page, prealloc);
3281 	else
3282 		/* Do new allocation to attach subpage */
3283 		ret = btrfs_attach_subpage(fs_info, page,
3284 					   BTRFS_SUBPAGE_METADATA);
3285 	return ret;
3286 }
3287 
3288 int set_page_extent_mapped(struct page *page)
3289 {
3290 	struct btrfs_fs_info *fs_info;
3291 
3292 	ASSERT(page->mapping);
3293 
3294 	if (PagePrivate(page))
3295 		return 0;
3296 
3297 	fs_info = btrfs_sb(page->mapping->host->i_sb);
3298 
3299 	if (fs_info->sectorsize < PAGE_SIZE)
3300 		return btrfs_attach_subpage(fs_info, page, BTRFS_SUBPAGE_DATA);
3301 
3302 	attach_page_private(page, (void *)EXTENT_PAGE_PRIVATE);
3303 	return 0;
3304 }
3305 
3306 void clear_page_extent_mapped(struct page *page)
3307 {
3308 	struct btrfs_fs_info *fs_info;
3309 
3310 	ASSERT(page->mapping);
3311 
3312 	if (!PagePrivate(page))
3313 		return;
3314 
3315 	fs_info = btrfs_sb(page->mapping->host->i_sb);
3316 	if (fs_info->sectorsize < PAGE_SIZE)
3317 		return btrfs_detach_subpage(fs_info, page);
3318 
3319 	detach_page_private(page);
3320 }
3321 
3322 static struct extent_map *
3323 __get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
3324 		 u64 start, u64 len, struct extent_map **em_cached)
3325 {
3326 	struct extent_map *em;
3327 
3328 	if (em_cached && *em_cached) {
3329 		em = *em_cached;
3330 		if (extent_map_in_tree(em) && start >= em->start &&
3331 		    start < extent_map_end(em)) {
3332 			refcount_inc(&em->refs);
3333 			return em;
3334 		}
3335 
3336 		free_extent_map(em);
3337 		*em_cached = NULL;
3338 	}
3339 
3340 	em = btrfs_get_extent(BTRFS_I(inode), page, pg_offset, start, len);
3341 	if (em_cached && !IS_ERR_OR_NULL(em)) {
3342 		BUG_ON(*em_cached);
3343 		refcount_inc(&em->refs);
3344 		*em_cached = em;
3345 	}
3346 	return em;
3347 }
3348 /*
3349  * basic readpage implementation.  Locked extent state structs are inserted
3350  * into the tree that are removed when the IO is done (by the end_io
3351  * handlers)
3352  * XXX JDM: This needs looking at to ensure proper page locking
3353  * return 0 on success, otherwise return error
3354  */
3355 int btrfs_do_readpage(struct page *page, struct extent_map **em_cached,
3356 		      struct bio **bio, unsigned long *bio_flags,
3357 		      unsigned int read_flags, u64 *prev_em_start)
3358 {
3359 	struct inode *inode = page->mapping->host;
3360 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3361 	u64 start = page_offset(page);
3362 	const u64 end = start + PAGE_SIZE - 1;
3363 	u64 cur = start;
3364 	u64 extent_offset;
3365 	u64 last_byte = i_size_read(inode);
3366 	u64 block_start;
3367 	u64 cur_end;
3368 	struct extent_map *em;
3369 	int ret = 0;
3370 	int nr = 0;
3371 	size_t pg_offset = 0;
3372 	size_t iosize;
3373 	size_t blocksize = inode->i_sb->s_blocksize;
3374 	unsigned long this_bio_flag = 0;
3375 	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
3376 
3377 	ret = set_page_extent_mapped(page);
3378 	if (ret < 0) {
3379 		unlock_extent(tree, start, end);
3380 		btrfs_page_set_error(fs_info, page, start, PAGE_SIZE);
3381 		unlock_page(page);
3382 		goto out;
3383 	}
3384 
3385 	if (!PageUptodate(page)) {
3386 		if (cleancache_get_page(page) == 0) {
3387 			BUG_ON(blocksize != PAGE_SIZE);
3388 			unlock_extent(tree, start, end);
3389 			unlock_page(page);
3390 			goto out;
3391 		}
3392 	}
3393 
3394 	if (page->index == last_byte >> PAGE_SHIFT) {
3395 		char *userpage;
3396 		size_t zero_offset = offset_in_page(last_byte);
3397 
3398 		if (zero_offset) {
3399 			iosize = PAGE_SIZE - zero_offset;
3400 			userpage = kmap_atomic(page);
3401 			memset(userpage + zero_offset, 0, iosize);
3402 			flush_dcache_page(page);
3403 			kunmap_atomic(userpage);
3404 		}
3405 	}
3406 	begin_page_read(fs_info, page);
3407 	while (cur <= end) {
3408 		bool force_bio_submit = false;
3409 		u64 disk_bytenr;
3410 
3411 		if (cur >= last_byte) {
3412 			char *userpage;
3413 			struct extent_state *cached = NULL;
3414 
3415 			iosize = PAGE_SIZE - pg_offset;
3416 			userpage = kmap_atomic(page);
3417 			memset(userpage + pg_offset, 0, iosize);
3418 			flush_dcache_page(page);
3419 			kunmap_atomic(userpage);
3420 			set_extent_uptodate(tree, cur, cur + iosize - 1,
3421 					    &cached, GFP_NOFS);
3422 			unlock_extent_cached(tree, cur,
3423 					     cur + iosize - 1, &cached);
3424 			end_page_read(page, true, cur, iosize);
3425 			break;
3426 		}
3427 		em = __get_extent_map(inode, page, pg_offset, cur,
3428 				      end - cur + 1, em_cached);
3429 		if (IS_ERR_OR_NULL(em)) {
3430 			unlock_extent(tree, cur, end);
3431 			end_page_read(page, false, cur, end + 1 - cur);
3432 			break;
3433 		}
3434 		extent_offset = cur - em->start;
3435 		BUG_ON(extent_map_end(em) <= cur);
3436 		BUG_ON(end < cur);
3437 
3438 		if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
3439 			this_bio_flag |= EXTENT_BIO_COMPRESSED;
3440 			extent_set_compress_type(&this_bio_flag,
3441 						 em->compress_type);
3442 		}
3443 
3444 		iosize = min(extent_map_end(em) - cur, end - cur + 1);
3445 		cur_end = min(extent_map_end(em) - 1, end);
3446 		iosize = ALIGN(iosize, blocksize);
3447 		if (this_bio_flag & EXTENT_BIO_COMPRESSED)
3448 			disk_bytenr = em->block_start;
3449 		else
3450 			disk_bytenr = em->block_start + extent_offset;
3451 		block_start = em->block_start;
3452 		if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
3453 			block_start = EXTENT_MAP_HOLE;
3454 
3455 		/*
3456 		 * If we have a file range that points to a compressed extent
3457 		 * and it's followed by a consecutive file range that points
3458 		 * to the same compressed extent (possibly with a different
3459 		 * offset and/or length, so it either points to the whole extent
3460 		 * or only part of it), we must make sure we do not submit a
3461 		 * single bio to populate the pages for the 2 ranges because
3462 		 * this makes the compressed extent read zero out the pages
3463 		 * belonging to the 2nd range. Imagine the following scenario:
3464 		 *
3465 		 *  File layout
3466 		 *  [0 - 8K]                     [8K - 24K]
3467 		 *    |                               |
3468 		 *    |                               |
3469 		 * points to extent X,         points to extent X,
3470 		 * offset 4K, length of 8K     offset 0, length 16K
3471 		 *
3472 		 * [extent X, compressed length = 4K uncompressed length = 16K]
3473 		 *
3474 		 * If the bio to read the compressed extent covers both ranges,
3475 		 * it will decompress extent X into the pages belonging to the
3476 		 * first range and then it will stop, zeroing out the remaining
3477 		 * pages that belong to the other range that points to extent X.
3478 		 * So here we make sure we submit 2 bios, one for the first
3479 		 * range and another one for the third range. Both will target
3480 		 * the same physical extent from disk, but we can't currently
3481 		 * make the compressed bio endio callback populate the pages
3482 		 * for both ranges because each compressed bio is tightly
3483 		 * coupled with a single extent map, and each range can have
3484 		 * an extent map with a different offset value relative to the
3485 		 * uncompressed data of our extent and different lengths. This
3486 		 * is a corner case so we prioritize correctness over
3487 		 * non-optimal behavior (submitting 2 bios for the same extent).
3488 		 */
3489 		if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) &&
3490 		    prev_em_start && *prev_em_start != (u64)-1 &&
3491 		    *prev_em_start != em->start)
3492 			force_bio_submit = true;
3493 
3494 		if (prev_em_start)
3495 			*prev_em_start = em->start;
3496 
3497 		free_extent_map(em);
3498 		em = NULL;
3499 
3500 		/* we've found a hole, just zero and go on */
3501 		if (block_start == EXTENT_MAP_HOLE) {
3502 			char *userpage;
3503 			struct extent_state *cached = NULL;
3504 
3505 			userpage = kmap_atomic(page);
3506 			memset(userpage + pg_offset, 0, iosize);
3507 			flush_dcache_page(page);
3508 			kunmap_atomic(userpage);
3509 
3510 			set_extent_uptodate(tree, cur, cur + iosize - 1,
3511 					    &cached, GFP_NOFS);
3512 			unlock_extent_cached(tree, cur,
3513 					     cur + iosize - 1, &cached);
3514 			end_page_read(page, true, cur, iosize);
3515 			cur = cur + iosize;
3516 			pg_offset += iosize;
3517 			continue;
3518 		}
3519 		/* the get_extent function already copied into the page */
3520 		if (test_range_bit(tree, cur, cur_end,
3521 				   EXTENT_UPTODATE, 1, NULL)) {
3522 			check_page_uptodate(tree, page);
3523 			unlock_extent(tree, cur, cur + iosize - 1);
3524 			end_page_read(page, true, cur, iosize);
3525 			cur = cur + iosize;
3526 			pg_offset += iosize;
3527 			continue;
3528 		}
3529 		/* we have an inline extent but it didn't get marked up
3530 		 * to date.  Error out
3531 		 */
3532 		if (block_start == EXTENT_MAP_INLINE) {
3533 			unlock_extent(tree, cur, cur + iosize - 1);
3534 			end_page_read(page, false, cur, iosize);
3535 			cur = cur + iosize;
3536 			pg_offset += iosize;
3537 			continue;
3538 		}
3539 
3540 		ret = submit_extent_page(REQ_OP_READ | read_flags, NULL,
3541 					 page, disk_bytenr, iosize,
3542 					 pg_offset, bio,
3543 					 end_bio_extent_readpage, 0,
3544 					 *bio_flags,
3545 					 this_bio_flag,
3546 					 force_bio_submit);
3547 		if (!ret) {
3548 			nr++;
3549 			*bio_flags = this_bio_flag;
3550 		} else {
3551 			unlock_extent(tree, cur, cur + iosize - 1);
3552 			end_page_read(page, false, cur, iosize);
3553 			goto out;
3554 		}
3555 		cur = cur + iosize;
3556 		pg_offset += iosize;
3557 	}
3558 out:
3559 	return ret;
3560 }
3561 
3562 static inline void contiguous_readpages(struct page *pages[], int nr_pages,
3563 					     u64 start, u64 end,
3564 					     struct extent_map **em_cached,
3565 					     struct bio **bio,
3566 					     unsigned long *bio_flags,
3567 					     u64 *prev_em_start)
3568 {
3569 	struct btrfs_inode *inode = BTRFS_I(pages[0]->mapping->host);
3570 	int index;
3571 
3572 	btrfs_lock_and_flush_ordered_range(inode, start, end, NULL);
3573 
3574 	for (index = 0; index < nr_pages; index++) {
3575 		btrfs_do_readpage(pages[index], em_cached, bio, bio_flags,
3576 				  REQ_RAHEAD, prev_em_start);
3577 		put_page(pages[index]);
3578 	}
3579 }
3580 
3581 static void update_nr_written(struct writeback_control *wbc,
3582 			      unsigned long nr_written)
3583 {
3584 	wbc->nr_to_write -= nr_written;
3585 }
3586 
3587 /*
3588  * helper for __extent_writepage, doing all of the delayed allocation setup.
3589  *
3590  * This returns 1 if btrfs_run_delalloc_range function did all the work required
3591  * to write the page (copy into inline extent).  In this case the IO has
3592  * been started and the page is already unlocked.
3593  *
3594  * This returns 0 if all went well (page still locked)
3595  * This returns < 0 if there were errors (page still locked)
3596  */
3597 static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode,
3598 		struct page *page, struct writeback_control *wbc,
3599 		u64 delalloc_start, unsigned long *nr_written)
3600 {
3601 	u64 page_end = delalloc_start + PAGE_SIZE - 1;
3602 	bool found;
3603 	u64 delalloc_to_write = 0;
3604 	u64 delalloc_end = 0;
3605 	int ret;
3606 	int page_started = 0;
3607 
3608 
3609 	while (delalloc_end < page_end) {
3610 		found = find_lock_delalloc_range(&inode->vfs_inode, page,
3611 					       &delalloc_start,
3612 					       &delalloc_end);
3613 		if (!found) {
3614 			delalloc_start = delalloc_end + 1;
3615 			continue;
3616 		}
3617 		ret = btrfs_run_delalloc_range(inode, page, delalloc_start,
3618 				delalloc_end, &page_started, nr_written, wbc);
3619 		if (ret) {
3620 			SetPageError(page);
3621 			/*
3622 			 * btrfs_run_delalloc_range should return < 0 for error
3623 			 * but just in case, we use > 0 here meaning the IO is
3624 			 * started, so we don't want to return > 0 unless
3625 			 * things are going well.
3626 			 */
3627 			return ret < 0 ? ret : -EIO;
3628 		}
3629 		/*
3630 		 * delalloc_end is already one less than the total length, so
3631 		 * we don't subtract one from PAGE_SIZE
3632 		 */
3633 		delalloc_to_write += (delalloc_end - delalloc_start +
3634 				      PAGE_SIZE) >> PAGE_SHIFT;
3635 		delalloc_start = delalloc_end + 1;
3636 	}
3637 	if (wbc->nr_to_write < delalloc_to_write) {
3638 		int thresh = 8192;
3639 
3640 		if (delalloc_to_write < thresh * 2)
3641 			thresh = delalloc_to_write;
3642 		wbc->nr_to_write = min_t(u64, delalloc_to_write,
3643 					 thresh);
3644 	}
3645 
3646 	/* did the fill delalloc function already unlock and start
3647 	 * the IO?
3648 	 */
3649 	if (page_started) {
3650 		/*
3651 		 * we've unlocked the page, so we can't update
3652 		 * the mapping's writeback index, just update
3653 		 * nr_to_write.
3654 		 */
3655 		wbc->nr_to_write -= *nr_written;
3656 		return 1;
3657 	}
3658 
3659 	return 0;
3660 }
3661 
3662 /*
3663  * helper for __extent_writepage.  This calls the writepage start hooks,
3664  * and does the loop to map the page into extents and bios.
3665  *
3666  * We return 1 if the IO is started and the page is unlocked,
3667  * 0 if all went well (page still locked)
3668  * < 0 if there were errors (page still locked)
3669  */
3670 static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode,
3671 				 struct page *page,
3672 				 struct writeback_control *wbc,
3673 				 struct extent_page_data *epd,
3674 				 loff_t i_size,
3675 				 unsigned long nr_written,
3676 				 int *nr_ret)
3677 {
3678 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
3679 	struct extent_io_tree *tree = &inode->io_tree;
3680 	u64 start = page_offset(page);
3681 	u64 end = start + PAGE_SIZE - 1;
3682 	u64 cur = start;
3683 	u64 extent_offset;
3684 	u64 block_start;
3685 	struct extent_map *em;
3686 	int ret = 0;
3687 	int nr = 0;
3688 	u32 opf = REQ_OP_WRITE;
3689 	const unsigned int write_flags = wbc_to_write_flags(wbc);
3690 	bool compressed;
3691 
3692 	ret = btrfs_writepage_cow_fixup(page, start, end);
3693 	if (ret) {
3694 		/* Fixup worker will requeue */
3695 		redirty_page_for_writepage(wbc, page);
3696 		update_nr_written(wbc, nr_written);
3697 		unlock_page(page);
3698 		return 1;
3699 	}
3700 
3701 	/*
3702 	 * we don't want to touch the inode after unlocking the page,
3703 	 * so we update the mapping writeback index now
3704 	 */
3705 	update_nr_written(wbc, nr_written + 1);
3706 
3707 	while (cur <= end) {
3708 		u64 disk_bytenr;
3709 		u64 em_end;
3710 		u32 iosize;
3711 
3712 		if (cur >= i_size) {
3713 			btrfs_writepage_endio_finish_ordered(page, cur, end, 1);
3714 			break;
3715 		}
3716 		em = btrfs_get_extent(inode, NULL, 0, cur, end - cur + 1);
3717 		if (IS_ERR_OR_NULL(em)) {
3718 			SetPageError(page);
3719 			ret = PTR_ERR_OR_ZERO(em);
3720 			break;
3721 		}
3722 
3723 		extent_offset = cur - em->start;
3724 		em_end = extent_map_end(em);
3725 		ASSERT(cur <= em_end);
3726 		ASSERT(cur < end);
3727 		ASSERT(IS_ALIGNED(em->start, fs_info->sectorsize));
3728 		ASSERT(IS_ALIGNED(em->len, fs_info->sectorsize));
3729 		block_start = em->block_start;
3730 		compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
3731 		disk_bytenr = em->block_start + extent_offset;
3732 
3733 		/* Note that em_end from extent_map_end() is exclusive */
3734 		iosize = min(em_end, end + 1) - cur;
3735 
3736 		if (btrfs_use_zone_append(inode, em))
3737 			opf = REQ_OP_ZONE_APPEND;
3738 
3739 		free_extent_map(em);
3740 		em = NULL;
3741 
3742 		/*
3743 		 * compressed and inline extents are written through other
3744 		 * paths in the FS
3745 		 */
3746 		if (compressed || block_start == EXTENT_MAP_HOLE ||
3747 		    block_start == EXTENT_MAP_INLINE) {
3748 			if (compressed)
3749 				nr++;
3750 			else
3751 				btrfs_writepage_endio_finish_ordered(page, cur,
3752 							cur + iosize - 1, 1);
3753 			cur += iosize;
3754 			continue;
3755 		}
3756 
3757 		btrfs_set_range_writeback(tree, cur, cur + iosize - 1);
3758 		if (!PageWriteback(page)) {
3759 			btrfs_err(inode->root->fs_info,
3760 				   "page %lu not writeback, cur %llu end %llu",
3761 			       page->index, cur, end);
3762 		}
3763 
3764 		ret = submit_extent_page(opf | write_flags, wbc, page,
3765 					 disk_bytenr, iosize,
3766 					 cur - page_offset(page), &epd->bio,
3767 					 end_bio_extent_writepage,
3768 					 0, 0, 0, false);
3769 		if (ret) {
3770 			SetPageError(page);
3771 			if (PageWriteback(page))
3772 				end_page_writeback(page);
3773 		}
3774 
3775 		cur += iosize;
3776 		nr++;
3777 	}
3778 	*nr_ret = nr;
3779 	return ret;
3780 }
3781 
3782 /*
3783  * the writepage semantics are similar to regular writepage.  extent
3784  * records are inserted to lock ranges in the tree, and as dirty areas
3785  * are found, they are marked writeback.  Then the lock bits are removed
3786  * and the end_io handler clears the writeback ranges
3787  *
3788  * Return 0 if everything goes well.
3789  * Return <0 for error.
3790  */
3791 static int __extent_writepage(struct page *page, struct writeback_control *wbc,
3792 			      struct extent_page_data *epd)
3793 {
3794 	struct inode *inode = page->mapping->host;
3795 	u64 start = page_offset(page);
3796 	u64 page_end = start + PAGE_SIZE - 1;
3797 	int ret;
3798 	int nr = 0;
3799 	size_t pg_offset;
3800 	loff_t i_size = i_size_read(inode);
3801 	unsigned long end_index = i_size >> PAGE_SHIFT;
3802 	unsigned long nr_written = 0;
3803 
3804 	trace___extent_writepage(page, inode, wbc);
3805 
3806 	WARN_ON(!PageLocked(page));
3807 
3808 	ClearPageError(page);
3809 
3810 	pg_offset = offset_in_page(i_size);
3811 	if (page->index > end_index ||
3812 	   (page->index == end_index && !pg_offset)) {
3813 		page->mapping->a_ops->invalidatepage(page, 0, PAGE_SIZE);
3814 		unlock_page(page);
3815 		return 0;
3816 	}
3817 
3818 	if (page->index == end_index) {
3819 		char *userpage;
3820 
3821 		userpage = kmap_atomic(page);
3822 		memset(userpage + pg_offset, 0,
3823 		       PAGE_SIZE - pg_offset);
3824 		kunmap_atomic(userpage);
3825 		flush_dcache_page(page);
3826 	}
3827 
3828 	ret = set_page_extent_mapped(page);
3829 	if (ret < 0) {
3830 		SetPageError(page);
3831 		goto done;
3832 	}
3833 
3834 	if (!epd->extent_locked) {
3835 		ret = writepage_delalloc(BTRFS_I(inode), page, wbc, start,
3836 					 &nr_written);
3837 		if (ret == 1)
3838 			return 0;
3839 		if (ret)
3840 			goto done;
3841 	}
3842 
3843 	ret = __extent_writepage_io(BTRFS_I(inode), page, wbc, epd, i_size,
3844 				    nr_written, &nr);
3845 	if (ret == 1)
3846 		return 0;
3847 
3848 done:
3849 	if (nr == 0) {
3850 		/* make sure the mapping tag for page dirty gets cleared */
3851 		set_page_writeback(page);
3852 		end_page_writeback(page);
3853 	}
3854 	if (PageError(page)) {
3855 		ret = ret < 0 ? ret : -EIO;
3856 		end_extent_writepage(page, ret, start, page_end);
3857 	}
3858 	unlock_page(page);
3859 	ASSERT(ret <= 0);
3860 	return ret;
3861 }
3862 
3863 void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
3864 {
3865 	wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_WRITEBACK,
3866 		       TASK_UNINTERRUPTIBLE);
3867 }
3868 
3869 static void end_extent_buffer_writeback(struct extent_buffer *eb)
3870 {
3871 	clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
3872 	smp_mb__after_atomic();
3873 	wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
3874 }
3875 
3876 /*
3877  * Lock extent buffer status and pages for writeback.
3878  *
3879  * May try to flush write bio if we can't get the lock.
3880  *
3881  * Return  0 if the extent buffer doesn't need to be submitted.
3882  *           (E.g. the extent buffer is not dirty)
3883  * Return >0 is the extent buffer is submitted to bio.
3884  * Return <0 if something went wrong, no page is locked.
3885  */
3886 static noinline_for_stack int lock_extent_buffer_for_io(struct extent_buffer *eb,
3887 			  struct extent_page_data *epd)
3888 {
3889 	struct btrfs_fs_info *fs_info = eb->fs_info;
3890 	int i, num_pages, failed_page_nr;
3891 	int flush = 0;
3892 	int ret = 0;
3893 
3894 	if (!btrfs_try_tree_write_lock(eb)) {
3895 		ret = flush_write_bio(epd);
3896 		if (ret < 0)
3897 			return ret;
3898 		flush = 1;
3899 		btrfs_tree_lock(eb);
3900 	}
3901 
3902 	if (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
3903 		btrfs_tree_unlock(eb);
3904 		if (!epd->sync_io)
3905 			return 0;
3906 		if (!flush) {
3907 			ret = flush_write_bio(epd);
3908 			if (ret < 0)
3909 				return ret;
3910 			flush = 1;
3911 		}
3912 		while (1) {
3913 			wait_on_extent_buffer_writeback(eb);
3914 			btrfs_tree_lock(eb);
3915 			if (!test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags))
3916 				break;
3917 			btrfs_tree_unlock(eb);
3918 		}
3919 	}
3920 
3921 	/*
3922 	 * We need to do this to prevent races in people who check if the eb is
3923 	 * under IO since we can end up having no IO bits set for a short period
3924 	 * of time.
3925 	 */
3926 	spin_lock(&eb->refs_lock);
3927 	if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
3928 		set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
3929 		spin_unlock(&eb->refs_lock);
3930 		btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
3931 		percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
3932 					 -eb->len,
3933 					 fs_info->dirty_metadata_batch);
3934 		ret = 1;
3935 	} else {
3936 		spin_unlock(&eb->refs_lock);
3937 	}
3938 
3939 	btrfs_tree_unlock(eb);
3940 
3941 	if (!ret)
3942 		return ret;
3943 
3944 	num_pages = num_extent_pages(eb);
3945 	for (i = 0; i < num_pages; i++) {
3946 		struct page *p = eb->pages[i];
3947 
3948 		if (!trylock_page(p)) {
3949 			if (!flush) {
3950 				int err;
3951 
3952 				err = flush_write_bio(epd);
3953 				if (err < 0) {
3954 					ret = err;
3955 					failed_page_nr = i;
3956 					goto err_unlock;
3957 				}
3958 				flush = 1;
3959 			}
3960 			lock_page(p);
3961 		}
3962 	}
3963 
3964 	return ret;
3965 err_unlock:
3966 	/* Unlock already locked pages */
3967 	for (i = 0; i < failed_page_nr; i++)
3968 		unlock_page(eb->pages[i]);
3969 	/*
3970 	 * Clear EXTENT_BUFFER_WRITEBACK and wake up anyone waiting on it.
3971 	 * Also set back EXTENT_BUFFER_DIRTY so future attempts to this eb can
3972 	 * be made and undo everything done before.
3973 	 */
3974 	btrfs_tree_lock(eb);
3975 	spin_lock(&eb->refs_lock);
3976 	set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
3977 	end_extent_buffer_writeback(eb);
3978 	spin_unlock(&eb->refs_lock);
3979 	percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, eb->len,
3980 				 fs_info->dirty_metadata_batch);
3981 	btrfs_clear_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
3982 	btrfs_tree_unlock(eb);
3983 	return ret;
3984 }
3985 
3986 static void set_btree_ioerr(struct page *page)
3987 {
3988 	struct extent_buffer *eb = (struct extent_buffer *)page->private;
3989 	struct btrfs_fs_info *fs_info;
3990 
3991 	SetPageError(page);
3992 	if (test_and_set_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags))
3993 		return;
3994 
3995 	/*
3996 	 * If we error out, we should add back the dirty_metadata_bytes
3997 	 * to make it consistent.
3998 	 */
3999 	fs_info = eb->fs_info;
4000 	percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
4001 				 eb->len, fs_info->dirty_metadata_batch);
4002 
4003 	/*
4004 	 * If writeback for a btree extent that doesn't belong to a log tree
4005 	 * failed, increment the counter transaction->eb_write_errors.
4006 	 * We do this because while the transaction is running and before it's
4007 	 * committing (when we call filemap_fdata[write|wait]_range against
4008 	 * the btree inode), we might have
4009 	 * btree_inode->i_mapping->a_ops->writepages() called by the VM - if it
4010 	 * returns an error or an error happens during writeback, when we're
4011 	 * committing the transaction we wouldn't know about it, since the pages
4012 	 * can be no longer dirty nor marked anymore for writeback (if a
4013 	 * subsequent modification to the extent buffer didn't happen before the
4014 	 * transaction commit), which makes filemap_fdata[write|wait]_range not
4015 	 * able to find the pages tagged with SetPageError at transaction
4016 	 * commit time. So if this happens we must abort the transaction,
4017 	 * otherwise we commit a super block with btree roots that point to
4018 	 * btree nodes/leafs whose content on disk is invalid - either garbage
4019 	 * or the content of some node/leaf from a past generation that got
4020 	 * cowed or deleted and is no longer valid.
4021 	 *
4022 	 * Note: setting AS_EIO/AS_ENOSPC in the btree inode's i_mapping would
4023 	 * not be enough - we need to distinguish between log tree extents vs
4024 	 * non-log tree extents, and the next filemap_fdatawait_range() call
4025 	 * will catch and clear such errors in the mapping - and that call might
4026 	 * be from a log sync and not from a transaction commit. Also, checking
4027 	 * for the eb flag EXTENT_BUFFER_WRITE_ERR at transaction commit time is
4028 	 * not done and would not be reliable - the eb might have been released
4029 	 * from memory and reading it back again means that flag would not be
4030 	 * set (since it's a runtime flag, not persisted on disk).
4031 	 *
4032 	 * Using the flags below in the btree inode also makes us achieve the
4033 	 * goal of AS_EIO/AS_ENOSPC when writepages() returns success, started
4034 	 * writeback for all dirty pages and before filemap_fdatawait_range()
4035 	 * is called, the writeback for all dirty pages had already finished
4036 	 * with errors - because we were not using AS_EIO/AS_ENOSPC,
4037 	 * filemap_fdatawait_range() would return success, as it could not know
4038 	 * that writeback errors happened (the pages were no longer tagged for
4039 	 * writeback).
4040 	 */
4041 	switch (eb->log_index) {
4042 	case -1:
4043 		set_bit(BTRFS_FS_BTREE_ERR, &eb->fs_info->flags);
4044 		break;
4045 	case 0:
4046 		set_bit(BTRFS_FS_LOG1_ERR, &eb->fs_info->flags);
4047 		break;
4048 	case 1:
4049 		set_bit(BTRFS_FS_LOG2_ERR, &eb->fs_info->flags);
4050 		break;
4051 	default:
4052 		BUG(); /* unexpected, logic error */
4053 	}
4054 }
4055 
4056 static void end_bio_extent_buffer_writepage(struct bio *bio)
4057 {
4058 	struct bio_vec *bvec;
4059 	struct extent_buffer *eb;
4060 	int done;
4061 	struct bvec_iter_all iter_all;
4062 
4063 	ASSERT(!bio_flagged(bio, BIO_CLONED));
4064 	bio_for_each_segment_all(bvec, bio, iter_all) {
4065 		struct page *page = bvec->bv_page;
4066 
4067 		eb = (struct extent_buffer *)page->private;
4068 		BUG_ON(!eb);
4069 		done = atomic_dec_and_test(&eb->io_pages);
4070 
4071 		if (bio->bi_status ||
4072 		    test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)) {
4073 			ClearPageUptodate(page);
4074 			set_btree_ioerr(page);
4075 		}
4076 
4077 		end_page_writeback(page);
4078 
4079 		if (!done)
4080 			continue;
4081 
4082 		end_extent_buffer_writeback(eb);
4083 	}
4084 
4085 	bio_put(bio);
4086 }
4087 
4088 static noinline_for_stack int write_one_eb(struct extent_buffer *eb,
4089 			struct writeback_control *wbc,
4090 			struct extent_page_data *epd)
4091 {
4092 	u64 disk_bytenr = eb->start;
4093 	u32 nritems;
4094 	int i, num_pages;
4095 	unsigned long start, end;
4096 	unsigned int write_flags = wbc_to_write_flags(wbc) | REQ_META;
4097 	int ret = 0;
4098 
4099 	clear_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags);
4100 	num_pages = num_extent_pages(eb);
4101 	atomic_set(&eb->io_pages, num_pages);
4102 
4103 	/* set btree blocks beyond nritems with 0 to avoid stale content. */
4104 	nritems = btrfs_header_nritems(eb);
4105 	if (btrfs_header_level(eb) > 0) {
4106 		end = btrfs_node_key_ptr_offset(nritems);
4107 
4108 		memzero_extent_buffer(eb, end, eb->len - end);
4109 	} else {
4110 		/*
4111 		 * leaf:
4112 		 * header 0 1 2 .. N ... data_N .. data_2 data_1 data_0
4113 		 */
4114 		start = btrfs_item_nr_offset(nritems);
4115 		end = BTRFS_LEAF_DATA_OFFSET + leaf_data_end(eb);
4116 		memzero_extent_buffer(eb, start, end - start);
4117 	}
4118 
4119 	for (i = 0; i < num_pages; i++) {
4120 		struct page *p = eb->pages[i];
4121 
4122 		clear_page_dirty_for_io(p);
4123 		set_page_writeback(p);
4124 		ret = submit_extent_page(REQ_OP_WRITE | write_flags, wbc,
4125 					 p, disk_bytenr, PAGE_SIZE, 0,
4126 					 &epd->bio,
4127 					 end_bio_extent_buffer_writepage,
4128 					 0, 0, 0, false);
4129 		if (ret) {
4130 			set_btree_ioerr(p);
4131 			if (PageWriteback(p))
4132 				end_page_writeback(p);
4133 			if (atomic_sub_and_test(num_pages - i, &eb->io_pages))
4134 				end_extent_buffer_writeback(eb);
4135 			ret = -EIO;
4136 			break;
4137 		}
4138 		disk_bytenr += PAGE_SIZE;
4139 		update_nr_written(wbc, 1);
4140 		unlock_page(p);
4141 	}
4142 
4143 	if (unlikely(ret)) {
4144 		for (; i < num_pages; i++) {
4145 			struct page *p = eb->pages[i];
4146 			clear_page_dirty_for_io(p);
4147 			unlock_page(p);
4148 		}
4149 	}
4150 
4151 	return ret;
4152 }
4153 
4154 /*
4155  * Submit all page(s) of one extent buffer.
4156  *
4157  * @page:	the page of one extent buffer
4158  * @eb_context:	to determine if we need to submit this page, if current page
4159  *		belongs to this eb, we don't need to submit
4160  *
4161  * The caller should pass each page in their bytenr order, and here we use
4162  * @eb_context to determine if we have submitted pages of one extent buffer.
4163  *
4164  * If we have, we just skip until we hit a new page that doesn't belong to
4165  * current @eb_context.
4166  *
4167  * If not, we submit all the page(s) of the extent buffer.
4168  *
4169  * Return >0 if we have submitted the extent buffer successfully.
4170  * Return 0 if we don't need to submit the page, as it's already submitted by
4171  * previous call.
4172  * Return <0 for fatal error.
4173  */
4174 static int submit_eb_page(struct page *page, struct writeback_control *wbc,
4175 			  struct extent_page_data *epd,
4176 			  struct extent_buffer **eb_context)
4177 {
4178 	struct address_space *mapping = page->mapping;
4179 	struct btrfs_block_group *cache = NULL;
4180 	struct extent_buffer *eb;
4181 	int ret;
4182 
4183 	if (!PagePrivate(page))
4184 		return 0;
4185 
4186 	spin_lock(&mapping->private_lock);
4187 	if (!PagePrivate(page)) {
4188 		spin_unlock(&mapping->private_lock);
4189 		return 0;
4190 	}
4191 
4192 	eb = (struct extent_buffer *)page->private;
4193 
4194 	/*
4195 	 * Shouldn't happen and normally this would be a BUG_ON but no point
4196 	 * crashing the machine for something we can survive anyway.
4197 	 */
4198 	if (WARN_ON(!eb)) {
4199 		spin_unlock(&mapping->private_lock);
4200 		return 0;
4201 	}
4202 
4203 	if (eb == *eb_context) {
4204 		spin_unlock(&mapping->private_lock);
4205 		return 0;
4206 	}
4207 	ret = atomic_inc_not_zero(&eb->refs);
4208 	spin_unlock(&mapping->private_lock);
4209 	if (!ret)
4210 		return 0;
4211 
4212 	if (!btrfs_check_meta_write_pointer(eb->fs_info, eb, &cache)) {
4213 		/*
4214 		 * If for_sync, this hole will be filled with
4215 		 * trasnsaction commit.
4216 		 */
4217 		if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync)
4218 			ret = -EAGAIN;
4219 		else
4220 			ret = 0;
4221 		free_extent_buffer(eb);
4222 		return ret;
4223 	}
4224 
4225 	*eb_context = eb;
4226 
4227 	ret = lock_extent_buffer_for_io(eb, epd);
4228 	if (ret <= 0) {
4229 		btrfs_revert_meta_write_pointer(cache, eb);
4230 		if (cache)
4231 			btrfs_put_block_group(cache);
4232 		free_extent_buffer(eb);
4233 		return ret;
4234 	}
4235 	if (cache)
4236 		btrfs_put_block_group(cache);
4237 	ret = write_one_eb(eb, wbc, epd);
4238 	free_extent_buffer(eb);
4239 	if (ret < 0)
4240 		return ret;
4241 	return 1;
4242 }
4243 
4244 int btree_write_cache_pages(struct address_space *mapping,
4245 				   struct writeback_control *wbc)
4246 {
4247 	struct extent_buffer *eb_context = NULL;
4248 	struct extent_page_data epd = {
4249 		.bio = NULL,
4250 		.extent_locked = 0,
4251 		.sync_io = wbc->sync_mode == WB_SYNC_ALL,
4252 	};
4253 	struct btrfs_fs_info *fs_info = BTRFS_I(mapping->host)->root->fs_info;
4254 	int ret = 0;
4255 	int done = 0;
4256 	int nr_to_write_done = 0;
4257 	struct pagevec pvec;
4258 	int nr_pages;
4259 	pgoff_t index;
4260 	pgoff_t end;		/* Inclusive */
4261 	int scanned = 0;
4262 	xa_mark_t tag;
4263 
4264 	pagevec_init(&pvec);
4265 	if (wbc->range_cyclic) {
4266 		index = mapping->writeback_index; /* Start from prev offset */
4267 		end = -1;
4268 		/*
4269 		 * Start from the beginning does not need to cycle over the
4270 		 * range, mark it as scanned.
4271 		 */
4272 		scanned = (index == 0);
4273 	} else {
4274 		index = wbc->range_start >> PAGE_SHIFT;
4275 		end = wbc->range_end >> PAGE_SHIFT;
4276 		scanned = 1;
4277 	}
4278 	if (wbc->sync_mode == WB_SYNC_ALL)
4279 		tag = PAGECACHE_TAG_TOWRITE;
4280 	else
4281 		tag = PAGECACHE_TAG_DIRTY;
4282 	btrfs_zoned_meta_io_lock(fs_info);
4283 retry:
4284 	if (wbc->sync_mode == WB_SYNC_ALL)
4285 		tag_pages_for_writeback(mapping, index, end);
4286 	while (!done && !nr_to_write_done && (index <= end) &&
4287 	       (nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end,
4288 			tag))) {
4289 		unsigned i;
4290 
4291 		for (i = 0; i < nr_pages; i++) {
4292 			struct page *page = pvec.pages[i];
4293 
4294 			ret = submit_eb_page(page, wbc, &epd, &eb_context);
4295 			if (ret == 0)
4296 				continue;
4297 			if (ret < 0) {
4298 				done = 1;
4299 				break;
4300 			}
4301 
4302 			/*
4303 			 * the filesystem may choose to bump up nr_to_write.
4304 			 * We have to make sure to honor the new nr_to_write
4305 			 * at any time
4306 			 */
4307 			nr_to_write_done = wbc->nr_to_write <= 0;
4308 		}
4309 		pagevec_release(&pvec);
4310 		cond_resched();
4311 	}
4312 	if (!scanned && !done) {
4313 		/*
4314 		 * We hit the last page and there is more work to be done: wrap
4315 		 * back to the start of the file
4316 		 */
4317 		scanned = 1;
4318 		index = 0;
4319 		goto retry;
4320 	}
4321 	if (ret < 0) {
4322 		end_write_bio(&epd, ret);
4323 		goto out;
4324 	}
4325 	/*
4326 	 * If something went wrong, don't allow any metadata write bio to be
4327 	 * submitted.
4328 	 *
4329 	 * This would prevent use-after-free if we had dirty pages not
4330 	 * cleaned up, which can still happen by fuzzed images.
4331 	 *
4332 	 * - Bad extent tree
4333 	 *   Allowing existing tree block to be allocated for other trees.
4334 	 *
4335 	 * - Log tree operations
4336 	 *   Exiting tree blocks get allocated to log tree, bumps its
4337 	 *   generation, then get cleaned in tree re-balance.
4338 	 *   Such tree block will not be written back, since it's clean,
4339 	 *   thus no WRITTEN flag set.
4340 	 *   And after log writes back, this tree block is not traced by
4341 	 *   any dirty extent_io_tree.
4342 	 *
4343 	 * - Offending tree block gets re-dirtied from its original owner
4344 	 *   Since it has bumped generation, no WRITTEN flag, it can be
4345 	 *   reused without COWing. This tree block will not be traced
4346 	 *   by btrfs_transaction::dirty_pages.
4347 	 *
4348 	 *   Now such dirty tree block will not be cleaned by any dirty
4349 	 *   extent io tree. Thus we don't want to submit such wild eb
4350 	 *   if the fs already has error.
4351 	 */
4352 	if (!test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) {
4353 		ret = flush_write_bio(&epd);
4354 	} else {
4355 		ret = -EROFS;
4356 		end_write_bio(&epd, ret);
4357 	}
4358 out:
4359 	btrfs_zoned_meta_io_unlock(fs_info);
4360 	return ret;
4361 }
4362 
4363 /**
4364  * Walk the list of dirty pages of the given address space and write all of them.
4365  *
4366  * @mapping: address space structure to write
4367  * @wbc:     subtract the number of written pages from *@wbc->nr_to_write
4368  * @epd:     holds context for the write, namely the bio
4369  *
4370  * If a page is already under I/O, write_cache_pages() skips it, even
4371  * if it's dirty.  This is desirable behaviour for memory-cleaning writeback,
4372  * but it is INCORRECT for data-integrity system calls such as fsync().  fsync()
4373  * and msync() need to guarantee that all the data which was dirty at the time
4374  * the call was made get new I/O started against them.  If wbc->sync_mode is
4375  * WB_SYNC_ALL then we were called for data integrity and we must wait for
4376  * existing IO to complete.
4377  */
4378 static int extent_write_cache_pages(struct address_space *mapping,
4379 			     struct writeback_control *wbc,
4380 			     struct extent_page_data *epd)
4381 {
4382 	struct inode *inode = mapping->host;
4383 	int ret = 0;
4384 	int done = 0;
4385 	int nr_to_write_done = 0;
4386 	struct pagevec pvec;
4387 	int nr_pages;
4388 	pgoff_t index;
4389 	pgoff_t end;		/* Inclusive */
4390 	pgoff_t done_index;
4391 	int range_whole = 0;
4392 	int scanned = 0;
4393 	xa_mark_t tag;
4394 
4395 	/*
4396 	 * We have to hold onto the inode so that ordered extents can do their
4397 	 * work when the IO finishes.  The alternative to this is failing to add
4398 	 * an ordered extent if the igrab() fails there and that is a huge pain
4399 	 * to deal with, so instead just hold onto the inode throughout the
4400 	 * writepages operation.  If it fails here we are freeing up the inode
4401 	 * anyway and we'd rather not waste our time writing out stuff that is
4402 	 * going to be truncated anyway.
4403 	 */
4404 	if (!igrab(inode))
4405 		return 0;
4406 
4407 	pagevec_init(&pvec);
4408 	if (wbc->range_cyclic) {
4409 		index = mapping->writeback_index; /* Start from prev offset */
4410 		end = -1;
4411 		/*
4412 		 * Start from the beginning does not need to cycle over the
4413 		 * range, mark it as scanned.
4414 		 */
4415 		scanned = (index == 0);
4416 	} else {
4417 		index = wbc->range_start >> PAGE_SHIFT;
4418 		end = wbc->range_end >> PAGE_SHIFT;
4419 		if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
4420 			range_whole = 1;
4421 		scanned = 1;
4422 	}
4423 
4424 	/*
4425 	 * We do the tagged writepage as long as the snapshot flush bit is set
4426 	 * and we are the first one who do the filemap_flush() on this inode.
4427 	 *
4428 	 * The nr_to_write == LONG_MAX is needed to make sure other flushers do
4429 	 * not race in and drop the bit.
4430 	 */
4431 	if (range_whole && wbc->nr_to_write == LONG_MAX &&
4432 	    test_and_clear_bit(BTRFS_INODE_SNAPSHOT_FLUSH,
4433 			       &BTRFS_I(inode)->runtime_flags))
4434 		wbc->tagged_writepages = 1;
4435 
4436 	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
4437 		tag = PAGECACHE_TAG_TOWRITE;
4438 	else
4439 		tag = PAGECACHE_TAG_DIRTY;
4440 retry:
4441 	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
4442 		tag_pages_for_writeback(mapping, index, end);
4443 	done_index = index;
4444 	while (!done && !nr_to_write_done && (index <= end) &&
4445 			(nr_pages = pagevec_lookup_range_tag(&pvec, mapping,
4446 						&index, end, tag))) {
4447 		unsigned i;
4448 
4449 		for (i = 0; i < nr_pages; i++) {
4450 			struct page *page = pvec.pages[i];
4451 
4452 			done_index = page->index + 1;
4453 			/*
4454 			 * At this point we hold neither the i_pages lock nor
4455 			 * the page lock: the page may be truncated or
4456 			 * invalidated (changing page->mapping to NULL),
4457 			 * or even swizzled back from swapper_space to
4458 			 * tmpfs file mapping
4459 			 */
4460 			if (!trylock_page(page)) {
4461 				ret = flush_write_bio(epd);
4462 				BUG_ON(ret < 0);
4463 				lock_page(page);
4464 			}
4465 
4466 			if (unlikely(page->mapping != mapping)) {
4467 				unlock_page(page);
4468 				continue;
4469 			}
4470 
4471 			if (wbc->sync_mode != WB_SYNC_NONE) {
4472 				if (PageWriteback(page)) {
4473 					ret = flush_write_bio(epd);
4474 					BUG_ON(ret < 0);
4475 				}
4476 				wait_on_page_writeback(page);
4477 			}
4478 
4479 			if (PageWriteback(page) ||
4480 			    !clear_page_dirty_for_io(page)) {
4481 				unlock_page(page);
4482 				continue;
4483 			}
4484 
4485 			ret = __extent_writepage(page, wbc, epd);
4486 			if (ret < 0) {
4487 				done = 1;
4488 				break;
4489 			}
4490 
4491 			/*
4492 			 * the filesystem may choose to bump up nr_to_write.
4493 			 * We have to make sure to honor the new nr_to_write
4494 			 * at any time
4495 			 */
4496 			nr_to_write_done = wbc->nr_to_write <= 0;
4497 		}
4498 		pagevec_release(&pvec);
4499 		cond_resched();
4500 	}
4501 	if (!scanned && !done) {
4502 		/*
4503 		 * We hit the last page and there is more work to be done: wrap
4504 		 * back to the start of the file
4505 		 */
4506 		scanned = 1;
4507 		index = 0;
4508 
4509 		/*
4510 		 * If we're looping we could run into a page that is locked by a
4511 		 * writer and that writer could be waiting on writeback for a
4512 		 * page in our current bio, and thus deadlock, so flush the
4513 		 * write bio here.
4514 		 */
4515 		ret = flush_write_bio(epd);
4516 		if (!ret)
4517 			goto retry;
4518 	}
4519 
4520 	if (wbc->range_cyclic || (wbc->nr_to_write > 0 && range_whole))
4521 		mapping->writeback_index = done_index;
4522 
4523 	btrfs_add_delayed_iput(inode);
4524 	return ret;
4525 }
4526 
4527 int extent_write_full_page(struct page *page, struct writeback_control *wbc)
4528 {
4529 	int ret;
4530 	struct extent_page_data epd = {
4531 		.bio = NULL,
4532 		.extent_locked = 0,
4533 		.sync_io = wbc->sync_mode == WB_SYNC_ALL,
4534 	};
4535 
4536 	ret = __extent_writepage(page, wbc, &epd);
4537 	ASSERT(ret <= 0);
4538 	if (ret < 0) {
4539 		end_write_bio(&epd, ret);
4540 		return ret;
4541 	}
4542 
4543 	ret = flush_write_bio(&epd);
4544 	ASSERT(ret <= 0);
4545 	return ret;
4546 }
4547 
4548 int extent_write_locked_range(struct inode *inode, u64 start, u64 end,
4549 			      int mode)
4550 {
4551 	int ret = 0;
4552 	struct address_space *mapping = inode->i_mapping;
4553 	struct page *page;
4554 	unsigned long nr_pages = (end - start + PAGE_SIZE) >>
4555 		PAGE_SHIFT;
4556 
4557 	struct extent_page_data epd = {
4558 		.bio = NULL,
4559 		.extent_locked = 1,
4560 		.sync_io = mode == WB_SYNC_ALL,
4561 	};
4562 	struct writeback_control wbc_writepages = {
4563 		.sync_mode	= mode,
4564 		.nr_to_write	= nr_pages * 2,
4565 		.range_start	= start,
4566 		.range_end	= end + 1,
4567 		/* We're called from an async helper function */
4568 		.punt_to_cgroup	= 1,
4569 		.no_cgroup_owner = 1,
4570 	};
4571 
4572 	wbc_attach_fdatawrite_inode(&wbc_writepages, inode);
4573 	while (start <= end) {
4574 		page = find_get_page(mapping, start >> PAGE_SHIFT);
4575 		if (clear_page_dirty_for_io(page))
4576 			ret = __extent_writepage(page, &wbc_writepages, &epd);
4577 		else {
4578 			btrfs_writepage_endio_finish_ordered(page, start,
4579 						    start + PAGE_SIZE - 1, 1);
4580 			unlock_page(page);
4581 		}
4582 		put_page(page);
4583 		start += PAGE_SIZE;
4584 	}
4585 
4586 	ASSERT(ret <= 0);
4587 	if (ret == 0)
4588 		ret = flush_write_bio(&epd);
4589 	else
4590 		end_write_bio(&epd, ret);
4591 
4592 	wbc_detach_inode(&wbc_writepages);
4593 	return ret;
4594 }
4595 
4596 int extent_writepages(struct address_space *mapping,
4597 		      struct writeback_control *wbc)
4598 {
4599 	int ret = 0;
4600 	struct extent_page_data epd = {
4601 		.bio = NULL,
4602 		.extent_locked = 0,
4603 		.sync_io = wbc->sync_mode == WB_SYNC_ALL,
4604 	};
4605 
4606 	ret = extent_write_cache_pages(mapping, wbc, &epd);
4607 	ASSERT(ret <= 0);
4608 	if (ret < 0) {
4609 		end_write_bio(&epd, ret);
4610 		return ret;
4611 	}
4612 	ret = flush_write_bio(&epd);
4613 	return ret;
4614 }
4615 
4616 void extent_readahead(struct readahead_control *rac)
4617 {
4618 	struct bio *bio = NULL;
4619 	unsigned long bio_flags = 0;
4620 	struct page *pagepool[16];
4621 	struct extent_map *em_cached = NULL;
4622 	u64 prev_em_start = (u64)-1;
4623 	int nr;
4624 
4625 	while ((nr = readahead_page_batch(rac, pagepool))) {
4626 		u64 contig_start = page_offset(pagepool[0]);
4627 		u64 contig_end = page_offset(pagepool[nr - 1]) + PAGE_SIZE - 1;
4628 
4629 		ASSERT(contig_start + nr * PAGE_SIZE - 1 == contig_end);
4630 
4631 		contiguous_readpages(pagepool, nr, contig_start, contig_end,
4632 				&em_cached, &bio, &bio_flags, &prev_em_start);
4633 	}
4634 
4635 	if (em_cached)
4636 		free_extent_map(em_cached);
4637 
4638 	if (bio) {
4639 		if (submit_one_bio(bio, 0, bio_flags))
4640 			return;
4641 	}
4642 }
4643 
4644 /*
4645  * basic invalidatepage code, this waits on any locked or writeback
4646  * ranges corresponding to the page, and then deletes any extent state
4647  * records from the tree
4648  */
4649 int extent_invalidatepage(struct extent_io_tree *tree,
4650 			  struct page *page, unsigned long offset)
4651 {
4652 	struct extent_state *cached_state = NULL;
4653 	u64 start = page_offset(page);
4654 	u64 end = start + PAGE_SIZE - 1;
4655 	size_t blocksize = page->mapping->host->i_sb->s_blocksize;
4656 
4657 	/* This function is only called for the btree inode */
4658 	ASSERT(tree->owner == IO_TREE_BTREE_INODE_IO);
4659 
4660 	start += ALIGN(offset, blocksize);
4661 	if (start > end)
4662 		return 0;
4663 
4664 	lock_extent_bits(tree, start, end, &cached_state);
4665 	wait_on_page_writeback(page);
4666 
4667 	/*
4668 	 * Currently for btree io tree, only EXTENT_LOCKED is utilized,
4669 	 * so here we only need to unlock the extent range to free any
4670 	 * existing extent state.
4671 	 */
4672 	unlock_extent_cached(tree, start, end, &cached_state);
4673 	return 0;
4674 }
4675 
4676 /*
4677  * a helper for releasepage, this tests for areas of the page that
4678  * are locked or under IO and drops the related state bits if it is safe
4679  * to drop the page.
4680  */
4681 static int try_release_extent_state(struct extent_io_tree *tree,
4682 				    struct page *page, gfp_t mask)
4683 {
4684 	u64 start = page_offset(page);
4685 	u64 end = start + PAGE_SIZE - 1;
4686 	int ret = 1;
4687 
4688 	if (test_range_bit(tree, start, end, EXTENT_LOCKED, 0, NULL)) {
4689 		ret = 0;
4690 	} else {
4691 		/*
4692 		 * At this point we can safely clear everything except the
4693 		 * locked bit, the nodatasum bit and the delalloc new bit.
4694 		 * The delalloc new bit will be cleared by ordered extent
4695 		 * completion.
4696 		 */
4697 		ret = __clear_extent_bit(tree, start, end,
4698 			 ~(EXTENT_LOCKED | EXTENT_NODATASUM | EXTENT_DELALLOC_NEW),
4699 			 0, 0, NULL, mask, NULL);
4700 
4701 		/* if clear_extent_bit failed for enomem reasons,
4702 		 * we can't allow the release to continue.
4703 		 */
4704 		if (ret < 0)
4705 			ret = 0;
4706 		else
4707 			ret = 1;
4708 	}
4709 	return ret;
4710 }
4711 
4712 /*
4713  * a helper for releasepage.  As long as there are no locked extents
4714  * in the range corresponding to the page, both state records and extent
4715  * map records are removed
4716  */
4717 int try_release_extent_mapping(struct page *page, gfp_t mask)
4718 {
4719 	struct extent_map *em;
4720 	u64 start = page_offset(page);
4721 	u64 end = start + PAGE_SIZE - 1;
4722 	struct btrfs_inode *btrfs_inode = BTRFS_I(page->mapping->host);
4723 	struct extent_io_tree *tree = &btrfs_inode->io_tree;
4724 	struct extent_map_tree *map = &btrfs_inode->extent_tree;
4725 
4726 	if (gfpflags_allow_blocking(mask) &&
4727 	    page->mapping->host->i_size > SZ_16M) {
4728 		u64 len;
4729 		while (start <= end) {
4730 			struct btrfs_fs_info *fs_info;
4731 			u64 cur_gen;
4732 
4733 			len = end - start + 1;
4734 			write_lock(&map->lock);
4735 			em = lookup_extent_mapping(map, start, len);
4736 			if (!em) {
4737 				write_unlock(&map->lock);
4738 				break;
4739 			}
4740 			if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
4741 			    em->start != start) {
4742 				write_unlock(&map->lock);
4743 				free_extent_map(em);
4744 				break;
4745 			}
4746 			if (test_range_bit(tree, em->start,
4747 					   extent_map_end(em) - 1,
4748 					   EXTENT_LOCKED, 0, NULL))
4749 				goto next;
4750 			/*
4751 			 * If it's not in the list of modified extents, used
4752 			 * by a fast fsync, we can remove it. If it's being
4753 			 * logged we can safely remove it since fsync took an
4754 			 * extra reference on the em.
4755 			 */
4756 			if (list_empty(&em->list) ||
4757 			    test_bit(EXTENT_FLAG_LOGGING, &em->flags))
4758 				goto remove_em;
4759 			/*
4760 			 * If it's in the list of modified extents, remove it
4761 			 * only if its generation is older then the current one,
4762 			 * in which case we don't need it for a fast fsync.
4763 			 * Otherwise don't remove it, we could be racing with an
4764 			 * ongoing fast fsync that could miss the new extent.
4765 			 */
4766 			fs_info = btrfs_inode->root->fs_info;
4767 			spin_lock(&fs_info->trans_lock);
4768 			cur_gen = fs_info->generation;
4769 			spin_unlock(&fs_info->trans_lock);
4770 			if (em->generation >= cur_gen)
4771 				goto next;
4772 remove_em:
4773 			/*
4774 			 * We only remove extent maps that are not in the list of
4775 			 * modified extents or that are in the list but with a
4776 			 * generation lower then the current generation, so there
4777 			 * is no need to set the full fsync flag on the inode (it
4778 			 * hurts the fsync performance for workloads with a data
4779 			 * size that exceeds or is close to the system's memory).
4780 			 */
4781 			remove_extent_mapping(map, em);
4782 			/* once for the rb tree */
4783 			free_extent_map(em);
4784 next:
4785 			start = extent_map_end(em);
4786 			write_unlock(&map->lock);
4787 
4788 			/* once for us */
4789 			free_extent_map(em);
4790 
4791 			cond_resched(); /* Allow large-extent preemption. */
4792 		}
4793 	}
4794 	return try_release_extent_state(tree, page, mask);
4795 }
4796 
4797 /*
4798  * helper function for fiemap, which doesn't want to see any holes.
4799  * This maps until we find something past 'last'
4800  */
4801 static struct extent_map *get_extent_skip_holes(struct btrfs_inode *inode,
4802 						u64 offset, u64 last)
4803 {
4804 	u64 sectorsize = btrfs_inode_sectorsize(inode);
4805 	struct extent_map *em;
4806 	u64 len;
4807 
4808 	if (offset >= last)
4809 		return NULL;
4810 
4811 	while (1) {
4812 		len = last - offset;
4813 		if (len == 0)
4814 			break;
4815 		len = ALIGN(len, sectorsize);
4816 		em = btrfs_get_extent_fiemap(inode, offset, len);
4817 		if (IS_ERR_OR_NULL(em))
4818 			return em;
4819 
4820 		/* if this isn't a hole return it */
4821 		if (em->block_start != EXTENT_MAP_HOLE)
4822 			return em;
4823 
4824 		/* this is a hole, advance to the next extent */
4825 		offset = extent_map_end(em);
4826 		free_extent_map(em);
4827 		if (offset >= last)
4828 			break;
4829 	}
4830 	return NULL;
4831 }
4832 
4833 /*
4834  * To cache previous fiemap extent
4835  *
4836  * Will be used for merging fiemap extent
4837  */
4838 struct fiemap_cache {
4839 	u64 offset;
4840 	u64 phys;
4841 	u64 len;
4842 	u32 flags;
4843 	bool cached;
4844 };
4845 
4846 /*
4847  * Helper to submit fiemap extent.
4848  *
4849  * Will try to merge current fiemap extent specified by @offset, @phys,
4850  * @len and @flags with cached one.
4851  * And only when we fails to merge, cached one will be submitted as
4852  * fiemap extent.
4853  *
4854  * Return value is the same as fiemap_fill_next_extent().
4855  */
4856 static int emit_fiemap_extent(struct fiemap_extent_info *fieinfo,
4857 				struct fiemap_cache *cache,
4858 				u64 offset, u64 phys, u64 len, u32 flags)
4859 {
4860 	int ret = 0;
4861 
4862 	if (!cache->cached)
4863 		goto assign;
4864 
4865 	/*
4866 	 * Sanity check, extent_fiemap() should have ensured that new
4867 	 * fiemap extent won't overlap with cached one.
4868 	 * Not recoverable.
4869 	 *
4870 	 * NOTE: Physical address can overlap, due to compression
4871 	 */
4872 	if (cache->offset + cache->len > offset) {
4873 		WARN_ON(1);
4874 		return -EINVAL;
4875 	}
4876 
4877 	/*
4878 	 * Only merges fiemap extents if
4879 	 * 1) Their logical addresses are continuous
4880 	 *
4881 	 * 2) Their physical addresses are continuous
4882 	 *    So truly compressed (physical size smaller than logical size)
4883 	 *    extents won't get merged with each other
4884 	 *
4885 	 * 3) Share same flags except FIEMAP_EXTENT_LAST
4886 	 *    So regular extent won't get merged with prealloc extent
4887 	 */
4888 	if (cache->offset + cache->len  == offset &&
4889 	    cache->phys + cache->len == phys  &&
4890 	    (cache->flags & ~FIEMAP_EXTENT_LAST) ==
4891 			(flags & ~FIEMAP_EXTENT_LAST)) {
4892 		cache->len += len;
4893 		cache->flags |= flags;
4894 		goto try_submit_last;
4895 	}
4896 
4897 	/* Not mergeable, need to submit cached one */
4898 	ret = fiemap_fill_next_extent(fieinfo, cache->offset, cache->phys,
4899 				      cache->len, cache->flags);
4900 	cache->cached = false;
4901 	if (ret)
4902 		return ret;
4903 assign:
4904 	cache->cached = true;
4905 	cache->offset = offset;
4906 	cache->phys = phys;
4907 	cache->len = len;
4908 	cache->flags = flags;
4909 try_submit_last:
4910 	if (cache->flags & FIEMAP_EXTENT_LAST) {
4911 		ret = fiemap_fill_next_extent(fieinfo, cache->offset,
4912 				cache->phys, cache->len, cache->flags);
4913 		cache->cached = false;
4914 	}
4915 	return ret;
4916 }
4917 
4918 /*
4919  * Emit last fiemap cache
4920  *
4921  * The last fiemap cache may still be cached in the following case:
4922  * 0		      4k		    8k
4923  * |<- Fiemap range ->|
4924  * |<------------  First extent ----------->|
4925  *
4926  * In this case, the first extent range will be cached but not emitted.
4927  * So we must emit it before ending extent_fiemap().
4928  */
4929 static int emit_last_fiemap_cache(struct fiemap_extent_info *fieinfo,
4930 				  struct fiemap_cache *cache)
4931 {
4932 	int ret;
4933 
4934 	if (!cache->cached)
4935 		return 0;
4936 
4937 	ret = fiemap_fill_next_extent(fieinfo, cache->offset, cache->phys,
4938 				      cache->len, cache->flags);
4939 	cache->cached = false;
4940 	if (ret > 0)
4941 		ret = 0;
4942 	return ret;
4943 }
4944 
4945 int extent_fiemap(struct btrfs_inode *inode, struct fiemap_extent_info *fieinfo,
4946 		  u64 start, u64 len)
4947 {
4948 	int ret = 0;
4949 	u64 off = start;
4950 	u64 max = start + len;
4951 	u32 flags = 0;
4952 	u32 found_type;
4953 	u64 last;
4954 	u64 last_for_get_extent = 0;
4955 	u64 disko = 0;
4956 	u64 isize = i_size_read(&inode->vfs_inode);
4957 	struct btrfs_key found_key;
4958 	struct extent_map *em = NULL;
4959 	struct extent_state *cached_state = NULL;
4960 	struct btrfs_path *path;
4961 	struct btrfs_root *root = inode->root;
4962 	struct fiemap_cache cache = { 0 };
4963 	struct ulist *roots;
4964 	struct ulist *tmp_ulist;
4965 	int end = 0;
4966 	u64 em_start = 0;
4967 	u64 em_len = 0;
4968 	u64 em_end = 0;
4969 
4970 	if (len == 0)
4971 		return -EINVAL;
4972 
4973 	path = btrfs_alloc_path();
4974 	if (!path)
4975 		return -ENOMEM;
4976 
4977 	roots = ulist_alloc(GFP_KERNEL);
4978 	tmp_ulist = ulist_alloc(GFP_KERNEL);
4979 	if (!roots || !tmp_ulist) {
4980 		ret = -ENOMEM;
4981 		goto out_free_ulist;
4982 	}
4983 
4984 	start = round_down(start, btrfs_inode_sectorsize(inode));
4985 	len = round_up(max, btrfs_inode_sectorsize(inode)) - start;
4986 
4987 	/*
4988 	 * lookup the last file extent.  We're not using i_size here
4989 	 * because there might be preallocation past i_size
4990 	 */
4991 	ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(inode), -1,
4992 				       0);
4993 	if (ret < 0) {
4994 		goto out_free_ulist;
4995 	} else {
4996 		WARN_ON(!ret);
4997 		if (ret == 1)
4998 			ret = 0;
4999 	}
5000 
5001 	path->slots[0]--;
5002 	btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]);
5003 	found_type = found_key.type;
5004 
5005 	/* No extents, but there might be delalloc bits */
5006 	if (found_key.objectid != btrfs_ino(inode) ||
5007 	    found_type != BTRFS_EXTENT_DATA_KEY) {
5008 		/* have to trust i_size as the end */
5009 		last = (u64)-1;
5010 		last_for_get_extent = isize;
5011 	} else {
5012 		/*
5013 		 * remember the start of the last extent.  There are a
5014 		 * bunch of different factors that go into the length of the
5015 		 * extent, so its much less complex to remember where it started
5016 		 */
5017 		last = found_key.offset;
5018 		last_for_get_extent = last + 1;
5019 	}
5020 	btrfs_release_path(path);
5021 
5022 	/*
5023 	 * we might have some extents allocated but more delalloc past those
5024 	 * extents.  so, we trust isize unless the start of the last extent is
5025 	 * beyond isize
5026 	 */
5027 	if (last < isize) {
5028 		last = (u64)-1;
5029 		last_for_get_extent = isize;
5030 	}
5031 
5032 	lock_extent_bits(&inode->io_tree, start, start + len - 1,
5033 			 &cached_state);
5034 
5035 	em = get_extent_skip_holes(inode, start, last_for_get_extent);
5036 	if (!em)
5037 		goto out;
5038 	if (IS_ERR(em)) {
5039 		ret = PTR_ERR(em);
5040 		goto out;
5041 	}
5042 
5043 	while (!end) {
5044 		u64 offset_in_extent = 0;
5045 
5046 		/* break if the extent we found is outside the range */
5047 		if (em->start >= max || extent_map_end(em) < off)
5048 			break;
5049 
5050 		/*
5051 		 * get_extent may return an extent that starts before our
5052 		 * requested range.  We have to make sure the ranges
5053 		 * we return to fiemap always move forward and don't
5054 		 * overlap, so adjust the offsets here
5055 		 */
5056 		em_start = max(em->start, off);
5057 
5058 		/*
5059 		 * record the offset from the start of the extent
5060 		 * for adjusting the disk offset below.  Only do this if the
5061 		 * extent isn't compressed since our in ram offset may be past
5062 		 * what we have actually allocated on disk.
5063 		 */
5064 		if (!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
5065 			offset_in_extent = em_start - em->start;
5066 		em_end = extent_map_end(em);
5067 		em_len = em_end - em_start;
5068 		flags = 0;
5069 		if (em->block_start < EXTENT_MAP_LAST_BYTE)
5070 			disko = em->block_start + offset_in_extent;
5071 		else
5072 			disko = 0;
5073 
5074 		/*
5075 		 * bump off for our next call to get_extent
5076 		 */
5077 		off = extent_map_end(em);
5078 		if (off >= max)
5079 			end = 1;
5080 
5081 		if (em->block_start == EXTENT_MAP_LAST_BYTE) {
5082 			end = 1;
5083 			flags |= FIEMAP_EXTENT_LAST;
5084 		} else if (em->block_start == EXTENT_MAP_INLINE) {
5085 			flags |= (FIEMAP_EXTENT_DATA_INLINE |
5086 				  FIEMAP_EXTENT_NOT_ALIGNED);
5087 		} else if (em->block_start == EXTENT_MAP_DELALLOC) {
5088 			flags |= (FIEMAP_EXTENT_DELALLOC |
5089 				  FIEMAP_EXTENT_UNKNOWN);
5090 		} else if (fieinfo->fi_extents_max) {
5091 			u64 bytenr = em->block_start -
5092 				(em->start - em->orig_start);
5093 
5094 			/*
5095 			 * As btrfs supports shared space, this information
5096 			 * can be exported to userspace tools via
5097 			 * flag FIEMAP_EXTENT_SHARED.  If fi_extents_max == 0
5098 			 * then we're just getting a count and we can skip the
5099 			 * lookup stuff.
5100 			 */
5101 			ret = btrfs_check_shared(root, btrfs_ino(inode),
5102 						 bytenr, roots, tmp_ulist);
5103 			if (ret < 0)
5104 				goto out_free;
5105 			if (ret)
5106 				flags |= FIEMAP_EXTENT_SHARED;
5107 			ret = 0;
5108 		}
5109 		if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
5110 			flags |= FIEMAP_EXTENT_ENCODED;
5111 		if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
5112 			flags |= FIEMAP_EXTENT_UNWRITTEN;
5113 
5114 		free_extent_map(em);
5115 		em = NULL;
5116 		if ((em_start >= last) || em_len == (u64)-1 ||
5117 		   (last == (u64)-1 && isize <= em_end)) {
5118 			flags |= FIEMAP_EXTENT_LAST;
5119 			end = 1;
5120 		}
5121 
5122 		/* now scan forward to see if this is really the last extent. */
5123 		em = get_extent_skip_holes(inode, off, last_for_get_extent);
5124 		if (IS_ERR(em)) {
5125 			ret = PTR_ERR(em);
5126 			goto out;
5127 		}
5128 		if (!em) {
5129 			flags |= FIEMAP_EXTENT_LAST;
5130 			end = 1;
5131 		}
5132 		ret = emit_fiemap_extent(fieinfo, &cache, em_start, disko,
5133 					   em_len, flags);
5134 		if (ret) {
5135 			if (ret == 1)
5136 				ret = 0;
5137 			goto out_free;
5138 		}
5139 	}
5140 out_free:
5141 	if (!ret)
5142 		ret = emit_last_fiemap_cache(fieinfo, &cache);
5143 	free_extent_map(em);
5144 out:
5145 	unlock_extent_cached(&inode->io_tree, start, start + len - 1,
5146 			     &cached_state);
5147 
5148 out_free_ulist:
5149 	btrfs_free_path(path);
5150 	ulist_free(roots);
5151 	ulist_free(tmp_ulist);
5152 	return ret;
5153 }
5154 
5155 static void __free_extent_buffer(struct extent_buffer *eb)
5156 {
5157 	kmem_cache_free(extent_buffer_cache, eb);
5158 }
5159 
5160 int extent_buffer_under_io(const struct extent_buffer *eb)
5161 {
5162 	return (atomic_read(&eb->io_pages) ||
5163 		test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
5164 		test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
5165 }
5166 
5167 static bool page_range_has_eb(struct btrfs_fs_info *fs_info, struct page *page)
5168 {
5169 	struct btrfs_subpage *subpage;
5170 
5171 	lockdep_assert_held(&page->mapping->private_lock);
5172 
5173 	if (PagePrivate(page)) {
5174 		subpage = (struct btrfs_subpage *)page->private;
5175 		if (atomic_read(&subpage->eb_refs))
5176 			return true;
5177 	}
5178 	return false;
5179 }
5180 
5181 static void detach_extent_buffer_page(struct extent_buffer *eb, struct page *page)
5182 {
5183 	struct btrfs_fs_info *fs_info = eb->fs_info;
5184 	const bool mapped = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
5185 
5186 	/*
5187 	 * For mapped eb, we're going to change the page private, which should
5188 	 * be done under the private_lock.
5189 	 */
5190 	if (mapped)
5191 		spin_lock(&page->mapping->private_lock);
5192 
5193 	if (!PagePrivate(page)) {
5194 		if (mapped)
5195 			spin_unlock(&page->mapping->private_lock);
5196 		return;
5197 	}
5198 
5199 	if (fs_info->sectorsize == PAGE_SIZE) {
5200 		/*
5201 		 * We do this since we'll remove the pages after we've
5202 		 * removed the eb from the radix tree, so we could race
5203 		 * and have this page now attached to the new eb.  So
5204 		 * only clear page_private if it's still connected to
5205 		 * this eb.
5206 		 */
5207 		if (PagePrivate(page) &&
5208 		    page->private == (unsigned long)eb) {
5209 			BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
5210 			BUG_ON(PageDirty(page));
5211 			BUG_ON(PageWriteback(page));
5212 			/*
5213 			 * We need to make sure we haven't be attached
5214 			 * to a new eb.
5215 			 */
5216 			detach_page_private(page);
5217 		}
5218 		if (mapped)
5219 			spin_unlock(&page->mapping->private_lock);
5220 		return;
5221 	}
5222 
5223 	/*
5224 	 * For subpage, we can have dummy eb with page private.  In this case,
5225 	 * we can directly detach the private as such page is only attached to
5226 	 * one dummy eb, no sharing.
5227 	 */
5228 	if (!mapped) {
5229 		btrfs_detach_subpage(fs_info, page);
5230 		return;
5231 	}
5232 
5233 	btrfs_page_dec_eb_refs(fs_info, page);
5234 
5235 	/*
5236 	 * We can only detach the page private if there are no other ebs in the
5237 	 * page range.
5238 	 */
5239 	if (!page_range_has_eb(fs_info, page))
5240 		btrfs_detach_subpage(fs_info, page);
5241 
5242 	spin_unlock(&page->mapping->private_lock);
5243 }
5244 
5245 /* Release all pages attached to the extent buffer */
5246 static void btrfs_release_extent_buffer_pages(struct extent_buffer *eb)
5247 {
5248 	int i;
5249 	int num_pages;
5250 
5251 	ASSERT(!extent_buffer_under_io(eb));
5252 
5253 	num_pages = num_extent_pages(eb);
5254 	for (i = 0; i < num_pages; i++) {
5255 		struct page *page = eb->pages[i];
5256 
5257 		if (!page)
5258 			continue;
5259 
5260 		detach_extent_buffer_page(eb, page);
5261 
5262 		/* One for when we allocated the page */
5263 		put_page(page);
5264 	}
5265 }
5266 
5267 /*
5268  * Helper for releasing the extent buffer.
5269  */
5270 static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
5271 {
5272 	btrfs_release_extent_buffer_pages(eb);
5273 	btrfs_leak_debug_del(&eb->fs_info->eb_leak_lock, &eb->leak_list);
5274 	__free_extent_buffer(eb);
5275 }
5276 
5277 static struct extent_buffer *
5278 __alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start,
5279 		      unsigned long len)
5280 {
5281 	struct extent_buffer *eb = NULL;
5282 
5283 	eb = kmem_cache_zalloc(extent_buffer_cache, GFP_NOFS|__GFP_NOFAIL);
5284 	eb->start = start;
5285 	eb->len = len;
5286 	eb->fs_info = fs_info;
5287 	eb->bflags = 0;
5288 	init_rwsem(&eb->lock);
5289 
5290 	btrfs_leak_debug_add(&fs_info->eb_leak_lock, &eb->leak_list,
5291 			     &fs_info->allocated_ebs);
5292 	INIT_LIST_HEAD(&eb->release_list);
5293 
5294 	spin_lock_init(&eb->refs_lock);
5295 	atomic_set(&eb->refs, 1);
5296 	atomic_set(&eb->io_pages, 0);
5297 
5298 	ASSERT(len <= BTRFS_MAX_METADATA_BLOCKSIZE);
5299 
5300 	return eb;
5301 }
5302 
5303 struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src)
5304 {
5305 	int i;
5306 	struct page *p;
5307 	struct extent_buffer *new;
5308 	int num_pages = num_extent_pages(src);
5309 
5310 	new = __alloc_extent_buffer(src->fs_info, src->start, src->len);
5311 	if (new == NULL)
5312 		return NULL;
5313 
5314 	/*
5315 	 * Set UNMAPPED before calling btrfs_release_extent_buffer(), as
5316 	 * btrfs_release_extent_buffer() have different behavior for
5317 	 * UNMAPPED subpage extent buffer.
5318 	 */
5319 	set_bit(EXTENT_BUFFER_UNMAPPED, &new->bflags);
5320 
5321 	for (i = 0; i < num_pages; i++) {
5322 		int ret;
5323 
5324 		p = alloc_page(GFP_NOFS);
5325 		if (!p) {
5326 			btrfs_release_extent_buffer(new);
5327 			return NULL;
5328 		}
5329 		ret = attach_extent_buffer_page(new, p, NULL);
5330 		if (ret < 0) {
5331 			put_page(p);
5332 			btrfs_release_extent_buffer(new);
5333 			return NULL;
5334 		}
5335 		WARN_ON(PageDirty(p));
5336 		new->pages[i] = p;
5337 		copy_page(page_address(p), page_address(src->pages[i]));
5338 	}
5339 	set_extent_buffer_uptodate(new);
5340 
5341 	return new;
5342 }
5343 
5344 struct extent_buffer *__alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
5345 						  u64 start, unsigned long len)
5346 {
5347 	struct extent_buffer *eb;
5348 	int num_pages;
5349 	int i;
5350 
5351 	eb = __alloc_extent_buffer(fs_info, start, len);
5352 	if (!eb)
5353 		return NULL;
5354 
5355 	num_pages = num_extent_pages(eb);
5356 	for (i = 0; i < num_pages; i++) {
5357 		int ret;
5358 
5359 		eb->pages[i] = alloc_page(GFP_NOFS);
5360 		if (!eb->pages[i])
5361 			goto err;
5362 		ret = attach_extent_buffer_page(eb, eb->pages[i], NULL);
5363 		if (ret < 0)
5364 			goto err;
5365 	}
5366 	set_extent_buffer_uptodate(eb);
5367 	btrfs_set_header_nritems(eb, 0);
5368 	set_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
5369 
5370 	return eb;
5371 err:
5372 	for (; i > 0; i--) {
5373 		detach_extent_buffer_page(eb, eb->pages[i - 1]);
5374 		__free_page(eb->pages[i - 1]);
5375 	}
5376 	__free_extent_buffer(eb);
5377 	return NULL;
5378 }
5379 
5380 struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
5381 						u64 start)
5382 {
5383 	return __alloc_dummy_extent_buffer(fs_info, start, fs_info->nodesize);
5384 }
5385 
5386 static void check_buffer_tree_ref(struct extent_buffer *eb)
5387 {
5388 	int refs;
5389 	/*
5390 	 * The TREE_REF bit is first set when the extent_buffer is added
5391 	 * to the radix tree. It is also reset, if unset, when a new reference
5392 	 * is created by find_extent_buffer.
5393 	 *
5394 	 * It is only cleared in two cases: freeing the last non-tree
5395 	 * reference to the extent_buffer when its STALE bit is set or
5396 	 * calling releasepage when the tree reference is the only reference.
5397 	 *
5398 	 * In both cases, care is taken to ensure that the extent_buffer's
5399 	 * pages are not under io. However, releasepage can be concurrently
5400 	 * called with creating new references, which is prone to race
5401 	 * conditions between the calls to check_buffer_tree_ref in those
5402 	 * codepaths and clearing TREE_REF in try_release_extent_buffer.
5403 	 *
5404 	 * The actual lifetime of the extent_buffer in the radix tree is
5405 	 * adequately protected by the refcount, but the TREE_REF bit and
5406 	 * its corresponding reference are not. To protect against this
5407 	 * class of races, we call check_buffer_tree_ref from the codepaths
5408 	 * which trigger io after they set eb->io_pages. Note that once io is
5409 	 * initiated, TREE_REF can no longer be cleared, so that is the
5410 	 * moment at which any such race is best fixed.
5411 	 */
5412 	refs = atomic_read(&eb->refs);
5413 	if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
5414 		return;
5415 
5416 	spin_lock(&eb->refs_lock);
5417 	if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
5418 		atomic_inc(&eb->refs);
5419 	spin_unlock(&eb->refs_lock);
5420 }
5421 
5422 static void mark_extent_buffer_accessed(struct extent_buffer *eb,
5423 		struct page *accessed)
5424 {
5425 	int num_pages, i;
5426 
5427 	check_buffer_tree_ref(eb);
5428 
5429 	num_pages = num_extent_pages(eb);
5430 	for (i = 0; i < num_pages; i++) {
5431 		struct page *p = eb->pages[i];
5432 
5433 		if (p != accessed)
5434 			mark_page_accessed(p);
5435 	}
5436 }
5437 
5438 struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info,
5439 					 u64 start)
5440 {
5441 	struct extent_buffer *eb;
5442 
5443 	rcu_read_lock();
5444 	eb = radix_tree_lookup(&fs_info->buffer_radix,
5445 			       start >> fs_info->sectorsize_bits);
5446 	if (eb && atomic_inc_not_zero(&eb->refs)) {
5447 		rcu_read_unlock();
5448 		/*
5449 		 * Lock our eb's refs_lock to avoid races with
5450 		 * free_extent_buffer. When we get our eb it might be flagged
5451 		 * with EXTENT_BUFFER_STALE and another task running
5452 		 * free_extent_buffer might have seen that flag set,
5453 		 * eb->refs == 2, that the buffer isn't under IO (dirty and
5454 		 * writeback flags not set) and it's still in the tree (flag
5455 		 * EXTENT_BUFFER_TREE_REF set), therefore being in the process
5456 		 * of decrementing the extent buffer's reference count twice.
5457 		 * So here we could race and increment the eb's reference count,
5458 		 * clear its stale flag, mark it as dirty and drop our reference
5459 		 * before the other task finishes executing free_extent_buffer,
5460 		 * which would later result in an attempt to free an extent
5461 		 * buffer that is dirty.
5462 		 */
5463 		if (test_bit(EXTENT_BUFFER_STALE, &eb->bflags)) {
5464 			spin_lock(&eb->refs_lock);
5465 			spin_unlock(&eb->refs_lock);
5466 		}
5467 		mark_extent_buffer_accessed(eb, NULL);
5468 		return eb;
5469 	}
5470 	rcu_read_unlock();
5471 
5472 	return NULL;
5473 }
5474 
5475 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
5476 struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info,
5477 					u64 start)
5478 {
5479 	struct extent_buffer *eb, *exists = NULL;
5480 	int ret;
5481 
5482 	eb = find_extent_buffer(fs_info, start);
5483 	if (eb)
5484 		return eb;
5485 	eb = alloc_dummy_extent_buffer(fs_info, start);
5486 	if (!eb)
5487 		return ERR_PTR(-ENOMEM);
5488 	eb->fs_info = fs_info;
5489 again:
5490 	ret = radix_tree_preload(GFP_NOFS);
5491 	if (ret) {
5492 		exists = ERR_PTR(ret);
5493 		goto free_eb;
5494 	}
5495 	spin_lock(&fs_info->buffer_lock);
5496 	ret = radix_tree_insert(&fs_info->buffer_radix,
5497 				start >> fs_info->sectorsize_bits, eb);
5498 	spin_unlock(&fs_info->buffer_lock);
5499 	radix_tree_preload_end();
5500 	if (ret == -EEXIST) {
5501 		exists = find_extent_buffer(fs_info, start);
5502 		if (exists)
5503 			goto free_eb;
5504 		else
5505 			goto again;
5506 	}
5507 	check_buffer_tree_ref(eb);
5508 	set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
5509 
5510 	return eb;
5511 free_eb:
5512 	btrfs_release_extent_buffer(eb);
5513 	return exists;
5514 }
5515 #endif
5516 
5517 static struct extent_buffer *grab_extent_buffer(
5518 		struct btrfs_fs_info *fs_info, struct page *page)
5519 {
5520 	struct extent_buffer *exists;
5521 
5522 	/*
5523 	 * For subpage case, we completely rely on radix tree to ensure we
5524 	 * don't try to insert two ebs for the same bytenr.  So here we always
5525 	 * return NULL and just continue.
5526 	 */
5527 	if (fs_info->sectorsize < PAGE_SIZE)
5528 		return NULL;
5529 
5530 	/* Page not yet attached to an extent buffer */
5531 	if (!PagePrivate(page))
5532 		return NULL;
5533 
5534 	/*
5535 	 * We could have already allocated an eb for this page and attached one
5536 	 * so lets see if we can get a ref on the existing eb, and if we can we
5537 	 * know it's good and we can just return that one, else we know we can
5538 	 * just overwrite page->private.
5539 	 */
5540 	exists = (struct extent_buffer *)page->private;
5541 	if (atomic_inc_not_zero(&exists->refs))
5542 		return exists;
5543 
5544 	WARN_ON(PageDirty(page));
5545 	detach_page_private(page);
5546 	return NULL;
5547 }
5548 
5549 struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
5550 					  u64 start, u64 owner_root, int level)
5551 {
5552 	unsigned long len = fs_info->nodesize;
5553 	int num_pages;
5554 	int i;
5555 	unsigned long index = start >> PAGE_SHIFT;
5556 	struct extent_buffer *eb;
5557 	struct extent_buffer *exists = NULL;
5558 	struct page *p;
5559 	struct address_space *mapping = fs_info->btree_inode->i_mapping;
5560 	int uptodate = 1;
5561 	int ret;
5562 
5563 	if (!IS_ALIGNED(start, fs_info->sectorsize)) {
5564 		btrfs_err(fs_info, "bad tree block start %llu", start);
5565 		return ERR_PTR(-EINVAL);
5566 	}
5567 
5568 	if (fs_info->sectorsize < PAGE_SIZE &&
5569 	    offset_in_page(start) + len > PAGE_SIZE) {
5570 		btrfs_err(fs_info,
5571 		"tree block crosses page boundary, start %llu nodesize %lu",
5572 			  start, len);
5573 		return ERR_PTR(-EINVAL);
5574 	}
5575 
5576 	eb = find_extent_buffer(fs_info, start);
5577 	if (eb)
5578 		return eb;
5579 
5580 	eb = __alloc_extent_buffer(fs_info, start, len);
5581 	if (!eb)
5582 		return ERR_PTR(-ENOMEM);
5583 	btrfs_set_buffer_lockdep_class(owner_root, eb, level);
5584 
5585 	num_pages = num_extent_pages(eb);
5586 	for (i = 0; i < num_pages; i++, index++) {
5587 		struct btrfs_subpage *prealloc = NULL;
5588 
5589 		p = find_or_create_page(mapping, index, GFP_NOFS|__GFP_NOFAIL);
5590 		if (!p) {
5591 			exists = ERR_PTR(-ENOMEM);
5592 			goto free_eb;
5593 		}
5594 
5595 		/*
5596 		 * Preallocate page->private for subpage case, so that we won't
5597 		 * allocate memory with private_lock hold.  The memory will be
5598 		 * freed by attach_extent_buffer_page() or freed manually if
5599 		 * we exit earlier.
5600 		 *
5601 		 * Although we have ensured one subpage eb can only have one
5602 		 * page, but it may change in the future for 16K page size
5603 		 * support, so we still preallocate the memory in the loop.
5604 		 */
5605 		ret = btrfs_alloc_subpage(fs_info, &prealloc,
5606 					  BTRFS_SUBPAGE_METADATA);
5607 		if (ret < 0) {
5608 			unlock_page(p);
5609 			put_page(p);
5610 			exists = ERR_PTR(ret);
5611 			goto free_eb;
5612 		}
5613 
5614 		spin_lock(&mapping->private_lock);
5615 		exists = grab_extent_buffer(fs_info, p);
5616 		if (exists) {
5617 			spin_unlock(&mapping->private_lock);
5618 			unlock_page(p);
5619 			put_page(p);
5620 			mark_extent_buffer_accessed(exists, p);
5621 			btrfs_free_subpage(prealloc);
5622 			goto free_eb;
5623 		}
5624 		/* Should not fail, as we have preallocated the memory */
5625 		ret = attach_extent_buffer_page(eb, p, prealloc);
5626 		ASSERT(!ret);
5627 		/*
5628 		 * To inform we have extra eb under allocation, so that
5629 		 * detach_extent_buffer_page() won't release the page private
5630 		 * when the eb hasn't yet been inserted into radix tree.
5631 		 *
5632 		 * The ref will be decreased when the eb released the page, in
5633 		 * detach_extent_buffer_page().
5634 		 * Thus needs no special handling in error path.
5635 		 */
5636 		btrfs_page_inc_eb_refs(fs_info, p);
5637 		spin_unlock(&mapping->private_lock);
5638 
5639 		WARN_ON(PageDirty(p));
5640 		eb->pages[i] = p;
5641 		if (!PageUptodate(p))
5642 			uptodate = 0;
5643 
5644 		/*
5645 		 * We can't unlock the pages just yet since the extent buffer
5646 		 * hasn't been properly inserted in the radix tree, this
5647 		 * opens a race with btree_releasepage which can free a page
5648 		 * while we are still filling in all pages for the buffer and
5649 		 * we could crash.
5650 		 */
5651 	}
5652 	if (uptodate)
5653 		set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
5654 again:
5655 	ret = radix_tree_preload(GFP_NOFS);
5656 	if (ret) {
5657 		exists = ERR_PTR(ret);
5658 		goto free_eb;
5659 	}
5660 
5661 	spin_lock(&fs_info->buffer_lock);
5662 	ret = radix_tree_insert(&fs_info->buffer_radix,
5663 				start >> fs_info->sectorsize_bits, eb);
5664 	spin_unlock(&fs_info->buffer_lock);
5665 	radix_tree_preload_end();
5666 	if (ret == -EEXIST) {
5667 		exists = find_extent_buffer(fs_info, start);
5668 		if (exists)
5669 			goto free_eb;
5670 		else
5671 			goto again;
5672 	}
5673 	/* add one reference for the tree */
5674 	check_buffer_tree_ref(eb);
5675 	set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
5676 
5677 	/*
5678 	 * Now it's safe to unlock the pages because any calls to
5679 	 * btree_releasepage will correctly detect that a page belongs to a
5680 	 * live buffer and won't free them prematurely.
5681 	 */
5682 	for (i = 0; i < num_pages; i++)
5683 		unlock_page(eb->pages[i]);
5684 	return eb;
5685 
5686 free_eb:
5687 	WARN_ON(!atomic_dec_and_test(&eb->refs));
5688 	for (i = 0; i < num_pages; i++) {
5689 		if (eb->pages[i])
5690 			unlock_page(eb->pages[i]);
5691 	}
5692 
5693 	btrfs_release_extent_buffer(eb);
5694 	return exists;
5695 }
5696 
5697 static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
5698 {
5699 	struct extent_buffer *eb =
5700 			container_of(head, struct extent_buffer, rcu_head);
5701 
5702 	__free_extent_buffer(eb);
5703 }
5704 
5705 static int release_extent_buffer(struct extent_buffer *eb)
5706 	__releases(&eb->refs_lock)
5707 {
5708 	lockdep_assert_held(&eb->refs_lock);
5709 
5710 	WARN_ON(atomic_read(&eb->refs) == 0);
5711 	if (atomic_dec_and_test(&eb->refs)) {
5712 		if (test_and_clear_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags)) {
5713 			struct btrfs_fs_info *fs_info = eb->fs_info;
5714 
5715 			spin_unlock(&eb->refs_lock);
5716 
5717 			spin_lock(&fs_info->buffer_lock);
5718 			radix_tree_delete(&fs_info->buffer_radix,
5719 					  eb->start >> fs_info->sectorsize_bits);
5720 			spin_unlock(&fs_info->buffer_lock);
5721 		} else {
5722 			spin_unlock(&eb->refs_lock);
5723 		}
5724 
5725 		btrfs_leak_debug_del(&eb->fs_info->eb_leak_lock, &eb->leak_list);
5726 		/* Should be safe to release our pages at this point */
5727 		btrfs_release_extent_buffer_pages(eb);
5728 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
5729 		if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags))) {
5730 			__free_extent_buffer(eb);
5731 			return 1;
5732 		}
5733 #endif
5734 		call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
5735 		return 1;
5736 	}
5737 	spin_unlock(&eb->refs_lock);
5738 
5739 	return 0;
5740 }
5741 
5742 void free_extent_buffer(struct extent_buffer *eb)
5743 {
5744 	int refs;
5745 	int old;
5746 	if (!eb)
5747 		return;
5748 
5749 	while (1) {
5750 		refs = atomic_read(&eb->refs);
5751 		if ((!test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) && refs <= 3)
5752 		    || (test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) &&
5753 			refs == 1))
5754 			break;
5755 		old = atomic_cmpxchg(&eb->refs, refs, refs - 1);
5756 		if (old == refs)
5757 			return;
5758 	}
5759 
5760 	spin_lock(&eb->refs_lock);
5761 	if (atomic_read(&eb->refs) == 2 &&
5762 	    test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
5763 	    !extent_buffer_under_io(eb) &&
5764 	    test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
5765 		atomic_dec(&eb->refs);
5766 
5767 	/*
5768 	 * I know this is terrible, but it's temporary until we stop tracking
5769 	 * the uptodate bits and such for the extent buffers.
5770 	 */
5771 	release_extent_buffer(eb);
5772 }
5773 
5774 void free_extent_buffer_stale(struct extent_buffer *eb)
5775 {
5776 	if (!eb)
5777 		return;
5778 
5779 	spin_lock(&eb->refs_lock);
5780 	set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
5781 
5782 	if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
5783 	    test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
5784 		atomic_dec(&eb->refs);
5785 	release_extent_buffer(eb);
5786 }
5787 
5788 void clear_extent_buffer_dirty(const struct extent_buffer *eb)
5789 {
5790 	int i;
5791 	int num_pages;
5792 	struct page *page;
5793 
5794 	num_pages = num_extent_pages(eb);
5795 
5796 	for (i = 0; i < num_pages; i++) {
5797 		page = eb->pages[i];
5798 		if (!PageDirty(page))
5799 			continue;
5800 
5801 		lock_page(page);
5802 		WARN_ON(!PagePrivate(page));
5803 
5804 		clear_page_dirty_for_io(page);
5805 		xa_lock_irq(&page->mapping->i_pages);
5806 		if (!PageDirty(page))
5807 			__xa_clear_mark(&page->mapping->i_pages,
5808 					page_index(page), PAGECACHE_TAG_DIRTY);
5809 		xa_unlock_irq(&page->mapping->i_pages);
5810 		ClearPageError(page);
5811 		unlock_page(page);
5812 	}
5813 	WARN_ON(atomic_read(&eb->refs) == 0);
5814 }
5815 
5816 bool set_extent_buffer_dirty(struct extent_buffer *eb)
5817 {
5818 	int i;
5819 	int num_pages;
5820 	bool was_dirty;
5821 
5822 	check_buffer_tree_ref(eb);
5823 
5824 	was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
5825 
5826 	num_pages = num_extent_pages(eb);
5827 	WARN_ON(atomic_read(&eb->refs) == 0);
5828 	WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
5829 
5830 	if (!was_dirty)
5831 		for (i = 0; i < num_pages; i++)
5832 			set_page_dirty(eb->pages[i]);
5833 
5834 #ifdef CONFIG_BTRFS_DEBUG
5835 	for (i = 0; i < num_pages; i++)
5836 		ASSERT(PageDirty(eb->pages[i]));
5837 #endif
5838 
5839 	return was_dirty;
5840 }
5841 
5842 void clear_extent_buffer_uptodate(struct extent_buffer *eb)
5843 {
5844 	struct btrfs_fs_info *fs_info = eb->fs_info;
5845 	struct page *page;
5846 	int num_pages;
5847 	int i;
5848 
5849 	clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
5850 	num_pages = num_extent_pages(eb);
5851 	for (i = 0; i < num_pages; i++) {
5852 		page = eb->pages[i];
5853 		if (page)
5854 			btrfs_page_clear_uptodate(fs_info, page,
5855 						  eb->start, eb->len);
5856 	}
5857 }
5858 
5859 void set_extent_buffer_uptodate(struct extent_buffer *eb)
5860 {
5861 	struct btrfs_fs_info *fs_info = eb->fs_info;
5862 	struct page *page;
5863 	int num_pages;
5864 	int i;
5865 
5866 	set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
5867 	num_pages = num_extent_pages(eb);
5868 	for (i = 0; i < num_pages; i++) {
5869 		page = eb->pages[i];
5870 		btrfs_page_set_uptodate(fs_info, page, eb->start, eb->len);
5871 	}
5872 }
5873 
5874 static int read_extent_buffer_subpage(struct extent_buffer *eb, int wait,
5875 				      int mirror_num)
5876 {
5877 	struct btrfs_fs_info *fs_info = eb->fs_info;
5878 	struct extent_io_tree *io_tree;
5879 	struct page *page = eb->pages[0];
5880 	struct bio *bio = NULL;
5881 	int ret = 0;
5882 
5883 	ASSERT(!test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags));
5884 	ASSERT(PagePrivate(page));
5885 	io_tree = &BTRFS_I(fs_info->btree_inode)->io_tree;
5886 
5887 	if (wait == WAIT_NONE) {
5888 		ret = try_lock_extent(io_tree, eb->start,
5889 				      eb->start + eb->len - 1);
5890 		if (ret <= 0)
5891 			return ret;
5892 	} else {
5893 		ret = lock_extent(io_tree, eb->start, eb->start + eb->len - 1);
5894 		if (ret < 0)
5895 			return ret;
5896 	}
5897 
5898 	ret = 0;
5899 	if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags) ||
5900 	    PageUptodate(page) ||
5901 	    btrfs_subpage_test_uptodate(fs_info, page, eb->start, eb->len)) {
5902 		set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
5903 		unlock_extent(io_tree, eb->start, eb->start + eb->len - 1);
5904 		return ret;
5905 	}
5906 
5907 	clear_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
5908 	eb->read_mirror = 0;
5909 	atomic_set(&eb->io_pages, 1);
5910 	check_buffer_tree_ref(eb);
5911 	btrfs_subpage_clear_error(fs_info, page, eb->start, eb->len);
5912 
5913 	ret = submit_extent_page(REQ_OP_READ | REQ_META, NULL, page, eb->start,
5914 				 eb->len, eb->start - page_offset(page), &bio,
5915 				 end_bio_extent_readpage, mirror_num, 0, 0,
5916 				 true);
5917 	if (ret) {
5918 		/*
5919 		 * In the endio function, if we hit something wrong we will
5920 		 * increase the io_pages, so here we need to decrease it for
5921 		 * error path.
5922 		 */
5923 		atomic_dec(&eb->io_pages);
5924 	}
5925 	if (bio) {
5926 		int tmp;
5927 
5928 		tmp = submit_one_bio(bio, mirror_num, 0);
5929 		if (tmp < 0)
5930 			return tmp;
5931 	}
5932 	if (ret || wait != WAIT_COMPLETE)
5933 		return ret;
5934 
5935 	wait_extent_bit(io_tree, eb->start, eb->start + eb->len - 1, EXTENT_LOCKED);
5936 	if (!test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
5937 		ret = -EIO;
5938 	return ret;
5939 }
5940 
5941 int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num)
5942 {
5943 	int i;
5944 	struct page *page;
5945 	int err;
5946 	int ret = 0;
5947 	int locked_pages = 0;
5948 	int all_uptodate = 1;
5949 	int num_pages;
5950 	unsigned long num_reads = 0;
5951 	struct bio *bio = NULL;
5952 	unsigned long bio_flags = 0;
5953 
5954 	if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
5955 		return 0;
5956 
5957 	if (eb->fs_info->sectorsize < PAGE_SIZE)
5958 		return read_extent_buffer_subpage(eb, wait, mirror_num);
5959 
5960 	num_pages = num_extent_pages(eb);
5961 	for (i = 0; i < num_pages; i++) {
5962 		page = eb->pages[i];
5963 		if (wait == WAIT_NONE) {
5964 			/*
5965 			 * WAIT_NONE is only utilized by readahead. If we can't
5966 			 * acquire the lock atomically it means either the eb
5967 			 * is being read out or under modification.
5968 			 * Either way the eb will be or has been cached,
5969 			 * readahead can exit safely.
5970 			 */
5971 			if (!trylock_page(page))
5972 				goto unlock_exit;
5973 		} else {
5974 			lock_page(page);
5975 		}
5976 		locked_pages++;
5977 	}
5978 	/*
5979 	 * We need to firstly lock all pages to make sure that
5980 	 * the uptodate bit of our pages won't be affected by
5981 	 * clear_extent_buffer_uptodate().
5982 	 */
5983 	for (i = 0; i < num_pages; i++) {
5984 		page = eb->pages[i];
5985 		if (!PageUptodate(page)) {
5986 			num_reads++;
5987 			all_uptodate = 0;
5988 		}
5989 	}
5990 
5991 	if (all_uptodate) {
5992 		set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
5993 		goto unlock_exit;
5994 	}
5995 
5996 	clear_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
5997 	eb->read_mirror = 0;
5998 	atomic_set(&eb->io_pages, num_reads);
5999 	/*
6000 	 * It is possible for releasepage to clear the TREE_REF bit before we
6001 	 * set io_pages. See check_buffer_tree_ref for a more detailed comment.
6002 	 */
6003 	check_buffer_tree_ref(eb);
6004 	for (i = 0; i < num_pages; i++) {
6005 		page = eb->pages[i];
6006 
6007 		if (!PageUptodate(page)) {
6008 			if (ret) {
6009 				atomic_dec(&eb->io_pages);
6010 				unlock_page(page);
6011 				continue;
6012 			}
6013 
6014 			ClearPageError(page);
6015 			err = submit_extent_page(REQ_OP_READ | REQ_META, NULL,
6016 					 page, page_offset(page), PAGE_SIZE, 0,
6017 					 &bio, end_bio_extent_readpage,
6018 					 mirror_num, 0, 0, false);
6019 			if (err) {
6020 				/*
6021 				 * We failed to submit the bio so it's the
6022 				 * caller's responsibility to perform cleanup
6023 				 * i.e unlock page/set error bit.
6024 				 */
6025 				ret = err;
6026 				SetPageError(page);
6027 				unlock_page(page);
6028 				atomic_dec(&eb->io_pages);
6029 			}
6030 		} else {
6031 			unlock_page(page);
6032 		}
6033 	}
6034 
6035 	if (bio) {
6036 		err = submit_one_bio(bio, mirror_num, bio_flags);
6037 		if (err)
6038 			return err;
6039 	}
6040 
6041 	if (ret || wait != WAIT_COMPLETE)
6042 		return ret;
6043 
6044 	for (i = 0; i < num_pages; i++) {
6045 		page = eb->pages[i];
6046 		wait_on_page_locked(page);
6047 		if (!PageUptodate(page))
6048 			ret = -EIO;
6049 	}
6050 
6051 	return ret;
6052 
6053 unlock_exit:
6054 	while (locked_pages > 0) {
6055 		locked_pages--;
6056 		page = eb->pages[locked_pages];
6057 		unlock_page(page);
6058 	}
6059 	return ret;
6060 }
6061 
6062 static bool report_eb_range(const struct extent_buffer *eb, unsigned long start,
6063 			    unsigned long len)
6064 {
6065 	btrfs_warn(eb->fs_info,
6066 		"access to eb bytenr %llu len %lu out of range start %lu len %lu",
6067 		eb->start, eb->len, start, len);
6068 	WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
6069 
6070 	return true;
6071 }
6072 
6073 /*
6074  * Check if the [start, start + len) range is valid before reading/writing
6075  * the eb.
6076  * NOTE: @start and @len are offset inside the eb, not logical address.
6077  *
6078  * Caller should not touch the dst/src memory if this function returns error.
6079  */
6080 static inline int check_eb_range(const struct extent_buffer *eb,
6081 				 unsigned long start, unsigned long len)
6082 {
6083 	unsigned long offset;
6084 
6085 	/* start, start + len should not go beyond eb->len nor overflow */
6086 	if (unlikely(check_add_overflow(start, len, &offset) || offset > eb->len))
6087 		return report_eb_range(eb, start, len);
6088 
6089 	return false;
6090 }
6091 
6092 void read_extent_buffer(const struct extent_buffer *eb, void *dstv,
6093 			unsigned long start, unsigned long len)
6094 {
6095 	size_t cur;
6096 	size_t offset;
6097 	struct page *page;
6098 	char *kaddr;
6099 	char *dst = (char *)dstv;
6100 	unsigned long i = get_eb_page_index(start);
6101 
6102 	if (check_eb_range(eb, start, len))
6103 		return;
6104 
6105 	offset = get_eb_offset_in_page(eb, start);
6106 
6107 	while (len > 0) {
6108 		page = eb->pages[i];
6109 
6110 		cur = min(len, (PAGE_SIZE - offset));
6111 		kaddr = page_address(page);
6112 		memcpy(dst, kaddr + offset, cur);
6113 
6114 		dst += cur;
6115 		len -= cur;
6116 		offset = 0;
6117 		i++;
6118 	}
6119 }
6120 
6121 int read_extent_buffer_to_user_nofault(const struct extent_buffer *eb,
6122 				       void __user *dstv,
6123 				       unsigned long start, unsigned long len)
6124 {
6125 	size_t cur;
6126 	size_t offset;
6127 	struct page *page;
6128 	char *kaddr;
6129 	char __user *dst = (char __user *)dstv;
6130 	unsigned long i = get_eb_page_index(start);
6131 	int ret = 0;
6132 
6133 	WARN_ON(start > eb->len);
6134 	WARN_ON(start + len > eb->start + eb->len);
6135 
6136 	offset = get_eb_offset_in_page(eb, start);
6137 
6138 	while (len > 0) {
6139 		page = eb->pages[i];
6140 
6141 		cur = min(len, (PAGE_SIZE - offset));
6142 		kaddr = page_address(page);
6143 		if (copy_to_user_nofault(dst, kaddr + offset, cur)) {
6144 			ret = -EFAULT;
6145 			break;
6146 		}
6147 
6148 		dst += cur;
6149 		len -= cur;
6150 		offset = 0;
6151 		i++;
6152 	}
6153 
6154 	return ret;
6155 }
6156 
6157 int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv,
6158 			 unsigned long start, unsigned long len)
6159 {
6160 	size_t cur;
6161 	size_t offset;
6162 	struct page *page;
6163 	char *kaddr;
6164 	char *ptr = (char *)ptrv;
6165 	unsigned long i = get_eb_page_index(start);
6166 	int ret = 0;
6167 
6168 	if (check_eb_range(eb, start, len))
6169 		return -EINVAL;
6170 
6171 	offset = get_eb_offset_in_page(eb, start);
6172 
6173 	while (len > 0) {
6174 		page = eb->pages[i];
6175 
6176 		cur = min(len, (PAGE_SIZE - offset));
6177 
6178 		kaddr = page_address(page);
6179 		ret = memcmp(ptr, kaddr + offset, cur);
6180 		if (ret)
6181 			break;
6182 
6183 		ptr += cur;
6184 		len -= cur;
6185 		offset = 0;
6186 		i++;
6187 	}
6188 	return ret;
6189 }
6190 
6191 void write_extent_buffer_chunk_tree_uuid(const struct extent_buffer *eb,
6192 		const void *srcv)
6193 {
6194 	char *kaddr;
6195 
6196 	WARN_ON(!PageUptodate(eb->pages[0]));
6197 	kaddr = page_address(eb->pages[0]) + get_eb_offset_in_page(eb, 0);
6198 	memcpy(kaddr + offsetof(struct btrfs_header, chunk_tree_uuid), srcv,
6199 			BTRFS_FSID_SIZE);
6200 }
6201 
6202 void write_extent_buffer_fsid(const struct extent_buffer *eb, const void *srcv)
6203 {
6204 	char *kaddr;
6205 
6206 	WARN_ON(!PageUptodate(eb->pages[0]));
6207 	kaddr = page_address(eb->pages[0]) + get_eb_offset_in_page(eb, 0);
6208 	memcpy(kaddr + offsetof(struct btrfs_header, fsid), srcv,
6209 			BTRFS_FSID_SIZE);
6210 }
6211 
6212 void write_extent_buffer(const struct extent_buffer *eb, const void *srcv,
6213 			 unsigned long start, unsigned long len)
6214 {
6215 	size_t cur;
6216 	size_t offset;
6217 	struct page *page;
6218 	char *kaddr;
6219 	char *src = (char *)srcv;
6220 	unsigned long i = get_eb_page_index(start);
6221 
6222 	WARN_ON(test_bit(EXTENT_BUFFER_NO_CHECK, &eb->bflags));
6223 
6224 	if (check_eb_range(eb, start, len))
6225 		return;
6226 
6227 	offset = get_eb_offset_in_page(eb, start);
6228 
6229 	while (len > 0) {
6230 		page = eb->pages[i];
6231 		WARN_ON(!PageUptodate(page));
6232 
6233 		cur = min(len, PAGE_SIZE - offset);
6234 		kaddr = page_address(page);
6235 		memcpy(kaddr + offset, src, cur);
6236 
6237 		src += cur;
6238 		len -= cur;
6239 		offset = 0;
6240 		i++;
6241 	}
6242 }
6243 
6244 void memzero_extent_buffer(const struct extent_buffer *eb, unsigned long start,
6245 		unsigned long len)
6246 {
6247 	size_t cur;
6248 	size_t offset;
6249 	struct page *page;
6250 	char *kaddr;
6251 	unsigned long i = get_eb_page_index(start);
6252 
6253 	if (check_eb_range(eb, start, len))
6254 		return;
6255 
6256 	offset = get_eb_offset_in_page(eb, start);
6257 
6258 	while (len > 0) {
6259 		page = eb->pages[i];
6260 		WARN_ON(!PageUptodate(page));
6261 
6262 		cur = min(len, PAGE_SIZE - offset);
6263 		kaddr = page_address(page);
6264 		memset(kaddr + offset, 0, cur);
6265 
6266 		len -= cur;
6267 		offset = 0;
6268 		i++;
6269 	}
6270 }
6271 
6272 void copy_extent_buffer_full(const struct extent_buffer *dst,
6273 			     const struct extent_buffer *src)
6274 {
6275 	int i;
6276 	int num_pages;
6277 
6278 	ASSERT(dst->len == src->len);
6279 
6280 	if (dst->fs_info->sectorsize == PAGE_SIZE) {
6281 		num_pages = num_extent_pages(dst);
6282 		for (i = 0; i < num_pages; i++)
6283 			copy_page(page_address(dst->pages[i]),
6284 				  page_address(src->pages[i]));
6285 	} else {
6286 		size_t src_offset = get_eb_offset_in_page(src, 0);
6287 		size_t dst_offset = get_eb_offset_in_page(dst, 0);
6288 
6289 		ASSERT(src->fs_info->sectorsize < PAGE_SIZE);
6290 		memcpy(page_address(dst->pages[0]) + dst_offset,
6291 		       page_address(src->pages[0]) + src_offset,
6292 		       src->len);
6293 	}
6294 }
6295 
6296 void copy_extent_buffer(const struct extent_buffer *dst,
6297 			const struct extent_buffer *src,
6298 			unsigned long dst_offset, unsigned long src_offset,
6299 			unsigned long len)
6300 {
6301 	u64 dst_len = dst->len;
6302 	size_t cur;
6303 	size_t offset;
6304 	struct page *page;
6305 	char *kaddr;
6306 	unsigned long i = get_eb_page_index(dst_offset);
6307 
6308 	if (check_eb_range(dst, dst_offset, len) ||
6309 	    check_eb_range(src, src_offset, len))
6310 		return;
6311 
6312 	WARN_ON(src->len != dst_len);
6313 
6314 	offset = get_eb_offset_in_page(dst, dst_offset);
6315 
6316 	while (len > 0) {
6317 		page = dst->pages[i];
6318 		WARN_ON(!PageUptodate(page));
6319 
6320 		cur = min(len, (unsigned long)(PAGE_SIZE - offset));
6321 
6322 		kaddr = page_address(page);
6323 		read_extent_buffer(src, kaddr + offset, src_offset, cur);
6324 
6325 		src_offset += cur;
6326 		len -= cur;
6327 		offset = 0;
6328 		i++;
6329 	}
6330 }
6331 
6332 /*
6333  * eb_bitmap_offset() - calculate the page and offset of the byte containing the
6334  * given bit number
6335  * @eb: the extent buffer
6336  * @start: offset of the bitmap item in the extent buffer
6337  * @nr: bit number
6338  * @page_index: return index of the page in the extent buffer that contains the
6339  * given bit number
6340  * @page_offset: return offset into the page given by page_index
6341  *
6342  * This helper hides the ugliness of finding the byte in an extent buffer which
6343  * contains a given bit.
6344  */
6345 static inline void eb_bitmap_offset(const struct extent_buffer *eb,
6346 				    unsigned long start, unsigned long nr,
6347 				    unsigned long *page_index,
6348 				    size_t *page_offset)
6349 {
6350 	size_t byte_offset = BIT_BYTE(nr);
6351 	size_t offset;
6352 
6353 	/*
6354 	 * The byte we want is the offset of the extent buffer + the offset of
6355 	 * the bitmap item in the extent buffer + the offset of the byte in the
6356 	 * bitmap item.
6357 	 */
6358 	offset = start + offset_in_page(eb->start) + byte_offset;
6359 
6360 	*page_index = offset >> PAGE_SHIFT;
6361 	*page_offset = offset_in_page(offset);
6362 }
6363 
6364 /**
6365  * extent_buffer_test_bit - determine whether a bit in a bitmap item is set
6366  * @eb: the extent buffer
6367  * @start: offset of the bitmap item in the extent buffer
6368  * @nr: bit number to test
6369  */
6370 int extent_buffer_test_bit(const struct extent_buffer *eb, unsigned long start,
6371 			   unsigned long nr)
6372 {
6373 	u8 *kaddr;
6374 	struct page *page;
6375 	unsigned long i;
6376 	size_t offset;
6377 
6378 	eb_bitmap_offset(eb, start, nr, &i, &offset);
6379 	page = eb->pages[i];
6380 	WARN_ON(!PageUptodate(page));
6381 	kaddr = page_address(page);
6382 	return 1U & (kaddr[offset] >> (nr & (BITS_PER_BYTE - 1)));
6383 }
6384 
6385 /**
6386  * extent_buffer_bitmap_set - set an area of a bitmap
6387  * @eb: the extent buffer
6388  * @start: offset of the bitmap item in the extent buffer
6389  * @pos: bit number of the first bit
6390  * @len: number of bits to set
6391  */
6392 void extent_buffer_bitmap_set(const struct extent_buffer *eb, unsigned long start,
6393 			      unsigned long pos, unsigned long len)
6394 {
6395 	u8 *kaddr;
6396 	struct page *page;
6397 	unsigned long i;
6398 	size_t offset;
6399 	const unsigned int size = pos + len;
6400 	int bits_to_set = BITS_PER_BYTE - (pos % BITS_PER_BYTE);
6401 	u8 mask_to_set = BITMAP_FIRST_BYTE_MASK(pos);
6402 
6403 	eb_bitmap_offset(eb, start, pos, &i, &offset);
6404 	page = eb->pages[i];
6405 	WARN_ON(!PageUptodate(page));
6406 	kaddr = page_address(page);
6407 
6408 	while (len >= bits_to_set) {
6409 		kaddr[offset] |= mask_to_set;
6410 		len -= bits_to_set;
6411 		bits_to_set = BITS_PER_BYTE;
6412 		mask_to_set = ~0;
6413 		if (++offset >= PAGE_SIZE && len > 0) {
6414 			offset = 0;
6415 			page = eb->pages[++i];
6416 			WARN_ON(!PageUptodate(page));
6417 			kaddr = page_address(page);
6418 		}
6419 	}
6420 	if (len) {
6421 		mask_to_set &= BITMAP_LAST_BYTE_MASK(size);
6422 		kaddr[offset] |= mask_to_set;
6423 	}
6424 }
6425 
6426 
6427 /**
6428  * extent_buffer_bitmap_clear - clear an area of a bitmap
6429  * @eb: the extent buffer
6430  * @start: offset of the bitmap item in the extent buffer
6431  * @pos: bit number of the first bit
6432  * @len: number of bits to clear
6433  */
6434 void extent_buffer_bitmap_clear(const struct extent_buffer *eb,
6435 				unsigned long start, unsigned long pos,
6436 				unsigned long len)
6437 {
6438 	u8 *kaddr;
6439 	struct page *page;
6440 	unsigned long i;
6441 	size_t offset;
6442 	const unsigned int size = pos + len;
6443 	int bits_to_clear = BITS_PER_BYTE - (pos % BITS_PER_BYTE);
6444 	u8 mask_to_clear = BITMAP_FIRST_BYTE_MASK(pos);
6445 
6446 	eb_bitmap_offset(eb, start, pos, &i, &offset);
6447 	page = eb->pages[i];
6448 	WARN_ON(!PageUptodate(page));
6449 	kaddr = page_address(page);
6450 
6451 	while (len >= bits_to_clear) {
6452 		kaddr[offset] &= ~mask_to_clear;
6453 		len -= bits_to_clear;
6454 		bits_to_clear = BITS_PER_BYTE;
6455 		mask_to_clear = ~0;
6456 		if (++offset >= PAGE_SIZE && len > 0) {
6457 			offset = 0;
6458 			page = eb->pages[++i];
6459 			WARN_ON(!PageUptodate(page));
6460 			kaddr = page_address(page);
6461 		}
6462 	}
6463 	if (len) {
6464 		mask_to_clear &= BITMAP_LAST_BYTE_MASK(size);
6465 		kaddr[offset] &= ~mask_to_clear;
6466 	}
6467 }
6468 
6469 static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
6470 {
6471 	unsigned long distance = (src > dst) ? src - dst : dst - src;
6472 	return distance < len;
6473 }
6474 
6475 static void copy_pages(struct page *dst_page, struct page *src_page,
6476 		       unsigned long dst_off, unsigned long src_off,
6477 		       unsigned long len)
6478 {
6479 	char *dst_kaddr = page_address(dst_page);
6480 	char *src_kaddr;
6481 	int must_memmove = 0;
6482 
6483 	if (dst_page != src_page) {
6484 		src_kaddr = page_address(src_page);
6485 	} else {
6486 		src_kaddr = dst_kaddr;
6487 		if (areas_overlap(src_off, dst_off, len))
6488 			must_memmove = 1;
6489 	}
6490 
6491 	if (must_memmove)
6492 		memmove(dst_kaddr + dst_off, src_kaddr + src_off, len);
6493 	else
6494 		memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
6495 }
6496 
6497 void memcpy_extent_buffer(const struct extent_buffer *dst,
6498 			  unsigned long dst_offset, unsigned long src_offset,
6499 			  unsigned long len)
6500 {
6501 	size_t cur;
6502 	size_t dst_off_in_page;
6503 	size_t src_off_in_page;
6504 	unsigned long dst_i;
6505 	unsigned long src_i;
6506 
6507 	if (check_eb_range(dst, dst_offset, len) ||
6508 	    check_eb_range(dst, src_offset, len))
6509 		return;
6510 
6511 	while (len > 0) {
6512 		dst_off_in_page = get_eb_offset_in_page(dst, dst_offset);
6513 		src_off_in_page = get_eb_offset_in_page(dst, src_offset);
6514 
6515 		dst_i = get_eb_page_index(dst_offset);
6516 		src_i = get_eb_page_index(src_offset);
6517 
6518 		cur = min(len, (unsigned long)(PAGE_SIZE -
6519 					       src_off_in_page));
6520 		cur = min_t(unsigned long, cur,
6521 			(unsigned long)(PAGE_SIZE - dst_off_in_page));
6522 
6523 		copy_pages(dst->pages[dst_i], dst->pages[src_i],
6524 			   dst_off_in_page, src_off_in_page, cur);
6525 
6526 		src_offset += cur;
6527 		dst_offset += cur;
6528 		len -= cur;
6529 	}
6530 }
6531 
6532 void memmove_extent_buffer(const struct extent_buffer *dst,
6533 			   unsigned long dst_offset, unsigned long src_offset,
6534 			   unsigned long len)
6535 {
6536 	size_t cur;
6537 	size_t dst_off_in_page;
6538 	size_t src_off_in_page;
6539 	unsigned long dst_end = dst_offset + len - 1;
6540 	unsigned long src_end = src_offset + len - 1;
6541 	unsigned long dst_i;
6542 	unsigned long src_i;
6543 
6544 	if (check_eb_range(dst, dst_offset, len) ||
6545 	    check_eb_range(dst, src_offset, len))
6546 		return;
6547 	if (dst_offset < src_offset) {
6548 		memcpy_extent_buffer(dst, dst_offset, src_offset, len);
6549 		return;
6550 	}
6551 	while (len > 0) {
6552 		dst_i = get_eb_page_index(dst_end);
6553 		src_i = get_eb_page_index(src_end);
6554 
6555 		dst_off_in_page = get_eb_offset_in_page(dst, dst_end);
6556 		src_off_in_page = get_eb_offset_in_page(dst, src_end);
6557 
6558 		cur = min_t(unsigned long, len, src_off_in_page + 1);
6559 		cur = min(cur, dst_off_in_page + 1);
6560 		copy_pages(dst->pages[dst_i], dst->pages[src_i],
6561 			   dst_off_in_page - cur + 1,
6562 			   src_off_in_page - cur + 1, cur);
6563 
6564 		dst_end -= cur;
6565 		src_end -= cur;
6566 		len -= cur;
6567 	}
6568 }
6569 
6570 static struct extent_buffer *get_next_extent_buffer(
6571 		struct btrfs_fs_info *fs_info, struct page *page, u64 bytenr)
6572 {
6573 	struct extent_buffer *gang[BTRFS_SUBPAGE_BITMAP_SIZE];
6574 	struct extent_buffer *found = NULL;
6575 	u64 page_start = page_offset(page);
6576 	int ret;
6577 	int i;
6578 
6579 	ASSERT(in_range(bytenr, page_start, PAGE_SIZE));
6580 	ASSERT(PAGE_SIZE / fs_info->nodesize <= BTRFS_SUBPAGE_BITMAP_SIZE);
6581 	lockdep_assert_held(&fs_info->buffer_lock);
6582 
6583 	ret = radix_tree_gang_lookup(&fs_info->buffer_radix, (void **)gang,
6584 			bytenr >> fs_info->sectorsize_bits,
6585 			PAGE_SIZE / fs_info->nodesize);
6586 	for (i = 0; i < ret; i++) {
6587 		/* Already beyond page end */
6588 		if (gang[i]->start >= page_start + PAGE_SIZE)
6589 			break;
6590 		/* Found one */
6591 		if (gang[i]->start >= bytenr) {
6592 			found = gang[i];
6593 			break;
6594 		}
6595 	}
6596 	return found;
6597 }
6598 
6599 static int try_release_subpage_extent_buffer(struct page *page)
6600 {
6601 	struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb);
6602 	u64 cur = page_offset(page);
6603 	const u64 end = page_offset(page) + PAGE_SIZE;
6604 	int ret;
6605 
6606 	while (cur < end) {
6607 		struct extent_buffer *eb = NULL;
6608 
6609 		/*
6610 		 * Unlike try_release_extent_buffer() which uses page->private
6611 		 * to grab buffer, for subpage case we rely on radix tree, thus
6612 		 * we need to ensure radix tree consistency.
6613 		 *
6614 		 * We also want an atomic snapshot of the radix tree, thus go
6615 		 * with spinlock rather than RCU.
6616 		 */
6617 		spin_lock(&fs_info->buffer_lock);
6618 		eb = get_next_extent_buffer(fs_info, page, cur);
6619 		if (!eb) {
6620 			/* No more eb in the page range after or at cur */
6621 			spin_unlock(&fs_info->buffer_lock);
6622 			break;
6623 		}
6624 		cur = eb->start + eb->len;
6625 
6626 		/*
6627 		 * The same as try_release_extent_buffer(), to ensure the eb
6628 		 * won't disappear out from under us.
6629 		 */
6630 		spin_lock(&eb->refs_lock);
6631 		if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
6632 			spin_unlock(&eb->refs_lock);
6633 			spin_unlock(&fs_info->buffer_lock);
6634 			break;
6635 		}
6636 		spin_unlock(&fs_info->buffer_lock);
6637 
6638 		/*
6639 		 * If tree ref isn't set then we know the ref on this eb is a
6640 		 * real ref, so just return, this eb will likely be freed soon
6641 		 * anyway.
6642 		 */
6643 		if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
6644 			spin_unlock(&eb->refs_lock);
6645 			break;
6646 		}
6647 
6648 		/*
6649 		 * Here we don't care about the return value, we will always
6650 		 * check the page private at the end.  And
6651 		 * release_extent_buffer() will release the refs_lock.
6652 		 */
6653 		release_extent_buffer(eb);
6654 	}
6655 	/*
6656 	 * Finally to check if we have cleared page private, as if we have
6657 	 * released all ebs in the page, the page private should be cleared now.
6658 	 */
6659 	spin_lock(&page->mapping->private_lock);
6660 	if (!PagePrivate(page))
6661 		ret = 1;
6662 	else
6663 		ret = 0;
6664 	spin_unlock(&page->mapping->private_lock);
6665 	return ret;
6666 
6667 }
6668 
6669 int try_release_extent_buffer(struct page *page)
6670 {
6671 	struct extent_buffer *eb;
6672 
6673 	if (btrfs_sb(page->mapping->host->i_sb)->sectorsize < PAGE_SIZE)
6674 		return try_release_subpage_extent_buffer(page);
6675 
6676 	/*
6677 	 * We need to make sure nobody is changing page->private, as we rely on
6678 	 * page->private as the pointer to extent buffer.
6679 	 */
6680 	spin_lock(&page->mapping->private_lock);
6681 	if (!PagePrivate(page)) {
6682 		spin_unlock(&page->mapping->private_lock);
6683 		return 1;
6684 	}
6685 
6686 	eb = (struct extent_buffer *)page->private;
6687 	BUG_ON(!eb);
6688 
6689 	/*
6690 	 * This is a little awful but should be ok, we need to make sure that
6691 	 * the eb doesn't disappear out from under us while we're looking at
6692 	 * this page.
6693 	 */
6694 	spin_lock(&eb->refs_lock);
6695 	if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
6696 		spin_unlock(&eb->refs_lock);
6697 		spin_unlock(&page->mapping->private_lock);
6698 		return 0;
6699 	}
6700 	spin_unlock(&page->mapping->private_lock);
6701 
6702 	/*
6703 	 * If tree ref isn't set then we know the ref on this eb is a real ref,
6704 	 * so just return, this page will likely be freed soon anyway.
6705 	 */
6706 	if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
6707 		spin_unlock(&eb->refs_lock);
6708 		return 0;
6709 	}
6710 
6711 	return release_extent_buffer(eb);
6712 }
6713 
6714 /*
6715  * btrfs_readahead_tree_block - attempt to readahead a child block
6716  * @fs_info:	the fs_info
6717  * @bytenr:	bytenr to read
6718  * @owner_root: objectid of the root that owns this eb
6719  * @gen:	generation for the uptodate check, can be 0
6720  * @level:	level for the eb
6721  *
6722  * Attempt to readahead a tree block at @bytenr.  If @gen is 0 then we do a
6723  * normal uptodate check of the eb, without checking the generation.  If we have
6724  * to read the block we will not block on anything.
6725  */
6726 void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info,
6727 				u64 bytenr, u64 owner_root, u64 gen, int level)
6728 {
6729 	struct extent_buffer *eb;
6730 	int ret;
6731 
6732 	eb = btrfs_find_create_tree_block(fs_info, bytenr, owner_root, level);
6733 	if (IS_ERR(eb))
6734 		return;
6735 
6736 	if (btrfs_buffer_uptodate(eb, gen, 1)) {
6737 		free_extent_buffer(eb);
6738 		return;
6739 	}
6740 
6741 	ret = read_extent_buffer_pages(eb, WAIT_NONE, 0);
6742 	if (ret < 0)
6743 		free_extent_buffer_stale(eb);
6744 	else
6745 		free_extent_buffer(eb);
6746 }
6747 
6748 /*
6749  * btrfs_readahead_node_child - readahead a node's child block
6750  * @node:	parent node we're reading from
6751  * @slot:	slot in the parent node for the child we want to read
6752  *
6753  * A helper for btrfs_readahead_tree_block, we simply read the bytenr pointed at
6754  * the slot in the node provided.
6755  */
6756 void btrfs_readahead_node_child(struct extent_buffer *node, int slot)
6757 {
6758 	btrfs_readahead_tree_block(node->fs_info,
6759 				   btrfs_node_blockptr(node, slot),
6760 				   btrfs_header_owner(node),
6761 				   btrfs_node_ptr_generation(node, slot),
6762 				   btrfs_header_level(node) - 1);
6763 }
6764