xref: /openbmc/linux/fs/btrfs/ctree.c (revision 4fd786e6c3d67b1348e0ad4f450efe9fc9d7a306)
1c1d7c514SDavid Sterba // SPDX-License-Identifier: GPL-2.0
26cbd5570SChris Mason /*
3d352ac68SChris Mason  * Copyright (C) 2007,2008 Oracle.  All rights reserved.
46cbd5570SChris Mason  */
56cbd5570SChris Mason 
6a6b6e75eSChris Mason #include <linux/sched.h>
75a0e3ad6STejun Heo #include <linux/slab.h>
8bd989ba3SJan Schmidt #include <linux/rbtree.h>
9adf02123SDavid Sterba #include <linux/mm.h>
10eb60ceacSChris Mason #include "ctree.h"
11eb60ceacSChris Mason #include "disk-io.h"
127f5c1516SChris Mason #include "transaction.h"
135f39d397SChris Mason #include "print-tree.h"
14925baeddSChris Mason #include "locking.h"
159a8dd150SChris Mason 
16e089f05cSChris Mason static int split_node(struct btrfs_trans_handle *trans, struct btrfs_root
17e089f05cSChris Mason 		      *root, struct btrfs_path *path, int level);
18310712b2SOmar Sandoval static int split_leaf(struct btrfs_trans_handle *trans, struct btrfs_root *root,
19310712b2SOmar Sandoval 		      const struct btrfs_key *ins_key, struct btrfs_path *path,
20310712b2SOmar Sandoval 		      int data_size, int extend);
215f39d397SChris Mason static int push_node_left(struct btrfs_trans_handle *trans,
222ff7e61eSJeff Mahoney 			  struct btrfs_fs_info *fs_info,
232ff7e61eSJeff Mahoney 			  struct extent_buffer *dst,
24971a1f66SChris Mason 			  struct extent_buffer *src, int empty);
255f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans,
262ff7e61eSJeff Mahoney 			      struct btrfs_fs_info *fs_info,
275f39d397SChris Mason 			      struct extent_buffer *dst_buf,
285f39d397SChris Mason 			      struct extent_buffer *src_buf);
29afe5fea7STsutomu Itoh static void del_ptr(struct btrfs_root *root, struct btrfs_path *path,
30afe5fea7STsutomu Itoh 		    int level, int slot);
31d97e63b6SChris Mason 
322c90e5d6SChris Mason struct btrfs_path *btrfs_alloc_path(void)
332c90e5d6SChris Mason {
34e2c89907SMasahiro Yamada 	return kmem_cache_zalloc(btrfs_path_cachep, GFP_NOFS);
352c90e5d6SChris Mason }
362c90e5d6SChris Mason 
37b4ce94deSChris Mason /*
38b4ce94deSChris Mason  * set all locked nodes in the path to blocking locks.  This should
39b4ce94deSChris Mason  * be done before scheduling
40b4ce94deSChris Mason  */
41b4ce94deSChris Mason noinline void btrfs_set_path_blocking(struct btrfs_path *p)
42b4ce94deSChris Mason {
43b4ce94deSChris Mason 	int i;
44b4ce94deSChris Mason 	for (i = 0; i < BTRFS_MAX_LEVEL; i++) {
45bd681513SChris Mason 		if (!p->nodes[i] || !p->locks[i])
46bd681513SChris Mason 			continue;
47bd681513SChris Mason 		btrfs_set_lock_blocking_rw(p->nodes[i], p->locks[i]);
48bd681513SChris Mason 		if (p->locks[i] == BTRFS_READ_LOCK)
49bd681513SChris Mason 			p->locks[i] = BTRFS_READ_LOCK_BLOCKING;
50bd681513SChris Mason 		else if (p->locks[i] == BTRFS_WRITE_LOCK)
51bd681513SChris Mason 			p->locks[i] = BTRFS_WRITE_LOCK_BLOCKING;
52b4ce94deSChris Mason 	}
53b4ce94deSChris Mason }
54b4ce94deSChris Mason 
55b4ce94deSChris Mason /*
56b4ce94deSChris Mason  * reset all the locked nodes in the patch to spinning locks.
574008c04aSChris Mason  *
584008c04aSChris Mason  * held is used to keep lockdep happy, when lockdep is enabled
594008c04aSChris Mason  * we set held to a blocking lock before we go around and
604008c04aSChris Mason  * retake all the spinlocks in the path.  You can safely use NULL
614008c04aSChris Mason  * for held
62b4ce94deSChris Mason  */
634008c04aSChris Mason noinline void btrfs_clear_path_blocking(struct btrfs_path *p,
64bd681513SChris Mason 					struct extent_buffer *held, int held_rw)
65b4ce94deSChris Mason {
66b4ce94deSChris Mason 	int i;
674008c04aSChris Mason 
68bd681513SChris Mason 	if (held) {
69bd681513SChris Mason 		btrfs_set_lock_blocking_rw(held, held_rw);
70bd681513SChris Mason 		if (held_rw == BTRFS_WRITE_LOCK)
71bd681513SChris Mason 			held_rw = BTRFS_WRITE_LOCK_BLOCKING;
72bd681513SChris Mason 		else if (held_rw == BTRFS_READ_LOCK)
73bd681513SChris Mason 			held_rw = BTRFS_READ_LOCK_BLOCKING;
74bd681513SChris Mason 	}
754008c04aSChris Mason 	btrfs_set_path_blocking(p);
764008c04aSChris Mason 
774008c04aSChris Mason 	for (i = BTRFS_MAX_LEVEL - 1; i >= 0; i--) {
78bd681513SChris Mason 		if (p->nodes[i] && p->locks[i]) {
79bd681513SChris Mason 			btrfs_clear_lock_blocking_rw(p->nodes[i], p->locks[i]);
80bd681513SChris Mason 			if (p->locks[i] == BTRFS_WRITE_LOCK_BLOCKING)
81bd681513SChris Mason 				p->locks[i] = BTRFS_WRITE_LOCK;
82bd681513SChris Mason 			else if (p->locks[i] == BTRFS_READ_LOCK_BLOCKING)
83bd681513SChris Mason 				p->locks[i] = BTRFS_READ_LOCK;
84bd681513SChris Mason 		}
85b4ce94deSChris Mason 	}
864008c04aSChris Mason 
874008c04aSChris Mason 	if (held)
88bd681513SChris Mason 		btrfs_clear_lock_blocking_rw(held, held_rw);
89b4ce94deSChris Mason }
90b4ce94deSChris Mason 
91d352ac68SChris Mason /* this also releases the path */
922c90e5d6SChris Mason void btrfs_free_path(struct btrfs_path *p)
932c90e5d6SChris Mason {
94ff175d57SJesper Juhl 	if (!p)
95ff175d57SJesper Juhl 		return;
96b3b4aa74SDavid Sterba 	btrfs_release_path(p);
972c90e5d6SChris Mason 	kmem_cache_free(btrfs_path_cachep, p);
982c90e5d6SChris Mason }
992c90e5d6SChris Mason 
100d352ac68SChris Mason /*
101d352ac68SChris Mason  * path release drops references on the extent buffers in the path
102d352ac68SChris Mason  * and it drops any locks held by this path
103d352ac68SChris Mason  *
104d352ac68SChris Mason  * It is safe to call this on paths that no locks or extent buffers held.
105d352ac68SChris Mason  */
106b3b4aa74SDavid Sterba noinline void btrfs_release_path(struct btrfs_path *p)
107eb60ceacSChris Mason {
108eb60ceacSChris Mason 	int i;
109a2135011SChris Mason 
110234b63a0SChris Mason 	for (i = 0; i < BTRFS_MAX_LEVEL; i++) {
1113f157a2fSChris Mason 		p->slots[i] = 0;
112eb60ceacSChris Mason 		if (!p->nodes[i])
113925baeddSChris Mason 			continue;
114925baeddSChris Mason 		if (p->locks[i]) {
115bd681513SChris Mason 			btrfs_tree_unlock_rw(p->nodes[i], p->locks[i]);
116925baeddSChris Mason 			p->locks[i] = 0;
117925baeddSChris Mason 		}
1185f39d397SChris Mason 		free_extent_buffer(p->nodes[i]);
1193f157a2fSChris Mason 		p->nodes[i] = NULL;
120eb60ceacSChris Mason 	}
121eb60ceacSChris Mason }
122eb60ceacSChris Mason 
123d352ac68SChris Mason /*
124d352ac68SChris Mason  * safely gets a reference on the root node of a tree.  A lock
125d352ac68SChris Mason  * is not taken, so a concurrent writer may put a different node
126d352ac68SChris Mason  * at the root of the tree.  See btrfs_lock_root_node for the
127d352ac68SChris Mason  * looping required.
128d352ac68SChris Mason  *
129d352ac68SChris Mason  * The extent buffer returned by this has a reference taken, so
130d352ac68SChris Mason  * it won't disappear.  It may stop being the root of the tree
131d352ac68SChris Mason  * at any time because there are no locks held.
132d352ac68SChris Mason  */
133925baeddSChris Mason struct extent_buffer *btrfs_root_node(struct btrfs_root *root)
134925baeddSChris Mason {
135925baeddSChris Mason 	struct extent_buffer *eb;
136240f62c8SChris Mason 
1373083ee2eSJosef Bacik 	while (1) {
138240f62c8SChris Mason 		rcu_read_lock();
139240f62c8SChris Mason 		eb = rcu_dereference(root->node);
1403083ee2eSJosef Bacik 
1413083ee2eSJosef Bacik 		/*
1423083ee2eSJosef Bacik 		 * RCU really hurts here, we could free up the root node because
14301327610SNicholas D Steeves 		 * it was COWed but we may not get the new root node yet so do
1443083ee2eSJosef Bacik 		 * the inc_not_zero dance and if it doesn't work then
1453083ee2eSJosef Bacik 		 * synchronize_rcu and try again.
1463083ee2eSJosef Bacik 		 */
1473083ee2eSJosef Bacik 		if (atomic_inc_not_zero(&eb->refs)) {
148240f62c8SChris Mason 			rcu_read_unlock();
1493083ee2eSJosef Bacik 			break;
1503083ee2eSJosef Bacik 		}
1513083ee2eSJosef Bacik 		rcu_read_unlock();
1523083ee2eSJosef Bacik 		synchronize_rcu();
1533083ee2eSJosef Bacik 	}
154925baeddSChris Mason 	return eb;
155925baeddSChris Mason }
156925baeddSChris Mason 
157d352ac68SChris Mason /* loop around taking references on and locking the root node of the
158d352ac68SChris Mason  * tree until you end up with a lock on the root.  A locked buffer
159d352ac68SChris Mason  * is returned, with a reference held.
160d352ac68SChris Mason  */
161925baeddSChris Mason struct extent_buffer *btrfs_lock_root_node(struct btrfs_root *root)
162925baeddSChris Mason {
163925baeddSChris Mason 	struct extent_buffer *eb;
164925baeddSChris Mason 
165925baeddSChris Mason 	while (1) {
166925baeddSChris Mason 		eb = btrfs_root_node(root);
167925baeddSChris Mason 		btrfs_tree_lock(eb);
168240f62c8SChris Mason 		if (eb == root->node)
169925baeddSChris Mason 			break;
170925baeddSChris Mason 		btrfs_tree_unlock(eb);
171925baeddSChris Mason 		free_extent_buffer(eb);
172925baeddSChris Mason 	}
173925baeddSChris Mason 	return eb;
174925baeddSChris Mason }
175925baeddSChris Mason 
176bd681513SChris Mason /* loop around taking references on and locking the root node of the
177bd681513SChris Mason  * tree until you end up with a lock on the root.  A locked buffer
178bd681513SChris Mason  * is returned, with a reference held.
179bd681513SChris Mason  */
18084f7d8e6SJosef Bacik struct extent_buffer *btrfs_read_lock_root_node(struct btrfs_root *root)
181bd681513SChris Mason {
182bd681513SChris Mason 	struct extent_buffer *eb;
183bd681513SChris Mason 
184bd681513SChris Mason 	while (1) {
185bd681513SChris Mason 		eb = btrfs_root_node(root);
186bd681513SChris Mason 		btrfs_tree_read_lock(eb);
187bd681513SChris Mason 		if (eb == root->node)
188bd681513SChris Mason 			break;
189bd681513SChris Mason 		btrfs_tree_read_unlock(eb);
190bd681513SChris Mason 		free_extent_buffer(eb);
191bd681513SChris Mason 	}
192bd681513SChris Mason 	return eb;
193bd681513SChris Mason }
194bd681513SChris Mason 
195d352ac68SChris Mason /* cowonly root (everything not a reference counted cow subvolume), just get
196d352ac68SChris Mason  * put onto a simple dirty list.  transaction.c walks this to make sure they
197d352ac68SChris Mason  * get properly updated on disk.
198d352ac68SChris Mason  */
1990b86a832SChris Mason static void add_root_to_dirty_list(struct btrfs_root *root)
2000b86a832SChris Mason {
2010b246afaSJeff Mahoney 	struct btrfs_fs_info *fs_info = root->fs_info;
2020b246afaSJeff Mahoney 
203e7070be1SJosef Bacik 	if (test_bit(BTRFS_ROOT_DIRTY, &root->state) ||
204e7070be1SJosef Bacik 	    !test_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state))
205e7070be1SJosef Bacik 		return;
206e7070be1SJosef Bacik 
2070b246afaSJeff Mahoney 	spin_lock(&fs_info->trans_lock);
208e7070be1SJosef Bacik 	if (!test_and_set_bit(BTRFS_ROOT_DIRTY, &root->state)) {
209e7070be1SJosef Bacik 		/* Want the extent tree to be the last on the list */
210*4fd786e6SMisono Tomohiro 		if (root->root_key.objectid == BTRFS_EXTENT_TREE_OBJECTID)
211e7070be1SJosef Bacik 			list_move_tail(&root->dirty_list,
2120b246afaSJeff Mahoney 				       &fs_info->dirty_cowonly_roots);
213e7070be1SJosef Bacik 		else
214e7070be1SJosef Bacik 			list_move(&root->dirty_list,
2150b246afaSJeff Mahoney 				  &fs_info->dirty_cowonly_roots);
2160b86a832SChris Mason 	}
2170b246afaSJeff Mahoney 	spin_unlock(&fs_info->trans_lock);
2180b86a832SChris Mason }
2190b86a832SChris Mason 
220d352ac68SChris Mason /*
221d352ac68SChris Mason  * used by snapshot creation to make a copy of a root for a tree with
222d352ac68SChris Mason  * a given objectid.  The buffer with the new root node is returned in
223d352ac68SChris Mason  * cow_ret, and this func returns zero on success or a negative error code.
224d352ac68SChris Mason  */
225be20aa9dSChris Mason int btrfs_copy_root(struct btrfs_trans_handle *trans,
226be20aa9dSChris Mason 		      struct btrfs_root *root,
227be20aa9dSChris Mason 		      struct extent_buffer *buf,
228be20aa9dSChris Mason 		      struct extent_buffer **cow_ret, u64 new_root_objectid)
229be20aa9dSChris Mason {
2300b246afaSJeff Mahoney 	struct btrfs_fs_info *fs_info = root->fs_info;
231be20aa9dSChris Mason 	struct extent_buffer *cow;
232be20aa9dSChris Mason 	int ret = 0;
233be20aa9dSChris Mason 	int level;
2345d4f98a2SYan Zheng 	struct btrfs_disk_key disk_key;
235be20aa9dSChris Mason 
23627cdeb70SMiao Xie 	WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
2370b246afaSJeff Mahoney 		trans->transid != fs_info->running_transaction->transid);
23827cdeb70SMiao Xie 	WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
23927cdeb70SMiao Xie 		trans->transid != root->last_trans);
240be20aa9dSChris Mason 
241be20aa9dSChris Mason 	level = btrfs_header_level(buf);
2425d4f98a2SYan Zheng 	if (level == 0)
2435d4f98a2SYan Zheng 		btrfs_item_key(buf, &disk_key, 0);
2445d4f98a2SYan Zheng 	else
2455d4f98a2SYan Zheng 		btrfs_node_key(buf, &disk_key, 0);
24631840ae1SZheng Yan 
2474d75f8a9SDavid Sterba 	cow = btrfs_alloc_tree_block(trans, root, 0, new_root_objectid,
2484d75f8a9SDavid Sterba 			&disk_key, level, buf->start, 0);
2495d4f98a2SYan Zheng 	if (IS_ERR(cow))
250be20aa9dSChris Mason 		return PTR_ERR(cow);
251be20aa9dSChris Mason 
25258e8012cSDavid Sterba 	copy_extent_buffer_full(cow, buf);
253be20aa9dSChris Mason 	btrfs_set_header_bytenr(cow, cow->start);
254be20aa9dSChris Mason 	btrfs_set_header_generation(cow, trans->transid);
2555d4f98a2SYan Zheng 	btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV);
2565d4f98a2SYan Zheng 	btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN |
2575d4f98a2SYan Zheng 				     BTRFS_HEADER_FLAG_RELOC);
2585d4f98a2SYan Zheng 	if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID)
2595d4f98a2SYan Zheng 		btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC);
2605d4f98a2SYan Zheng 	else
261be20aa9dSChris Mason 		btrfs_set_header_owner(cow, new_root_objectid);
262be20aa9dSChris Mason 
2630b246afaSJeff Mahoney 	write_extent_buffer_fsid(cow, fs_info->fsid);
2642b82032cSYan Zheng 
265be20aa9dSChris Mason 	WARN_ON(btrfs_header_generation(buf) > trans->transid);
2665d4f98a2SYan Zheng 	if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID)
267e339a6b0SJosef Bacik 		ret = btrfs_inc_ref(trans, root, cow, 1);
2685d4f98a2SYan Zheng 	else
269e339a6b0SJosef Bacik 		ret = btrfs_inc_ref(trans, root, cow, 0);
2704aec2b52SChris Mason 
271be20aa9dSChris Mason 	if (ret)
272be20aa9dSChris Mason 		return ret;
273be20aa9dSChris Mason 
274be20aa9dSChris Mason 	btrfs_mark_buffer_dirty(cow);
275be20aa9dSChris Mason 	*cow_ret = cow;
276be20aa9dSChris Mason 	return 0;
277be20aa9dSChris Mason }
278be20aa9dSChris Mason 
279bd989ba3SJan Schmidt enum mod_log_op {
280bd989ba3SJan Schmidt 	MOD_LOG_KEY_REPLACE,
281bd989ba3SJan Schmidt 	MOD_LOG_KEY_ADD,
282bd989ba3SJan Schmidt 	MOD_LOG_KEY_REMOVE,
283bd989ba3SJan Schmidt 	MOD_LOG_KEY_REMOVE_WHILE_FREEING,
284bd989ba3SJan Schmidt 	MOD_LOG_KEY_REMOVE_WHILE_MOVING,
285bd989ba3SJan Schmidt 	MOD_LOG_MOVE_KEYS,
286bd989ba3SJan Schmidt 	MOD_LOG_ROOT_REPLACE,
287bd989ba3SJan Schmidt };
288bd989ba3SJan Schmidt 
289bd989ba3SJan Schmidt struct tree_mod_root {
290bd989ba3SJan Schmidt 	u64 logical;
291bd989ba3SJan Schmidt 	u8 level;
292bd989ba3SJan Schmidt };
293bd989ba3SJan Schmidt 
294bd989ba3SJan Schmidt struct tree_mod_elem {
295bd989ba3SJan Schmidt 	struct rb_node node;
296298cfd36SChandan Rajendra 	u64 logical;
297097b8a7cSJan Schmidt 	u64 seq;
298bd989ba3SJan Schmidt 	enum mod_log_op op;
299bd989ba3SJan Schmidt 
300bd989ba3SJan Schmidt 	/* this is used for MOD_LOG_KEY_* and MOD_LOG_MOVE_KEYS operations */
301bd989ba3SJan Schmidt 	int slot;
302bd989ba3SJan Schmidt 
303bd989ba3SJan Schmidt 	/* this is used for MOD_LOG_KEY* and MOD_LOG_ROOT_REPLACE */
304bd989ba3SJan Schmidt 	u64 generation;
305bd989ba3SJan Schmidt 
306bd989ba3SJan Schmidt 	/* those are used for op == MOD_LOG_KEY_{REPLACE,REMOVE} */
307bd989ba3SJan Schmidt 	struct btrfs_disk_key key;
308bd989ba3SJan Schmidt 	u64 blockptr;
309bd989ba3SJan Schmidt 
310bd989ba3SJan Schmidt 	/* this is used for op == MOD_LOG_MOVE_KEYS */
311b6dfa35bSDavid Sterba 	struct {
312b6dfa35bSDavid Sterba 		int dst_slot;
313b6dfa35bSDavid Sterba 		int nr_items;
314b6dfa35bSDavid Sterba 	} move;
315bd989ba3SJan Schmidt 
316bd989ba3SJan Schmidt 	/* this is used for op == MOD_LOG_ROOT_REPLACE */
317bd989ba3SJan Schmidt 	struct tree_mod_root old_root;
318bd989ba3SJan Schmidt };
319bd989ba3SJan Schmidt 
320097b8a7cSJan Schmidt /*
321fcebe456SJosef Bacik  * Pull a new tree mod seq number for our operation.
322fc36ed7eSJan Schmidt  */
323fcebe456SJosef Bacik static inline u64 btrfs_inc_tree_mod_seq(struct btrfs_fs_info *fs_info)
324fc36ed7eSJan Schmidt {
325fc36ed7eSJan Schmidt 	return atomic64_inc_return(&fs_info->tree_mod_seq);
326fc36ed7eSJan Schmidt }
327fc36ed7eSJan Schmidt 
328fc36ed7eSJan Schmidt /*
329097b8a7cSJan Schmidt  * This adds a new blocker to the tree mod log's blocker list if the @elem
330097b8a7cSJan Schmidt  * passed does not already have a sequence number set. So when a caller expects
331097b8a7cSJan Schmidt  * to record tree modifications, it should ensure to set elem->seq to zero
332097b8a7cSJan Schmidt  * before calling btrfs_get_tree_mod_seq.
333097b8a7cSJan Schmidt  * Returns a fresh, unused tree log modification sequence number, even if no new
334097b8a7cSJan Schmidt  * blocker was added.
335097b8a7cSJan Schmidt  */
336097b8a7cSJan Schmidt u64 btrfs_get_tree_mod_seq(struct btrfs_fs_info *fs_info,
337bd989ba3SJan Schmidt 			   struct seq_list *elem)
338bd989ba3SJan Schmidt {
339b1a09f1eSDavid Sterba 	write_lock(&fs_info->tree_mod_log_lock);
340bd989ba3SJan Schmidt 	spin_lock(&fs_info->tree_mod_seq_lock);
341097b8a7cSJan Schmidt 	if (!elem->seq) {
342fcebe456SJosef Bacik 		elem->seq = btrfs_inc_tree_mod_seq(fs_info);
343097b8a7cSJan Schmidt 		list_add_tail(&elem->list, &fs_info->tree_mod_seq_list);
344097b8a7cSJan Schmidt 	}
345bd989ba3SJan Schmidt 	spin_unlock(&fs_info->tree_mod_seq_lock);
346b1a09f1eSDavid Sterba 	write_unlock(&fs_info->tree_mod_log_lock);
347097b8a7cSJan Schmidt 
348fcebe456SJosef Bacik 	return elem->seq;
349bd989ba3SJan Schmidt }
350bd989ba3SJan Schmidt 
351bd989ba3SJan Schmidt void btrfs_put_tree_mod_seq(struct btrfs_fs_info *fs_info,
352bd989ba3SJan Schmidt 			    struct seq_list *elem)
353bd989ba3SJan Schmidt {
354bd989ba3SJan Schmidt 	struct rb_root *tm_root;
355bd989ba3SJan Schmidt 	struct rb_node *node;
356bd989ba3SJan Schmidt 	struct rb_node *next;
357bd989ba3SJan Schmidt 	struct seq_list *cur_elem;
358bd989ba3SJan Schmidt 	struct tree_mod_elem *tm;
359bd989ba3SJan Schmidt 	u64 min_seq = (u64)-1;
360bd989ba3SJan Schmidt 	u64 seq_putting = elem->seq;
361bd989ba3SJan Schmidt 
362bd989ba3SJan Schmidt 	if (!seq_putting)
363bd989ba3SJan Schmidt 		return;
364bd989ba3SJan Schmidt 
365bd989ba3SJan Schmidt 	spin_lock(&fs_info->tree_mod_seq_lock);
366bd989ba3SJan Schmidt 	list_del(&elem->list);
367097b8a7cSJan Schmidt 	elem->seq = 0;
368bd989ba3SJan Schmidt 
369bd989ba3SJan Schmidt 	list_for_each_entry(cur_elem, &fs_info->tree_mod_seq_list, list) {
370097b8a7cSJan Schmidt 		if (cur_elem->seq < min_seq) {
371bd989ba3SJan Schmidt 			if (seq_putting > cur_elem->seq) {
372bd989ba3SJan Schmidt 				/*
373bd989ba3SJan Schmidt 				 * blocker with lower sequence number exists, we
374bd989ba3SJan Schmidt 				 * cannot remove anything from the log
375bd989ba3SJan Schmidt 				 */
376097b8a7cSJan Schmidt 				spin_unlock(&fs_info->tree_mod_seq_lock);
377097b8a7cSJan Schmidt 				return;
378bd989ba3SJan Schmidt 			}
379bd989ba3SJan Schmidt 			min_seq = cur_elem->seq;
380bd989ba3SJan Schmidt 		}
381bd989ba3SJan Schmidt 	}
382097b8a7cSJan Schmidt 	spin_unlock(&fs_info->tree_mod_seq_lock);
383097b8a7cSJan Schmidt 
384097b8a7cSJan Schmidt 	/*
385bd989ba3SJan Schmidt 	 * anything that's lower than the lowest existing (read: blocked)
386bd989ba3SJan Schmidt 	 * sequence number can be removed from the tree.
387bd989ba3SJan Schmidt 	 */
388b1a09f1eSDavid Sterba 	write_lock(&fs_info->tree_mod_log_lock);
389bd989ba3SJan Schmidt 	tm_root = &fs_info->tree_mod_log;
390bd989ba3SJan Schmidt 	for (node = rb_first(tm_root); node; node = next) {
391bd989ba3SJan Schmidt 		next = rb_next(node);
3926b4df8b6SGeliang Tang 		tm = rb_entry(node, struct tree_mod_elem, node);
393097b8a7cSJan Schmidt 		if (tm->seq > min_seq)
394bd989ba3SJan Schmidt 			continue;
395bd989ba3SJan Schmidt 		rb_erase(node, tm_root);
396bd989ba3SJan Schmidt 		kfree(tm);
397bd989ba3SJan Schmidt 	}
398b1a09f1eSDavid Sterba 	write_unlock(&fs_info->tree_mod_log_lock);
399bd989ba3SJan Schmidt }
400bd989ba3SJan Schmidt 
401bd989ba3SJan Schmidt /*
402bd989ba3SJan Schmidt  * key order of the log:
403298cfd36SChandan Rajendra  *       node/leaf start address -> sequence
404bd989ba3SJan Schmidt  *
405298cfd36SChandan Rajendra  * The 'start address' is the logical address of the *new* root node
406298cfd36SChandan Rajendra  * for root replace operations, or the logical address of the affected
407298cfd36SChandan Rajendra  * block for all other operations.
4085de865eeSFilipe David Borba Manana  *
409b1a09f1eSDavid Sterba  * Note: must be called with write lock for fs_info::tree_mod_log_lock.
410bd989ba3SJan Schmidt  */
411bd989ba3SJan Schmidt static noinline int
412bd989ba3SJan Schmidt __tree_mod_log_insert(struct btrfs_fs_info *fs_info, struct tree_mod_elem *tm)
413bd989ba3SJan Schmidt {
414bd989ba3SJan Schmidt 	struct rb_root *tm_root;
415bd989ba3SJan Schmidt 	struct rb_node **new;
416bd989ba3SJan Schmidt 	struct rb_node *parent = NULL;
417bd989ba3SJan Schmidt 	struct tree_mod_elem *cur;
418bd989ba3SJan Schmidt 
419fcebe456SJosef Bacik 	tm->seq = btrfs_inc_tree_mod_seq(fs_info);
420bd989ba3SJan Schmidt 
421bd989ba3SJan Schmidt 	tm_root = &fs_info->tree_mod_log;
422bd989ba3SJan Schmidt 	new = &tm_root->rb_node;
423bd989ba3SJan Schmidt 	while (*new) {
4246b4df8b6SGeliang Tang 		cur = rb_entry(*new, struct tree_mod_elem, node);
425bd989ba3SJan Schmidt 		parent = *new;
426298cfd36SChandan Rajendra 		if (cur->logical < tm->logical)
427bd989ba3SJan Schmidt 			new = &((*new)->rb_left);
428298cfd36SChandan Rajendra 		else if (cur->logical > tm->logical)
429bd989ba3SJan Schmidt 			new = &((*new)->rb_right);
430097b8a7cSJan Schmidt 		else if (cur->seq < tm->seq)
431bd989ba3SJan Schmidt 			new = &((*new)->rb_left);
432097b8a7cSJan Schmidt 		else if (cur->seq > tm->seq)
433bd989ba3SJan Schmidt 			new = &((*new)->rb_right);
4345de865eeSFilipe David Borba Manana 		else
4355de865eeSFilipe David Borba Manana 			return -EEXIST;
436bd989ba3SJan Schmidt 	}
437bd989ba3SJan Schmidt 
438bd989ba3SJan Schmidt 	rb_link_node(&tm->node, parent, new);
439bd989ba3SJan Schmidt 	rb_insert_color(&tm->node, tm_root);
4405de865eeSFilipe David Borba Manana 	return 0;
441bd989ba3SJan Schmidt }
442bd989ba3SJan Schmidt 
443097b8a7cSJan Schmidt /*
444097b8a7cSJan Schmidt  * Determines if logging can be omitted. Returns 1 if it can. Otherwise, it
445097b8a7cSJan Schmidt  * returns zero with the tree_mod_log_lock acquired. The caller must hold
446097b8a7cSJan Schmidt  * this until all tree mod log insertions are recorded in the rb tree and then
447b1a09f1eSDavid Sterba  * write unlock fs_info::tree_mod_log_lock.
448097b8a7cSJan Schmidt  */
449e9b7fd4dSJan Schmidt static inline int tree_mod_dont_log(struct btrfs_fs_info *fs_info,
450e9b7fd4dSJan Schmidt 				    struct extent_buffer *eb) {
451e9b7fd4dSJan Schmidt 	smp_mb();
452e9b7fd4dSJan Schmidt 	if (list_empty(&(fs_info)->tree_mod_seq_list))
453e9b7fd4dSJan Schmidt 		return 1;
454097b8a7cSJan Schmidt 	if (eb && btrfs_header_level(eb) == 0)
455e9b7fd4dSJan Schmidt 		return 1;
4565de865eeSFilipe David Borba Manana 
457b1a09f1eSDavid Sterba 	write_lock(&fs_info->tree_mod_log_lock);
4585de865eeSFilipe David Borba Manana 	if (list_empty(&(fs_info)->tree_mod_seq_list)) {
459b1a09f1eSDavid Sterba 		write_unlock(&fs_info->tree_mod_log_lock);
4605de865eeSFilipe David Borba Manana 		return 1;
4615de865eeSFilipe David Borba Manana 	}
4625de865eeSFilipe David Borba Manana 
463e9b7fd4dSJan Schmidt 	return 0;
464e9b7fd4dSJan Schmidt }
465e9b7fd4dSJan Schmidt 
4665de865eeSFilipe David Borba Manana /* Similar to tree_mod_dont_log, but doesn't acquire any locks. */
4675de865eeSFilipe David Borba Manana static inline int tree_mod_need_log(const struct btrfs_fs_info *fs_info,
4685de865eeSFilipe David Borba Manana 				    struct extent_buffer *eb)
4695de865eeSFilipe David Borba Manana {
4705de865eeSFilipe David Borba Manana 	smp_mb();
4715de865eeSFilipe David Borba Manana 	if (list_empty(&(fs_info)->tree_mod_seq_list))
4725de865eeSFilipe David Borba Manana 		return 0;
4735de865eeSFilipe David Borba Manana 	if (eb && btrfs_header_level(eb) == 0)
4745de865eeSFilipe David Borba Manana 		return 0;
4755de865eeSFilipe David Borba Manana 
4765de865eeSFilipe David Borba Manana 	return 1;
4775de865eeSFilipe David Borba Manana }
4785de865eeSFilipe David Borba Manana 
4795de865eeSFilipe David Borba Manana static struct tree_mod_elem *
4805de865eeSFilipe David Borba Manana alloc_tree_mod_elem(struct extent_buffer *eb, int slot,
481bd989ba3SJan Schmidt 		    enum mod_log_op op, gfp_t flags)
482bd989ba3SJan Schmidt {
483097b8a7cSJan Schmidt 	struct tree_mod_elem *tm;
484bd989ba3SJan Schmidt 
485c8cc6341SJosef Bacik 	tm = kzalloc(sizeof(*tm), flags);
486c8cc6341SJosef Bacik 	if (!tm)
4875de865eeSFilipe David Borba Manana 		return NULL;
488bd989ba3SJan Schmidt 
489298cfd36SChandan Rajendra 	tm->logical = eb->start;
490bd989ba3SJan Schmidt 	if (op != MOD_LOG_KEY_ADD) {
491bd989ba3SJan Schmidt 		btrfs_node_key(eb, &tm->key, slot);
492bd989ba3SJan Schmidt 		tm->blockptr = btrfs_node_blockptr(eb, slot);
493bd989ba3SJan Schmidt 	}
494bd989ba3SJan Schmidt 	tm->op = op;
495bd989ba3SJan Schmidt 	tm->slot = slot;
496bd989ba3SJan Schmidt 	tm->generation = btrfs_node_ptr_generation(eb, slot);
4975de865eeSFilipe David Borba Manana 	RB_CLEAR_NODE(&tm->node);
498bd989ba3SJan Schmidt 
4995de865eeSFilipe David Borba Manana 	return tm;
500097b8a7cSJan Schmidt }
501097b8a7cSJan Schmidt 
502e09c2efeSDavid Sterba static noinline int tree_mod_log_insert_key(struct extent_buffer *eb, int slot,
503097b8a7cSJan Schmidt 		enum mod_log_op op, gfp_t flags)
504097b8a7cSJan Schmidt {
5055de865eeSFilipe David Borba Manana 	struct tree_mod_elem *tm;
5065de865eeSFilipe David Borba Manana 	int ret;
5075de865eeSFilipe David Borba Manana 
508e09c2efeSDavid Sterba 	if (!tree_mod_need_log(eb->fs_info, eb))
509097b8a7cSJan Schmidt 		return 0;
510097b8a7cSJan Schmidt 
5115de865eeSFilipe David Borba Manana 	tm = alloc_tree_mod_elem(eb, slot, op, flags);
5125de865eeSFilipe David Borba Manana 	if (!tm)
5135de865eeSFilipe David Borba Manana 		return -ENOMEM;
5145de865eeSFilipe David Borba Manana 
515e09c2efeSDavid Sterba 	if (tree_mod_dont_log(eb->fs_info, eb)) {
5165de865eeSFilipe David Borba Manana 		kfree(tm);
5175de865eeSFilipe David Borba Manana 		return 0;
5185de865eeSFilipe David Borba Manana 	}
5195de865eeSFilipe David Borba Manana 
520e09c2efeSDavid Sterba 	ret = __tree_mod_log_insert(eb->fs_info, tm);
521b1a09f1eSDavid Sterba 	write_unlock(&eb->fs_info->tree_mod_log_lock);
5225de865eeSFilipe David Borba Manana 	if (ret)
5235de865eeSFilipe David Borba Manana 		kfree(tm);
5245de865eeSFilipe David Borba Manana 
5255de865eeSFilipe David Borba Manana 	return ret;
526097b8a7cSJan Schmidt }
527097b8a7cSJan Schmidt 
5286074d45fSDavid Sterba static noinline int tree_mod_log_insert_move(struct extent_buffer *eb,
5296074d45fSDavid Sterba 		int dst_slot, int src_slot, int nr_items)
530bd989ba3SJan Schmidt {
5315de865eeSFilipe David Borba Manana 	struct tree_mod_elem *tm = NULL;
5325de865eeSFilipe David Borba Manana 	struct tree_mod_elem **tm_list = NULL;
5335de865eeSFilipe David Borba Manana 	int ret = 0;
534bd989ba3SJan Schmidt 	int i;
5355de865eeSFilipe David Borba Manana 	int locked = 0;
536bd989ba3SJan Schmidt 
5376074d45fSDavid Sterba 	if (!tree_mod_need_log(eb->fs_info, eb))
538f395694cSJan Schmidt 		return 0;
539bd989ba3SJan Schmidt 
540176ef8f5SDavid Sterba 	tm_list = kcalloc(nr_items, sizeof(struct tree_mod_elem *), GFP_NOFS);
5415de865eeSFilipe David Borba Manana 	if (!tm_list)
5425de865eeSFilipe David Borba Manana 		return -ENOMEM;
543bd989ba3SJan Schmidt 
544176ef8f5SDavid Sterba 	tm = kzalloc(sizeof(*tm), GFP_NOFS);
5455de865eeSFilipe David Borba Manana 	if (!tm) {
5465de865eeSFilipe David Borba Manana 		ret = -ENOMEM;
5475de865eeSFilipe David Borba Manana 		goto free_tms;
5485de865eeSFilipe David Borba Manana 	}
549f395694cSJan Schmidt 
550298cfd36SChandan Rajendra 	tm->logical = eb->start;
551bd989ba3SJan Schmidt 	tm->slot = src_slot;
552bd989ba3SJan Schmidt 	tm->move.dst_slot = dst_slot;
553bd989ba3SJan Schmidt 	tm->move.nr_items = nr_items;
554bd989ba3SJan Schmidt 	tm->op = MOD_LOG_MOVE_KEYS;
555bd989ba3SJan Schmidt 
5565de865eeSFilipe David Borba Manana 	for (i = 0; i + dst_slot < src_slot && i < nr_items; i++) {
5575de865eeSFilipe David Borba Manana 		tm_list[i] = alloc_tree_mod_elem(eb, i + dst_slot,
558176ef8f5SDavid Sterba 		    MOD_LOG_KEY_REMOVE_WHILE_MOVING, GFP_NOFS);
5595de865eeSFilipe David Borba Manana 		if (!tm_list[i]) {
5605de865eeSFilipe David Borba Manana 			ret = -ENOMEM;
5615de865eeSFilipe David Borba Manana 			goto free_tms;
5625de865eeSFilipe David Borba Manana 		}
563bd989ba3SJan Schmidt 	}
564bd989ba3SJan Schmidt 
5656074d45fSDavid Sterba 	if (tree_mod_dont_log(eb->fs_info, eb))
5665de865eeSFilipe David Borba Manana 		goto free_tms;
5675de865eeSFilipe David Borba Manana 	locked = 1;
5685de865eeSFilipe David Borba Manana 
5695de865eeSFilipe David Borba Manana 	/*
5705de865eeSFilipe David Borba Manana 	 * When we override something during the move, we log these removals.
5715de865eeSFilipe David Borba Manana 	 * This can only happen when we move towards the beginning of the
5725de865eeSFilipe David Borba Manana 	 * buffer, i.e. dst_slot < src_slot.
5735de865eeSFilipe David Borba Manana 	 */
5745de865eeSFilipe David Borba Manana 	for (i = 0; i + dst_slot < src_slot && i < nr_items; i++) {
5756074d45fSDavid Sterba 		ret = __tree_mod_log_insert(eb->fs_info, tm_list[i]);
5765de865eeSFilipe David Borba Manana 		if (ret)
5775de865eeSFilipe David Borba Manana 			goto free_tms;
5785de865eeSFilipe David Borba Manana 	}
5795de865eeSFilipe David Borba Manana 
5806074d45fSDavid Sterba 	ret = __tree_mod_log_insert(eb->fs_info, tm);
5815de865eeSFilipe David Borba Manana 	if (ret)
5825de865eeSFilipe David Borba Manana 		goto free_tms;
583b1a09f1eSDavid Sterba 	write_unlock(&eb->fs_info->tree_mod_log_lock);
5845de865eeSFilipe David Borba Manana 	kfree(tm_list);
5855de865eeSFilipe David Borba Manana 
5865de865eeSFilipe David Borba Manana 	return 0;
5875de865eeSFilipe David Borba Manana free_tms:
5885de865eeSFilipe David Borba Manana 	for (i = 0; i < nr_items; i++) {
5895de865eeSFilipe David Borba Manana 		if (tm_list[i] && !RB_EMPTY_NODE(&tm_list[i]->node))
5906074d45fSDavid Sterba 			rb_erase(&tm_list[i]->node, &eb->fs_info->tree_mod_log);
5915de865eeSFilipe David Borba Manana 		kfree(tm_list[i]);
5925de865eeSFilipe David Borba Manana 	}
5935de865eeSFilipe David Borba Manana 	if (locked)
594b1a09f1eSDavid Sterba 		write_unlock(&eb->fs_info->tree_mod_log_lock);
5955de865eeSFilipe David Borba Manana 	kfree(tm_list);
5965de865eeSFilipe David Borba Manana 	kfree(tm);
5975de865eeSFilipe David Borba Manana 
5985de865eeSFilipe David Borba Manana 	return ret;
5995de865eeSFilipe David Borba Manana }
6005de865eeSFilipe David Borba Manana 
6015de865eeSFilipe David Borba Manana static inline int
6025de865eeSFilipe David Borba Manana __tree_mod_log_free_eb(struct btrfs_fs_info *fs_info,
6035de865eeSFilipe David Borba Manana 		       struct tree_mod_elem **tm_list,
6045de865eeSFilipe David Borba Manana 		       int nritems)
605097b8a7cSJan Schmidt {
6065de865eeSFilipe David Borba Manana 	int i, j;
607097b8a7cSJan Schmidt 	int ret;
608097b8a7cSJan Schmidt 
609097b8a7cSJan Schmidt 	for (i = nritems - 1; i >= 0; i--) {
6105de865eeSFilipe David Borba Manana 		ret = __tree_mod_log_insert(fs_info, tm_list[i]);
6115de865eeSFilipe David Borba Manana 		if (ret) {
6125de865eeSFilipe David Borba Manana 			for (j = nritems - 1; j > i; j--)
6135de865eeSFilipe David Borba Manana 				rb_erase(&tm_list[j]->node,
6145de865eeSFilipe David Borba Manana 					 &fs_info->tree_mod_log);
6155de865eeSFilipe David Borba Manana 			return ret;
616097b8a7cSJan Schmidt 		}
617097b8a7cSJan Schmidt 	}
618097b8a7cSJan Schmidt 
6195de865eeSFilipe David Borba Manana 	return 0;
6205de865eeSFilipe David Borba Manana }
6215de865eeSFilipe David Borba Manana 
62295b757c1SDavid Sterba static noinline int tree_mod_log_insert_root(struct extent_buffer *old_root,
62395b757c1SDavid Sterba 			 struct extent_buffer *new_root, int log_removal)
624bd989ba3SJan Schmidt {
62595b757c1SDavid Sterba 	struct btrfs_fs_info *fs_info = old_root->fs_info;
6265de865eeSFilipe David Borba Manana 	struct tree_mod_elem *tm = NULL;
6275de865eeSFilipe David Borba Manana 	struct tree_mod_elem **tm_list = NULL;
6285de865eeSFilipe David Borba Manana 	int nritems = 0;
6295de865eeSFilipe David Borba Manana 	int ret = 0;
6305de865eeSFilipe David Borba Manana 	int i;
631bd989ba3SJan Schmidt 
6325de865eeSFilipe David Borba Manana 	if (!tree_mod_need_log(fs_info, NULL))
633097b8a7cSJan Schmidt 		return 0;
634097b8a7cSJan Schmidt 
6355de865eeSFilipe David Borba Manana 	if (log_removal && btrfs_header_level(old_root) > 0) {
6365de865eeSFilipe David Borba Manana 		nritems = btrfs_header_nritems(old_root);
63731e818feSDavid Sterba 		tm_list = kcalloc(nritems, sizeof(struct tree_mod_elem *),
638bcc8e07fSDavid Sterba 				  GFP_NOFS);
6395de865eeSFilipe David Borba Manana 		if (!tm_list) {
6405de865eeSFilipe David Borba Manana 			ret = -ENOMEM;
6415de865eeSFilipe David Borba Manana 			goto free_tms;
6425de865eeSFilipe David Borba Manana 		}
6435de865eeSFilipe David Borba Manana 		for (i = 0; i < nritems; i++) {
6445de865eeSFilipe David Borba Manana 			tm_list[i] = alloc_tree_mod_elem(old_root, i,
645bcc8e07fSDavid Sterba 			    MOD_LOG_KEY_REMOVE_WHILE_FREEING, GFP_NOFS);
6465de865eeSFilipe David Borba Manana 			if (!tm_list[i]) {
6475de865eeSFilipe David Borba Manana 				ret = -ENOMEM;
6485de865eeSFilipe David Borba Manana 				goto free_tms;
6495de865eeSFilipe David Borba Manana 			}
6505de865eeSFilipe David Borba Manana 		}
6515de865eeSFilipe David Borba Manana 	}
652d9abbf1cSJan Schmidt 
653bcc8e07fSDavid Sterba 	tm = kzalloc(sizeof(*tm), GFP_NOFS);
6545de865eeSFilipe David Borba Manana 	if (!tm) {
6555de865eeSFilipe David Borba Manana 		ret = -ENOMEM;
6565de865eeSFilipe David Borba Manana 		goto free_tms;
6575de865eeSFilipe David Borba Manana 	}
658bd989ba3SJan Schmidt 
659298cfd36SChandan Rajendra 	tm->logical = new_root->start;
660bd989ba3SJan Schmidt 	tm->old_root.logical = old_root->start;
661bd989ba3SJan Schmidt 	tm->old_root.level = btrfs_header_level(old_root);
662bd989ba3SJan Schmidt 	tm->generation = btrfs_header_generation(old_root);
663bd989ba3SJan Schmidt 	tm->op = MOD_LOG_ROOT_REPLACE;
664bd989ba3SJan Schmidt 
6655de865eeSFilipe David Borba Manana 	if (tree_mod_dont_log(fs_info, NULL))
6665de865eeSFilipe David Borba Manana 		goto free_tms;
6675de865eeSFilipe David Borba Manana 
6685de865eeSFilipe David Borba Manana 	if (tm_list)
6695de865eeSFilipe David Borba Manana 		ret = __tree_mod_log_free_eb(fs_info, tm_list, nritems);
6705de865eeSFilipe David Borba Manana 	if (!ret)
6715de865eeSFilipe David Borba Manana 		ret = __tree_mod_log_insert(fs_info, tm);
6725de865eeSFilipe David Borba Manana 
673b1a09f1eSDavid Sterba 	write_unlock(&fs_info->tree_mod_log_lock);
6745de865eeSFilipe David Borba Manana 	if (ret)
6755de865eeSFilipe David Borba Manana 		goto free_tms;
6765de865eeSFilipe David Borba Manana 	kfree(tm_list);
6775de865eeSFilipe David Borba Manana 
6785de865eeSFilipe David Borba Manana 	return ret;
6795de865eeSFilipe David Borba Manana 
6805de865eeSFilipe David Borba Manana free_tms:
6815de865eeSFilipe David Borba Manana 	if (tm_list) {
6825de865eeSFilipe David Borba Manana 		for (i = 0; i < nritems; i++)
6835de865eeSFilipe David Borba Manana 			kfree(tm_list[i]);
6845de865eeSFilipe David Borba Manana 		kfree(tm_list);
6855de865eeSFilipe David Borba Manana 	}
6865de865eeSFilipe David Borba Manana 	kfree(tm);
6875de865eeSFilipe David Borba Manana 
6885de865eeSFilipe David Borba Manana 	return ret;
689bd989ba3SJan Schmidt }
690bd989ba3SJan Schmidt 
691bd989ba3SJan Schmidt static struct tree_mod_elem *
692bd989ba3SJan Schmidt __tree_mod_log_search(struct btrfs_fs_info *fs_info, u64 start, u64 min_seq,
693bd989ba3SJan Schmidt 		      int smallest)
694bd989ba3SJan Schmidt {
695bd989ba3SJan Schmidt 	struct rb_root *tm_root;
696bd989ba3SJan Schmidt 	struct rb_node *node;
697bd989ba3SJan Schmidt 	struct tree_mod_elem *cur = NULL;
698bd989ba3SJan Schmidt 	struct tree_mod_elem *found = NULL;
699bd989ba3SJan Schmidt 
700b1a09f1eSDavid Sterba 	read_lock(&fs_info->tree_mod_log_lock);
701bd989ba3SJan Schmidt 	tm_root = &fs_info->tree_mod_log;
702bd989ba3SJan Schmidt 	node = tm_root->rb_node;
703bd989ba3SJan Schmidt 	while (node) {
7046b4df8b6SGeliang Tang 		cur = rb_entry(node, struct tree_mod_elem, node);
705298cfd36SChandan Rajendra 		if (cur->logical < start) {
706bd989ba3SJan Schmidt 			node = node->rb_left;
707298cfd36SChandan Rajendra 		} else if (cur->logical > start) {
708bd989ba3SJan Schmidt 			node = node->rb_right;
709097b8a7cSJan Schmidt 		} else if (cur->seq < min_seq) {
710bd989ba3SJan Schmidt 			node = node->rb_left;
711bd989ba3SJan Schmidt 		} else if (!smallest) {
712bd989ba3SJan Schmidt 			/* we want the node with the highest seq */
713bd989ba3SJan Schmidt 			if (found)
714097b8a7cSJan Schmidt 				BUG_ON(found->seq > cur->seq);
715bd989ba3SJan Schmidt 			found = cur;
716bd989ba3SJan Schmidt 			node = node->rb_left;
717097b8a7cSJan Schmidt 		} else if (cur->seq > min_seq) {
718bd989ba3SJan Schmidt 			/* we want the node with the smallest seq */
719bd989ba3SJan Schmidt 			if (found)
720097b8a7cSJan Schmidt 				BUG_ON(found->seq < cur->seq);
721bd989ba3SJan Schmidt 			found = cur;
722bd989ba3SJan Schmidt 			node = node->rb_right;
723bd989ba3SJan Schmidt 		} else {
724bd989ba3SJan Schmidt 			found = cur;
725bd989ba3SJan Schmidt 			break;
726bd989ba3SJan Schmidt 		}
727bd989ba3SJan Schmidt 	}
728b1a09f1eSDavid Sterba 	read_unlock(&fs_info->tree_mod_log_lock);
729bd989ba3SJan Schmidt 
730bd989ba3SJan Schmidt 	return found;
731bd989ba3SJan Schmidt }
732bd989ba3SJan Schmidt 
733bd989ba3SJan Schmidt /*
734bd989ba3SJan Schmidt  * this returns the element from the log with the smallest time sequence
735bd989ba3SJan Schmidt  * value that's in the log (the oldest log item). any element with a time
736bd989ba3SJan Schmidt  * sequence lower than min_seq will be ignored.
737bd989ba3SJan Schmidt  */
738bd989ba3SJan Schmidt static struct tree_mod_elem *
739bd989ba3SJan Schmidt tree_mod_log_search_oldest(struct btrfs_fs_info *fs_info, u64 start,
740bd989ba3SJan Schmidt 			   u64 min_seq)
741bd989ba3SJan Schmidt {
742bd989ba3SJan Schmidt 	return __tree_mod_log_search(fs_info, start, min_seq, 1);
743bd989ba3SJan Schmidt }
744bd989ba3SJan Schmidt 
745bd989ba3SJan Schmidt /*
746bd989ba3SJan Schmidt  * this returns the element from the log with the largest time sequence
747bd989ba3SJan Schmidt  * value that's in the log (the most recent log item). any element with
748bd989ba3SJan Schmidt  * a time sequence lower than min_seq will be ignored.
749bd989ba3SJan Schmidt  */
750bd989ba3SJan Schmidt static struct tree_mod_elem *
751bd989ba3SJan Schmidt tree_mod_log_search(struct btrfs_fs_info *fs_info, u64 start, u64 min_seq)
752bd989ba3SJan Schmidt {
753bd989ba3SJan Schmidt 	return __tree_mod_log_search(fs_info, start, min_seq, 0);
754bd989ba3SJan Schmidt }
755bd989ba3SJan Schmidt 
7565de865eeSFilipe David Borba Manana static noinline int
757bd989ba3SJan Schmidt tree_mod_log_eb_copy(struct btrfs_fs_info *fs_info, struct extent_buffer *dst,
758bd989ba3SJan Schmidt 		     struct extent_buffer *src, unsigned long dst_offset,
75990f8d62eSJan Schmidt 		     unsigned long src_offset, int nr_items)
760bd989ba3SJan Schmidt {
7615de865eeSFilipe David Borba Manana 	int ret = 0;
7625de865eeSFilipe David Borba Manana 	struct tree_mod_elem **tm_list = NULL;
7635de865eeSFilipe David Borba Manana 	struct tree_mod_elem **tm_list_add, **tm_list_rem;
764bd989ba3SJan Schmidt 	int i;
7655de865eeSFilipe David Borba Manana 	int locked = 0;
766bd989ba3SJan Schmidt 
7675de865eeSFilipe David Borba Manana 	if (!tree_mod_need_log(fs_info, NULL))
7685de865eeSFilipe David Borba Manana 		return 0;
769bd989ba3SJan Schmidt 
770c8cc6341SJosef Bacik 	if (btrfs_header_level(dst) == 0 && btrfs_header_level(src) == 0)
7715de865eeSFilipe David Borba Manana 		return 0;
7725de865eeSFilipe David Borba Manana 
77331e818feSDavid Sterba 	tm_list = kcalloc(nr_items * 2, sizeof(struct tree_mod_elem *),
7745de865eeSFilipe David Borba Manana 			  GFP_NOFS);
7755de865eeSFilipe David Borba Manana 	if (!tm_list)
7765de865eeSFilipe David Borba Manana 		return -ENOMEM;
7775de865eeSFilipe David Borba Manana 
7785de865eeSFilipe David Borba Manana 	tm_list_add = tm_list;
7795de865eeSFilipe David Borba Manana 	tm_list_rem = tm_list + nr_items;
7805de865eeSFilipe David Borba Manana 	for (i = 0; i < nr_items; i++) {
7815de865eeSFilipe David Borba Manana 		tm_list_rem[i] = alloc_tree_mod_elem(src, i + src_offset,
7825de865eeSFilipe David Borba Manana 		    MOD_LOG_KEY_REMOVE, GFP_NOFS);
7835de865eeSFilipe David Borba Manana 		if (!tm_list_rem[i]) {
7845de865eeSFilipe David Borba Manana 			ret = -ENOMEM;
7855de865eeSFilipe David Borba Manana 			goto free_tms;
7865de865eeSFilipe David Borba Manana 		}
7875de865eeSFilipe David Borba Manana 
7885de865eeSFilipe David Borba Manana 		tm_list_add[i] = alloc_tree_mod_elem(dst, i + dst_offset,
7895de865eeSFilipe David Borba Manana 		    MOD_LOG_KEY_ADD, GFP_NOFS);
7905de865eeSFilipe David Borba Manana 		if (!tm_list_add[i]) {
7915de865eeSFilipe David Borba Manana 			ret = -ENOMEM;
7925de865eeSFilipe David Borba Manana 			goto free_tms;
7935de865eeSFilipe David Borba Manana 		}
7945de865eeSFilipe David Borba Manana 	}
7955de865eeSFilipe David Borba Manana 
7965de865eeSFilipe David Borba Manana 	if (tree_mod_dont_log(fs_info, NULL))
7975de865eeSFilipe David Borba Manana 		goto free_tms;
7985de865eeSFilipe David Borba Manana 	locked = 1;
799bd989ba3SJan Schmidt 
800bd989ba3SJan Schmidt 	for (i = 0; i < nr_items; i++) {
8015de865eeSFilipe David Borba Manana 		ret = __tree_mod_log_insert(fs_info, tm_list_rem[i]);
8025de865eeSFilipe David Borba Manana 		if (ret)
8035de865eeSFilipe David Borba Manana 			goto free_tms;
8045de865eeSFilipe David Borba Manana 		ret = __tree_mod_log_insert(fs_info, tm_list_add[i]);
8055de865eeSFilipe David Borba Manana 		if (ret)
8065de865eeSFilipe David Borba Manana 			goto free_tms;
807bd989ba3SJan Schmidt 	}
8085de865eeSFilipe David Borba Manana 
809b1a09f1eSDavid Sterba 	write_unlock(&fs_info->tree_mod_log_lock);
8105de865eeSFilipe David Borba Manana 	kfree(tm_list);
8115de865eeSFilipe David Borba Manana 
8125de865eeSFilipe David Borba Manana 	return 0;
8135de865eeSFilipe David Borba Manana 
8145de865eeSFilipe David Borba Manana free_tms:
8155de865eeSFilipe David Borba Manana 	for (i = 0; i < nr_items * 2; i++) {
8165de865eeSFilipe David Borba Manana 		if (tm_list[i] && !RB_EMPTY_NODE(&tm_list[i]->node))
8175de865eeSFilipe David Borba Manana 			rb_erase(&tm_list[i]->node, &fs_info->tree_mod_log);
8185de865eeSFilipe David Borba Manana 		kfree(tm_list[i]);
8195de865eeSFilipe David Borba Manana 	}
8205de865eeSFilipe David Borba Manana 	if (locked)
821b1a09f1eSDavid Sterba 		write_unlock(&fs_info->tree_mod_log_lock);
8225de865eeSFilipe David Borba Manana 	kfree(tm_list);
8235de865eeSFilipe David Borba Manana 
8245de865eeSFilipe David Borba Manana 	return ret;
825bd989ba3SJan Schmidt }
826bd989ba3SJan Schmidt 
827db7279a2SDavid Sterba static noinline int tree_mod_log_free_eb(struct extent_buffer *eb)
828bd989ba3SJan Schmidt {
8295de865eeSFilipe David Borba Manana 	struct tree_mod_elem **tm_list = NULL;
8305de865eeSFilipe David Borba Manana 	int nritems = 0;
8315de865eeSFilipe David Borba Manana 	int i;
8325de865eeSFilipe David Borba Manana 	int ret = 0;
8335de865eeSFilipe David Borba Manana 
8345de865eeSFilipe David Borba Manana 	if (btrfs_header_level(eb) == 0)
8355de865eeSFilipe David Borba Manana 		return 0;
8365de865eeSFilipe David Borba Manana 
837db7279a2SDavid Sterba 	if (!tree_mod_need_log(eb->fs_info, NULL))
8385de865eeSFilipe David Borba Manana 		return 0;
8395de865eeSFilipe David Borba Manana 
8405de865eeSFilipe David Borba Manana 	nritems = btrfs_header_nritems(eb);
84131e818feSDavid Sterba 	tm_list = kcalloc(nritems, sizeof(struct tree_mod_elem *), GFP_NOFS);
8425de865eeSFilipe David Borba Manana 	if (!tm_list)
8435de865eeSFilipe David Borba Manana 		return -ENOMEM;
8445de865eeSFilipe David Borba Manana 
8455de865eeSFilipe David Borba Manana 	for (i = 0; i < nritems; i++) {
8465de865eeSFilipe David Borba Manana 		tm_list[i] = alloc_tree_mod_elem(eb, i,
8475de865eeSFilipe David Borba Manana 		    MOD_LOG_KEY_REMOVE_WHILE_FREEING, GFP_NOFS);
8485de865eeSFilipe David Borba Manana 		if (!tm_list[i]) {
8495de865eeSFilipe David Borba Manana 			ret = -ENOMEM;
8505de865eeSFilipe David Borba Manana 			goto free_tms;
8515de865eeSFilipe David Borba Manana 		}
8525de865eeSFilipe David Borba Manana 	}
8535de865eeSFilipe David Borba Manana 
854db7279a2SDavid Sterba 	if (tree_mod_dont_log(eb->fs_info, eb))
8555de865eeSFilipe David Borba Manana 		goto free_tms;
8565de865eeSFilipe David Borba Manana 
857db7279a2SDavid Sterba 	ret = __tree_mod_log_free_eb(eb->fs_info, tm_list, nritems);
858b1a09f1eSDavid Sterba 	write_unlock(&eb->fs_info->tree_mod_log_lock);
8595de865eeSFilipe David Borba Manana 	if (ret)
8605de865eeSFilipe David Borba Manana 		goto free_tms;
8615de865eeSFilipe David Borba Manana 	kfree(tm_list);
8625de865eeSFilipe David Borba Manana 
8635de865eeSFilipe David Borba Manana 	return 0;
8645de865eeSFilipe David Borba Manana 
8655de865eeSFilipe David Borba Manana free_tms:
8665de865eeSFilipe David Borba Manana 	for (i = 0; i < nritems; i++)
8675de865eeSFilipe David Borba Manana 		kfree(tm_list[i]);
8685de865eeSFilipe David Borba Manana 	kfree(tm_list);
8695de865eeSFilipe David Borba Manana 
8705de865eeSFilipe David Borba Manana 	return ret;
871bd989ba3SJan Schmidt }
872bd989ba3SJan Schmidt 
873d352ac68SChris Mason /*
8745d4f98a2SYan Zheng  * check if the tree block can be shared by multiple trees
8755d4f98a2SYan Zheng  */
8765d4f98a2SYan Zheng int btrfs_block_can_be_shared(struct btrfs_root *root,
8775d4f98a2SYan Zheng 			      struct extent_buffer *buf)
8785d4f98a2SYan Zheng {
8795d4f98a2SYan Zheng 	/*
88001327610SNicholas D Steeves 	 * Tree blocks not in reference counted trees and tree roots
8815d4f98a2SYan Zheng 	 * are never shared. If a block was allocated after the last
8825d4f98a2SYan Zheng 	 * snapshot and the block was not allocated by tree relocation,
8835d4f98a2SYan Zheng 	 * we know the block is not shared.
8845d4f98a2SYan Zheng 	 */
88527cdeb70SMiao Xie 	if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
8865d4f98a2SYan Zheng 	    buf != root->node && buf != root->commit_root &&
8875d4f98a2SYan Zheng 	    (btrfs_header_generation(buf) <=
8885d4f98a2SYan Zheng 	     btrfs_root_last_snapshot(&root->root_item) ||
8895d4f98a2SYan Zheng 	     btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)))
8905d4f98a2SYan Zheng 		return 1;
891a79865c6SNikolay Borisov 
8925d4f98a2SYan Zheng 	return 0;
8935d4f98a2SYan Zheng }
8945d4f98a2SYan Zheng 
8955d4f98a2SYan Zheng static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans,
8965d4f98a2SYan Zheng 				       struct btrfs_root *root,
8975d4f98a2SYan Zheng 				       struct extent_buffer *buf,
898f0486c68SYan, Zheng 				       struct extent_buffer *cow,
899f0486c68SYan, Zheng 				       int *last_ref)
9005d4f98a2SYan Zheng {
9010b246afaSJeff Mahoney 	struct btrfs_fs_info *fs_info = root->fs_info;
9025d4f98a2SYan Zheng 	u64 refs;
9035d4f98a2SYan Zheng 	u64 owner;
9045d4f98a2SYan Zheng 	u64 flags;
9055d4f98a2SYan Zheng 	u64 new_flags = 0;
9065d4f98a2SYan Zheng 	int ret;
9075d4f98a2SYan Zheng 
9085d4f98a2SYan Zheng 	/*
9095d4f98a2SYan Zheng 	 * Backrefs update rules:
9105d4f98a2SYan Zheng 	 *
9115d4f98a2SYan Zheng 	 * Always use full backrefs for extent pointers in tree block
9125d4f98a2SYan Zheng 	 * allocated by tree relocation.
9135d4f98a2SYan Zheng 	 *
9145d4f98a2SYan Zheng 	 * If a shared tree block is no longer referenced by its owner
9155d4f98a2SYan Zheng 	 * tree (btrfs_header_owner(buf) == root->root_key.objectid),
9165d4f98a2SYan Zheng 	 * use full backrefs for extent pointers in tree block.
9175d4f98a2SYan Zheng 	 *
9185d4f98a2SYan Zheng 	 * If a tree block is been relocating
9195d4f98a2SYan Zheng 	 * (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID),
9205d4f98a2SYan Zheng 	 * use full backrefs for extent pointers in tree block.
9215d4f98a2SYan Zheng 	 * The reason for this is some operations (such as drop tree)
9225d4f98a2SYan Zheng 	 * are only allowed for blocks use full backrefs.
9235d4f98a2SYan Zheng 	 */
9245d4f98a2SYan Zheng 
9255d4f98a2SYan Zheng 	if (btrfs_block_can_be_shared(root, buf)) {
9262ff7e61eSJeff Mahoney 		ret = btrfs_lookup_extent_info(trans, fs_info, buf->start,
9273173a18fSJosef Bacik 					       btrfs_header_level(buf), 1,
9283173a18fSJosef Bacik 					       &refs, &flags);
929be1a5564SMark Fasheh 		if (ret)
930be1a5564SMark Fasheh 			return ret;
931e5df9573SMark Fasheh 		if (refs == 0) {
932e5df9573SMark Fasheh 			ret = -EROFS;
9330b246afaSJeff Mahoney 			btrfs_handle_fs_error(fs_info, ret, NULL);
934e5df9573SMark Fasheh 			return ret;
935e5df9573SMark Fasheh 		}
9365d4f98a2SYan Zheng 	} else {
9375d4f98a2SYan Zheng 		refs = 1;
9385d4f98a2SYan Zheng 		if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID ||
9395d4f98a2SYan Zheng 		    btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV)
9405d4f98a2SYan Zheng 			flags = BTRFS_BLOCK_FLAG_FULL_BACKREF;
9415d4f98a2SYan Zheng 		else
9425d4f98a2SYan Zheng 			flags = 0;
9435d4f98a2SYan Zheng 	}
9445d4f98a2SYan Zheng 
9455d4f98a2SYan Zheng 	owner = btrfs_header_owner(buf);
9465d4f98a2SYan Zheng 	BUG_ON(owner == BTRFS_TREE_RELOC_OBJECTID &&
9475d4f98a2SYan Zheng 	       !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF));
9485d4f98a2SYan Zheng 
9495d4f98a2SYan Zheng 	if (refs > 1) {
9505d4f98a2SYan Zheng 		if ((owner == root->root_key.objectid ||
9515d4f98a2SYan Zheng 		     root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) &&
9525d4f98a2SYan Zheng 		    !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) {
953e339a6b0SJosef Bacik 			ret = btrfs_inc_ref(trans, root, buf, 1);
954692826b2SJeff Mahoney 			if (ret)
955692826b2SJeff Mahoney 				return ret;
9565d4f98a2SYan Zheng 
9575d4f98a2SYan Zheng 			if (root->root_key.objectid ==
9585d4f98a2SYan Zheng 			    BTRFS_TREE_RELOC_OBJECTID) {
959e339a6b0SJosef Bacik 				ret = btrfs_dec_ref(trans, root, buf, 0);
960692826b2SJeff Mahoney 				if (ret)
961692826b2SJeff Mahoney 					return ret;
962e339a6b0SJosef Bacik 				ret = btrfs_inc_ref(trans, root, cow, 1);
963692826b2SJeff Mahoney 				if (ret)
964692826b2SJeff Mahoney 					return ret;
9655d4f98a2SYan Zheng 			}
9665d4f98a2SYan Zheng 			new_flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF;
9675d4f98a2SYan Zheng 		} else {
9685d4f98a2SYan Zheng 
9695d4f98a2SYan Zheng 			if (root->root_key.objectid ==
9705d4f98a2SYan Zheng 			    BTRFS_TREE_RELOC_OBJECTID)
971e339a6b0SJosef Bacik 				ret = btrfs_inc_ref(trans, root, cow, 1);
9725d4f98a2SYan Zheng 			else
973e339a6b0SJosef Bacik 				ret = btrfs_inc_ref(trans, root, cow, 0);
974692826b2SJeff Mahoney 			if (ret)
975692826b2SJeff Mahoney 				return ret;
9765d4f98a2SYan Zheng 		}
9775d4f98a2SYan Zheng 		if (new_flags != 0) {
978b1c79e09SJosef Bacik 			int level = btrfs_header_level(buf);
979b1c79e09SJosef Bacik 
9802ff7e61eSJeff Mahoney 			ret = btrfs_set_disk_extent_flags(trans, fs_info,
9815d4f98a2SYan Zheng 							  buf->start,
9825d4f98a2SYan Zheng 							  buf->len,
983b1c79e09SJosef Bacik 							  new_flags, level, 0);
984be1a5564SMark Fasheh 			if (ret)
985be1a5564SMark Fasheh 				return ret;
9865d4f98a2SYan Zheng 		}
9875d4f98a2SYan Zheng 	} else {
9885d4f98a2SYan Zheng 		if (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
9895d4f98a2SYan Zheng 			if (root->root_key.objectid ==
9905d4f98a2SYan Zheng 			    BTRFS_TREE_RELOC_OBJECTID)
991e339a6b0SJosef Bacik 				ret = btrfs_inc_ref(trans, root, cow, 1);
9925d4f98a2SYan Zheng 			else
993e339a6b0SJosef Bacik 				ret = btrfs_inc_ref(trans, root, cow, 0);
994692826b2SJeff Mahoney 			if (ret)
995692826b2SJeff Mahoney 				return ret;
996e339a6b0SJosef Bacik 			ret = btrfs_dec_ref(trans, root, buf, 1);
997692826b2SJeff Mahoney 			if (ret)
998692826b2SJeff Mahoney 				return ret;
9995d4f98a2SYan Zheng 		}
10007c302b49SDavid Sterba 		clean_tree_block(fs_info, buf);
1001f0486c68SYan, Zheng 		*last_ref = 1;
10025d4f98a2SYan Zheng 	}
10035d4f98a2SYan Zheng 	return 0;
10045d4f98a2SYan Zheng }
10055d4f98a2SYan Zheng 
10065d4f98a2SYan Zheng /*
1007d397712bSChris Mason  * does the dirty work in cow of a single block.  The parent block (if
1008d397712bSChris Mason  * supplied) is updated to point to the new cow copy.  The new buffer is marked
1009d397712bSChris Mason  * dirty and returned locked.  If you modify the block it needs to be marked
1010d397712bSChris Mason  * dirty again.
1011d352ac68SChris Mason  *
1012d352ac68SChris Mason  * search_start -- an allocation hint for the new block
1013d352ac68SChris Mason  *
1014d397712bSChris Mason  * empty_size -- a hint that you plan on doing more cow.  This is the size in
1015d397712bSChris Mason  * bytes the allocator should try to find free next to the block it returns.
1016d397712bSChris Mason  * This is just a hint and may be ignored by the allocator.
1017d352ac68SChris Mason  */
1018d397712bSChris Mason static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans,
10195f39d397SChris Mason 			     struct btrfs_root *root,
10205f39d397SChris Mason 			     struct extent_buffer *buf,
10215f39d397SChris Mason 			     struct extent_buffer *parent, int parent_slot,
10225f39d397SChris Mason 			     struct extent_buffer **cow_ret,
10239fa8cfe7SChris Mason 			     u64 search_start, u64 empty_size)
10246702ed49SChris Mason {
10250b246afaSJeff Mahoney 	struct btrfs_fs_info *fs_info = root->fs_info;
10265d4f98a2SYan Zheng 	struct btrfs_disk_key disk_key;
10275f39d397SChris Mason 	struct extent_buffer *cow;
1028be1a5564SMark Fasheh 	int level, ret;
1029f0486c68SYan, Zheng 	int last_ref = 0;
1030925baeddSChris Mason 	int unlock_orig = 0;
10310f5053ebSGoldwyn Rodrigues 	u64 parent_start = 0;
10326702ed49SChris Mason 
1033925baeddSChris Mason 	if (*cow_ret == buf)
1034925baeddSChris Mason 		unlock_orig = 1;
1035925baeddSChris Mason 
1036b9447ef8SChris Mason 	btrfs_assert_tree_locked(buf);
1037925baeddSChris Mason 
103827cdeb70SMiao Xie 	WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
10390b246afaSJeff Mahoney 		trans->transid != fs_info->running_transaction->transid);
104027cdeb70SMiao Xie 	WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
104127cdeb70SMiao Xie 		trans->transid != root->last_trans);
10425f39d397SChris Mason 
10437bb86316SChris Mason 	level = btrfs_header_level(buf);
104431840ae1SZheng Yan 
10455d4f98a2SYan Zheng 	if (level == 0)
10465d4f98a2SYan Zheng 		btrfs_item_key(buf, &disk_key, 0);
10475d4f98a2SYan Zheng 	else
10485d4f98a2SYan Zheng 		btrfs_node_key(buf, &disk_key, 0);
10495d4f98a2SYan Zheng 
10500f5053ebSGoldwyn Rodrigues 	if ((root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && parent)
10515d4f98a2SYan Zheng 		parent_start = parent->start;
10525d4f98a2SYan Zheng 
10534d75f8a9SDavid Sterba 	cow = btrfs_alloc_tree_block(trans, root, parent_start,
10544d75f8a9SDavid Sterba 			root->root_key.objectid, &disk_key, level,
10554d75f8a9SDavid Sterba 			search_start, empty_size);
10566702ed49SChris Mason 	if (IS_ERR(cow))
10576702ed49SChris Mason 		return PTR_ERR(cow);
10586702ed49SChris Mason 
1059b4ce94deSChris Mason 	/* cow is set to blocking by btrfs_init_new_buffer */
1060b4ce94deSChris Mason 
106158e8012cSDavid Sterba 	copy_extent_buffer_full(cow, buf);
1062db94535dSChris Mason 	btrfs_set_header_bytenr(cow, cow->start);
10635f39d397SChris Mason 	btrfs_set_header_generation(cow, trans->transid);
10645d4f98a2SYan Zheng 	btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV);
10655d4f98a2SYan Zheng 	btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN |
10665d4f98a2SYan Zheng 				     BTRFS_HEADER_FLAG_RELOC);
10675d4f98a2SYan Zheng 	if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID)
10685d4f98a2SYan Zheng 		btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC);
10695d4f98a2SYan Zheng 	else
10705f39d397SChris Mason 		btrfs_set_header_owner(cow, root->root_key.objectid);
10716702ed49SChris Mason 
10720b246afaSJeff Mahoney 	write_extent_buffer_fsid(cow, fs_info->fsid);
10732b82032cSYan Zheng 
1074be1a5564SMark Fasheh 	ret = update_ref_for_cow(trans, root, buf, cow, &last_ref);
1075b68dc2a9SMark Fasheh 	if (ret) {
107666642832SJeff Mahoney 		btrfs_abort_transaction(trans, ret);
1077b68dc2a9SMark Fasheh 		return ret;
1078b68dc2a9SMark Fasheh 	}
10791a40e23bSZheng Yan 
108027cdeb70SMiao Xie 	if (test_bit(BTRFS_ROOT_REF_COWS, &root->state)) {
108183d4cfd4SJosef Bacik 		ret = btrfs_reloc_cow_block(trans, root, buf, cow);
108293314e3bSZhaolei 		if (ret) {
108366642832SJeff Mahoney 			btrfs_abort_transaction(trans, ret);
108483d4cfd4SJosef Bacik 			return ret;
108583d4cfd4SJosef Bacik 		}
108693314e3bSZhaolei 	}
10873fd0a558SYan, Zheng 
10886702ed49SChris Mason 	if (buf == root->node) {
1089925baeddSChris Mason 		WARN_ON(parent && parent != buf);
10905d4f98a2SYan Zheng 		if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID ||
10915d4f98a2SYan Zheng 		    btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV)
10925d4f98a2SYan Zheng 			parent_start = buf->start;
1093925baeddSChris Mason 
10945f39d397SChris Mason 		extent_buffer_get(cow);
1095d9d19a01SDavid Sterba 		ret = tree_mod_log_insert_root(root->node, cow, 1);
1096d9d19a01SDavid Sterba 		BUG_ON(ret < 0);
1097240f62c8SChris Mason 		rcu_assign_pointer(root->node, cow);
1098925baeddSChris Mason 
1099f0486c68SYan, Zheng 		btrfs_free_tree_block(trans, root, buf, parent_start,
11005581a51aSJan Schmidt 				      last_ref);
11015f39d397SChris Mason 		free_extent_buffer(buf);
11020b86a832SChris Mason 		add_root_to_dirty_list(root);
11036702ed49SChris Mason 	} else {
11045d4f98a2SYan Zheng 		WARN_ON(trans->transid != btrfs_header_generation(parent));
1105e09c2efeSDavid Sterba 		tree_mod_log_insert_key(parent, parent_slot,
1106c8cc6341SJosef Bacik 					MOD_LOG_KEY_REPLACE, GFP_NOFS);
11075f39d397SChris Mason 		btrfs_set_node_blockptr(parent, parent_slot,
1108db94535dSChris Mason 					cow->start);
110974493f7aSChris Mason 		btrfs_set_node_ptr_generation(parent, parent_slot,
111074493f7aSChris Mason 					      trans->transid);
11116702ed49SChris Mason 		btrfs_mark_buffer_dirty(parent);
11125de865eeSFilipe David Borba Manana 		if (last_ref) {
1113db7279a2SDavid Sterba 			ret = tree_mod_log_free_eb(buf);
11145de865eeSFilipe David Borba Manana 			if (ret) {
111566642832SJeff Mahoney 				btrfs_abort_transaction(trans, ret);
11165de865eeSFilipe David Borba Manana 				return ret;
11175de865eeSFilipe David Borba Manana 			}
11185de865eeSFilipe David Borba Manana 		}
1119f0486c68SYan, Zheng 		btrfs_free_tree_block(trans, root, buf, parent_start,
11205581a51aSJan Schmidt 				      last_ref);
11216702ed49SChris Mason 	}
1122925baeddSChris Mason 	if (unlock_orig)
1123925baeddSChris Mason 		btrfs_tree_unlock(buf);
11243083ee2eSJosef Bacik 	free_extent_buffer_stale(buf);
11256702ed49SChris Mason 	btrfs_mark_buffer_dirty(cow);
11266702ed49SChris Mason 	*cow_ret = cow;
11276702ed49SChris Mason 	return 0;
11286702ed49SChris Mason }
11296702ed49SChris Mason 
11305d9e75c4SJan Schmidt /*
11315d9e75c4SJan Schmidt  * returns the logical address of the oldest predecessor of the given root.
11325d9e75c4SJan Schmidt  * entries older than time_seq are ignored.
11335d9e75c4SJan Schmidt  */
1134bcd24dabSDavid Sterba static struct tree_mod_elem *__tree_mod_log_oldest_root(
113530b0463aSJan Schmidt 		struct extent_buffer *eb_root, u64 time_seq)
11365d9e75c4SJan Schmidt {
11375d9e75c4SJan Schmidt 	struct tree_mod_elem *tm;
11385d9e75c4SJan Schmidt 	struct tree_mod_elem *found = NULL;
113930b0463aSJan Schmidt 	u64 root_logical = eb_root->start;
11405d9e75c4SJan Schmidt 	int looped = 0;
11415d9e75c4SJan Schmidt 
11425d9e75c4SJan Schmidt 	if (!time_seq)
114335a3621bSStefan Behrens 		return NULL;
11445d9e75c4SJan Schmidt 
11455d9e75c4SJan Schmidt 	/*
1146298cfd36SChandan Rajendra 	 * the very last operation that's logged for a root is the
1147298cfd36SChandan Rajendra 	 * replacement operation (if it is replaced at all). this has
1148298cfd36SChandan Rajendra 	 * the logical address of the *new* root, making it the very
1149298cfd36SChandan Rajendra 	 * first operation that's logged for this root.
11505d9e75c4SJan Schmidt 	 */
11515d9e75c4SJan Schmidt 	while (1) {
1152bcd24dabSDavid Sterba 		tm = tree_mod_log_search_oldest(eb_root->fs_info, root_logical,
11535d9e75c4SJan Schmidt 						time_seq);
11545d9e75c4SJan Schmidt 		if (!looped && !tm)
115535a3621bSStefan Behrens 			return NULL;
11565d9e75c4SJan Schmidt 		/*
115728da9fb4SJan Schmidt 		 * if there are no tree operation for the oldest root, we simply
115828da9fb4SJan Schmidt 		 * return it. this should only happen if that (old) root is at
115928da9fb4SJan Schmidt 		 * level 0.
11605d9e75c4SJan Schmidt 		 */
116128da9fb4SJan Schmidt 		if (!tm)
116228da9fb4SJan Schmidt 			break;
11635d9e75c4SJan Schmidt 
116428da9fb4SJan Schmidt 		/*
116528da9fb4SJan Schmidt 		 * if there's an operation that's not a root replacement, we
116628da9fb4SJan Schmidt 		 * found the oldest version of our root. normally, we'll find a
116728da9fb4SJan Schmidt 		 * MOD_LOG_KEY_REMOVE_WHILE_FREEING operation here.
116828da9fb4SJan Schmidt 		 */
11695d9e75c4SJan Schmidt 		if (tm->op != MOD_LOG_ROOT_REPLACE)
11705d9e75c4SJan Schmidt 			break;
11715d9e75c4SJan Schmidt 
11725d9e75c4SJan Schmidt 		found = tm;
11735d9e75c4SJan Schmidt 		root_logical = tm->old_root.logical;
11745d9e75c4SJan Schmidt 		looped = 1;
11755d9e75c4SJan Schmidt 	}
11765d9e75c4SJan Schmidt 
1177a95236d9SJan Schmidt 	/* if there's no old root to return, return what we found instead */
1178a95236d9SJan Schmidt 	if (!found)
1179a95236d9SJan Schmidt 		found = tm;
1180a95236d9SJan Schmidt 
11815d9e75c4SJan Schmidt 	return found;
11825d9e75c4SJan Schmidt }
11835d9e75c4SJan Schmidt 
11845d9e75c4SJan Schmidt /*
11855d9e75c4SJan Schmidt  * tm is a pointer to the first operation to rewind within eb. then, all
118601327610SNicholas D Steeves  * previous operations will be rewound (until we reach something older than
11875d9e75c4SJan Schmidt  * time_seq).
11885d9e75c4SJan Schmidt  */
11895d9e75c4SJan Schmidt static void
1190f1ca7e98SJosef Bacik __tree_mod_log_rewind(struct btrfs_fs_info *fs_info, struct extent_buffer *eb,
1191f1ca7e98SJosef Bacik 		      u64 time_seq, struct tree_mod_elem *first_tm)
11925d9e75c4SJan Schmidt {
11935d9e75c4SJan Schmidt 	u32 n;
11945d9e75c4SJan Schmidt 	struct rb_node *next;
11955d9e75c4SJan Schmidt 	struct tree_mod_elem *tm = first_tm;
11965d9e75c4SJan Schmidt 	unsigned long o_dst;
11975d9e75c4SJan Schmidt 	unsigned long o_src;
11985d9e75c4SJan Schmidt 	unsigned long p_size = sizeof(struct btrfs_key_ptr);
11995d9e75c4SJan Schmidt 
12005d9e75c4SJan Schmidt 	n = btrfs_header_nritems(eb);
1201b1a09f1eSDavid Sterba 	read_lock(&fs_info->tree_mod_log_lock);
1202097b8a7cSJan Schmidt 	while (tm && tm->seq >= time_seq) {
12035d9e75c4SJan Schmidt 		/*
12045d9e75c4SJan Schmidt 		 * all the operations are recorded with the operator used for
12055d9e75c4SJan Schmidt 		 * the modification. as we're going backwards, we do the
12065d9e75c4SJan Schmidt 		 * opposite of each operation here.
12075d9e75c4SJan Schmidt 		 */
12085d9e75c4SJan Schmidt 		switch (tm->op) {
12095d9e75c4SJan Schmidt 		case MOD_LOG_KEY_REMOVE_WHILE_FREEING:
12105d9e75c4SJan Schmidt 			BUG_ON(tm->slot < n);
12111c697d4aSEric Sandeen 			/* Fallthrough */
121295c80bb1SLiu Bo 		case MOD_LOG_KEY_REMOVE_WHILE_MOVING:
12134c3e6969SChris Mason 		case MOD_LOG_KEY_REMOVE:
12145d9e75c4SJan Schmidt 			btrfs_set_node_key(eb, &tm->key, tm->slot);
12155d9e75c4SJan Schmidt 			btrfs_set_node_blockptr(eb, tm->slot, tm->blockptr);
12165d9e75c4SJan Schmidt 			btrfs_set_node_ptr_generation(eb, tm->slot,
12175d9e75c4SJan Schmidt 						      tm->generation);
12184c3e6969SChris Mason 			n++;
12195d9e75c4SJan Schmidt 			break;
12205d9e75c4SJan Schmidt 		case MOD_LOG_KEY_REPLACE:
12215d9e75c4SJan Schmidt 			BUG_ON(tm->slot >= n);
12225d9e75c4SJan Schmidt 			btrfs_set_node_key(eb, &tm->key, tm->slot);
12235d9e75c4SJan Schmidt 			btrfs_set_node_blockptr(eb, tm->slot, tm->blockptr);
12245d9e75c4SJan Schmidt 			btrfs_set_node_ptr_generation(eb, tm->slot,
12255d9e75c4SJan Schmidt 						      tm->generation);
12265d9e75c4SJan Schmidt 			break;
12275d9e75c4SJan Schmidt 		case MOD_LOG_KEY_ADD:
122819956c7eSJan Schmidt 			/* if a move operation is needed it's in the log */
12295d9e75c4SJan Schmidt 			n--;
12305d9e75c4SJan Schmidt 			break;
12315d9e75c4SJan Schmidt 		case MOD_LOG_MOVE_KEYS:
1232c3193108SJan Schmidt 			o_dst = btrfs_node_key_ptr_offset(tm->slot);
1233c3193108SJan Schmidt 			o_src = btrfs_node_key_ptr_offset(tm->move.dst_slot);
1234c3193108SJan Schmidt 			memmove_extent_buffer(eb, o_dst, o_src,
12355d9e75c4SJan Schmidt 					      tm->move.nr_items * p_size);
12365d9e75c4SJan Schmidt 			break;
12375d9e75c4SJan Schmidt 		case MOD_LOG_ROOT_REPLACE:
12385d9e75c4SJan Schmidt 			/*
12395d9e75c4SJan Schmidt 			 * this operation is special. for roots, this must be
12405d9e75c4SJan Schmidt 			 * handled explicitly before rewinding.
12415d9e75c4SJan Schmidt 			 * for non-roots, this operation may exist if the node
12425d9e75c4SJan Schmidt 			 * was a root: root A -> child B; then A gets empty and
12435d9e75c4SJan Schmidt 			 * B is promoted to the new root. in the mod log, we'll
12445d9e75c4SJan Schmidt 			 * have a root-replace operation for B, a tree block
12455d9e75c4SJan Schmidt 			 * that is no root. we simply ignore that operation.
12465d9e75c4SJan Schmidt 			 */
12475d9e75c4SJan Schmidt 			break;
12485d9e75c4SJan Schmidt 		}
12495d9e75c4SJan Schmidt 		next = rb_next(&tm->node);
12505d9e75c4SJan Schmidt 		if (!next)
12515d9e75c4SJan Schmidt 			break;
12526b4df8b6SGeliang Tang 		tm = rb_entry(next, struct tree_mod_elem, node);
1253298cfd36SChandan Rajendra 		if (tm->logical != first_tm->logical)
12545d9e75c4SJan Schmidt 			break;
12555d9e75c4SJan Schmidt 	}
1256b1a09f1eSDavid Sterba 	read_unlock(&fs_info->tree_mod_log_lock);
12575d9e75c4SJan Schmidt 	btrfs_set_header_nritems(eb, n);
12585d9e75c4SJan Schmidt }
12595d9e75c4SJan Schmidt 
126047fb091fSJan Schmidt /*
126101327610SNicholas D Steeves  * Called with eb read locked. If the buffer cannot be rewound, the same buffer
126247fb091fSJan Schmidt  * is returned. If rewind operations happen, a fresh buffer is returned. The
126347fb091fSJan Schmidt  * returned buffer is always read-locked. If the returned buffer is not the
126447fb091fSJan Schmidt  * input buffer, the lock on the input buffer is released and the input buffer
126547fb091fSJan Schmidt  * is freed (its refcount is decremented).
126647fb091fSJan Schmidt  */
12675d9e75c4SJan Schmidt static struct extent_buffer *
12689ec72677SJosef Bacik tree_mod_log_rewind(struct btrfs_fs_info *fs_info, struct btrfs_path *path,
12699ec72677SJosef Bacik 		    struct extent_buffer *eb, u64 time_seq)
12705d9e75c4SJan Schmidt {
12715d9e75c4SJan Schmidt 	struct extent_buffer *eb_rewin;
12725d9e75c4SJan Schmidt 	struct tree_mod_elem *tm;
12735d9e75c4SJan Schmidt 
12745d9e75c4SJan Schmidt 	if (!time_seq)
12755d9e75c4SJan Schmidt 		return eb;
12765d9e75c4SJan Schmidt 
12775d9e75c4SJan Schmidt 	if (btrfs_header_level(eb) == 0)
12785d9e75c4SJan Schmidt 		return eb;
12795d9e75c4SJan Schmidt 
12805d9e75c4SJan Schmidt 	tm = tree_mod_log_search(fs_info, eb->start, time_seq);
12815d9e75c4SJan Schmidt 	if (!tm)
12825d9e75c4SJan Schmidt 		return eb;
12835d9e75c4SJan Schmidt 
12849ec72677SJosef Bacik 	btrfs_set_path_blocking(path);
12859ec72677SJosef Bacik 	btrfs_set_lock_blocking_rw(eb, BTRFS_READ_LOCK);
12869ec72677SJosef Bacik 
12875d9e75c4SJan Schmidt 	if (tm->op == MOD_LOG_KEY_REMOVE_WHILE_FREEING) {
12885d9e75c4SJan Schmidt 		BUG_ON(tm->slot != 0);
1289da17066cSJeff Mahoney 		eb_rewin = alloc_dummy_extent_buffer(fs_info, eb->start);
1290db7f3436SJosef Bacik 		if (!eb_rewin) {
12919ec72677SJosef Bacik 			btrfs_tree_read_unlock_blocking(eb);
1292db7f3436SJosef Bacik 			free_extent_buffer(eb);
1293db7f3436SJosef Bacik 			return NULL;
1294db7f3436SJosef Bacik 		}
12955d9e75c4SJan Schmidt 		btrfs_set_header_bytenr(eb_rewin, eb->start);
12965d9e75c4SJan Schmidt 		btrfs_set_header_backref_rev(eb_rewin,
12975d9e75c4SJan Schmidt 					     btrfs_header_backref_rev(eb));
12985d9e75c4SJan Schmidt 		btrfs_set_header_owner(eb_rewin, btrfs_header_owner(eb));
1299c3193108SJan Schmidt 		btrfs_set_header_level(eb_rewin, btrfs_header_level(eb));
13005d9e75c4SJan Schmidt 	} else {
13015d9e75c4SJan Schmidt 		eb_rewin = btrfs_clone_extent_buffer(eb);
1302db7f3436SJosef Bacik 		if (!eb_rewin) {
13039ec72677SJosef Bacik 			btrfs_tree_read_unlock_blocking(eb);
1304db7f3436SJosef Bacik 			free_extent_buffer(eb);
1305db7f3436SJosef Bacik 			return NULL;
1306db7f3436SJosef Bacik 		}
13075d9e75c4SJan Schmidt 	}
13085d9e75c4SJan Schmidt 
13099ec72677SJosef Bacik 	btrfs_clear_path_blocking(path, NULL, BTRFS_READ_LOCK);
13109ec72677SJosef Bacik 	btrfs_tree_read_unlock_blocking(eb);
13115d9e75c4SJan Schmidt 	free_extent_buffer(eb);
13125d9e75c4SJan Schmidt 
131347fb091fSJan Schmidt 	extent_buffer_get(eb_rewin);
131447fb091fSJan Schmidt 	btrfs_tree_read_lock(eb_rewin);
1315f1ca7e98SJosef Bacik 	__tree_mod_log_rewind(fs_info, eb_rewin, time_seq, tm);
131657911b8bSJan Schmidt 	WARN_ON(btrfs_header_nritems(eb_rewin) >
1317da17066cSJeff Mahoney 		BTRFS_NODEPTRS_PER_BLOCK(fs_info));
13185d9e75c4SJan Schmidt 
13195d9e75c4SJan Schmidt 	return eb_rewin;
13205d9e75c4SJan Schmidt }
13215d9e75c4SJan Schmidt 
13228ba97a15SJan Schmidt /*
13238ba97a15SJan Schmidt  * get_old_root() rewinds the state of @root's root node to the given @time_seq
13248ba97a15SJan Schmidt  * value. If there are no changes, the current root->root_node is returned. If
13258ba97a15SJan Schmidt  * anything changed in between, there's a fresh buffer allocated on which the
13268ba97a15SJan Schmidt  * rewind operations are done. In any case, the returned buffer is read locked.
13278ba97a15SJan Schmidt  * Returns NULL on error (with no locks held).
13288ba97a15SJan Schmidt  */
13295d9e75c4SJan Schmidt static inline struct extent_buffer *
13305d9e75c4SJan Schmidt get_old_root(struct btrfs_root *root, u64 time_seq)
13315d9e75c4SJan Schmidt {
13320b246afaSJeff Mahoney 	struct btrfs_fs_info *fs_info = root->fs_info;
13335d9e75c4SJan Schmidt 	struct tree_mod_elem *tm;
133430b0463aSJan Schmidt 	struct extent_buffer *eb = NULL;
133530b0463aSJan Schmidt 	struct extent_buffer *eb_root;
13367bfdcf7fSLiu Bo 	struct extent_buffer *old;
1337a95236d9SJan Schmidt 	struct tree_mod_root *old_root = NULL;
13384325edd0SChris Mason 	u64 old_generation = 0;
1339a95236d9SJan Schmidt 	u64 logical;
1340581c1760SQu Wenruo 	int level;
13415d9e75c4SJan Schmidt 
134230b0463aSJan Schmidt 	eb_root = btrfs_read_lock_root_node(root);
1343bcd24dabSDavid Sterba 	tm = __tree_mod_log_oldest_root(eb_root, time_seq);
13445d9e75c4SJan Schmidt 	if (!tm)
134530b0463aSJan Schmidt 		return eb_root;
13465d9e75c4SJan Schmidt 
1347a95236d9SJan Schmidt 	if (tm->op == MOD_LOG_ROOT_REPLACE) {
13485d9e75c4SJan Schmidt 		old_root = &tm->old_root;
13495d9e75c4SJan Schmidt 		old_generation = tm->generation;
1350a95236d9SJan Schmidt 		logical = old_root->logical;
1351581c1760SQu Wenruo 		level = old_root->level;
1352a95236d9SJan Schmidt 	} else {
135330b0463aSJan Schmidt 		logical = eb_root->start;
1354581c1760SQu Wenruo 		level = btrfs_header_level(eb_root);
1355a95236d9SJan Schmidt 	}
13565d9e75c4SJan Schmidt 
13570b246afaSJeff Mahoney 	tm = tree_mod_log_search(fs_info, logical, time_seq);
1358834328a8SJan Schmidt 	if (old_root && tm && tm->op != MOD_LOG_KEY_REMOVE_WHILE_FREEING) {
135930b0463aSJan Schmidt 		btrfs_tree_read_unlock(eb_root);
136030b0463aSJan Schmidt 		free_extent_buffer(eb_root);
1361581c1760SQu Wenruo 		old = read_tree_block(fs_info, logical, 0, level, NULL);
136264c043deSLiu Bo 		if (WARN_ON(IS_ERR(old) || !extent_buffer_uptodate(old))) {
136364c043deSLiu Bo 			if (!IS_ERR(old))
1364416bc658SJosef Bacik 				free_extent_buffer(old);
13650b246afaSJeff Mahoney 			btrfs_warn(fs_info,
13660b246afaSJeff Mahoney 				   "failed to read tree block %llu from get_old_root",
13670b246afaSJeff Mahoney 				   logical);
1368834328a8SJan Schmidt 		} else {
13697bfdcf7fSLiu Bo 			eb = btrfs_clone_extent_buffer(old);
13707bfdcf7fSLiu Bo 			free_extent_buffer(old);
1371834328a8SJan Schmidt 		}
1372834328a8SJan Schmidt 	} else if (old_root) {
137330b0463aSJan Schmidt 		btrfs_tree_read_unlock(eb_root);
137430b0463aSJan Schmidt 		free_extent_buffer(eb_root);
13750b246afaSJeff Mahoney 		eb = alloc_dummy_extent_buffer(fs_info, logical);
1376834328a8SJan Schmidt 	} else {
13779ec72677SJosef Bacik 		btrfs_set_lock_blocking_rw(eb_root, BTRFS_READ_LOCK);
137830b0463aSJan Schmidt 		eb = btrfs_clone_extent_buffer(eb_root);
13799ec72677SJosef Bacik 		btrfs_tree_read_unlock_blocking(eb_root);
138030b0463aSJan Schmidt 		free_extent_buffer(eb_root);
1381834328a8SJan Schmidt 	}
1382834328a8SJan Schmidt 
13838ba97a15SJan Schmidt 	if (!eb)
13848ba97a15SJan Schmidt 		return NULL;
1385d6381084SJan Schmidt 	extent_buffer_get(eb);
13868ba97a15SJan Schmidt 	btrfs_tree_read_lock(eb);
1387a95236d9SJan Schmidt 	if (old_root) {
13885d9e75c4SJan Schmidt 		btrfs_set_header_bytenr(eb, eb->start);
13895d9e75c4SJan Schmidt 		btrfs_set_header_backref_rev(eb, BTRFS_MIXED_BACKREF_REV);
139030b0463aSJan Schmidt 		btrfs_set_header_owner(eb, btrfs_header_owner(eb_root));
13915d9e75c4SJan Schmidt 		btrfs_set_header_level(eb, old_root->level);
13925d9e75c4SJan Schmidt 		btrfs_set_header_generation(eb, old_generation);
1393a95236d9SJan Schmidt 	}
139428da9fb4SJan Schmidt 	if (tm)
13950b246afaSJeff Mahoney 		__tree_mod_log_rewind(fs_info, eb, time_seq, tm);
139628da9fb4SJan Schmidt 	else
139728da9fb4SJan Schmidt 		WARN_ON(btrfs_header_level(eb) != 0);
13980b246afaSJeff Mahoney 	WARN_ON(btrfs_header_nritems(eb) > BTRFS_NODEPTRS_PER_BLOCK(fs_info));
13995d9e75c4SJan Schmidt 
14005d9e75c4SJan Schmidt 	return eb;
14015d9e75c4SJan Schmidt }
14025d9e75c4SJan Schmidt 
14035b6602e7SJan Schmidt int btrfs_old_root_level(struct btrfs_root *root, u64 time_seq)
14045b6602e7SJan Schmidt {
14055b6602e7SJan Schmidt 	struct tree_mod_elem *tm;
14065b6602e7SJan Schmidt 	int level;
140730b0463aSJan Schmidt 	struct extent_buffer *eb_root = btrfs_root_node(root);
14085b6602e7SJan Schmidt 
1409bcd24dabSDavid Sterba 	tm = __tree_mod_log_oldest_root(eb_root, time_seq);
14105b6602e7SJan Schmidt 	if (tm && tm->op == MOD_LOG_ROOT_REPLACE) {
14115b6602e7SJan Schmidt 		level = tm->old_root.level;
14125b6602e7SJan Schmidt 	} else {
141330b0463aSJan Schmidt 		level = btrfs_header_level(eb_root);
14145b6602e7SJan Schmidt 	}
141530b0463aSJan Schmidt 	free_extent_buffer(eb_root);
14165b6602e7SJan Schmidt 
14175b6602e7SJan Schmidt 	return level;
14185b6602e7SJan Schmidt }
14195b6602e7SJan Schmidt 
14205d4f98a2SYan Zheng static inline int should_cow_block(struct btrfs_trans_handle *trans,
14215d4f98a2SYan Zheng 				   struct btrfs_root *root,
14225d4f98a2SYan Zheng 				   struct extent_buffer *buf)
14235d4f98a2SYan Zheng {
1424f5ee5c9aSJeff Mahoney 	if (btrfs_is_testing(root->fs_info))
1425faa2dbf0SJosef Bacik 		return 0;
1426fccb84c9SDavid Sterba 
1427d1980131SDavid Sterba 	/* Ensure we can see the FORCE_COW bit */
1428d1980131SDavid Sterba 	smp_mb__before_atomic();
1429f1ebcc74SLiu Bo 
1430f1ebcc74SLiu Bo 	/*
1431f1ebcc74SLiu Bo 	 * We do not need to cow a block if
1432f1ebcc74SLiu Bo 	 * 1) this block is not created or changed in this transaction;
1433f1ebcc74SLiu Bo 	 * 2) this block does not belong to TREE_RELOC tree;
1434f1ebcc74SLiu Bo 	 * 3) the root is not forced COW.
1435f1ebcc74SLiu Bo 	 *
1436f1ebcc74SLiu Bo 	 * What is forced COW:
143701327610SNicholas D Steeves 	 *    when we create snapshot during committing the transaction,
1438f1ebcc74SLiu Bo 	 *    after we've finished coping src root, we must COW the shared
1439f1ebcc74SLiu Bo 	 *    block to ensure the metadata consistency.
1440f1ebcc74SLiu Bo 	 */
14415d4f98a2SYan Zheng 	if (btrfs_header_generation(buf) == trans->transid &&
14425d4f98a2SYan Zheng 	    !btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN) &&
14435d4f98a2SYan Zheng 	    !(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID &&
1444f1ebcc74SLiu Bo 	      btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)) &&
144527cdeb70SMiao Xie 	    !test_bit(BTRFS_ROOT_FORCE_COW, &root->state))
14465d4f98a2SYan Zheng 		return 0;
14475d4f98a2SYan Zheng 	return 1;
14485d4f98a2SYan Zheng }
14495d4f98a2SYan Zheng 
1450d352ac68SChris Mason /*
1451d352ac68SChris Mason  * cows a single block, see __btrfs_cow_block for the real work.
145201327610SNicholas D Steeves  * This version of it has extra checks so that a block isn't COWed more than
1453d352ac68SChris Mason  * once per transaction, as long as it hasn't been written yet
1454d352ac68SChris Mason  */
1455d397712bSChris Mason noinline int btrfs_cow_block(struct btrfs_trans_handle *trans,
14565f39d397SChris Mason 		    struct btrfs_root *root, struct extent_buffer *buf,
14575f39d397SChris Mason 		    struct extent_buffer *parent, int parent_slot,
14589fa8cfe7SChris Mason 		    struct extent_buffer **cow_ret)
145902217ed2SChris Mason {
14600b246afaSJeff Mahoney 	struct btrfs_fs_info *fs_info = root->fs_info;
14616702ed49SChris Mason 	u64 search_start;
1462f510cfecSChris Mason 	int ret;
1463dc17ff8fSChris Mason 
14640b246afaSJeff Mahoney 	if (trans->transaction != fs_info->running_transaction)
146531b1a2bdSJulia Lawall 		WARN(1, KERN_CRIT "trans %llu running %llu\n",
1466c1c9ff7cSGeert Uytterhoeven 		       trans->transid,
14670b246afaSJeff Mahoney 		       fs_info->running_transaction->transid);
146831b1a2bdSJulia Lawall 
14690b246afaSJeff Mahoney 	if (trans->transid != fs_info->generation)
147031b1a2bdSJulia Lawall 		WARN(1, KERN_CRIT "trans %llu running %llu\n",
14710b246afaSJeff Mahoney 		       trans->transid, fs_info->generation);
1472dc17ff8fSChris Mason 
14735d4f98a2SYan Zheng 	if (!should_cow_block(trans, root, buf)) {
147464c12921SJeff Mahoney 		trans->dirty = true;
147502217ed2SChris Mason 		*cow_ret = buf;
147602217ed2SChris Mason 		return 0;
147702217ed2SChris Mason 	}
1478c487685dSChris Mason 
1479ee22184bSByongho Lee 	search_start = buf->start & ~((u64)SZ_1G - 1);
1480b4ce94deSChris Mason 
1481b4ce94deSChris Mason 	if (parent)
1482b4ce94deSChris Mason 		btrfs_set_lock_blocking(parent);
1483b4ce94deSChris Mason 	btrfs_set_lock_blocking(buf);
1484b4ce94deSChris Mason 
1485f510cfecSChris Mason 	ret = __btrfs_cow_block(trans, root, buf, parent,
14869fa8cfe7SChris Mason 				 parent_slot, cow_ret, search_start, 0);
14871abe9b8aSliubo 
14881abe9b8aSliubo 	trace_btrfs_cow_block(root, buf, *cow_ret);
14891abe9b8aSliubo 
1490f510cfecSChris Mason 	return ret;
14912c90e5d6SChris Mason }
14926702ed49SChris Mason 
1493d352ac68SChris Mason /*
1494d352ac68SChris Mason  * helper function for defrag to decide if two blocks pointed to by a
1495d352ac68SChris Mason  * node are actually close by
1496d352ac68SChris Mason  */
14976b80053dSChris Mason static int close_blocks(u64 blocknr, u64 other, u32 blocksize)
14986702ed49SChris Mason {
14996b80053dSChris Mason 	if (blocknr < other && other - (blocknr + blocksize) < 32768)
15006702ed49SChris Mason 		return 1;
15016b80053dSChris Mason 	if (blocknr > other && blocknr - (other + blocksize) < 32768)
15026702ed49SChris Mason 		return 1;
150302217ed2SChris Mason 	return 0;
150402217ed2SChris Mason }
150502217ed2SChris Mason 
1506081e9573SChris Mason /*
1507081e9573SChris Mason  * compare two keys in a memcmp fashion
1508081e9573SChris Mason  */
1509310712b2SOmar Sandoval static int comp_keys(const struct btrfs_disk_key *disk,
1510310712b2SOmar Sandoval 		     const struct btrfs_key *k2)
1511081e9573SChris Mason {
1512081e9573SChris Mason 	struct btrfs_key k1;
1513081e9573SChris Mason 
1514081e9573SChris Mason 	btrfs_disk_key_to_cpu(&k1, disk);
1515081e9573SChris Mason 
151620736abaSDiego Calleja 	return btrfs_comp_cpu_keys(&k1, k2);
1517081e9573SChris Mason }
1518081e9573SChris Mason 
1519f3465ca4SJosef Bacik /*
1520f3465ca4SJosef Bacik  * same as comp_keys only with two btrfs_key's
1521f3465ca4SJosef Bacik  */
1522310712b2SOmar Sandoval int btrfs_comp_cpu_keys(const struct btrfs_key *k1, const struct btrfs_key *k2)
1523f3465ca4SJosef Bacik {
1524f3465ca4SJosef Bacik 	if (k1->objectid > k2->objectid)
1525f3465ca4SJosef Bacik 		return 1;
1526f3465ca4SJosef Bacik 	if (k1->objectid < k2->objectid)
1527f3465ca4SJosef Bacik 		return -1;
1528f3465ca4SJosef Bacik 	if (k1->type > k2->type)
1529f3465ca4SJosef Bacik 		return 1;
1530f3465ca4SJosef Bacik 	if (k1->type < k2->type)
1531f3465ca4SJosef Bacik 		return -1;
1532f3465ca4SJosef Bacik 	if (k1->offset > k2->offset)
1533f3465ca4SJosef Bacik 		return 1;
1534f3465ca4SJosef Bacik 	if (k1->offset < k2->offset)
1535f3465ca4SJosef Bacik 		return -1;
1536f3465ca4SJosef Bacik 	return 0;
1537f3465ca4SJosef Bacik }
1538081e9573SChris Mason 
1539d352ac68SChris Mason /*
1540d352ac68SChris Mason  * this is used by the defrag code to go through all the
1541d352ac68SChris Mason  * leaves pointed to by a node and reallocate them so that
1542d352ac68SChris Mason  * disk order is close to key order
1543d352ac68SChris Mason  */
15446702ed49SChris Mason int btrfs_realloc_node(struct btrfs_trans_handle *trans,
15455f39d397SChris Mason 		       struct btrfs_root *root, struct extent_buffer *parent,
1546de78b51aSEric Sandeen 		       int start_slot, u64 *last_ret,
1547a6b6e75eSChris Mason 		       struct btrfs_key *progress)
15486702ed49SChris Mason {
15490b246afaSJeff Mahoney 	struct btrfs_fs_info *fs_info = root->fs_info;
15506b80053dSChris Mason 	struct extent_buffer *cur;
15516702ed49SChris Mason 	u64 blocknr;
1552ca7a79adSChris Mason 	u64 gen;
1553e9d0b13bSChris Mason 	u64 search_start = *last_ret;
1554e9d0b13bSChris Mason 	u64 last_block = 0;
15556702ed49SChris Mason 	u64 other;
15566702ed49SChris Mason 	u32 parent_nritems;
15576702ed49SChris Mason 	int end_slot;
15586702ed49SChris Mason 	int i;
15596702ed49SChris Mason 	int err = 0;
1560f2183bdeSChris Mason 	int parent_level;
15616b80053dSChris Mason 	int uptodate;
15626b80053dSChris Mason 	u32 blocksize;
1563081e9573SChris Mason 	int progress_passed = 0;
1564081e9573SChris Mason 	struct btrfs_disk_key disk_key;
15656702ed49SChris Mason 
15665708b959SChris Mason 	parent_level = btrfs_header_level(parent);
15675708b959SChris Mason 
15680b246afaSJeff Mahoney 	WARN_ON(trans->transaction != fs_info->running_transaction);
15690b246afaSJeff Mahoney 	WARN_ON(trans->transid != fs_info->generation);
157086479a04SChris Mason 
15716b80053dSChris Mason 	parent_nritems = btrfs_header_nritems(parent);
15720b246afaSJeff Mahoney 	blocksize = fs_info->nodesize;
15735dfe2be7SFilipe Manana 	end_slot = parent_nritems - 1;
15746702ed49SChris Mason 
15755dfe2be7SFilipe Manana 	if (parent_nritems <= 1)
15766702ed49SChris Mason 		return 0;
15776702ed49SChris Mason 
1578b4ce94deSChris Mason 	btrfs_set_lock_blocking(parent);
1579b4ce94deSChris Mason 
15805dfe2be7SFilipe Manana 	for (i = start_slot; i <= end_slot; i++) {
1581581c1760SQu Wenruo 		struct btrfs_key first_key;
15826702ed49SChris Mason 		int close = 1;
1583a6b6e75eSChris Mason 
1584081e9573SChris Mason 		btrfs_node_key(parent, &disk_key, i);
1585081e9573SChris Mason 		if (!progress_passed && comp_keys(&disk_key, progress) < 0)
1586081e9573SChris Mason 			continue;
1587081e9573SChris Mason 
1588081e9573SChris Mason 		progress_passed = 1;
15896b80053dSChris Mason 		blocknr = btrfs_node_blockptr(parent, i);
1590ca7a79adSChris Mason 		gen = btrfs_node_ptr_generation(parent, i);
1591581c1760SQu Wenruo 		btrfs_node_key_to_cpu(parent, &first_key, i);
1592e9d0b13bSChris Mason 		if (last_block == 0)
1593e9d0b13bSChris Mason 			last_block = blocknr;
15945708b959SChris Mason 
15956702ed49SChris Mason 		if (i > 0) {
15966b80053dSChris Mason 			other = btrfs_node_blockptr(parent, i - 1);
15976b80053dSChris Mason 			close = close_blocks(blocknr, other, blocksize);
15986702ed49SChris Mason 		}
15995dfe2be7SFilipe Manana 		if (!close && i < end_slot) {
16006b80053dSChris Mason 			other = btrfs_node_blockptr(parent, i + 1);
16016b80053dSChris Mason 			close = close_blocks(blocknr, other, blocksize);
16026702ed49SChris Mason 		}
1603e9d0b13bSChris Mason 		if (close) {
1604e9d0b13bSChris Mason 			last_block = blocknr;
16056702ed49SChris Mason 			continue;
1606e9d0b13bSChris Mason 		}
16076702ed49SChris Mason 
16080b246afaSJeff Mahoney 		cur = find_extent_buffer(fs_info, blocknr);
16096b80053dSChris Mason 		if (cur)
1610b9fab919SChris Mason 			uptodate = btrfs_buffer_uptodate(cur, gen, 0);
16116b80053dSChris Mason 		else
16126b80053dSChris Mason 			uptodate = 0;
16135708b959SChris Mason 		if (!cur || !uptodate) {
16146b80053dSChris Mason 			if (!cur) {
1615581c1760SQu Wenruo 				cur = read_tree_block(fs_info, blocknr, gen,
1616581c1760SQu Wenruo 						      parent_level - 1,
1617581c1760SQu Wenruo 						      &first_key);
161864c043deSLiu Bo 				if (IS_ERR(cur)) {
161964c043deSLiu Bo 					return PTR_ERR(cur);
162064c043deSLiu Bo 				} else if (!extent_buffer_uptodate(cur)) {
1621416bc658SJosef Bacik 					free_extent_buffer(cur);
162297d9a8a4STsutomu Itoh 					return -EIO;
1623416bc658SJosef Bacik 				}
16246b80053dSChris Mason 			} else if (!uptodate) {
1625581c1760SQu Wenruo 				err = btrfs_read_buffer(cur, gen,
1626581c1760SQu Wenruo 						parent_level - 1,&first_key);
1627018642a1STsutomu Itoh 				if (err) {
1628018642a1STsutomu Itoh 					free_extent_buffer(cur);
1629018642a1STsutomu Itoh 					return err;
1630018642a1STsutomu Itoh 				}
16316702ed49SChris Mason 			}
1632f2183bdeSChris Mason 		}
1633e9d0b13bSChris Mason 		if (search_start == 0)
16346b80053dSChris Mason 			search_start = last_block;
1635e9d0b13bSChris Mason 
1636e7a84565SChris Mason 		btrfs_tree_lock(cur);
1637b4ce94deSChris Mason 		btrfs_set_lock_blocking(cur);
16386b80053dSChris Mason 		err = __btrfs_cow_block(trans, root, cur, parent, i,
1639e7a84565SChris Mason 					&cur, search_start,
16406b80053dSChris Mason 					min(16 * blocksize,
16419fa8cfe7SChris Mason 					    (end_slot - i) * blocksize));
1642252c38f0SYan 		if (err) {
1643e7a84565SChris Mason 			btrfs_tree_unlock(cur);
16446b80053dSChris Mason 			free_extent_buffer(cur);
16456702ed49SChris Mason 			break;
1646252c38f0SYan 		}
1647e7a84565SChris Mason 		search_start = cur->start;
1648e7a84565SChris Mason 		last_block = cur->start;
1649f2183bdeSChris Mason 		*last_ret = search_start;
1650e7a84565SChris Mason 		btrfs_tree_unlock(cur);
1651e7a84565SChris Mason 		free_extent_buffer(cur);
16526702ed49SChris Mason 	}
16536702ed49SChris Mason 	return err;
16546702ed49SChris Mason }
16556702ed49SChris Mason 
165674123bd7SChris Mason /*
16575f39d397SChris Mason  * search for key in the extent_buffer.  The items start at offset p,
16585f39d397SChris Mason  * and they are item_size apart.  There are 'max' items in p.
16595f39d397SChris Mason  *
166074123bd7SChris Mason  * the slot in the array is returned via slot, and it points to
166174123bd7SChris Mason  * the place where you would insert key if it is not found in
166274123bd7SChris Mason  * the array.
166374123bd7SChris Mason  *
166474123bd7SChris Mason  * slot may point to max if the key is bigger than all of the keys
166574123bd7SChris Mason  */
1666e02119d5SChris Mason static noinline int generic_bin_search(struct extent_buffer *eb,
1667310712b2SOmar Sandoval 				       unsigned long p, int item_size,
1668310712b2SOmar Sandoval 				       const struct btrfs_key *key,
1669be0e5c09SChris Mason 				       int max, int *slot)
1670be0e5c09SChris Mason {
1671be0e5c09SChris Mason 	int low = 0;
1672be0e5c09SChris Mason 	int high = max;
1673be0e5c09SChris Mason 	int mid;
1674be0e5c09SChris Mason 	int ret;
1675479965d6SChris Mason 	struct btrfs_disk_key *tmp = NULL;
16765f39d397SChris Mason 	struct btrfs_disk_key unaligned;
16775f39d397SChris Mason 	unsigned long offset;
16785f39d397SChris Mason 	char *kaddr = NULL;
16795f39d397SChris Mason 	unsigned long map_start = 0;
16805f39d397SChris Mason 	unsigned long map_len = 0;
1681479965d6SChris Mason 	int err;
1682be0e5c09SChris Mason 
16835e24e9afSLiu Bo 	if (low > high) {
16845e24e9afSLiu Bo 		btrfs_err(eb->fs_info,
16855e24e9afSLiu Bo 		 "%s: low (%d) > high (%d) eb %llu owner %llu level %d",
16865e24e9afSLiu Bo 			  __func__, low, high, eb->start,
16875e24e9afSLiu Bo 			  btrfs_header_owner(eb), btrfs_header_level(eb));
16885e24e9afSLiu Bo 		return -EINVAL;
16895e24e9afSLiu Bo 	}
16905e24e9afSLiu Bo 
1691be0e5c09SChris Mason 	while (low < high) {
1692be0e5c09SChris Mason 		mid = (low + high) / 2;
16935f39d397SChris Mason 		offset = p + mid * item_size;
16945f39d397SChris Mason 
1695a6591715SChris Mason 		if (!kaddr || offset < map_start ||
16965f39d397SChris Mason 		    (offset + sizeof(struct btrfs_disk_key)) >
16975f39d397SChris Mason 		    map_start + map_len) {
1698934d375bSChris Mason 
1699934d375bSChris Mason 			err = map_private_extent_buffer(eb, offset,
1700479965d6SChris Mason 						sizeof(struct btrfs_disk_key),
1701a6591715SChris Mason 						&kaddr, &map_start, &map_len);
17025f39d397SChris Mason 
1703479965d6SChris Mason 			if (!err) {
1704479965d6SChris Mason 				tmp = (struct btrfs_disk_key *)(kaddr + offset -
1705479965d6SChris Mason 							map_start);
1706415b35a5SLiu Bo 			} else if (err == 1) {
17075f39d397SChris Mason 				read_extent_buffer(eb, &unaligned,
17085f39d397SChris Mason 						   offset, sizeof(unaligned));
17095f39d397SChris Mason 				tmp = &unaligned;
1710415b35a5SLiu Bo 			} else {
1711415b35a5SLiu Bo 				return err;
1712479965d6SChris Mason 			}
1713479965d6SChris Mason 
17145f39d397SChris Mason 		} else {
17155f39d397SChris Mason 			tmp = (struct btrfs_disk_key *)(kaddr + offset -
17165f39d397SChris Mason 							map_start);
17175f39d397SChris Mason 		}
1718be0e5c09SChris Mason 		ret = comp_keys(tmp, key);
1719be0e5c09SChris Mason 
1720be0e5c09SChris Mason 		if (ret < 0)
1721be0e5c09SChris Mason 			low = mid + 1;
1722be0e5c09SChris Mason 		else if (ret > 0)
1723be0e5c09SChris Mason 			high = mid;
1724be0e5c09SChris Mason 		else {
1725be0e5c09SChris Mason 			*slot = mid;
1726be0e5c09SChris Mason 			return 0;
1727be0e5c09SChris Mason 		}
1728be0e5c09SChris Mason 	}
1729be0e5c09SChris Mason 	*slot = low;
1730be0e5c09SChris Mason 	return 1;
1731be0e5c09SChris Mason }
1732be0e5c09SChris Mason 
173397571fd0SChris Mason /*
173497571fd0SChris Mason  * simple bin_search frontend that does the right thing for
173597571fd0SChris Mason  * leaves vs nodes
173697571fd0SChris Mason  */
1737a74b35ecSNikolay Borisov int btrfs_bin_search(struct extent_buffer *eb, const struct btrfs_key *key,
17385f39d397SChris Mason 		     int level, int *slot)
1739be0e5c09SChris Mason {
1740f775738fSWang Sheng-Hui 	if (level == 0)
17415f39d397SChris Mason 		return generic_bin_search(eb,
17425f39d397SChris Mason 					  offsetof(struct btrfs_leaf, items),
17430783fcfcSChris Mason 					  sizeof(struct btrfs_item),
17445f39d397SChris Mason 					  key, btrfs_header_nritems(eb),
17457518a238SChris Mason 					  slot);
1746f775738fSWang Sheng-Hui 	else
17475f39d397SChris Mason 		return generic_bin_search(eb,
17485f39d397SChris Mason 					  offsetof(struct btrfs_node, ptrs),
1749123abc88SChris Mason 					  sizeof(struct btrfs_key_ptr),
17505f39d397SChris Mason 					  key, btrfs_header_nritems(eb),
17517518a238SChris Mason 					  slot);
1752be0e5c09SChris Mason }
1753be0e5c09SChris Mason 
1754f0486c68SYan, Zheng static void root_add_used(struct btrfs_root *root, u32 size)
1755f0486c68SYan, Zheng {
1756f0486c68SYan, Zheng 	spin_lock(&root->accounting_lock);
1757f0486c68SYan, Zheng 	btrfs_set_root_used(&root->root_item,
1758f0486c68SYan, Zheng 			    btrfs_root_used(&root->root_item) + size);
1759f0486c68SYan, Zheng 	spin_unlock(&root->accounting_lock);
1760f0486c68SYan, Zheng }
1761f0486c68SYan, Zheng 
1762f0486c68SYan, Zheng static void root_sub_used(struct btrfs_root *root, u32 size)
1763f0486c68SYan, Zheng {
1764f0486c68SYan, Zheng 	spin_lock(&root->accounting_lock);
1765f0486c68SYan, Zheng 	btrfs_set_root_used(&root->root_item,
1766f0486c68SYan, Zheng 			    btrfs_root_used(&root->root_item) - size);
1767f0486c68SYan, Zheng 	spin_unlock(&root->accounting_lock);
1768f0486c68SYan, Zheng }
1769f0486c68SYan, Zheng 
1770d352ac68SChris Mason /* given a node and slot number, this reads the blocks it points to.  The
1771d352ac68SChris Mason  * extent buffer is returned with a reference taken (but unlocked).
1772d352ac68SChris Mason  */
17732ff7e61eSJeff Mahoney static noinline struct extent_buffer *
17742ff7e61eSJeff Mahoney read_node_slot(struct btrfs_fs_info *fs_info, struct extent_buffer *parent,
17752ff7e61eSJeff Mahoney 	       int slot)
1776bb803951SChris Mason {
1777ca7a79adSChris Mason 	int level = btrfs_header_level(parent);
1778416bc658SJosef Bacik 	struct extent_buffer *eb;
1779581c1760SQu Wenruo 	struct btrfs_key first_key;
1780416bc658SJosef Bacik 
1781fb770ae4SLiu Bo 	if (slot < 0 || slot >= btrfs_header_nritems(parent))
1782fb770ae4SLiu Bo 		return ERR_PTR(-ENOENT);
1783ca7a79adSChris Mason 
1784ca7a79adSChris Mason 	BUG_ON(level == 0);
1785ca7a79adSChris Mason 
1786581c1760SQu Wenruo 	btrfs_node_key_to_cpu(parent, &first_key, slot);
17872ff7e61eSJeff Mahoney 	eb = read_tree_block(fs_info, btrfs_node_blockptr(parent, slot),
1788581c1760SQu Wenruo 			     btrfs_node_ptr_generation(parent, slot),
1789581c1760SQu Wenruo 			     level - 1, &first_key);
1790fb770ae4SLiu Bo 	if (!IS_ERR(eb) && !extent_buffer_uptodate(eb)) {
1791416bc658SJosef Bacik 		free_extent_buffer(eb);
1792fb770ae4SLiu Bo 		eb = ERR_PTR(-EIO);
1793416bc658SJosef Bacik 	}
1794416bc658SJosef Bacik 
1795416bc658SJosef Bacik 	return eb;
1796bb803951SChris Mason }
1797bb803951SChris Mason 
1798d352ac68SChris Mason /*
1799d352ac68SChris Mason  * node level balancing, used to make sure nodes are in proper order for
1800d352ac68SChris Mason  * item deletion.  We balance from the top down, so we have to make sure
1801d352ac68SChris Mason  * that a deletion won't leave an node completely empty later on.
1802d352ac68SChris Mason  */
1803e02119d5SChris Mason static noinline int balance_level(struct btrfs_trans_handle *trans,
180498ed5174SChris Mason 			 struct btrfs_root *root,
180598ed5174SChris Mason 			 struct btrfs_path *path, int level)
1806bb803951SChris Mason {
18070b246afaSJeff Mahoney 	struct btrfs_fs_info *fs_info = root->fs_info;
18085f39d397SChris Mason 	struct extent_buffer *right = NULL;
18095f39d397SChris Mason 	struct extent_buffer *mid;
18105f39d397SChris Mason 	struct extent_buffer *left = NULL;
18115f39d397SChris Mason 	struct extent_buffer *parent = NULL;
1812bb803951SChris Mason 	int ret = 0;
1813bb803951SChris Mason 	int wret;
1814bb803951SChris Mason 	int pslot;
1815bb803951SChris Mason 	int orig_slot = path->slots[level];
181679f95c82SChris Mason 	u64 orig_ptr;
1817bb803951SChris Mason 
1818bb803951SChris Mason 	if (level == 0)
1819bb803951SChris Mason 		return 0;
1820bb803951SChris Mason 
18215f39d397SChris Mason 	mid = path->nodes[level];
1822b4ce94deSChris Mason 
1823bd681513SChris Mason 	WARN_ON(path->locks[level] != BTRFS_WRITE_LOCK &&
1824bd681513SChris Mason 		path->locks[level] != BTRFS_WRITE_LOCK_BLOCKING);
18257bb86316SChris Mason 	WARN_ON(btrfs_header_generation(mid) != trans->transid);
18267bb86316SChris Mason 
18271d4f8a0cSChris Mason 	orig_ptr = btrfs_node_blockptr(mid, orig_slot);
182879f95c82SChris Mason 
1829a05a9bb1SLi Zefan 	if (level < BTRFS_MAX_LEVEL - 1) {
18305f39d397SChris Mason 		parent = path->nodes[level + 1];
1831bb803951SChris Mason 		pslot = path->slots[level + 1];
1832a05a9bb1SLi Zefan 	}
1833bb803951SChris Mason 
183440689478SChris Mason 	/*
183540689478SChris Mason 	 * deal with the case where there is only one pointer in the root
183640689478SChris Mason 	 * by promoting the node below to a root
183740689478SChris Mason 	 */
18385f39d397SChris Mason 	if (!parent) {
18395f39d397SChris Mason 		struct extent_buffer *child;
1840bb803951SChris Mason 
18415f39d397SChris Mason 		if (btrfs_header_nritems(mid) != 1)
1842bb803951SChris Mason 			return 0;
1843bb803951SChris Mason 
1844bb803951SChris Mason 		/* promote the child to a root */
18452ff7e61eSJeff Mahoney 		child = read_node_slot(fs_info, mid, 0);
1846fb770ae4SLiu Bo 		if (IS_ERR(child)) {
1847fb770ae4SLiu Bo 			ret = PTR_ERR(child);
18480b246afaSJeff Mahoney 			btrfs_handle_fs_error(fs_info, ret, NULL);
1849305a26afSMark Fasheh 			goto enospc;
1850305a26afSMark Fasheh 		}
1851305a26afSMark Fasheh 
1852925baeddSChris Mason 		btrfs_tree_lock(child);
1853b4ce94deSChris Mason 		btrfs_set_lock_blocking(child);
18549fa8cfe7SChris Mason 		ret = btrfs_cow_block(trans, root, child, mid, 0, &child);
1855f0486c68SYan, Zheng 		if (ret) {
1856f0486c68SYan, Zheng 			btrfs_tree_unlock(child);
1857f0486c68SYan, Zheng 			free_extent_buffer(child);
1858f0486c68SYan, Zheng 			goto enospc;
1859f0486c68SYan, Zheng 		}
18602f375ab9SYan 
1861d9d19a01SDavid Sterba 		ret = tree_mod_log_insert_root(root->node, child, 1);
1862d9d19a01SDavid Sterba 		BUG_ON(ret < 0);
1863240f62c8SChris Mason 		rcu_assign_pointer(root->node, child);
1864925baeddSChris Mason 
18650b86a832SChris Mason 		add_root_to_dirty_list(root);
1866925baeddSChris Mason 		btrfs_tree_unlock(child);
1867b4ce94deSChris Mason 
1868925baeddSChris Mason 		path->locks[level] = 0;
1869bb803951SChris Mason 		path->nodes[level] = NULL;
18707c302b49SDavid Sterba 		clean_tree_block(fs_info, mid);
1871925baeddSChris Mason 		btrfs_tree_unlock(mid);
1872bb803951SChris Mason 		/* once for the path */
18735f39d397SChris Mason 		free_extent_buffer(mid);
1874f0486c68SYan, Zheng 
1875f0486c68SYan, Zheng 		root_sub_used(root, mid->len);
18765581a51aSJan Schmidt 		btrfs_free_tree_block(trans, root, mid, 0, 1);
1877bb803951SChris Mason 		/* once for the root ptr */
18783083ee2eSJosef Bacik 		free_extent_buffer_stale(mid);
1879f0486c68SYan, Zheng 		return 0;
1880bb803951SChris Mason 	}
18815f39d397SChris Mason 	if (btrfs_header_nritems(mid) >
18820b246afaSJeff Mahoney 	    BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 4)
1883bb803951SChris Mason 		return 0;
1884bb803951SChris Mason 
18852ff7e61eSJeff Mahoney 	left = read_node_slot(fs_info, parent, pslot - 1);
1886fb770ae4SLiu Bo 	if (IS_ERR(left))
1887fb770ae4SLiu Bo 		left = NULL;
1888fb770ae4SLiu Bo 
18895f39d397SChris Mason 	if (left) {
1890925baeddSChris Mason 		btrfs_tree_lock(left);
1891b4ce94deSChris Mason 		btrfs_set_lock_blocking(left);
18925f39d397SChris Mason 		wret = btrfs_cow_block(trans, root, left,
18939fa8cfe7SChris Mason 				       parent, pslot - 1, &left);
189454aa1f4dSChris Mason 		if (wret) {
189554aa1f4dSChris Mason 			ret = wret;
189654aa1f4dSChris Mason 			goto enospc;
189754aa1f4dSChris Mason 		}
18982cc58cf2SChris Mason 	}
1899fb770ae4SLiu Bo 
19002ff7e61eSJeff Mahoney 	right = read_node_slot(fs_info, parent, pslot + 1);
1901fb770ae4SLiu Bo 	if (IS_ERR(right))
1902fb770ae4SLiu Bo 		right = NULL;
1903fb770ae4SLiu Bo 
19045f39d397SChris Mason 	if (right) {
1905925baeddSChris Mason 		btrfs_tree_lock(right);
1906b4ce94deSChris Mason 		btrfs_set_lock_blocking(right);
19075f39d397SChris Mason 		wret = btrfs_cow_block(trans, root, right,
19089fa8cfe7SChris Mason 				       parent, pslot + 1, &right);
19092cc58cf2SChris Mason 		if (wret) {
19102cc58cf2SChris Mason 			ret = wret;
19112cc58cf2SChris Mason 			goto enospc;
19122cc58cf2SChris Mason 		}
19132cc58cf2SChris Mason 	}
19142cc58cf2SChris Mason 
19152cc58cf2SChris Mason 	/* first, try to make some room in the middle buffer */
19165f39d397SChris Mason 	if (left) {
19175f39d397SChris Mason 		orig_slot += btrfs_header_nritems(left);
19182ff7e61eSJeff Mahoney 		wret = push_node_left(trans, fs_info, left, mid, 1);
191979f95c82SChris Mason 		if (wret < 0)
192079f95c82SChris Mason 			ret = wret;
1921bb803951SChris Mason 	}
192279f95c82SChris Mason 
192379f95c82SChris Mason 	/*
192479f95c82SChris Mason 	 * then try to empty the right most buffer into the middle
192579f95c82SChris Mason 	 */
19265f39d397SChris Mason 	if (right) {
19272ff7e61eSJeff Mahoney 		wret = push_node_left(trans, fs_info, mid, right, 1);
192854aa1f4dSChris Mason 		if (wret < 0 && wret != -ENOSPC)
192979f95c82SChris Mason 			ret = wret;
19305f39d397SChris Mason 		if (btrfs_header_nritems(right) == 0) {
19317c302b49SDavid Sterba 			clean_tree_block(fs_info, right);
1932925baeddSChris Mason 			btrfs_tree_unlock(right);
1933afe5fea7STsutomu Itoh 			del_ptr(root, path, level + 1, pslot + 1);
1934f0486c68SYan, Zheng 			root_sub_used(root, right->len);
19355581a51aSJan Schmidt 			btrfs_free_tree_block(trans, root, right, 0, 1);
19363083ee2eSJosef Bacik 			free_extent_buffer_stale(right);
1937f0486c68SYan, Zheng 			right = NULL;
1938bb803951SChris Mason 		} else {
19395f39d397SChris Mason 			struct btrfs_disk_key right_key;
19405f39d397SChris Mason 			btrfs_node_key(right, &right_key, 0);
19410e82bcfeSDavid Sterba 			ret = tree_mod_log_insert_key(parent, pslot + 1,
19420e82bcfeSDavid Sterba 					MOD_LOG_KEY_REPLACE, GFP_NOFS);
19430e82bcfeSDavid Sterba 			BUG_ON(ret < 0);
19445f39d397SChris Mason 			btrfs_set_node_key(parent, &right_key, pslot + 1);
19455f39d397SChris Mason 			btrfs_mark_buffer_dirty(parent);
1946bb803951SChris Mason 		}
1947bb803951SChris Mason 	}
19485f39d397SChris Mason 	if (btrfs_header_nritems(mid) == 1) {
194979f95c82SChris Mason 		/*
195079f95c82SChris Mason 		 * we're not allowed to leave a node with one item in the
195179f95c82SChris Mason 		 * tree during a delete.  A deletion from lower in the tree
195279f95c82SChris Mason 		 * could try to delete the only pointer in this node.
195379f95c82SChris Mason 		 * So, pull some keys from the left.
195479f95c82SChris Mason 		 * There has to be a left pointer at this point because
195579f95c82SChris Mason 		 * otherwise we would have pulled some pointers from the
195679f95c82SChris Mason 		 * right
195779f95c82SChris Mason 		 */
1958305a26afSMark Fasheh 		if (!left) {
1959305a26afSMark Fasheh 			ret = -EROFS;
19600b246afaSJeff Mahoney 			btrfs_handle_fs_error(fs_info, ret, NULL);
1961305a26afSMark Fasheh 			goto enospc;
1962305a26afSMark Fasheh 		}
19632ff7e61eSJeff Mahoney 		wret = balance_node_right(trans, fs_info, mid, left);
196454aa1f4dSChris Mason 		if (wret < 0) {
196579f95c82SChris Mason 			ret = wret;
196654aa1f4dSChris Mason 			goto enospc;
196754aa1f4dSChris Mason 		}
1968bce4eae9SChris Mason 		if (wret == 1) {
19692ff7e61eSJeff Mahoney 			wret = push_node_left(trans, fs_info, left, mid, 1);
1970bce4eae9SChris Mason 			if (wret < 0)
1971bce4eae9SChris Mason 				ret = wret;
1972bce4eae9SChris Mason 		}
197379f95c82SChris Mason 		BUG_ON(wret == 1);
197479f95c82SChris Mason 	}
19755f39d397SChris Mason 	if (btrfs_header_nritems(mid) == 0) {
19767c302b49SDavid Sterba 		clean_tree_block(fs_info, mid);
1977925baeddSChris Mason 		btrfs_tree_unlock(mid);
1978afe5fea7STsutomu Itoh 		del_ptr(root, path, level + 1, pslot);
1979f0486c68SYan, Zheng 		root_sub_used(root, mid->len);
19805581a51aSJan Schmidt 		btrfs_free_tree_block(trans, root, mid, 0, 1);
19813083ee2eSJosef Bacik 		free_extent_buffer_stale(mid);
1982f0486c68SYan, Zheng 		mid = NULL;
198379f95c82SChris Mason 	} else {
198479f95c82SChris Mason 		/* update the parent key to reflect our changes */
19855f39d397SChris Mason 		struct btrfs_disk_key mid_key;
19865f39d397SChris Mason 		btrfs_node_key(mid, &mid_key, 0);
19870e82bcfeSDavid Sterba 		ret = tree_mod_log_insert_key(parent, pslot,
19880e82bcfeSDavid Sterba 				MOD_LOG_KEY_REPLACE, GFP_NOFS);
19890e82bcfeSDavid Sterba 		BUG_ON(ret < 0);
19905f39d397SChris Mason 		btrfs_set_node_key(parent, &mid_key, pslot);
19915f39d397SChris Mason 		btrfs_mark_buffer_dirty(parent);
199279f95c82SChris Mason 	}
1993bb803951SChris Mason 
199479f95c82SChris Mason 	/* update the path */
19955f39d397SChris Mason 	if (left) {
19965f39d397SChris Mason 		if (btrfs_header_nritems(left) > orig_slot) {
19975f39d397SChris Mason 			extent_buffer_get(left);
1998925baeddSChris Mason 			/* left was locked after cow */
19995f39d397SChris Mason 			path->nodes[level] = left;
2000bb803951SChris Mason 			path->slots[level + 1] -= 1;
2001bb803951SChris Mason 			path->slots[level] = orig_slot;
2002925baeddSChris Mason 			if (mid) {
2003925baeddSChris Mason 				btrfs_tree_unlock(mid);
20045f39d397SChris Mason 				free_extent_buffer(mid);
2005925baeddSChris Mason 			}
2006bb803951SChris Mason 		} else {
20075f39d397SChris Mason 			orig_slot -= btrfs_header_nritems(left);
2008bb803951SChris Mason 			path->slots[level] = orig_slot;
2009bb803951SChris Mason 		}
2010bb803951SChris Mason 	}
201179f95c82SChris Mason 	/* double check we haven't messed things up */
2012e20d96d6SChris Mason 	if (orig_ptr !=
20135f39d397SChris Mason 	    btrfs_node_blockptr(path->nodes[level], path->slots[level]))
201479f95c82SChris Mason 		BUG();
201554aa1f4dSChris Mason enospc:
2016925baeddSChris Mason 	if (right) {
2017925baeddSChris Mason 		btrfs_tree_unlock(right);
20185f39d397SChris Mason 		free_extent_buffer(right);
2019925baeddSChris Mason 	}
2020925baeddSChris Mason 	if (left) {
2021925baeddSChris Mason 		if (path->nodes[level] != left)
2022925baeddSChris Mason 			btrfs_tree_unlock(left);
20235f39d397SChris Mason 		free_extent_buffer(left);
2024925baeddSChris Mason 	}
2025bb803951SChris Mason 	return ret;
2026bb803951SChris Mason }
2027bb803951SChris Mason 
2028d352ac68SChris Mason /* Node balancing for insertion.  Here we only split or push nodes around
2029d352ac68SChris Mason  * when they are completely full.  This is also done top down, so we
2030d352ac68SChris Mason  * have to be pessimistic.
2031d352ac68SChris Mason  */
2032d397712bSChris Mason static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans,
2033e66f709bSChris Mason 					  struct btrfs_root *root,
2034e66f709bSChris Mason 					  struct btrfs_path *path, int level)
2035e66f709bSChris Mason {
20360b246afaSJeff Mahoney 	struct btrfs_fs_info *fs_info = root->fs_info;
20375f39d397SChris Mason 	struct extent_buffer *right = NULL;
20385f39d397SChris Mason 	struct extent_buffer *mid;
20395f39d397SChris Mason 	struct extent_buffer *left = NULL;
20405f39d397SChris Mason 	struct extent_buffer *parent = NULL;
2041e66f709bSChris Mason 	int ret = 0;
2042e66f709bSChris Mason 	int wret;
2043e66f709bSChris Mason 	int pslot;
2044e66f709bSChris Mason 	int orig_slot = path->slots[level];
2045e66f709bSChris Mason 
2046e66f709bSChris Mason 	if (level == 0)
2047e66f709bSChris Mason 		return 1;
2048e66f709bSChris Mason 
20495f39d397SChris Mason 	mid = path->nodes[level];
20507bb86316SChris Mason 	WARN_ON(btrfs_header_generation(mid) != trans->transid);
2051e66f709bSChris Mason 
2052a05a9bb1SLi Zefan 	if (level < BTRFS_MAX_LEVEL - 1) {
20535f39d397SChris Mason 		parent = path->nodes[level + 1];
2054e66f709bSChris Mason 		pslot = path->slots[level + 1];
2055a05a9bb1SLi Zefan 	}
2056e66f709bSChris Mason 
20575f39d397SChris Mason 	if (!parent)
2058e66f709bSChris Mason 		return 1;
2059e66f709bSChris Mason 
20602ff7e61eSJeff Mahoney 	left = read_node_slot(fs_info, parent, pslot - 1);
2061fb770ae4SLiu Bo 	if (IS_ERR(left))
2062fb770ae4SLiu Bo 		left = NULL;
2063e66f709bSChris Mason 
2064e66f709bSChris Mason 	/* first, try to make some room in the middle buffer */
20655f39d397SChris Mason 	if (left) {
2066e66f709bSChris Mason 		u32 left_nr;
2067925baeddSChris Mason 
2068925baeddSChris Mason 		btrfs_tree_lock(left);
2069b4ce94deSChris Mason 		btrfs_set_lock_blocking(left);
2070b4ce94deSChris Mason 
20715f39d397SChris Mason 		left_nr = btrfs_header_nritems(left);
20720b246afaSJeff Mahoney 		if (left_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) {
207333ade1f8SChris Mason 			wret = 1;
207433ade1f8SChris Mason 		} else {
20755f39d397SChris Mason 			ret = btrfs_cow_block(trans, root, left, parent,
20769fa8cfe7SChris Mason 					      pslot - 1, &left);
207754aa1f4dSChris Mason 			if (ret)
207854aa1f4dSChris Mason 				wret = 1;
207954aa1f4dSChris Mason 			else {
20802ff7e61eSJeff Mahoney 				wret = push_node_left(trans, fs_info,
2081971a1f66SChris Mason 						      left, mid, 0);
208254aa1f4dSChris Mason 			}
208333ade1f8SChris Mason 		}
2084e66f709bSChris Mason 		if (wret < 0)
2085e66f709bSChris Mason 			ret = wret;
2086e66f709bSChris Mason 		if (wret == 0) {
20875f39d397SChris Mason 			struct btrfs_disk_key disk_key;
2088e66f709bSChris Mason 			orig_slot += left_nr;
20895f39d397SChris Mason 			btrfs_node_key(mid, &disk_key, 0);
20900e82bcfeSDavid Sterba 			ret = tree_mod_log_insert_key(parent, pslot,
20910e82bcfeSDavid Sterba 					MOD_LOG_KEY_REPLACE, GFP_NOFS);
20920e82bcfeSDavid Sterba 			BUG_ON(ret < 0);
20935f39d397SChris Mason 			btrfs_set_node_key(parent, &disk_key, pslot);
20945f39d397SChris Mason 			btrfs_mark_buffer_dirty(parent);
20955f39d397SChris Mason 			if (btrfs_header_nritems(left) > orig_slot) {
20965f39d397SChris Mason 				path->nodes[level] = left;
2097e66f709bSChris Mason 				path->slots[level + 1] -= 1;
2098e66f709bSChris Mason 				path->slots[level] = orig_slot;
2099925baeddSChris Mason 				btrfs_tree_unlock(mid);
21005f39d397SChris Mason 				free_extent_buffer(mid);
2101e66f709bSChris Mason 			} else {
2102e66f709bSChris Mason 				orig_slot -=
21035f39d397SChris Mason 					btrfs_header_nritems(left);
2104e66f709bSChris Mason 				path->slots[level] = orig_slot;
2105925baeddSChris Mason 				btrfs_tree_unlock(left);
21065f39d397SChris Mason 				free_extent_buffer(left);
2107e66f709bSChris Mason 			}
2108e66f709bSChris Mason 			return 0;
2109e66f709bSChris Mason 		}
2110925baeddSChris Mason 		btrfs_tree_unlock(left);
21115f39d397SChris Mason 		free_extent_buffer(left);
2112e66f709bSChris Mason 	}
21132ff7e61eSJeff Mahoney 	right = read_node_slot(fs_info, parent, pslot + 1);
2114fb770ae4SLiu Bo 	if (IS_ERR(right))
2115fb770ae4SLiu Bo 		right = NULL;
2116e66f709bSChris Mason 
2117e66f709bSChris Mason 	/*
2118e66f709bSChris Mason 	 * then try to empty the right most buffer into the middle
2119e66f709bSChris Mason 	 */
21205f39d397SChris Mason 	if (right) {
212133ade1f8SChris Mason 		u32 right_nr;
2122b4ce94deSChris Mason 
2123925baeddSChris Mason 		btrfs_tree_lock(right);
2124b4ce94deSChris Mason 		btrfs_set_lock_blocking(right);
2125b4ce94deSChris Mason 
21265f39d397SChris Mason 		right_nr = btrfs_header_nritems(right);
21270b246afaSJeff Mahoney 		if (right_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) {
212833ade1f8SChris Mason 			wret = 1;
212933ade1f8SChris Mason 		} else {
21305f39d397SChris Mason 			ret = btrfs_cow_block(trans, root, right,
21315f39d397SChris Mason 					      parent, pslot + 1,
21329fa8cfe7SChris Mason 					      &right);
213354aa1f4dSChris Mason 			if (ret)
213454aa1f4dSChris Mason 				wret = 1;
213554aa1f4dSChris Mason 			else {
21362ff7e61eSJeff Mahoney 				wret = balance_node_right(trans, fs_info,
21375f39d397SChris Mason 							  right, mid);
213833ade1f8SChris Mason 			}
213954aa1f4dSChris Mason 		}
2140e66f709bSChris Mason 		if (wret < 0)
2141e66f709bSChris Mason 			ret = wret;
2142e66f709bSChris Mason 		if (wret == 0) {
21435f39d397SChris Mason 			struct btrfs_disk_key disk_key;
21445f39d397SChris Mason 
21455f39d397SChris Mason 			btrfs_node_key(right, &disk_key, 0);
21460e82bcfeSDavid Sterba 			ret = tree_mod_log_insert_key(parent, pslot + 1,
21470e82bcfeSDavid Sterba 					MOD_LOG_KEY_REPLACE, GFP_NOFS);
21480e82bcfeSDavid Sterba 			BUG_ON(ret < 0);
21495f39d397SChris Mason 			btrfs_set_node_key(parent, &disk_key, pslot + 1);
21505f39d397SChris Mason 			btrfs_mark_buffer_dirty(parent);
21515f39d397SChris Mason 
21525f39d397SChris Mason 			if (btrfs_header_nritems(mid) <= orig_slot) {
21535f39d397SChris Mason 				path->nodes[level] = right;
2154e66f709bSChris Mason 				path->slots[level + 1] += 1;
2155e66f709bSChris Mason 				path->slots[level] = orig_slot -
21565f39d397SChris Mason 					btrfs_header_nritems(mid);
2157925baeddSChris Mason 				btrfs_tree_unlock(mid);
21585f39d397SChris Mason 				free_extent_buffer(mid);
2159e66f709bSChris Mason 			} else {
2160925baeddSChris Mason 				btrfs_tree_unlock(right);
21615f39d397SChris Mason 				free_extent_buffer(right);
2162e66f709bSChris Mason 			}
2163e66f709bSChris Mason 			return 0;
2164e66f709bSChris Mason 		}
2165925baeddSChris Mason 		btrfs_tree_unlock(right);
21665f39d397SChris Mason 		free_extent_buffer(right);
2167e66f709bSChris Mason 	}
2168e66f709bSChris Mason 	return 1;
2169e66f709bSChris Mason }
2170e66f709bSChris Mason 
217174123bd7SChris Mason /*
2172d352ac68SChris Mason  * readahead one full node of leaves, finding things that are close
2173d352ac68SChris Mason  * to the block in 'slot', and triggering ra on them.
21743c69faecSChris Mason  */
21752ff7e61eSJeff Mahoney static void reada_for_search(struct btrfs_fs_info *fs_info,
2176e02119d5SChris Mason 			     struct btrfs_path *path,
217701f46658SChris Mason 			     int level, int slot, u64 objectid)
21783c69faecSChris Mason {
21795f39d397SChris Mason 	struct extent_buffer *node;
218001f46658SChris Mason 	struct btrfs_disk_key disk_key;
21813c69faecSChris Mason 	u32 nritems;
21823c69faecSChris Mason 	u64 search;
2183a7175319SChris Mason 	u64 target;
21846b80053dSChris Mason 	u64 nread = 0;
21855f39d397SChris Mason 	struct extent_buffer *eb;
21866b80053dSChris Mason 	u32 nr;
21876b80053dSChris Mason 	u32 blocksize;
21886b80053dSChris Mason 	u32 nscan = 0;
2189db94535dSChris Mason 
2190a6b6e75eSChris Mason 	if (level != 1)
21913c69faecSChris Mason 		return;
21923c69faecSChris Mason 
21936702ed49SChris Mason 	if (!path->nodes[level])
21946702ed49SChris Mason 		return;
21956702ed49SChris Mason 
21965f39d397SChris Mason 	node = path->nodes[level];
2197925baeddSChris Mason 
21983c69faecSChris Mason 	search = btrfs_node_blockptr(node, slot);
21990b246afaSJeff Mahoney 	blocksize = fs_info->nodesize;
22000b246afaSJeff Mahoney 	eb = find_extent_buffer(fs_info, search);
22015f39d397SChris Mason 	if (eb) {
22025f39d397SChris Mason 		free_extent_buffer(eb);
22033c69faecSChris Mason 		return;
22043c69faecSChris Mason 	}
22053c69faecSChris Mason 
2206a7175319SChris Mason 	target = search;
22076b80053dSChris Mason 
22085f39d397SChris Mason 	nritems = btrfs_header_nritems(node);
22096b80053dSChris Mason 	nr = slot;
221025b8b936SJosef Bacik 
22113c69faecSChris Mason 	while (1) {
2212e4058b54SDavid Sterba 		if (path->reada == READA_BACK) {
22136b80053dSChris Mason 			if (nr == 0)
22143c69faecSChris Mason 				break;
22156b80053dSChris Mason 			nr--;
2216e4058b54SDavid Sterba 		} else if (path->reada == READA_FORWARD) {
22176b80053dSChris Mason 			nr++;
22186b80053dSChris Mason 			if (nr >= nritems)
22196b80053dSChris Mason 				break;
22203c69faecSChris Mason 		}
2221e4058b54SDavid Sterba 		if (path->reada == READA_BACK && objectid) {
222201f46658SChris Mason 			btrfs_node_key(node, &disk_key, nr);
222301f46658SChris Mason 			if (btrfs_disk_key_objectid(&disk_key) != objectid)
222401f46658SChris Mason 				break;
222501f46658SChris Mason 		}
22266b80053dSChris Mason 		search = btrfs_node_blockptr(node, nr);
2227a7175319SChris Mason 		if ((search <= target && target - search <= 65536) ||
2228a7175319SChris Mason 		    (search > target && search - target <= 65536)) {
22292ff7e61eSJeff Mahoney 			readahead_tree_block(fs_info, search);
22306b80053dSChris Mason 			nread += blocksize;
22313c69faecSChris Mason 		}
22326b80053dSChris Mason 		nscan++;
2233a7175319SChris Mason 		if ((nread > 65536 || nscan > 32))
22346b80053dSChris Mason 			break;
22353c69faecSChris Mason 	}
22363c69faecSChris Mason }
2237925baeddSChris Mason 
22382ff7e61eSJeff Mahoney static noinline void reada_for_balance(struct btrfs_fs_info *fs_info,
2239b4ce94deSChris Mason 				       struct btrfs_path *path, int level)
2240b4ce94deSChris Mason {
2241b4ce94deSChris Mason 	int slot;
2242b4ce94deSChris Mason 	int nritems;
2243b4ce94deSChris Mason 	struct extent_buffer *parent;
2244b4ce94deSChris Mason 	struct extent_buffer *eb;
2245b4ce94deSChris Mason 	u64 gen;
2246b4ce94deSChris Mason 	u64 block1 = 0;
2247b4ce94deSChris Mason 	u64 block2 = 0;
2248b4ce94deSChris Mason 
22498c594ea8SChris Mason 	parent = path->nodes[level + 1];
2250b4ce94deSChris Mason 	if (!parent)
22510b08851fSJosef Bacik 		return;
2252b4ce94deSChris Mason 
2253b4ce94deSChris Mason 	nritems = btrfs_header_nritems(parent);
22548c594ea8SChris Mason 	slot = path->slots[level + 1];
2255b4ce94deSChris Mason 
2256b4ce94deSChris Mason 	if (slot > 0) {
2257b4ce94deSChris Mason 		block1 = btrfs_node_blockptr(parent, slot - 1);
2258b4ce94deSChris Mason 		gen = btrfs_node_ptr_generation(parent, slot - 1);
22590b246afaSJeff Mahoney 		eb = find_extent_buffer(fs_info, block1);
2260b9fab919SChris Mason 		/*
2261b9fab919SChris Mason 		 * if we get -eagain from btrfs_buffer_uptodate, we
2262b9fab919SChris Mason 		 * don't want to return eagain here.  That will loop
2263b9fab919SChris Mason 		 * forever
2264b9fab919SChris Mason 		 */
2265b9fab919SChris Mason 		if (eb && btrfs_buffer_uptodate(eb, gen, 1) != 0)
2266b4ce94deSChris Mason 			block1 = 0;
2267b4ce94deSChris Mason 		free_extent_buffer(eb);
2268b4ce94deSChris Mason 	}
22698c594ea8SChris Mason 	if (slot + 1 < nritems) {
2270b4ce94deSChris Mason 		block2 = btrfs_node_blockptr(parent, slot + 1);
2271b4ce94deSChris Mason 		gen = btrfs_node_ptr_generation(parent, slot + 1);
22720b246afaSJeff Mahoney 		eb = find_extent_buffer(fs_info, block2);
2273b9fab919SChris Mason 		if (eb && btrfs_buffer_uptodate(eb, gen, 1) != 0)
2274b4ce94deSChris Mason 			block2 = 0;
2275b4ce94deSChris Mason 		free_extent_buffer(eb);
2276b4ce94deSChris Mason 	}
22778c594ea8SChris Mason 
2278b4ce94deSChris Mason 	if (block1)
22792ff7e61eSJeff Mahoney 		readahead_tree_block(fs_info, block1);
2280b4ce94deSChris Mason 	if (block2)
22812ff7e61eSJeff Mahoney 		readahead_tree_block(fs_info, block2);
2282b4ce94deSChris Mason }
2283b4ce94deSChris Mason 
2284b4ce94deSChris Mason 
2285b4ce94deSChris Mason /*
2286d397712bSChris Mason  * when we walk down the tree, it is usually safe to unlock the higher layers
2287d397712bSChris Mason  * in the tree.  The exceptions are when our path goes through slot 0, because
2288d397712bSChris Mason  * operations on the tree might require changing key pointers higher up in the
2289d397712bSChris Mason  * tree.
2290d352ac68SChris Mason  *
2291d397712bSChris Mason  * callers might also have set path->keep_locks, which tells this code to keep
2292d397712bSChris Mason  * the lock if the path points to the last slot in the block.  This is part of
2293d397712bSChris Mason  * walking through the tree, and selecting the next slot in the higher block.
2294d352ac68SChris Mason  *
2295d397712bSChris Mason  * lowest_unlock sets the lowest level in the tree we're allowed to unlock.  so
2296d397712bSChris Mason  * if lowest_unlock is 1, level 0 won't be unlocked
2297d352ac68SChris Mason  */
2298e02119d5SChris Mason static noinline void unlock_up(struct btrfs_path *path, int level,
2299f7c79f30SChris Mason 			       int lowest_unlock, int min_write_lock_level,
2300f7c79f30SChris Mason 			       int *write_lock_level)
2301925baeddSChris Mason {
2302925baeddSChris Mason 	int i;
2303925baeddSChris Mason 	int skip_level = level;
2304051e1b9fSChris Mason 	int no_skips = 0;
2305925baeddSChris Mason 	struct extent_buffer *t;
2306925baeddSChris Mason 
2307925baeddSChris Mason 	for (i = level; i < BTRFS_MAX_LEVEL; i++) {
2308925baeddSChris Mason 		if (!path->nodes[i])
2309925baeddSChris Mason 			break;
2310925baeddSChris Mason 		if (!path->locks[i])
2311925baeddSChris Mason 			break;
2312051e1b9fSChris Mason 		if (!no_skips && path->slots[i] == 0) {
2313925baeddSChris Mason 			skip_level = i + 1;
2314925baeddSChris Mason 			continue;
2315925baeddSChris Mason 		}
2316051e1b9fSChris Mason 		if (!no_skips && path->keep_locks) {
2317925baeddSChris Mason 			u32 nritems;
2318925baeddSChris Mason 			t = path->nodes[i];
2319925baeddSChris Mason 			nritems = btrfs_header_nritems(t);
2320051e1b9fSChris Mason 			if (nritems < 1 || path->slots[i] >= nritems - 1) {
2321925baeddSChris Mason 				skip_level = i + 1;
2322925baeddSChris Mason 				continue;
2323925baeddSChris Mason 			}
2324925baeddSChris Mason 		}
2325051e1b9fSChris Mason 		if (skip_level < i && i >= lowest_unlock)
2326051e1b9fSChris Mason 			no_skips = 1;
2327051e1b9fSChris Mason 
2328925baeddSChris Mason 		t = path->nodes[i];
2329d80bb3f9SLiu Bo 		if (i >= lowest_unlock && i > skip_level) {
2330bd681513SChris Mason 			btrfs_tree_unlock_rw(t, path->locks[i]);
2331925baeddSChris Mason 			path->locks[i] = 0;
2332f7c79f30SChris Mason 			if (write_lock_level &&
2333f7c79f30SChris Mason 			    i > min_write_lock_level &&
2334f7c79f30SChris Mason 			    i <= *write_lock_level) {
2335f7c79f30SChris Mason 				*write_lock_level = i - 1;
2336f7c79f30SChris Mason 			}
2337925baeddSChris Mason 		}
2338925baeddSChris Mason 	}
2339925baeddSChris Mason }
2340925baeddSChris Mason 
23413c69faecSChris Mason /*
2342b4ce94deSChris Mason  * This releases any locks held in the path starting at level and
2343b4ce94deSChris Mason  * going all the way up to the root.
2344b4ce94deSChris Mason  *
2345b4ce94deSChris Mason  * btrfs_search_slot will keep the lock held on higher nodes in a few
2346b4ce94deSChris Mason  * corner cases, such as COW of the block at slot zero in the node.  This
2347b4ce94deSChris Mason  * ignores those rules, and it should only be called when there are no
2348b4ce94deSChris Mason  * more updates to be done higher up in the tree.
2349b4ce94deSChris Mason  */
2350b4ce94deSChris Mason noinline void btrfs_unlock_up_safe(struct btrfs_path *path, int level)
2351b4ce94deSChris Mason {
2352b4ce94deSChris Mason 	int i;
2353b4ce94deSChris Mason 
235409a2a8f9SJosef Bacik 	if (path->keep_locks)
2355b4ce94deSChris Mason 		return;
2356b4ce94deSChris Mason 
2357b4ce94deSChris Mason 	for (i = level; i < BTRFS_MAX_LEVEL; i++) {
2358b4ce94deSChris Mason 		if (!path->nodes[i])
235912f4daccSChris Mason 			continue;
2360b4ce94deSChris Mason 		if (!path->locks[i])
236112f4daccSChris Mason 			continue;
2362bd681513SChris Mason 		btrfs_tree_unlock_rw(path->nodes[i], path->locks[i]);
2363b4ce94deSChris Mason 		path->locks[i] = 0;
2364b4ce94deSChris Mason 	}
2365b4ce94deSChris Mason }
2366b4ce94deSChris Mason 
2367b4ce94deSChris Mason /*
2368c8c42864SChris Mason  * helper function for btrfs_search_slot.  The goal is to find a block
2369c8c42864SChris Mason  * in cache without setting the path to blocking.  If we find the block
2370c8c42864SChris Mason  * we return zero and the path is unchanged.
2371c8c42864SChris Mason  *
2372c8c42864SChris Mason  * If we can't find the block, we set the path blocking and do some
2373c8c42864SChris Mason  * reada.  -EAGAIN is returned and the search must be repeated.
2374c8c42864SChris Mason  */
2375c8c42864SChris Mason static int
2376d07b8528SLiu Bo read_block_for_search(struct btrfs_root *root, struct btrfs_path *p,
2377c8c42864SChris Mason 		      struct extent_buffer **eb_ret, int level, int slot,
2378cda79c54SDavid Sterba 		      const struct btrfs_key *key)
2379c8c42864SChris Mason {
23800b246afaSJeff Mahoney 	struct btrfs_fs_info *fs_info = root->fs_info;
2381c8c42864SChris Mason 	u64 blocknr;
2382c8c42864SChris Mason 	u64 gen;
2383c8c42864SChris Mason 	struct extent_buffer *b = *eb_ret;
2384c8c42864SChris Mason 	struct extent_buffer *tmp;
2385581c1760SQu Wenruo 	struct btrfs_key first_key;
238676a05b35SChris Mason 	int ret;
2387581c1760SQu Wenruo 	int parent_level;
2388c8c42864SChris Mason 
2389c8c42864SChris Mason 	blocknr = btrfs_node_blockptr(b, slot);
2390c8c42864SChris Mason 	gen = btrfs_node_ptr_generation(b, slot);
2391581c1760SQu Wenruo 	parent_level = btrfs_header_level(b);
2392581c1760SQu Wenruo 	btrfs_node_key_to_cpu(b, &first_key, slot);
2393c8c42864SChris Mason 
23940b246afaSJeff Mahoney 	tmp = find_extent_buffer(fs_info, blocknr);
2395cb44921aSChris Mason 	if (tmp) {
2396b9fab919SChris Mason 		/* first we do an atomic uptodate check */
2397b9fab919SChris Mason 		if (btrfs_buffer_uptodate(tmp, gen, 1) > 0) {
2398c8c42864SChris Mason 			*eb_ret = tmp;
2399c8c42864SChris Mason 			return 0;
2400c8c42864SChris Mason 		}
2401bdf7c00eSJosef Bacik 
2402cb44921aSChris Mason 		/* the pages were up to date, but we failed
2403cb44921aSChris Mason 		 * the generation number check.  Do a full
2404cb44921aSChris Mason 		 * read for the generation number that is correct.
2405cb44921aSChris Mason 		 * We must do this without dropping locks so
2406cb44921aSChris Mason 		 * we can trust our generation number
2407cb44921aSChris Mason 		 */
2408bd681513SChris Mason 		btrfs_set_path_blocking(p);
2409bd681513SChris Mason 
2410b9fab919SChris Mason 		/* now we're allowed to do a blocking uptodate check */
2411581c1760SQu Wenruo 		ret = btrfs_read_buffer(tmp, gen, parent_level - 1, &first_key);
2412bdf7c00eSJosef Bacik 		if (!ret) {
2413cb44921aSChris Mason 			*eb_ret = tmp;
2414cb44921aSChris Mason 			return 0;
2415cb44921aSChris Mason 		}
2416cb44921aSChris Mason 		free_extent_buffer(tmp);
2417b3b4aa74SDavid Sterba 		btrfs_release_path(p);
2418cb44921aSChris Mason 		return -EIO;
2419cb44921aSChris Mason 	}
2420c8c42864SChris Mason 
2421c8c42864SChris Mason 	/*
2422c8c42864SChris Mason 	 * reduce lock contention at high levels
2423c8c42864SChris Mason 	 * of the btree by dropping locks before
242476a05b35SChris Mason 	 * we read.  Don't release the lock on the current
242576a05b35SChris Mason 	 * level because we need to walk this node to figure
242676a05b35SChris Mason 	 * out which blocks to read.
2427c8c42864SChris Mason 	 */
24288c594ea8SChris Mason 	btrfs_unlock_up_safe(p, level + 1);
24298c594ea8SChris Mason 	btrfs_set_path_blocking(p);
24308c594ea8SChris Mason 
2431e4058b54SDavid Sterba 	if (p->reada != READA_NONE)
24322ff7e61eSJeff Mahoney 		reada_for_search(fs_info, p, level, slot, key->objectid);
2433c8c42864SChris Mason 
243476a05b35SChris Mason 	ret = -EAGAIN;
243502a3307aSLiu Bo 	tmp = read_tree_block(fs_info, blocknr, gen, parent_level - 1,
2436581c1760SQu Wenruo 			      &first_key);
243764c043deSLiu Bo 	if (!IS_ERR(tmp)) {
243876a05b35SChris Mason 		/*
243976a05b35SChris Mason 		 * If the read above didn't mark this buffer up to date,
244076a05b35SChris Mason 		 * it will never end up being up to date.  Set ret to EIO now
244176a05b35SChris Mason 		 * and give up so that our caller doesn't loop forever
244276a05b35SChris Mason 		 * on our EAGAINs.
244376a05b35SChris Mason 		 */
2444e6a1d6fdSLiu Bo 		if (!extent_buffer_uptodate(tmp))
244576a05b35SChris Mason 			ret = -EIO;
2446c8c42864SChris Mason 		free_extent_buffer(tmp);
2447c871b0f2SLiu Bo 	} else {
2448c871b0f2SLiu Bo 		ret = PTR_ERR(tmp);
244976a05b35SChris Mason 	}
245002a3307aSLiu Bo 
245102a3307aSLiu Bo 	btrfs_release_path(p);
245276a05b35SChris Mason 	return ret;
2453c8c42864SChris Mason }
2454c8c42864SChris Mason 
2455c8c42864SChris Mason /*
2456c8c42864SChris Mason  * helper function for btrfs_search_slot.  This does all of the checks
2457c8c42864SChris Mason  * for node-level blocks and does any balancing required based on
2458c8c42864SChris Mason  * the ins_len.
2459c8c42864SChris Mason  *
2460c8c42864SChris Mason  * If no extra work was required, zero is returned.  If we had to
2461c8c42864SChris Mason  * drop the path, -EAGAIN is returned and btrfs_search_slot must
2462c8c42864SChris Mason  * start over
2463c8c42864SChris Mason  */
2464c8c42864SChris Mason static int
2465c8c42864SChris Mason setup_nodes_for_search(struct btrfs_trans_handle *trans,
2466c8c42864SChris Mason 		       struct btrfs_root *root, struct btrfs_path *p,
2467bd681513SChris Mason 		       struct extent_buffer *b, int level, int ins_len,
2468bd681513SChris Mason 		       int *write_lock_level)
2469c8c42864SChris Mason {
24700b246afaSJeff Mahoney 	struct btrfs_fs_info *fs_info = root->fs_info;
2471c8c42864SChris Mason 	int ret;
24720b246afaSJeff Mahoney 
2473c8c42864SChris Mason 	if ((p->search_for_split || ins_len > 0) && btrfs_header_nritems(b) >=
24740b246afaSJeff Mahoney 	    BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3) {
2475c8c42864SChris Mason 		int sret;
2476c8c42864SChris Mason 
2477bd681513SChris Mason 		if (*write_lock_level < level + 1) {
2478bd681513SChris Mason 			*write_lock_level = level + 1;
2479bd681513SChris Mason 			btrfs_release_path(p);
2480bd681513SChris Mason 			goto again;
2481bd681513SChris Mason 		}
2482bd681513SChris Mason 
2483c8c42864SChris Mason 		btrfs_set_path_blocking(p);
24842ff7e61eSJeff Mahoney 		reada_for_balance(fs_info, p, level);
2485c8c42864SChris Mason 		sret = split_node(trans, root, p, level);
2486bd681513SChris Mason 		btrfs_clear_path_blocking(p, NULL, 0);
2487c8c42864SChris Mason 
2488c8c42864SChris Mason 		BUG_ON(sret > 0);
2489c8c42864SChris Mason 		if (sret) {
2490c8c42864SChris Mason 			ret = sret;
2491c8c42864SChris Mason 			goto done;
2492c8c42864SChris Mason 		}
2493c8c42864SChris Mason 		b = p->nodes[level];
2494c8c42864SChris Mason 	} else if (ins_len < 0 && btrfs_header_nritems(b) <
24950b246afaSJeff Mahoney 		   BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 2) {
2496c8c42864SChris Mason 		int sret;
2497c8c42864SChris Mason 
2498bd681513SChris Mason 		if (*write_lock_level < level + 1) {
2499bd681513SChris Mason 			*write_lock_level = level + 1;
2500bd681513SChris Mason 			btrfs_release_path(p);
2501bd681513SChris Mason 			goto again;
2502bd681513SChris Mason 		}
2503bd681513SChris Mason 
2504c8c42864SChris Mason 		btrfs_set_path_blocking(p);
25052ff7e61eSJeff Mahoney 		reada_for_balance(fs_info, p, level);
2506c8c42864SChris Mason 		sret = balance_level(trans, root, p, level);
2507bd681513SChris Mason 		btrfs_clear_path_blocking(p, NULL, 0);
2508c8c42864SChris Mason 
2509c8c42864SChris Mason 		if (sret) {
2510c8c42864SChris Mason 			ret = sret;
2511c8c42864SChris Mason 			goto done;
2512c8c42864SChris Mason 		}
2513c8c42864SChris Mason 		b = p->nodes[level];
2514c8c42864SChris Mason 		if (!b) {
2515b3b4aa74SDavid Sterba 			btrfs_release_path(p);
2516c8c42864SChris Mason 			goto again;
2517c8c42864SChris Mason 		}
2518c8c42864SChris Mason 		BUG_ON(btrfs_header_nritems(b) == 1);
2519c8c42864SChris Mason 	}
2520c8c42864SChris Mason 	return 0;
2521c8c42864SChris Mason 
2522c8c42864SChris Mason again:
2523c8c42864SChris Mason 	ret = -EAGAIN;
2524c8c42864SChris Mason done:
2525c8c42864SChris Mason 	return ret;
2526c8c42864SChris Mason }
2527c8c42864SChris Mason 
2528d7396f07SFilipe David Borba Manana static void key_search_validate(struct extent_buffer *b,
2529310712b2SOmar Sandoval 				const struct btrfs_key *key,
2530d7396f07SFilipe David Borba Manana 				int level)
2531d7396f07SFilipe David Borba Manana {
2532d7396f07SFilipe David Borba Manana #ifdef CONFIG_BTRFS_ASSERT
2533d7396f07SFilipe David Borba Manana 	struct btrfs_disk_key disk_key;
2534d7396f07SFilipe David Borba Manana 
2535d7396f07SFilipe David Borba Manana 	btrfs_cpu_key_to_disk(&disk_key, key);
2536d7396f07SFilipe David Borba Manana 
2537d7396f07SFilipe David Borba Manana 	if (level == 0)
2538d7396f07SFilipe David Borba Manana 		ASSERT(!memcmp_extent_buffer(b, &disk_key,
2539d7396f07SFilipe David Borba Manana 		    offsetof(struct btrfs_leaf, items[0].key),
2540d7396f07SFilipe David Borba Manana 		    sizeof(disk_key)));
2541d7396f07SFilipe David Borba Manana 	else
2542d7396f07SFilipe David Borba Manana 		ASSERT(!memcmp_extent_buffer(b, &disk_key,
2543d7396f07SFilipe David Borba Manana 		    offsetof(struct btrfs_node, ptrs[0].key),
2544d7396f07SFilipe David Borba Manana 		    sizeof(disk_key)));
2545d7396f07SFilipe David Borba Manana #endif
2546d7396f07SFilipe David Borba Manana }
2547d7396f07SFilipe David Borba Manana 
2548310712b2SOmar Sandoval static int key_search(struct extent_buffer *b, const struct btrfs_key *key,
2549d7396f07SFilipe David Borba Manana 		      int level, int *prev_cmp, int *slot)
2550d7396f07SFilipe David Borba Manana {
2551d7396f07SFilipe David Borba Manana 	if (*prev_cmp != 0) {
2552a74b35ecSNikolay Borisov 		*prev_cmp = btrfs_bin_search(b, key, level, slot);
2553d7396f07SFilipe David Borba Manana 		return *prev_cmp;
2554d7396f07SFilipe David Borba Manana 	}
2555d7396f07SFilipe David Borba Manana 
2556d7396f07SFilipe David Borba Manana 	key_search_validate(b, key, level);
2557d7396f07SFilipe David Borba Manana 	*slot = 0;
2558d7396f07SFilipe David Borba Manana 
2559d7396f07SFilipe David Borba Manana 	return 0;
2560d7396f07SFilipe David Borba Manana }
2561d7396f07SFilipe David Borba Manana 
2562381cf658SDavid Sterba int btrfs_find_item(struct btrfs_root *fs_root, struct btrfs_path *path,
2563e33d5c3dSKelley Nielsen 		u64 iobjectid, u64 ioff, u8 key_type,
2564e33d5c3dSKelley Nielsen 		struct btrfs_key *found_key)
2565e33d5c3dSKelley Nielsen {
2566e33d5c3dSKelley Nielsen 	int ret;
2567e33d5c3dSKelley Nielsen 	struct btrfs_key key;
2568e33d5c3dSKelley Nielsen 	struct extent_buffer *eb;
2569381cf658SDavid Sterba 
2570381cf658SDavid Sterba 	ASSERT(path);
25711d4c08e0SDavid Sterba 	ASSERT(found_key);
2572e33d5c3dSKelley Nielsen 
2573e33d5c3dSKelley Nielsen 	key.type = key_type;
2574e33d5c3dSKelley Nielsen 	key.objectid = iobjectid;
2575e33d5c3dSKelley Nielsen 	key.offset = ioff;
2576e33d5c3dSKelley Nielsen 
2577e33d5c3dSKelley Nielsen 	ret = btrfs_search_slot(NULL, fs_root, &key, path, 0, 0);
25781d4c08e0SDavid Sterba 	if (ret < 0)
2579e33d5c3dSKelley Nielsen 		return ret;
2580e33d5c3dSKelley Nielsen 
2581e33d5c3dSKelley Nielsen 	eb = path->nodes[0];
2582e33d5c3dSKelley Nielsen 	if (ret && path->slots[0] >= btrfs_header_nritems(eb)) {
2583e33d5c3dSKelley Nielsen 		ret = btrfs_next_leaf(fs_root, path);
2584e33d5c3dSKelley Nielsen 		if (ret)
2585e33d5c3dSKelley Nielsen 			return ret;
2586e33d5c3dSKelley Nielsen 		eb = path->nodes[0];
2587e33d5c3dSKelley Nielsen 	}
2588e33d5c3dSKelley Nielsen 
2589e33d5c3dSKelley Nielsen 	btrfs_item_key_to_cpu(eb, found_key, path->slots[0]);
2590e33d5c3dSKelley Nielsen 	if (found_key->type != key.type ||
2591e33d5c3dSKelley Nielsen 			found_key->objectid != key.objectid)
2592e33d5c3dSKelley Nielsen 		return 1;
2593e33d5c3dSKelley Nielsen 
2594e33d5c3dSKelley Nielsen 	return 0;
2595e33d5c3dSKelley Nielsen }
2596e33d5c3dSKelley Nielsen 
25971fc28d8eSLiu Bo static struct extent_buffer *btrfs_search_slot_get_root(struct btrfs_root *root,
25981fc28d8eSLiu Bo 							struct btrfs_path *p,
25991fc28d8eSLiu Bo 							int write_lock_level)
26001fc28d8eSLiu Bo {
26011fc28d8eSLiu Bo 	struct btrfs_fs_info *fs_info = root->fs_info;
26021fc28d8eSLiu Bo 	struct extent_buffer *b;
26031fc28d8eSLiu Bo 	int root_lock;
26041fc28d8eSLiu Bo 	int level = 0;
26051fc28d8eSLiu Bo 
26061fc28d8eSLiu Bo 	/* We try very hard to do read locks on the root */
26071fc28d8eSLiu Bo 	root_lock = BTRFS_READ_LOCK;
26081fc28d8eSLiu Bo 
26091fc28d8eSLiu Bo 	if (p->search_commit_root) {
26101fc28d8eSLiu Bo 		/* The commit roots are read only so we always do read locks */
26111fc28d8eSLiu Bo 		if (p->need_commit_sem)
26121fc28d8eSLiu Bo 			down_read(&fs_info->commit_root_sem);
26131fc28d8eSLiu Bo 		b = root->commit_root;
26141fc28d8eSLiu Bo 		extent_buffer_get(b);
26151fc28d8eSLiu Bo 		level = btrfs_header_level(b);
26161fc28d8eSLiu Bo 		if (p->need_commit_sem)
26171fc28d8eSLiu Bo 			up_read(&fs_info->commit_root_sem);
2618f9ddfd05SLiu Bo 		/*
2619f9ddfd05SLiu Bo 		 * Ensure that all callers have set skip_locking when
2620f9ddfd05SLiu Bo 		 * p->search_commit_root = 1.
2621f9ddfd05SLiu Bo 		 */
2622f9ddfd05SLiu Bo 		ASSERT(p->skip_locking == 1);
26231fc28d8eSLiu Bo 
26241fc28d8eSLiu Bo 		goto out;
26251fc28d8eSLiu Bo 	}
26261fc28d8eSLiu Bo 
26271fc28d8eSLiu Bo 	if (p->skip_locking) {
26281fc28d8eSLiu Bo 		b = btrfs_root_node(root);
26291fc28d8eSLiu Bo 		level = btrfs_header_level(b);
26301fc28d8eSLiu Bo 		goto out;
26311fc28d8eSLiu Bo 	}
26321fc28d8eSLiu Bo 
26331fc28d8eSLiu Bo 	/*
2634662c653bSLiu Bo 	 * If the level is set to maximum, we can skip trying to get the read
2635662c653bSLiu Bo 	 * lock.
2636662c653bSLiu Bo 	 */
2637662c653bSLiu Bo 	if (write_lock_level < BTRFS_MAX_LEVEL) {
2638662c653bSLiu Bo 		/*
2639662c653bSLiu Bo 		 * We don't know the level of the root node until we actually
2640662c653bSLiu Bo 		 * have it read locked
26411fc28d8eSLiu Bo 		 */
26421fc28d8eSLiu Bo 		b = btrfs_read_lock_root_node(root);
26431fc28d8eSLiu Bo 		level = btrfs_header_level(b);
26441fc28d8eSLiu Bo 		if (level > write_lock_level)
26451fc28d8eSLiu Bo 			goto out;
26461fc28d8eSLiu Bo 
2647662c653bSLiu Bo 		/* Whoops, must trade for write lock */
26481fc28d8eSLiu Bo 		btrfs_tree_read_unlock(b);
26491fc28d8eSLiu Bo 		free_extent_buffer(b);
2650662c653bSLiu Bo 	}
2651662c653bSLiu Bo 
26521fc28d8eSLiu Bo 	b = btrfs_lock_root_node(root);
26531fc28d8eSLiu Bo 	root_lock = BTRFS_WRITE_LOCK;
26541fc28d8eSLiu Bo 
26551fc28d8eSLiu Bo 	/* The level might have changed, check again */
26561fc28d8eSLiu Bo 	level = btrfs_header_level(b);
26571fc28d8eSLiu Bo 
26581fc28d8eSLiu Bo out:
26591fc28d8eSLiu Bo 	p->nodes[level] = b;
26601fc28d8eSLiu Bo 	if (!p->skip_locking)
26611fc28d8eSLiu Bo 		p->locks[level] = root_lock;
26621fc28d8eSLiu Bo 	/*
26631fc28d8eSLiu Bo 	 * Callers are responsible for dropping b's references.
26641fc28d8eSLiu Bo 	 */
26651fc28d8eSLiu Bo 	return b;
26661fc28d8eSLiu Bo }
26671fc28d8eSLiu Bo 
26681fc28d8eSLiu Bo 
2669c8c42864SChris Mason /*
26704271eceaSNikolay Borisov  * btrfs_search_slot - look for a key in a tree and perform necessary
26714271eceaSNikolay Borisov  * modifications to preserve tree invariants.
267274123bd7SChris Mason  *
26734271eceaSNikolay Borisov  * @trans:	Handle of transaction, used when modifying the tree
26744271eceaSNikolay Borisov  * @p:		Holds all btree nodes along the search path
26754271eceaSNikolay Borisov  * @root:	The root node of the tree
26764271eceaSNikolay Borisov  * @key:	The key we are looking for
26774271eceaSNikolay Borisov  * @ins_len:	Indicates purpose of search, for inserts it is 1, for
26784271eceaSNikolay Borisov  *		deletions it's -1. 0 for plain searches
26794271eceaSNikolay Borisov  * @cow:	boolean should CoW operations be performed. Must always be 1
26804271eceaSNikolay Borisov  *		when modifying the tree.
268197571fd0SChris Mason  *
26824271eceaSNikolay Borisov  * If @ins_len > 0, nodes and leaves will be split as we walk down the tree.
26834271eceaSNikolay Borisov  * If @ins_len < 0, nodes will be merged as we walk down the tree (if possible)
26844271eceaSNikolay Borisov  *
26854271eceaSNikolay Borisov  * If @key is found, 0 is returned and you can find the item in the leaf level
26864271eceaSNikolay Borisov  * of the path (level 0)
26874271eceaSNikolay Borisov  *
26884271eceaSNikolay Borisov  * If @key isn't found, 1 is returned and the leaf level of the path (level 0)
26894271eceaSNikolay Borisov  * points to the slot where it should be inserted
26904271eceaSNikolay Borisov  *
26914271eceaSNikolay Borisov  * If an error is encountered while searching the tree a negative error number
26924271eceaSNikolay Borisov  * is returned
269374123bd7SChris Mason  */
2694310712b2SOmar Sandoval int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root,
2695310712b2SOmar Sandoval 		      const struct btrfs_key *key, struct btrfs_path *p,
2696310712b2SOmar Sandoval 		      int ins_len, int cow)
2697be0e5c09SChris Mason {
26980b246afaSJeff Mahoney 	struct btrfs_fs_info *fs_info = root->fs_info;
26995f39d397SChris Mason 	struct extent_buffer *b;
2700be0e5c09SChris Mason 	int slot;
2701be0e5c09SChris Mason 	int ret;
270233c66f43SYan Zheng 	int err;
2703be0e5c09SChris Mason 	int level;
2704925baeddSChris Mason 	int lowest_unlock = 1;
2705bd681513SChris Mason 	/* everything at write_lock_level or lower must be write locked */
2706bd681513SChris Mason 	int write_lock_level = 0;
27079f3a7427SChris Mason 	u8 lowest_level = 0;
2708f7c79f30SChris Mason 	int min_write_lock_level;
2709d7396f07SFilipe David Borba Manana 	int prev_cmp;
27109f3a7427SChris Mason 
27116702ed49SChris Mason 	lowest_level = p->lowest_level;
2712323ac95bSChris Mason 	WARN_ON(lowest_level && ins_len > 0);
271322b0ebdaSChris Mason 	WARN_ON(p->nodes[0] != NULL);
2714eb653de1SFilipe David Borba Manana 	BUG_ON(!cow && ins_len);
271525179201SJosef Bacik 
2716bd681513SChris Mason 	if (ins_len < 0) {
2717925baeddSChris Mason 		lowest_unlock = 2;
271865b51a00SChris Mason 
2719bd681513SChris Mason 		/* when we are removing items, we might have to go up to level
2720bd681513SChris Mason 		 * two as we update tree pointers  Make sure we keep write
2721bd681513SChris Mason 		 * for those levels as well
2722bd681513SChris Mason 		 */
2723bd681513SChris Mason 		write_lock_level = 2;
2724bd681513SChris Mason 	} else if (ins_len > 0) {
2725bd681513SChris Mason 		/*
2726bd681513SChris Mason 		 * for inserting items, make sure we have a write lock on
2727bd681513SChris Mason 		 * level 1 so we can update keys
2728bd681513SChris Mason 		 */
2729bd681513SChris Mason 		write_lock_level = 1;
2730bd681513SChris Mason 	}
2731bd681513SChris Mason 
2732bd681513SChris Mason 	if (!cow)
2733bd681513SChris Mason 		write_lock_level = -1;
2734bd681513SChris Mason 
273509a2a8f9SJosef Bacik 	if (cow && (p->keep_locks || p->lowest_level))
2736bd681513SChris Mason 		write_lock_level = BTRFS_MAX_LEVEL;
2737bd681513SChris Mason 
2738f7c79f30SChris Mason 	min_write_lock_level = write_lock_level;
2739f7c79f30SChris Mason 
2740bb803951SChris Mason again:
2741d7396f07SFilipe David Borba Manana 	prev_cmp = -1;
27421fc28d8eSLiu Bo 	b = btrfs_search_slot_get_root(root, p, write_lock_level);
2743925baeddSChris Mason 
2744eb60ceacSChris Mason 	while (b) {
27455f39d397SChris Mason 		level = btrfs_header_level(b);
274665b51a00SChris Mason 
274765b51a00SChris Mason 		/*
274865b51a00SChris Mason 		 * setup the path here so we can release it under lock
274965b51a00SChris Mason 		 * contention with the cow code
275065b51a00SChris Mason 		 */
275102217ed2SChris Mason 		if (cow) {
27529ea2c7c9SNikolay Borisov 			bool last_level = (level == (BTRFS_MAX_LEVEL - 1));
27539ea2c7c9SNikolay Borisov 
2754c8c42864SChris Mason 			/*
2755c8c42864SChris Mason 			 * if we don't really need to cow this block
2756c8c42864SChris Mason 			 * then we don't want to set the path blocking,
2757c8c42864SChris Mason 			 * so we test it here
2758c8c42864SChris Mason 			 */
275964c12921SJeff Mahoney 			if (!should_cow_block(trans, root, b)) {
276064c12921SJeff Mahoney 				trans->dirty = true;
276165b51a00SChris Mason 				goto cow_done;
276264c12921SJeff Mahoney 			}
27635d4f98a2SYan Zheng 
2764bd681513SChris Mason 			/*
2765bd681513SChris Mason 			 * must have write locks on this node and the
2766bd681513SChris Mason 			 * parent
2767bd681513SChris Mason 			 */
27685124e00eSJosef Bacik 			if (level > write_lock_level ||
27695124e00eSJosef Bacik 			    (level + 1 > write_lock_level &&
27705124e00eSJosef Bacik 			    level + 1 < BTRFS_MAX_LEVEL &&
27715124e00eSJosef Bacik 			    p->nodes[level + 1])) {
2772bd681513SChris Mason 				write_lock_level = level + 1;
2773bd681513SChris Mason 				btrfs_release_path(p);
2774bd681513SChris Mason 				goto again;
2775bd681513SChris Mason 			}
2776bd681513SChris Mason 
2777160f4089SFilipe Manana 			btrfs_set_path_blocking(p);
27789ea2c7c9SNikolay Borisov 			if (last_level)
27799ea2c7c9SNikolay Borisov 				err = btrfs_cow_block(trans, root, b, NULL, 0,
27809ea2c7c9SNikolay Borisov 						      &b);
27819ea2c7c9SNikolay Borisov 			else
278233c66f43SYan Zheng 				err = btrfs_cow_block(trans, root, b,
2783e20d96d6SChris Mason 						      p->nodes[level + 1],
27849fa8cfe7SChris Mason 						      p->slots[level + 1], &b);
278533c66f43SYan Zheng 			if (err) {
278633c66f43SYan Zheng 				ret = err;
278765b51a00SChris Mason 				goto done;
278854aa1f4dSChris Mason 			}
278902217ed2SChris Mason 		}
279065b51a00SChris Mason cow_done:
2791eb60ceacSChris Mason 		p->nodes[level] = b;
2792bd681513SChris Mason 		btrfs_clear_path_blocking(p, NULL, 0);
2793b4ce94deSChris Mason 
2794b4ce94deSChris Mason 		/*
2795b4ce94deSChris Mason 		 * we have a lock on b and as long as we aren't changing
2796b4ce94deSChris Mason 		 * the tree, there is no way to for the items in b to change.
2797b4ce94deSChris Mason 		 * It is safe to drop the lock on our parent before we
2798b4ce94deSChris Mason 		 * go through the expensive btree search on b.
2799b4ce94deSChris Mason 		 *
2800eb653de1SFilipe David Borba Manana 		 * If we're inserting or deleting (ins_len != 0), then we might
2801eb653de1SFilipe David Borba Manana 		 * be changing slot zero, which may require changing the parent.
2802eb653de1SFilipe David Borba Manana 		 * So, we can't drop the lock until after we know which slot
2803eb653de1SFilipe David Borba Manana 		 * we're operating on.
2804b4ce94deSChris Mason 		 */
2805eb653de1SFilipe David Borba Manana 		if (!ins_len && !p->keep_locks) {
2806eb653de1SFilipe David Borba Manana 			int u = level + 1;
2807eb653de1SFilipe David Borba Manana 
2808eb653de1SFilipe David Borba Manana 			if (u < BTRFS_MAX_LEVEL && p->locks[u]) {
2809eb653de1SFilipe David Borba Manana 				btrfs_tree_unlock_rw(p->nodes[u], p->locks[u]);
2810eb653de1SFilipe David Borba Manana 				p->locks[u] = 0;
2811eb653de1SFilipe David Borba Manana 			}
2812eb653de1SFilipe David Borba Manana 		}
2813b4ce94deSChris Mason 
2814d7396f07SFilipe David Borba Manana 		ret = key_search(b, key, level, &prev_cmp, &slot);
2815415b35a5SLiu Bo 		if (ret < 0)
2816415b35a5SLiu Bo 			goto done;
2817b4ce94deSChris Mason 
28185f39d397SChris Mason 		if (level != 0) {
281933c66f43SYan Zheng 			int dec = 0;
282033c66f43SYan Zheng 			if (ret && slot > 0) {
282133c66f43SYan Zheng 				dec = 1;
2822be0e5c09SChris Mason 				slot -= 1;
282333c66f43SYan Zheng 			}
2824be0e5c09SChris Mason 			p->slots[level] = slot;
282533c66f43SYan Zheng 			err = setup_nodes_for_search(trans, root, p, b, level,
2826bd681513SChris Mason 					     ins_len, &write_lock_level);
282733c66f43SYan Zheng 			if (err == -EAGAIN)
2828b4ce94deSChris Mason 				goto again;
282933c66f43SYan Zheng 			if (err) {
283033c66f43SYan Zheng 				ret = err;
283165b51a00SChris Mason 				goto done;
283233c66f43SYan Zheng 			}
28335c680ed6SChris Mason 			b = p->nodes[level];
28345c680ed6SChris Mason 			slot = p->slots[level];
2835b4ce94deSChris Mason 
2836bd681513SChris Mason 			/*
2837bd681513SChris Mason 			 * slot 0 is special, if we change the key
2838bd681513SChris Mason 			 * we have to update the parent pointer
2839bd681513SChris Mason 			 * which means we must have a write lock
2840bd681513SChris Mason 			 * on the parent
2841bd681513SChris Mason 			 */
2842eb653de1SFilipe David Borba Manana 			if (slot == 0 && ins_len &&
2843bd681513SChris Mason 			    write_lock_level < level + 1) {
2844bd681513SChris Mason 				write_lock_level = level + 1;
2845bd681513SChris Mason 				btrfs_release_path(p);
2846bd681513SChris Mason 				goto again;
2847bd681513SChris Mason 			}
2848bd681513SChris Mason 
2849f7c79f30SChris Mason 			unlock_up(p, level, lowest_unlock,
2850f7c79f30SChris Mason 				  min_write_lock_level, &write_lock_level);
2851f9efa9c7SChris Mason 
2852925baeddSChris Mason 			if (level == lowest_level) {
285333c66f43SYan Zheng 				if (dec)
285433c66f43SYan Zheng 					p->slots[level]++;
28555b21f2edSZheng Yan 				goto done;
2856925baeddSChris Mason 			}
2857ca7a79adSChris Mason 
2858d07b8528SLiu Bo 			err = read_block_for_search(root, p, &b, level,
2859cda79c54SDavid Sterba 						    slot, key);
286033c66f43SYan Zheng 			if (err == -EAGAIN)
2861051e1b9fSChris Mason 				goto again;
286233c66f43SYan Zheng 			if (err) {
286333c66f43SYan Zheng 				ret = err;
286476a05b35SChris Mason 				goto done;
286533c66f43SYan Zheng 			}
286676a05b35SChris Mason 
2867b4ce94deSChris Mason 			if (!p->skip_locking) {
2868bd681513SChris Mason 				level = btrfs_header_level(b);
2869bd681513SChris Mason 				if (level <= write_lock_level) {
2870bd681513SChris Mason 					err = btrfs_try_tree_write_lock(b);
287133c66f43SYan Zheng 					if (!err) {
2872b4ce94deSChris Mason 						btrfs_set_path_blocking(p);
2873925baeddSChris Mason 						btrfs_tree_lock(b);
2874bd681513SChris Mason 						btrfs_clear_path_blocking(p, b,
2875bd681513SChris Mason 								  BTRFS_WRITE_LOCK);
2876b4ce94deSChris Mason 					}
2877bd681513SChris Mason 					p->locks[level] = BTRFS_WRITE_LOCK;
2878bd681513SChris Mason 				} else {
2879f82c458aSChris Mason 					err = btrfs_tree_read_lock_atomic(b);
2880bd681513SChris Mason 					if (!err) {
2881bd681513SChris Mason 						btrfs_set_path_blocking(p);
2882bd681513SChris Mason 						btrfs_tree_read_lock(b);
2883bd681513SChris Mason 						btrfs_clear_path_blocking(p, b,
2884bd681513SChris Mason 								  BTRFS_READ_LOCK);
2885bd681513SChris Mason 					}
2886bd681513SChris Mason 					p->locks[level] = BTRFS_READ_LOCK;
2887bd681513SChris Mason 				}
2888bd681513SChris Mason 				p->nodes[level] = b;
2889b4ce94deSChris Mason 			}
2890be0e5c09SChris Mason 		} else {
2891be0e5c09SChris Mason 			p->slots[level] = slot;
289287b29b20SYan Zheng 			if (ins_len > 0 &&
28932ff7e61eSJeff Mahoney 			    btrfs_leaf_free_space(fs_info, b) < ins_len) {
2894bd681513SChris Mason 				if (write_lock_level < 1) {
2895bd681513SChris Mason 					write_lock_level = 1;
2896bd681513SChris Mason 					btrfs_release_path(p);
2897bd681513SChris Mason 					goto again;
2898bd681513SChris Mason 				}
2899bd681513SChris Mason 
2900b4ce94deSChris Mason 				btrfs_set_path_blocking(p);
290133c66f43SYan Zheng 				err = split_leaf(trans, root, key,
2902cc0c5538SChris Mason 						 p, ins_len, ret == 0);
2903bd681513SChris Mason 				btrfs_clear_path_blocking(p, NULL, 0);
2904b4ce94deSChris Mason 
290533c66f43SYan Zheng 				BUG_ON(err > 0);
290633c66f43SYan Zheng 				if (err) {
290733c66f43SYan Zheng 					ret = err;
290865b51a00SChris Mason 					goto done;
290965b51a00SChris Mason 				}
29105c680ed6SChris Mason 			}
2911459931ecSChris Mason 			if (!p->search_for_split)
2912f7c79f30SChris Mason 				unlock_up(p, level, lowest_unlock,
2913f7c79f30SChris Mason 					  min_write_lock_level, &write_lock_level);
291465b51a00SChris Mason 			goto done;
291565b51a00SChris Mason 		}
291665b51a00SChris Mason 	}
291765b51a00SChris Mason 	ret = 1;
291865b51a00SChris Mason done:
2919b4ce94deSChris Mason 	/*
2920b4ce94deSChris Mason 	 * we don't really know what they plan on doing with the path
2921b4ce94deSChris Mason 	 * from here on, so for now just mark it as blocking
2922b4ce94deSChris Mason 	 */
2923b9473439SChris Mason 	if (!p->leave_spinning)
2924b4ce94deSChris Mason 		btrfs_set_path_blocking(p);
29255f5bc6b1SFilipe Manana 	if (ret < 0 && !p->skip_release_on_error)
2926b3b4aa74SDavid Sterba 		btrfs_release_path(p);
2927be0e5c09SChris Mason 	return ret;
2928be0e5c09SChris Mason }
2929be0e5c09SChris Mason 
293074123bd7SChris Mason /*
29315d9e75c4SJan Schmidt  * Like btrfs_search_slot, this looks for a key in the given tree. It uses the
29325d9e75c4SJan Schmidt  * current state of the tree together with the operations recorded in the tree
29335d9e75c4SJan Schmidt  * modification log to search for the key in a previous version of this tree, as
29345d9e75c4SJan Schmidt  * denoted by the time_seq parameter.
29355d9e75c4SJan Schmidt  *
29365d9e75c4SJan Schmidt  * Naturally, there is no support for insert, delete or cow operations.
29375d9e75c4SJan Schmidt  *
29385d9e75c4SJan Schmidt  * The resulting path and return value will be set up as if we called
29395d9e75c4SJan Schmidt  * btrfs_search_slot at that point in time with ins_len and cow both set to 0.
29405d9e75c4SJan Schmidt  */
2941310712b2SOmar Sandoval int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key,
29425d9e75c4SJan Schmidt 			  struct btrfs_path *p, u64 time_seq)
29435d9e75c4SJan Schmidt {
29440b246afaSJeff Mahoney 	struct btrfs_fs_info *fs_info = root->fs_info;
29455d9e75c4SJan Schmidt 	struct extent_buffer *b;
29465d9e75c4SJan Schmidt 	int slot;
29475d9e75c4SJan Schmidt 	int ret;
29485d9e75c4SJan Schmidt 	int err;
29495d9e75c4SJan Schmidt 	int level;
29505d9e75c4SJan Schmidt 	int lowest_unlock = 1;
29515d9e75c4SJan Schmidt 	u8 lowest_level = 0;
2952d4b4087cSJosef Bacik 	int prev_cmp = -1;
29535d9e75c4SJan Schmidt 
29545d9e75c4SJan Schmidt 	lowest_level = p->lowest_level;
29555d9e75c4SJan Schmidt 	WARN_ON(p->nodes[0] != NULL);
29565d9e75c4SJan Schmidt 
29575d9e75c4SJan Schmidt 	if (p->search_commit_root) {
29585d9e75c4SJan Schmidt 		BUG_ON(time_seq);
29595d9e75c4SJan Schmidt 		return btrfs_search_slot(NULL, root, key, p, 0, 0);
29605d9e75c4SJan Schmidt 	}
29615d9e75c4SJan Schmidt 
29625d9e75c4SJan Schmidt again:
29635d9e75c4SJan Schmidt 	b = get_old_root(root, time_seq);
29645d9e75c4SJan Schmidt 	level = btrfs_header_level(b);
29655d9e75c4SJan Schmidt 	p->locks[level] = BTRFS_READ_LOCK;
29665d9e75c4SJan Schmidt 
29675d9e75c4SJan Schmidt 	while (b) {
29685d9e75c4SJan Schmidt 		level = btrfs_header_level(b);
29695d9e75c4SJan Schmidt 		p->nodes[level] = b;
29705d9e75c4SJan Schmidt 		btrfs_clear_path_blocking(p, NULL, 0);
29715d9e75c4SJan Schmidt 
29725d9e75c4SJan Schmidt 		/*
29735d9e75c4SJan Schmidt 		 * we have a lock on b and as long as we aren't changing
29745d9e75c4SJan Schmidt 		 * the tree, there is no way to for the items in b to change.
29755d9e75c4SJan Schmidt 		 * It is safe to drop the lock on our parent before we
29765d9e75c4SJan Schmidt 		 * go through the expensive btree search on b.
29775d9e75c4SJan Schmidt 		 */
29785d9e75c4SJan Schmidt 		btrfs_unlock_up_safe(p, level + 1);
29795d9e75c4SJan Schmidt 
2980d4b4087cSJosef Bacik 		/*
298101327610SNicholas D Steeves 		 * Since we can unwind ebs we want to do a real search every
2982d4b4087cSJosef Bacik 		 * time.
2983d4b4087cSJosef Bacik 		 */
2984d4b4087cSJosef Bacik 		prev_cmp = -1;
2985d7396f07SFilipe David Borba Manana 		ret = key_search(b, key, level, &prev_cmp, &slot);
29865d9e75c4SJan Schmidt 
29875d9e75c4SJan Schmidt 		if (level != 0) {
29885d9e75c4SJan Schmidt 			int dec = 0;
29895d9e75c4SJan Schmidt 			if (ret && slot > 0) {
29905d9e75c4SJan Schmidt 				dec = 1;
29915d9e75c4SJan Schmidt 				slot -= 1;
29925d9e75c4SJan Schmidt 			}
29935d9e75c4SJan Schmidt 			p->slots[level] = slot;
29945d9e75c4SJan Schmidt 			unlock_up(p, level, lowest_unlock, 0, NULL);
29955d9e75c4SJan Schmidt 
29965d9e75c4SJan Schmidt 			if (level == lowest_level) {
29975d9e75c4SJan Schmidt 				if (dec)
29985d9e75c4SJan Schmidt 					p->slots[level]++;
29995d9e75c4SJan Schmidt 				goto done;
30005d9e75c4SJan Schmidt 			}
30015d9e75c4SJan Schmidt 
3002d07b8528SLiu Bo 			err = read_block_for_search(root, p, &b, level,
3003cda79c54SDavid Sterba 						    slot, key);
30045d9e75c4SJan Schmidt 			if (err == -EAGAIN)
30055d9e75c4SJan Schmidt 				goto again;
30065d9e75c4SJan Schmidt 			if (err) {
30075d9e75c4SJan Schmidt 				ret = err;
30085d9e75c4SJan Schmidt 				goto done;
30095d9e75c4SJan Schmidt 			}
30105d9e75c4SJan Schmidt 
30115d9e75c4SJan Schmidt 			level = btrfs_header_level(b);
3012f82c458aSChris Mason 			err = btrfs_tree_read_lock_atomic(b);
30135d9e75c4SJan Schmidt 			if (!err) {
30145d9e75c4SJan Schmidt 				btrfs_set_path_blocking(p);
30155d9e75c4SJan Schmidt 				btrfs_tree_read_lock(b);
30165d9e75c4SJan Schmidt 				btrfs_clear_path_blocking(p, b,
30175d9e75c4SJan Schmidt 							  BTRFS_READ_LOCK);
30185d9e75c4SJan Schmidt 			}
30190b246afaSJeff Mahoney 			b = tree_mod_log_rewind(fs_info, p, b, time_seq);
3020db7f3436SJosef Bacik 			if (!b) {
3021db7f3436SJosef Bacik 				ret = -ENOMEM;
3022db7f3436SJosef Bacik 				goto done;
3023db7f3436SJosef Bacik 			}
30245d9e75c4SJan Schmidt 			p->locks[level] = BTRFS_READ_LOCK;
30255d9e75c4SJan Schmidt 			p->nodes[level] = b;
30265d9e75c4SJan Schmidt 		} else {
30275d9e75c4SJan Schmidt 			p->slots[level] = slot;
30285d9e75c4SJan Schmidt 			unlock_up(p, level, lowest_unlock, 0, NULL);
30295d9e75c4SJan Schmidt 			goto done;
30305d9e75c4SJan Schmidt 		}
30315d9e75c4SJan Schmidt 	}
30325d9e75c4SJan Schmidt 	ret = 1;
30335d9e75c4SJan Schmidt done:
30345d9e75c4SJan Schmidt 	if (!p->leave_spinning)
30355d9e75c4SJan Schmidt 		btrfs_set_path_blocking(p);
30365d9e75c4SJan Schmidt 	if (ret < 0)
30375d9e75c4SJan Schmidt 		btrfs_release_path(p);
30385d9e75c4SJan Schmidt 
30395d9e75c4SJan Schmidt 	return ret;
30405d9e75c4SJan Schmidt }
30415d9e75c4SJan Schmidt 
30425d9e75c4SJan Schmidt /*
30432f38b3e1SArne Jansen  * helper to use instead of search slot if no exact match is needed but
30442f38b3e1SArne Jansen  * instead the next or previous item should be returned.
30452f38b3e1SArne Jansen  * When find_higher is true, the next higher item is returned, the next lower
30462f38b3e1SArne Jansen  * otherwise.
30472f38b3e1SArne Jansen  * When return_any and find_higher are both true, and no higher item is found,
30482f38b3e1SArne Jansen  * return the next lower instead.
30492f38b3e1SArne Jansen  * When return_any is true and find_higher is false, and no lower item is found,
30502f38b3e1SArne Jansen  * return the next higher instead.
30512f38b3e1SArne Jansen  * It returns 0 if any item is found, 1 if none is found (tree empty), and
30522f38b3e1SArne Jansen  * < 0 on error
30532f38b3e1SArne Jansen  */
30542f38b3e1SArne Jansen int btrfs_search_slot_for_read(struct btrfs_root *root,
3055310712b2SOmar Sandoval 			       const struct btrfs_key *key,
3056310712b2SOmar Sandoval 			       struct btrfs_path *p, int find_higher,
3057310712b2SOmar Sandoval 			       int return_any)
30582f38b3e1SArne Jansen {
30592f38b3e1SArne Jansen 	int ret;
30602f38b3e1SArne Jansen 	struct extent_buffer *leaf;
30612f38b3e1SArne Jansen 
30622f38b3e1SArne Jansen again:
30632f38b3e1SArne Jansen 	ret = btrfs_search_slot(NULL, root, key, p, 0, 0);
30642f38b3e1SArne Jansen 	if (ret <= 0)
30652f38b3e1SArne Jansen 		return ret;
30662f38b3e1SArne Jansen 	/*
30672f38b3e1SArne Jansen 	 * a return value of 1 means the path is at the position where the
30682f38b3e1SArne Jansen 	 * item should be inserted. Normally this is the next bigger item,
30692f38b3e1SArne Jansen 	 * but in case the previous item is the last in a leaf, path points
30702f38b3e1SArne Jansen 	 * to the first free slot in the previous leaf, i.e. at an invalid
30712f38b3e1SArne Jansen 	 * item.
30722f38b3e1SArne Jansen 	 */
30732f38b3e1SArne Jansen 	leaf = p->nodes[0];
30742f38b3e1SArne Jansen 
30752f38b3e1SArne Jansen 	if (find_higher) {
30762f38b3e1SArne Jansen 		if (p->slots[0] >= btrfs_header_nritems(leaf)) {
30772f38b3e1SArne Jansen 			ret = btrfs_next_leaf(root, p);
30782f38b3e1SArne Jansen 			if (ret <= 0)
30792f38b3e1SArne Jansen 				return ret;
30802f38b3e1SArne Jansen 			if (!return_any)
30812f38b3e1SArne Jansen 				return 1;
30822f38b3e1SArne Jansen 			/*
30832f38b3e1SArne Jansen 			 * no higher item found, return the next
30842f38b3e1SArne Jansen 			 * lower instead
30852f38b3e1SArne Jansen 			 */
30862f38b3e1SArne Jansen 			return_any = 0;
30872f38b3e1SArne Jansen 			find_higher = 0;
30882f38b3e1SArne Jansen 			btrfs_release_path(p);
30892f38b3e1SArne Jansen 			goto again;
30902f38b3e1SArne Jansen 		}
30912f38b3e1SArne Jansen 	} else {
30922f38b3e1SArne Jansen 		if (p->slots[0] == 0) {
30932f38b3e1SArne Jansen 			ret = btrfs_prev_leaf(root, p);
3094e6793769SArne Jansen 			if (ret < 0)
30952f38b3e1SArne Jansen 				return ret;
3096e6793769SArne Jansen 			if (!ret) {
309723c6bf6aSFilipe David Borba Manana 				leaf = p->nodes[0];
309823c6bf6aSFilipe David Borba Manana 				if (p->slots[0] == btrfs_header_nritems(leaf))
309923c6bf6aSFilipe David Borba Manana 					p->slots[0]--;
3100e6793769SArne Jansen 				return 0;
3101e6793769SArne Jansen 			}
31022f38b3e1SArne Jansen 			if (!return_any)
31032f38b3e1SArne Jansen 				return 1;
31042f38b3e1SArne Jansen 			/*
31052f38b3e1SArne Jansen 			 * no lower item found, return the next
31062f38b3e1SArne Jansen 			 * higher instead
31072f38b3e1SArne Jansen 			 */
31082f38b3e1SArne Jansen 			return_any = 0;
31092f38b3e1SArne Jansen 			find_higher = 1;
31102f38b3e1SArne Jansen 			btrfs_release_path(p);
31112f38b3e1SArne Jansen 			goto again;
3112e6793769SArne Jansen 		} else {
31132f38b3e1SArne Jansen 			--p->slots[0];
31142f38b3e1SArne Jansen 		}
31152f38b3e1SArne Jansen 	}
31162f38b3e1SArne Jansen 	return 0;
31172f38b3e1SArne Jansen }
31182f38b3e1SArne Jansen 
31192f38b3e1SArne Jansen /*
312074123bd7SChris Mason  * adjust the pointers going up the tree, starting at level
312174123bd7SChris Mason  * making sure the right key of each node is points to 'key'.
312274123bd7SChris Mason  * This is used after shifting pointers to the left, so it stops
312374123bd7SChris Mason  * fixing up pointers when a given leaf/node is not in slot 0 of the
312474123bd7SChris Mason  * higher levels
3125aa5d6bedSChris Mason  *
312674123bd7SChris Mason  */
3127b167fa91SNikolay Borisov static void fixup_low_keys(struct btrfs_path *path,
31285f39d397SChris Mason 			   struct btrfs_disk_key *key, int level)
3129be0e5c09SChris Mason {
3130be0e5c09SChris Mason 	int i;
31315f39d397SChris Mason 	struct extent_buffer *t;
31320e82bcfeSDavid Sterba 	int ret;
31335f39d397SChris Mason 
3134234b63a0SChris Mason 	for (i = level; i < BTRFS_MAX_LEVEL; i++) {
3135be0e5c09SChris Mason 		int tslot = path->slots[i];
31360e82bcfeSDavid Sterba 
3137eb60ceacSChris Mason 		if (!path->nodes[i])
3138be0e5c09SChris Mason 			break;
31395f39d397SChris Mason 		t = path->nodes[i];
31400e82bcfeSDavid Sterba 		ret = tree_mod_log_insert_key(t, tslot, MOD_LOG_KEY_REPLACE,
31410e82bcfeSDavid Sterba 				GFP_ATOMIC);
31420e82bcfeSDavid Sterba 		BUG_ON(ret < 0);
31435f39d397SChris Mason 		btrfs_set_node_key(t, key, tslot);
3144d6025579SChris Mason 		btrfs_mark_buffer_dirty(path->nodes[i]);
3145be0e5c09SChris Mason 		if (tslot != 0)
3146be0e5c09SChris Mason 			break;
3147be0e5c09SChris Mason 	}
3148be0e5c09SChris Mason }
3149be0e5c09SChris Mason 
315074123bd7SChris Mason /*
315131840ae1SZheng Yan  * update item key.
315231840ae1SZheng Yan  *
315331840ae1SZheng Yan  * This function isn't completely safe. It's the caller's responsibility
315431840ae1SZheng Yan  * that the new key won't break the order
315531840ae1SZheng Yan  */
3156b7a0365eSDaniel Dressler void btrfs_set_item_key_safe(struct btrfs_fs_info *fs_info,
3157b7a0365eSDaniel Dressler 			     struct btrfs_path *path,
3158310712b2SOmar Sandoval 			     const struct btrfs_key *new_key)
315931840ae1SZheng Yan {
316031840ae1SZheng Yan 	struct btrfs_disk_key disk_key;
316131840ae1SZheng Yan 	struct extent_buffer *eb;
316231840ae1SZheng Yan 	int slot;
316331840ae1SZheng Yan 
316431840ae1SZheng Yan 	eb = path->nodes[0];
316531840ae1SZheng Yan 	slot = path->slots[0];
316631840ae1SZheng Yan 	if (slot > 0) {
316731840ae1SZheng Yan 		btrfs_item_key(eb, &disk_key, slot - 1);
3168143bede5SJeff Mahoney 		BUG_ON(comp_keys(&disk_key, new_key) >= 0);
316931840ae1SZheng Yan 	}
317031840ae1SZheng Yan 	if (slot < btrfs_header_nritems(eb) - 1) {
317131840ae1SZheng Yan 		btrfs_item_key(eb, &disk_key, slot + 1);
3172143bede5SJeff Mahoney 		BUG_ON(comp_keys(&disk_key, new_key) <= 0);
317331840ae1SZheng Yan 	}
317431840ae1SZheng Yan 
317531840ae1SZheng Yan 	btrfs_cpu_key_to_disk(&disk_key, new_key);
317631840ae1SZheng Yan 	btrfs_set_item_key(eb, &disk_key, slot);
317731840ae1SZheng Yan 	btrfs_mark_buffer_dirty(eb);
317831840ae1SZheng Yan 	if (slot == 0)
3179b167fa91SNikolay Borisov 		fixup_low_keys(path, &disk_key, 1);
318031840ae1SZheng Yan }
318131840ae1SZheng Yan 
318231840ae1SZheng Yan /*
318374123bd7SChris Mason  * try to push data from one node into the next node left in the
318479f95c82SChris Mason  * tree.
3185aa5d6bedSChris Mason  *
3186aa5d6bedSChris Mason  * returns 0 if some ptrs were pushed left, < 0 if there was some horrible
3187aa5d6bedSChris Mason  * error, and > 0 if there was no room in the left hand block.
318874123bd7SChris Mason  */
318998ed5174SChris Mason static int push_node_left(struct btrfs_trans_handle *trans,
31902ff7e61eSJeff Mahoney 			  struct btrfs_fs_info *fs_info,
31912ff7e61eSJeff Mahoney 			  struct extent_buffer *dst,
3192971a1f66SChris Mason 			  struct extent_buffer *src, int empty)
3193be0e5c09SChris Mason {
3194be0e5c09SChris Mason 	int push_items = 0;
3195bb803951SChris Mason 	int src_nritems;
3196bb803951SChris Mason 	int dst_nritems;
3197aa5d6bedSChris Mason 	int ret = 0;
3198be0e5c09SChris Mason 
31995f39d397SChris Mason 	src_nritems = btrfs_header_nritems(src);
32005f39d397SChris Mason 	dst_nritems = btrfs_header_nritems(dst);
32010b246afaSJeff Mahoney 	push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems;
32027bb86316SChris Mason 	WARN_ON(btrfs_header_generation(src) != trans->transid);
32037bb86316SChris Mason 	WARN_ON(btrfs_header_generation(dst) != trans->transid);
320454aa1f4dSChris Mason 
3205bce4eae9SChris Mason 	if (!empty && src_nritems <= 8)
3206971a1f66SChris Mason 		return 1;
3207971a1f66SChris Mason 
3208d397712bSChris Mason 	if (push_items <= 0)
3209be0e5c09SChris Mason 		return 1;
3210be0e5c09SChris Mason 
3211bce4eae9SChris Mason 	if (empty) {
3212971a1f66SChris Mason 		push_items = min(src_nritems, push_items);
3213bce4eae9SChris Mason 		if (push_items < src_nritems) {
3214bce4eae9SChris Mason 			/* leave at least 8 pointers in the node if
3215bce4eae9SChris Mason 			 * we aren't going to empty it
3216bce4eae9SChris Mason 			 */
3217bce4eae9SChris Mason 			if (src_nritems - push_items < 8) {
3218bce4eae9SChris Mason 				if (push_items <= 8)
3219bce4eae9SChris Mason 					return 1;
3220bce4eae9SChris Mason 				push_items -= 8;
3221bce4eae9SChris Mason 			}
3222bce4eae9SChris Mason 		}
3223bce4eae9SChris Mason 	} else
3224bce4eae9SChris Mason 		push_items = min(src_nritems - 8, push_items);
322579f95c82SChris Mason 
32260b246afaSJeff Mahoney 	ret = tree_mod_log_eb_copy(fs_info, dst, src, dst_nritems, 0,
322790f8d62eSJan Schmidt 				   push_items);
32285de865eeSFilipe David Borba Manana 	if (ret) {
322966642832SJeff Mahoney 		btrfs_abort_transaction(trans, ret);
32305de865eeSFilipe David Borba Manana 		return ret;
32315de865eeSFilipe David Borba Manana 	}
32325f39d397SChris Mason 	copy_extent_buffer(dst, src,
32335f39d397SChris Mason 			   btrfs_node_key_ptr_offset(dst_nritems),
32345f39d397SChris Mason 			   btrfs_node_key_ptr_offset(0),
3235123abc88SChris Mason 			   push_items * sizeof(struct btrfs_key_ptr));
32365f39d397SChris Mason 
3237bb803951SChris Mason 	if (push_items < src_nritems) {
323857911b8bSJan Schmidt 		/*
3239bf1d3425SDavid Sterba 		 * Don't call tree_mod_log_insert_move here, key removal was
3240bf1d3425SDavid Sterba 		 * already fully logged by tree_mod_log_eb_copy above.
324157911b8bSJan Schmidt 		 */
32425f39d397SChris Mason 		memmove_extent_buffer(src, btrfs_node_key_ptr_offset(0),
32435f39d397SChris Mason 				      btrfs_node_key_ptr_offset(push_items),
3244e2fa7227SChris Mason 				      (src_nritems - push_items) *
3245123abc88SChris Mason 				      sizeof(struct btrfs_key_ptr));
3246bb803951SChris Mason 	}
32475f39d397SChris Mason 	btrfs_set_header_nritems(src, src_nritems - push_items);
32485f39d397SChris Mason 	btrfs_set_header_nritems(dst, dst_nritems + push_items);
32495f39d397SChris Mason 	btrfs_mark_buffer_dirty(src);
32505f39d397SChris Mason 	btrfs_mark_buffer_dirty(dst);
325131840ae1SZheng Yan 
3252bb803951SChris Mason 	return ret;
3253be0e5c09SChris Mason }
3254be0e5c09SChris Mason 
325597571fd0SChris Mason /*
325679f95c82SChris Mason  * try to push data from one node into the next node right in the
325779f95c82SChris Mason  * tree.
325879f95c82SChris Mason  *
325979f95c82SChris Mason  * returns 0 if some ptrs were pushed, < 0 if there was some horrible
326079f95c82SChris Mason  * error, and > 0 if there was no room in the right hand block.
326179f95c82SChris Mason  *
326279f95c82SChris Mason  * this will  only push up to 1/2 the contents of the left node over
326379f95c82SChris Mason  */
32645f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans,
32652ff7e61eSJeff Mahoney 			      struct btrfs_fs_info *fs_info,
32665f39d397SChris Mason 			      struct extent_buffer *dst,
32675f39d397SChris Mason 			      struct extent_buffer *src)
326879f95c82SChris Mason {
326979f95c82SChris Mason 	int push_items = 0;
327079f95c82SChris Mason 	int max_push;
327179f95c82SChris Mason 	int src_nritems;
327279f95c82SChris Mason 	int dst_nritems;
327379f95c82SChris Mason 	int ret = 0;
327479f95c82SChris Mason 
32757bb86316SChris Mason 	WARN_ON(btrfs_header_generation(src) != trans->transid);
32767bb86316SChris Mason 	WARN_ON(btrfs_header_generation(dst) != trans->transid);
32777bb86316SChris Mason 
32785f39d397SChris Mason 	src_nritems = btrfs_header_nritems(src);
32795f39d397SChris Mason 	dst_nritems = btrfs_header_nritems(dst);
32800b246afaSJeff Mahoney 	push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems;
3281d397712bSChris Mason 	if (push_items <= 0)
328279f95c82SChris Mason 		return 1;
3283bce4eae9SChris Mason 
3284d397712bSChris Mason 	if (src_nritems < 4)
3285bce4eae9SChris Mason 		return 1;
328679f95c82SChris Mason 
328779f95c82SChris Mason 	max_push = src_nritems / 2 + 1;
328879f95c82SChris Mason 	/* don't try to empty the node */
3289d397712bSChris Mason 	if (max_push >= src_nritems)
329079f95c82SChris Mason 		return 1;
3291252c38f0SYan 
329279f95c82SChris Mason 	if (max_push < push_items)
329379f95c82SChris Mason 		push_items = max_push;
329479f95c82SChris Mason 
3295bf1d3425SDavid Sterba 	ret = tree_mod_log_insert_move(dst, push_items, 0, dst_nritems);
3296bf1d3425SDavid Sterba 	BUG_ON(ret < 0);
32975f39d397SChris Mason 	memmove_extent_buffer(dst, btrfs_node_key_ptr_offset(push_items),
32985f39d397SChris Mason 				      btrfs_node_key_ptr_offset(0),
32995f39d397SChris Mason 				      (dst_nritems) *
33005f39d397SChris Mason 				      sizeof(struct btrfs_key_ptr));
3301d6025579SChris Mason 
33020b246afaSJeff Mahoney 	ret = tree_mod_log_eb_copy(fs_info, dst, src, 0,
330390f8d62eSJan Schmidt 				   src_nritems - push_items, push_items);
33045de865eeSFilipe David Borba Manana 	if (ret) {
330566642832SJeff Mahoney 		btrfs_abort_transaction(trans, ret);
33065de865eeSFilipe David Borba Manana 		return ret;
33075de865eeSFilipe David Borba Manana 	}
33085f39d397SChris Mason 	copy_extent_buffer(dst, src,
33095f39d397SChris Mason 			   btrfs_node_key_ptr_offset(0),
33105f39d397SChris Mason 			   btrfs_node_key_ptr_offset(src_nritems - push_items),
3311123abc88SChris Mason 			   push_items * sizeof(struct btrfs_key_ptr));
331279f95c82SChris Mason 
33135f39d397SChris Mason 	btrfs_set_header_nritems(src, src_nritems - push_items);
33145f39d397SChris Mason 	btrfs_set_header_nritems(dst, dst_nritems + push_items);
331579f95c82SChris Mason 
33165f39d397SChris Mason 	btrfs_mark_buffer_dirty(src);
33175f39d397SChris Mason 	btrfs_mark_buffer_dirty(dst);
331831840ae1SZheng Yan 
331979f95c82SChris Mason 	return ret;
332079f95c82SChris Mason }
332179f95c82SChris Mason 
332279f95c82SChris Mason /*
332397571fd0SChris Mason  * helper function to insert a new root level in the tree.
332497571fd0SChris Mason  * A new node is allocated, and a single item is inserted to
332597571fd0SChris Mason  * point to the existing root
3326aa5d6bedSChris Mason  *
3327aa5d6bedSChris Mason  * returns zero on success or < 0 on failure.
332897571fd0SChris Mason  */
3329d397712bSChris Mason static noinline int insert_new_root(struct btrfs_trans_handle *trans,
33305f39d397SChris Mason 			   struct btrfs_root *root,
3331fdd99c72SLiu Bo 			   struct btrfs_path *path, int level)
333274123bd7SChris Mason {
33330b246afaSJeff Mahoney 	struct btrfs_fs_info *fs_info = root->fs_info;
33347bb86316SChris Mason 	u64 lower_gen;
33355f39d397SChris Mason 	struct extent_buffer *lower;
33365f39d397SChris Mason 	struct extent_buffer *c;
3337925baeddSChris Mason 	struct extent_buffer *old;
33385f39d397SChris Mason 	struct btrfs_disk_key lower_key;
3339d9d19a01SDavid Sterba 	int ret;
33405c680ed6SChris Mason 
33415c680ed6SChris Mason 	BUG_ON(path->nodes[level]);
33425c680ed6SChris Mason 	BUG_ON(path->nodes[level-1] != root->node);
33435c680ed6SChris Mason 
33447bb86316SChris Mason 	lower = path->nodes[level-1];
33457bb86316SChris Mason 	if (level == 1)
33467bb86316SChris Mason 		btrfs_item_key(lower, &lower_key, 0);
33477bb86316SChris Mason 	else
33487bb86316SChris Mason 		btrfs_node_key(lower, &lower_key, 0);
33497bb86316SChris Mason 
33504d75f8a9SDavid Sterba 	c = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid,
33514d75f8a9SDavid Sterba 				   &lower_key, level, root->node->start, 0);
33525f39d397SChris Mason 	if (IS_ERR(c))
33535f39d397SChris Mason 		return PTR_ERR(c);
3354925baeddSChris Mason 
33550b246afaSJeff Mahoney 	root_add_used(root, fs_info->nodesize);
3356f0486c68SYan, Zheng 
33575f39d397SChris Mason 	btrfs_set_header_nritems(c, 1);
33585f39d397SChris Mason 	btrfs_set_node_key(c, &lower_key, 0);
3359db94535dSChris Mason 	btrfs_set_node_blockptr(c, 0, lower->start);
33607bb86316SChris Mason 	lower_gen = btrfs_header_generation(lower);
336131840ae1SZheng Yan 	WARN_ON(lower_gen != trans->transid);
33627bb86316SChris Mason 
33637bb86316SChris Mason 	btrfs_set_node_ptr_generation(c, 0, lower_gen);
33645f39d397SChris Mason 
33655f39d397SChris Mason 	btrfs_mark_buffer_dirty(c);
3366d5719762SChris Mason 
3367925baeddSChris Mason 	old = root->node;
3368d9d19a01SDavid Sterba 	ret = tree_mod_log_insert_root(root->node, c, 0);
3369d9d19a01SDavid Sterba 	BUG_ON(ret < 0);
3370240f62c8SChris Mason 	rcu_assign_pointer(root->node, c);
3371925baeddSChris Mason 
3372925baeddSChris Mason 	/* the super has an extra ref to root->node */
3373925baeddSChris Mason 	free_extent_buffer(old);
3374925baeddSChris Mason 
33750b86a832SChris Mason 	add_root_to_dirty_list(root);
33765f39d397SChris Mason 	extent_buffer_get(c);
33775f39d397SChris Mason 	path->nodes[level] = c;
337895449a16Schandan 	path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
337974123bd7SChris Mason 	path->slots[level] = 0;
338074123bd7SChris Mason 	return 0;
338174123bd7SChris Mason }
33825c680ed6SChris Mason 
33835c680ed6SChris Mason /*
33845c680ed6SChris Mason  * worker function to insert a single pointer in a node.
33855c680ed6SChris Mason  * the node should have enough room for the pointer already
338697571fd0SChris Mason  *
33875c680ed6SChris Mason  * slot and level indicate where you want the key to go, and
33885c680ed6SChris Mason  * blocknr is the block the key points to.
33895c680ed6SChris Mason  */
3390143bede5SJeff Mahoney static void insert_ptr(struct btrfs_trans_handle *trans,
33912ff7e61eSJeff Mahoney 		       struct btrfs_fs_info *fs_info, struct btrfs_path *path,
3392143bede5SJeff Mahoney 		       struct btrfs_disk_key *key, u64 bytenr,
3393c3e06965SJan Schmidt 		       int slot, int level)
33945c680ed6SChris Mason {
33955f39d397SChris Mason 	struct extent_buffer *lower;
33965c680ed6SChris Mason 	int nritems;
3397f3ea38daSJan Schmidt 	int ret;
33985c680ed6SChris Mason 
33995c680ed6SChris Mason 	BUG_ON(!path->nodes[level]);
3400f0486c68SYan, Zheng 	btrfs_assert_tree_locked(path->nodes[level]);
34015f39d397SChris Mason 	lower = path->nodes[level];
34025f39d397SChris Mason 	nritems = btrfs_header_nritems(lower);
3403c293498bSStoyan Gaydarov 	BUG_ON(slot > nritems);
34040b246afaSJeff Mahoney 	BUG_ON(nritems == BTRFS_NODEPTRS_PER_BLOCK(fs_info));
340574123bd7SChris Mason 	if (slot != nritems) {
3406bf1d3425SDavid Sterba 		if (level) {
3407bf1d3425SDavid Sterba 			ret = tree_mod_log_insert_move(lower, slot + 1, slot,
3408a446a979SDavid Sterba 					nritems - slot);
3409bf1d3425SDavid Sterba 			BUG_ON(ret < 0);
3410bf1d3425SDavid Sterba 		}
34115f39d397SChris Mason 		memmove_extent_buffer(lower,
34125f39d397SChris Mason 			      btrfs_node_key_ptr_offset(slot + 1),
34135f39d397SChris Mason 			      btrfs_node_key_ptr_offset(slot),
3414123abc88SChris Mason 			      (nritems - slot) * sizeof(struct btrfs_key_ptr));
341574123bd7SChris Mason 	}
3416c3e06965SJan Schmidt 	if (level) {
3417e09c2efeSDavid Sterba 		ret = tree_mod_log_insert_key(lower, slot, MOD_LOG_KEY_ADD,
3418e09c2efeSDavid Sterba 				GFP_NOFS);
3419f3ea38daSJan Schmidt 		BUG_ON(ret < 0);
3420f3ea38daSJan Schmidt 	}
34215f39d397SChris Mason 	btrfs_set_node_key(lower, key, slot);
3422db94535dSChris Mason 	btrfs_set_node_blockptr(lower, slot, bytenr);
342374493f7aSChris Mason 	WARN_ON(trans->transid == 0);
342474493f7aSChris Mason 	btrfs_set_node_ptr_generation(lower, slot, trans->transid);
34255f39d397SChris Mason 	btrfs_set_header_nritems(lower, nritems + 1);
34265f39d397SChris Mason 	btrfs_mark_buffer_dirty(lower);
342774123bd7SChris Mason }
342874123bd7SChris Mason 
342997571fd0SChris Mason /*
343097571fd0SChris Mason  * split the node at the specified level in path in two.
343197571fd0SChris Mason  * The path is corrected to point to the appropriate node after the split
343297571fd0SChris Mason  *
343397571fd0SChris Mason  * Before splitting this tries to make some room in the node by pushing
343497571fd0SChris Mason  * left and right, if either one works, it returns right away.
3435aa5d6bedSChris Mason  *
3436aa5d6bedSChris Mason  * returns 0 on success and < 0 on failure
343797571fd0SChris Mason  */
3438e02119d5SChris Mason static noinline int split_node(struct btrfs_trans_handle *trans,
3439e02119d5SChris Mason 			       struct btrfs_root *root,
3440e02119d5SChris Mason 			       struct btrfs_path *path, int level)
3441be0e5c09SChris Mason {
34420b246afaSJeff Mahoney 	struct btrfs_fs_info *fs_info = root->fs_info;
34435f39d397SChris Mason 	struct extent_buffer *c;
34445f39d397SChris Mason 	struct extent_buffer *split;
34455f39d397SChris Mason 	struct btrfs_disk_key disk_key;
3446be0e5c09SChris Mason 	int mid;
34475c680ed6SChris Mason 	int ret;
34487518a238SChris Mason 	u32 c_nritems;
3449be0e5c09SChris Mason 
34505f39d397SChris Mason 	c = path->nodes[level];
34517bb86316SChris Mason 	WARN_ON(btrfs_header_generation(c) != trans->transid);
34525f39d397SChris Mason 	if (c == root->node) {
3453d9abbf1cSJan Schmidt 		/*
345490f8d62eSJan Schmidt 		 * trying to split the root, lets make a new one
345590f8d62eSJan Schmidt 		 *
3456fdd99c72SLiu Bo 		 * tree mod log: We don't log_removal old root in
345790f8d62eSJan Schmidt 		 * insert_new_root, because that root buffer will be kept as a
345890f8d62eSJan Schmidt 		 * normal node. We are going to log removal of half of the
345990f8d62eSJan Schmidt 		 * elements below with tree_mod_log_eb_copy. We're holding a
346090f8d62eSJan Schmidt 		 * tree lock on the buffer, which is why we cannot race with
346190f8d62eSJan Schmidt 		 * other tree_mod_log users.
3462d9abbf1cSJan Schmidt 		 */
3463fdd99c72SLiu Bo 		ret = insert_new_root(trans, root, path, level + 1);
34645c680ed6SChris Mason 		if (ret)
34655c680ed6SChris Mason 			return ret;
3466b3612421SChris Mason 	} else {
3467e66f709bSChris Mason 		ret = push_nodes_for_insert(trans, root, path, level);
34685f39d397SChris Mason 		c = path->nodes[level];
34695f39d397SChris Mason 		if (!ret && btrfs_header_nritems(c) <
34700b246afaSJeff Mahoney 		    BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3)
3471e66f709bSChris Mason 			return 0;
347254aa1f4dSChris Mason 		if (ret < 0)
347354aa1f4dSChris Mason 			return ret;
34745c680ed6SChris Mason 	}
3475e66f709bSChris Mason 
34765f39d397SChris Mason 	c_nritems = btrfs_header_nritems(c);
34775d4f98a2SYan Zheng 	mid = (c_nritems + 1) / 2;
34785d4f98a2SYan Zheng 	btrfs_node_key(c, &disk_key, mid);
34797bb86316SChris Mason 
34804d75f8a9SDavid Sterba 	split = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid,
34815581a51aSJan Schmidt 			&disk_key, level, c->start, 0);
34825f39d397SChris Mason 	if (IS_ERR(split))
34835f39d397SChris Mason 		return PTR_ERR(split);
348454aa1f4dSChris Mason 
34850b246afaSJeff Mahoney 	root_add_used(root, fs_info->nodesize);
3486bc877d28SNikolay Borisov 	ASSERT(btrfs_header_level(c) == level);
34875f39d397SChris Mason 
34880b246afaSJeff Mahoney 	ret = tree_mod_log_eb_copy(fs_info, split, c, 0, mid, c_nritems - mid);
34895de865eeSFilipe David Borba Manana 	if (ret) {
349066642832SJeff Mahoney 		btrfs_abort_transaction(trans, ret);
34915de865eeSFilipe David Borba Manana 		return ret;
34925de865eeSFilipe David Borba Manana 	}
34935f39d397SChris Mason 	copy_extent_buffer(split, c,
34945f39d397SChris Mason 			   btrfs_node_key_ptr_offset(0),
34955f39d397SChris Mason 			   btrfs_node_key_ptr_offset(mid),
3496123abc88SChris Mason 			   (c_nritems - mid) * sizeof(struct btrfs_key_ptr));
34975f39d397SChris Mason 	btrfs_set_header_nritems(split, c_nritems - mid);
34985f39d397SChris Mason 	btrfs_set_header_nritems(c, mid);
3499aa5d6bedSChris Mason 	ret = 0;
3500aa5d6bedSChris Mason 
35015f39d397SChris Mason 	btrfs_mark_buffer_dirty(c);
35025f39d397SChris Mason 	btrfs_mark_buffer_dirty(split);
35035f39d397SChris Mason 
35042ff7e61eSJeff Mahoney 	insert_ptr(trans, fs_info, path, &disk_key, split->start,
3505c3e06965SJan Schmidt 		   path->slots[level + 1] + 1, level + 1);
3506aa5d6bedSChris Mason 
35075de08d7dSChris Mason 	if (path->slots[level] >= mid) {
35085c680ed6SChris Mason 		path->slots[level] -= mid;
3509925baeddSChris Mason 		btrfs_tree_unlock(c);
35105f39d397SChris Mason 		free_extent_buffer(c);
35115f39d397SChris Mason 		path->nodes[level] = split;
35125c680ed6SChris Mason 		path->slots[level + 1] += 1;
3513eb60ceacSChris Mason 	} else {
3514925baeddSChris Mason 		btrfs_tree_unlock(split);
35155f39d397SChris Mason 		free_extent_buffer(split);
3516be0e5c09SChris Mason 	}
3517aa5d6bedSChris Mason 	return ret;
3518be0e5c09SChris Mason }
3519be0e5c09SChris Mason 
352074123bd7SChris Mason /*
352174123bd7SChris Mason  * how many bytes are required to store the items in a leaf.  start
352274123bd7SChris Mason  * and nr indicate which items in the leaf to check.  This totals up the
352374123bd7SChris Mason  * space used both by the item structs and the item data
352474123bd7SChris Mason  */
35255f39d397SChris Mason static int leaf_space_used(struct extent_buffer *l, int start, int nr)
3526be0e5c09SChris Mason {
352741be1f3bSJosef Bacik 	struct btrfs_item *start_item;
352841be1f3bSJosef Bacik 	struct btrfs_item *end_item;
352941be1f3bSJosef Bacik 	struct btrfs_map_token token;
3530be0e5c09SChris Mason 	int data_len;
35315f39d397SChris Mason 	int nritems = btrfs_header_nritems(l);
3532d4dbff95SChris Mason 	int end = min(nritems, start + nr) - 1;
3533be0e5c09SChris Mason 
3534be0e5c09SChris Mason 	if (!nr)
3535be0e5c09SChris Mason 		return 0;
353641be1f3bSJosef Bacik 	btrfs_init_map_token(&token);
3537dd3cc16bSRoss Kirk 	start_item = btrfs_item_nr(start);
3538dd3cc16bSRoss Kirk 	end_item = btrfs_item_nr(end);
353941be1f3bSJosef Bacik 	data_len = btrfs_token_item_offset(l, start_item, &token) +
354041be1f3bSJosef Bacik 		btrfs_token_item_size(l, start_item, &token);
354141be1f3bSJosef Bacik 	data_len = data_len - btrfs_token_item_offset(l, end_item, &token);
35420783fcfcSChris Mason 	data_len += sizeof(struct btrfs_item) * nr;
3543d4dbff95SChris Mason 	WARN_ON(data_len < 0);
3544be0e5c09SChris Mason 	return data_len;
3545be0e5c09SChris Mason }
3546be0e5c09SChris Mason 
354774123bd7SChris Mason /*
3548d4dbff95SChris Mason  * The space between the end of the leaf items and
3549d4dbff95SChris Mason  * the start of the leaf data.  IOW, how much room
3550d4dbff95SChris Mason  * the leaf has left for both items and data
3551d4dbff95SChris Mason  */
35522ff7e61eSJeff Mahoney noinline int btrfs_leaf_free_space(struct btrfs_fs_info *fs_info,
3553e02119d5SChris Mason 				   struct extent_buffer *leaf)
3554d4dbff95SChris Mason {
35555f39d397SChris Mason 	int nritems = btrfs_header_nritems(leaf);
35565f39d397SChris Mason 	int ret;
35570b246afaSJeff Mahoney 
35580b246afaSJeff Mahoney 	ret = BTRFS_LEAF_DATA_SIZE(fs_info) - leaf_space_used(leaf, 0, nritems);
35595f39d397SChris Mason 	if (ret < 0) {
35600b246afaSJeff Mahoney 		btrfs_crit(fs_info,
3561efe120a0SFrank Holton 			   "leaf free space ret %d, leaf data size %lu, used %d nritems %d",
3562da17066cSJeff Mahoney 			   ret,
35630b246afaSJeff Mahoney 			   (unsigned long) BTRFS_LEAF_DATA_SIZE(fs_info),
35645f39d397SChris Mason 			   leaf_space_used(leaf, 0, nritems), nritems);
35655f39d397SChris Mason 	}
35665f39d397SChris Mason 	return ret;
3567d4dbff95SChris Mason }
3568d4dbff95SChris Mason 
356999d8f83cSChris Mason /*
357099d8f83cSChris Mason  * min slot controls the lowest index we're willing to push to the
357199d8f83cSChris Mason  * right.  We'll push up to and including min_slot, but no lower
357299d8f83cSChris Mason  */
35731e47eef2SDavid Sterba static noinline int __push_leaf_right(struct btrfs_fs_info *fs_info,
357444871b1bSChris Mason 				      struct btrfs_path *path,
357544871b1bSChris Mason 				      int data_size, int empty,
357644871b1bSChris Mason 				      struct extent_buffer *right,
357799d8f83cSChris Mason 				      int free_space, u32 left_nritems,
357899d8f83cSChris Mason 				      u32 min_slot)
357900ec4c51SChris Mason {
35805f39d397SChris Mason 	struct extent_buffer *left = path->nodes[0];
358144871b1bSChris Mason 	struct extent_buffer *upper = path->nodes[1];
3582cfed81a0SChris Mason 	struct btrfs_map_token token;
35835f39d397SChris Mason 	struct btrfs_disk_key disk_key;
358400ec4c51SChris Mason 	int slot;
358534a38218SChris Mason 	u32 i;
358600ec4c51SChris Mason 	int push_space = 0;
358700ec4c51SChris Mason 	int push_items = 0;
35880783fcfcSChris Mason 	struct btrfs_item *item;
358934a38218SChris Mason 	u32 nr;
35907518a238SChris Mason 	u32 right_nritems;
35915f39d397SChris Mason 	u32 data_end;
3592db94535dSChris Mason 	u32 this_item_size;
359300ec4c51SChris Mason 
3594cfed81a0SChris Mason 	btrfs_init_map_token(&token);
3595cfed81a0SChris Mason 
359634a38218SChris Mason 	if (empty)
359734a38218SChris Mason 		nr = 0;
359834a38218SChris Mason 	else
359999d8f83cSChris Mason 		nr = max_t(u32, 1, min_slot);
360034a38218SChris Mason 
360131840ae1SZheng Yan 	if (path->slots[0] >= left_nritems)
360287b29b20SYan Zheng 		push_space += data_size;
360331840ae1SZheng Yan 
360444871b1bSChris Mason 	slot = path->slots[1];
360534a38218SChris Mason 	i = left_nritems - 1;
360634a38218SChris Mason 	while (i >= nr) {
3607dd3cc16bSRoss Kirk 		item = btrfs_item_nr(i);
3608db94535dSChris Mason 
360931840ae1SZheng Yan 		if (!empty && push_items > 0) {
361031840ae1SZheng Yan 			if (path->slots[0] > i)
361131840ae1SZheng Yan 				break;
361231840ae1SZheng Yan 			if (path->slots[0] == i) {
36132ff7e61eSJeff Mahoney 				int space = btrfs_leaf_free_space(fs_info, left);
361431840ae1SZheng Yan 				if (space + push_space * 2 > free_space)
361531840ae1SZheng Yan 					break;
361631840ae1SZheng Yan 			}
361731840ae1SZheng Yan 		}
361831840ae1SZheng Yan 
361900ec4c51SChris Mason 		if (path->slots[0] == i)
362087b29b20SYan Zheng 			push_space += data_size;
3621db94535dSChris Mason 
3622db94535dSChris Mason 		this_item_size = btrfs_item_size(left, item);
3623db94535dSChris Mason 		if (this_item_size + sizeof(*item) + push_space > free_space)
362400ec4c51SChris Mason 			break;
362531840ae1SZheng Yan 
362600ec4c51SChris Mason 		push_items++;
3627db94535dSChris Mason 		push_space += this_item_size + sizeof(*item);
362834a38218SChris Mason 		if (i == 0)
362934a38218SChris Mason 			break;
363034a38218SChris Mason 		i--;
3631db94535dSChris Mason 	}
36325f39d397SChris Mason 
3633925baeddSChris Mason 	if (push_items == 0)
3634925baeddSChris Mason 		goto out_unlock;
36355f39d397SChris Mason 
36366c1500f2SJulia Lawall 	WARN_ON(!empty && push_items == left_nritems);
36375f39d397SChris Mason 
363800ec4c51SChris Mason 	/* push left to right */
36395f39d397SChris Mason 	right_nritems = btrfs_header_nritems(right);
364034a38218SChris Mason 
36415f39d397SChris Mason 	push_space = btrfs_item_end_nr(left, left_nritems - push_items);
36422ff7e61eSJeff Mahoney 	push_space -= leaf_data_end(fs_info, left);
36435f39d397SChris Mason 
364400ec4c51SChris Mason 	/* make room in the right data area */
36452ff7e61eSJeff Mahoney 	data_end = leaf_data_end(fs_info, right);
36465f39d397SChris Mason 	memmove_extent_buffer(right,
36473d9ec8c4SNikolay Borisov 			      BTRFS_LEAF_DATA_OFFSET + data_end - push_space,
36483d9ec8c4SNikolay Borisov 			      BTRFS_LEAF_DATA_OFFSET + data_end,
36490b246afaSJeff Mahoney 			      BTRFS_LEAF_DATA_SIZE(fs_info) - data_end);
36505f39d397SChris Mason 
365100ec4c51SChris Mason 	/* copy from the left data area */
36523d9ec8c4SNikolay Borisov 	copy_extent_buffer(right, left, BTRFS_LEAF_DATA_OFFSET +
36530b246afaSJeff Mahoney 		     BTRFS_LEAF_DATA_SIZE(fs_info) - push_space,
36543d9ec8c4SNikolay Borisov 		     BTRFS_LEAF_DATA_OFFSET + leaf_data_end(fs_info, left),
3655d6025579SChris Mason 		     push_space);
36565f39d397SChris Mason 
36575f39d397SChris Mason 	memmove_extent_buffer(right, btrfs_item_nr_offset(push_items),
36585f39d397SChris Mason 			      btrfs_item_nr_offset(0),
36590783fcfcSChris Mason 			      right_nritems * sizeof(struct btrfs_item));
36605f39d397SChris Mason 
366100ec4c51SChris Mason 	/* copy the items from left to right */
36625f39d397SChris Mason 	copy_extent_buffer(right, left, btrfs_item_nr_offset(0),
36635f39d397SChris Mason 		   btrfs_item_nr_offset(left_nritems - push_items),
36640783fcfcSChris Mason 		   push_items * sizeof(struct btrfs_item));
366500ec4c51SChris Mason 
366600ec4c51SChris Mason 	/* update the item pointers */
36677518a238SChris Mason 	right_nritems += push_items;
36685f39d397SChris Mason 	btrfs_set_header_nritems(right, right_nritems);
36690b246afaSJeff Mahoney 	push_space = BTRFS_LEAF_DATA_SIZE(fs_info);
36707518a238SChris Mason 	for (i = 0; i < right_nritems; i++) {
3671dd3cc16bSRoss Kirk 		item = btrfs_item_nr(i);
3672cfed81a0SChris Mason 		push_space -= btrfs_token_item_size(right, item, &token);
3673cfed81a0SChris Mason 		btrfs_set_token_item_offset(right, item, push_space, &token);
3674db94535dSChris Mason 	}
3675db94535dSChris Mason 
36767518a238SChris Mason 	left_nritems -= push_items;
36775f39d397SChris Mason 	btrfs_set_header_nritems(left, left_nritems);
367800ec4c51SChris Mason 
367934a38218SChris Mason 	if (left_nritems)
36805f39d397SChris Mason 		btrfs_mark_buffer_dirty(left);
3681f0486c68SYan, Zheng 	else
36827c302b49SDavid Sterba 		clean_tree_block(fs_info, left);
3683f0486c68SYan, Zheng 
36845f39d397SChris Mason 	btrfs_mark_buffer_dirty(right);
3685a429e513SChris Mason 
36865f39d397SChris Mason 	btrfs_item_key(right, &disk_key, 0);
36875f39d397SChris Mason 	btrfs_set_node_key(upper, &disk_key, slot + 1);
3688d6025579SChris Mason 	btrfs_mark_buffer_dirty(upper);
368902217ed2SChris Mason 
369000ec4c51SChris Mason 	/* then fixup the leaf pointer in the path */
36917518a238SChris Mason 	if (path->slots[0] >= left_nritems) {
36927518a238SChris Mason 		path->slots[0] -= left_nritems;
3693925baeddSChris Mason 		if (btrfs_header_nritems(path->nodes[0]) == 0)
36947c302b49SDavid Sterba 			clean_tree_block(fs_info, path->nodes[0]);
3695925baeddSChris Mason 		btrfs_tree_unlock(path->nodes[0]);
36965f39d397SChris Mason 		free_extent_buffer(path->nodes[0]);
36975f39d397SChris Mason 		path->nodes[0] = right;
369800ec4c51SChris Mason 		path->slots[1] += 1;
369900ec4c51SChris Mason 	} else {
3700925baeddSChris Mason 		btrfs_tree_unlock(right);
37015f39d397SChris Mason 		free_extent_buffer(right);
370200ec4c51SChris Mason 	}
370300ec4c51SChris Mason 	return 0;
3704925baeddSChris Mason 
3705925baeddSChris Mason out_unlock:
3706925baeddSChris Mason 	btrfs_tree_unlock(right);
3707925baeddSChris Mason 	free_extent_buffer(right);
3708925baeddSChris Mason 	return 1;
370900ec4c51SChris Mason }
3710925baeddSChris Mason 
371100ec4c51SChris Mason /*
371244871b1bSChris Mason  * push some data in the path leaf to the right, trying to free up at
371374123bd7SChris Mason  * least data_size bytes.  returns zero if the push worked, nonzero otherwise
371444871b1bSChris Mason  *
371544871b1bSChris Mason  * returns 1 if the push failed because the other node didn't have enough
371644871b1bSChris Mason  * room, 0 if everything worked out and < 0 if there were major errors.
371799d8f83cSChris Mason  *
371899d8f83cSChris Mason  * this will push starting from min_slot to the end of the leaf.  It won't
371999d8f83cSChris Mason  * push any slot lower than min_slot
372074123bd7SChris Mason  */
372144871b1bSChris Mason static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root
372299d8f83cSChris Mason 			   *root, struct btrfs_path *path,
372399d8f83cSChris Mason 			   int min_data_size, int data_size,
372499d8f83cSChris Mason 			   int empty, u32 min_slot)
3725be0e5c09SChris Mason {
37262ff7e61eSJeff Mahoney 	struct btrfs_fs_info *fs_info = root->fs_info;
372744871b1bSChris Mason 	struct extent_buffer *left = path->nodes[0];
372844871b1bSChris Mason 	struct extent_buffer *right;
372944871b1bSChris Mason 	struct extent_buffer *upper;
373044871b1bSChris Mason 	int slot;
373144871b1bSChris Mason 	int free_space;
373244871b1bSChris Mason 	u32 left_nritems;
373344871b1bSChris Mason 	int ret;
373444871b1bSChris Mason 
373544871b1bSChris Mason 	if (!path->nodes[1])
373644871b1bSChris Mason 		return 1;
373744871b1bSChris Mason 
373844871b1bSChris Mason 	slot = path->slots[1];
373944871b1bSChris Mason 	upper = path->nodes[1];
374044871b1bSChris Mason 	if (slot >= btrfs_header_nritems(upper) - 1)
374144871b1bSChris Mason 		return 1;
374244871b1bSChris Mason 
374344871b1bSChris Mason 	btrfs_assert_tree_locked(path->nodes[1]);
374444871b1bSChris Mason 
37452ff7e61eSJeff Mahoney 	right = read_node_slot(fs_info, upper, slot + 1);
3746fb770ae4SLiu Bo 	/*
3747fb770ae4SLiu Bo 	 * slot + 1 is not valid or we fail to read the right node,
3748fb770ae4SLiu Bo 	 * no big deal, just return.
3749fb770ae4SLiu Bo 	 */
3750fb770ae4SLiu Bo 	if (IS_ERR(right))
375191ca338dSTsutomu Itoh 		return 1;
375291ca338dSTsutomu Itoh 
375344871b1bSChris Mason 	btrfs_tree_lock(right);
375444871b1bSChris Mason 	btrfs_set_lock_blocking(right);
375544871b1bSChris Mason 
37562ff7e61eSJeff Mahoney 	free_space = btrfs_leaf_free_space(fs_info, right);
375744871b1bSChris Mason 	if (free_space < data_size)
375844871b1bSChris Mason 		goto out_unlock;
375944871b1bSChris Mason 
376044871b1bSChris Mason 	/* cow and double check */
376144871b1bSChris Mason 	ret = btrfs_cow_block(trans, root, right, upper,
376244871b1bSChris Mason 			      slot + 1, &right);
376344871b1bSChris Mason 	if (ret)
376444871b1bSChris Mason 		goto out_unlock;
376544871b1bSChris Mason 
37662ff7e61eSJeff Mahoney 	free_space = btrfs_leaf_free_space(fs_info, right);
376744871b1bSChris Mason 	if (free_space < data_size)
376844871b1bSChris Mason 		goto out_unlock;
376944871b1bSChris Mason 
377044871b1bSChris Mason 	left_nritems = btrfs_header_nritems(left);
377144871b1bSChris Mason 	if (left_nritems == 0)
377244871b1bSChris Mason 		goto out_unlock;
377344871b1bSChris Mason 
37742ef1fed2SFilipe David Borba Manana 	if (path->slots[0] == left_nritems && !empty) {
37752ef1fed2SFilipe David Borba Manana 		/* Key greater than all keys in the leaf, right neighbor has
37762ef1fed2SFilipe David Borba Manana 		 * enough room for it and we're not emptying our leaf to delete
37772ef1fed2SFilipe David Borba Manana 		 * it, therefore use right neighbor to insert the new item and
37782ef1fed2SFilipe David Borba Manana 		 * no need to touch/dirty our left leaft. */
37792ef1fed2SFilipe David Borba Manana 		btrfs_tree_unlock(left);
37802ef1fed2SFilipe David Borba Manana 		free_extent_buffer(left);
37812ef1fed2SFilipe David Borba Manana 		path->nodes[0] = right;
37822ef1fed2SFilipe David Borba Manana 		path->slots[0] = 0;
37832ef1fed2SFilipe David Borba Manana 		path->slots[1]++;
37842ef1fed2SFilipe David Borba Manana 		return 0;
37852ef1fed2SFilipe David Borba Manana 	}
37862ef1fed2SFilipe David Borba Manana 
37871e47eef2SDavid Sterba 	return __push_leaf_right(fs_info, path, min_data_size, empty,
378899d8f83cSChris Mason 				right, free_space, left_nritems, min_slot);
378944871b1bSChris Mason out_unlock:
379044871b1bSChris Mason 	btrfs_tree_unlock(right);
379144871b1bSChris Mason 	free_extent_buffer(right);
379244871b1bSChris Mason 	return 1;
379344871b1bSChris Mason }
379444871b1bSChris Mason 
379544871b1bSChris Mason /*
379644871b1bSChris Mason  * push some data in the path leaf to the left, trying to free up at
379744871b1bSChris Mason  * least data_size bytes.  returns zero if the push worked, nonzero otherwise
379899d8f83cSChris Mason  *
379999d8f83cSChris Mason  * max_slot can put a limit on how far into the leaf we'll push items.  The
380099d8f83cSChris Mason  * item at 'max_slot' won't be touched.  Use (u32)-1 to make us do all the
380199d8f83cSChris Mason  * items
380244871b1bSChris Mason  */
380366cb7ddbSDavid Sterba static noinline int __push_leaf_left(struct btrfs_fs_info *fs_info,
380444871b1bSChris Mason 				     struct btrfs_path *path, int data_size,
380544871b1bSChris Mason 				     int empty, struct extent_buffer *left,
380699d8f83cSChris Mason 				     int free_space, u32 right_nritems,
380799d8f83cSChris Mason 				     u32 max_slot)
380844871b1bSChris Mason {
38095f39d397SChris Mason 	struct btrfs_disk_key disk_key;
38105f39d397SChris Mason 	struct extent_buffer *right = path->nodes[0];
3811be0e5c09SChris Mason 	int i;
3812be0e5c09SChris Mason 	int push_space = 0;
3813be0e5c09SChris Mason 	int push_items = 0;
38140783fcfcSChris Mason 	struct btrfs_item *item;
38157518a238SChris Mason 	u32 old_left_nritems;
381634a38218SChris Mason 	u32 nr;
3817aa5d6bedSChris Mason 	int ret = 0;
3818db94535dSChris Mason 	u32 this_item_size;
3819db94535dSChris Mason 	u32 old_left_item_size;
3820cfed81a0SChris Mason 	struct btrfs_map_token token;
3821cfed81a0SChris Mason 
3822cfed81a0SChris Mason 	btrfs_init_map_token(&token);
3823be0e5c09SChris Mason 
382434a38218SChris Mason 	if (empty)
382599d8f83cSChris Mason 		nr = min(right_nritems, max_slot);
382634a38218SChris Mason 	else
382799d8f83cSChris Mason 		nr = min(right_nritems - 1, max_slot);
382834a38218SChris Mason 
382934a38218SChris Mason 	for (i = 0; i < nr; i++) {
3830dd3cc16bSRoss Kirk 		item = btrfs_item_nr(i);
3831db94535dSChris Mason 
383231840ae1SZheng Yan 		if (!empty && push_items > 0) {
383331840ae1SZheng Yan 			if (path->slots[0] < i)
383431840ae1SZheng Yan 				break;
383531840ae1SZheng Yan 			if (path->slots[0] == i) {
38362ff7e61eSJeff Mahoney 				int space = btrfs_leaf_free_space(fs_info, right);
383731840ae1SZheng Yan 				if (space + push_space * 2 > free_space)
383831840ae1SZheng Yan 					break;
383931840ae1SZheng Yan 			}
384031840ae1SZheng Yan 		}
384131840ae1SZheng Yan 
3842be0e5c09SChris Mason 		if (path->slots[0] == i)
384387b29b20SYan Zheng 			push_space += data_size;
3844db94535dSChris Mason 
3845db94535dSChris Mason 		this_item_size = btrfs_item_size(right, item);
3846db94535dSChris Mason 		if (this_item_size + sizeof(*item) + push_space > free_space)
3847be0e5c09SChris Mason 			break;
3848db94535dSChris Mason 
3849be0e5c09SChris Mason 		push_items++;
3850db94535dSChris Mason 		push_space += this_item_size + sizeof(*item);
3851be0e5c09SChris Mason 	}
3852db94535dSChris Mason 
3853be0e5c09SChris Mason 	if (push_items == 0) {
3854925baeddSChris Mason 		ret = 1;
3855925baeddSChris Mason 		goto out;
3856be0e5c09SChris Mason 	}
3857fae7f21cSDulshani Gunawardhana 	WARN_ON(!empty && push_items == btrfs_header_nritems(right));
38585f39d397SChris Mason 
3859be0e5c09SChris Mason 	/* push data from right to left */
38605f39d397SChris Mason 	copy_extent_buffer(left, right,
38615f39d397SChris Mason 			   btrfs_item_nr_offset(btrfs_header_nritems(left)),
38625f39d397SChris Mason 			   btrfs_item_nr_offset(0),
38635f39d397SChris Mason 			   push_items * sizeof(struct btrfs_item));
38645f39d397SChris Mason 
38650b246afaSJeff Mahoney 	push_space = BTRFS_LEAF_DATA_SIZE(fs_info) -
38665f39d397SChris Mason 		     btrfs_item_offset_nr(right, push_items - 1);
38675f39d397SChris Mason 
38683d9ec8c4SNikolay Borisov 	copy_extent_buffer(left, right, BTRFS_LEAF_DATA_OFFSET +
38692ff7e61eSJeff Mahoney 		     leaf_data_end(fs_info, left) - push_space,
38703d9ec8c4SNikolay Borisov 		     BTRFS_LEAF_DATA_OFFSET +
38715f39d397SChris Mason 		     btrfs_item_offset_nr(right, push_items - 1),
3872be0e5c09SChris Mason 		     push_space);
38735f39d397SChris Mason 	old_left_nritems = btrfs_header_nritems(left);
387487b29b20SYan Zheng 	BUG_ON(old_left_nritems <= 0);
3875eb60ceacSChris Mason 
3876db94535dSChris Mason 	old_left_item_size = btrfs_item_offset_nr(left, old_left_nritems - 1);
3877be0e5c09SChris Mason 	for (i = old_left_nritems; i < old_left_nritems + push_items; i++) {
38785f39d397SChris Mason 		u32 ioff;
3879db94535dSChris Mason 
3880dd3cc16bSRoss Kirk 		item = btrfs_item_nr(i);
3881db94535dSChris Mason 
3882cfed81a0SChris Mason 		ioff = btrfs_token_item_offset(left, item, &token);
3883cfed81a0SChris Mason 		btrfs_set_token_item_offset(left, item,
38840b246afaSJeff Mahoney 		      ioff - (BTRFS_LEAF_DATA_SIZE(fs_info) - old_left_item_size),
3885cfed81a0SChris Mason 		      &token);
3886be0e5c09SChris Mason 	}
38875f39d397SChris Mason 	btrfs_set_header_nritems(left, old_left_nritems + push_items);
3888be0e5c09SChris Mason 
3889be0e5c09SChris Mason 	/* fixup right node */
389031b1a2bdSJulia Lawall 	if (push_items > right_nritems)
389131b1a2bdSJulia Lawall 		WARN(1, KERN_CRIT "push items %d nr %u\n", push_items,
3892d397712bSChris Mason 		       right_nritems);
389334a38218SChris Mason 
389434a38218SChris Mason 	if (push_items < right_nritems) {
38955f39d397SChris Mason 		push_space = btrfs_item_offset_nr(right, push_items - 1) -
38962ff7e61eSJeff Mahoney 						  leaf_data_end(fs_info, right);
38973d9ec8c4SNikolay Borisov 		memmove_extent_buffer(right, BTRFS_LEAF_DATA_OFFSET +
38980b246afaSJeff Mahoney 				      BTRFS_LEAF_DATA_SIZE(fs_info) - push_space,
38993d9ec8c4SNikolay Borisov 				      BTRFS_LEAF_DATA_OFFSET +
39002ff7e61eSJeff Mahoney 				      leaf_data_end(fs_info, right), push_space);
39015f39d397SChris Mason 
39025f39d397SChris Mason 		memmove_extent_buffer(right, btrfs_item_nr_offset(0),
39035f39d397SChris Mason 			      btrfs_item_nr_offset(push_items),
39045f39d397SChris Mason 			     (btrfs_header_nritems(right) - push_items) *
39050783fcfcSChris Mason 			     sizeof(struct btrfs_item));
390634a38218SChris Mason 	}
3907eef1c494SYan 	right_nritems -= push_items;
3908eef1c494SYan 	btrfs_set_header_nritems(right, right_nritems);
39090b246afaSJeff Mahoney 	push_space = BTRFS_LEAF_DATA_SIZE(fs_info);
39105f39d397SChris Mason 	for (i = 0; i < right_nritems; i++) {
3911dd3cc16bSRoss Kirk 		item = btrfs_item_nr(i);
3912db94535dSChris Mason 
3913cfed81a0SChris Mason 		push_space = push_space - btrfs_token_item_size(right,
3914cfed81a0SChris Mason 								item, &token);
3915cfed81a0SChris Mason 		btrfs_set_token_item_offset(right, item, push_space, &token);
3916db94535dSChris Mason 	}
3917eb60ceacSChris Mason 
39185f39d397SChris Mason 	btrfs_mark_buffer_dirty(left);
391934a38218SChris Mason 	if (right_nritems)
39205f39d397SChris Mason 		btrfs_mark_buffer_dirty(right);
3921f0486c68SYan, Zheng 	else
39227c302b49SDavid Sterba 		clean_tree_block(fs_info, right);
3923098f59c2SChris Mason 
39245f39d397SChris Mason 	btrfs_item_key(right, &disk_key, 0);
3925b167fa91SNikolay Borisov 	fixup_low_keys(path, &disk_key, 1);
3926be0e5c09SChris Mason 
3927be0e5c09SChris Mason 	/* then fixup the leaf pointer in the path */
3928be0e5c09SChris Mason 	if (path->slots[0] < push_items) {
3929be0e5c09SChris Mason 		path->slots[0] += old_left_nritems;
3930925baeddSChris Mason 		btrfs_tree_unlock(path->nodes[0]);
39315f39d397SChris Mason 		free_extent_buffer(path->nodes[0]);
39325f39d397SChris Mason 		path->nodes[0] = left;
3933be0e5c09SChris Mason 		path->slots[1] -= 1;
3934be0e5c09SChris Mason 	} else {
3935925baeddSChris Mason 		btrfs_tree_unlock(left);
39365f39d397SChris Mason 		free_extent_buffer(left);
3937be0e5c09SChris Mason 		path->slots[0] -= push_items;
3938be0e5c09SChris Mason 	}
3939eb60ceacSChris Mason 	BUG_ON(path->slots[0] < 0);
3940aa5d6bedSChris Mason 	return ret;
3941925baeddSChris Mason out:
3942925baeddSChris Mason 	btrfs_tree_unlock(left);
3943925baeddSChris Mason 	free_extent_buffer(left);
3944925baeddSChris Mason 	return ret;
3945be0e5c09SChris Mason }
3946be0e5c09SChris Mason 
394774123bd7SChris Mason /*
394844871b1bSChris Mason  * push some data in the path leaf to the left, trying to free up at
394944871b1bSChris Mason  * least data_size bytes.  returns zero if the push worked, nonzero otherwise
395099d8f83cSChris Mason  *
395199d8f83cSChris Mason  * max_slot can put a limit on how far into the leaf we'll push items.  The
395299d8f83cSChris Mason  * item at 'max_slot' won't be touched.  Use (u32)-1 to make us push all the
395399d8f83cSChris Mason  * items
395444871b1bSChris Mason  */
395544871b1bSChris Mason static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root
395699d8f83cSChris Mason 			  *root, struct btrfs_path *path, int min_data_size,
395799d8f83cSChris Mason 			  int data_size, int empty, u32 max_slot)
395844871b1bSChris Mason {
39592ff7e61eSJeff Mahoney 	struct btrfs_fs_info *fs_info = root->fs_info;
396044871b1bSChris Mason 	struct extent_buffer *right = path->nodes[0];
396144871b1bSChris Mason 	struct extent_buffer *left;
396244871b1bSChris Mason 	int slot;
396344871b1bSChris Mason 	int free_space;
396444871b1bSChris Mason 	u32 right_nritems;
396544871b1bSChris Mason 	int ret = 0;
396644871b1bSChris Mason 
396744871b1bSChris Mason 	slot = path->slots[1];
396844871b1bSChris Mason 	if (slot == 0)
396944871b1bSChris Mason 		return 1;
397044871b1bSChris Mason 	if (!path->nodes[1])
397144871b1bSChris Mason 		return 1;
397244871b1bSChris Mason 
397344871b1bSChris Mason 	right_nritems = btrfs_header_nritems(right);
397444871b1bSChris Mason 	if (right_nritems == 0)
397544871b1bSChris Mason 		return 1;
397644871b1bSChris Mason 
397744871b1bSChris Mason 	btrfs_assert_tree_locked(path->nodes[1]);
397844871b1bSChris Mason 
39792ff7e61eSJeff Mahoney 	left = read_node_slot(fs_info, path->nodes[1], slot - 1);
3980fb770ae4SLiu Bo 	/*
3981fb770ae4SLiu Bo 	 * slot - 1 is not valid or we fail to read the left node,
3982fb770ae4SLiu Bo 	 * no big deal, just return.
3983fb770ae4SLiu Bo 	 */
3984fb770ae4SLiu Bo 	if (IS_ERR(left))
398591ca338dSTsutomu Itoh 		return 1;
398691ca338dSTsutomu Itoh 
398744871b1bSChris Mason 	btrfs_tree_lock(left);
398844871b1bSChris Mason 	btrfs_set_lock_blocking(left);
398944871b1bSChris Mason 
39902ff7e61eSJeff Mahoney 	free_space = btrfs_leaf_free_space(fs_info, left);
399144871b1bSChris Mason 	if (free_space < data_size) {
399244871b1bSChris Mason 		ret = 1;
399344871b1bSChris Mason 		goto out;
399444871b1bSChris Mason 	}
399544871b1bSChris Mason 
399644871b1bSChris Mason 	/* cow and double check */
399744871b1bSChris Mason 	ret = btrfs_cow_block(trans, root, left,
399844871b1bSChris Mason 			      path->nodes[1], slot - 1, &left);
399944871b1bSChris Mason 	if (ret) {
400044871b1bSChris Mason 		/* we hit -ENOSPC, but it isn't fatal here */
400179787eaaSJeff Mahoney 		if (ret == -ENOSPC)
400244871b1bSChris Mason 			ret = 1;
400344871b1bSChris Mason 		goto out;
400444871b1bSChris Mason 	}
400544871b1bSChris Mason 
40062ff7e61eSJeff Mahoney 	free_space = btrfs_leaf_free_space(fs_info, left);
400744871b1bSChris Mason 	if (free_space < data_size) {
400844871b1bSChris Mason 		ret = 1;
400944871b1bSChris Mason 		goto out;
401044871b1bSChris Mason 	}
401144871b1bSChris Mason 
401266cb7ddbSDavid Sterba 	return __push_leaf_left(fs_info, path, min_data_size,
401399d8f83cSChris Mason 			       empty, left, free_space, right_nritems,
401499d8f83cSChris Mason 			       max_slot);
401544871b1bSChris Mason out:
401644871b1bSChris Mason 	btrfs_tree_unlock(left);
401744871b1bSChris Mason 	free_extent_buffer(left);
401844871b1bSChris Mason 	return ret;
401944871b1bSChris Mason }
402044871b1bSChris Mason 
402144871b1bSChris Mason /*
402274123bd7SChris Mason  * split the path's leaf in two, making sure there is at least data_size
402374123bd7SChris Mason  * available for the resulting leaf level of the path.
402474123bd7SChris Mason  */
4025143bede5SJeff Mahoney static noinline void copy_for_split(struct btrfs_trans_handle *trans,
40262ff7e61eSJeff Mahoney 				    struct btrfs_fs_info *fs_info,
402744871b1bSChris Mason 				    struct btrfs_path *path,
402844871b1bSChris Mason 				    struct extent_buffer *l,
402944871b1bSChris Mason 				    struct extent_buffer *right,
403044871b1bSChris Mason 				    int slot, int mid, int nritems)
4031be0e5c09SChris Mason {
4032be0e5c09SChris Mason 	int data_copy_size;
4033be0e5c09SChris Mason 	int rt_data_off;
4034be0e5c09SChris Mason 	int i;
4035d4dbff95SChris Mason 	struct btrfs_disk_key disk_key;
4036cfed81a0SChris Mason 	struct btrfs_map_token token;
4037cfed81a0SChris Mason 
4038cfed81a0SChris Mason 	btrfs_init_map_token(&token);
4039be0e5c09SChris Mason 
40405f39d397SChris Mason 	nritems = nritems - mid;
40415f39d397SChris Mason 	btrfs_set_header_nritems(right, nritems);
40422ff7e61eSJeff Mahoney 	data_copy_size = btrfs_item_end_nr(l, mid) - leaf_data_end(fs_info, l);
40435f39d397SChris Mason 
40445f39d397SChris Mason 	copy_extent_buffer(right, l, btrfs_item_nr_offset(0),
40455f39d397SChris Mason 			   btrfs_item_nr_offset(mid),
40465f39d397SChris Mason 			   nritems * sizeof(struct btrfs_item));
40475f39d397SChris Mason 
40485f39d397SChris Mason 	copy_extent_buffer(right, l,
40493d9ec8c4SNikolay Borisov 		     BTRFS_LEAF_DATA_OFFSET + BTRFS_LEAF_DATA_SIZE(fs_info) -
40503d9ec8c4SNikolay Borisov 		     data_copy_size, BTRFS_LEAF_DATA_OFFSET +
40512ff7e61eSJeff Mahoney 		     leaf_data_end(fs_info, l), data_copy_size);
405274123bd7SChris Mason 
40530b246afaSJeff Mahoney 	rt_data_off = BTRFS_LEAF_DATA_SIZE(fs_info) - btrfs_item_end_nr(l, mid);
40545f39d397SChris Mason 
40555f39d397SChris Mason 	for (i = 0; i < nritems; i++) {
4056dd3cc16bSRoss Kirk 		struct btrfs_item *item = btrfs_item_nr(i);
4057db94535dSChris Mason 		u32 ioff;
4058db94535dSChris Mason 
4059cfed81a0SChris Mason 		ioff = btrfs_token_item_offset(right, item, &token);
4060cfed81a0SChris Mason 		btrfs_set_token_item_offset(right, item,
4061cfed81a0SChris Mason 					    ioff + rt_data_off, &token);
40620783fcfcSChris Mason 	}
406374123bd7SChris Mason 
40645f39d397SChris Mason 	btrfs_set_header_nritems(l, mid);
40655f39d397SChris Mason 	btrfs_item_key(right, &disk_key, 0);
40662ff7e61eSJeff Mahoney 	insert_ptr(trans, fs_info, path, &disk_key, right->start,
4067c3e06965SJan Schmidt 		   path->slots[1] + 1, 1);
40685f39d397SChris Mason 
40695f39d397SChris Mason 	btrfs_mark_buffer_dirty(right);
40705f39d397SChris Mason 	btrfs_mark_buffer_dirty(l);
4071eb60ceacSChris Mason 	BUG_ON(path->slots[0] != slot);
40725f39d397SChris Mason 
4073be0e5c09SChris Mason 	if (mid <= slot) {
4074925baeddSChris Mason 		btrfs_tree_unlock(path->nodes[0]);
40755f39d397SChris Mason 		free_extent_buffer(path->nodes[0]);
40765f39d397SChris Mason 		path->nodes[0] = right;
4077be0e5c09SChris Mason 		path->slots[0] -= mid;
4078be0e5c09SChris Mason 		path->slots[1] += 1;
4079925baeddSChris Mason 	} else {
4080925baeddSChris Mason 		btrfs_tree_unlock(right);
40815f39d397SChris Mason 		free_extent_buffer(right);
4082925baeddSChris Mason 	}
40835f39d397SChris Mason 
4084eb60ceacSChris Mason 	BUG_ON(path->slots[0] < 0);
408544871b1bSChris Mason }
408644871b1bSChris Mason 
408744871b1bSChris Mason /*
408899d8f83cSChris Mason  * double splits happen when we need to insert a big item in the middle
408999d8f83cSChris Mason  * of a leaf.  A double split can leave us with 3 mostly empty leaves:
409099d8f83cSChris Mason  * leaf: [ slots 0 - N] [ our target ] [ N + 1 - total in leaf ]
409199d8f83cSChris Mason  *          A                 B                 C
409299d8f83cSChris Mason  *
409399d8f83cSChris Mason  * We avoid this by trying to push the items on either side of our target
409499d8f83cSChris Mason  * into the adjacent leaves.  If all goes well we can avoid the double split
409599d8f83cSChris Mason  * completely.
409699d8f83cSChris Mason  */
409799d8f83cSChris Mason static noinline int push_for_double_split(struct btrfs_trans_handle *trans,
409899d8f83cSChris Mason 					  struct btrfs_root *root,
409999d8f83cSChris Mason 					  struct btrfs_path *path,
410099d8f83cSChris Mason 					  int data_size)
410199d8f83cSChris Mason {
41022ff7e61eSJeff Mahoney 	struct btrfs_fs_info *fs_info = root->fs_info;
410399d8f83cSChris Mason 	int ret;
410499d8f83cSChris Mason 	int progress = 0;
410599d8f83cSChris Mason 	int slot;
410699d8f83cSChris Mason 	u32 nritems;
41075a4267caSFilipe David Borba Manana 	int space_needed = data_size;
410899d8f83cSChris Mason 
410999d8f83cSChris Mason 	slot = path->slots[0];
41105a4267caSFilipe David Borba Manana 	if (slot < btrfs_header_nritems(path->nodes[0]))
41112ff7e61eSJeff Mahoney 		space_needed -= btrfs_leaf_free_space(fs_info, path->nodes[0]);
411299d8f83cSChris Mason 
411399d8f83cSChris Mason 	/*
411499d8f83cSChris Mason 	 * try to push all the items after our slot into the
411599d8f83cSChris Mason 	 * right leaf
411699d8f83cSChris Mason 	 */
41175a4267caSFilipe David Borba Manana 	ret = push_leaf_right(trans, root, path, 1, space_needed, 0, slot);
411899d8f83cSChris Mason 	if (ret < 0)
411999d8f83cSChris Mason 		return ret;
412099d8f83cSChris Mason 
412199d8f83cSChris Mason 	if (ret == 0)
412299d8f83cSChris Mason 		progress++;
412399d8f83cSChris Mason 
412499d8f83cSChris Mason 	nritems = btrfs_header_nritems(path->nodes[0]);
412599d8f83cSChris Mason 	/*
412699d8f83cSChris Mason 	 * our goal is to get our slot at the start or end of a leaf.  If
412799d8f83cSChris Mason 	 * we've done so we're done
412899d8f83cSChris Mason 	 */
412999d8f83cSChris Mason 	if (path->slots[0] == 0 || path->slots[0] == nritems)
413099d8f83cSChris Mason 		return 0;
413199d8f83cSChris Mason 
41322ff7e61eSJeff Mahoney 	if (btrfs_leaf_free_space(fs_info, path->nodes[0]) >= data_size)
413399d8f83cSChris Mason 		return 0;
413499d8f83cSChris Mason 
413599d8f83cSChris Mason 	/* try to push all the items before our slot into the next leaf */
413699d8f83cSChris Mason 	slot = path->slots[0];
4137263d3995SFilipe Manana 	space_needed = data_size;
4138263d3995SFilipe Manana 	if (slot > 0)
4139263d3995SFilipe Manana 		space_needed -= btrfs_leaf_free_space(fs_info, path->nodes[0]);
41405a4267caSFilipe David Borba Manana 	ret = push_leaf_left(trans, root, path, 1, space_needed, 0, slot);
414199d8f83cSChris Mason 	if (ret < 0)
414299d8f83cSChris Mason 		return ret;
414399d8f83cSChris Mason 
414499d8f83cSChris Mason 	if (ret == 0)
414599d8f83cSChris Mason 		progress++;
414699d8f83cSChris Mason 
414799d8f83cSChris Mason 	if (progress)
414899d8f83cSChris Mason 		return 0;
414999d8f83cSChris Mason 	return 1;
415099d8f83cSChris Mason }
415199d8f83cSChris Mason 
415299d8f83cSChris Mason /*
415344871b1bSChris Mason  * split the path's leaf in two, making sure there is at least data_size
415444871b1bSChris Mason  * available for the resulting leaf level of the path.
415544871b1bSChris Mason  *
415644871b1bSChris Mason  * returns 0 if all went well and < 0 on failure.
415744871b1bSChris Mason  */
415844871b1bSChris Mason static noinline int split_leaf(struct btrfs_trans_handle *trans,
415944871b1bSChris Mason 			       struct btrfs_root *root,
4160310712b2SOmar Sandoval 			       const struct btrfs_key *ins_key,
416144871b1bSChris Mason 			       struct btrfs_path *path, int data_size,
416244871b1bSChris Mason 			       int extend)
416344871b1bSChris Mason {
41645d4f98a2SYan Zheng 	struct btrfs_disk_key disk_key;
416544871b1bSChris Mason 	struct extent_buffer *l;
416644871b1bSChris Mason 	u32 nritems;
416744871b1bSChris Mason 	int mid;
416844871b1bSChris Mason 	int slot;
416944871b1bSChris Mason 	struct extent_buffer *right;
4170b7a0365eSDaniel Dressler 	struct btrfs_fs_info *fs_info = root->fs_info;
417144871b1bSChris Mason 	int ret = 0;
417244871b1bSChris Mason 	int wret;
41735d4f98a2SYan Zheng 	int split;
417444871b1bSChris Mason 	int num_doubles = 0;
417599d8f83cSChris Mason 	int tried_avoid_double = 0;
417644871b1bSChris Mason 
4177a5719521SYan, Zheng 	l = path->nodes[0];
4178a5719521SYan, Zheng 	slot = path->slots[0];
4179a5719521SYan, Zheng 	if (extend && data_size + btrfs_item_size_nr(l, slot) +
41800b246afaSJeff Mahoney 	    sizeof(struct btrfs_item) > BTRFS_LEAF_DATA_SIZE(fs_info))
4181a5719521SYan, Zheng 		return -EOVERFLOW;
4182a5719521SYan, Zheng 
418344871b1bSChris Mason 	/* first try to make some room by pushing left and right */
418433157e05SLiu Bo 	if (data_size && path->nodes[1]) {
41855a4267caSFilipe David Borba Manana 		int space_needed = data_size;
41865a4267caSFilipe David Borba Manana 
41875a4267caSFilipe David Borba Manana 		if (slot < btrfs_header_nritems(l))
41882ff7e61eSJeff Mahoney 			space_needed -= btrfs_leaf_free_space(fs_info, l);
41895a4267caSFilipe David Borba Manana 
41905a4267caSFilipe David Borba Manana 		wret = push_leaf_right(trans, root, path, space_needed,
41915a4267caSFilipe David Borba Manana 				       space_needed, 0, 0);
419244871b1bSChris Mason 		if (wret < 0)
419344871b1bSChris Mason 			return wret;
419444871b1bSChris Mason 		if (wret) {
4195263d3995SFilipe Manana 			space_needed = data_size;
4196263d3995SFilipe Manana 			if (slot > 0)
4197263d3995SFilipe Manana 				space_needed -= btrfs_leaf_free_space(fs_info,
4198263d3995SFilipe Manana 								      l);
41995a4267caSFilipe David Borba Manana 			wret = push_leaf_left(trans, root, path, space_needed,
42005a4267caSFilipe David Borba Manana 					      space_needed, 0, (u32)-1);
420144871b1bSChris Mason 			if (wret < 0)
420244871b1bSChris Mason 				return wret;
420344871b1bSChris Mason 		}
420444871b1bSChris Mason 		l = path->nodes[0];
420544871b1bSChris Mason 
420644871b1bSChris Mason 		/* did the pushes work? */
42072ff7e61eSJeff Mahoney 		if (btrfs_leaf_free_space(fs_info, l) >= data_size)
420844871b1bSChris Mason 			return 0;
420944871b1bSChris Mason 	}
421044871b1bSChris Mason 
421144871b1bSChris Mason 	if (!path->nodes[1]) {
4212fdd99c72SLiu Bo 		ret = insert_new_root(trans, root, path, 1);
421344871b1bSChris Mason 		if (ret)
421444871b1bSChris Mason 			return ret;
421544871b1bSChris Mason 	}
421644871b1bSChris Mason again:
42175d4f98a2SYan Zheng 	split = 1;
421844871b1bSChris Mason 	l = path->nodes[0];
421944871b1bSChris Mason 	slot = path->slots[0];
422044871b1bSChris Mason 	nritems = btrfs_header_nritems(l);
422144871b1bSChris Mason 	mid = (nritems + 1) / 2;
422244871b1bSChris Mason 
42235d4f98a2SYan Zheng 	if (mid <= slot) {
42245d4f98a2SYan Zheng 		if (nritems == 1 ||
42255d4f98a2SYan Zheng 		    leaf_space_used(l, mid, nritems - mid) + data_size >
42260b246afaSJeff Mahoney 			BTRFS_LEAF_DATA_SIZE(fs_info)) {
42275d4f98a2SYan Zheng 			if (slot >= nritems) {
42285d4f98a2SYan Zheng 				split = 0;
42295d4f98a2SYan Zheng 			} else {
42305d4f98a2SYan Zheng 				mid = slot;
42315d4f98a2SYan Zheng 				if (mid != nritems &&
42325d4f98a2SYan Zheng 				    leaf_space_used(l, mid, nritems - mid) +
42330b246afaSJeff Mahoney 				    data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) {
423499d8f83cSChris Mason 					if (data_size && !tried_avoid_double)
423599d8f83cSChris Mason 						goto push_for_double;
42365d4f98a2SYan Zheng 					split = 2;
42375d4f98a2SYan Zheng 				}
42385d4f98a2SYan Zheng 			}
42395d4f98a2SYan Zheng 		}
42405d4f98a2SYan Zheng 	} else {
42415d4f98a2SYan Zheng 		if (leaf_space_used(l, 0, mid) + data_size >
42420b246afaSJeff Mahoney 			BTRFS_LEAF_DATA_SIZE(fs_info)) {
42435d4f98a2SYan Zheng 			if (!extend && data_size && slot == 0) {
42445d4f98a2SYan Zheng 				split = 0;
42455d4f98a2SYan Zheng 			} else if ((extend || !data_size) && slot == 0) {
42465d4f98a2SYan Zheng 				mid = 1;
42475d4f98a2SYan Zheng 			} else {
42485d4f98a2SYan Zheng 				mid = slot;
42495d4f98a2SYan Zheng 				if (mid != nritems &&
42505d4f98a2SYan Zheng 				    leaf_space_used(l, mid, nritems - mid) +
42510b246afaSJeff Mahoney 				    data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) {
425299d8f83cSChris Mason 					if (data_size && !tried_avoid_double)
425399d8f83cSChris Mason 						goto push_for_double;
42545d4f98a2SYan Zheng 					split = 2;
42555d4f98a2SYan Zheng 				}
42565d4f98a2SYan Zheng 			}
42575d4f98a2SYan Zheng 		}
42585d4f98a2SYan Zheng 	}
42595d4f98a2SYan Zheng 
42605d4f98a2SYan Zheng 	if (split == 0)
42615d4f98a2SYan Zheng 		btrfs_cpu_key_to_disk(&disk_key, ins_key);
42625d4f98a2SYan Zheng 	else
42635d4f98a2SYan Zheng 		btrfs_item_key(l, &disk_key, mid);
42645d4f98a2SYan Zheng 
42654d75f8a9SDavid Sterba 	right = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid,
42665581a51aSJan Schmidt 			&disk_key, 0, l->start, 0);
4267f0486c68SYan, Zheng 	if (IS_ERR(right))
426844871b1bSChris Mason 		return PTR_ERR(right);
4269f0486c68SYan, Zheng 
42700b246afaSJeff Mahoney 	root_add_used(root, fs_info->nodesize);
427144871b1bSChris Mason 
42725d4f98a2SYan Zheng 	if (split == 0) {
427344871b1bSChris Mason 		if (mid <= slot) {
427444871b1bSChris Mason 			btrfs_set_header_nritems(right, 0);
42752ff7e61eSJeff Mahoney 			insert_ptr(trans, fs_info, path, &disk_key,
42762ff7e61eSJeff Mahoney 				   right->start, path->slots[1] + 1, 1);
427744871b1bSChris Mason 			btrfs_tree_unlock(path->nodes[0]);
427844871b1bSChris Mason 			free_extent_buffer(path->nodes[0]);
427944871b1bSChris Mason 			path->nodes[0] = right;
428044871b1bSChris Mason 			path->slots[0] = 0;
428144871b1bSChris Mason 			path->slots[1] += 1;
428244871b1bSChris Mason 		} else {
428344871b1bSChris Mason 			btrfs_set_header_nritems(right, 0);
42842ff7e61eSJeff Mahoney 			insert_ptr(trans, fs_info, path, &disk_key,
42852ff7e61eSJeff Mahoney 				   right->start, path->slots[1], 1);
428644871b1bSChris Mason 			btrfs_tree_unlock(path->nodes[0]);
428744871b1bSChris Mason 			free_extent_buffer(path->nodes[0]);
428844871b1bSChris Mason 			path->nodes[0] = right;
428944871b1bSChris Mason 			path->slots[0] = 0;
4290143bede5SJeff Mahoney 			if (path->slots[1] == 0)
4291b167fa91SNikolay Borisov 				fixup_low_keys(path, &disk_key, 1);
42925d4f98a2SYan Zheng 		}
4293196e0249SLiu Bo 		/*
4294196e0249SLiu Bo 		 * We create a new leaf 'right' for the required ins_len and
4295196e0249SLiu Bo 		 * we'll do btrfs_mark_buffer_dirty() on this leaf after copying
4296196e0249SLiu Bo 		 * the content of ins_len to 'right'.
4297196e0249SLiu Bo 		 */
429844871b1bSChris Mason 		return ret;
429944871b1bSChris Mason 	}
430044871b1bSChris Mason 
43012ff7e61eSJeff Mahoney 	copy_for_split(trans, fs_info, path, l, right, slot, mid, nritems);
430244871b1bSChris Mason 
43035d4f98a2SYan Zheng 	if (split == 2) {
4304cc0c5538SChris Mason 		BUG_ON(num_doubles != 0);
4305cc0c5538SChris Mason 		num_doubles++;
4306cc0c5538SChris Mason 		goto again;
43073326d1b0SChris Mason 	}
430844871b1bSChris Mason 
4309143bede5SJeff Mahoney 	return 0;
431099d8f83cSChris Mason 
431199d8f83cSChris Mason push_for_double:
431299d8f83cSChris Mason 	push_for_double_split(trans, root, path, data_size);
431399d8f83cSChris Mason 	tried_avoid_double = 1;
43142ff7e61eSJeff Mahoney 	if (btrfs_leaf_free_space(fs_info, path->nodes[0]) >= data_size)
431599d8f83cSChris Mason 		return 0;
431699d8f83cSChris Mason 	goto again;
4317be0e5c09SChris Mason }
4318be0e5c09SChris Mason 
4319ad48fd75SYan, Zheng static noinline int setup_leaf_for_split(struct btrfs_trans_handle *trans,
4320ad48fd75SYan, Zheng 					 struct btrfs_root *root,
4321ad48fd75SYan, Zheng 					 struct btrfs_path *path, int ins_len)
4322ad48fd75SYan, Zheng {
43232ff7e61eSJeff Mahoney 	struct btrfs_fs_info *fs_info = root->fs_info;
4324ad48fd75SYan, Zheng 	struct btrfs_key key;
4325ad48fd75SYan, Zheng 	struct extent_buffer *leaf;
4326ad48fd75SYan, Zheng 	struct btrfs_file_extent_item *fi;
4327ad48fd75SYan, Zheng 	u64 extent_len = 0;
4328ad48fd75SYan, Zheng 	u32 item_size;
4329ad48fd75SYan, Zheng 	int ret;
4330ad48fd75SYan, Zheng 
4331ad48fd75SYan, Zheng 	leaf = path->nodes[0];
4332ad48fd75SYan, Zheng 	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
4333ad48fd75SYan, Zheng 
4334ad48fd75SYan, Zheng 	BUG_ON(key.type != BTRFS_EXTENT_DATA_KEY &&
4335ad48fd75SYan, Zheng 	       key.type != BTRFS_EXTENT_CSUM_KEY);
4336ad48fd75SYan, Zheng 
43372ff7e61eSJeff Mahoney 	if (btrfs_leaf_free_space(fs_info, leaf) >= ins_len)
4338ad48fd75SYan, Zheng 		return 0;
4339ad48fd75SYan, Zheng 
4340ad48fd75SYan, Zheng 	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
4341ad48fd75SYan, Zheng 	if (key.type == BTRFS_EXTENT_DATA_KEY) {
4342ad48fd75SYan, Zheng 		fi = btrfs_item_ptr(leaf, path->slots[0],
4343ad48fd75SYan, Zheng 				    struct btrfs_file_extent_item);
4344ad48fd75SYan, Zheng 		extent_len = btrfs_file_extent_num_bytes(leaf, fi);
4345ad48fd75SYan, Zheng 	}
4346b3b4aa74SDavid Sterba 	btrfs_release_path(path);
4347ad48fd75SYan, Zheng 
4348ad48fd75SYan, Zheng 	path->keep_locks = 1;
4349ad48fd75SYan, Zheng 	path->search_for_split = 1;
4350ad48fd75SYan, Zheng 	ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
4351ad48fd75SYan, Zheng 	path->search_for_split = 0;
4352a8df6fe6SFilipe Manana 	if (ret > 0)
4353a8df6fe6SFilipe Manana 		ret = -EAGAIN;
4354ad48fd75SYan, Zheng 	if (ret < 0)
4355ad48fd75SYan, Zheng 		goto err;
4356ad48fd75SYan, Zheng 
4357ad48fd75SYan, Zheng 	ret = -EAGAIN;
4358ad48fd75SYan, Zheng 	leaf = path->nodes[0];
4359a8df6fe6SFilipe Manana 	/* if our item isn't there, return now */
4360a8df6fe6SFilipe Manana 	if (item_size != btrfs_item_size_nr(leaf, path->slots[0]))
4361ad48fd75SYan, Zheng 		goto err;
4362ad48fd75SYan, Zheng 
4363109f6aefSChris Mason 	/* the leaf has  changed, it now has room.  return now */
43642ff7e61eSJeff Mahoney 	if (btrfs_leaf_free_space(fs_info, path->nodes[0]) >= ins_len)
4365109f6aefSChris Mason 		goto err;
4366109f6aefSChris Mason 
4367ad48fd75SYan, Zheng 	if (key.type == BTRFS_EXTENT_DATA_KEY) {
4368ad48fd75SYan, Zheng 		fi = btrfs_item_ptr(leaf, path->slots[0],
4369ad48fd75SYan, Zheng 				    struct btrfs_file_extent_item);
4370ad48fd75SYan, Zheng 		if (extent_len != btrfs_file_extent_num_bytes(leaf, fi))
4371ad48fd75SYan, Zheng 			goto err;
4372ad48fd75SYan, Zheng 	}
4373ad48fd75SYan, Zheng 
4374ad48fd75SYan, Zheng 	btrfs_set_path_blocking(path);
4375ad48fd75SYan, Zheng 	ret = split_leaf(trans, root, &key, path, ins_len, 1);
4376f0486c68SYan, Zheng 	if (ret)
4377f0486c68SYan, Zheng 		goto err;
4378ad48fd75SYan, Zheng 
4379ad48fd75SYan, Zheng 	path->keep_locks = 0;
4380ad48fd75SYan, Zheng 	btrfs_unlock_up_safe(path, 1);
4381ad48fd75SYan, Zheng 	return 0;
4382ad48fd75SYan, Zheng err:
4383ad48fd75SYan, Zheng 	path->keep_locks = 0;
4384ad48fd75SYan, Zheng 	return ret;
4385ad48fd75SYan, Zheng }
4386ad48fd75SYan, Zheng 
43874961e293SDavid Sterba static noinline int split_item(struct btrfs_fs_info *fs_info,
4388459931ecSChris Mason 			       struct btrfs_path *path,
4389310712b2SOmar Sandoval 			       const struct btrfs_key *new_key,
4390459931ecSChris Mason 			       unsigned long split_offset)
4391459931ecSChris Mason {
4392459931ecSChris Mason 	struct extent_buffer *leaf;
4393459931ecSChris Mason 	struct btrfs_item *item;
4394459931ecSChris Mason 	struct btrfs_item *new_item;
4395459931ecSChris Mason 	int slot;
4396ad48fd75SYan, Zheng 	char *buf;
4397459931ecSChris Mason 	u32 nritems;
4398ad48fd75SYan, Zheng 	u32 item_size;
4399459931ecSChris Mason 	u32 orig_offset;
4400459931ecSChris Mason 	struct btrfs_disk_key disk_key;
4401459931ecSChris Mason 
4402459931ecSChris Mason 	leaf = path->nodes[0];
44032ff7e61eSJeff Mahoney 	BUG_ON(btrfs_leaf_free_space(fs_info, leaf) < sizeof(struct btrfs_item));
4404b9473439SChris Mason 
4405b4ce94deSChris Mason 	btrfs_set_path_blocking(path);
4406b4ce94deSChris Mason 
4407dd3cc16bSRoss Kirk 	item = btrfs_item_nr(path->slots[0]);
4408459931ecSChris Mason 	orig_offset = btrfs_item_offset(leaf, item);
4409459931ecSChris Mason 	item_size = btrfs_item_size(leaf, item);
4410459931ecSChris Mason 
4411459931ecSChris Mason 	buf = kmalloc(item_size, GFP_NOFS);
4412ad48fd75SYan, Zheng 	if (!buf)
4413ad48fd75SYan, Zheng 		return -ENOMEM;
4414ad48fd75SYan, Zheng 
4415459931ecSChris Mason 	read_extent_buffer(leaf, buf, btrfs_item_ptr_offset(leaf,
4416459931ecSChris Mason 			    path->slots[0]), item_size);
4417ad48fd75SYan, Zheng 
4418459931ecSChris Mason 	slot = path->slots[0] + 1;
4419459931ecSChris Mason 	nritems = btrfs_header_nritems(leaf);
4420459931ecSChris Mason 	if (slot != nritems) {
4421459931ecSChris Mason 		/* shift the items */
4422459931ecSChris Mason 		memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + 1),
4423459931ecSChris Mason 				btrfs_item_nr_offset(slot),
4424459931ecSChris Mason 				(nritems - slot) * sizeof(struct btrfs_item));
4425459931ecSChris Mason 	}
4426459931ecSChris Mason 
4427459931ecSChris Mason 	btrfs_cpu_key_to_disk(&disk_key, new_key);
4428459931ecSChris Mason 	btrfs_set_item_key(leaf, &disk_key, slot);
4429459931ecSChris Mason 
4430dd3cc16bSRoss Kirk 	new_item = btrfs_item_nr(slot);
4431459931ecSChris Mason 
4432459931ecSChris Mason 	btrfs_set_item_offset(leaf, new_item, orig_offset);
4433459931ecSChris Mason 	btrfs_set_item_size(leaf, new_item, item_size - split_offset);
4434459931ecSChris Mason 
4435459931ecSChris Mason 	btrfs_set_item_offset(leaf, item,
4436459931ecSChris Mason 			      orig_offset + item_size - split_offset);
4437459931ecSChris Mason 	btrfs_set_item_size(leaf, item, split_offset);
4438459931ecSChris Mason 
4439459931ecSChris Mason 	btrfs_set_header_nritems(leaf, nritems + 1);
4440459931ecSChris Mason 
4441459931ecSChris Mason 	/* write the data for the start of the original item */
4442459931ecSChris Mason 	write_extent_buffer(leaf, buf,
4443459931ecSChris Mason 			    btrfs_item_ptr_offset(leaf, path->slots[0]),
4444459931ecSChris Mason 			    split_offset);
4445459931ecSChris Mason 
4446459931ecSChris Mason 	/* write the data for the new item */
4447459931ecSChris Mason 	write_extent_buffer(leaf, buf + split_offset,
4448459931ecSChris Mason 			    btrfs_item_ptr_offset(leaf, slot),
4449459931ecSChris Mason 			    item_size - split_offset);
4450459931ecSChris Mason 	btrfs_mark_buffer_dirty(leaf);
4451459931ecSChris Mason 
44522ff7e61eSJeff Mahoney 	BUG_ON(btrfs_leaf_free_space(fs_info, leaf) < 0);
4453459931ecSChris Mason 	kfree(buf);
4454ad48fd75SYan, Zheng 	return 0;
4455ad48fd75SYan, Zheng }
4456ad48fd75SYan, Zheng 
4457ad48fd75SYan, Zheng /*
4458ad48fd75SYan, Zheng  * This function splits a single item into two items,
4459ad48fd75SYan, Zheng  * giving 'new_key' to the new item and splitting the
4460ad48fd75SYan, Zheng  * old one at split_offset (from the start of the item).
4461ad48fd75SYan, Zheng  *
4462ad48fd75SYan, Zheng  * The path may be released by this operation.  After
4463ad48fd75SYan, Zheng  * the split, the path is pointing to the old item.  The
4464ad48fd75SYan, Zheng  * new item is going to be in the same node as the old one.
4465ad48fd75SYan, Zheng  *
4466ad48fd75SYan, Zheng  * Note, the item being split must be smaller enough to live alone on
4467ad48fd75SYan, Zheng  * a tree block with room for one extra struct btrfs_item
4468ad48fd75SYan, Zheng  *
4469ad48fd75SYan, Zheng  * This allows us to split the item in place, keeping a lock on the
4470ad48fd75SYan, Zheng  * leaf the entire time.
4471ad48fd75SYan, Zheng  */
4472ad48fd75SYan, Zheng int btrfs_split_item(struct btrfs_trans_handle *trans,
4473ad48fd75SYan, Zheng 		     struct btrfs_root *root,
4474ad48fd75SYan, Zheng 		     struct btrfs_path *path,
4475310712b2SOmar Sandoval 		     const struct btrfs_key *new_key,
4476ad48fd75SYan, Zheng 		     unsigned long split_offset)
4477ad48fd75SYan, Zheng {
4478ad48fd75SYan, Zheng 	int ret;
4479ad48fd75SYan, Zheng 	ret = setup_leaf_for_split(trans, root, path,
4480ad48fd75SYan, Zheng 				   sizeof(struct btrfs_item));
4481ad48fd75SYan, Zheng 	if (ret)
4482459931ecSChris Mason 		return ret;
4483ad48fd75SYan, Zheng 
44844961e293SDavid Sterba 	ret = split_item(root->fs_info, path, new_key, split_offset);
4485ad48fd75SYan, Zheng 	return ret;
4486ad48fd75SYan, Zheng }
4487ad48fd75SYan, Zheng 
4488ad48fd75SYan, Zheng /*
4489ad48fd75SYan, Zheng  * This function duplicate a item, giving 'new_key' to the new item.
4490ad48fd75SYan, Zheng  * It guarantees both items live in the same tree leaf and the new item
4491ad48fd75SYan, Zheng  * is contiguous with the original item.
4492ad48fd75SYan, Zheng  *
4493ad48fd75SYan, Zheng  * This allows us to split file extent in place, keeping a lock on the
4494ad48fd75SYan, Zheng  * leaf the entire time.
4495ad48fd75SYan, Zheng  */
4496ad48fd75SYan, Zheng int btrfs_duplicate_item(struct btrfs_trans_handle *trans,
4497ad48fd75SYan, Zheng 			 struct btrfs_root *root,
4498ad48fd75SYan, Zheng 			 struct btrfs_path *path,
4499310712b2SOmar Sandoval 			 const struct btrfs_key *new_key)
4500ad48fd75SYan, Zheng {
4501ad48fd75SYan, Zheng 	struct extent_buffer *leaf;
4502ad48fd75SYan, Zheng 	int ret;
4503ad48fd75SYan, Zheng 	u32 item_size;
4504ad48fd75SYan, Zheng 
4505ad48fd75SYan, Zheng 	leaf = path->nodes[0];
4506ad48fd75SYan, Zheng 	item_size = btrfs_item_size_nr(leaf, path->slots[0]);
4507ad48fd75SYan, Zheng 	ret = setup_leaf_for_split(trans, root, path,
4508ad48fd75SYan, Zheng 				   item_size + sizeof(struct btrfs_item));
4509ad48fd75SYan, Zheng 	if (ret)
4510ad48fd75SYan, Zheng 		return ret;
4511ad48fd75SYan, Zheng 
4512ad48fd75SYan, Zheng 	path->slots[0]++;
4513afe5fea7STsutomu Itoh 	setup_items_for_insert(root, path, new_key, &item_size,
4514ad48fd75SYan, Zheng 			       item_size, item_size +
4515ad48fd75SYan, Zheng 			       sizeof(struct btrfs_item), 1);
4516ad48fd75SYan, Zheng 	leaf = path->nodes[0];
4517ad48fd75SYan, Zheng 	memcpy_extent_buffer(leaf,
4518ad48fd75SYan, Zheng 			     btrfs_item_ptr_offset(leaf, path->slots[0]),
4519ad48fd75SYan, Zheng 			     btrfs_item_ptr_offset(leaf, path->slots[0] - 1),
4520ad48fd75SYan, Zheng 			     item_size);
4521ad48fd75SYan, Zheng 	return 0;
4522459931ecSChris Mason }
4523459931ecSChris Mason 
4524459931ecSChris Mason /*
4525d352ac68SChris Mason  * make the item pointed to by the path smaller.  new_size indicates
4526d352ac68SChris Mason  * how small to make it, and from_end tells us if we just chop bytes
4527d352ac68SChris Mason  * off the end of the item or if we shift the item to chop bytes off
4528d352ac68SChris Mason  * the front.
4529d352ac68SChris Mason  */
45302ff7e61eSJeff Mahoney void btrfs_truncate_item(struct btrfs_fs_info *fs_info,
45312ff7e61eSJeff Mahoney 			 struct btrfs_path *path, u32 new_size, int from_end)
4532b18c6685SChris Mason {
4533b18c6685SChris Mason 	int slot;
45345f39d397SChris Mason 	struct extent_buffer *leaf;
45355f39d397SChris Mason 	struct btrfs_item *item;
4536b18c6685SChris Mason 	u32 nritems;
4537b18c6685SChris Mason 	unsigned int data_end;
4538b18c6685SChris Mason 	unsigned int old_data_start;
4539b18c6685SChris Mason 	unsigned int old_size;
4540b18c6685SChris Mason 	unsigned int size_diff;
4541b18c6685SChris Mason 	int i;
4542cfed81a0SChris Mason 	struct btrfs_map_token token;
4543cfed81a0SChris Mason 
4544cfed81a0SChris Mason 	btrfs_init_map_token(&token);
4545b18c6685SChris Mason 
45465f39d397SChris Mason 	leaf = path->nodes[0];
4547179e29e4SChris Mason 	slot = path->slots[0];
4548179e29e4SChris Mason 
4549179e29e4SChris Mason 	old_size = btrfs_item_size_nr(leaf, slot);
4550179e29e4SChris Mason 	if (old_size == new_size)
4551143bede5SJeff Mahoney 		return;
4552b18c6685SChris Mason 
45535f39d397SChris Mason 	nritems = btrfs_header_nritems(leaf);
45542ff7e61eSJeff Mahoney 	data_end = leaf_data_end(fs_info, leaf);
4555b18c6685SChris Mason 
45565f39d397SChris Mason 	old_data_start = btrfs_item_offset_nr(leaf, slot);
4557179e29e4SChris Mason 
4558b18c6685SChris Mason 	size_diff = old_size - new_size;
4559b18c6685SChris Mason 
4560b18c6685SChris Mason 	BUG_ON(slot < 0);
4561b18c6685SChris Mason 	BUG_ON(slot >= nritems);
4562b18c6685SChris Mason 
4563b18c6685SChris Mason 	/*
4564b18c6685SChris Mason 	 * item0..itemN ... dataN.offset..dataN.size .. data0.size
4565b18c6685SChris Mason 	 */
4566b18c6685SChris Mason 	/* first correct the data pointers */
4567b18c6685SChris Mason 	for (i = slot; i < nritems; i++) {
45685f39d397SChris Mason 		u32 ioff;
4569dd3cc16bSRoss Kirk 		item = btrfs_item_nr(i);
4570db94535dSChris Mason 
4571cfed81a0SChris Mason 		ioff = btrfs_token_item_offset(leaf, item, &token);
4572cfed81a0SChris Mason 		btrfs_set_token_item_offset(leaf, item,
4573cfed81a0SChris Mason 					    ioff + size_diff, &token);
4574b18c6685SChris Mason 	}
4575db94535dSChris Mason 
4576b18c6685SChris Mason 	/* shift the data */
4577179e29e4SChris Mason 	if (from_end) {
45783d9ec8c4SNikolay Borisov 		memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET +
45793d9ec8c4SNikolay Borisov 			      data_end + size_diff, BTRFS_LEAF_DATA_OFFSET +
4580b18c6685SChris Mason 			      data_end, old_data_start + new_size - data_end);
4581179e29e4SChris Mason 	} else {
4582179e29e4SChris Mason 		struct btrfs_disk_key disk_key;
4583179e29e4SChris Mason 		u64 offset;
4584179e29e4SChris Mason 
4585179e29e4SChris Mason 		btrfs_item_key(leaf, &disk_key, slot);
4586179e29e4SChris Mason 
4587179e29e4SChris Mason 		if (btrfs_disk_key_type(&disk_key) == BTRFS_EXTENT_DATA_KEY) {
4588179e29e4SChris Mason 			unsigned long ptr;
4589179e29e4SChris Mason 			struct btrfs_file_extent_item *fi;
4590179e29e4SChris Mason 
4591179e29e4SChris Mason 			fi = btrfs_item_ptr(leaf, slot,
4592179e29e4SChris Mason 					    struct btrfs_file_extent_item);
4593179e29e4SChris Mason 			fi = (struct btrfs_file_extent_item *)(
4594179e29e4SChris Mason 			     (unsigned long)fi - size_diff);
4595179e29e4SChris Mason 
4596179e29e4SChris Mason 			if (btrfs_file_extent_type(leaf, fi) ==
4597179e29e4SChris Mason 			    BTRFS_FILE_EXTENT_INLINE) {
4598179e29e4SChris Mason 				ptr = btrfs_item_ptr_offset(leaf, slot);
4599179e29e4SChris Mason 				memmove_extent_buffer(leaf, ptr,
4600179e29e4SChris Mason 				      (unsigned long)fi,
46017ec20afbSDavid Sterba 				      BTRFS_FILE_EXTENT_INLINE_DATA_START);
4602179e29e4SChris Mason 			}
4603179e29e4SChris Mason 		}
4604179e29e4SChris Mason 
46053d9ec8c4SNikolay Borisov 		memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET +
46063d9ec8c4SNikolay Borisov 			      data_end + size_diff, BTRFS_LEAF_DATA_OFFSET +
4607179e29e4SChris Mason 			      data_end, old_data_start - data_end);
4608179e29e4SChris Mason 
4609179e29e4SChris Mason 		offset = btrfs_disk_key_offset(&disk_key);
4610179e29e4SChris Mason 		btrfs_set_disk_key_offset(&disk_key, offset + size_diff);
4611179e29e4SChris Mason 		btrfs_set_item_key(leaf, &disk_key, slot);
4612179e29e4SChris Mason 		if (slot == 0)
4613b167fa91SNikolay Borisov 			fixup_low_keys(path, &disk_key, 1);
4614179e29e4SChris Mason 	}
46155f39d397SChris Mason 
4616dd3cc16bSRoss Kirk 	item = btrfs_item_nr(slot);
46175f39d397SChris Mason 	btrfs_set_item_size(leaf, item, new_size);
46185f39d397SChris Mason 	btrfs_mark_buffer_dirty(leaf);
4619b18c6685SChris Mason 
46202ff7e61eSJeff Mahoney 	if (btrfs_leaf_free_space(fs_info, leaf) < 0) {
4621a4f78750SDavid Sterba 		btrfs_print_leaf(leaf);
4622b18c6685SChris Mason 		BUG();
46235f39d397SChris Mason 	}
4624b18c6685SChris Mason }
4625b18c6685SChris Mason 
4626d352ac68SChris Mason /*
46278f69dbd2SStefan Behrens  * make the item pointed to by the path bigger, data_size is the added size.
4628d352ac68SChris Mason  */
46292ff7e61eSJeff Mahoney void btrfs_extend_item(struct btrfs_fs_info *fs_info, struct btrfs_path *path,
46305f39d397SChris Mason 		       u32 data_size)
46316567e837SChris Mason {
46326567e837SChris Mason 	int slot;
46335f39d397SChris Mason 	struct extent_buffer *leaf;
46345f39d397SChris Mason 	struct btrfs_item *item;
46356567e837SChris Mason 	u32 nritems;
46366567e837SChris Mason 	unsigned int data_end;
46376567e837SChris Mason 	unsigned int old_data;
46386567e837SChris Mason 	unsigned int old_size;
46396567e837SChris Mason 	int i;
4640cfed81a0SChris Mason 	struct btrfs_map_token token;
4641cfed81a0SChris Mason 
4642cfed81a0SChris Mason 	btrfs_init_map_token(&token);
46436567e837SChris Mason 
46445f39d397SChris Mason 	leaf = path->nodes[0];
46456567e837SChris Mason 
46465f39d397SChris Mason 	nritems = btrfs_header_nritems(leaf);
46472ff7e61eSJeff Mahoney 	data_end = leaf_data_end(fs_info, leaf);
46486567e837SChris Mason 
46492ff7e61eSJeff Mahoney 	if (btrfs_leaf_free_space(fs_info, leaf) < data_size) {
4650a4f78750SDavid Sterba 		btrfs_print_leaf(leaf);
46516567e837SChris Mason 		BUG();
46525f39d397SChris Mason 	}
46536567e837SChris Mason 	slot = path->slots[0];
46545f39d397SChris Mason 	old_data = btrfs_item_end_nr(leaf, slot);
46556567e837SChris Mason 
46566567e837SChris Mason 	BUG_ON(slot < 0);
46573326d1b0SChris Mason 	if (slot >= nritems) {
4658a4f78750SDavid Sterba 		btrfs_print_leaf(leaf);
46590b246afaSJeff Mahoney 		btrfs_crit(fs_info, "slot %d too large, nritems %d",
4660d397712bSChris Mason 			   slot, nritems);
46613326d1b0SChris Mason 		BUG_ON(1);
46623326d1b0SChris Mason 	}
46636567e837SChris Mason 
46646567e837SChris Mason 	/*
46656567e837SChris Mason 	 * item0..itemN ... dataN.offset..dataN.size .. data0.size
46666567e837SChris Mason 	 */
46676567e837SChris Mason 	/* first correct the data pointers */
46686567e837SChris Mason 	for (i = slot; i < nritems; i++) {
46695f39d397SChris Mason 		u32 ioff;
4670dd3cc16bSRoss Kirk 		item = btrfs_item_nr(i);
4671db94535dSChris Mason 
4672cfed81a0SChris Mason 		ioff = btrfs_token_item_offset(leaf, item, &token);
4673cfed81a0SChris Mason 		btrfs_set_token_item_offset(leaf, item,
4674cfed81a0SChris Mason 					    ioff - data_size, &token);
46756567e837SChris Mason 	}
46765f39d397SChris Mason 
46776567e837SChris Mason 	/* shift the data */
46783d9ec8c4SNikolay Borisov 	memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET +
46793d9ec8c4SNikolay Borisov 		      data_end - data_size, BTRFS_LEAF_DATA_OFFSET +
46806567e837SChris Mason 		      data_end, old_data - data_end);
46815f39d397SChris Mason 
46826567e837SChris Mason 	data_end = old_data;
46835f39d397SChris Mason 	old_size = btrfs_item_size_nr(leaf, slot);
4684dd3cc16bSRoss Kirk 	item = btrfs_item_nr(slot);
46855f39d397SChris Mason 	btrfs_set_item_size(leaf, item, old_size + data_size);
46865f39d397SChris Mason 	btrfs_mark_buffer_dirty(leaf);
46876567e837SChris Mason 
46882ff7e61eSJeff Mahoney 	if (btrfs_leaf_free_space(fs_info, leaf) < 0) {
4689a4f78750SDavid Sterba 		btrfs_print_leaf(leaf);
46906567e837SChris Mason 		BUG();
46915f39d397SChris Mason 	}
46926567e837SChris Mason }
46936567e837SChris Mason 
469474123bd7SChris Mason /*
469544871b1bSChris Mason  * this is a helper for btrfs_insert_empty_items, the main goal here is
469644871b1bSChris Mason  * to save stack depth by doing the bulk of the work in a function
469744871b1bSChris Mason  * that doesn't call btrfs_search_slot
469874123bd7SChris Mason  */
4699afe5fea7STsutomu Itoh void setup_items_for_insert(struct btrfs_root *root, struct btrfs_path *path,
4700310712b2SOmar Sandoval 			    const struct btrfs_key *cpu_key, u32 *data_size,
470144871b1bSChris Mason 			    u32 total_data, u32 total_size, int nr)
4702be0e5c09SChris Mason {
47030b246afaSJeff Mahoney 	struct btrfs_fs_info *fs_info = root->fs_info;
47045f39d397SChris Mason 	struct btrfs_item *item;
47059c58309dSChris Mason 	int i;
47067518a238SChris Mason 	u32 nritems;
4707be0e5c09SChris Mason 	unsigned int data_end;
4708e2fa7227SChris Mason 	struct btrfs_disk_key disk_key;
470944871b1bSChris Mason 	struct extent_buffer *leaf;
471044871b1bSChris Mason 	int slot;
4711cfed81a0SChris Mason 	struct btrfs_map_token token;
4712cfed81a0SChris Mason 
471324cdc847SFilipe Manana 	if (path->slots[0] == 0) {
471424cdc847SFilipe Manana 		btrfs_cpu_key_to_disk(&disk_key, cpu_key);
4715b167fa91SNikolay Borisov 		fixup_low_keys(path, &disk_key, 1);
471624cdc847SFilipe Manana 	}
471724cdc847SFilipe Manana 	btrfs_unlock_up_safe(path, 1);
471824cdc847SFilipe Manana 
4719cfed81a0SChris Mason 	btrfs_init_map_token(&token);
4720e2fa7227SChris Mason 
47215f39d397SChris Mason 	leaf = path->nodes[0];
472244871b1bSChris Mason 	slot = path->slots[0];
472374123bd7SChris Mason 
47245f39d397SChris Mason 	nritems = btrfs_header_nritems(leaf);
47252ff7e61eSJeff Mahoney 	data_end = leaf_data_end(fs_info, leaf);
4726eb60ceacSChris Mason 
47272ff7e61eSJeff Mahoney 	if (btrfs_leaf_free_space(fs_info, leaf) < total_size) {
4728a4f78750SDavid Sterba 		btrfs_print_leaf(leaf);
47290b246afaSJeff Mahoney 		btrfs_crit(fs_info, "not enough freespace need %u have %d",
47302ff7e61eSJeff Mahoney 			   total_size, btrfs_leaf_free_space(fs_info, leaf));
4731be0e5c09SChris Mason 		BUG();
4732d4dbff95SChris Mason 	}
47335f39d397SChris Mason 
4734be0e5c09SChris Mason 	if (slot != nritems) {
47355f39d397SChris Mason 		unsigned int old_data = btrfs_item_end_nr(leaf, slot);
4736be0e5c09SChris Mason 
47375f39d397SChris Mason 		if (old_data < data_end) {
4738a4f78750SDavid Sterba 			btrfs_print_leaf(leaf);
47390b246afaSJeff Mahoney 			btrfs_crit(fs_info, "slot %d old_data %d data_end %d",
47405f39d397SChris Mason 				   slot, old_data, data_end);
47415f39d397SChris Mason 			BUG_ON(1);
47425f39d397SChris Mason 		}
4743be0e5c09SChris Mason 		/*
4744be0e5c09SChris Mason 		 * item0..itemN ... dataN.offset..dataN.size .. data0.size
4745be0e5c09SChris Mason 		 */
4746be0e5c09SChris Mason 		/* first correct the data pointers */
47470783fcfcSChris Mason 		for (i = slot; i < nritems; i++) {
47485f39d397SChris Mason 			u32 ioff;
4749db94535dSChris Mason 
4750dd3cc16bSRoss Kirk 			item = btrfs_item_nr(i);
4751cfed81a0SChris Mason 			ioff = btrfs_token_item_offset(leaf, item, &token);
4752cfed81a0SChris Mason 			btrfs_set_token_item_offset(leaf, item,
4753cfed81a0SChris Mason 						    ioff - total_data, &token);
47540783fcfcSChris Mason 		}
4755be0e5c09SChris Mason 		/* shift the items */
47569c58309dSChris Mason 		memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + nr),
47575f39d397SChris Mason 			      btrfs_item_nr_offset(slot),
47580783fcfcSChris Mason 			      (nritems - slot) * sizeof(struct btrfs_item));
4759be0e5c09SChris Mason 
4760be0e5c09SChris Mason 		/* shift the data */
47613d9ec8c4SNikolay Borisov 		memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET +
47623d9ec8c4SNikolay Borisov 			      data_end - total_data, BTRFS_LEAF_DATA_OFFSET +
4763be0e5c09SChris Mason 			      data_end, old_data - data_end);
4764be0e5c09SChris Mason 		data_end = old_data;
4765be0e5c09SChris Mason 	}
47665f39d397SChris Mason 
476762e2749eSChris Mason 	/* setup the item for the new data */
47689c58309dSChris Mason 	for (i = 0; i < nr; i++) {
47699c58309dSChris Mason 		btrfs_cpu_key_to_disk(&disk_key, cpu_key + i);
47709c58309dSChris Mason 		btrfs_set_item_key(leaf, &disk_key, slot + i);
4771dd3cc16bSRoss Kirk 		item = btrfs_item_nr(slot + i);
4772cfed81a0SChris Mason 		btrfs_set_token_item_offset(leaf, item,
4773cfed81a0SChris Mason 					    data_end - data_size[i], &token);
47749c58309dSChris Mason 		data_end -= data_size[i];
4775cfed81a0SChris Mason 		btrfs_set_token_item_size(leaf, item, data_size[i], &token);
47769c58309dSChris Mason 	}
477744871b1bSChris Mason 
47789c58309dSChris Mason 	btrfs_set_header_nritems(leaf, nritems + nr);
4779b9473439SChris Mason 	btrfs_mark_buffer_dirty(leaf);
4780aa5d6bedSChris Mason 
47812ff7e61eSJeff Mahoney 	if (btrfs_leaf_free_space(fs_info, leaf) < 0) {
4782a4f78750SDavid Sterba 		btrfs_print_leaf(leaf);
4783be0e5c09SChris Mason 		BUG();
47845f39d397SChris Mason 	}
478544871b1bSChris Mason }
478644871b1bSChris Mason 
478744871b1bSChris Mason /*
478844871b1bSChris Mason  * Given a key and some data, insert items into the tree.
478944871b1bSChris Mason  * This does all the path init required, making room in the tree if needed.
479044871b1bSChris Mason  */
479144871b1bSChris Mason int btrfs_insert_empty_items(struct btrfs_trans_handle *trans,
479244871b1bSChris Mason 			    struct btrfs_root *root,
479344871b1bSChris Mason 			    struct btrfs_path *path,
4794310712b2SOmar Sandoval 			    const struct btrfs_key *cpu_key, u32 *data_size,
479544871b1bSChris Mason 			    int nr)
479644871b1bSChris Mason {
479744871b1bSChris Mason 	int ret = 0;
479844871b1bSChris Mason 	int slot;
479944871b1bSChris Mason 	int i;
480044871b1bSChris Mason 	u32 total_size = 0;
480144871b1bSChris Mason 	u32 total_data = 0;
480244871b1bSChris Mason 
480344871b1bSChris Mason 	for (i = 0; i < nr; i++)
480444871b1bSChris Mason 		total_data += data_size[i];
480544871b1bSChris Mason 
480644871b1bSChris Mason 	total_size = total_data + (nr * sizeof(struct btrfs_item));
480744871b1bSChris Mason 	ret = btrfs_search_slot(trans, root, cpu_key, path, total_size, 1);
480844871b1bSChris Mason 	if (ret == 0)
480944871b1bSChris Mason 		return -EEXIST;
481044871b1bSChris Mason 	if (ret < 0)
4811143bede5SJeff Mahoney 		return ret;
481244871b1bSChris Mason 
481344871b1bSChris Mason 	slot = path->slots[0];
481444871b1bSChris Mason 	BUG_ON(slot < 0);
481544871b1bSChris Mason 
4816afe5fea7STsutomu Itoh 	setup_items_for_insert(root, path, cpu_key, data_size,
481744871b1bSChris Mason 			       total_data, total_size, nr);
4818143bede5SJeff Mahoney 	return 0;
481962e2749eSChris Mason }
482062e2749eSChris Mason 
482162e2749eSChris Mason /*
482262e2749eSChris Mason  * Given a key and some data, insert an item into the tree.
482362e2749eSChris Mason  * This does all the path init required, making room in the tree if needed.
482462e2749eSChris Mason  */
4825310712b2SOmar Sandoval int btrfs_insert_item(struct btrfs_trans_handle *trans, struct btrfs_root *root,
4826310712b2SOmar Sandoval 		      const struct btrfs_key *cpu_key, void *data,
4827310712b2SOmar Sandoval 		      u32 data_size)
482862e2749eSChris Mason {
482962e2749eSChris Mason 	int ret = 0;
48302c90e5d6SChris Mason 	struct btrfs_path *path;
48315f39d397SChris Mason 	struct extent_buffer *leaf;
48325f39d397SChris Mason 	unsigned long ptr;
483362e2749eSChris Mason 
48342c90e5d6SChris Mason 	path = btrfs_alloc_path();
4835db5b493aSTsutomu Itoh 	if (!path)
4836db5b493aSTsutomu Itoh 		return -ENOMEM;
48372c90e5d6SChris Mason 	ret = btrfs_insert_empty_item(trans, root, path, cpu_key, data_size);
483862e2749eSChris Mason 	if (!ret) {
48395f39d397SChris Mason 		leaf = path->nodes[0];
48405f39d397SChris Mason 		ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);
48415f39d397SChris Mason 		write_extent_buffer(leaf, data, ptr, data_size);
48425f39d397SChris Mason 		btrfs_mark_buffer_dirty(leaf);
484362e2749eSChris Mason 	}
48442c90e5d6SChris Mason 	btrfs_free_path(path);
4845aa5d6bedSChris Mason 	return ret;
4846be0e5c09SChris Mason }
4847be0e5c09SChris Mason 
484874123bd7SChris Mason /*
48495de08d7dSChris Mason  * delete the pointer from a given node.
485074123bd7SChris Mason  *
4851d352ac68SChris Mason  * the tree should have been previously balanced so the deletion does not
4852d352ac68SChris Mason  * empty a node.
485374123bd7SChris Mason  */
4854afe5fea7STsutomu Itoh static void del_ptr(struct btrfs_root *root, struct btrfs_path *path,
4855afe5fea7STsutomu Itoh 		    int level, int slot)
4856be0e5c09SChris Mason {
48575f39d397SChris Mason 	struct extent_buffer *parent = path->nodes[level];
48587518a238SChris Mason 	u32 nritems;
4859f3ea38daSJan Schmidt 	int ret;
4860be0e5c09SChris Mason 
48615f39d397SChris Mason 	nritems = btrfs_header_nritems(parent);
4862be0e5c09SChris Mason 	if (slot != nritems - 1) {
4863bf1d3425SDavid Sterba 		if (level) {
4864bf1d3425SDavid Sterba 			ret = tree_mod_log_insert_move(parent, slot, slot + 1,
4865a446a979SDavid Sterba 					nritems - slot - 1);
4866bf1d3425SDavid Sterba 			BUG_ON(ret < 0);
4867bf1d3425SDavid Sterba 		}
48685f39d397SChris Mason 		memmove_extent_buffer(parent,
48695f39d397SChris Mason 			      btrfs_node_key_ptr_offset(slot),
48705f39d397SChris Mason 			      btrfs_node_key_ptr_offset(slot + 1),
4871d6025579SChris Mason 			      sizeof(struct btrfs_key_ptr) *
4872d6025579SChris Mason 			      (nritems - slot - 1));
487357ba86c0SChris Mason 	} else if (level) {
4874e09c2efeSDavid Sterba 		ret = tree_mod_log_insert_key(parent, slot, MOD_LOG_KEY_REMOVE,
4875e09c2efeSDavid Sterba 				GFP_NOFS);
487657ba86c0SChris Mason 		BUG_ON(ret < 0);
4877be0e5c09SChris Mason 	}
4878f3ea38daSJan Schmidt 
48797518a238SChris Mason 	nritems--;
48805f39d397SChris Mason 	btrfs_set_header_nritems(parent, nritems);
48817518a238SChris Mason 	if (nritems == 0 && parent == root->node) {
48825f39d397SChris Mason 		BUG_ON(btrfs_header_level(root->node) != 1);
4883eb60ceacSChris Mason 		/* just turn the root into a leaf and break */
48845f39d397SChris Mason 		btrfs_set_header_level(root->node, 0);
4885bb803951SChris Mason 	} else if (slot == 0) {
48865f39d397SChris Mason 		struct btrfs_disk_key disk_key;
48875f39d397SChris Mason 
48885f39d397SChris Mason 		btrfs_node_key(parent, &disk_key, 0);
4889b167fa91SNikolay Borisov 		fixup_low_keys(path, &disk_key, level + 1);
4890be0e5c09SChris Mason 	}
4891d6025579SChris Mason 	btrfs_mark_buffer_dirty(parent);
4892be0e5c09SChris Mason }
4893be0e5c09SChris Mason 
489474123bd7SChris Mason /*
4895323ac95bSChris Mason  * a helper function to delete the leaf pointed to by path->slots[1] and
48965d4f98a2SYan Zheng  * path->nodes[1].
4897323ac95bSChris Mason  *
4898323ac95bSChris Mason  * This deletes the pointer in path->nodes[1] and frees the leaf
4899323ac95bSChris Mason  * block extent.  zero is returned if it all worked out, < 0 otherwise.
4900323ac95bSChris Mason  *
4901323ac95bSChris Mason  * The path must have already been setup for deleting the leaf, including
4902323ac95bSChris Mason  * all the proper balancing.  path->nodes[1] must be locked.
4903323ac95bSChris Mason  */
4904143bede5SJeff Mahoney static noinline void btrfs_del_leaf(struct btrfs_trans_handle *trans,
4905323ac95bSChris Mason 				    struct btrfs_root *root,
49065d4f98a2SYan Zheng 				    struct btrfs_path *path,
49075d4f98a2SYan Zheng 				    struct extent_buffer *leaf)
4908323ac95bSChris Mason {
49095d4f98a2SYan Zheng 	WARN_ON(btrfs_header_generation(leaf) != trans->transid);
4910afe5fea7STsutomu Itoh 	del_ptr(root, path, 1, path->slots[1]);
4911323ac95bSChris Mason 
49124d081c41SChris Mason 	/*
49134d081c41SChris Mason 	 * btrfs_free_extent is expensive, we want to make sure we
49144d081c41SChris Mason 	 * aren't holding any locks when we call it
49154d081c41SChris Mason 	 */
49164d081c41SChris Mason 	btrfs_unlock_up_safe(path, 0);
49174d081c41SChris Mason 
4918f0486c68SYan, Zheng 	root_sub_used(root, leaf->len);
4919f0486c68SYan, Zheng 
49203083ee2eSJosef Bacik 	extent_buffer_get(leaf);
49215581a51aSJan Schmidt 	btrfs_free_tree_block(trans, root, leaf, 0, 1);
49223083ee2eSJosef Bacik 	free_extent_buffer_stale(leaf);
4923323ac95bSChris Mason }
4924323ac95bSChris Mason /*
492574123bd7SChris Mason  * delete the item at the leaf level in path.  If that empties
492674123bd7SChris Mason  * the leaf, remove it from the tree
492774123bd7SChris Mason  */
492885e21bacSChris Mason int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root,
492985e21bacSChris Mason 		    struct btrfs_path *path, int slot, int nr)
4930be0e5c09SChris Mason {
49310b246afaSJeff Mahoney 	struct btrfs_fs_info *fs_info = root->fs_info;
49325f39d397SChris Mason 	struct extent_buffer *leaf;
49335f39d397SChris Mason 	struct btrfs_item *item;
4934ce0eac2aSAlexandru Moise 	u32 last_off;
4935ce0eac2aSAlexandru Moise 	u32 dsize = 0;
4936aa5d6bedSChris Mason 	int ret = 0;
4937aa5d6bedSChris Mason 	int wret;
493885e21bacSChris Mason 	int i;
49397518a238SChris Mason 	u32 nritems;
4940cfed81a0SChris Mason 	struct btrfs_map_token token;
4941cfed81a0SChris Mason 
4942cfed81a0SChris Mason 	btrfs_init_map_token(&token);
4943be0e5c09SChris Mason 
49445f39d397SChris Mason 	leaf = path->nodes[0];
494585e21bacSChris Mason 	last_off = btrfs_item_offset_nr(leaf, slot + nr - 1);
494685e21bacSChris Mason 
494785e21bacSChris Mason 	for (i = 0; i < nr; i++)
494885e21bacSChris Mason 		dsize += btrfs_item_size_nr(leaf, slot + i);
494985e21bacSChris Mason 
49505f39d397SChris Mason 	nritems = btrfs_header_nritems(leaf);
4951be0e5c09SChris Mason 
495285e21bacSChris Mason 	if (slot + nr != nritems) {
49532ff7e61eSJeff Mahoney 		int data_end = leaf_data_end(fs_info, leaf);
49545f39d397SChris Mason 
49553d9ec8c4SNikolay Borisov 		memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET +
4956d6025579SChris Mason 			      data_end + dsize,
49573d9ec8c4SNikolay Borisov 			      BTRFS_LEAF_DATA_OFFSET + data_end,
495885e21bacSChris Mason 			      last_off - data_end);
49595f39d397SChris Mason 
496085e21bacSChris Mason 		for (i = slot + nr; i < nritems; i++) {
49615f39d397SChris Mason 			u32 ioff;
4962db94535dSChris Mason 
4963dd3cc16bSRoss Kirk 			item = btrfs_item_nr(i);
4964cfed81a0SChris Mason 			ioff = btrfs_token_item_offset(leaf, item, &token);
4965cfed81a0SChris Mason 			btrfs_set_token_item_offset(leaf, item,
4966cfed81a0SChris Mason 						    ioff + dsize, &token);
49670783fcfcSChris Mason 		}
4968db94535dSChris Mason 
49695f39d397SChris Mason 		memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot),
497085e21bacSChris Mason 			      btrfs_item_nr_offset(slot + nr),
49710783fcfcSChris Mason 			      sizeof(struct btrfs_item) *
497285e21bacSChris Mason 			      (nritems - slot - nr));
4973be0e5c09SChris Mason 	}
497485e21bacSChris Mason 	btrfs_set_header_nritems(leaf, nritems - nr);
497585e21bacSChris Mason 	nritems -= nr;
49765f39d397SChris Mason 
497774123bd7SChris Mason 	/* delete the leaf if we've emptied it */
49787518a238SChris Mason 	if (nritems == 0) {
49795f39d397SChris Mason 		if (leaf == root->node) {
49805f39d397SChris Mason 			btrfs_set_header_level(leaf, 0);
49819a8dd150SChris Mason 		} else {
4982f0486c68SYan, Zheng 			btrfs_set_path_blocking(path);
49837c302b49SDavid Sterba 			clean_tree_block(fs_info, leaf);
4984143bede5SJeff Mahoney 			btrfs_del_leaf(trans, root, path, leaf);
49859a8dd150SChris Mason 		}
4986be0e5c09SChris Mason 	} else {
49877518a238SChris Mason 		int used = leaf_space_used(leaf, 0, nritems);
4988aa5d6bedSChris Mason 		if (slot == 0) {
49895f39d397SChris Mason 			struct btrfs_disk_key disk_key;
49905f39d397SChris Mason 
49915f39d397SChris Mason 			btrfs_item_key(leaf, &disk_key, 0);
4992b167fa91SNikolay Borisov 			fixup_low_keys(path, &disk_key, 1);
4993aa5d6bedSChris Mason 		}
4994aa5d6bedSChris Mason 
499574123bd7SChris Mason 		/* delete the leaf if it is mostly empty */
49960b246afaSJeff Mahoney 		if (used < BTRFS_LEAF_DATA_SIZE(fs_info) / 3) {
4997be0e5c09SChris Mason 			/* push_leaf_left fixes the path.
4998be0e5c09SChris Mason 			 * make sure the path still points to our leaf
4999be0e5c09SChris Mason 			 * for possible call to del_ptr below
5000be0e5c09SChris Mason 			 */
50014920c9acSChris Mason 			slot = path->slots[1];
50025f39d397SChris Mason 			extent_buffer_get(leaf);
50035f39d397SChris Mason 
5004b9473439SChris Mason 			btrfs_set_path_blocking(path);
500599d8f83cSChris Mason 			wret = push_leaf_left(trans, root, path, 1, 1,
500699d8f83cSChris Mason 					      1, (u32)-1);
500754aa1f4dSChris Mason 			if (wret < 0 && wret != -ENOSPC)
5008aa5d6bedSChris Mason 				ret = wret;
50095f39d397SChris Mason 
50105f39d397SChris Mason 			if (path->nodes[0] == leaf &&
50115f39d397SChris Mason 			    btrfs_header_nritems(leaf)) {
501299d8f83cSChris Mason 				wret = push_leaf_right(trans, root, path, 1,
501399d8f83cSChris Mason 						       1, 1, 0);
501454aa1f4dSChris Mason 				if (wret < 0 && wret != -ENOSPC)
5015aa5d6bedSChris Mason 					ret = wret;
5016aa5d6bedSChris Mason 			}
50175f39d397SChris Mason 
50185f39d397SChris Mason 			if (btrfs_header_nritems(leaf) == 0) {
5019323ac95bSChris Mason 				path->slots[1] = slot;
5020143bede5SJeff Mahoney 				btrfs_del_leaf(trans, root, path, leaf);
50215f39d397SChris Mason 				free_extent_buffer(leaf);
5022143bede5SJeff Mahoney 				ret = 0;
50235de08d7dSChris Mason 			} else {
5024925baeddSChris Mason 				/* if we're still in the path, make sure
5025925baeddSChris Mason 				 * we're dirty.  Otherwise, one of the
5026925baeddSChris Mason 				 * push_leaf functions must have already
5027925baeddSChris Mason 				 * dirtied this buffer
5028925baeddSChris Mason 				 */
5029925baeddSChris Mason 				if (path->nodes[0] == leaf)
50305f39d397SChris Mason 					btrfs_mark_buffer_dirty(leaf);
50315f39d397SChris Mason 				free_extent_buffer(leaf);
5032be0e5c09SChris Mason 			}
5033d5719762SChris Mason 		} else {
50345f39d397SChris Mason 			btrfs_mark_buffer_dirty(leaf);
5035be0e5c09SChris Mason 		}
50369a8dd150SChris Mason 	}
5037aa5d6bedSChris Mason 	return ret;
50389a8dd150SChris Mason }
50399a8dd150SChris Mason 
504097571fd0SChris Mason /*
5041925baeddSChris Mason  * search the tree again to find a leaf with lesser keys
50427bb86316SChris Mason  * returns 0 if it found something or 1 if there are no lesser leaves.
50437bb86316SChris Mason  * returns < 0 on io errors.
5044d352ac68SChris Mason  *
5045d352ac68SChris Mason  * This may release the path, and so you may lose any locks held at the
5046d352ac68SChris Mason  * time you call it.
50477bb86316SChris Mason  */
504816e7549fSJosef Bacik int btrfs_prev_leaf(struct btrfs_root *root, struct btrfs_path *path)
50497bb86316SChris Mason {
5050925baeddSChris Mason 	struct btrfs_key key;
5051925baeddSChris Mason 	struct btrfs_disk_key found_key;
5052925baeddSChris Mason 	int ret;
50537bb86316SChris Mason 
5054925baeddSChris Mason 	btrfs_item_key_to_cpu(path->nodes[0], &key, 0);
5055925baeddSChris Mason 
5056e8b0d724SFilipe David Borba Manana 	if (key.offset > 0) {
5057925baeddSChris Mason 		key.offset--;
5058e8b0d724SFilipe David Borba Manana 	} else if (key.type > 0) {
5059925baeddSChris Mason 		key.type--;
5060e8b0d724SFilipe David Borba Manana 		key.offset = (u64)-1;
5061e8b0d724SFilipe David Borba Manana 	} else if (key.objectid > 0) {
5062925baeddSChris Mason 		key.objectid--;
5063e8b0d724SFilipe David Borba Manana 		key.type = (u8)-1;
5064e8b0d724SFilipe David Borba Manana 		key.offset = (u64)-1;
5065e8b0d724SFilipe David Borba Manana 	} else {
50667bb86316SChris Mason 		return 1;
5067e8b0d724SFilipe David Borba Manana 	}
50687bb86316SChris Mason 
5069b3b4aa74SDavid Sterba 	btrfs_release_path(path);
5070925baeddSChris Mason 	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
5071925baeddSChris Mason 	if (ret < 0)
5072925baeddSChris Mason 		return ret;
5073925baeddSChris Mason 	btrfs_item_key(path->nodes[0], &found_key, 0);
5074925baeddSChris Mason 	ret = comp_keys(&found_key, &key);
5075337c6f68SFilipe Manana 	/*
5076337c6f68SFilipe Manana 	 * We might have had an item with the previous key in the tree right
5077337c6f68SFilipe Manana 	 * before we released our path. And after we released our path, that
5078337c6f68SFilipe Manana 	 * item might have been pushed to the first slot (0) of the leaf we
5079337c6f68SFilipe Manana 	 * were holding due to a tree balance. Alternatively, an item with the
5080337c6f68SFilipe Manana 	 * previous key can exist as the only element of a leaf (big fat item).
5081337c6f68SFilipe Manana 	 * Therefore account for these 2 cases, so that our callers (like
5082337c6f68SFilipe Manana 	 * btrfs_previous_item) don't miss an existing item with a key matching
5083337c6f68SFilipe Manana 	 * the previous key we computed above.
5084337c6f68SFilipe Manana 	 */
5085337c6f68SFilipe Manana 	if (ret <= 0)
50867bb86316SChris Mason 		return 0;
5087925baeddSChris Mason 	return 1;
50887bb86316SChris Mason }
50897bb86316SChris Mason 
50903f157a2fSChris Mason /*
50913f157a2fSChris Mason  * A helper function to walk down the tree starting at min_key, and looking
5092de78b51aSEric Sandeen  * for nodes or leaves that are have a minimum transaction id.
5093de78b51aSEric Sandeen  * This is used by the btree defrag code, and tree logging
50943f157a2fSChris Mason  *
50953f157a2fSChris Mason  * This does not cow, but it does stuff the starting key it finds back
50963f157a2fSChris Mason  * into min_key, so you can call btrfs_search_slot with cow=1 on the
50973f157a2fSChris Mason  * key and get a writable path.
50983f157a2fSChris Mason  *
50993f157a2fSChris Mason  * This honors path->lowest_level to prevent descent past a given level
51003f157a2fSChris Mason  * of the tree.
51013f157a2fSChris Mason  *
5102d352ac68SChris Mason  * min_trans indicates the oldest transaction that you are interested
5103d352ac68SChris Mason  * in walking through.  Any nodes or leaves older than min_trans are
5104d352ac68SChris Mason  * skipped over (without reading them).
5105d352ac68SChris Mason  *
51063f157a2fSChris Mason  * returns zero if something useful was found, < 0 on error and 1 if there
51073f157a2fSChris Mason  * was nothing in the tree that matched the search criteria.
51083f157a2fSChris Mason  */
51093f157a2fSChris Mason int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key,
5110de78b51aSEric Sandeen 			 struct btrfs_path *path,
51113f157a2fSChris Mason 			 u64 min_trans)
51123f157a2fSChris Mason {
51132ff7e61eSJeff Mahoney 	struct btrfs_fs_info *fs_info = root->fs_info;
51143f157a2fSChris Mason 	struct extent_buffer *cur;
51153f157a2fSChris Mason 	struct btrfs_key found_key;
51163f157a2fSChris Mason 	int slot;
51179652480bSYan 	int sret;
51183f157a2fSChris Mason 	u32 nritems;
51193f157a2fSChris Mason 	int level;
51203f157a2fSChris Mason 	int ret = 1;
5121f98de9b9SFilipe Manana 	int keep_locks = path->keep_locks;
51223f157a2fSChris Mason 
5123f98de9b9SFilipe Manana 	path->keep_locks = 1;
51243f157a2fSChris Mason again:
5125bd681513SChris Mason 	cur = btrfs_read_lock_root_node(root);
51263f157a2fSChris Mason 	level = btrfs_header_level(cur);
5127e02119d5SChris Mason 	WARN_ON(path->nodes[level]);
51283f157a2fSChris Mason 	path->nodes[level] = cur;
5129bd681513SChris Mason 	path->locks[level] = BTRFS_READ_LOCK;
51303f157a2fSChris Mason 
51313f157a2fSChris Mason 	if (btrfs_header_generation(cur) < min_trans) {
51323f157a2fSChris Mason 		ret = 1;
51333f157a2fSChris Mason 		goto out;
51343f157a2fSChris Mason 	}
51353f157a2fSChris Mason 	while (1) {
51363f157a2fSChris Mason 		nritems = btrfs_header_nritems(cur);
51373f157a2fSChris Mason 		level = btrfs_header_level(cur);
5138a74b35ecSNikolay Borisov 		sret = btrfs_bin_search(cur, min_key, level, &slot);
51393f157a2fSChris Mason 
5140323ac95bSChris Mason 		/* at the lowest level, we're done, setup the path and exit */
5141323ac95bSChris Mason 		if (level == path->lowest_level) {
5142e02119d5SChris Mason 			if (slot >= nritems)
5143e02119d5SChris Mason 				goto find_next_key;
51443f157a2fSChris Mason 			ret = 0;
51453f157a2fSChris Mason 			path->slots[level] = slot;
51463f157a2fSChris Mason 			btrfs_item_key_to_cpu(cur, &found_key, slot);
51473f157a2fSChris Mason 			goto out;
51483f157a2fSChris Mason 		}
51499652480bSYan 		if (sret && slot > 0)
51509652480bSYan 			slot--;
51513f157a2fSChris Mason 		/*
5152de78b51aSEric Sandeen 		 * check this node pointer against the min_trans parameters.
5153de78b51aSEric Sandeen 		 * If it is too old, old, skip to the next one.
51543f157a2fSChris Mason 		 */
51553f157a2fSChris Mason 		while (slot < nritems) {
51563f157a2fSChris Mason 			u64 gen;
5157e02119d5SChris Mason 
51583f157a2fSChris Mason 			gen = btrfs_node_ptr_generation(cur, slot);
51593f157a2fSChris Mason 			if (gen < min_trans) {
51603f157a2fSChris Mason 				slot++;
51613f157a2fSChris Mason 				continue;
51623f157a2fSChris Mason 			}
51633f157a2fSChris Mason 			break;
51643f157a2fSChris Mason 		}
5165e02119d5SChris Mason find_next_key:
51663f157a2fSChris Mason 		/*
51673f157a2fSChris Mason 		 * we didn't find a candidate key in this node, walk forward
51683f157a2fSChris Mason 		 * and find another one
51693f157a2fSChris Mason 		 */
51703f157a2fSChris Mason 		if (slot >= nritems) {
5171e02119d5SChris Mason 			path->slots[level] = slot;
5172b4ce94deSChris Mason 			btrfs_set_path_blocking(path);
5173e02119d5SChris Mason 			sret = btrfs_find_next_key(root, path, min_key, level,
5174de78b51aSEric Sandeen 						  min_trans);
5175e02119d5SChris Mason 			if (sret == 0) {
5176b3b4aa74SDavid Sterba 				btrfs_release_path(path);
51773f157a2fSChris Mason 				goto again;
51783f157a2fSChris Mason 			} else {
51793f157a2fSChris Mason 				goto out;
51803f157a2fSChris Mason 			}
51813f157a2fSChris Mason 		}
51823f157a2fSChris Mason 		/* save our key for returning back */
51833f157a2fSChris Mason 		btrfs_node_key_to_cpu(cur, &found_key, slot);
51843f157a2fSChris Mason 		path->slots[level] = slot;
51853f157a2fSChris Mason 		if (level == path->lowest_level) {
51863f157a2fSChris Mason 			ret = 0;
51873f157a2fSChris Mason 			goto out;
51883f157a2fSChris Mason 		}
5189b4ce94deSChris Mason 		btrfs_set_path_blocking(path);
51902ff7e61eSJeff Mahoney 		cur = read_node_slot(fs_info, cur, slot);
5191fb770ae4SLiu Bo 		if (IS_ERR(cur)) {
5192fb770ae4SLiu Bo 			ret = PTR_ERR(cur);
5193fb770ae4SLiu Bo 			goto out;
5194fb770ae4SLiu Bo 		}
51953f157a2fSChris Mason 
5196bd681513SChris Mason 		btrfs_tree_read_lock(cur);
5197b4ce94deSChris Mason 
5198bd681513SChris Mason 		path->locks[level - 1] = BTRFS_READ_LOCK;
51993f157a2fSChris Mason 		path->nodes[level - 1] = cur;
5200f7c79f30SChris Mason 		unlock_up(path, level, 1, 0, NULL);
5201bd681513SChris Mason 		btrfs_clear_path_blocking(path, NULL, 0);
52023f157a2fSChris Mason 	}
52033f157a2fSChris Mason out:
5204f98de9b9SFilipe Manana 	path->keep_locks = keep_locks;
5205f98de9b9SFilipe Manana 	if (ret == 0) {
5206f98de9b9SFilipe Manana 		btrfs_unlock_up_safe(path, path->lowest_level + 1);
5207b4ce94deSChris Mason 		btrfs_set_path_blocking(path);
5208f98de9b9SFilipe Manana 		memcpy(min_key, &found_key, sizeof(found_key));
5209f98de9b9SFilipe Manana 	}
52103f157a2fSChris Mason 	return ret;
52113f157a2fSChris Mason }
52123f157a2fSChris Mason 
52132ff7e61eSJeff Mahoney static int tree_move_down(struct btrfs_fs_info *fs_info,
52147069830aSAlexander Block 			   struct btrfs_path *path,
5215ab6a43e1SDavid Sterba 			   int *level)
52167069830aSAlexander Block {
5217fb770ae4SLiu Bo 	struct extent_buffer *eb;
5218fb770ae4SLiu Bo 
521974dd17fbSChris Mason 	BUG_ON(*level == 0);
52202ff7e61eSJeff Mahoney 	eb = read_node_slot(fs_info, path->nodes[*level], path->slots[*level]);
5221fb770ae4SLiu Bo 	if (IS_ERR(eb))
5222fb770ae4SLiu Bo 		return PTR_ERR(eb);
5223fb770ae4SLiu Bo 
5224fb770ae4SLiu Bo 	path->nodes[*level - 1] = eb;
52257069830aSAlexander Block 	path->slots[*level - 1] = 0;
52267069830aSAlexander Block 	(*level)--;
5227fb770ae4SLiu Bo 	return 0;
52287069830aSAlexander Block }
52297069830aSAlexander Block 
5230f1e30261SDavid Sterba static int tree_move_next_or_upnext(struct btrfs_path *path,
52317069830aSAlexander Block 				    int *level, int root_level)
52327069830aSAlexander Block {
52337069830aSAlexander Block 	int ret = 0;
52347069830aSAlexander Block 	int nritems;
52357069830aSAlexander Block 	nritems = btrfs_header_nritems(path->nodes[*level]);
52367069830aSAlexander Block 
52377069830aSAlexander Block 	path->slots[*level]++;
52387069830aSAlexander Block 
523974dd17fbSChris Mason 	while (path->slots[*level] >= nritems) {
52407069830aSAlexander Block 		if (*level == root_level)
52417069830aSAlexander Block 			return -1;
52427069830aSAlexander Block 
52437069830aSAlexander Block 		/* move upnext */
52447069830aSAlexander Block 		path->slots[*level] = 0;
52457069830aSAlexander Block 		free_extent_buffer(path->nodes[*level]);
52467069830aSAlexander Block 		path->nodes[*level] = NULL;
52477069830aSAlexander Block 		(*level)++;
52487069830aSAlexander Block 		path->slots[*level]++;
52497069830aSAlexander Block 
52507069830aSAlexander Block 		nritems = btrfs_header_nritems(path->nodes[*level]);
52517069830aSAlexander Block 		ret = 1;
52527069830aSAlexander Block 	}
52537069830aSAlexander Block 	return ret;
52547069830aSAlexander Block }
52557069830aSAlexander Block 
52567069830aSAlexander Block /*
52577069830aSAlexander Block  * Returns 1 if it had to move up and next. 0 is returned if it moved only next
52587069830aSAlexander Block  * or down.
52597069830aSAlexander Block  */
52602ff7e61eSJeff Mahoney static int tree_advance(struct btrfs_fs_info *fs_info,
52617069830aSAlexander Block 			struct btrfs_path *path,
52627069830aSAlexander Block 			int *level, int root_level,
52637069830aSAlexander Block 			int allow_down,
52647069830aSAlexander Block 			struct btrfs_key *key)
52657069830aSAlexander Block {
52667069830aSAlexander Block 	int ret;
52677069830aSAlexander Block 
52687069830aSAlexander Block 	if (*level == 0 || !allow_down) {
5269f1e30261SDavid Sterba 		ret = tree_move_next_or_upnext(path, level, root_level);
52707069830aSAlexander Block 	} else {
5271ab6a43e1SDavid Sterba 		ret = tree_move_down(fs_info, path, level);
52727069830aSAlexander Block 	}
52737069830aSAlexander Block 	if (ret >= 0) {
52747069830aSAlexander Block 		if (*level == 0)
52757069830aSAlexander Block 			btrfs_item_key_to_cpu(path->nodes[*level], key,
52767069830aSAlexander Block 					path->slots[*level]);
52777069830aSAlexander Block 		else
52787069830aSAlexander Block 			btrfs_node_key_to_cpu(path->nodes[*level], key,
52797069830aSAlexander Block 					path->slots[*level]);
52807069830aSAlexander Block 	}
52817069830aSAlexander Block 	return ret;
52827069830aSAlexander Block }
52837069830aSAlexander Block 
52842ff7e61eSJeff Mahoney static int tree_compare_item(struct btrfs_path *left_path,
52857069830aSAlexander Block 			     struct btrfs_path *right_path,
52867069830aSAlexander Block 			     char *tmp_buf)
52877069830aSAlexander Block {
52887069830aSAlexander Block 	int cmp;
52897069830aSAlexander Block 	int len1, len2;
52907069830aSAlexander Block 	unsigned long off1, off2;
52917069830aSAlexander Block 
52927069830aSAlexander Block 	len1 = btrfs_item_size_nr(left_path->nodes[0], left_path->slots[0]);
52937069830aSAlexander Block 	len2 = btrfs_item_size_nr(right_path->nodes[0], right_path->slots[0]);
52947069830aSAlexander Block 	if (len1 != len2)
52957069830aSAlexander Block 		return 1;
52967069830aSAlexander Block 
52977069830aSAlexander Block 	off1 = btrfs_item_ptr_offset(left_path->nodes[0], left_path->slots[0]);
52987069830aSAlexander Block 	off2 = btrfs_item_ptr_offset(right_path->nodes[0],
52997069830aSAlexander Block 				right_path->slots[0]);
53007069830aSAlexander Block 
53017069830aSAlexander Block 	read_extent_buffer(left_path->nodes[0], tmp_buf, off1, len1);
53027069830aSAlexander Block 
53037069830aSAlexander Block 	cmp = memcmp_extent_buffer(right_path->nodes[0], tmp_buf, off2, len1);
53047069830aSAlexander Block 	if (cmp)
53057069830aSAlexander Block 		return 1;
53067069830aSAlexander Block 	return 0;
53077069830aSAlexander Block }
53087069830aSAlexander Block 
53097069830aSAlexander Block #define ADVANCE 1
53107069830aSAlexander Block #define ADVANCE_ONLY_NEXT -1
53117069830aSAlexander Block 
53127069830aSAlexander Block /*
53137069830aSAlexander Block  * This function compares two trees and calls the provided callback for
53147069830aSAlexander Block  * every changed/new/deleted item it finds.
53157069830aSAlexander Block  * If shared tree blocks are encountered, whole subtrees are skipped, making
53167069830aSAlexander Block  * the compare pretty fast on snapshotted subvolumes.
53177069830aSAlexander Block  *
53187069830aSAlexander Block  * This currently works on commit roots only. As commit roots are read only,
53197069830aSAlexander Block  * we don't do any locking. The commit roots are protected with transactions.
53207069830aSAlexander Block  * Transactions are ended and rejoined when a commit is tried in between.
53217069830aSAlexander Block  *
53227069830aSAlexander Block  * This function checks for modifications done to the trees while comparing.
53237069830aSAlexander Block  * If it detects a change, it aborts immediately.
53247069830aSAlexander Block  */
53257069830aSAlexander Block int btrfs_compare_trees(struct btrfs_root *left_root,
53267069830aSAlexander Block 			struct btrfs_root *right_root,
53277069830aSAlexander Block 			btrfs_changed_cb_t changed_cb, void *ctx)
53287069830aSAlexander Block {
53290b246afaSJeff Mahoney 	struct btrfs_fs_info *fs_info = left_root->fs_info;
53307069830aSAlexander Block 	int ret;
53317069830aSAlexander Block 	int cmp;
53327069830aSAlexander Block 	struct btrfs_path *left_path = NULL;
53337069830aSAlexander Block 	struct btrfs_path *right_path = NULL;
53347069830aSAlexander Block 	struct btrfs_key left_key;
53357069830aSAlexander Block 	struct btrfs_key right_key;
53367069830aSAlexander Block 	char *tmp_buf = NULL;
53377069830aSAlexander Block 	int left_root_level;
53387069830aSAlexander Block 	int right_root_level;
53397069830aSAlexander Block 	int left_level;
53407069830aSAlexander Block 	int right_level;
53417069830aSAlexander Block 	int left_end_reached;
53427069830aSAlexander Block 	int right_end_reached;
53437069830aSAlexander Block 	int advance_left;
53447069830aSAlexander Block 	int advance_right;
53457069830aSAlexander Block 	u64 left_blockptr;
53467069830aSAlexander Block 	u64 right_blockptr;
53476baa4293SFilipe Manana 	u64 left_gen;
53486baa4293SFilipe Manana 	u64 right_gen;
53497069830aSAlexander Block 
53507069830aSAlexander Block 	left_path = btrfs_alloc_path();
53517069830aSAlexander Block 	if (!left_path) {
53527069830aSAlexander Block 		ret = -ENOMEM;
53537069830aSAlexander Block 		goto out;
53547069830aSAlexander Block 	}
53557069830aSAlexander Block 	right_path = btrfs_alloc_path();
53567069830aSAlexander Block 	if (!right_path) {
53577069830aSAlexander Block 		ret = -ENOMEM;
53587069830aSAlexander Block 		goto out;
53597069830aSAlexander Block 	}
53607069830aSAlexander Block 
5361752ade68SMichal Hocko 	tmp_buf = kvmalloc(fs_info->nodesize, GFP_KERNEL);
53627069830aSAlexander Block 	if (!tmp_buf) {
53637069830aSAlexander Block 		ret = -ENOMEM;
53647069830aSAlexander Block 		goto out;
53657069830aSAlexander Block 	}
53667069830aSAlexander Block 
53677069830aSAlexander Block 	left_path->search_commit_root = 1;
53687069830aSAlexander Block 	left_path->skip_locking = 1;
53697069830aSAlexander Block 	right_path->search_commit_root = 1;
53707069830aSAlexander Block 	right_path->skip_locking = 1;
53717069830aSAlexander Block 
53727069830aSAlexander Block 	/*
53737069830aSAlexander Block 	 * Strategy: Go to the first items of both trees. Then do
53747069830aSAlexander Block 	 *
53757069830aSAlexander Block 	 * If both trees are at level 0
53767069830aSAlexander Block 	 *   Compare keys of current items
53777069830aSAlexander Block 	 *     If left < right treat left item as new, advance left tree
53787069830aSAlexander Block 	 *       and repeat
53797069830aSAlexander Block 	 *     If left > right treat right item as deleted, advance right tree
53807069830aSAlexander Block 	 *       and repeat
53817069830aSAlexander Block 	 *     If left == right do deep compare of items, treat as changed if
53827069830aSAlexander Block 	 *       needed, advance both trees and repeat
53837069830aSAlexander Block 	 * If both trees are at the same level but not at level 0
53847069830aSAlexander Block 	 *   Compare keys of current nodes/leafs
53857069830aSAlexander Block 	 *     If left < right advance left tree and repeat
53867069830aSAlexander Block 	 *     If left > right advance right tree and repeat
53877069830aSAlexander Block 	 *     If left == right compare blockptrs of the next nodes/leafs
53887069830aSAlexander Block 	 *       If they match advance both trees but stay at the same level
53897069830aSAlexander Block 	 *         and repeat
53907069830aSAlexander Block 	 *       If they don't match advance both trees while allowing to go
53917069830aSAlexander Block 	 *         deeper and repeat
53927069830aSAlexander Block 	 * If tree levels are different
53937069830aSAlexander Block 	 *   Advance the tree that needs it and repeat
53947069830aSAlexander Block 	 *
53957069830aSAlexander Block 	 * Advancing a tree means:
53967069830aSAlexander Block 	 *   If we are at level 0, try to go to the next slot. If that's not
53977069830aSAlexander Block 	 *   possible, go one level up and repeat. Stop when we found a level
53987069830aSAlexander Block 	 *   where we could go to the next slot. We may at this point be on a
53997069830aSAlexander Block 	 *   node or a leaf.
54007069830aSAlexander Block 	 *
54017069830aSAlexander Block 	 *   If we are not at level 0 and not on shared tree blocks, go one
54027069830aSAlexander Block 	 *   level deeper.
54037069830aSAlexander Block 	 *
54047069830aSAlexander Block 	 *   If we are not at level 0 and on shared tree blocks, go one slot to
54057069830aSAlexander Block 	 *   the right if possible or go up and right.
54067069830aSAlexander Block 	 */
54077069830aSAlexander Block 
54080b246afaSJeff Mahoney 	down_read(&fs_info->commit_root_sem);
54097069830aSAlexander Block 	left_level = btrfs_header_level(left_root->commit_root);
54107069830aSAlexander Block 	left_root_level = left_level;
54116f2f0b39SRobbie Ko 	left_path->nodes[left_level] =
54126f2f0b39SRobbie Ko 			btrfs_clone_extent_buffer(left_root->commit_root);
54136f2f0b39SRobbie Ko 	if (!left_path->nodes[left_level]) {
54146f2f0b39SRobbie Ko 		up_read(&fs_info->commit_root_sem);
54156f2f0b39SRobbie Ko 		ret = -ENOMEM;
54166f2f0b39SRobbie Ko 		goto out;
54176f2f0b39SRobbie Ko 	}
54187069830aSAlexander Block 	extent_buffer_get(left_path->nodes[left_level]);
54197069830aSAlexander Block 
54207069830aSAlexander Block 	right_level = btrfs_header_level(right_root->commit_root);
54217069830aSAlexander Block 	right_root_level = right_level;
54226f2f0b39SRobbie Ko 	right_path->nodes[right_level] =
54236f2f0b39SRobbie Ko 			btrfs_clone_extent_buffer(right_root->commit_root);
54246f2f0b39SRobbie Ko 	if (!right_path->nodes[right_level]) {
54256f2f0b39SRobbie Ko 		up_read(&fs_info->commit_root_sem);
54266f2f0b39SRobbie Ko 		ret = -ENOMEM;
54276f2f0b39SRobbie Ko 		goto out;
54286f2f0b39SRobbie Ko 	}
54297069830aSAlexander Block 	extent_buffer_get(right_path->nodes[right_level]);
54300b246afaSJeff Mahoney 	up_read(&fs_info->commit_root_sem);
54317069830aSAlexander Block 
54327069830aSAlexander Block 	if (left_level == 0)
54337069830aSAlexander Block 		btrfs_item_key_to_cpu(left_path->nodes[left_level],
54347069830aSAlexander Block 				&left_key, left_path->slots[left_level]);
54357069830aSAlexander Block 	else
54367069830aSAlexander Block 		btrfs_node_key_to_cpu(left_path->nodes[left_level],
54377069830aSAlexander Block 				&left_key, left_path->slots[left_level]);
54387069830aSAlexander Block 	if (right_level == 0)
54397069830aSAlexander Block 		btrfs_item_key_to_cpu(right_path->nodes[right_level],
54407069830aSAlexander Block 				&right_key, right_path->slots[right_level]);
54417069830aSAlexander Block 	else
54427069830aSAlexander Block 		btrfs_node_key_to_cpu(right_path->nodes[right_level],
54437069830aSAlexander Block 				&right_key, right_path->slots[right_level]);
54447069830aSAlexander Block 
54457069830aSAlexander Block 	left_end_reached = right_end_reached = 0;
54467069830aSAlexander Block 	advance_left = advance_right = 0;
54477069830aSAlexander Block 
54487069830aSAlexander Block 	while (1) {
54497069830aSAlexander Block 		if (advance_left && !left_end_reached) {
54502ff7e61eSJeff Mahoney 			ret = tree_advance(fs_info, left_path, &left_level,
54517069830aSAlexander Block 					left_root_level,
54527069830aSAlexander Block 					advance_left != ADVANCE_ONLY_NEXT,
54537069830aSAlexander Block 					&left_key);
5454fb770ae4SLiu Bo 			if (ret == -1)
54557069830aSAlexander Block 				left_end_reached = ADVANCE;
5456fb770ae4SLiu Bo 			else if (ret < 0)
5457fb770ae4SLiu Bo 				goto out;
54587069830aSAlexander Block 			advance_left = 0;
54597069830aSAlexander Block 		}
54607069830aSAlexander Block 		if (advance_right && !right_end_reached) {
54612ff7e61eSJeff Mahoney 			ret = tree_advance(fs_info, right_path, &right_level,
54627069830aSAlexander Block 					right_root_level,
54637069830aSAlexander Block 					advance_right != ADVANCE_ONLY_NEXT,
54647069830aSAlexander Block 					&right_key);
5465fb770ae4SLiu Bo 			if (ret == -1)
54667069830aSAlexander Block 				right_end_reached = ADVANCE;
5467fb770ae4SLiu Bo 			else if (ret < 0)
5468fb770ae4SLiu Bo 				goto out;
54697069830aSAlexander Block 			advance_right = 0;
54707069830aSAlexander Block 		}
54717069830aSAlexander Block 
54727069830aSAlexander Block 		if (left_end_reached && right_end_reached) {
54737069830aSAlexander Block 			ret = 0;
54747069830aSAlexander Block 			goto out;
54757069830aSAlexander Block 		} else if (left_end_reached) {
54767069830aSAlexander Block 			if (right_level == 0) {
5477ee8c494fSNikolay Borisov 				ret = changed_cb(left_path, right_path,
54787069830aSAlexander Block 						&right_key,
54797069830aSAlexander Block 						BTRFS_COMPARE_TREE_DELETED,
54807069830aSAlexander Block 						ctx);
54817069830aSAlexander Block 				if (ret < 0)
54827069830aSAlexander Block 					goto out;
54837069830aSAlexander Block 			}
54847069830aSAlexander Block 			advance_right = ADVANCE;
54857069830aSAlexander Block 			continue;
54867069830aSAlexander Block 		} else if (right_end_reached) {
54877069830aSAlexander Block 			if (left_level == 0) {
5488ee8c494fSNikolay Borisov 				ret = changed_cb(left_path, right_path,
54897069830aSAlexander Block 						&left_key,
54907069830aSAlexander Block 						BTRFS_COMPARE_TREE_NEW,
54917069830aSAlexander Block 						ctx);
54927069830aSAlexander Block 				if (ret < 0)
54937069830aSAlexander Block 					goto out;
54947069830aSAlexander Block 			}
54957069830aSAlexander Block 			advance_left = ADVANCE;
54967069830aSAlexander Block 			continue;
54977069830aSAlexander Block 		}
54987069830aSAlexander Block 
54997069830aSAlexander Block 		if (left_level == 0 && right_level == 0) {
55007069830aSAlexander Block 			cmp = btrfs_comp_cpu_keys(&left_key, &right_key);
55017069830aSAlexander Block 			if (cmp < 0) {
5502ee8c494fSNikolay Borisov 				ret = changed_cb(left_path, right_path,
55037069830aSAlexander Block 						&left_key,
55047069830aSAlexander Block 						BTRFS_COMPARE_TREE_NEW,
55057069830aSAlexander Block 						ctx);
55067069830aSAlexander Block 				if (ret < 0)
55077069830aSAlexander Block 					goto out;
55087069830aSAlexander Block 				advance_left = ADVANCE;
55097069830aSAlexander Block 			} else if (cmp > 0) {
5510ee8c494fSNikolay Borisov 				ret = changed_cb(left_path, right_path,
55117069830aSAlexander Block 						&right_key,
55127069830aSAlexander Block 						BTRFS_COMPARE_TREE_DELETED,
55137069830aSAlexander Block 						ctx);
55147069830aSAlexander Block 				if (ret < 0)
55157069830aSAlexander Block 					goto out;
55167069830aSAlexander Block 				advance_right = ADVANCE;
55177069830aSAlexander Block 			} else {
5518b99d9a6aSFabian Frederick 				enum btrfs_compare_tree_result result;
5519ba5e8f2eSJosef Bacik 
552074dd17fbSChris Mason 				WARN_ON(!extent_buffer_uptodate(left_path->nodes[0]));
55212ff7e61eSJeff Mahoney 				ret = tree_compare_item(left_path, right_path,
55222ff7e61eSJeff Mahoney 							tmp_buf);
5523ba5e8f2eSJosef Bacik 				if (ret)
5524b99d9a6aSFabian Frederick 					result = BTRFS_COMPARE_TREE_CHANGED;
5525ba5e8f2eSJosef Bacik 				else
5526b99d9a6aSFabian Frederick 					result = BTRFS_COMPARE_TREE_SAME;
5527ee8c494fSNikolay Borisov 				ret = changed_cb(left_path, right_path,
5528b99d9a6aSFabian Frederick 						 &left_key, result, ctx);
55297069830aSAlexander Block 				if (ret < 0)
55307069830aSAlexander Block 					goto out;
55317069830aSAlexander Block 				advance_left = ADVANCE;
55327069830aSAlexander Block 				advance_right = ADVANCE;
55337069830aSAlexander Block 			}
55347069830aSAlexander Block 		} else if (left_level == right_level) {
55357069830aSAlexander Block 			cmp = btrfs_comp_cpu_keys(&left_key, &right_key);
55367069830aSAlexander Block 			if (cmp < 0) {
55377069830aSAlexander Block 				advance_left = ADVANCE;
55387069830aSAlexander Block 			} else if (cmp > 0) {
55397069830aSAlexander Block 				advance_right = ADVANCE;
55407069830aSAlexander Block 			} else {
55417069830aSAlexander Block 				left_blockptr = btrfs_node_blockptr(
55427069830aSAlexander Block 						left_path->nodes[left_level],
55437069830aSAlexander Block 						left_path->slots[left_level]);
55447069830aSAlexander Block 				right_blockptr = btrfs_node_blockptr(
55457069830aSAlexander Block 						right_path->nodes[right_level],
55467069830aSAlexander Block 						right_path->slots[right_level]);
55476baa4293SFilipe Manana 				left_gen = btrfs_node_ptr_generation(
55486baa4293SFilipe Manana 						left_path->nodes[left_level],
55496baa4293SFilipe Manana 						left_path->slots[left_level]);
55506baa4293SFilipe Manana 				right_gen = btrfs_node_ptr_generation(
55516baa4293SFilipe Manana 						right_path->nodes[right_level],
55526baa4293SFilipe Manana 						right_path->slots[right_level]);
55536baa4293SFilipe Manana 				if (left_blockptr == right_blockptr &&
55546baa4293SFilipe Manana 				    left_gen == right_gen) {
55557069830aSAlexander Block 					/*
55567069830aSAlexander Block 					 * As we're on a shared block, don't
55577069830aSAlexander Block 					 * allow to go deeper.
55587069830aSAlexander Block 					 */
55597069830aSAlexander Block 					advance_left = ADVANCE_ONLY_NEXT;
55607069830aSAlexander Block 					advance_right = ADVANCE_ONLY_NEXT;
55617069830aSAlexander Block 				} else {
55627069830aSAlexander Block 					advance_left = ADVANCE;
55637069830aSAlexander Block 					advance_right = ADVANCE;
55647069830aSAlexander Block 				}
55657069830aSAlexander Block 			}
55667069830aSAlexander Block 		} else if (left_level < right_level) {
55677069830aSAlexander Block 			advance_right = ADVANCE;
55687069830aSAlexander Block 		} else {
55697069830aSAlexander Block 			advance_left = ADVANCE;
55707069830aSAlexander Block 		}
55717069830aSAlexander Block 	}
55727069830aSAlexander Block 
55737069830aSAlexander Block out:
55747069830aSAlexander Block 	btrfs_free_path(left_path);
55757069830aSAlexander Block 	btrfs_free_path(right_path);
55768f282f71SDavid Sterba 	kvfree(tmp_buf);
55777069830aSAlexander Block 	return ret;
55787069830aSAlexander Block }
55797069830aSAlexander Block 
55803f157a2fSChris Mason /*
55813f157a2fSChris Mason  * this is similar to btrfs_next_leaf, but does not try to preserve
55823f157a2fSChris Mason  * and fixup the path.  It looks for and returns the next key in the
5583de78b51aSEric Sandeen  * tree based on the current path and the min_trans parameters.
55843f157a2fSChris Mason  *
55853f157a2fSChris Mason  * 0 is returned if another key is found, < 0 if there are any errors
55863f157a2fSChris Mason  * and 1 is returned if there are no higher keys in the tree
55873f157a2fSChris Mason  *
55883f157a2fSChris Mason  * path->keep_locks should be set to 1 on the search made before
55893f157a2fSChris Mason  * calling this function.
55903f157a2fSChris Mason  */
5591e7a84565SChris Mason int btrfs_find_next_key(struct btrfs_root *root, struct btrfs_path *path,
5592de78b51aSEric Sandeen 			struct btrfs_key *key, int level, u64 min_trans)
5593e7a84565SChris Mason {
5594e7a84565SChris Mason 	int slot;
5595e7a84565SChris Mason 	struct extent_buffer *c;
5596e7a84565SChris Mason 
5597934d375bSChris Mason 	WARN_ON(!path->keep_locks);
5598e7a84565SChris Mason 	while (level < BTRFS_MAX_LEVEL) {
5599e7a84565SChris Mason 		if (!path->nodes[level])
5600e7a84565SChris Mason 			return 1;
5601e7a84565SChris Mason 
5602e7a84565SChris Mason 		slot = path->slots[level] + 1;
5603e7a84565SChris Mason 		c = path->nodes[level];
56043f157a2fSChris Mason next:
5605e7a84565SChris Mason 		if (slot >= btrfs_header_nritems(c)) {
560633c66f43SYan Zheng 			int ret;
560733c66f43SYan Zheng 			int orig_lowest;
560833c66f43SYan Zheng 			struct btrfs_key cur_key;
560933c66f43SYan Zheng 			if (level + 1 >= BTRFS_MAX_LEVEL ||
561033c66f43SYan Zheng 			    !path->nodes[level + 1])
5611e7a84565SChris Mason 				return 1;
561233c66f43SYan Zheng 
561333c66f43SYan Zheng 			if (path->locks[level + 1]) {
561433c66f43SYan Zheng 				level++;
5615e7a84565SChris Mason 				continue;
5616e7a84565SChris Mason 			}
561733c66f43SYan Zheng 
561833c66f43SYan Zheng 			slot = btrfs_header_nritems(c) - 1;
561933c66f43SYan Zheng 			if (level == 0)
562033c66f43SYan Zheng 				btrfs_item_key_to_cpu(c, &cur_key, slot);
562133c66f43SYan Zheng 			else
562233c66f43SYan Zheng 				btrfs_node_key_to_cpu(c, &cur_key, slot);
562333c66f43SYan Zheng 
562433c66f43SYan Zheng 			orig_lowest = path->lowest_level;
5625b3b4aa74SDavid Sterba 			btrfs_release_path(path);
562633c66f43SYan Zheng 			path->lowest_level = level;
562733c66f43SYan Zheng 			ret = btrfs_search_slot(NULL, root, &cur_key, path,
562833c66f43SYan Zheng 						0, 0);
562933c66f43SYan Zheng 			path->lowest_level = orig_lowest;
563033c66f43SYan Zheng 			if (ret < 0)
563133c66f43SYan Zheng 				return ret;
563233c66f43SYan Zheng 
563333c66f43SYan Zheng 			c = path->nodes[level];
563433c66f43SYan Zheng 			slot = path->slots[level];
563533c66f43SYan Zheng 			if (ret == 0)
563633c66f43SYan Zheng 				slot++;
563733c66f43SYan Zheng 			goto next;
563833c66f43SYan Zheng 		}
563933c66f43SYan Zheng 
5640e7a84565SChris Mason 		if (level == 0)
5641e7a84565SChris Mason 			btrfs_item_key_to_cpu(c, key, slot);
56423f157a2fSChris Mason 		else {
56433f157a2fSChris Mason 			u64 gen = btrfs_node_ptr_generation(c, slot);
56443f157a2fSChris Mason 
56453f157a2fSChris Mason 			if (gen < min_trans) {
56463f157a2fSChris Mason 				slot++;
56473f157a2fSChris Mason 				goto next;
56483f157a2fSChris Mason 			}
5649e7a84565SChris Mason 			btrfs_node_key_to_cpu(c, key, slot);
56503f157a2fSChris Mason 		}
5651e7a84565SChris Mason 		return 0;
5652e7a84565SChris Mason 	}
5653e7a84565SChris Mason 	return 1;
5654e7a84565SChris Mason }
5655e7a84565SChris Mason 
56567bb86316SChris Mason /*
5657925baeddSChris Mason  * search the tree again to find a leaf with greater keys
56580f70abe2SChris Mason  * returns 0 if it found something or 1 if there are no greater leaves.
56590f70abe2SChris Mason  * returns < 0 on io errors.
566097571fd0SChris Mason  */
5661234b63a0SChris Mason int btrfs_next_leaf(struct btrfs_root *root, struct btrfs_path *path)
5662d97e63b6SChris Mason {
56633d7806ecSJan Schmidt 	return btrfs_next_old_leaf(root, path, 0);
56643d7806ecSJan Schmidt }
56653d7806ecSJan Schmidt 
56663d7806ecSJan Schmidt int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path,
56673d7806ecSJan Schmidt 			u64 time_seq)
56683d7806ecSJan Schmidt {
5669d97e63b6SChris Mason 	int slot;
56708e73f275SChris Mason 	int level;
56715f39d397SChris Mason 	struct extent_buffer *c;
56728e73f275SChris Mason 	struct extent_buffer *next;
5673925baeddSChris Mason 	struct btrfs_key key;
5674925baeddSChris Mason 	u32 nritems;
5675925baeddSChris Mason 	int ret;
56768e73f275SChris Mason 	int old_spinning = path->leave_spinning;
5677bd681513SChris Mason 	int next_rw_lock = 0;
5678925baeddSChris Mason 
5679925baeddSChris Mason 	nritems = btrfs_header_nritems(path->nodes[0]);
5680d397712bSChris Mason 	if (nritems == 0)
5681925baeddSChris Mason 		return 1;
5682925baeddSChris Mason 
56838e73f275SChris Mason 	btrfs_item_key_to_cpu(path->nodes[0], &key, nritems - 1);
56848e73f275SChris Mason again:
56858e73f275SChris Mason 	level = 1;
56868e73f275SChris Mason 	next = NULL;
5687bd681513SChris Mason 	next_rw_lock = 0;
5688b3b4aa74SDavid Sterba 	btrfs_release_path(path);
56898e73f275SChris Mason 
5690a2135011SChris Mason 	path->keep_locks = 1;
56918e73f275SChris Mason 	path->leave_spinning = 1;
56928e73f275SChris Mason 
56933d7806ecSJan Schmidt 	if (time_seq)
56943d7806ecSJan Schmidt 		ret = btrfs_search_old_slot(root, &key, path, time_seq);
56953d7806ecSJan Schmidt 	else
5696925baeddSChris Mason 		ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
5697925baeddSChris Mason 	path->keep_locks = 0;
5698925baeddSChris Mason 
5699925baeddSChris Mason 	if (ret < 0)
5700925baeddSChris Mason 		return ret;
5701925baeddSChris Mason 
5702a2135011SChris Mason 	nritems = btrfs_header_nritems(path->nodes[0]);
5703168fd7d2SChris Mason 	/*
5704168fd7d2SChris Mason 	 * by releasing the path above we dropped all our locks.  A balance
5705168fd7d2SChris Mason 	 * could have added more items next to the key that used to be
5706168fd7d2SChris Mason 	 * at the very end of the block.  So, check again here and
5707168fd7d2SChris Mason 	 * advance the path if there are now more items available.
5708168fd7d2SChris Mason 	 */
5709a2135011SChris Mason 	if (nritems > 0 && path->slots[0] < nritems - 1) {
5710e457afecSYan Zheng 		if (ret == 0)
5711168fd7d2SChris Mason 			path->slots[0]++;
57128e73f275SChris Mason 		ret = 0;
5713925baeddSChris Mason 		goto done;
5714925baeddSChris Mason 	}
57150b43e04fSLiu Bo 	/*
57160b43e04fSLiu Bo 	 * So the above check misses one case:
57170b43e04fSLiu Bo 	 * - after releasing the path above, someone has removed the item that
57180b43e04fSLiu Bo 	 *   used to be at the very end of the block, and balance between leafs
57190b43e04fSLiu Bo 	 *   gets another one with bigger key.offset to replace it.
57200b43e04fSLiu Bo 	 *
57210b43e04fSLiu Bo 	 * This one should be returned as well, or we can get leaf corruption
57220b43e04fSLiu Bo 	 * later(esp. in __btrfs_drop_extents()).
57230b43e04fSLiu Bo 	 *
57240b43e04fSLiu Bo 	 * And a bit more explanation about this check,
57250b43e04fSLiu Bo 	 * with ret > 0, the key isn't found, the path points to the slot
57260b43e04fSLiu Bo 	 * where it should be inserted, so the path->slots[0] item must be the
57270b43e04fSLiu Bo 	 * bigger one.
57280b43e04fSLiu Bo 	 */
57290b43e04fSLiu Bo 	if (nritems > 0 && ret > 0 && path->slots[0] == nritems - 1) {
57300b43e04fSLiu Bo 		ret = 0;
57310b43e04fSLiu Bo 		goto done;
57320b43e04fSLiu Bo 	}
5733d97e63b6SChris Mason 
5734234b63a0SChris Mason 	while (level < BTRFS_MAX_LEVEL) {
57358e73f275SChris Mason 		if (!path->nodes[level]) {
57368e73f275SChris Mason 			ret = 1;
57378e73f275SChris Mason 			goto done;
57388e73f275SChris Mason 		}
57395f39d397SChris Mason 
5740d97e63b6SChris Mason 		slot = path->slots[level] + 1;
5741d97e63b6SChris Mason 		c = path->nodes[level];
57425f39d397SChris Mason 		if (slot >= btrfs_header_nritems(c)) {
5743d97e63b6SChris Mason 			level++;
57448e73f275SChris Mason 			if (level == BTRFS_MAX_LEVEL) {
57458e73f275SChris Mason 				ret = 1;
57468e73f275SChris Mason 				goto done;
57478e73f275SChris Mason 			}
5748d97e63b6SChris Mason 			continue;
5749d97e63b6SChris Mason 		}
57505f39d397SChris Mason 
5751925baeddSChris Mason 		if (next) {
5752bd681513SChris Mason 			btrfs_tree_unlock_rw(next, next_rw_lock);
57535f39d397SChris Mason 			free_extent_buffer(next);
5754925baeddSChris Mason 		}
57555f39d397SChris Mason 
57568e73f275SChris Mason 		next = c;
5757bd681513SChris Mason 		next_rw_lock = path->locks[level];
5758d07b8528SLiu Bo 		ret = read_block_for_search(root, path, &next, level,
5759cda79c54SDavid Sterba 					    slot, &key);
57608e73f275SChris Mason 		if (ret == -EAGAIN)
57618e73f275SChris Mason 			goto again;
57625f39d397SChris Mason 
576376a05b35SChris Mason 		if (ret < 0) {
5764b3b4aa74SDavid Sterba 			btrfs_release_path(path);
576576a05b35SChris Mason 			goto done;
576676a05b35SChris Mason 		}
576776a05b35SChris Mason 
57685cd57b2cSChris Mason 		if (!path->skip_locking) {
5769bd681513SChris Mason 			ret = btrfs_try_tree_read_lock(next);
5770d42244a0SJan Schmidt 			if (!ret && time_seq) {
5771d42244a0SJan Schmidt 				/*
5772d42244a0SJan Schmidt 				 * If we don't get the lock, we may be racing
5773d42244a0SJan Schmidt 				 * with push_leaf_left, holding that lock while
5774d42244a0SJan Schmidt 				 * itself waiting for the leaf we've currently
5775d42244a0SJan Schmidt 				 * locked. To solve this situation, we give up
5776d42244a0SJan Schmidt 				 * on our lock and cycle.
5777d42244a0SJan Schmidt 				 */
5778cf538830SJan Schmidt 				free_extent_buffer(next);
5779d42244a0SJan Schmidt 				btrfs_release_path(path);
5780d42244a0SJan Schmidt 				cond_resched();
5781d42244a0SJan Schmidt 				goto again;
5782d42244a0SJan Schmidt 			}
57838e73f275SChris Mason 			if (!ret) {
57848e73f275SChris Mason 				btrfs_set_path_blocking(path);
5785bd681513SChris Mason 				btrfs_tree_read_lock(next);
5786bd681513SChris Mason 				btrfs_clear_path_blocking(path, next,
5787bd681513SChris Mason 							  BTRFS_READ_LOCK);
57888e73f275SChris Mason 			}
5789bd681513SChris Mason 			next_rw_lock = BTRFS_READ_LOCK;
5790bd681513SChris Mason 		}
5791d97e63b6SChris Mason 		break;
5792d97e63b6SChris Mason 	}
5793d97e63b6SChris Mason 	path->slots[level] = slot;
5794d97e63b6SChris Mason 	while (1) {
5795d97e63b6SChris Mason 		level--;
5796d97e63b6SChris Mason 		c = path->nodes[level];
5797925baeddSChris Mason 		if (path->locks[level])
5798bd681513SChris Mason 			btrfs_tree_unlock_rw(c, path->locks[level]);
57998e73f275SChris Mason 
58005f39d397SChris Mason 		free_extent_buffer(c);
5801d97e63b6SChris Mason 		path->nodes[level] = next;
5802d97e63b6SChris Mason 		path->slots[level] = 0;
5803a74a4b97SChris Mason 		if (!path->skip_locking)
5804bd681513SChris Mason 			path->locks[level] = next_rw_lock;
5805d97e63b6SChris Mason 		if (!level)
5806d97e63b6SChris Mason 			break;
5807b4ce94deSChris Mason 
5808d07b8528SLiu Bo 		ret = read_block_for_search(root, path, &next, level,
5809cda79c54SDavid Sterba 					    0, &key);
58108e73f275SChris Mason 		if (ret == -EAGAIN)
58118e73f275SChris Mason 			goto again;
58128e73f275SChris Mason 
581376a05b35SChris Mason 		if (ret < 0) {
5814b3b4aa74SDavid Sterba 			btrfs_release_path(path);
581576a05b35SChris Mason 			goto done;
581676a05b35SChris Mason 		}
581776a05b35SChris Mason 
58185cd57b2cSChris Mason 		if (!path->skip_locking) {
5819bd681513SChris Mason 			ret = btrfs_try_tree_read_lock(next);
58208e73f275SChris Mason 			if (!ret) {
58218e73f275SChris Mason 				btrfs_set_path_blocking(path);
5822bd681513SChris Mason 				btrfs_tree_read_lock(next);
5823bd681513SChris Mason 				btrfs_clear_path_blocking(path, next,
5824bd681513SChris Mason 							  BTRFS_READ_LOCK);
58258e73f275SChris Mason 			}
5826bd681513SChris Mason 			next_rw_lock = BTRFS_READ_LOCK;
5827bd681513SChris Mason 		}
5828d97e63b6SChris Mason 	}
58298e73f275SChris Mason 	ret = 0;
5830925baeddSChris Mason done:
5831f7c79f30SChris Mason 	unlock_up(path, 0, 1, 0, NULL);
58328e73f275SChris Mason 	path->leave_spinning = old_spinning;
58338e73f275SChris Mason 	if (!old_spinning)
58348e73f275SChris Mason 		btrfs_set_path_blocking(path);
58358e73f275SChris Mason 
58368e73f275SChris Mason 	return ret;
5837d97e63b6SChris Mason }
58380b86a832SChris Mason 
58393f157a2fSChris Mason /*
58403f157a2fSChris Mason  * this uses btrfs_prev_leaf to walk backwards in the tree, and keeps
58413f157a2fSChris Mason  * searching until it gets past min_objectid or finds an item of 'type'
58423f157a2fSChris Mason  *
58433f157a2fSChris Mason  * returns 0 if something is found, 1 if nothing was found and < 0 on error
58443f157a2fSChris Mason  */
58450b86a832SChris Mason int btrfs_previous_item(struct btrfs_root *root,
58460b86a832SChris Mason 			struct btrfs_path *path, u64 min_objectid,
58470b86a832SChris Mason 			int type)
58480b86a832SChris Mason {
58490b86a832SChris Mason 	struct btrfs_key found_key;
58500b86a832SChris Mason 	struct extent_buffer *leaf;
5851e02119d5SChris Mason 	u32 nritems;
58520b86a832SChris Mason 	int ret;
58530b86a832SChris Mason 
58540b86a832SChris Mason 	while (1) {
58550b86a832SChris Mason 		if (path->slots[0] == 0) {
5856b4ce94deSChris Mason 			btrfs_set_path_blocking(path);
58570b86a832SChris Mason 			ret = btrfs_prev_leaf(root, path);
58580b86a832SChris Mason 			if (ret != 0)
58590b86a832SChris Mason 				return ret;
58600b86a832SChris Mason 		} else {
58610b86a832SChris Mason 			path->slots[0]--;
58620b86a832SChris Mason 		}
58630b86a832SChris Mason 		leaf = path->nodes[0];
5864e02119d5SChris Mason 		nritems = btrfs_header_nritems(leaf);
5865e02119d5SChris Mason 		if (nritems == 0)
5866e02119d5SChris Mason 			return 1;
5867e02119d5SChris Mason 		if (path->slots[0] == nritems)
5868e02119d5SChris Mason 			path->slots[0]--;
5869e02119d5SChris Mason 
58700b86a832SChris Mason 		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
5871e02119d5SChris Mason 		if (found_key.objectid < min_objectid)
5872e02119d5SChris Mason 			break;
58730a4eefbbSYan Zheng 		if (found_key.type == type)
58740a4eefbbSYan Zheng 			return 0;
5875e02119d5SChris Mason 		if (found_key.objectid == min_objectid &&
5876e02119d5SChris Mason 		    found_key.type < type)
5877e02119d5SChris Mason 			break;
58780b86a832SChris Mason 	}
58790b86a832SChris Mason 	return 1;
58800b86a832SChris Mason }
5881ade2e0b3SWang Shilong 
5882ade2e0b3SWang Shilong /*
5883ade2e0b3SWang Shilong  * search in extent tree to find a previous Metadata/Data extent item with
5884ade2e0b3SWang Shilong  * min objecitd.
5885ade2e0b3SWang Shilong  *
5886ade2e0b3SWang Shilong  * returns 0 if something is found, 1 if nothing was found and < 0 on error
5887ade2e0b3SWang Shilong  */
5888ade2e0b3SWang Shilong int btrfs_previous_extent_item(struct btrfs_root *root,
5889ade2e0b3SWang Shilong 			struct btrfs_path *path, u64 min_objectid)
5890ade2e0b3SWang Shilong {
5891ade2e0b3SWang Shilong 	struct btrfs_key found_key;
5892ade2e0b3SWang Shilong 	struct extent_buffer *leaf;
5893ade2e0b3SWang Shilong 	u32 nritems;
5894ade2e0b3SWang Shilong 	int ret;
5895ade2e0b3SWang Shilong 
5896ade2e0b3SWang Shilong 	while (1) {
5897ade2e0b3SWang Shilong 		if (path->slots[0] == 0) {
5898ade2e0b3SWang Shilong 			btrfs_set_path_blocking(path);
5899ade2e0b3SWang Shilong 			ret = btrfs_prev_leaf(root, path);
5900ade2e0b3SWang Shilong 			if (ret != 0)
5901ade2e0b3SWang Shilong 				return ret;
5902ade2e0b3SWang Shilong 		} else {
5903ade2e0b3SWang Shilong 			path->slots[0]--;
5904ade2e0b3SWang Shilong 		}
5905ade2e0b3SWang Shilong 		leaf = path->nodes[0];
5906ade2e0b3SWang Shilong 		nritems = btrfs_header_nritems(leaf);
5907ade2e0b3SWang Shilong 		if (nritems == 0)
5908ade2e0b3SWang Shilong 			return 1;
5909ade2e0b3SWang Shilong 		if (path->slots[0] == nritems)
5910ade2e0b3SWang Shilong 			path->slots[0]--;
5911ade2e0b3SWang Shilong 
5912ade2e0b3SWang Shilong 		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
5913ade2e0b3SWang Shilong 		if (found_key.objectid < min_objectid)
5914ade2e0b3SWang Shilong 			break;
5915ade2e0b3SWang Shilong 		if (found_key.type == BTRFS_EXTENT_ITEM_KEY ||
5916ade2e0b3SWang Shilong 		    found_key.type == BTRFS_METADATA_ITEM_KEY)
5917ade2e0b3SWang Shilong 			return 0;
5918ade2e0b3SWang Shilong 		if (found_key.objectid == min_objectid &&
5919ade2e0b3SWang Shilong 		    found_key.type < BTRFS_EXTENT_ITEM_KEY)
5920ade2e0b3SWang Shilong 			break;
5921ade2e0b3SWang Shilong 	}
5922ade2e0b3SWang Shilong 	return 1;
5923ade2e0b3SWang Shilong }
5924