1c1d7c514SDavid Sterba // SPDX-License-Identifier: GPL-2.0
26cbd5570SChris Mason /*
3d352ac68SChris Mason * Copyright (C) 2007,2008 Oracle. All rights reserved.
46cbd5570SChris Mason */
56cbd5570SChris Mason
6a6b6e75eSChris Mason #include <linux/sched.h>
75a0e3ad6STejun Heo #include <linux/slab.h>
8bd989ba3SJan Schmidt #include <linux/rbtree.h>
9adf02123SDavid Sterba #include <linux/mm.h>
10e41d12f5SChristoph Hellwig #include <linux/error-injection.h>
119b569ea0SJosef Bacik #include "messages.h"
12eb60ceacSChris Mason #include "ctree.h"
13eb60ceacSChris Mason #include "disk-io.h"
147f5c1516SChris Mason #include "transaction.h"
155f39d397SChris Mason #include "print-tree.h"
16925baeddSChris Mason #include "locking.h"
17de37aa51SNikolay Borisov #include "volumes.h"
18f616f5cdSQu Wenruo #include "qgroup.h"
19f3a84ccdSFilipe Manana #include "tree-mod-log.h"
2088c602abSQu Wenruo #include "tree-checker.h"
21ec8eb376SJosef Bacik #include "fs.h"
22ad1ac501SJosef Bacik #include "accessors.h"
23a0231804SJosef Bacik #include "extent-tree.h"
2467707479SJosef Bacik #include "relocation.h"
256bfd0ffaSJosef Bacik #include "file-item.h"
269a8dd150SChris Mason
27226463d7SJosef Bacik static struct kmem_cache *btrfs_path_cachep;
28226463d7SJosef Bacik
29e089f05cSChris Mason static int split_node(struct btrfs_trans_handle *trans, struct btrfs_root
30e089f05cSChris Mason *root, struct btrfs_path *path, int level);
31310712b2SOmar Sandoval static int split_leaf(struct btrfs_trans_handle *trans, struct btrfs_root *root,
32310712b2SOmar Sandoval const struct btrfs_key *ins_key, struct btrfs_path *path,
33310712b2SOmar Sandoval int data_size, int extend);
345f39d397SChris Mason static int push_node_left(struct btrfs_trans_handle *trans,
352ff7e61eSJeff Mahoney struct extent_buffer *dst,
36971a1f66SChris Mason struct extent_buffer *src, int empty);
375f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans,
385f39d397SChris Mason struct extent_buffer *dst_buf,
395f39d397SChris Mason struct extent_buffer *src_buf);
40d97e63b6SChris Mason
41af024ed2SJohannes Thumshirn static const struct btrfs_csums {
42af024ed2SJohannes Thumshirn u16 size;
4359a0fcdbSDavid Sterba const char name[10];
4459a0fcdbSDavid Sterba const char driver[12];
45af024ed2SJohannes Thumshirn } btrfs_csums[] = {
46af024ed2SJohannes Thumshirn [BTRFS_CSUM_TYPE_CRC32] = { .size = 4, .name = "crc32c" },
473951e7f0SJohannes Thumshirn [BTRFS_CSUM_TYPE_XXHASH] = { .size = 8, .name = "xxhash64" },
483831bf00SJohannes Thumshirn [BTRFS_CSUM_TYPE_SHA256] = { .size = 32, .name = "sha256" },
49352ae07bSDavid Sterba [BTRFS_CSUM_TYPE_BLAKE2] = { .size = 32, .name = "blake2b",
50352ae07bSDavid Sterba .driver = "blake2b-256" },
51af024ed2SJohannes Thumshirn };
52af024ed2SJohannes Thumshirn
533a3178c7SJosef Bacik /*
543a3178c7SJosef Bacik * The leaf data grows from end-to-front in the node. this returns the address
553a3178c7SJosef Bacik * of the start of the last item, which is the stop of the leaf data stack.
563a3178c7SJosef Bacik */
leaf_data_end(const struct extent_buffer * leaf)573a3178c7SJosef Bacik static unsigned int leaf_data_end(const struct extent_buffer *leaf)
583a3178c7SJosef Bacik {
593a3178c7SJosef Bacik u32 nr = btrfs_header_nritems(leaf);
603a3178c7SJosef Bacik
613a3178c7SJosef Bacik if (nr == 0)
623a3178c7SJosef Bacik return BTRFS_LEAF_DATA_SIZE(leaf->fs_info);
633a3178c7SJosef Bacik return btrfs_item_offset(leaf, nr - 1);
643a3178c7SJosef Bacik }
653a3178c7SJosef Bacik
66637e3b48SJosef Bacik /*
67637e3b48SJosef Bacik * Move data in a @leaf (using memmove, safe for overlapping ranges).
68637e3b48SJosef Bacik *
69637e3b48SJosef Bacik * @leaf: leaf that we're doing a memmove on
70637e3b48SJosef Bacik * @dst_offset: item data offset we're moving to
71637e3b48SJosef Bacik * @src_offset: item data offset were' moving from
72637e3b48SJosef Bacik * @len: length of the data we're moving
73637e3b48SJosef Bacik *
74637e3b48SJosef Bacik * Wrapper around memmove_extent_buffer() that takes into account the header on
75637e3b48SJosef Bacik * the leaf. The btrfs_item offset's start directly after the header, so we
76637e3b48SJosef Bacik * have to adjust any offsets to account for the header in the leaf. This
77637e3b48SJosef Bacik * handles that math to simplify the callers.
78637e3b48SJosef Bacik */
memmove_leaf_data(const struct extent_buffer * leaf,unsigned long dst_offset,unsigned long src_offset,unsigned long len)79637e3b48SJosef Bacik static inline void memmove_leaf_data(const struct extent_buffer *leaf,
80637e3b48SJosef Bacik unsigned long dst_offset,
81637e3b48SJosef Bacik unsigned long src_offset,
82637e3b48SJosef Bacik unsigned long len)
83637e3b48SJosef Bacik {
848009adf3SJosef Bacik memmove_extent_buffer(leaf, btrfs_item_nr_offset(leaf, 0) + dst_offset,
858009adf3SJosef Bacik btrfs_item_nr_offset(leaf, 0) + src_offset, len);
86637e3b48SJosef Bacik }
87637e3b48SJosef Bacik
88637e3b48SJosef Bacik /*
89637e3b48SJosef Bacik * Copy item data from @src into @dst at the given @offset.
90637e3b48SJosef Bacik *
91637e3b48SJosef Bacik * @dst: destination leaf that we're copying into
92637e3b48SJosef Bacik * @src: source leaf that we're copying from
93637e3b48SJosef Bacik * @dst_offset: item data offset we're copying to
94637e3b48SJosef Bacik * @src_offset: item data offset were' copying from
95637e3b48SJosef Bacik * @len: length of the data we're copying
96637e3b48SJosef Bacik *
97637e3b48SJosef Bacik * Wrapper around copy_extent_buffer() that takes into account the header on
98637e3b48SJosef Bacik * the leaf. The btrfs_item offset's start directly after the header, so we
99637e3b48SJosef Bacik * have to adjust any offsets to account for the header in the leaf. This
100637e3b48SJosef Bacik * handles that math to simplify the callers.
101637e3b48SJosef Bacik */
copy_leaf_data(const struct extent_buffer * dst,const struct extent_buffer * src,unsigned long dst_offset,unsigned long src_offset,unsigned long len)102637e3b48SJosef Bacik static inline void copy_leaf_data(const struct extent_buffer *dst,
103637e3b48SJosef Bacik const struct extent_buffer *src,
104637e3b48SJosef Bacik unsigned long dst_offset,
105637e3b48SJosef Bacik unsigned long src_offset, unsigned long len)
106637e3b48SJosef Bacik {
1078009adf3SJosef Bacik copy_extent_buffer(dst, src, btrfs_item_nr_offset(dst, 0) + dst_offset,
1088009adf3SJosef Bacik btrfs_item_nr_offset(src, 0) + src_offset, len);
109637e3b48SJosef Bacik }
110637e3b48SJosef Bacik
111637e3b48SJosef Bacik /*
112637e3b48SJosef Bacik * Move items in a @leaf (using memmove).
113637e3b48SJosef Bacik *
114637e3b48SJosef Bacik * @dst: destination leaf for the items
115637e3b48SJosef Bacik * @dst_item: the item nr we're copying into
116637e3b48SJosef Bacik * @src_item: the item nr we're copying from
117637e3b48SJosef Bacik * @nr_items: the number of items to copy
118637e3b48SJosef Bacik *
119637e3b48SJosef Bacik * Wrapper around memmove_extent_buffer() that does the math to get the
120637e3b48SJosef Bacik * appropriate offsets into the leaf from the item numbers.
121637e3b48SJosef Bacik */
memmove_leaf_items(const struct extent_buffer * leaf,int dst_item,int src_item,int nr_items)122637e3b48SJosef Bacik static inline void memmove_leaf_items(const struct extent_buffer *leaf,
123637e3b48SJosef Bacik int dst_item, int src_item, int nr_items)
124637e3b48SJosef Bacik {
125637e3b48SJosef Bacik memmove_extent_buffer(leaf, btrfs_item_nr_offset(leaf, dst_item),
126637e3b48SJosef Bacik btrfs_item_nr_offset(leaf, src_item),
127637e3b48SJosef Bacik nr_items * sizeof(struct btrfs_item));
128637e3b48SJosef Bacik }
129637e3b48SJosef Bacik
130637e3b48SJosef Bacik /*
131637e3b48SJosef Bacik * Copy items from @src into @dst at the given @offset.
132637e3b48SJosef Bacik *
133637e3b48SJosef Bacik * @dst: destination leaf for the items
134637e3b48SJosef Bacik * @src: source leaf for the items
135637e3b48SJosef Bacik * @dst_item: the item nr we're copying into
136637e3b48SJosef Bacik * @src_item: the item nr we're copying from
137637e3b48SJosef Bacik * @nr_items: the number of items to copy
138637e3b48SJosef Bacik *
139637e3b48SJosef Bacik * Wrapper around copy_extent_buffer() that does the math to get the
140637e3b48SJosef Bacik * appropriate offsets into the leaf from the item numbers.
141637e3b48SJosef Bacik */
copy_leaf_items(const struct extent_buffer * dst,const struct extent_buffer * src,int dst_item,int src_item,int nr_items)142637e3b48SJosef Bacik static inline void copy_leaf_items(const struct extent_buffer *dst,
143637e3b48SJosef Bacik const struct extent_buffer *src,
144637e3b48SJosef Bacik int dst_item, int src_item, int nr_items)
145637e3b48SJosef Bacik {
146637e3b48SJosef Bacik copy_extent_buffer(dst, src, btrfs_item_nr_offset(dst, dst_item),
147637e3b48SJosef Bacik btrfs_item_nr_offset(src, src_item),
148637e3b48SJosef Bacik nr_items * sizeof(struct btrfs_item));
149637e3b48SJosef Bacik }
150637e3b48SJosef Bacik
151b3cbfb0dSJosef Bacik /* This exists for btrfs-progs usages. */
btrfs_csum_type_size(u16 type)152b3cbfb0dSJosef Bacik u16 btrfs_csum_type_size(u16 type)
153b3cbfb0dSJosef Bacik {
154b3cbfb0dSJosef Bacik return btrfs_csums[type].size;
155b3cbfb0dSJosef Bacik }
156b3cbfb0dSJosef Bacik
btrfs_super_csum_size(const struct btrfs_super_block * s)157af024ed2SJohannes Thumshirn int btrfs_super_csum_size(const struct btrfs_super_block *s)
158af024ed2SJohannes Thumshirn {
159af024ed2SJohannes Thumshirn u16 t = btrfs_super_csum_type(s);
160af024ed2SJohannes Thumshirn /*
161af024ed2SJohannes Thumshirn * csum type is validated at mount time
162af024ed2SJohannes Thumshirn */
163b3cbfb0dSJosef Bacik return btrfs_csum_type_size(t);
164af024ed2SJohannes Thumshirn }
165af024ed2SJohannes Thumshirn
btrfs_super_csum_name(u16 csum_type)166af024ed2SJohannes Thumshirn const char *btrfs_super_csum_name(u16 csum_type)
167af024ed2SJohannes Thumshirn {
168af024ed2SJohannes Thumshirn /* csum type is validated at mount time */
169af024ed2SJohannes Thumshirn return btrfs_csums[csum_type].name;
170af024ed2SJohannes Thumshirn }
171af024ed2SJohannes Thumshirn
172b4e967beSDavid Sterba /*
173b4e967beSDavid Sterba * Return driver name if defined, otherwise the name that's also a valid driver
174b4e967beSDavid Sterba * name
175b4e967beSDavid Sterba */
btrfs_super_csum_driver(u16 csum_type)176b4e967beSDavid Sterba const char *btrfs_super_csum_driver(u16 csum_type)
177b4e967beSDavid Sterba {
178b4e967beSDavid Sterba /* csum type is validated at mount time */
17959a0fcdbSDavid Sterba return btrfs_csums[csum_type].driver[0] ?
18059a0fcdbSDavid Sterba btrfs_csums[csum_type].driver :
181b4e967beSDavid Sterba btrfs_csums[csum_type].name;
182b4e967beSDavid Sterba }
183b4e967beSDavid Sterba
btrfs_get_num_csums(void)184604997b4SDavid Sterba size_t __attribute_const__ btrfs_get_num_csums(void)
185f7cea56cSDavid Sterba {
186f7cea56cSDavid Sterba return ARRAY_SIZE(btrfs_csums);
187f7cea56cSDavid Sterba }
188f7cea56cSDavid Sterba
btrfs_alloc_path(void)1892c90e5d6SChris Mason struct btrfs_path *btrfs_alloc_path(void)
1902c90e5d6SChris Mason {
191a4c853afSChenXiaoSong might_sleep();
192a4c853afSChenXiaoSong
193e2c89907SMasahiro Yamada return kmem_cache_zalloc(btrfs_path_cachep, GFP_NOFS);
1942c90e5d6SChris Mason }
1952c90e5d6SChris Mason
196d352ac68SChris Mason /* this also releases the path */
btrfs_free_path(struct btrfs_path * p)1972c90e5d6SChris Mason void btrfs_free_path(struct btrfs_path *p)
1982c90e5d6SChris Mason {
199ff175d57SJesper Juhl if (!p)
200ff175d57SJesper Juhl return;
201b3b4aa74SDavid Sterba btrfs_release_path(p);
2022c90e5d6SChris Mason kmem_cache_free(btrfs_path_cachep, p);
2032c90e5d6SChris Mason }
2042c90e5d6SChris Mason
205d352ac68SChris Mason /*
206d352ac68SChris Mason * path release drops references on the extent buffers in the path
207d352ac68SChris Mason * and it drops any locks held by this path
208d352ac68SChris Mason *
209d352ac68SChris Mason * It is safe to call this on paths that no locks or extent buffers held.
210d352ac68SChris Mason */
btrfs_release_path(struct btrfs_path * p)211b3b4aa74SDavid Sterba noinline void btrfs_release_path(struct btrfs_path *p)
212eb60ceacSChris Mason {
213eb60ceacSChris Mason int i;
214a2135011SChris Mason
215234b63a0SChris Mason for (i = 0; i < BTRFS_MAX_LEVEL; i++) {
2163f157a2fSChris Mason p->slots[i] = 0;
217eb60ceacSChris Mason if (!p->nodes[i])
218925baeddSChris Mason continue;
219925baeddSChris Mason if (p->locks[i]) {
220bd681513SChris Mason btrfs_tree_unlock_rw(p->nodes[i], p->locks[i]);
221925baeddSChris Mason p->locks[i] = 0;
222925baeddSChris Mason }
2235f39d397SChris Mason free_extent_buffer(p->nodes[i]);
2243f157a2fSChris Mason p->nodes[i] = NULL;
225eb60ceacSChris Mason }
226eb60ceacSChris Mason }
227eb60ceacSChris Mason
228d352ac68SChris Mason /*
2298bb808c6SDavid Sterba * We want the transaction abort to print stack trace only for errors where the
2308bb808c6SDavid Sterba * cause could be a bug, eg. due to ENOSPC, and not for common errors that are
2318bb808c6SDavid Sterba * caused by external factors.
2328bb808c6SDavid Sterba */
abort_should_print_stack(int errno)2338bb808c6SDavid Sterba bool __cold abort_should_print_stack(int errno)
2348bb808c6SDavid Sterba {
2358bb808c6SDavid Sterba switch (errno) {
2368bb808c6SDavid Sterba case -EIO:
2378bb808c6SDavid Sterba case -EROFS:
2388bb808c6SDavid Sterba case -ENOMEM:
2398bb808c6SDavid Sterba return false;
2408bb808c6SDavid Sterba }
2418bb808c6SDavid Sterba return true;
2428bb808c6SDavid Sterba }
2438bb808c6SDavid Sterba
2448bb808c6SDavid Sterba /*
245d352ac68SChris Mason * safely gets a reference on the root node of a tree. A lock
246d352ac68SChris Mason * is not taken, so a concurrent writer may put a different node
247d352ac68SChris Mason * at the root of the tree. See btrfs_lock_root_node for the
248d352ac68SChris Mason * looping required.
249d352ac68SChris Mason *
250d352ac68SChris Mason * The extent buffer returned by this has a reference taken, so
251d352ac68SChris Mason * it won't disappear. It may stop being the root of the tree
252d352ac68SChris Mason * at any time because there are no locks held.
253d352ac68SChris Mason */
btrfs_root_node(struct btrfs_root * root)254925baeddSChris Mason struct extent_buffer *btrfs_root_node(struct btrfs_root *root)
255925baeddSChris Mason {
256925baeddSChris Mason struct extent_buffer *eb;
257240f62c8SChris Mason
2583083ee2eSJosef Bacik while (1) {
259240f62c8SChris Mason rcu_read_lock();
260240f62c8SChris Mason eb = rcu_dereference(root->node);
2613083ee2eSJosef Bacik
2623083ee2eSJosef Bacik /*
2633083ee2eSJosef Bacik * RCU really hurts here, we could free up the root node because
26401327610SNicholas D Steeves * it was COWed but we may not get the new root node yet so do
2653083ee2eSJosef Bacik * the inc_not_zero dance and if it doesn't work then
2663083ee2eSJosef Bacik * synchronize_rcu and try again.
2673083ee2eSJosef Bacik */
2683083ee2eSJosef Bacik if (atomic_inc_not_zero(&eb->refs)) {
269240f62c8SChris Mason rcu_read_unlock();
2703083ee2eSJosef Bacik break;
2713083ee2eSJosef Bacik }
2723083ee2eSJosef Bacik rcu_read_unlock();
2733083ee2eSJosef Bacik synchronize_rcu();
2743083ee2eSJosef Bacik }
275925baeddSChris Mason return eb;
276925baeddSChris Mason }
277925baeddSChris Mason
27892a7cc42SQu Wenruo /*
27992a7cc42SQu Wenruo * Cowonly root (not-shareable trees, everything not subvolume or reloc roots),
28092a7cc42SQu Wenruo * just get put onto a simple dirty list. Transaction walks this list to make
28192a7cc42SQu Wenruo * sure they get properly updated on disk.
282d352ac68SChris Mason */
add_root_to_dirty_list(struct btrfs_root * root)2830b86a832SChris Mason static void add_root_to_dirty_list(struct btrfs_root *root)
2840b86a832SChris Mason {
2850b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info;
2860b246afaSJeff Mahoney
287e7070be1SJosef Bacik if (test_bit(BTRFS_ROOT_DIRTY, &root->state) ||
288e7070be1SJosef Bacik !test_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state))
289e7070be1SJosef Bacik return;
290e7070be1SJosef Bacik
2910b246afaSJeff Mahoney spin_lock(&fs_info->trans_lock);
292e7070be1SJosef Bacik if (!test_and_set_bit(BTRFS_ROOT_DIRTY, &root->state)) {
293e7070be1SJosef Bacik /* Want the extent tree to be the last on the list */
2944fd786e6SMisono Tomohiro if (root->root_key.objectid == BTRFS_EXTENT_TREE_OBJECTID)
295e7070be1SJosef Bacik list_move_tail(&root->dirty_list,
2960b246afaSJeff Mahoney &fs_info->dirty_cowonly_roots);
297e7070be1SJosef Bacik else
298e7070be1SJosef Bacik list_move(&root->dirty_list,
2990b246afaSJeff Mahoney &fs_info->dirty_cowonly_roots);
3000b86a832SChris Mason }
3010b246afaSJeff Mahoney spin_unlock(&fs_info->trans_lock);
3020b86a832SChris Mason }
3030b86a832SChris Mason
304d352ac68SChris Mason /*
305d352ac68SChris Mason * used by snapshot creation to make a copy of a root for a tree with
306d352ac68SChris Mason * a given objectid. The buffer with the new root node is returned in
307d352ac68SChris Mason * cow_ret, and this func returns zero on success or a negative error code.
308d352ac68SChris Mason */
btrfs_copy_root(struct btrfs_trans_handle * trans,struct btrfs_root * root,struct extent_buffer * buf,struct extent_buffer ** cow_ret,u64 new_root_objectid)309be20aa9dSChris Mason int btrfs_copy_root(struct btrfs_trans_handle *trans,
310be20aa9dSChris Mason struct btrfs_root *root,
311be20aa9dSChris Mason struct extent_buffer *buf,
312be20aa9dSChris Mason struct extent_buffer **cow_ret, u64 new_root_objectid)
313be20aa9dSChris Mason {
3140b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info;
315be20aa9dSChris Mason struct extent_buffer *cow;
316be20aa9dSChris Mason int ret = 0;
317be20aa9dSChris Mason int level;
3185d4f98a2SYan Zheng struct btrfs_disk_key disk_key;
319be20aa9dSChris Mason
32092a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) &&
3210b246afaSJeff Mahoney trans->transid != fs_info->running_transaction->transid);
32292a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) &&
32327cdeb70SMiao Xie trans->transid != root->last_trans);
324be20aa9dSChris Mason
325be20aa9dSChris Mason level = btrfs_header_level(buf);
3265d4f98a2SYan Zheng if (level == 0)
3275d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0);
3285d4f98a2SYan Zheng else
3295d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0);
33031840ae1SZheng Yan
3314d75f8a9SDavid Sterba cow = btrfs_alloc_tree_block(trans, root, 0, new_root_objectid,
332cf6f34aaSJosef Bacik &disk_key, level, buf->start, 0,
333cf6f34aaSJosef Bacik BTRFS_NESTING_NEW_ROOT);
3345d4f98a2SYan Zheng if (IS_ERR(cow))
335be20aa9dSChris Mason return PTR_ERR(cow);
336be20aa9dSChris Mason
33758e8012cSDavid Sterba copy_extent_buffer_full(cow, buf);
338be20aa9dSChris Mason btrfs_set_header_bytenr(cow, cow->start);
339be20aa9dSChris Mason btrfs_set_header_generation(cow, trans->transid);
3405d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV);
3415d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN |
3425d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC);
3435d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID)
3445d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC);
3455d4f98a2SYan Zheng else
346be20aa9dSChris Mason btrfs_set_header_owner(cow, new_root_objectid);
347be20aa9dSChris Mason
348de37aa51SNikolay Borisov write_extent_buffer_fsid(cow, fs_info->fs_devices->metadata_uuid);
3492b82032cSYan Zheng
350be20aa9dSChris Mason WARN_ON(btrfs_header_generation(buf) > trans->transid);
3515d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID)
352e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1);
3535d4f98a2SYan Zheng else
354e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0);
355867ed321SJosef Bacik if (ret) {
35672c9925fSFilipe Manana btrfs_tree_unlock(cow);
35772c9925fSFilipe Manana free_extent_buffer(cow);
358867ed321SJosef Bacik btrfs_abort_transaction(trans, ret);
359be20aa9dSChris Mason return ret;
360867ed321SJosef Bacik }
361be20aa9dSChris Mason
362*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, cow);
363be20aa9dSChris Mason *cow_ret = cow;
364be20aa9dSChris Mason return 0;
365be20aa9dSChris Mason }
366be20aa9dSChris Mason
367d352ac68SChris Mason /*
3685d4f98a2SYan Zheng * check if the tree block can be shared by multiple trees
3695d4f98a2SYan Zheng */
btrfs_block_can_be_shared(struct btrfs_trans_handle * trans,struct btrfs_root * root,struct extent_buffer * buf)370eb96e221SFilipe Manana int btrfs_block_can_be_shared(struct btrfs_trans_handle *trans,
371eb96e221SFilipe Manana struct btrfs_root *root,
3725d4f98a2SYan Zheng struct extent_buffer *buf)
3735d4f98a2SYan Zheng {
3745d4f98a2SYan Zheng /*
37592a7cc42SQu Wenruo * Tree blocks not in shareable trees and tree roots are never shared.
37692a7cc42SQu Wenruo * If a block was allocated after the last snapshot and the block was
37792a7cc42SQu Wenruo * not allocated by tree relocation, we know the block is not shared.
3785d4f98a2SYan Zheng */
37992a7cc42SQu Wenruo if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state) &&
380eb96e221SFilipe Manana buf != root->node &&
3815d4f98a2SYan Zheng (btrfs_header_generation(buf) <=
3825d4f98a2SYan Zheng btrfs_root_last_snapshot(&root->root_item) ||
383eb96e221SFilipe Manana btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC))) {
384eb96e221SFilipe Manana if (buf != root->commit_root)
3855d4f98a2SYan Zheng return 1;
386eb96e221SFilipe Manana /*
387eb96e221SFilipe Manana * An extent buffer that used to be the commit root may still be
388eb96e221SFilipe Manana * shared because the tree height may have increased and it
389eb96e221SFilipe Manana * became a child of a higher level root. This can happen when
390eb96e221SFilipe Manana * snapshotting a subvolume created in the current transaction.
391eb96e221SFilipe Manana */
392eb96e221SFilipe Manana if (btrfs_header_generation(buf) == trans->transid)
393eb96e221SFilipe Manana return 1;
394eb96e221SFilipe Manana }
395a79865c6SNikolay Borisov
3965d4f98a2SYan Zheng return 0;
3975d4f98a2SYan Zheng }
3985d4f98a2SYan Zheng
update_ref_for_cow(struct btrfs_trans_handle * trans,struct btrfs_root * root,struct extent_buffer * buf,struct extent_buffer * cow,int * last_ref)3995d4f98a2SYan Zheng static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans,
4005d4f98a2SYan Zheng struct btrfs_root *root,
4015d4f98a2SYan Zheng struct extent_buffer *buf,
402f0486c68SYan, Zheng struct extent_buffer *cow,
403f0486c68SYan, Zheng int *last_ref)
4045d4f98a2SYan Zheng {
4050b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info;
4065d4f98a2SYan Zheng u64 refs;
4075d4f98a2SYan Zheng u64 owner;
4085d4f98a2SYan Zheng u64 flags;
4095d4f98a2SYan Zheng u64 new_flags = 0;
4105d4f98a2SYan Zheng int ret;
4115d4f98a2SYan Zheng
4125d4f98a2SYan Zheng /*
4135d4f98a2SYan Zheng * Backrefs update rules:
4145d4f98a2SYan Zheng *
4155d4f98a2SYan Zheng * Always use full backrefs for extent pointers in tree block
4165d4f98a2SYan Zheng * allocated by tree relocation.
4175d4f98a2SYan Zheng *
4185d4f98a2SYan Zheng * If a shared tree block is no longer referenced by its owner
4195d4f98a2SYan Zheng * tree (btrfs_header_owner(buf) == root->root_key.objectid),
4205d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block.
4215d4f98a2SYan Zheng *
4225d4f98a2SYan Zheng * If a tree block is been relocating
4235d4f98a2SYan Zheng * (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID),
4245d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block.
4255d4f98a2SYan Zheng * The reason for this is some operations (such as drop tree)
4265d4f98a2SYan Zheng * are only allowed for blocks use full backrefs.
4275d4f98a2SYan Zheng */
4285d4f98a2SYan Zheng
429eb96e221SFilipe Manana if (btrfs_block_can_be_shared(trans, root, buf)) {
4302ff7e61eSJeff Mahoney ret = btrfs_lookup_extent_info(trans, fs_info, buf->start,
4313173a18fSJosef Bacik btrfs_header_level(buf), 1,
4323173a18fSJosef Bacik &refs, &flags);
433be1a5564SMark Fasheh if (ret)
434be1a5564SMark Fasheh return ret;
435eced687eSFilipe Manana if (unlikely(refs == 0)) {
436eced687eSFilipe Manana btrfs_crit(fs_info,
437eced687eSFilipe Manana "found 0 references for tree block at bytenr %llu level %d root %llu",
438eced687eSFilipe Manana buf->start, btrfs_header_level(buf),
439eced687eSFilipe Manana btrfs_root_id(root));
440eced687eSFilipe Manana ret = -EUCLEAN;
441eced687eSFilipe Manana btrfs_abort_transaction(trans, ret);
442e5df9573SMark Fasheh return ret;
443e5df9573SMark Fasheh }
4445d4f98a2SYan Zheng } else {
4455d4f98a2SYan Zheng refs = 1;
4465d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID ||
4475d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV)
4485d4f98a2SYan Zheng flags = BTRFS_BLOCK_FLAG_FULL_BACKREF;
4495d4f98a2SYan Zheng else
4505d4f98a2SYan Zheng flags = 0;
4515d4f98a2SYan Zheng }
4525d4f98a2SYan Zheng
4535d4f98a2SYan Zheng owner = btrfs_header_owner(buf);
4545d4f98a2SYan Zheng BUG_ON(owner == BTRFS_TREE_RELOC_OBJECTID &&
4555d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF));
4565d4f98a2SYan Zheng
4575d4f98a2SYan Zheng if (refs > 1) {
4585d4f98a2SYan Zheng if ((owner == root->root_key.objectid ||
4595d4f98a2SYan Zheng root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) &&
4605d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) {
461e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, buf, 1);
462692826b2SJeff Mahoney if (ret)
463692826b2SJeff Mahoney return ret;
4645d4f98a2SYan Zheng
4655d4f98a2SYan Zheng if (root->root_key.objectid ==
4665d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) {
467e339a6b0SJosef Bacik ret = btrfs_dec_ref(trans, root, buf, 0);
468692826b2SJeff Mahoney if (ret)
469692826b2SJeff Mahoney return ret;
470e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1);
471692826b2SJeff Mahoney if (ret)
472692826b2SJeff Mahoney return ret;
4735d4f98a2SYan Zheng }
4745d4f98a2SYan Zheng new_flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF;
4755d4f98a2SYan Zheng } else {
4765d4f98a2SYan Zheng
4775d4f98a2SYan Zheng if (root->root_key.objectid ==
4785d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID)
479e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1);
4805d4f98a2SYan Zheng else
481e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0);
482692826b2SJeff Mahoney if (ret)
483692826b2SJeff Mahoney return ret;
4845d4f98a2SYan Zheng }
4855d4f98a2SYan Zheng if (new_flags != 0) {
4864aec05faSJosef Bacik ret = btrfs_set_disk_extent_flags(trans, buf, new_flags);
487be1a5564SMark Fasheh if (ret)
488be1a5564SMark Fasheh return ret;
4895d4f98a2SYan Zheng }
4905d4f98a2SYan Zheng } else {
4915d4f98a2SYan Zheng if (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
4925d4f98a2SYan Zheng if (root->root_key.objectid ==
4935d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID)
494e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1);
4955d4f98a2SYan Zheng else
496e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0);
497692826b2SJeff Mahoney if (ret)
498692826b2SJeff Mahoney return ret;
499e339a6b0SJosef Bacik ret = btrfs_dec_ref(trans, root, buf, 1);
500692826b2SJeff Mahoney if (ret)
501692826b2SJeff Mahoney return ret;
5025d4f98a2SYan Zheng }
503190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, buf);
504f0486c68SYan, Zheng *last_ref = 1;
5055d4f98a2SYan Zheng }
5065d4f98a2SYan Zheng return 0;
5075d4f98a2SYan Zheng }
5085d4f98a2SYan Zheng
5095d4f98a2SYan Zheng /*
510d397712bSChris Mason * does the dirty work in cow of a single block. The parent block (if
511d397712bSChris Mason * supplied) is updated to point to the new cow copy. The new buffer is marked
512d397712bSChris Mason * dirty and returned locked. If you modify the block it needs to be marked
513d397712bSChris Mason * dirty again.
514d352ac68SChris Mason *
515d352ac68SChris Mason * search_start -- an allocation hint for the new block
516d352ac68SChris Mason *
517d397712bSChris Mason * empty_size -- a hint that you plan on doing more cow. This is the size in
518d397712bSChris Mason * bytes the allocator should try to find free next to the block it returns.
519d397712bSChris Mason * This is just a hint and may be ignored by the allocator.
520d352ac68SChris Mason */
__btrfs_cow_block(struct btrfs_trans_handle * trans,struct btrfs_root * root,struct extent_buffer * buf,struct extent_buffer * parent,int parent_slot,struct extent_buffer ** cow_ret,u64 search_start,u64 empty_size,enum btrfs_lock_nesting nest)521d397712bSChris Mason static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans,
5225f39d397SChris Mason struct btrfs_root *root,
5235f39d397SChris Mason struct extent_buffer *buf,
5245f39d397SChris Mason struct extent_buffer *parent, int parent_slot,
5255f39d397SChris Mason struct extent_buffer **cow_ret,
5269631e4ccSJosef Bacik u64 search_start, u64 empty_size,
5279631e4ccSJosef Bacik enum btrfs_lock_nesting nest)
5286702ed49SChris Mason {
5290b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info;
5305d4f98a2SYan Zheng struct btrfs_disk_key disk_key;
5315f39d397SChris Mason struct extent_buffer *cow;
532be1a5564SMark Fasheh int level, ret;
533f0486c68SYan, Zheng int last_ref = 0;
534925baeddSChris Mason int unlock_orig = 0;
5350f5053ebSGoldwyn Rodrigues u64 parent_start = 0;
5366702ed49SChris Mason
537925baeddSChris Mason if (*cow_ret == buf)
538925baeddSChris Mason unlock_orig = 1;
539925baeddSChris Mason
54049d0c642SFilipe Manana btrfs_assert_tree_write_locked(buf);
541925baeddSChris Mason
54292a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) &&
5430b246afaSJeff Mahoney trans->transid != fs_info->running_transaction->transid);
54492a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) &&
54527cdeb70SMiao Xie trans->transid != root->last_trans);
5465f39d397SChris Mason
5477bb86316SChris Mason level = btrfs_header_level(buf);
54831840ae1SZheng Yan
5495d4f98a2SYan Zheng if (level == 0)
5505d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0);
5515d4f98a2SYan Zheng else
5525d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0);
5535d4f98a2SYan Zheng
5540f5053ebSGoldwyn Rodrigues if ((root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && parent)
5555d4f98a2SYan Zheng parent_start = parent->start;
5565d4f98a2SYan Zheng
55779bd3712SFilipe Manana cow = btrfs_alloc_tree_block(trans, root, parent_start,
55879bd3712SFilipe Manana root->root_key.objectid, &disk_key, level,
55979bd3712SFilipe Manana search_start, empty_size, nest);
5606702ed49SChris Mason if (IS_ERR(cow))
5616702ed49SChris Mason return PTR_ERR(cow);
5626702ed49SChris Mason
563b4ce94deSChris Mason /* cow is set to blocking by btrfs_init_new_buffer */
564b4ce94deSChris Mason
56558e8012cSDavid Sterba copy_extent_buffer_full(cow, buf);
566db94535dSChris Mason btrfs_set_header_bytenr(cow, cow->start);
5675f39d397SChris Mason btrfs_set_header_generation(cow, trans->transid);
5685d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV);
5695d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN |
5705d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC);
5715d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID)
5725d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC);
5735d4f98a2SYan Zheng else
5745f39d397SChris Mason btrfs_set_header_owner(cow, root->root_key.objectid);
5756702ed49SChris Mason
576de37aa51SNikolay Borisov write_extent_buffer_fsid(cow, fs_info->fs_devices->metadata_uuid);
5772b82032cSYan Zheng
578be1a5564SMark Fasheh ret = update_ref_for_cow(trans, root, buf, cow, &last_ref);
579b68dc2a9SMark Fasheh if (ret) {
580572c83acSJosef Bacik btrfs_tree_unlock(cow);
581572c83acSJosef Bacik free_extent_buffer(cow);
58266642832SJeff Mahoney btrfs_abort_transaction(trans, ret);
583b68dc2a9SMark Fasheh return ret;
584b68dc2a9SMark Fasheh }
5851a40e23bSZheng Yan
58692a7cc42SQu Wenruo if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) {
58783d4cfd4SJosef Bacik ret = btrfs_reloc_cow_block(trans, root, buf, cow);
58893314e3bSZhaolei if (ret) {
589572c83acSJosef Bacik btrfs_tree_unlock(cow);
590572c83acSJosef Bacik free_extent_buffer(cow);
59166642832SJeff Mahoney btrfs_abort_transaction(trans, ret);
59283d4cfd4SJosef Bacik return ret;
59383d4cfd4SJosef Bacik }
59493314e3bSZhaolei }
5953fd0a558SYan, Zheng
5966702ed49SChris Mason if (buf == root->node) {
597925baeddSChris Mason WARN_ON(parent && parent != buf);
5985d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID ||
5995d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV)
6005d4f98a2SYan Zheng parent_start = buf->start;
601925baeddSChris Mason
602406808abSFilipe Manana ret = btrfs_tree_mod_log_insert_root(root->node, cow, true);
60340b0a749SFilipe Manana if (ret < 0) {
60440b0a749SFilipe Manana btrfs_tree_unlock(cow);
60540b0a749SFilipe Manana free_extent_buffer(cow);
60640b0a749SFilipe Manana btrfs_abort_transaction(trans, ret);
60740b0a749SFilipe Manana return ret;
60840b0a749SFilipe Manana }
60940b0a749SFilipe Manana atomic_inc(&cow->refs);
610240f62c8SChris Mason rcu_assign_pointer(root->node, cow);
611925baeddSChris Mason
6127a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), buf,
6137a163608SFilipe Manana parent_start, last_ref);
6145f39d397SChris Mason free_extent_buffer(buf);
6150b86a832SChris Mason add_root_to_dirty_list(root);
6166702ed49SChris Mason } else {
6175d4f98a2SYan Zheng WARN_ON(trans->transid != btrfs_header_generation(parent));
618d09c5152SFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, parent_slot,
61933cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE);
620d09c5152SFilipe Manana if (ret) {
621d09c5152SFilipe Manana btrfs_tree_unlock(cow);
622d09c5152SFilipe Manana free_extent_buffer(cow);
623d09c5152SFilipe Manana btrfs_abort_transaction(trans, ret);
624d09c5152SFilipe Manana return ret;
625d09c5152SFilipe Manana }
6265f39d397SChris Mason btrfs_set_node_blockptr(parent, parent_slot,
627db94535dSChris Mason cow->start);
62874493f7aSChris Mason btrfs_set_node_ptr_generation(parent, parent_slot,
62974493f7aSChris Mason trans->transid);
630*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, parent);
6315de865eeSFilipe David Borba Manana if (last_ref) {
632f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_free_eb(buf);
6335de865eeSFilipe David Borba Manana if (ret) {
634572c83acSJosef Bacik btrfs_tree_unlock(cow);
635572c83acSJosef Bacik free_extent_buffer(cow);
63666642832SJeff Mahoney btrfs_abort_transaction(trans, ret);
6375de865eeSFilipe David Borba Manana return ret;
6385de865eeSFilipe David Borba Manana }
6395de865eeSFilipe David Borba Manana }
6407a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), buf,
6417a163608SFilipe Manana parent_start, last_ref);
6426702ed49SChris Mason }
643925baeddSChris Mason if (unlock_orig)
644925baeddSChris Mason btrfs_tree_unlock(buf);
6453083ee2eSJosef Bacik free_extent_buffer_stale(buf);
646*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, cow);
6476702ed49SChris Mason *cow_ret = cow;
6486702ed49SChris Mason return 0;
6496702ed49SChris Mason }
6506702ed49SChris Mason
should_cow_block(struct btrfs_trans_handle * trans,struct btrfs_root * root,struct extent_buffer * buf)6515d4f98a2SYan Zheng static inline int should_cow_block(struct btrfs_trans_handle *trans,
6525d4f98a2SYan Zheng struct btrfs_root *root,
6535d4f98a2SYan Zheng struct extent_buffer *buf)
6545d4f98a2SYan Zheng {
655f5ee5c9aSJeff Mahoney if (btrfs_is_testing(root->fs_info))
656faa2dbf0SJosef Bacik return 0;
657fccb84c9SDavid Sterba
658d1980131SDavid Sterba /* Ensure we can see the FORCE_COW bit */
659d1980131SDavid Sterba smp_mb__before_atomic();
660f1ebcc74SLiu Bo
661f1ebcc74SLiu Bo /*
662f1ebcc74SLiu Bo * We do not need to cow a block if
663f1ebcc74SLiu Bo * 1) this block is not created or changed in this transaction;
664f1ebcc74SLiu Bo * 2) this block does not belong to TREE_RELOC tree;
665f1ebcc74SLiu Bo * 3) the root is not forced COW.
666f1ebcc74SLiu Bo *
667f1ebcc74SLiu Bo * What is forced COW:
66801327610SNicholas D Steeves * when we create snapshot during committing the transaction,
66952042d8eSAndrea Gelmini * after we've finished copying src root, we must COW the shared
670f1ebcc74SLiu Bo * block to ensure the metadata consistency.
671f1ebcc74SLiu Bo */
6725d4f98a2SYan Zheng if (btrfs_header_generation(buf) == trans->transid &&
6735d4f98a2SYan Zheng !btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN) &&
6745d4f98a2SYan Zheng !(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID &&
675f1ebcc74SLiu Bo btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)) &&
67627cdeb70SMiao Xie !test_bit(BTRFS_ROOT_FORCE_COW, &root->state))
6775d4f98a2SYan Zheng return 0;
6785d4f98a2SYan Zheng return 1;
6795d4f98a2SYan Zheng }
6805d4f98a2SYan Zheng
681d352ac68SChris Mason /*
682d352ac68SChris Mason * cows a single block, see __btrfs_cow_block for the real work.
68301327610SNicholas D Steeves * This version of it has extra checks so that a block isn't COWed more than
684d352ac68SChris Mason * once per transaction, as long as it hasn't been written yet
685d352ac68SChris Mason */
btrfs_cow_block(struct btrfs_trans_handle * trans,struct btrfs_root * root,struct extent_buffer * buf,struct extent_buffer * parent,int parent_slot,struct extent_buffer ** cow_ret,enum btrfs_lock_nesting nest)686d397712bSChris Mason noinline int btrfs_cow_block(struct btrfs_trans_handle *trans,
6875f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *buf,
6885f39d397SChris Mason struct extent_buffer *parent, int parent_slot,
6899631e4ccSJosef Bacik struct extent_buffer **cow_ret,
6909631e4ccSJosef Bacik enum btrfs_lock_nesting nest)
69102217ed2SChris Mason {
6920b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info;
6936702ed49SChris Mason u64 search_start;
694f510cfecSChris Mason int ret;
695dc17ff8fSChris Mason
696a2caab29SFilipe Manana if (unlikely(test_bit(BTRFS_ROOT_DELETING, &root->state))) {
697a2caab29SFilipe Manana btrfs_abort_transaction(trans, -EUCLEAN);
698a2caab29SFilipe Manana btrfs_crit(fs_info,
699a2caab29SFilipe Manana "attempt to COW block %llu on root %llu that is being deleted",
700a2caab29SFilipe Manana buf->start, btrfs_root_id(root));
701a2caab29SFilipe Manana return -EUCLEAN;
702a2caab29SFilipe Manana }
70383354f07SJosef Bacik
70448774f3bSFilipe Manana /*
70548774f3bSFilipe Manana * COWing must happen through a running transaction, which always
70648774f3bSFilipe Manana * matches the current fs generation (it's a transaction with a state
70748774f3bSFilipe Manana * less than TRANS_STATE_UNBLOCKED). If it doesn't, then turn the fs
70848774f3bSFilipe Manana * into error state to prevent the commit of any transaction.
70948774f3bSFilipe Manana */
71048774f3bSFilipe Manana if (unlikely(trans->transaction != fs_info->running_transaction ||
71148774f3bSFilipe Manana trans->transid != fs_info->generation)) {
71248774f3bSFilipe Manana btrfs_abort_transaction(trans, -EUCLEAN);
71348774f3bSFilipe Manana btrfs_crit(fs_info,
71448774f3bSFilipe Manana "unexpected transaction when attempting to COW block %llu on root %llu, transaction %llu running transaction %llu fs generation %llu",
71548774f3bSFilipe Manana buf->start, btrfs_root_id(root), trans->transid,
71648774f3bSFilipe Manana fs_info->running_transaction->transid,
71748774f3bSFilipe Manana fs_info->generation);
71848774f3bSFilipe Manana return -EUCLEAN;
71948774f3bSFilipe Manana }
720dc17ff8fSChris Mason
7215d4f98a2SYan Zheng if (!should_cow_block(trans, root, buf)) {
72202217ed2SChris Mason *cow_ret = buf;
72302217ed2SChris Mason return 0;
72402217ed2SChris Mason }
725c487685dSChris Mason
726ee22184bSByongho Lee search_start = buf->start & ~((u64)SZ_1G - 1);
727b4ce94deSChris Mason
728f616f5cdSQu Wenruo /*
729f616f5cdSQu Wenruo * Before CoWing this block for later modification, check if it's
730f616f5cdSQu Wenruo * the subtree root and do the delayed subtree trace if needed.
731f616f5cdSQu Wenruo *
732f616f5cdSQu Wenruo * Also We don't care about the error, as it's handled internally.
733f616f5cdSQu Wenruo */
734f616f5cdSQu Wenruo btrfs_qgroup_trace_subtree_after_cow(trans, root, buf);
735f510cfecSChris Mason ret = __btrfs_cow_block(trans, root, buf, parent,
7369631e4ccSJosef Bacik parent_slot, cow_ret, search_start, 0, nest);
7371abe9b8aSliubo
7381abe9b8aSliubo trace_btrfs_cow_block(root, buf, *cow_ret);
7391abe9b8aSliubo
740f510cfecSChris Mason return ret;
7412c90e5d6SChris Mason }
742f75e2b79SJosef Bacik ALLOW_ERROR_INJECTION(btrfs_cow_block, ERRNO);
7436702ed49SChris Mason
744d352ac68SChris Mason /*
745d352ac68SChris Mason * helper function for defrag to decide if two blocks pointed to by a
746d352ac68SChris Mason * node are actually close by
747d352ac68SChris Mason */
close_blocks(u64 blocknr,u64 other,u32 blocksize)7486b80053dSChris Mason static int close_blocks(u64 blocknr, u64 other, u32 blocksize)
7496702ed49SChris Mason {
7506b80053dSChris Mason if (blocknr < other && other - (blocknr + blocksize) < 32768)
7516702ed49SChris Mason return 1;
7526b80053dSChris Mason if (blocknr > other && blocknr - (other + blocksize) < 32768)
7536702ed49SChris Mason return 1;
75402217ed2SChris Mason return 0;
75502217ed2SChris Mason }
75602217ed2SChris Mason
757ce6ef5abSDavid Sterba #ifdef __LITTLE_ENDIAN
758ce6ef5abSDavid Sterba
759ce6ef5abSDavid Sterba /*
760ce6ef5abSDavid Sterba * Compare two keys, on little-endian the disk order is same as CPU order and
761ce6ef5abSDavid Sterba * we can avoid the conversion.
762ce6ef5abSDavid Sterba */
comp_keys(const struct btrfs_disk_key * disk_key,const struct btrfs_key * k2)763ce6ef5abSDavid Sterba static int comp_keys(const struct btrfs_disk_key *disk_key,
764ce6ef5abSDavid Sterba const struct btrfs_key *k2)
765ce6ef5abSDavid Sterba {
766ce6ef5abSDavid Sterba const struct btrfs_key *k1 = (const struct btrfs_key *)disk_key;
767ce6ef5abSDavid Sterba
768ce6ef5abSDavid Sterba return btrfs_comp_cpu_keys(k1, k2);
769ce6ef5abSDavid Sterba }
770ce6ef5abSDavid Sterba
771ce6ef5abSDavid Sterba #else
772ce6ef5abSDavid Sterba
773081e9573SChris Mason /*
774081e9573SChris Mason * compare two keys in a memcmp fashion
775081e9573SChris Mason */
comp_keys(const struct btrfs_disk_key * disk,const struct btrfs_key * k2)776310712b2SOmar Sandoval static int comp_keys(const struct btrfs_disk_key *disk,
777310712b2SOmar Sandoval const struct btrfs_key *k2)
778081e9573SChris Mason {
779081e9573SChris Mason struct btrfs_key k1;
780081e9573SChris Mason
781081e9573SChris Mason btrfs_disk_key_to_cpu(&k1, disk);
782081e9573SChris Mason
78320736abaSDiego Calleja return btrfs_comp_cpu_keys(&k1, k2);
784081e9573SChris Mason }
785ce6ef5abSDavid Sterba #endif
786081e9573SChris Mason
787f3465ca4SJosef Bacik /*
788f3465ca4SJosef Bacik * same as comp_keys only with two btrfs_key's
789f3465ca4SJosef Bacik */
btrfs_comp_cpu_keys(const struct btrfs_key * k1,const struct btrfs_key * k2)790e1f60a65SDavid Sterba int __pure btrfs_comp_cpu_keys(const struct btrfs_key *k1, const struct btrfs_key *k2)
791f3465ca4SJosef Bacik {
792f3465ca4SJosef Bacik if (k1->objectid > k2->objectid)
793f3465ca4SJosef Bacik return 1;
794f3465ca4SJosef Bacik if (k1->objectid < k2->objectid)
795f3465ca4SJosef Bacik return -1;
796f3465ca4SJosef Bacik if (k1->type > k2->type)
797f3465ca4SJosef Bacik return 1;
798f3465ca4SJosef Bacik if (k1->type < k2->type)
799f3465ca4SJosef Bacik return -1;
800f3465ca4SJosef Bacik if (k1->offset > k2->offset)
801f3465ca4SJosef Bacik return 1;
802f3465ca4SJosef Bacik if (k1->offset < k2->offset)
803f3465ca4SJosef Bacik return -1;
804f3465ca4SJosef Bacik return 0;
805f3465ca4SJosef Bacik }
806081e9573SChris Mason
807d352ac68SChris Mason /*
808d352ac68SChris Mason * this is used by the defrag code to go through all the
809d352ac68SChris Mason * leaves pointed to by a node and reallocate them so that
810d352ac68SChris Mason * disk order is close to key order
811d352ac68SChris Mason */
btrfs_realloc_node(struct btrfs_trans_handle * trans,struct btrfs_root * root,struct extent_buffer * parent,int start_slot,u64 * last_ret,struct btrfs_key * progress)8126702ed49SChris Mason int btrfs_realloc_node(struct btrfs_trans_handle *trans,
8135f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *parent,
814de78b51aSEric Sandeen int start_slot, u64 *last_ret,
815a6b6e75eSChris Mason struct btrfs_key *progress)
8166702ed49SChris Mason {
8170b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info;
8186b80053dSChris Mason struct extent_buffer *cur;
8196702ed49SChris Mason u64 blocknr;
820e9d0b13bSChris Mason u64 search_start = *last_ret;
821e9d0b13bSChris Mason u64 last_block = 0;
8226702ed49SChris Mason u64 other;
8236702ed49SChris Mason u32 parent_nritems;
8246702ed49SChris Mason int end_slot;
8256702ed49SChris Mason int i;
8266702ed49SChris Mason int err = 0;
8276b80053dSChris Mason u32 blocksize;
828081e9573SChris Mason int progress_passed = 0;
829081e9573SChris Mason struct btrfs_disk_key disk_key;
8306702ed49SChris Mason
831e36f9491SFilipe Manana /*
832e36f9491SFilipe Manana * COWing must happen through a running transaction, which always
833e36f9491SFilipe Manana * matches the current fs generation (it's a transaction with a state
834e36f9491SFilipe Manana * less than TRANS_STATE_UNBLOCKED). If it doesn't, then turn the fs
835e36f9491SFilipe Manana * into error state to prevent the commit of any transaction.
836e36f9491SFilipe Manana */
837e36f9491SFilipe Manana if (unlikely(trans->transaction != fs_info->running_transaction ||
838e36f9491SFilipe Manana trans->transid != fs_info->generation)) {
839e36f9491SFilipe Manana btrfs_abort_transaction(trans, -EUCLEAN);
840e36f9491SFilipe Manana btrfs_crit(fs_info,
841e36f9491SFilipe Manana "unexpected transaction when attempting to reallocate parent %llu for root %llu, transaction %llu running transaction %llu fs generation %llu",
842e36f9491SFilipe Manana parent->start, btrfs_root_id(root), trans->transid,
843e36f9491SFilipe Manana fs_info->running_transaction->transid,
844e36f9491SFilipe Manana fs_info->generation);
845e36f9491SFilipe Manana return -EUCLEAN;
846e36f9491SFilipe Manana }
84786479a04SChris Mason
8486b80053dSChris Mason parent_nritems = btrfs_header_nritems(parent);
8490b246afaSJeff Mahoney blocksize = fs_info->nodesize;
8505dfe2be7SFilipe Manana end_slot = parent_nritems - 1;
8516702ed49SChris Mason
8525dfe2be7SFilipe Manana if (parent_nritems <= 1)
8536702ed49SChris Mason return 0;
8546702ed49SChris Mason
8555dfe2be7SFilipe Manana for (i = start_slot; i <= end_slot; i++) {
8566702ed49SChris Mason int close = 1;
857a6b6e75eSChris Mason
858081e9573SChris Mason btrfs_node_key(parent, &disk_key, i);
859081e9573SChris Mason if (!progress_passed && comp_keys(&disk_key, progress) < 0)
860081e9573SChris Mason continue;
861081e9573SChris Mason
862081e9573SChris Mason progress_passed = 1;
8636b80053dSChris Mason blocknr = btrfs_node_blockptr(parent, i);
864e9d0b13bSChris Mason if (last_block == 0)
865e9d0b13bSChris Mason last_block = blocknr;
8665708b959SChris Mason
8676702ed49SChris Mason if (i > 0) {
8686b80053dSChris Mason other = btrfs_node_blockptr(parent, i - 1);
8696b80053dSChris Mason close = close_blocks(blocknr, other, blocksize);
8706702ed49SChris Mason }
8715dfe2be7SFilipe Manana if (!close && i < end_slot) {
8726b80053dSChris Mason other = btrfs_node_blockptr(parent, i + 1);
8736b80053dSChris Mason close = close_blocks(blocknr, other, blocksize);
8746702ed49SChris Mason }
875e9d0b13bSChris Mason if (close) {
876e9d0b13bSChris Mason last_block = blocknr;
8776702ed49SChris Mason continue;
878e9d0b13bSChris Mason }
8796702ed49SChris Mason
880206983b7SJosef Bacik cur = btrfs_read_node_slot(parent, i);
881206983b7SJosef Bacik if (IS_ERR(cur))
88264c043deSLiu Bo return PTR_ERR(cur);
883e9d0b13bSChris Mason if (search_start == 0)
8846b80053dSChris Mason search_start = last_block;
885e9d0b13bSChris Mason
886e7a84565SChris Mason btrfs_tree_lock(cur);
8876b80053dSChris Mason err = __btrfs_cow_block(trans, root, cur, parent, i,
888e7a84565SChris Mason &cur, search_start,
8896b80053dSChris Mason min(16 * blocksize,
8909631e4ccSJosef Bacik (end_slot - i) * blocksize),
8919631e4ccSJosef Bacik BTRFS_NESTING_COW);
892252c38f0SYan if (err) {
893e7a84565SChris Mason btrfs_tree_unlock(cur);
8946b80053dSChris Mason free_extent_buffer(cur);
8956702ed49SChris Mason break;
896252c38f0SYan }
897e7a84565SChris Mason search_start = cur->start;
898e7a84565SChris Mason last_block = cur->start;
899f2183bdeSChris Mason *last_ret = search_start;
900e7a84565SChris Mason btrfs_tree_unlock(cur);
901e7a84565SChris Mason free_extent_buffer(cur);
9026702ed49SChris Mason }
9036702ed49SChris Mason return err;
9046702ed49SChris Mason }
9056702ed49SChris Mason
90674123bd7SChris Mason /*
907fb81212cSFilipe Manana * Search for a key in the given extent_buffer.
9085f39d397SChris Mason *
909a724f313SFilipe Manana * The lower boundary for the search is specified by the slot number @first_slot.
910fdf8d595SAnand Jain * Use a value of 0 to search over the whole extent buffer. Works for both
911fdf8d595SAnand Jain * leaves and nodes.
91274123bd7SChris Mason *
913fb81212cSFilipe Manana * The slot in the extent buffer is returned via @slot. If the key exists in the
914fb81212cSFilipe Manana * extent buffer, then @slot will point to the slot where the key is, otherwise
915fb81212cSFilipe Manana * it points to the slot where you would insert the key.
916fb81212cSFilipe Manana *
917fb81212cSFilipe Manana * Slot may point to the total number of items (i.e. one position beyond the last
918fb81212cSFilipe Manana * key) if the key is bigger than the last key in the extent buffer.
91974123bd7SChris Mason */
btrfs_bin_search(struct extent_buffer * eb,int first_slot,const struct btrfs_key * key,int * slot)920fdf8d595SAnand Jain int btrfs_bin_search(struct extent_buffer *eb, int first_slot,
92167d5e289SMarcos Paulo de Souza const struct btrfs_key *key, int *slot)
922be0e5c09SChris Mason {
923fb81212cSFilipe Manana unsigned long p;
924fb81212cSFilipe Manana int item_size;
925a724f313SFilipe Manana /*
926a724f313SFilipe Manana * Use unsigned types for the low and high slots, so that we get a more
927a724f313SFilipe Manana * efficient division in the search loop below.
928a724f313SFilipe Manana */
929a724f313SFilipe Manana u32 low = first_slot;
930a724f313SFilipe Manana u32 high = btrfs_header_nritems(eb);
931be0e5c09SChris Mason int ret;
9325cd17f34SDavid Sterba const int key_size = sizeof(struct btrfs_disk_key);
933be0e5c09SChris Mason
934a724f313SFilipe Manana if (unlikely(low > high)) {
9355e24e9afSLiu Bo btrfs_err(eb->fs_info,
936a724f313SFilipe Manana "%s: low (%u) > high (%u) eb %llu owner %llu level %d",
9375e24e9afSLiu Bo __func__, low, high, eb->start,
9385e24e9afSLiu Bo btrfs_header_owner(eb), btrfs_header_level(eb));
9395e24e9afSLiu Bo return -EINVAL;
9405e24e9afSLiu Bo }
9415e24e9afSLiu Bo
942fb81212cSFilipe Manana if (btrfs_header_level(eb) == 0) {
943fb81212cSFilipe Manana p = offsetof(struct btrfs_leaf, items);
944fb81212cSFilipe Manana item_size = sizeof(struct btrfs_item);
945fb81212cSFilipe Manana } else {
946fb81212cSFilipe Manana p = offsetof(struct btrfs_node, ptrs);
947fb81212cSFilipe Manana item_size = sizeof(struct btrfs_key_ptr);
948fb81212cSFilipe Manana }
949fb81212cSFilipe Manana
950be0e5c09SChris Mason while (low < high) {
9515cd17f34SDavid Sterba unsigned long oip;
9525cd17f34SDavid Sterba unsigned long offset;
9535cd17f34SDavid Sterba struct btrfs_disk_key *tmp;
9545cd17f34SDavid Sterba struct btrfs_disk_key unaligned;
9555cd17f34SDavid Sterba int mid;
9565cd17f34SDavid Sterba
957be0e5c09SChris Mason mid = (low + high) / 2;
9585f39d397SChris Mason offset = p + mid * item_size;
9595cd17f34SDavid Sterba oip = offset_in_page(offset);
9605f39d397SChris Mason
9615cd17f34SDavid Sterba if (oip + key_size <= PAGE_SIZE) {
962884b07d0SQu Wenruo const unsigned long idx = get_eb_page_index(offset);
9635cd17f34SDavid Sterba char *kaddr = page_address(eb->pages[idx]);
964934d375bSChris Mason
965884b07d0SQu Wenruo oip = get_eb_offset_in_page(eb, offset);
9665cd17f34SDavid Sterba tmp = (struct btrfs_disk_key *)(kaddr + oip);
9675cd17f34SDavid Sterba } else {
9685cd17f34SDavid Sterba read_extent_buffer(eb, &unaligned, offset, key_size);
9695f39d397SChris Mason tmp = &unaligned;
970479965d6SChris Mason }
971479965d6SChris Mason
972be0e5c09SChris Mason ret = comp_keys(tmp, key);
973be0e5c09SChris Mason
974be0e5c09SChris Mason if (ret < 0)
975be0e5c09SChris Mason low = mid + 1;
976be0e5c09SChris Mason else if (ret > 0)
977be0e5c09SChris Mason high = mid;
978be0e5c09SChris Mason else {
979be0e5c09SChris Mason *slot = mid;
980be0e5c09SChris Mason return 0;
981be0e5c09SChris Mason }
982be0e5c09SChris Mason }
983be0e5c09SChris Mason *slot = low;
984be0e5c09SChris Mason return 1;
985be0e5c09SChris Mason }
986be0e5c09SChris Mason
root_add_used(struct btrfs_root * root,u32 size)987f0486c68SYan, Zheng static void root_add_used(struct btrfs_root *root, u32 size)
988f0486c68SYan, Zheng {
989f0486c68SYan, Zheng spin_lock(&root->accounting_lock);
990f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item,
991f0486c68SYan, Zheng btrfs_root_used(&root->root_item) + size);
992f0486c68SYan, Zheng spin_unlock(&root->accounting_lock);
993f0486c68SYan, Zheng }
994f0486c68SYan, Zheng
root_sub_used(struct btrfs_root * root,u32 size)995f0486c68SYan, Zheng static void root_sub_used(struct btrfs_root *root, u32 size)
996f0486c68SYan, Zheng {
997f0486c68SYan, Zheng spin_lock(&root->accounting_lock);
998f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item,
999f0486c68SYan, Zheng btrfs_root_used(&root->root_item) - size);
1000f0486c68SYan, Zheng spin_unlock(&root->accounting_lock);
1001f0486c68SYan, Zheng }
1002f0486c68SYan, Zheng
1003d352ac68SChris Mason /* given a node and slot number, this reads the blocks it points to. The
1004d352ac68SChris Mason * extent buffer is returned with a reference taken (but unlocked).
1005d352ac68SChris Mason */
btrfs_read_node_slot(struct extent_buffer * parent,int slot)10064b231ae4SDavid Sterba struct extent_buffer *btrfs_read_node_slot(struct extent_buffer *parent,
10074b231ae4SDavid Sterba int slot)
1008bb803951SChris Mason {
1009ca7a79adSChris Mason int level = btrfs_header_level(parent);
1010789d6a3aSQu Wenruo struct btrfs_tree_parent_check check = { 0 };
1011416bc658SJosef Bacik struct extent_buffer *eb;
1012416bc658SJosef Bacik
1013fb770ae4SLiu Bo if (slot < 0 || slot >= btrfs_header_nritems(parent))
1014fb770ae4SLiu Bo return ERR_PTR(-ENOENT);
1015ca7a79adSChris Mason
1016d4694728SJosef Bacik ASSERT(level);
1017ca7a79adSChris Mason
1018789d6a3aSQu Wenruo check.level = level - 1;
1019789d6a3aSQu Wenruo check.transid = btrfs_node_ptr_generation(parent, slot);
1020789d6a3aSQu Wenruo check.owner_root = btrfs_header_owner(parent);
1021789d6a3aSQu Wenruo check.has_first_key = true;
1022789d6a3aSQu Wenruo btrfs_node_key_to_cpu(parent, &check.first_key, slot);
1023789d6a3aSQu Wenruo
1024d0d20b0fSDavid Sterba eb = read_tree_block(parent->fs_info, btrfs_node_blockptr(parent, slot),
1025789d6a3aSQu Wenruo &check);
10264eb150d6SQu Wenruo if (IS_ERR(eb))
10274eb150d6SQu Wenruo return eb;
10284eb150d6SQu Wenruo if (!extent_buffer_uptodate(eb)) {
1029416bc658SJosef Bacik free_extent_buffer(eb);
10304eb150d6SQu Wenruo return ERR_PTR(-EIO);
1031416bc658SJosef Bacik }
1032416bc658SJosef Bacik
1033416bc658SJosef Bacik return eb;
1034bb803951SChris Mason }
1035bb803951SChris Mason
1036d352ac68SChris Mason /*
1037d352ac68SChris Mason * node level balancing, used to make sure nodes are in proper order for
1038d352ac68SChris Mason * item deletion. We balance from the top down, so we have to make sure
1039d352ac68SChris Mason * that a deletion won't leave an node completely empty later on.
1040d352ac68SChris Mason */
balance_level(struct btrfs_trans_handle * trans,struct btrfs_root * root,struct btrfs_path * path,int level)1041e02119d5SChris Mason static noinline int balance_level(struct btrfs_trans_handle *trans,
104298ed5174SChris Mason struct btrfs_root *root,
104398ed5174SChris Mason struct btrfs_path *path, int level)
1044bb803951SChris Mason {
10450b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info;
10465f39d397SChris Mason struct extent_buffer *right = NULL;
10475f39d397SChris Mason struct extent_buffer *mid;
10485f39d397SChris Mason struct extent_buffer *left = NULL;
10495f39d397SChris Mason struct extent_buffer *parent = NULL;
1050bb803951SChris Mason int ret = 0;
1051bb803951SChris Mason int wret;
1052bb803951SChris Mason int pslot;
1053bb803951SChris Mason int orig_slot = path->slots[level];
105479f95c82SChris Mason u64 orig_ptr;
1055bb803951SChris Mason
105698e6b1ebSLiu Bo ASSERT(level > 0);
1057bb803951SChris Mason
10585f39d397SChris Mason mid = path->nodes[level];
1059b4ce94deSChris Mason
1060ac5887c8SJosef Bacik WARN_ON(path->locks[level] != BTRFS_WRITE_LOCK);
10617bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid);
10627bb86316SChris Mason
10631d4f8a0cSChris Mason orig_ptr = btrfs_node_blockptr(mid, orig_slot);
106479f95c82SChris Mason
1065a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) {
10665f39d397SChris Mason parent = path->nodes[level + 1];
1067bb803951SChris Mason pslot = path->slots[level + 1];
1068a05a9bb1SLi Zefan }
1069bb803951SChris Mason
107040689478SChris Mason /*
107140689478SChris Mason * deal with the case where there is only one pointer in the root
107240689478SChris Mason * by promoting the node below to a root
107340689478SChris Mason */
10745f39d397SChris Mason if (!parent) {
10755f39d397SChris Mason struct extent_buffer *child;
1076bb803951SChris Mason
10775f39d397SChris Mason if (btrfs_header_nritems(mid) != 1)
1078bb803951SChris Mason return 0;
1079bb803951SChris Mason
1080bb803951SChris Mason /* promote the child to a root */
10814b231ae4SDavid Sterba child = btrfs_read_node_slot(mid, 0);
1082fb770ae4SLiu Bo if (IS_ERR(child)) {
1083fb770ae4SLiu Bo ret = PTR_ERR(child);
1084daefe4d4SFilipe Manana goto out;
1085305a26afSMark Fasheh }
1086305a26afSMark Fasheh
1087925baeddSChris Mason btrfs_tree_lock(child);
10889631e4ccSJosef Bacik ret = btrfs_cow_block(trans, root, child, mid, 0, &child,
10899631e4ccSJosef Bacik BTRFS_NESTING_COW);
1090f0486c68SYan, Zheng if (ret) {
1091f0486c68SYan, Zheng btrfs_tree_unlock(child);
1092f0486c68SYan, Zheng free_extent_buffer(child);
1093daefe4d4SFilipe Manana goto out;
1094f0486c68SYan, Zheng }
10952f375ab9SYan
1096406808abSFilipe Manana ret = btrfs_tree_mod_log_insert_root(root->node, child, true);
109739020d8aSFilipe Manana if (ret < 0) {
109839020d8aSFilipe Manana btrfs_tree_unlock(child);
109939020d8aSFilipe Manana free_extent_buffer(child);
110039020d8aSFilipe Manana btrfs_abort_transaction(trans, ret);
1101daefe4d4SFilipe Manana goto out;
110239020d8aSFilipe Manana }
1103240f62c8SChris Mason rcu_assign_pointer(root->node, child);
1104925baeddSChris Mason
11050b86a832SChris Mason add_root_to_dirty_list(root);
1106925baeddSChris Mason btrfs_tree_unlock(child);
1107b4ce94deSChris Mason
1108925baeddSChris Mason path->locks[level] = 0;
1109bb803951SChris Mason path->nodes[level] = NULL;
1110190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, mid);
1111925baeddSChris Mason btrfs_tree_unlock(mid);
1112bb803951SChris Mason /* once for the path */
11135f39d397SChris Mason free_extent_buffer(mid);
1114f0486c68SYan, Zheng
1115f0486c68SYan, Zheng root_sub_used(root, mid->len);
11167a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), mid, 0, 1);
1117bb803951SChris Mason /* once for the root ptr */
11183083ee2eSJosef Bacik free_extent_buffer_stale(mid);
1119f0486c68SYan, Zheng return 0;
1120bb803951SChris Mason }
11215f39d397SChris Mason if (btrfs_header_nritems(mid) >
11220b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 4)
1123bb803951SChris Mason return 0;
1124bb803951SChris Mason
11259cf14029SJosef Bacik if (pslot) {
11264b231ae4SDavid Sterba left = btrfs_read_node_slot(parent, pslot - 1);
11279cf14029SJosef Bacik if (IS_ERR(left)) {
11289cf14029SJosef Bacik ret = PTR_ERR(left);
1129fb770ae4SLiu Bo left = NULL;
1130daefe4d4SFilipe Manana goto out;
11319cf14029SJosef Bacik }
1132fb770ae4SLiu Bo
1133bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT);
11345f39d397SChris Mason wret = btrfs_cow_block(trans, root, left,
11359631e4ccSJosef Bacik parent, pslot - 1, &left,
1136bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW);
113754aa1f4dSChris Mason if (wret) {
113854aa1f4dSChris Mason ret = wret;
1139daefe4d4SFilipe Manana goto out;
114054aa1f4dSChris Mason }
11412cc58cf2SChris Mason }
1142fb770ae4SLiu Bo
11439cf14029SJosef Bacik if (pslot + 1 < btrfs_header_nritems(parent)) {
11444b231ae4SDavid Sterba right = btrfs_read_node_slot(parent, pslot + 1);
11459cf14029SJosef Bacik if (IS_ERR(right)) {
11469cf14029SJosef Bacik ret = PTR_ERR(right);
1147fb770ae4SLiu Bo right = NULL;
1148daefe4d4SFilipe Manana goto out;
11499cf14029SJosef Bacik }
1150fb770ae4SLiu Bo
1151bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT);
11525f39d397SChris Mason wret = btrfs_cow_block(trans, root, right,
11539631e4ccSJosef Bacik parent, pslot + 1, &right,
1154bf59a5a2SJosef Bacik BTRFS_NESTING_RIGHT_COW);
11552cc58cf2SChris Mason if (wret) {
11562cc58cf2SChris Mason ret = wret;
1157daefe4d4SFilipe Manana goto out;
11582cc58cf2SChris Mason }
11592cc58cf2SChris Mason }
11602cc58cf2SChris Mason
11612cc58cf2SChris Mason /* first, try to make some room in the middle buffer */
11625f39d397SChris Mason if (left) {
11635f39d397SChris Mason orig_slot += btrfs_header_nritems(left);
1164d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 1);
116579f95c82SChris Mason if (wret < 0)
116679f95c82SChris Mason ret = wret;
1167bb803951SChris Mason }
116879f95c82SChris Mason
116979f95c82SChris Mason /*
117079f95c82SChris Mason * then try to empty the right most buffer into the middle
117179f95c82SChris Mason */
11725f39d397SChris Mason if (right) {
1173d30a668fSDavid Sterba wret = push_node_left(trans, mid, right, 1);
117454aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC)
117579f95c82SChris Mason ret = wret;
11765f39d397SChris Mason if (btrfs_header_nritems(right) == 0) {
1177190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, right);
1178925baeddSChris Mason btrfs_tree_unlock(right);
1179751a2761SFilipe Manana ret = btrfs_del_ptr(trans, root, path, level + 1, pslot + 1);
1180751a2761SFilipe Manana if (ret < 0) {
1181751a2761SFilipe Manana free_extent_buffer_stale(right);
1182751a2761SFilipe Manana right = NULL;
1183751a2761SFilipe Manana goto out;
1184751a2761SFilipe Manana }
1185f0486c68SYan, Zheng root_sub_used(root, right->len);
11867a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), right,
11877a163608SFilipe Manana 0, 1);
11883083ee2eSJosef Bacik free_extent_buffer_stale(right);
1189f0486c68SYan, Zheng right = NULL;
1190bb803951SChris Mason } else {
11915f39d397SChris Mason struct btrfs_disk_key right_key;
11925f39d397SChris Mason btrfs_node_key(right, &right_key, 0);
1193f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot + 1,
119433cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE);
119539020d8aSFilipe Manana if (ret < 0) {
119639020d8aSFilipe Manana btrfs_abort_transaction(trans, ret);
1197daefe4d4SFilipe Manana goto out;
119839020d8aSFilipe Manana }
11995f39d397SChris Mason btrfs_set_node_key(parent, &right_key, pslot + 1);
1200*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, parent);
1201bb803951SChris Mason }
1202bb803951SChris Mason }
12035f39d397SChris Mason if (btrfs_header_nritems(mid) == 1) {
120479f95c82SChris Mason /*
120579f95c82SChris Mason * we're not allowed to leave a node with one item in the
120679f95c82SChris Mason * tree during a delete. A deletion from lower in the tree
120779f95c82SChris Mason * could try to delete the only pointer in this node.
120879f95c82SChris Mason * So, pull some keys from the left.
120979f95c82SChris Mason * There has to be a left pointer at this point because
121079f95c82SChris Mason * otherwise we would have pulled some pointers from the
121179f95c82SChris Mason * right
121279f95c82SChris Mason */
1213725026edSFilipe Manana if (unlikely(!left)) {
1214725026edSFilipe Manana btrfs_crit(fs_info,
1215725026edSFilipe Manana "missing left child when middle child only has 1 item, parent bytenr %llu level %d mid bytenr %llu root %llu",
1216725026edSFilipe Manana parent->start, btrfs_header_level(parent),
1217725026edSFilipe Manana mid->start, btrfs_root_id(root));
1218725026edSFilipe Manana ret = -EUCLEAN;
1219725026edSFilipe Manana btrfs_abort_transaction(trans, ret);
1220daefe4d4SFilipe Manana goto out;
1221305a26afSMark Fasheh }
122255d32ed8SDavid Sterba wret = balance_node_right(trans, mid, left);
122354aa1f4dSChris Mason if (wret < 0) {
122479f95c82SChris Mason ret = wret;
1225daefe4d4SFilipe Manana goto out;
122654aa1f4dSChris Mason }
1227bce4eae9SChris Mason if (wret == 1) {
1228d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 1);
1229bce4eae9SChris Mason if (wret < 0)
1230bce4eae9SChris Mason ret = wret;
1231bce4eae9SChris Mason }
123279f95c82SChris Mason BUG_ON(wret == 1);
123379f95c82SChris Mason }
12345f39d397SChris Mason if (btrfs_header_nritems(mid) == 0) {
1235190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, mid);
1236925baeddSChris Mason btrfs_tree_unlock(mid);
1237751a2761SFilipe Manana ret = btrfs_del_ptr(trans, root, path, level + 1, pslot);
1238751a2761SFilipe Manana if (ret < 0) {
1239751a2761SFilipe Manana free_extent_buffer_stale(mid);
1240751a2761SFilipe Manana mid = NULL;
1241751a2761SFilipe Manana goto out;
1242751a2761SFilipe Manana }
1243f0486c68SYan, Zheng root_sub_used(root, mid->len);
12447a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), mid, 0, 1);
12453083ee2eSJosef Bacik free_extent_buffer_stale(mid);
1246f0486c68SYan, Zheng mid = NULL;
124779f95c82SChris Mason } else {
124879f95c82SChris Mason /* update the parent key to reflect our changes */
12495f39d397SChris Mason struct btrfs_disk_key mid_key;
12505f39d397SChris Mason btrfs_node_key(mid, &mid_key, 0);
1251f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot,
125233cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE);
125339020d8aSFilipe Manana if (ret < 0) {
125439020d8aSFilipe Manana btrfs_abort_transaction(trans, ret);
1255daefe4d4SFilipe Manana goto out;
125639020d8aSFilipe Manana }
12575f39d397SChris Mason btrfs_set_node_key(parent, &mid_key, pslot);
1258*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, parent);
125979f95c82SChris Mason }
1260bb803951SChris Mason
126179f95c82SChris Mason /* update the path */
12625f39d397SChris Mason if (left) {
12635f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) {
126467439dadSDavid Sterba atomic_inc(&left->refs);
1265925baeddSChris Mason /* left was locked after cow */
12665f39d397SChris Mason path->nodes[level] = left;
1267bb803951SChris Mason path->slots[level + 1] -= 1;
1268bb803951SChris Mason path->slots[level] = orig_slot;
1269925baeddSChris Mason if (mid) {
1270925baeddSChris Mason btrfs_tree_unlock(mid);
12715f39d397SChris Mason free_extent_buffer(mid);
1272925baeddSChris Mason }
1273bb803951SChris Mason } else {
12745f39d397SChris Mason orig_slot -= btrfs_header_nritems(left);
1275bb803951SChris Mason path->slots[level] = orig_slot;
1276bb803951SChris Mason }
1277bb803951SChris Mason }
127879f95c82SChris Mason /* double check we haven't messed things up */
1279e20d96d6SChris Mason if (orig_ptr !=
12805f39d397SChris Mason btrfs_node_blockptr(path->nodes[level], path->slots[level]))
128179f95c82SChris Mason BUG();
1282daefe4d4SFilipe Manana out:
1283925baeddSChris Mason if (right) {
1284925baeddSChris Mason btrfs_tree_unlock(right);
12855f39d397SChris Mason free_extent_buffer(right);
1286925baeddSChris Mason }
1287925baeddSChris Mason if (left) {
1288925baeddSChris Mason if (path->nodes[level] != left)
1289925baeddSChris Mason btrfs_tree_unlock(left);
12905f39d397SChris Mason free_extent_buffer(left);
1291925baeddSChris Mason }
1292bb803951SChris Mason return ret;
1293bb803951SChris Mason }
1294bb803951SChris Mason
1295d352ac68SChris Mason /* Node balancing for insertion. Here we only split or push nodes around
1296d352ac68SChris Mason * when they are completely full. This is also done top down, so we
1297d352ac68SChris Mason * have to be pessimistic.
1298d352ac68SChris Mason */
push_nodes_for_insert(struct btrfs_trans_handle * trans,struct btrfs_root * root,struct btrfs_path * path,int level)1299d397712bSChris Mason static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans,
1300e66f709bSChris Mason struct btrfs_root *root,
1301e66f709bSChris Mason struct btrfs_path *path, int level)
1302e66f709bSChris Mason {
13030b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info;
13045f39d397SChris Mason struct extent_buffer *right = NULL;
13055f39d397SChris Mason struct extent_buffer *mid;
13065f39d397SChris Mason struct extent_buffer *left = NULL;
13075f39d397SChris Mason struct extent_buffer *parent = NULL;
1308e66f709bSChris Mason int ret = 0;
1309e66f709bSChris Mason int wret;
1310e66f709bSChris Mason int pslot;
1311e66f709bSChris Mason int orig_slot = path->slots[level];
1312e66f709bSChris Mason
1313e66f709bSChris Mason if (level == 0)
1314e66f709bSChris Mason return 1;
1315e66f709bSChris Mason
13165f39d397SChris Mason mid = path->nodes[level];
13177bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid);
1318e66f709bSChris Mason
1319a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) {
13205f39d397SChris Mason parent = path->nodes[level + 1];
1321e66f709bSChris Mason pslot = path->slots[level + 1];
1322a05a9bb1SLi Zefan }
1323e66f709bSChris Mason
13245f39d397SChris Mason if (!parent)
1325e66f709bSChris Mason return 1;
1326e66f709bSChris Mason
13279cf14029SJosef Bacik /* first, try to make some room in the middle buffer */
13289cf14029SJosef Bacik if (pslot) {
13299cf14029SJosef Bacik u32 left_nr;
13309cf14029SJosef Bacik
13314b231ae4SDavid Sterba left = btrfs_read_node_slot(parent, pslot - 1);
1332fb770ae4SLiu Bo if (IS_ERR(left))
13339cf14029SJosef Bacik return PTR_ERR(left);
1334925baeddSChris Mason
1335bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT);
1336b4ce94deSChris Mason
13375f39d397SChris Mason left_nr = btrfs_header_nritems(left);
13380b246afaSJeff Mahoney if (left_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) {
133933ade1f8SChris Mason wret = 1;
134033ade1f8SChris Mason } else {
13415f39d397SChris Mason ret = btrfs_cow_block(trans, root, left, parent,
13429631e4ccSJosef Bacik pslot - 1, &left,
1343bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW);
134454aa1f4dSChris Mason if (ret)
134554aa1f4dSChris Mason wret = 1;
134654aa1f4dSChris Mason else {
1347d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 0);
134854aa1f4dSChris Mason }
134933ade1f8SChris Mason }
1350e66f709bSChris Mason if (wret < 0)
1351e66f709bSChris Mason ret = wret;
1352e66f709bSChris Mason if (wret == 0) {
13535f39d397SChris Mason struct btrfs_disk_key disk_key;
1354e66f709bSChris Mason orig_slot += left_nr;
13555f39d397SChris Mason btrfs_node_key(mid, &disk_key, 0);
1356f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot,
135733cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE);
135811d6ae03SFilipe Manana if (ret < 0) {
135911d6ae03SFilipe Manana btrfs_tree_unlock(left);
136011d6ae03SFilipe Manana free_extent_buffer(left);
136111d6ae03SFilipe Manana btrfs_abort_transaction(trans, ret);
136211d6ae03SFilipe Manana return ret;
136311d6ae03SFilipe Manana }
13645f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot);
1365*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, parent);
13665f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) {
13675f39d397SChris Mason path->nodes[level] = left;
1368e66f709bSChris Mason path->slots[level + 1] -= 1;
1369e66f709bSChris Mason path->slots[level] = orig_slot;
1370925baeddSChris Mason btrfs_tree_unlock(mid);
13715f39d397SChris Mason free_extent_buffer(mid);
1372e66f709bSChris Mason } else {
1373e66f709bSChris Mason orig_slot -=
13745f39d397SChris Mason btrfs_header_nritems(left);
1375e66f709bSChris Mason path->slots[level] = orig_slot;
1376925baeddSChris Mason btrfs_tree_unlock(left);
13775f39d397SChris Mason free_extent_buffer(left);
1378e66f709bSChris Mason }
1379e66f709bSChris Mason return 0;
1380e66f709bSChris Mason }
1381925baeddSChris Mason btrfs_tree_unlock(left);
13825f39d397SChris Mason free_extent_buffer(left);
1383e66f709bSChris Mason }
1384e66f709bSChris Mason
1385e66f709bSChris Mason /*
1386e66f709bSChris Mason * then try to empty the right most buffer into the middle
1387e66f709bSChris Mason */
13889cf14029SJosef Bacik if (pslot + 1 < btrfs_header_nritems(parent)) {
138933ade1f8SChris Mason u32 right_nr;
1390b4ce94deSChris Mason
13919cf14029SJosef Bacik right = btrfs_read_node_slot(parent, pslot + 1);
13929cf14029SJosef Bacik if (IS_ERR(right))
13939cf14029SJosef Bacik return PTR_ERR(right);
13949cf14029SJosef Bacik
1395bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT);
1396b4ce94deSChris Mason
13975f39d397SChris Mason right_nr = btrfs_header_nritems(right);
13980b246afaSJeff Mahoney if (right_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) {
139933ade1f8SChris Mason wret = 1;
140033ade1f8SChris Mason } else {
14015f39d397SChris Mason ret = btrfs_cow_block(trans, root, right,
14025f39d397SChris Mason parent, pslot + 1,
1403bf59a5a2SJosef Bacik &right, BTRFS_NESTING_RIGHT_COW);
140454aa1f4dSChris Mason if (ret)
140554aa1f4dSChris Mason wret = 1;
140654aa1f4dSChris Mason else {
140755d32ed8SDavid Sterba wret = balance_node_right(trans, right, mid);
140833ade1f8SChris Mason }
140954aa1f4dSChris Mason }
1410e66f709bSChris Mason if (wret < 0)
1411e66f709bSChris Mason ret = wret;
1412e66f709bSChris Mason if (wret == 0) {
14135f39d397SChris Mason struct btrfs_disk_key disk_key;
14145f39d397SChris Mason
14155f39d397SChris Mason btrfs_node_key(right, &disk_key, 0);
1416f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot + 1,
141733cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE);
141811d6ae03SFilipe Manana if (ret < 0) {
141911d6ae03SFilipe Manana btrfs_tree_unlock(right);
142011d6ae03SFilipe Manana free_extent_buffer(right);
142111d6ae03SFilipe Manana btrfs_abort_transaction(trans, ret);
142211d6ae03SFilipe Manana return ret;
142311d6ae03SFilipe Manana }
14245f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot + 1);
1425*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, parent);
14265f39d397SChris Mason
14275f39d397SChris Mason if (btrfs_header_nritems(mid) <= orig_slot) {
14285f39d397SChris Mason path->nodes[level] = right;
1429e66f709bSChris Mason path->slots[level + 1] += 1;
1430e66f709bSChris Mason path->slots[level] = orig_slot -
14315f39d397SChris Mason btrfs_header_nritems(mid);
1432925baeddSChris Mason btrfs_tree_unlock(mid);
14335f39d397SChris Mason free_extent_buffer(mid);
1434e66f709bSChris Mason } else {
1435925baeddSChris Mason btrfs_tree_unlock(right);
14365f39d397SChris Mason free_extent_buffer(right);
1437e66f709bSChris Mason }
1438e66f709bSChris Mason return 0;
1439e66f709bSChris Mason }
1440925baeddSChris Mason btrfs_tree_unlock(right);
14415f39d397SChris Mason free_extent_buffer(right);
1442e66f709bSChris Mason }
1443e66f709bSChris Mason return 1;
1444e66f709bSChris Mason }
1445e66f709bSChris Mason
144674123bd7SChris Mason /*
1447d352ac68SChris Mason * readahead one full node of leaves, finding things that are close
1448d352ac68SChris Mason * to the block in 'slot', and triggering ra on them.
14493c69faecSChris Mason */
reada_for_search(struct btrfs_fs_info * fs_info,struct btrfs_path * path,int level,int slot,u64 objectid)14502ff7e61eSJeff Mahoney static void reada_for_search(struct btrfs_fs_info *fs_info,
1451e02119d5SChris Mason struct btrfs_path *path,
145201f46658SChris Mason int level, int slot, u64 objectid)
14533c69faecSChris Mason {
14545f39d397SChris Mason struct extent_buffer *node;
145501f46658SChris Mason struct btrfs_disk_key disk_key;
14563c69faecSChris Mason u32 nritems;
14573c69faecSChris Mason u64 search;
1458a7175319SChris Mason u64 target;
14596b80053dSChris Mason u64 nread = 0;
1460ace75066SFilipe Manana u64 nread_max;
14616b80053dSChris Mason u32 nr;
14626b80053dSChris Mason u32 blocksize;
14636b80053dSChris Mason u32 nscan = 0;
1464db94535dSChris Mason
1465ace75066SFilipe Manana if (level != 1 && path->reada != READA_FORWARD_ALWAYS)
14663c69faecSChris Mason return;
14673c69faecSChris Mason
14686702ed49SChris Mason if (!path->nodes[level])
14696702ed49SChris Mason return;
14706702ed49SChris Mason
14715f39d397SChris Mason node = path->nodes[level];
1472925baeddSChris Mason
1473ace75066SFilipe Manana /*
1474ace75066SFilipe Manana * Since the time between visiting leaves is much shorter than the time
1475ace75066SFilipe Manana * between visiting nodes, limit read ahead of nodes to 1, to avoid too
1476ace75066SFilipe Manana * much IO at once (possibly random).
1477ace75066SFilipe Manana */
1478ace75066SFilipe Manana if (path->reada == READA_FORWARD_ALWAYS) {
1479ace75066SFilipe Manana if (level > 1)
1480ace75066SFilipe Manana nread_max = node->fs_info->nodesize;
1481ace75066SFilipe Manana else
1482ace75066SFilipe Manana nread_max = SZ_128K;
1483ace75066SFilipe Manana } else {
1484ace75066SFilipe Manana nread_max = SZ_64K;
1485ace75066SFilipe Manana }
1486ace75066SFilipe Manana
14873c69faecSChris Mason search = btrfs_node_blockptr(node, slot);
14880b246afaSJeff Mahoney blocksize = fs_info->nodesize;
1489069a2e37SFilipe Manana if (path->reada != READA_FORWARD_ALWAYS) {
1490069a2e37SFilipe Manana struct extent_buffer *eb;
1491069a2e37SFilipe Manana
14920b246afaSJeff Mahoney eb = find_extent_buffer(fs_info, search);
14935f39d397SChris Mason if (eb) {
14945f39d397SChris Mason free_extent_buffer(eb);
14953c69faecSChris Mason return;
14963c69faecSChris Mason }
1497069a2e37SFilipe Manana }
14983c69faecSChris Mason
1499a7175319SChris Mason target = search;
15006b80053dSChris Mason
15015f39d397SChris Mason nritems = btrfs_header_nritems(node);
15026b80053dSChris Mason nr = slot;
150325b8b936SJosef Bacik
15043c69faecSChris Mason while (1) {
1505e4058b54SDavid Sterba if (path->reada == READA_BACK) {
15066b80053dSChris Mason if (nr == 0)
15073c69faecSChris Mason break;
15086b80053dSChris Mason nr--;
1509ace75066SFilipe Manana } else if (path->reada == READA_FORWARD ||
1510ace75066SFilipe Manana path->reada == READA_FORWARD_ALWAYS) {
15116b80053dSChris Mason nr++;
15126b80053dSChris Mason if (nr >= nritems)
15136b80053dSChris Mason break;
15143c69faecSChris Mason }
1515e4058b54SDavid Sterba if (path->reada == READA_BACK && objectid) {
151601f46658SChris Mason btrfs_node_key(node, &disk_key, nr);
151701f46658SChris Mason if (btrfs_disk_key_objectid(&disk_key) != objectid)
151801f46658SChris Mason break;
151901f46658SChris Mason }
15206b80053dSChris Mason search = btrfs_node_blockptr(node, nr);
1521ace75066SFilipe Manana if (path->reada == READA_FORWARD_ALWAYS ||
1522ace75066SFilipe Manana (search <= target && target - search <= 65536) ||
1523a7175319SChris Mason (search > target && search - target <= 65536)) {
1524bfb484d9SJosef Bacik btrfs_readahead_node_child(node, nr);
15256b80053dSChris Mason nread += blocksize;
15263c69faecSChris Mason }
15276b80053dSChris Mason nscan++;
1528ace75066SFilipe Manana if (nread > nread_max || nscan > 32)
15296b80053dSChris Mason break;
15303c69faecSChris Mason }
15313c69faecSChris Mason }
1532925baeddSChris Mason
reada_for_balance(struct btrfs_path * path,int level)1533bfb484d9SJosef Bacik static noinline void reada_for_balance(struct btrfs_path *path, int level)
1534b4ce94deSChris Mason {
1535bfb484d9SJosef Bacik struct extent_buffer *parent;
1536b4ce94deSChris Mason int slot;
1537b4ce94deSChris Mason int nritems;
1538b4ce94deSChris Mason
15398c594ea8SChris Mason parent = path->nodes[level + 1];
1540b4ce94deSChris Mason if (!parent)
15410b08851fSJosef Bacik return;
1542b4ce94deSChris Mason
1543b4ce94deSChris Mason nritems = btrfs_header_nritems(parent);
15448c594ea8SChris Mason slot = path->slots[level + 1];
1545b4ce94deSChris Mason
1546bfb484d9SJosef Bacik if (slot > 0)
1547bfb484d9SJosef Bacik btrfs_readahead_node_child(parent, slot - 1);
1548bfb484d9SJosef Bacik if (slot + 1 < nritems)
1549bfb484d9SJosef Bacik btrfs_readahead_node_child(parent, slot + 1);
1550b4ce94deSChris Mason }
1551b4ce94deSChris Mason
1552b4ce94deSChris Mason
1553b4ce94deSChris Mason /*
1554d397712bSChris Mason * when we walk down the tree, it is usually safe to unlock the higher layers
1555d397712bSChris Mason * in the tree. The exceptions are when our path goes through slot 0, because
1556d397712bSChris Mason * operations on the tree might require changing key pointers higher up in the
1557d397712bSChris Mason * tree.
1558d352ac68SChris Mason *
1559d397712bSChris Mason * callers might also have set path->keep_locks, which tells this code to keep
1560d397712bSChris Mason * the lock if the path points to the last slot in the block. This is part of
1561d397712bSChris Mason * walking through the tree, and selecting the next slot in the higher block.
1562d352ac68SChris Mason *
1563d397712bSChris Mason * lowest_unlock sets the lowest level in the tree we're allowed to unlock. so
1564d397712bSChris Mason * if lowest_unlock is 1, level 0 won't be unlocked
1565d352ac68SChris Mason */
unlock_up(struct btrfs_path * path,int level,int lowest_unlock,int min_write_lock_level,int * write_lock_level)1566e02119d5SChris Mason static noinline void unlock_up(struct btrfs_path *path, int level,
1567f7c79f30SChris Mason int lowest_unlock, int min_write_lock_level,
1568f7c79f30SChris Mason int *write_lock_level)
1569925baeddSChris Mason {
1570925baeddSChris Mason int i;
1571925baeddSChris Mason int skip_level = level;
1572c1227996SNikolay Borisov bool check_skip = true;
1573925baeddSChris Mason
1574925baeddSChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) {
1575925baeddSChris Mason if (!path->nodes[i])
1576925baeddSChris Mason break;
1577925baeddSChris Mason if (!path->locks[i])
1578925baeddSChris Mason break;
1579c1227996SNikolay Borisov
1580c1227996SNikolay Borisov if (check_skip) {
1581c1227996SNikolay Borisov if (path->slots[i] == 0) {
1582925baeddSChris Mason skip_level = i + 1;
1583925baeddSChris Mason continue;
1584925baeddSChris Mason }
1585c1227996SNikolay Borisov
1586c1227996SNikolay Borisov if (path->keep_locks) {
1587925baeddSChris Mason u32 nritems;
1588c1227996SNikolay Borisov
1589c1227996SNikolay Borisov nritems = btrfs_header_nritems(path->nodes[i]);
1590051e1b9fSChris Mason if (nritems < 1 || path->slots[i] >= nritems - 1) {
1591925baeddSChris Mason skip_level = i + 1;
1592925baeddSChris Mason continue;
1593925baeddSChris Mason }
1594925baeddSChris Mason }
1595c1227996SNikolay Borisov }
1596051e1b9fSChris Mason
1597d80bb3f9SLiu Bo if (i >= lowest_unlock && i > skip_level) {
1598c1227996SNikolay Borisov check_skip = false;
1599c1227996SNikolay Borisov btrfs_tree_unlock_rw(path->nodes[i], path->locks[i]);
1600925baeddSChris Mason path->locks[i] = 0;
1601f7c79f30SChris Mason if (write_lock_level &&
1602f7c79f30SChris Mason i > min_write_lock_level &&
1603f7c79f30SChris Mason i <= *write_lock_level) {
1604f7c79f30SChris Mason *write_lock_level = i - 1;
1605f7c79f30SChris Mason }
1606925baeddSChris Mason }
1607925baeddSChris Mason }
1608925baeddSChris Mason }
1609925baeddSChris Mason
16103c69faecSChris Mason /*
1611376a21d7SFilipe Manana * Helper function for btrfs_search_slot() and other functions that do a search
1612376a21d7SFilipe Manana * on a btree. The goal is to find a tree block in the cache (the radix tree at
1613376a21d7SFilipe Manana * fs_info->buffer_radix), but if we can't find it, or it's not up to date, read
1614376a21d7SFilipe Manana * its pages from disk.
1615c8c42864SChris Mason *
1616376a21d7SFilipe Manana * Returns -EAGAIN, with the path unlocked, if the caller needs to repeat the
1617376a21d7SFilipe Manana * whole btree search, starting again from the current root node.
1618c8c42864SChris Mason */
1619c8c42864SChris Mason static int
read_block_for_search(struct btrfs_root * root,struct btrfs_path * p,struct extent_buffer ** eb_ret,int level,int slot,const struct btrfs_key * key)1620d07b8528SLiu Bo read_block_for_search(struct btrfs_root *root, struct btrfs_path *p,
1621c8c42864SChris Mason struct extent_buffer **eb_ret, int level, int slot,
1622cda79c54SDavid Sterba const struct btrfs_key *key)
1623c8c42864SChris Mason {
16240b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info;
1625789d6a3aSQu Wenruo struct btrfs_tree_parent_check check = { 0 };
1626c8c42864SChris Mason u64 blocknr;
1627c8c42864SChris Mason u64 gen;
1628c8c42864SChris Mason struct extent_buffer *tmp;
162976a05b35SChris Mason int ret;
1630581c1760SQu Wenruo int parent_level;
1631b246666eSFilipe Manana bool unlock_up;
1632c8c42864SChris Mason
1633b246666eSFilipe Manana unlock_up = ((level + 1 < BTRFS_MAX_LEVEL) && p->locks[level + 1]);
1634213ff4b7SNikolay Borisov blocknr = btrfs_node_blockptr(*eb_ret, slot);
1635213ff4b7SNikolay Borisov gen = btrfs_node_ptr_generation(*eb_ret, slot);
1636213ff4b7SNikolay Borisov parent_level = btrfs_header_level(*eb_ret);
1637789d6a3aSQu Wenruo btrfs_node_key_to_cpu(*eb_ret, &check.first_key, slot);
1638789d6a3aSQu Wenruo check.has_first_key = true;
1639789d6a3aSQu Wenruo check.level = parent_level - 1;
1640789d6a3aSQu Wenruo check.transid = gen;
1641789d6a3aSQu Wenruo check.owner_root = root->root_key.objectid;
1642c8c42864SChris Mason
1643b246666eSFilipe Manana /*
1644b246666eSFilipe Manana * If we need to read an extent buffer from disk and we are holding locks
1645b246666eSFilipe Manana * on upper level nodes, we unlock all the upper nodes before reading the
1646b246666eSFilipe Manana * extent buffer, and then return -EAGAIN to the caller as it needs to
1647b246666eSFilipe Manana * restart the search. We don't release the lock on the current level
1648b246666eSFilipe Manana * because we need to walk this node to figure out which blocks to read.
1649b246666eSFilipe Manana */
16500b246afaSJeff Mahoney tmp = find_extent_buffer(fs_info, blocknr);
1651cb44921aSChris Mason if (tmp) {
1652ace75066SFilipe Manana if (p->reada == READA_FORWARD_ALWAYS)
1653ace75066SFilipe Manana reada_for_search(fs_info, p, level, slot, key->objectid);
1654ace75066SFilipe Manana
1655b9fab919SChris Mason /* first we do an atomic uptodate check */
1656b9fab919SChris Mason if (btrfs_buffer_uptodate(tmp, gen, 1) > 0) {
1657448de471SQu Wenruo /*
1658448de471SQu Wenruo * Do extra check for first_key, eb can be stale due to
1659448de471SQu Wenruo * being cached, read from scrub, or have multiple
1660448de471SQu Wenruo * parents (shared tree blocks).
1661448de471SQu Wenruo */
1662e064d5e9SDavid Sterba if (btrfs_verify_level_key(tmp,
1663789d6a3aSQu Wenruo parent_level - 1, &check.first_key, gen)) {
1664448de471SQu Wenruo free_extent_buffer(tmp);
1665448de471SQu Wenruo return -EUCLEAN;
1666448de471SQu Wenruo }
1667c8c42864SChris Mason *eb_ret = tmp;
1668c8c42864SChris Mason return 0;
1669c8c42864SChris Mason }
1670bdf7c00eSJosef Bacik
1671857bc13fSJosef Bacik if (p->nowait) {
1672857bc13fSJosef Bacik free_extent_buffer(tmp);
1673857bc13fSJosef Bacik return -EAGAIN;
1674857bc13fSJosef Bacik }
1675857bc13fSJosef Bacik
1676b246666eSFilipe Manana if (unlock_up)
1677b246666eSFilipe Manana btrfs_unlock_up_safe(p, level + 1);
1678b246666eSFilipe Manana
1679b9fab919SChris Mason /* now we're allowed to do a blocking uptodate check */
1680789d6a3aSQu Wenruo ret = btrfs_read_extent_buffer(tmp, &check);
16819a4ffa1bSQu Wenruo if (ret) {
1682cb44921aSChris Mason free_extent_buffer(tmp);
1683b3b4aa74SDavid Sterba btrfs_release_path(p);
1684cb44921aSChris Mason return -EIO;
1685cb44921aSChris Mason }
168688c602abSQu Wenruo if (btrfs_check_eb_owner(tmp, root->root_key.objectid)) {
168788c602abSQu Wenruo free_extent_buffer(tmp);
168888c602abSQu Wenruo btrfs_release_path(p);
168988c602abSQu Wenruo return -EUCLEAN;
169088c602abSQu Wenruo }
1691b246666eSFilipe Manana
1692b246666eSFilipe Manana if (unlock_up)
1693b246666eSFilipe Manana ret = -EAGAIN;
1694b246666eSFilipe Manana
1695b246666eSFilipe Manana goto out;
1696857bc13fSJosef Bacik } else if (p->nowait) {
1697857bc13fSJosef Bacik return -EAGAIN;
16989a4ffa1bSQu Wenruo }
1699c8c42864SChris Mason
1700b246666eSFilipe Manana if (unlock_up) {
17018c594ea8SChris Mason btrfs_unlock_up_safe(p, level + 1);
17024bb59055SFilipe Manana ret = -EAGAIN;
17034bb59055SFilipe Manana } else {
17044bb59055SFilipe Manana ret = 0;
17054bb59055SFilipe Manana }
17068c594ea8SChris Mason
1707e4058b54SDavid Sterba if (p->reada != READA_NONE)
17082ff7e61eSJeff Mahoney reada_for_search(fs_info, p, level, slot, key->objectid);
1709c8c42864SChris Mason
1710789d6a3aSQu Wenruo tmp = read_tree_block(fs_info, blocknr, &check);
17114eb150d6SQu Wenruo if (IS_ERR(tmp)) {
17124eb150d6SQu Wenruo btrfs_release_path(p);
17134eb150d6SQu Wenruo return PTR_ERR(tmp);
17144eb150d6SQu Wenruo }
171576a05b35SChris Mason /*
171676a05b35SChris Mason * If the read above didn't mark this buffer up to date,
171776a05b35SChris Mason * it will never end up being up to date. Set ret to EIO now
171876a05b35SChris Mason * and give up so that our caller doesn't loop forever
171976a05b35SChris Mason * on our EAGAINs.
172076a05b35SChris Mason */
1721e6a1d6fdSLiu Bo if (!extent_buffer_uptodate(tmp))
172276a05b35SChris Mason ret = -EIO;
172302a3307aSLiu Bo
1724b246666eSFilipe Manana out:
17254bb59055SFilipe Manana if (ret == 0) {
17264bb59055SFilipe Manana *eb_ret = tmp;
17274bb59055SFilipe Manana } else {
17284bb59055SFilipe Manana free_extent_buffer(tmp);
172902a3307aSLiu Bo btrfs_release_path(p);
17304bb59055SFilipe Manana }
17314bb59055SFilipe Manana
173276a05b35SChris Mason return ret;
1733c8c42864SChris Mason }
1734c8c42864SChris Mason
1735c8c42864SChris Mason /*
1736c8c42864SChris Mason * helper function for btrfs_search_slot. This does all of the checks
1737c8c42864SChris Mason * for node-level blocks and does any balancing required based on
1738c8c42864SChris Mason * the ins_len.
1739c8c42864SChris Mason *
1740c8c42864SChris Mason * If no extra work was required, zero is returned. If we had to
1741c8c42864SChris Mason * drop the path, -EAGAIN is returned and btrfs_search_slot must
1742c8c42864SChris Mason * start over
1743c8c42864SChris Mason */
1744c8c42864SChris Mason static int
setup_nodes_for_search(struct btrfs_trans_handle * trans,struct btrfs_root * root,struct btrfs_path * p,struct extent_buffer * b,int level,int ins_len,int * write_lock_level)1745c8c42864SChris Mason setup_nodes_for_search(struct btrfs_trans_handle *trans,
1746c8c42864SChris Mason struct btrfs_root *root, struct btrfs_path *p,
1747bd681513SChris Mason struct extent_buffer *b, int level, int ins_len,
1748bd681513SChris Mason int *write_lock_level)
1749c8c42864SChris Mason {
17500b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info;
175195b982deSNikolay Borisov int ret = 0;
17520b246afaSJeff Mahoney
1753c8c42864SChris Mason if ((p->search_for_split || ins_len > 0) && btrfs_header_nritems(b) >=
17540b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3) {
1755c8c42864SChris Mason
1756bd681513SChris Mason if (*write_lock_level < level + 1) {
1757bd681513SChris Mason *write_lock_level = level + 1;
1758bd681513SChris Mason btrfs_release_path(p);
175995b982deSNikolay Borisov return -EAGAIN;
1760bd681513SChris Mason }
1761bd681513SChris Mason
1762bfb484d9SJosef Bacik reada_for_balance(p, level);
176395b982deSNikolay Borisov ret = split_node(trans, root, p, level);
1764c8c42864SChris Mason
1765c8c42864SChris Mason b = p->nodes[level];
1766c8c42864SChris Mason } else if (ins_len < 0 && btrfs_header_nritems(b) <
17670b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 2) {
1768c8c42864SChris Mason
1769bd681513SChris Mason if (*write_lock_level < level + 1) {
1770bd681513SChris Mason *write_lock_level = level + 1;
1771bd681513SChris Mason btrfs_release_path(p);
177295b982deSNikolay Borisov return -EAGAIN;
1773bd681513SChris Mason }
1774bd681513SChris Mason
1775bfb484d9SJosef Bacik reada_for_balance(p, level);
177695b982deSNikolay Borisov ret = balance_level(trans, root, p, level);
177795b982deSNikolay Borisov if (ret)
177895b982deSNikolay Borisov return ret;
1779c8c42864SChris Mason
1780c8c42864SChris Mason b = p->nodes[level];
1781c8c42864SChris Mason if (!b) {
1782b3b4aa74SDavid Sterba btrfs_release_path(p);
178395b982deSNikolay Borisov return -EAGAIN;
1784c8c42864SChris Mason }
1785c8c42864SChris Mason BUG_ON(btrfs_header_nritems(b) == 1);
1786c8c42864SChris Mason }
1787c8c42864SChris Mason return ret;
1788c8c42864SChris Mason }
1789c8c42864SChris Mason
btrfs_find_item(struct btrfs_root * fs_root,struct btrfs_path * path,u64 iobjectid,u64 ioff,u8 key_type,struct btrfs_key * found_key)1790381cf658SDavid Sterba int btrfs_find_item(struct btrfs_root *fs_root, struct btrfs_path *path,
1791e33d5c3dSKelley Nielsen u64 iobjectid, u64 ioff, u8 key_type,
1792e33d5c3dSKelley Nielsen struct btrfs_key *found_key)
1793e33d5c3dSKelley Nielsen {
1794e33d5c3dSKelley Nielsen int ret;
1795e33d5c3dSKelley Nielsen struct btrfs_key key;
1796e33d5c3dSKelley Nielsen struct extent_buffer *eb;
1797381cf658SDavid Sterba
1798381cf658SDavid Sterba ASSERT(path);
17991d4c08e0SDavid Sterba ASSERT(found_key);
1800e33d5c3dSKelley Nielsen
1801e33d5c3dSKelley Nielsen key.type = key_type;
1802e33d5c3dSKelley Nielsen key.objectid = iobjectid;
1803e33d5c3dSKelley Nielsen key.offset = ioff;
1804e33d5c3dSKelley Nielsen
1805e33d5c3dSKelley Nielsen ret = btrfs_search_slot(NULL, fs_root, &key, path, 0, 0);
18061d4c08e0SDavid Sterba if (ret < 0)
1807e33d5c3dSKelley Nielsen return ret;
1808e33d5c3dSKelley Nielsen
1809e33d5c3dSKelley Nielsen eb = path->nodes[0];
1810e33d5c3dSKelley Nielsen if (ret && path->slots[0] >= btrfs_header_nritems(eb)) {
1811e33d5c3dSKelley Nielsen ret = btrfs_next_leaf(fs_root, path);
1812e33d5c3dSKelley Nielsen if (ret)
1813e33d5c3dSKelley Nielsen return ret;
1814e33d5c3dSKelley Nielsen eb = path->nodes[0];
1815e33d5c3dSKelley Nielsen }
1816e33d5c3dSKelley Nielsen
1817e33d5c3dSKelley Nielsen btrfs_item_key_to_cpu(eb, found_key, path->slots[0]);
1818e33d5c3dSKelley Nielsen if (found_key->type != key.type ||
1819e33d5c3dSKelley Nielsen found_key->objectid != key.objectid)
1820e33d5c3dSKelley Nielsen return 1;
1821e33d5c3dSKelley Nielsen
1822e33d5c3dSKelley Nielsen return 0;
1823e33d5c3dSKelley Nielsen }
1824e33d5c3dSKelley Nielsen
btrfs_search_slot_get_root(struct btrfs_root * root,struct btrfs_path * p,int write_lock_level)18251fc28d8eSLiu Bo static struct extent_buffer *btrfs_search_slot_get_root(struct btrfs_root *root,
18261fc28d8eSLiu Bo struct btrfs_path *p,
18271fc28d8eSLiu Bo int write_lock_level)
18281fc28d8eSLiu Bo {
18291fc28d8eSLiu Bo struct extent_buffer *b;
1830120de408SJosef Bacik int root_lock = 0;
18311fc28d8eSLiu Bo int level = 0;
18321fc28d8eSLiu Bo
18331fc28d8eSLiu Bo if (p->search_commit_root) {
18341fc28d8eSLiu Bo b = root->commit_root;
183567439dadSDavid Sterba atomic_inc(&b->refs);
18361fc28d8eSLiu Bo level = btrfs_header_level(b);
1837f9ddfd05SLiu Bo /*
1838f9ddfd05SLiu Bo * Ensure that all callers have set skip_locking when
1839f9ddfd05SLiu Bo * p->search_commit_root = 1.
1840f9ddfd05SLiu Bo */
1841f9ddfd05SLiu Bo ASSERT(p->skip_locking == 1);
18421fc28d8eSLiu Bo
18431fc28d8eSLiu Bo goto out;
18441fc28d8eSLiu Bo }
18451fc28d8eSLiu Bo
18461fc28d8eSLiu Bo if (p->skip_locking) {
18471fc28d8eSLiu Bo b = btrfs_root_node(root);
18481fc28d8eSLiu Bo level = btrfs_header_level(b);
18491fc28d8eSLiu Bo goto out;
18501fc28d8eSLiu Bo }
18511fc28d8eSLiu Bo
1852120de408SJosef Bacik /* We try very hard to do read locks on the root */
1853120de408SJosef Bacik root_lock = BTRFS_READ_LOCK;
1854120de408SJosef Bacik
18551fc28d8eSLiu Bo /*
1856662c653bSLiu Bo * If the level is set to maximum, we can skip trying to get the read
1857662c653bSLiu Bo * lock.
1858662c653bSLiu Bo */
1859662c653bSLiu Bo if (write_lock_level < BTRFS_MAX_LEVEL) {
1860662c653bSLiu Bo /*
1861662c653bSLiu Bo * We don't know the level of the root node until we actually
1862662c653bSLiu Bo * have it read locked
18631fc28d8eSLiu Bo */
1864857bc13fSJosef Bacik if (p->nowait) {
1865857bc13fSJosef Bacik b = btrfs_try_read_lock_root_node(root);
1866857bc13fSJosef Bacik if (IS_ERR(b))
1867857bc13fSJosef Bacik return b;
1868857bc13fSJosef Bacik } else {
18691bb96598SJosef Bacik b = btrfs_read_lock_root_node(root);
1870857bc13fSJosef Bacik }
18711fc28d8eSLiu Bo level = btrfs_header_level(b);
18721fc28d8eSLiu Bo if (level > write_lock_level)
18731fc28d8eSLiu Bo goto out;
18741fc28d8eSLiu Bo
1875662c653bSLiu Bo /* Whoops, must trade for write lock */
18761fc28d8eSLiu Bo btrfs_tree_read_unlock(b);
18771fc28d8eSLiu Bo free_extent_buffer(b);
1878662c653bSLiu Bo }
1879662c653bSLiu Bo
18801fc28d8eSLiu Bo b = btrfs_lock_root_node(root);
18811fc28d8eSLiu Bo root_lock = BTRFS_WRITE_LOCK;
18821fc28d8eSLiu Bo
18831fc28d8eSLiu Bo /* The level might have changed, check again */
18841fc28d8eSLiu Bo level = btrfs_header_level(b);
18851fc28d8eSLiu Bo
18861fc28d8eSLiu Bo out:
1887120de408SJosef Bacik /*
1888120de408SJosef Bacik * The root may have failed to write out at some point, and thus is no
1889120de408SJosef Bacik * longer valid, return an error in this case.
1890120de408SJosef Bacik */
1891120de408SJosef Bacik if (!extent_buffer_uptodate(b)) {
1892120de408SJosef Bacik if (root_lock)
1893120de408SJosef Bacik btrfs_tree_unlock_rw(b, root_lock);
1894120de408SJosef Bacik free_extent_buffer(b);
1895120de408SJosef Bacik return ERR_PTR(-EIO);
1896120de408SJosef Bacik }
1897120de408SJosef Bacik
18981fc28d8eSLiu Bo p->nodes[level] = b;
18991fc28d8eSLiu Bo if (!p->skip_locking)
19001fc28d8eSLiu Bo p->locks[level] = root_lock;
19011fc28d8eSLiu Bo /*
19021fc28d8eSLiu Bo * Callers are responsible for dropping b's references.
19031fc28d8eSLiu Bo */
19041fc28d8eSLiu Bo return b;
19051fc28d8eSLiu Bo }
19061fc28d8eSLiu Bo
1907d96b3424SFilipe Manana /*
1908d96b3424SFilipe Manana * Replace the extent buffer at the lowest level of the path with a cloned
1909d96b3424SFilipe Manana * version. The purpose is to be able to use it safely, after releasing the
1910d96b3424SFilipe Manana * commit root semaphore, even if relocation is happening in parallel, the
1911d96b3424SFilipe Manana * transaction used for relocation is committed and the extent buffer is
1912d96b3424SFilipe Manana * reallocated in the next transaction.
1913d96b3424SFilipe Manana *
1914d96b3424SFilipe Manana * This is used in a context where the caller does not prevent transaction
1915d96b3424SFilipe Manana * commits from happening, either by holding a transaction handle or holding
1916d96b3424SFilipe Manana * some lock, while it's doing searches through a commit root.
1917d96b3424SFilipe Manana * At the moment it's only used for send operations.
1918d96b3424SFilipe Manana */
finish_need_commit_sem_search(struct btrfs_path * path)1919d96b3424SFilipe Manana static int finish_need_commit_sem_search(struct btrfs_path *path)
1920d96b3424SFilipe Manana {
1921d96b3424SFilipe Manana const int i = path->lowest_level;
1922d96b3424SFilipe Manana const int slot = path->slots[i];
1923d96b3424SFilipe Manana struct extent_buffer *lowest = path->nodes[i];
1924d96b3424SFilipe Manana struct extent_buffer *clone;
1925d96b3424SFilipe Manana
1926d96b3424SFilipe Manana ASSERT(path->need_commit_sem);
1927d96b3424SFilipe Manana
1928d96b3424SFilipe Manana if (!lowest)
1929d96b3424SFilipe Manana return 0;
1930d96b3424SFilipe Manana
1931d96b3424SFilipe Manana lockdep_assert_held_read(&lowest->fs_info->commit_root_sem);
1932d96b3424SFilipe Manana
1933d96b3424SFilipe Manana clone = btrfs_clone_extent_buffer(lowest);
1934d96b3424SFilipe Manana if (!clone)
1935d96b3424SFilipe Manana return -ENOMEM;
1936d96b3424SFilipe Manana
1937d96b3424SFilipe Manana btrfs_release_path(path);
1938d96b3424SFilipe Manana path->nodes[i] = clone;
1939d96b3424SFilipe Manana path->slots[i] = slot;
1940d96b3424SFilipe Manana
1941d96b3424SFilipe Manana return 0;
1942d96b3424SFilipe Manana }
19431fc28d8eSLiu Bo
search_for_key_slot(struct extent_buffer * eb,int search_low_slot,const struct btrfs_key * key,int prev_cmp,int * slot)1944e2e58d0fSFilipe Manana static inline int search_for_key_slot(struct extent_buffer *eb,
1945e2e58d0fSFilipe Manana int search_low_slot,
1946e2e58d0fSFilipe Manana const struct btrfs_key *key,
1947e2e58d0fSFilipe Manana int prev_cmp,
1948e2e58d0fSFilipe Manana int *slot)
1949e2e58d0fSFilipe Manana {
1950e2e58d0fSFilipe Manana /*
1951e2e58d0fSFilipe Manana * If a previous call to btrfs_bin_search() on a parent node returned an
1952e2e58d0fSFilipe Manana * exact match (prev_cmp == 0), we can safely assume the target key will
1953e2e58d0fSFilipe Manana * always be at slot 0 on lower levels, since each key pointer
1954e2e58d0fSFilipe Manana * (struct btrfs_key_ptr) refers to the lowest key accessible from the
1955e2e58d0fSFilipe Manana * subtree it points to. Thus we can skip searching lower levels.
1956e2e58d0fSFilipe Manana */
1957e2e58d0fSFilipe Manana if (prev_cmp == 0) {
1958e2e58d0fSFilipe Manana *slot = 0;
1959e2e58d0fSFilipe Manana return 0;
1960e2e58d0fSFilipe Manana }
1961e2e58d0fSFilipe Manana
1962fdf8d595SAnand Jain return btrfs_bin_search(eb, search_low_slot, key, slot);
1963e2e58d0fSFilipe Manana }
1964e2e58d0fSFilipe Manana
search_leaf(struct btrfs_trans_handle * trans,struct btrfs_root * root,const struct btrfs_key * key,struct btrfs_path * path,int ins_len,int prev_cmp)1965109324cfSFilipe Manana static int search_leaf(struct btrfs_trans_handle *trans,
1966109324cfSFilipe Manana struct btrfs_root *root,
1967109324cfSFilipe Manana const struct btrfs_key *key,
1968109324cfSFilipe Manana struct btrfs_path *path,
1969109324cfSFilipe Manana int ins_len,
1970109324cfSFilipe Manana int prev_cmp)
1971109324cfSFilipe Manana {
1972109324cfSFilipe Manana struct extent_buffer *leaf = path->nodes[0];
1973109324cfSFilipe Manana int leaf_free_space = -1;
1974109324cfSFilipe Manana int search_low_slot = 0;
1975109324cfSFilipe Manana int ret;
1976109324cfSFilipe Manana bool do_bin_search = true;
1977109324cfSFilipe Manana
1978109324cfSFilipe Manana /*
1979109324cfSFilipe Manana * If we are doing an insertion, the leaf has enough free space and the
1980109324cfSFilipe Manana * destination slot for the key is not slot 0, then we can unlock our
1981109324cfSFilipe Manana * write lock on the parent, and any other upper nodes, before doing the
1982109324cfSFilipe Manana * binary search on the leaf (with search_for_key_slot()), allowing other
1983109324cfSFilipe Manana * tasks to lock the parent and any other upper nodes.
1984109324cfSFilipe Manana */
1985109324cfSFilipe Manana if (ins_len > 0) {
1986109324cfSFilipe Manana /*
1987109324cfSFilipe Manana * Cache the leaf free space, since we will need it later and it
1988109324cfSFilipe Manana * will not change until then.
1989109324cfSFilipe Manana */
1990109324cfSFilipe Manana leaf_free_space = btrfs_leaf_free_space(leaf);
1991109324cfSFilipe Manana
1992109324cfSFilipe Manana /*
1993109324cfSFilipe Manana * !path->locks[1] means we have a single node tree, the leaf is
1994109324cfSFilipe Manana * the root of the tree.
1995109324cfSFilipe Manana */
1996109324cfSFilipe Manana if (path->locks[1] && leaf_free_space >= ins_len) {
1997109324cfSFilipe Manana struct btrfs_disk_key first_key;
1998109324cfSFilipe Manana
1999109324cfSFilipe Manana ASSERT(btrfs_header_nritems(leaf) > 0);
2000109324cfSFilipe Manana btrfs_item_key(leaf, &first_key, 0);
2001109324cfSFilipe Manana
2002109324cfSFilipe Manana /*
2003109324cfSFilipe Manana * Doing the extra comparison with the first key is cheap,
2004109324cfSFilipe Manana * taking into account that the first key is very likely
2005109324cfSFilipe Manana * already in a cache line because it immediately follows
2006109324cfSFilipe Manana * the extent buffer's header and we have recently accessed
2007109324cfSFilipe Manana * the header's level field.
2008109324cfSFilipe Manana */
2009109324cfSFilipe Manana ret = comp_keys(&first_key, key);
2010109324cfSFilipe Manana if (ret < 0) {
2011109324cfSFilipe Manana /*
2012109324cfSFilipe Manana * The first key is smaller than the key we want
2013109324cfSFilipe Manana * to insert, so we are safe to unlock all upper
2014109324cfSFilipe Manana * nodes and we have to do the binary search.
2015109324cfSFilipe Manana *
2016109324cfSFilipe Manana * We do use btrfs_unlock_up_safe() and not
2017109324cfSFilipe Manana * unlock_up() because the later does not unlock
2018109324cfSFilipe Manana * nodes with a slot of 0 - we can safely unlock
2019109324cfSFilipe Manana * any node even if its slot is 0 since in this
2020109324cfSFilipe Manana * case the key does not end up at slot 0 of the
2021109324cfSFilipe Manana * leaf and there's no need to split the leaf.
2022109324cfSFilipe Manana */
2023109324cfSFilipe Manana btrfs_unlock_up_safe(path, 1);
2024109324cfSFilipe Manana search_low_slot = 1;
2025109324cfSFilipe Manana } else {
2026109324cfSFilipe Manana /*
2027109324cfSFilipe Manana * The first key is >= then the key we want to
2028109324cfSFilipe Manana * insert, so we can skip the binary search as
2029109324cfSFilipe Manana * the target key will be at slot 0.
2030109324cfSFilipe Manana *
2031109324cfSFilipe Manana * We can not unlock upper nodes when the key is
2032109324cfSFilipe Manana * less than the first key, because we will need
2033109324cfSFilipe Manana * to update the key at slot 0 of the parent node
2034109324cfSFilipe Manana * and possibly of other upper nodes too.
2035109324cfSFilipe Manana * If the key matches the first key, then we can
2036109324cfSFilipe Manana * unlock all the upper nodes, using
2037109324cfSFilipe Manana * btrfs_unlock_up_safe() instead of unlock_up()
2038109324cfSFilipe Manana * as stated above.
2039109324cfSFilipe Manana */
2040109324cfSFilipe Manana if (ret == 0)
2041109324cfSFilipe Manana btrfs_unlock_up_safe(path, 1);
2042109324cfSFilipe Manana /*
2043109324cfSFilipe Manana * ret is already 0 or 1, matching the result of
2044109324cfSFilipe Manana * a btrfs_bin_search() call, so there is no need
2045109324cfSFilipe Manana * to adjust it.
2046109324cfSFilipe Manana */
2047109324cfSFilipe Manana do_bin_search = false;
2048109324cfSFilipe Manana path->slots[0] = 0;
2049109324cfSFilipe Manana }
2050109324cfSFilipe Manana }
2051109324cfSFilipe Manana }
2052109324cfSFilipe Manana
2053109324cfSFilipe Manana if (do_bin_search) {
2054109324cfSFilipe Manana ret = search_for_key_slot(leaf, search_low_slot, key,
2055109324cfSFilipe Manana prev_cmp, &path->slots[0]);
2056109324cfSFilipe Manana if (ret < 0)
2057109324cfSFilipe Manana return ret;
2058109324cfSFilipe Manana }
2059109324cfSFilipe Manana
2060109324cfSFilipe Manana if (ins_len > 0) {
2061109324cfSFilipe Manana /*
2062109324cfSFilipe Manana * Item key already exists. In this case, if we are allowed to
2063109324cfSFilipe Manana * insert the item (for example, in dir_item case, item key
2064109324cfSFilipe Manana * collision is allowed), it will be merged with the original
2065109324cfSFilipe Manana * item. Only the item size grows, no new btrfs item will be
2066109324cfSFilipe Manana * added. If search_for_extension is not set, ins_len already
2067109324cfSFilipe Manana * accounts the size btrfs_item, deduct it here so leaf space
2068109324cfSFilipe Manana * check will be correct.
2069109324cfSFilipe Manana */
2070109324cfSFilipe Manana if (ret == 0 && !path->search_for_extension) {
2071109324cfSFilipe Manana ASSERT(ins_len >= sizeof(struct btrfs_item));
2072109324cfSFilipe Manana ins_len -= sizeof(struct btrfs_item);
2073109324cfSFilipe Manana }
2074109324cfSFilipe Manana
2075109324cfSFilipe Manana ASSERT(leaf_free_space >= 0);
2076109324cfSFilipe Manana
2077109324cfSFilipe Manana if (leaf_free_space < ins_len) {
2078109324cfSFilipe Manana int err;
2079109324cfSFilipe Manana
2080109324cfSFilipe Manana err = split_leaf(trans, root, key, path, ins_len,
2081109324cfSFilipe Manana (ret == 0));
2082bb8e9a60SFilipe Manana ASSERT(err <= 0);
2083bb8e9a60SFilipe Manana if (WARN_ON(err > 0))
2084bb8e9a60SFilipe Manana err = -EUCLEAN;
2085109324cfSFilipe Manana if (err)
2086109324cfSFilipe Manana ret = err;
2087109324cfSFilipe Manana }
2088109324cfSFilipe Manana }
2089109324cfSFilipe Manana
2090109324cfSFilipe Manana return ret;
2091109324cfSFilipe Manana }
2092109324cfSFilipe Manana
2093c8c42864SChris Mason /*
20944271eceaSNikolay Borisov * btrfs_search_slot - look for a key in a tree and perform necessary
20954271eceaSNikolay Borisov * modifications to preserve tree invariants.
209674123bd7SChris Mason *
20974271eceaSNikolay Borisov * @trans: Handle of transaction, used when modifying the tree
20984271eceaSNikolay Borisov * @p: Holds all btree nodes along the search path
20994271eceaSNikolay Borisov * @root: The root node of the tree
21004271eceaSNikolay Borisov * @key: The key we are looking for
21019a664971Sethanwu * @ins_len: Indicates purpose of search:
21029a664971Sethanwu * >0 for inserts it's size of item inserted (*)
21039a664971Sethanwu * <0 for deletions
21049a664971Sethanwu * 0 for plain searches, not modifying the tree
21059a664971Sethanwu *
21069a664971Sethanwu * (*) If size of item inserted doesn't include
21079a664971Sethanwu * sizeof(struct btrfs_item), then p->search_for_extension must
21089a664971Sethanwu * be set.
21094271eceaSNikolay Borisov * @cow: boolean should CoW operations be performed. Must always be 1
21104271eceaSNikolay Borisov * when modifying the tree.
211197571fd0SChris Mason *
21124271eceaSNikolay Borisov * If @ins_len > 0, nodes and leaves will be split as we walk down the tree.
21134271eceaSNikolay Borisov * If @ins_len < 0, nodes will be merged as we walk down the tree (if possible)
21144271eceaSNikolay Borisov *
21154271eceaSNikolay Borisov * If @key is found, 0 is returned and you can find the item in the leaf level
21164271eceaSNikolay Borisov * of the path (level 0)
21174271eceaSNikolay Borisov *
21184271eceaSNikolay Borisov * If @key isn't found, 1 is returned and the leaf level of the path (level 0)
21194271eceaSNikolay Borisov * points to the slot where it should be inserted
21204271eceaSNikolay Borisov *
21214271eceaSNikolay Borisov * If an error is encountered while searching the tree a negative error number
21224271eceaSNikolay Borisov * is returned
212374123bd7SChris Mason */
btrfs_search_slot(struct btrfs_trans_handle * trans,struct btrfs_root * root,const struct btrfs_key * key,struct btrfs_path * p,int ins_len,int cow)2124310712b2SOmar Sandoval int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root,
2125310712b2SOmar Sandoval const struct btrfs_key *key, struct btrfs_path *p,
2126310712b2SOmar Sandoval int ins_len, int cow)
2127be0e5c09SChris Mason {
2128d96b3424SFilipe Manana struct btrfs_fs_info *fs_info = root->fs_info;
21295f39d397SChris Mason struct extent_buffer *b;
2130be0e5c09SChris Mason int slot;
2131be0e5c09SChris Mason int ret;
213233c66f43SYan Zheng int err;
2133be0e5c09SChris Mason int level;
2134925baeddSChris Mason int lowest_unlock = 1;
2135bd681513SChris Mason /* everything at write_lock_level or lower must be write locked */
2136bd681513SChris Mason int write_lock_level = 0;
21379f3a7427SChris Mason u8 lowest_level = 0;
2138f7c79f30SChris Mason int min_write_lock_level;
2139d7396f07SFilipe David Borba Manana int prev_cmp;
21409f3a7427SChris Mason
2141a4c853afSChenXiaoSong might_sleep();
2142a4c853afSChenXiaoSong
21436702ed49SChris Mason lowest_level = p->lowest_level;
2144323ac95bSChris Mason WARN_ON(lowest_level && ins_len > 0);
214522b0ebdaSChris Mason WARN_ON(p->nodes[0] != NULL);
2146eb653de1SFilipe David Borba Manana BUG_ON(!cow && ins_len);
214725179201SJosef Bacik
2148857bc13fSJosef Bacik /*
2149857bc13fSJosef Bacik * For now only allow nowait for read only operations. There's no
2150857bc13fSJosef Bacik * strict reason why we can't, we just only need it for reads so it's
2151857bc13fSJosef Bacik * only implemented for reads.
2152857bc13fSJosef Bacik */
2153857bc13fSJosef Bacik ASSERT(!p->nowait || !cow);
2154857bc13fSJosef Bacik
2155bd681513SChris Mason if (ins_len < 0) {
2156925baeddSChris Mason lowest_unlock = 2;
215765b51a00SChris Mason
2158bd681513SChris Mason /* when we are removing items, we might have to go up to level
2159bd681513SChris Mason * two as we update tree pointers Make sure we keep write
2160bd681513SChris Mason * for those levels as well
2161bd681513SChris Mason */
2162bd681513SChris Mason write_lock_level = 2;
2163bd681513SChris Mason } else if (ins_len > 0) {
2164bd681513SChris Mason /*
2165bd681513SChris Mason * for inserting items, make sure we have a write lock on
2166bd681513SChris Mason * level 1 so we can update keys
2167bd681513SChris Mason */
2168bd681513SChris Mason write_lock_level = 1;
2169bd681513SChris Mason }
2170bd681513SChris Mason
2171bd681513SChris Mason if (!cow)
2172bd681513SChris Mason write_lock_level = -1;
2173bd681513SChris Mason
217409a2a8f9SJosef Bacik if (cow && (p->keep_locks || p->lowest_level))
2175bd681513SChris Mason write_lock_level = BTRFS_MAX_LEVEL;
2176bd681513SChris Mason
2177f7c79f30SChris Mason min_write_lock_level = write_lock_level;
2178f7c79f30SChris Mason
2179d96b3424SFilipe Manana if (p->need_commit_sem) {
2180d96b3424SFilipe Manana ASSERT(p->search_commit_root);
2181857bc13fSJosef Bacik if (p->nowait) {
2182857bc13fSJosef Bacik if (!down_read_trylock(&fs_info->commit_root_sem))
2183857bc13fSJosef Bacik return -EAGAIN;
2184857bc13fSJosef Bacik } else {
2185d96b3424SFilipe Manana down_read(&fs_info->commit_root_sem);
2186d96b3424SFilipe Manana }
2187857bc13fSJosef Bacik }
2188d96b3424SFilipe Manana
2189bb803951SChris Mason again:
2190d7396f07SFilipe David Borba Manana prev_cmp = -1;
21911fc28d8eSLiu Bo b = btrfs_search_slot_get_root(root, p, write_lock_level);
2192be6821f8SFilipe Manana if (IS_ERR(b)) {
2193be6821f8SFilipe Manana ret = PTR_ERR(b);
2194be6821f8SFilipe Manana goto done;
2195be6821f8SFilipe Manana }
2196925baeddSChris Mason
2197eb60ceacSChris Mason while (b) {
2198f624d976SQu Wenruo int dec = 0;
2199f624d976SQu Wenruo
22005f39d397SChris Mason level = btrfs_header_level(b);
220165b51a00SChris Mason
220202217ed2SChris Mason if (cow) {
22039ea2c7c9SNikolay Borisov bool last_level = (level == (BTRFS_MAX_LEVEL - 1));
22049ea2c7c9SNikolay Borisov
2205c8c42864SChris Mason /*
2206c8c42864SChris Mason * if we don't really need to cow this block
2207c8c42864SChris Mason * then we don't want to set the path blocking,
2208c8c42864SChris Mason * so we test it here
2209c8c42864SChris Mason */
22105963ffcaSJosef Bacik if (!should_cow_block(trans, root, b))
221165b51a00SChris Mason goto cow_done;
22125d4f98a2SYan Zheng
2213bd681513SChris Mason /*
2214bd681513SChris Mason * must have write locks on this node and the
2215bd681513SChris Mason * parent
2216bd681513SChris Mason */
22175124e00eSJosef Bacik if (level > write_lock_level ||
22185124e00eSJosef Bacik (level + 1 > write_lock_level &&
22195124e00eSJosef Bacik level + 1 < BTRFS_MAX_LEVEL &&
22205124e00eSJosef Bacik p->nodes[level + 1])) {
2221bd681513SChris Mason write_lock_level = level + 1;
2222bd681513SChris Mason btrfs_release_path(p);
2223bd681513SChris Mason goto again;
2224bd681513SChris Mason }
2225bd681513SChris Mason
22269ea2c7c9SNikolay Borisov if (last_level)
22279ea2c7c9SNikolay Borisov err = btrfs_cow_block(trans, root, b, NULL, 0,
22289631e4ccSJosef Bacik &b,
22299631e4ccSJosef Bacik BTRFS_NESTING_COW);
22309ea2c7c9SNikolay Borisov else
223133c66f43SYan Zheng err = btrfs_cow_block(trans, root, b,
2232e20d96d6SChris Mason p->nodes[level + 1],
22339631e4ccSJosef Bacik p->slots[level + 1], &b,
22349631e4ccSJosef Bacik BTRFS_NESTING_COW);
223533c66f43SYan Zheng if (err) {
223633c66f43SYan Zheng ret = err;
223765b51a00SChris Mason goto done;
223854aa1f4dSChris Mason }
223902217ed2SChris Mason }
224065b51a00SChris Mason cow_done:
2241eb60ceacSChris Mason p->nodes[level] = b;
2242b4ce94deSChris Mason
2243b4ce94deSChris Mason /*
2244b4ce94deSChris Mason * we have a lock on b and as long as we aren't changing
2245b4ce94deSChris Mason * the tree, there is no way to for the items in b to change.
2246b4ce94deSChris Mason * It is safe to drop the lock on our parent before we
2247b4ce94deSChris Mason * go through the expensive btree search on b.
2248b4ce94deSChris Mason *
2249eb653de1SFilipe David Borba Manana * If we're inserting or deleting (ins_len != 0), then we might
2250eb653de1SFilipe David Borba Manana * be changing slot zero, which may require changing the parent.
2251eb653de1SFilipe David Borba Manana * So, we can't drop the lock until after we know which slot
2252eb653de1SFilipe David Borba Manana * we're operating on.
2253b4ce94deSChris Mason */
2254eb653de1SFilipe David Borba Manana if (!ins_len && !p->keep_locks) {
2255eb653de1SFilipe David Borba Manana int u = level + 1;
2256eb653de1SFilipe David Borba Manana
2257eb653de1SFilipe David Borba Manana if (u < BTRFS_MAX_LEVEL && p->locks[u]) {
2258eb653de1SFilipe David Borba Manana btrfs_tree_unlock_rw(p->nodes[u], p->locks[u]);
2259eb653de1SFilipe David Borba Manana p->locks[u] = 0;
2260eb653de1SFilipe David Borba Manana }
2261eb653de1SFilipe David Borba Manana }
2262b4ce94deSChris Mason
2263e2e58d0fSFilipe Manana if (level == 0) {
2264109324cfSFilipe Manana if (ins_len > 0)
2265e5e1c174SFilipe Manana ASSERT(write_lock_level >= 1);
2266bd681513SChris Mason
2267109324cfSFilipe Manana ret = search_leaf(trans, root, key, p, ins_len, prev_cmp);
2268459931ecSChris Mason if (!p->search_for_split)
2269f7c79f30SChris Mason unlock_up(p, level, lowest_unlock,
22704b6f8e96SLiu Bo min_write_lock_level, NULL);
227165b51a00SChris Mason goto done;
227265b51a00SChris Mason }
2273e2e58d0fSFilipe Manana
2274e2e58d0fSFilipe Manana ret = search_for_key_slot(b, 0, key, prev_cmp, &slot);
2275e2e58d0fSFilipe Manana if (ret < 0)
2276e2e58d0fSFilipe Manana goto done;
2277e2e58d0fSFilipe Manana prev_cmp = ret;
2278e2e58d0fSFilipe Manana
2279f624d976SQu Wenruo if (ret && slot > 0) {
2280f624d976SQu Wenruo dec = 1;
2281f624d976SQu Wenruo slot--;
2282f624d976SQu Wenruo }
2283f624d976SQu Wenruo p->slots[level] = slot;
2284f624d976SQu Wenruo err = setup_nodes_for_search(trans, root, p, b, level, ins_len,
2285f624d976SQu Wenruo &write_lock_level);
2286f624d976SQu Wenruo if (err == -EAGAIN)
2287f624d976SQu Wenruo goto again;
2288f624d976SQu Wenruo if (err) {
2289f624d976SQu Wenruo ret = err;
2290f624d976SQu Wenruo goto done;
2291f624d976SQu Wenruo }
2292f624d976SQu Wenruo b = p->nodes[level];
2293f624d976SQu Wenruo slot = p->slots[level];
2294f624d976SQu Wenruo
2295f624d976SQu Wenruo /*
2296f624d976SQu Wenruo * Slot 0 is special, if we change the key we have to update
2297f624d976SQu Wenruo * the parent pointer which means we must have a write lock on
2298f624d976SQu Wenruo * the parent
2299f624d976SQu Wenruo */
2300f624d976SQu Wenruo if (slot == 0 && ins_len && write_lock_level < level + 1) {
2301f624d976SQu Wenruo write_lock_level = level + 1;
2302f624d976SQu Wenruo btrfs_release_path(p);
2303f624d976SQu Wenruo goto again;
2304f624d976SQu Wenruo }
2305f624d976SQu Wenruo
2306f624d976SQu Wenruo unlock_up(p, level, lowest_unlock, min_write_lock_level,
2307f624d976SQu Wenruo &write_lock_level);
2308f624d976SQu Wenruo
2309f624d976SQu Wenruo if (level == lowest_level) {
2310f624d976SQu Wenruo if (dec)
2311f624d976SQu Wenruo p->slots[level]++;
2312f624d976SQu Wenruo goto done;
2313f624d976SQu Wenruo }
2314f624d976SQu Wenruo
2315f624d976SQu Wenruo err = read_block_for_search(root, p, &b, level, slot, key);
2316f624d976SQu Wenruo if (err == -EAGAIN)
2317f624d976SQu Wenruo goto again;
2318f624d976SQu Wenruo if (err) {
2319f624d976SQu Wenruo ret = err;
2320f624d976SQu Wenruo goto done;
2321f624d976SQu Wenruo }
2322f624d976SQu Wenruo
2323f624d976SQu Wenruo if (!p->skip_locking) {
2324f624d976SQu Wenruo level = btrfs_header_level(b);
2325b40130b2SJosef Bacik
2326b40130b2SJosef Bacik btrfs_maybe_reset_lockdep_class(root, b);
2327b40130b2SJosef Bacik
2328f624d976SQu Wenruo if (level <= write_lock_level) {
2329f624d976SQu Wenruo btrfs_tree_lock(b);
2330f624d976SQu Wenruo p->locks[level] = BTRFS_WRITE_LOCK;
2331f624d976SQu Wenruo } else {
2332857bc13fSJosef Bacik if (p->nowait) {
2333857bc13fSJosef Bacik if (!btrfs_try_tree_read_lock(b)) {
2334857bc13fSJosef Bacik free_extent_buffer(b);
2335857bc13fSJosef Bacik ret = -EAGAIN;
2336857bc13fSJosef Bacik goto done;
2337857bc13fSJosef Bacik }
2338857bc13fSJosef Bacik } else {
2339fe596ca3SJosef Bacik btrfs_tree_read_lock(b);
2340857bc13fSJosef Bacik }
2341f624d976SQu Wenruo p->locks[level] = BTRFS_READ_LOCK;
2342f624d976SQu Wenruo }
2343f624d976SQu Wenruo p->nodes[level] = b;
2344f624d976SQu Wenruo }
234565b51a00SChris Mason }
234665b51a00SChris Mason ret = 1;
234765b51a00SChris Mason done:
23485f5bc6b1SFilipe Manana if (ret < 0 && !p->skip_release_on_error)
2349b3b4aa74SDavid Sterba btrfs_release_path(p);
2350d96b3424SFilipe Manana
2351d96b3424SFilipe Manana if (p->need_commit_sem) {
2352d96b3424SFilipe Manana int ret2;
2353d96b3424SFilipe Manana
2354d96b3424SFilipe Manana ret2 = finish_need_commit_sem_search(p);
2355d96b3424SFilipe Manana up_read(&fs_info->commit_root_sem);
2356d96b3424SFilipe Manana if (ret2)
2357d96b3424SFilipe Manana ret = ret2;
2358d96b3424SFilipe Manana }
2359d96b3424SFilipe Manana
2360be0e5c09SChris Mason return ret;
2361be0e5c09SChris Mason }
2362f75e2b79SJosef Bacik ALLOW_ERROR_INJECTION(btrfs_search_slot, ERRNO);
2363be0e5c09SChris Mason
236474123bd7SChris Mason /*
23655d9e75c4SJan Schmidt * Like btrfs_search_slot, this looks for a key in the given tree. It uses the
23665d9e75c4SJan Schmidt * current state of the tree together with the operations recorded in the tree
23675d9e75c4SJan Schmidt * modification log to search for the key in a previous version of this tree, as
23685d9e75c4SJan Schmidt * denoted by the time_seq parameter.
23695d9e75c4SJan Schmidt *
23705d9e75c4SJan Schmidt * Naturally, there is no support for insert, delete or cow operations.
23715d9e75c4SJan Schmidt *
23725d9e75c4SJan Schmidt * The resulting path and return value will be set up as if we called
23735d9e75c4SJan Schmidt * btrfs_search_slot at that point in time with ins_len and cow both set to 0.
23745d9e75c4SJan Schmidt */
btrfs_search_old_slot(struct btrfs_root * root,const struct btrfs_key * key,struct btrfs_path * p,u64 time_seq)2375310712b2SOmar Sandoval int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key,
23765d9e75c4SJan Schmidt struct btrfs_path *p, u64 time_seq)
23775d9e75c4SJan Schmidt {
23780b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info;
23795d9e75c4SJan Schmidt struct extent_buffer *b;
23805d9e75c4SJan Schmidt int slot;
23815d9e75c4SJan Schmidt int ret;
23825d9e75c4SJan Schmidt int err;
23835d9e75c4SJan Schmidt int level;
23845d9e75c4SJan Schmidt int lowest_unlock = 1;
23855d9e75c4SJan Schmidt u8 lowest_level = 0;
23865d9e75c4SJan Schmidt
23875d9e75c4SJan Schmidt lowest_level = p->lowest_level;
23885d9e75c4SJan Schmidt WARN_ON(p->nodes[0] != NULL);
2389c922b016SStefan Roesch ASSERT(!p->nowait);
23905d9e75c4SJan Schmidt
23915d9e75c4SJan Schmidt if (p->search_commit_root) {
23925d9e75c4SJan Schmidt BUG_ON(time_seq);
23935d9e75c4SJan Schmidt return btrfs_search_slot(NULL, root, key, p, 0, 0);
23945d9e75c4SJan Schmidt }
23955d9e75c4SJan Schmidt
23965d9e75c4SJan Schmidt again:
2397f3a84ccdSFilipe Manana b = btrfs_get_old_root(root, time_seq);
2398315bed43SNikolay Borisov if (!b) {
2399315bed43SNikolay Borisov ret = -EIO;
2400315bed43SNikolay Borisov goto done;
2401315bed43SNikolay Borisov }
24025d9e75c4SJan Schmidt level = btrfs_header_level(b);
24035d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK;
24045d9e75c4SJan Schmidt
24055d9e75c4SJan Schmidt while (b) {
2406abe9339dSQu Wenruo int dec = 0;
2407abe9339dSQu Wenruo
24085d9e75c4SJan Schmidt level = btrfs_header_level(b);
24095d9e75c4SJan Schmidt p->nodes[level] = b;
24105d9e75c4SJan Schmidt
24115d9e75c4SJan Schmidt /*
24125d9e75c4SJan Schmidt * we have a lock on b and as long as we aren't changing
24135d9e75c4SJan Schmidt * the tree, there is no way to for the items in b to change.
24145d9e75c4SJan Schmidt * It is safe to drop the lock on our parent before we
24155d9e75c4SJan Schmidt * go through the expensive btree search on b.
24165d9e75c4SJan Schmidt */
24175d9e75c4SJan Schmidt btrfs_unlock_up_safe(p, level + 1);
24185d9e75c4SJan Schmidt
2419fdf8d595SAnand Jain ret = btrfs_bin_search(b, 0, key, &slot);
2420cbca7d59SFilipe Manana if (ret < 0)
2421cbca7d59SFilipe Manana goto done;
24225d9e75c4SJan Schmidt
2423abe9339dSQu Wenruo if (level == 0) {
2424abe9339dSQu Wenruo p->slots[level] = slot;
2425abe9339dSQu Wenruo unlock_up(p, level, lowest_unlock, 0, NULL);
2426abe9339dSQu Wenruo goto done;
2427abe9339dSQu Wenruo }
2428abe9339dSQu Wenruo
24295d9e75c4SJan Schmidt if (ret && slot > 0) {
24305d9e75c4SJan Schmidt dec = 1;
2431abe9339dSQu Wenruo slot--;
24325d9e75c4SJan Schmidt }
24335d9e75c4SJan Schmidt p->slots[level] = slot;
24345d9e75c4SJan Schmidt unlock_up(p, level, lowest_unlock, 0, NULL);
24355d9e75c4SJan Schmidt
24365d9e75c4SJan Schmidt if (level == lowest_level) {
24375d9e75c4SJan Schmidt if (dec)
24385d9e75c4SJan Schmidt p->slots[level]++;
24395d9e75c4SJan Schmidt goto done;
24405d9e75c4SJan Schmidt }
24415d9e75c4SJan Schmidt
2442abe9339dSQu Wenruo err = read_block_for_search(root, p, &b, level, slot, key);
24435d9e75c4SJan Schmidt if (err == -EAGAIN)
24445d9e75c4SJan Schmidt goto again;
24455d9e75c4SJan Schmidt if (err) {
24465d9e75c4SJan Schmidt ret = err;
24475d9e75c4SJan Schmidt goto done;
24485d9e75c4SJan Schmidt }
24495d9e75c4SJan Schmidt
24505d9e75c4SJan Schmidt level = btrfs_header_level(b);
24515d9e75c4SJan Schmidt btrfs_tree_read_lock(b);
2452f3a84ccdSFilipe Manana b = btrfs_tree_mod_log_rewind(fs_info, p, b, time_seq);
2453db7f3436SJosef Bacik if (!b) {
2454db7f3436SJosef Bacik ret = -ENOMEM;
2455db7f3436SJosef Bacik goto done;
2456db7f3436SJosef Bacik }
24575d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK;
24585d9e75c4SJan Schmidt p->nodes[level] = b;
24595d9e75c4SJan Schmidt }
24605d9e75c4SJan Schmidt ret = 1;
24615d9e75c4SJan Schmidt done:
24625d9e75c4SJan Schmidt if (ret < 0)
24635d9e75c4SJan Schmidt btrfs_release_path(p);
24645d9e75c4SJan Schmidt
24655d9e75c4SJan Schmidt return ret;
24665d9e75c4SJan Schmidt }
24675d9e75c4SJan Schmidt
24685d9e75c4SJan Schmidt /*
2469f469c8bdSFilipe Manana * Search the tree again to find a leaf with smaller keys.
2470f469c8bdSFilipe Manana * Returns 0 if it found something.
2471f469c8bdSFilipe Manana * Returns 1 if there are no smaller keys.
2472f469c8bdSFilipe Manana * Returns < 0 on error.
2473f469c8bdSFilipe Manana *
2474f469c8bdSFilipe Manana * This may release the path, and so you may lose any locks held at the
2475f469c8bdSFilipe Manana * time you call it.
2476f469c8bdSFilipe Manana */
btrfs_prev_leaf(struct btrfs_root * root,struct btrfs_path * path)2477f469c8bdSFilipe Manana static int btrfs_prev_leaf(struct btrfs_root *root, struct btrfs_path *path)
2478f469c8bdSFilipe Manana {
2479f469c8bdSFilipe Manana struct btrfs_key key;
2480f469c8bdSFilipe Manana struct btrfs_key orig_key;
2481f469c8bdSFilipe Manana struct btrfs_disk_key found_key;
2482f469c8bdSFilipe Manana int ret;
2483f469c8bdSFilipe Manana
2484f469c8bdSFilipe Manana btrfs_item_key_to_cpu(path->nodes[0], &key, 0);
2485f469c8bdSFilipe Manana orig_key = key;
2486f469c8bdSFilipe Manana
2487f469c8bdSFilipe Manana if (key.offset > 0) {
2488f469c8bdSFilipe Manana key.offset--;
2489f469c8bdSFilipe Manana } else if (key.type > 0) {
2490f469c8bdSFilipe Manana key.type--;
2491f469c8bdSFilipe Manana key.offset = (u64)-1;
2492f469c8bdSFilipe Manana } else if (key.objectid > 0) {
2493f469c8bdSFilipe Manana key.objectid--;
2494f469c8bdSFilipe Manana key.type = (u8)-1;
2495f469c8bdSFilipe Manana key.offset = (u64)-1;
2496f469c8bdSFilipe Manana } else {
2497f469c8bdSFilipe Manana return 1;
2498f469c8bdSFilipe Manana }
2499f469c8bdSFilipe Manana
2500f469c8bdSFilipe Manana btrfs_release_path(path);
2501f469c8bdSFilipe Manana ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2502f469c8bdSFilipe Manana if (ret <= 0)
2503f469c8bdSFilipe Manana return ret;
2504f469c8bdSFilipe Manana
2505f469c8bdSFilipe Manana /*
2506f469c8bdSFilipe Manana * Previous key not found. Even if we were at slot 0 of the leaf we had
2507f469c8bdSFilipe Manana * before releasing the path and calling btrfs_search_slot(), we now may
2508f469c8bdSFilipe Manana * be in a slot pointing to the same original key - this can happen if
2509f469c8bdSFilipe Manana * after we released the path, one of more items were moved from a
2510f469c8bdSFilipe Manana * sibling leaf into the front of the leaf we had due to an insertion
2511f469c8bdSFilipe Manana * (see push_leaf_right()).
2512f469c8bdSFilipe Manana * If we hit this case and our slot is > 0 and just decrement the slot
2513f469c8bdSFilipe Manana * so that the caller does not process the same key again, which may or
2514f469c8bdSFilipe Manana * may not break the caller, depending on its logic.
2515f469c8bdSFilipe Manana */
2516f469c8bdSFilipe Manana if (path->slots[0] < btrfs_header_nritems(path->nodes[0])) {
2517f469c8bdSFilipe Manana btrfs_item_key(path->nodes[0], &found_key, path->slots[0]);
2518f469c8bdSFilipe Manana ret = comp_keys(&found_key, &orig_key);
2519f469c8bdSFilipe Manana if (ret == 0) {
2520f469c8bdSFilipe Manana if (path->slots[0] > 0) {
2521f469c8bdSFilipe Manana path->slots[0]--;
2522f469c8bdSFilipe Manana return 0;
2523f469c8bdSFilipe Manana }
2524f469c8bdSFilipe Manana /*
2525f469c8bdSFilipe Manana * At slot 0, same key as before, it means orig_key is
2526f469c8bdSFilipe Manana * the lowest, leftmost, key in the tree. We're done.
2527f469c8bdSFilipe Manana */
2528f469c8bdSFilipe Manana return 1;
2529f469c8bdSFilipe Manana }
2530f469c8bdSFilipe Manana }
2531f469c8bdSFilipe Manana
2532f469c8bdSFilipe Manana btrfs_item_key(path->nodes[0], &found_key, 0);
2533f469c8bdSFilipe Manana ret = comp_keys(&found_key, &key);
2534f469c8bdSFilipe Manana /*
2535f469c8bdSFilipe Manana * We might have had an item with the previous key in the tree right
2536f469c8bdSFilipe Manana * before we released our path. And after we released our path, that
2537f469c8bdSFilipe Manana * item might have been pushed to the first slot (0) of the leaf we
2538f469c8bdSFilipe Manana * were holding due to a tree balance. Alternatively, an item with the
2539f469c8bdSFilipe Manana * previous key can exist as the only element of a leaf (big fat item).
2540f469c8bdSFilipe Manana * Therefore account for these 2 cases, so that our callers (like
2541f469c8bdSFilipe Manana * btrfs_previous_item) don't miss an existing item with a key matching
2542f469c8bdSFilipe Manana * the previous key we computed above.
2543f469c8bdSFilipe Manana */
2544f469c8bdSFilipe Manana if (ret <= 0)
2545f469c8bdSFilipe Manana return 0;
2546f469c8bdSFilipe Manana return 1;
2547f469c8bdSFilipe Manana }
2548f469c8bdSFilipe Manana
2549f469c8bdSFilipe Manana /*
25502f38b3e1SArne Jansen * helper to use instead of search slot if no exact match is needed but
25512f38b3e1SArne Jansen * instead the next or previous item should be returned.
25522f38b3e1SArne Jansen * When find_higher is true, the next higher item is returned, the next lower
25532f38b3e1SArne Jansen * otherwise.
25542f38b3e1SArne Jansen * When return_any and find_higher are both true, and no higher item is found,
25552f38b3e1SArne Jansen * return the next lower instead.
25562f38b3e1SArne Jansen * When return_any is true and find_higher is false, and no lower item is found,
25572f38b3e1SArne Jansen * return the next higher instead.
25582f38b3e1SArne Jansen * It returns 0 if any item is found, 1 if none is found (tree empty), and
25592f38b3e1SArne Jansen * < 0 on error
25602f38b3e1SArne Jansen */
btrfs_search_slot_for_read(struct btrfs_root * root,const struct btrfs_key * key,struct btrfs_path * p,int find_higher,int return_any)25612f38b3e1SArne Jansen int btrfs_search_slot_for_read(struct btrfs_root *root,
2562310712b2SOmar Sandoval const struct btrfs_key *key,
2563310712b2SOmar Sandoval struct btrfs_path *p, int find_higher,
2564310712b2SOmar Sandoval int return_any)
25652f38b3e1SArne Jansen {
25662f38b3e1SArne Jansen int ret;
25672f38b3e1SArne Jansen struct extent_buffer *leaf;
25682f38b3e1SArne Jansen
25692f38b3e1SArne Jansen again:
25702f38b3e1SArne Jansen ret = btrfs_search_slot(NULL, root, key, p, 0, 0);
25712f38b3e1SArne Jansen if (ret <= 0)
25722f38b3e1SArne Jansen return ret;
25732f38b3e1SArne Jansen /*
25742f38b3e1SArne Jansen * a return value of 1 means the path is at the position where the
25752f38b3e1SArne Jansen * item should be inserted. Normally this is the next bigger item,
25762f38b3e1SArne Jansen * but in case the previous item is the last in a leaf, path points
25772f38b3e1SArne Jansen * to the first free slot in the previous leaf, i.e. at an invalid
25782f38b3e1SArne Jansen * item.
25792f38b3e1SArne Jansen */
25802f38b3e1SArne Jansen leaf = p->nodes[0];
25812f38b3e1SArne Jansen
25822f38b3e1SArne Jansen if (find_higher) {
25832f38b3e1SArne Jansen if (p->slots[0] >= btrfs_header_nritems(leaf)) {
25842f38b3e1SArne Jansen ret = btrfs_next_leaf(root, p);
25852f38b3e1SArne Jansen if (ret <= 0)
25862f38b3e1SArne Jansen return ret;
25872f38b3e1SArne Jansen if (!return_any)
25882f38b3e1SArne Jansen return 1;
25892f38b3e1SArne Jansen /*
25902f38b3e1SArne Jansen * no higher item found, return the next
25912f38b3e1SArne Jansen * lower instead
25922f38b3e1SArne Jansen */
25932f38b3e1SArne Jansen return_any = 0;
25942f38b3e1SArne Jansen find_higher = 0;
25952f38b3e1SArne Jansen btrfs_release_path(p);
25962f38b3e1SArne Jansen goto again;
25972f38b3e1SArne Jansen }
25982f38b3e1SArne Jansen } else {
25992f38b3e1SArne Jansen if (p->slots[0] == 0) {
26002f38b3e1SArne Jansen ret = btrfs_prev_leaf(root, p);
2601e6793769SArne Jansen if (ret < 0)
26022f38b3e1SArne Jansen return ret;
2603e6793769SArne Jansen if (!ret) {
260423c6bf6aSFilipe David Borba Manana leaf = p->nodes[0];
260523c6bf6aSFilipe David Borba Manana if (p->slots[0] == btrfs_header_nritems(leaf))
260623c6bf6aSFilipe David Borba Manana p->slots[0]--;
2607e6793769SArne Jansen return 0;
2608e6793769SArne Jansen }
26092f38b3e1SArne Jansen if (!return_any)
26102f38b3e1SArne Jansen return 1;
26112f38b3e1SArne Jansen /*
26122f38b3e1SArne Jansen * no lower item found, return the next
26132f38b3e1SArne Jansen * higher instead
26142f38b3e1SArne Jansen */
26152f38b3e1SArne Jansen return_any = 0;
26162f38b3e1SArne Jansen find_higher = 1;
26172f38b3e1SArne Jansen btrfs_release_path(p);
26182f38b3e1SArne Jansen goto again;
2619e6793769SArne Jansen } else {
26202f38b3e1SArne Jansen --p->slots[0];
26212f38b3e1SArne Jansen }
26222f38b3e1SArne Jansen }
26232f38b3e1SArne Jansen return 0;
26242f38b3e1SArne Jansen }
26252f38b3e1SArne Jansen
26262f38b3e1SArne Jansen /*
26270ff40a91SMarcos Paulo de Souza * Execute search and call btrfs_previous_item to traverse backwards if the item
26280ff40a91SMarcos Paulo de Souza * was not found.
26290ff40a91SMarcos Paulo de Souza *
26300ff40a91SMarcos Paulo de Souza * Return 0 if found, 1 if not found and < 0 if error.
26310ff40a91SMarcos Paulo de Souza */
btrfs_search_backwards(struct btrfs_root * root,struct btrfs_key * key,struct btrfs_path * path)26320ff40a91SMarcos Paulo de Souza int btrfs_search_backwards(struct btrfs_root *root, struct btrfs_key *key,
26330ff40a91SMarcos Paulo de Souza struct btrfs_path *path)
26340ff40a91SMarcos Paulo de Souza {
26350ff40a91SMarcos Paulo de Souza int ret;
26360ff40a91SMarcos Paulo de Souza
26370ff40a91SMarcos Paulo de Souza ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
26380ff40a91SMarcos Paulo de Souza if (ret > 0)
26390ff40a91SMarcos Paulo de Souza ret = btrfs_previous_item(root, path, key->objectid, key->type);
26400ff40a91SMarcos Paulo de Souza
26410ff40a91SMarcos Paulo de Souza if (ret == 0)
26420ff40a91SMarcos Paulo de Souza btrfs_item_key_to_cpu(path->nodes[0], key, path->slots[0]);
26430ff40a91SMarcos Paulo de Souza
26440ff40a91SMarcos Paulo de Souza return ret;
26450ff40a91SMarcos Paulo de Souza }
26460ff40a91SMarcos Paulo de Souza
264743dd529aSDavid Sterba /*
264862142be3SGabriel Niebler * Search for a valid slot for the given path.
264962142be3SGabriel Niebler *
265062142be3SGabriel Niebler * @root: The root node of the tree.
265162142be3SGabriel Niebler * @key: Will contain a valid item if found.
265262142be3SGabriel Niebler * @path: The starting point to validate the slot.
265362142be3SGabriel Niebler *
265462142be3SGabriel Niebler * Return: 0 if the item is valid
265562142be3SGabriel Niebler * 1 if not found
265662142be3SGabriel Niebler * <0 if error.
265762142be3SGabriel Niebler */
btrfs_get_next_valid_item(struct btrfs_root * root,struct btrfs_key * key,struct btrfs_path * path)265862142be3SGabriel Niebler int btrfs_get_next_valid_item(struct btrfs_root *root, struct btrfs_key *key,
265962142be3SGabriel Niebler struct btrfs_path *path)
266062142be3SGabriel Niebler {
2661524f14bbSFilipe Manana if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) {
266262142be3SGabriel Niebler int ret;
266362142be3SGabriel Niebler
266462142be3SGabriel Niebler ret = btrfs_next_leaf(root, path);
266562142be3SGabriel Niebler if (ret)
266662142be3SGabriel Niebler return ret;
266762142be3SGabriel Niebler }
2668524f14bbSFilipe Manana
2669524f14bbSFilipe Manana btrfs_item_key_to_cpu(path->nodes[0], key, path->slots[0]);
267062142be3SGabriel Niebler return 0;
267162142be3SGabriel Niebler }
267262142be3SGabriel Niebler
26730ff40a91SMarcos Paulo de Souza /*
267474123bd7SChris Mason * adjust the pointers going up the tree, starting at level
267574123bd7SChris Mason * making sure the right key of each node is points to 'key'.
267674123bd7SChris Mason * This is used after shifting pointers to the left, so it stops
267774123bd7SChris Mason * fixing up pointers when a given leaf/node is not in slot 0 of the
267874123bd7SChris Mason * higher levels
2679aa5d6bedSChris Mason *
268074123bd7SChris Mason */
fixup_low_keys(struct btrfs_trans_handle * trans,struct btrfs_path * path,struct btrfs_disk_key * key,int level)2681*d5e09e38SFilipe Manana static void fixup_low_keys(struct btrfs_trans_handle *trans,
2682*d5e09e38SFilipe Manana struct btrfs_path *path,
26835f39d397SChris Mason struct btrfs_disk_key *key, int level)
2684be0e5c09SChris Mason {
2685be0e5c09SChris Mason int i;
26865f39d397SChris Mason struct extent_buffer *t;
26870e82bcfeSDavid Sterba int ret;
26885f39d397SChris Mason
2689234b63a0SChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) {
2690be0e5c09SChris Mason int tslot = path->slots[i];
26910e82bcfeSDavid Sterba
2692eb60ceacSChris Mason if (!path->nodes[i])
2693be0e5c09SChris Mason break;
26945f39d397SChris Mason t = path->nodes[i];
2695f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(t, tslot,
269633cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE);
26970e82bcfeSDavid Sterba BUG_ON(ret < 0);
26985f39d397SChris Mason btrfs_set_node_key(t, key, tslot);
2699*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, path->nodes[i]);
2700be0e5c09SChris Mason if (tslot != 0)
2701be0e5c09SChris Mason break;
2702be0e5c09SChris Mason }
2703be0e5c09SChris Mason }
2704be0e5c09SChris Mason
270574123bd7SChris Mason /*
270631840ae1SZheng Yan * update item key.
270731840ae1SZheng Yan *
270831840ae1SZheng Yan * This function isn't completely safe. It's the caller's responsibility
270931840ae1SZheng Yan * that the new key won't break the order
271031840ae1SZheng Yan */
btrfs_set_item_key_safe(struct btrfs_trans_handle * trans,struct btrfs_path * path,const struct btrfs_key * new_key)2711*d5e09e38SFilipe Manana void btrfs_set_item_key_safe(struct btrfs_trans_handle *trans,
2712b7a0365eSDaniel Dressler struct btrfs_path *path,
2713310712b2SOmar Sandoval const struct btrfs_key *new_key)
271431840ae1SZheng Yan {
2715*d5e09e38SFilipe Manana struct btrfs_fs_info *fs_info = trans->fs_info;
271631840ae1SZheng Yan struct btrfs_disk_key disk_key;
271731840ae1SZheng Yan struct extent_buffer *eb;
271831840ae1SZheng Yan int slot;
271931840ae1SZheng Yan
272031840ae1SZheng Yan eb = path->nodes[0];
272131840ae1SZheng Yan slot = path->slots[0];
272231840ae1SZheng Yan if (slot > 0) {
272331840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot - 1);
27247c15d410SQu Wenruo if (unlikely(comp_keys(&disk_key, new_key) >= 0)) {
2725eee3b811SQu Wenruo btrfs_print_leaf(eb);
27267c15d410SQu Wenruo btrfs_crit(fs_info,
27277c15d410SQu Wenruo "slot %u key (%llu %u %llu) new key (%llu %u %llu)",
27287c15d410SQu Wenruo slot, btrfs_disk_key_objectid(&disk_key),
27297c15d410SQu Wenruo btrfs_disk_key_type(&disk_key),
27307c15d410SQu Wenruo btrfs_disk_key_offset(&disk_key),
27317c15d410SQu Wenruo new_key->objectid, new_key->type,
27327c15d410SQu Wenruo new_key->offset);
27337c15d410SQu Wenruo BUG();
27347c15d410SQu Wenruo }
273531840ae1SZheng Yan }
273631840ae1SZheng Yan if (slot < btrfs_header_nritems(eb) - 1) {
273731840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot + 1);
27387c15d410SQu Wenruo if (unlikely(comp_keys(&disk_key, new_key) <= 0)) {
2739eee3b811SQu Wenruo btrfs_print_leaf(eb);
27407c15d410SQu Wenruo btrfs_crit(fs_info,
27417c15d410SQu Wenruo "slot %u key (%llu %u %llu) new key (%llu %u %llu)",
27427c15d410SQu Wenruo slot, btrfs_disk_key_objectid(&disk_key),
27437c15d410SQu Wenruo btrfs_disk_key_type(&disk_key),
27447c15d410SQu Wenruo btrfs_disk_key_offset(&disk_key),
27457c15d410SQu Wenruo new_key->objectid, new_key->type,
27467c15d410SQu Wenruo new_key->offset);
27477c15d410SQu Wenruo BUG();
27487c15d410SQu Wenruo }
274931840ae1SZheng Yan }
275031840ae1SZheng Yan
275131840ae1SZheng Yan btrfs_cpu_key_to_disk(&disk_key, new_key);
275231840ae1SZheng Yan btrfs_set_item_key(eb, &disk_key, slot);
2753*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, eb);
275431840ae1SZheng Yan if (slot == 0)
2755*d5e09e38SFilipe Manana fixup_low_keys(trans, path, &disk_key, 1);
275631840ae1SZheng Yan }
275731840ae1SZheng Yan
275831840ae1SZheng Yan /*
2759d16c702fSQu Wenruo * Check key order of two sibling extent buffers.
2760d16c702fSQu Wenruo *
2761d16c702fSQu Wenruo * Return true if something is wrong.
2762d16c702fSQu Wenruo * Return false if everything is fine.
2763d16c702fSQu Wenruo *
2764d16c702fSQu Wenruo * Tree-checker only works inside one tree block, thus the following
2765d16c702fSQu Wenruo * corruption can not be detected by tree-checker:
2766d16c702fSQu Wenruo *
2767d16c702fSQu Wenruo * Leaf @left | Leaf @right
2768d16c702fSQu Wenruo * --------------------------------------------------------------
2769d16c702fSQu Wenruo * | 1 | 2 | 3 | 4 | 5 | f6 | | 7 | 8 |
2770d16c702fSQu Wenruo *
2771d16c702fSQu Wenruo * Key f6 in leaf @left itself is valid, but not valid when the next
2772d16c702fSQu Wenruo * key in leaf @right is 7.
2773d16c702fSQu Wenruo * This can only be checked at tree block merge time.
2774d16c702fSQu Wenruo * And since tree checker has ensured all key order in each tree block
2775d16c702fSQu Wenruo * is correct, we only need to bother the last key of @left and the first
2776d16c702fSQu Wenruo * key of @right.
2777d16c702fSQu Wenruo */
check_sibling_keys(struct extent_buffer * left,struct extent_buffer * right)2778d16c702fSQu Wenruo static bool check_sibling_keys(struct extent_buffer *left,
2779d16c702fSQu Wenruo struct extent_buffer *right)
2780d16c702fSQu Wenruo {
2781d16c702fSQu Wenruo struct btrfs_key left_last;
2782d16c702fSQu Wenruo struct btrfs_key right_first;
2783d16c702fSQu Wenruo int level = btrfs_header_level(left);
2784d16c702fSQu Wenruo int nr_left = btrfs_header_nritems(left);
2785d16c702fSQu Wenruo int nr_right = btrfs_header_nritems(right);
2786d16c702fSQu Wenruo
2787d16c702fSQu Wenruo /* No key to check in one of the tree blocks */
2788d16c702fSQu Wenruo if (!nr_left || !nr_right)
2789d16c702fSQu Wenruo return false;
2790d16c702fSQu Wenruo
2791d16c702fSQu Wenruo if (level) {
2792d16c702fSQu Wenruo btrfs_node_key_to_cpu(left, &left_last, nr_left - 1);
2793d16c702fSQu Wenruo btrfs_node_key_to_cpu(right, &right_first, 0);
2794d16c702fSQu Wenruo } else {
2795d16c702fSQu Wenruo btrfs_item_key_to_cpu(left, &left_last, nr_left - 1);
2796d16c702fSQu Wenruo btrfs_item_key_to_cpu(right, &right_first, 0);
2797d16c702fSQu Wenruo }
2798d16c702fSQu Wenruo
279988ad95b0SFilipe Manana if (unlikely(btrfs_comp_cpu_keys(&left_last, &right_first) >= 0)) {
2800a2cea677SFilipe Manana btrfs_crit(left->fs_info, "left extent buffer:");
2801a2cea677SFilipe Manana btrfs_print_tree(left, false);
2802a2cea677SFilipe Manana btrfs_crit(left->fs_info, "right extent buffer:");
2803a2cea677SFilipe Manana btrfs_print_tree(right, false);
2804d16c702fSQu Wenruo btrfs_crit(left->fs_info,
2805d16c702fSQu Wenruo "bad key order, sibling blocks, left last (%llu %u %llu) right first (%llu %u %llu)",
2806d16c702fSQu Wenruo left_last.objectid, left_last.type,
2807d16c702fSQu Wenruo left_last.offset, right_first.objectid,
2808d16c702fSQu Wenruo right_first.type, right_first.offset);
2809d16c702fSQu Wenruo return true;
2810d16c702fSQu Wenruo }
2811d16c702fSQu Wenruo return false;
2812d16c702fSQu Wenruo }
2813d16c702fSQu Wenruo
2814d16c702fSQu Wenruo /*
281574123bd7SChris Mason * try to push data from one node into the next node left in the
281679f95c82SChris Mason * tree.
2817aa5d6bedSChris Mason *
2818aa5d6bedSChris Mason * returns 0 if some ptrs were pushed left, < 0 if there was some horrible
2819aa5d6bedSChris Mason * error, and > 0 if there was no room in the left hand block.
282074123bd7SChris Mason */
push_node_left(struct btrfs_trans_handle * trans,struct extent_buffer * dst,struct extent_buffer * src,int empty)282198ed5174SChris Mason static int push_node_left(struct btrfs_trans_handle *trans,
28222ff7e61eSJeff Mahoney struct extent_buffer *dst,
2823971a1f66SChris Mason struct extent_buffer *src, int empty)
2824be0e5c09SChris Mason {
2825d30a668fSDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info;
2826be0e5c09SChris Mason int push_items = 0;
2827bb803951SChris Mason int src_nritems;
2828bb803951SChris Mason int dst_nritems;
2829aa5d6bedSChris Mason int ret = 0;
2830be0e5c09SChris Mason
28315f39d397SChris Mason src_nritems = btrfs_header_nritems(src);
28325f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst);
28330b246afaSJeff Mahoney push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems;
28347bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid);
28357bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid);
283654aa1f4dSChris Mason
2837bce4eae9SChris Mason if (!empty && src_nritems <= 8)
2838971a1f66SChris Mason return 1;
2839971a1f66SChris Mason
2840d397712bSChris Mason if (push_items <= 0)
2841be0e5c09SChris Mason return 1;
2842be0e5c09SChris Mason
2843bce4eae9SChris Mason if (empty) {
2844971a1f66SChris Mason push_items = min(src_nritems, push_items);
2845bce4eae9SChris Mason if (push_items < src_nritems) {
2846bce4eae9SChris Mason /* leave at least 8 pointers in the node if
2847bce4eae9SChris Mason * we aren't going to empty it
2848bce4eae9SChris Mason */
2849bce4eae9SChris Mason if (src_nritems - push_items < 8) {
2850bce4eae9SChris Mason if (push_items <= 8)
2851bce4eae9SChris Mason return 1;
2852bce4eae9SChris Mason push_items -= 8;
2853bce4eae9SChris Mason }
2854bce4eae9SChris Mason }
2855bce4eae9SChris Mason } else
2856bce4eae9SChris Mason push_items = min(src_nritems - 8, push_items);
285779f95c82SChris Mason
2858d16c702fSQu Wenruo /* dst is the left eb, src is the middle eb */
2859d16c702fSQu Wenruo if (check_sibling_keys(dst, src)) {
2860d16c702fSQu Wenruo ret = -EUCLEAN;
2861d16c702fSQu Wenruo btrfs_abort_transaction(trans, ret);
2862d16c702fSQu Wenruo return ret;
2863d16c702fSQu Wenruo }
2864f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_eb_copy(dst, src, dst_nritems, 0, push_items);
28655de865eeSFilipe David Borba Manana if (ret) {
286666642832SJeff Mahoney btrfs_abort_transaction(trans, ret);
28675de865eeSFilipe David Borba Manana return ret;
28685de865eeSFilipe David Borba Manana }
28695f39d397SChris Mason copy_extent_buffer(dst, src,
2870e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(dst, dst_nritems),
2871e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(src, 0),
2872123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr));
28735f39d397SChris Mason
2874bb803951SChris Mason if (push_items < src_nritems) {
287557911b8bSJan Schmidt /*
28765cead542SBoris Burkov * btrfs_tree_mod_log_eb_copy handles logging the move, so we
28775cead542SBoris Burkov * don't need to do an explicit tree mod log operation for it.
287857911b8bSJan Schmidt */
2879e23efd8eSJosef Bacik memmove_extent_buffer(src, btrfs_node_key_ptr_offset(src, 0),
2880e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(src, push_items),
2881e2fa7227SChris Mason (src_nritems - push_items) *
2882123abc88SChris Mason sizeof(struct btrfs_key_ptr));
2883bb803951SChris Mason }
28845f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items);
28855f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items);
2886*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, src);
2887*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, dst);
288831840ae1SZheng Yan
2889bb803951SChris Mason return ret;
2890be0e5c09SChris Mason }
2891be0e5c09SChris Mason
289297571fd0SChris Mason /*
289379f95c82SChris Mason * try to push data from one node into the next node right in the
289479f95c82SChris Mason * tree.
289579f95c82SChris Mason *
289679f95c82SChris Mason * returns 0 if some ptrs were pushed, < 0 if there was some horrible
289779f95c82SChris Mason * error, and > 0 if there was no room in the right hand block.
289879f95c82SChris Mason *
289979f95c82SChris Mason * this will only push up to 1/2 the contents of the left node over
290079f95c82SChris Mason */
balance_node_right(struct btrfs_trans_handle * trans,struct extent_buffer * dst,struct extent_buffer * src)29015f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans,
29025f39d397SChris Mason struct extent_buffer *dst,
29035f39d397SChris Mason struct extent_buffer *src)
290479f95c82SChris Mason {
290555d32ed8SDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info;
290679f95c82SChris Mason int push_items = 0;
290779f95c82SChris Mason int max_push;
290879f95c82SChris Mason int src_nritems;
290979f95c82SChris Mason int dst_nritems;
291079f95c82SChris Mason int ret = 0;
291179f95c82SChris Mason
29127bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid);
29137bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid);
29147bb86316SChris Mason
29155f39d397SChris Mason src_nritems = btrfs_header_nritems(src);
29165f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst);
29170b246afaSJeff Mahoney push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems;
2918d397712bSChris Mason if (push_items <= 0)
291979f95c82SChris Mason return 1;
2920bce4eae9SChris Mason
2921d397712bSChris Mason if (src_nritems < 4)
2922bce4eae9SChris Mason return 1;
292379f95c82SChris Mason
292479f95c82SChris Mason max_push = src_nritems / 2 + 1;
292579f95c82SChris Mason /* don't try to empty the node */
2926d397712bSChris Mason if (max_push >= src_nritems)
292779f95c82SChris Mason return 1;
2928252c38f0SYan
292979f95c82SChris Mason if (max_push < push_items)
293079f95c82SChris Mason push_items = max_push;
293179f95c82SChris Mason
2932d16c702fSQu Wenruo /* dst is the right eb, src is the middle eb */
2933d16c702fSQu Wenruo if (check_sibling_keys(src, dst)) {
2934d16c702fSQu Wenruo ret = -EUCLEAN;
2935d16c702fSQu Wenruo btrfs_abort_transaction(trans, ret);
2936d16c702fSQu Wenruo return ret;
2937d16c702fSQu Wenruo }
29385cead542SBoris Burkov
29395cead542SBoris Burkov /*
29405cead542SBoris Burkov * btrfs_tree_mod_log_eb_copy handles logging the move, so we don't
29415cead542SBoris Burkov * need to do an explicit tree mod log operation for it.
29425cead542SBoris Burkov */
2943e23efd8eSJosef Bacik memmove_extent_buffer(dst, btrfs_node_key_ptr_offset(dst, push_items),
2944e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(dst, 0),
29455f39d397SChris Mason (dst_nritems) *
29465f39d397SChris Mason sizeof(struct btrfs_key_ptr));
2947d6025579SChris Mason
2948f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_eb_copy(dst, src, 0, src_nritems - push_items,
2949ed874f0dSDavid Sterba push_items);
29505de865eeSFilipe David Borba Manana if (ret) {
295166642832SJeff Mahoney btrfs_abort_transaction(trans, ret);
29525de865eeSFilipe David Borba Manana return ret;
29535de865eeSFilipe David Borba Manana }
29545f39d397SChris Mason copy_extent_buffer(dst, src,
2955e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(dst, 0),
2956e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(src, src_nritems - push_items),
2957123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr));
295879f95c82SChris Mason
29595f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items);
29605f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items);
296179f95c82SChris Mason
2962*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, src);
2963*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, dst);
296431840ae1SZheng Yan
296579f95c82SChris Mason return ret;
296679f95c82SChris Mason }
296779f95c82SChris Mason
296879f95c82SChris Mason /*
296997571fd0SChris Mason * helper function to insert a new root level in the tree.
297097571fd0SChris Mason * A new node is allocated, and a single item is inserted to
297197571fd0SChris Mason * point to the existing root
2972aa5d6bedSChris Mason *
2973aa5d6bedSChris Mason * returns zero on success or < 0 on failure.
297497571fd0SChris Mason */
insert_new_root(struct btrfs_trans_handle * trans,struct btrfs_root * root,struct btrfs_path * path,int level)2975d397712bSChris Mason static noinline int insert_new_root(struct btrfs_trans_handle *trans,
29765f39d397SChris Mason struct btrfs_root *root,
2977fdd99c72SLiu Bo struct btrfs_path *path, int level)
297874123bd7SChris Mason {
29790b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info;
29807bb86316SChris Mason u64 lower_gen;
29815f39d397SChris Mason struct extent_buffer *lower;
29825f39d397SChris Mason struct extent_buffer *c;
2983925baeddSChris Mason struct extent_buffer *old;
29845f39d397SChris Mason struct btrfs_disk_key lower_key;
2985d9d19a01SDavid Sterba int ret;
29865c680ed6SChris Mason
29875c680ed6SChris Mason BUG_ON(path->nodes[level]);
29885c680ed6SChris Mason BUG_ON(path->nodes[level-1] != root->node);
29895c680ed6SChris Mason
29907bb86316SChris Mason lower = path->nodes[level-1];
29917bb86316SChris Mason if (level == 1)
29927bb86316SChris Mason btrfs_item_key(lower, &lower_key, 0);
29937bb86316SChris Mason else
29947bb86316SChris Mason btrfs_node_key(lower, &lower_key, 0);
29957bb86316SChris Mason
299679bd3712SFilipe Manana c = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid,
299779bd3712SFilipe Manana &lower_key, level, root->node->start, 0,
2998cf6f34aaSJosef Bacik BTRFS_NESTING_NEW_ROOT);
29995f39d397SChris Mason if (IS_ERR(c))
30005f39d397SChris Mason return PTR_ERR(c);
3001925baeddSChris Mason
30020b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize);
3003f0486c68SYan, Zheng
30045f39d397SChris Mason btrfs_set_header_nritems(c, 1);
30055f39d397SChris Mason btrfs_set_node_key(c, &lower_key, 0);
3006db94535dSChris Mason btrfs_set_node_blockptr(c, 0, lower->start);
30077bb86316SChris Mason lower_gen = btrfs_header_generation(lower);
300831840ae1SZheng Yan WARN_ON(lower_gen != trans->transid);
30097bb86316SChris Mason
30107bb86316SChris Mason btrfs_set_node_ptr_generation(c, 0, lower_gen);
30115f39d397SChris Mason
3012*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, c);
3013d5719762SChris Mason
3014925baeddSChris Mason old = root->node;
3015406808abSFilipe Manana ret = btrfs_tree_mod_log_insert_root(root->node, c, false);
3016f61aa7baSFilipe Manana if (ret < 0) {
3017f61aa7baSFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), c, 0, 1);
3018f61aa7baSFilipe Manana btrfs_tree_unlock(c);
3019f61aa7baSFilipe Manana free_extent_buffer(c);
3020f61aa7baSFilipe Manana return ret;
3021f61aa7baSFilipe Manana }
3022240f62c8SChris Mason rcu_assign_pointer(root->node, c);
3023925baeddSChris Mason
3024925baeddSChris Mason /* the super has an extra ref to root->node */
3025925baeddSChris Mason free_extent_buffer(old);
3026925baeddSChris Mason
30270b86a832SChris Mason add_root_to_dirty_list(root);
302867439dadSDavid Sterba atomic_inc(&c->refs);
30295f39d397SChris Mason path->nodes[level] = c;
3030ac5887c8SJosef Bacik path->locks[level] = BTRFS_WRITE_LOCK;
303174123bd7SChris Mason path->slots[level] = 0;
303274123bd7SChris Mason return 0;
303374123bd7SChris Mason }
30345c680ed6SChris Mason
30355c680ed6SChris Mason /*
30365c680ed6SChris Mason * worker function to insert a single pointer in a node.
30375c680ed6SChris Mason * the node should have enough room for the pointer already
303897571fd0SChris Mason *
30395c680ed6SChris Mason * slot and level indicate where you want the key to go, and
30405c680ed6SChris Mason * blocknr is the block the key points to.
30415c680ed6SChris Mason */
insert_ptr(struct btrfs_trans_handle * trans,struct btrfs_path * path,struct btrfs_disk_key * key,u64 bytenr,int slot,int level)304250b5d1fcSFilipe Manana static int insert_ptr(struct btrfs_trans_handle *trans,
30436ad3cf6dSDavid Sterba struct btrfs_path *path,
3044143bede5SJeff Mahoney struct btrfs_disk_key *key, u64 bytenr,
3045c3e06965SJan Schmidt int slot, int level)
30465c680ed6SChris Mason {
30475f39d397SChris Mason struct extent_buffer *lower;
30485c680ed6SChris Mason int nritems;
3049f3ea38daSJan Schmidt int ret;
30505c680ed6SChris Mason
30515c680ed6SChris Mason BUG_ON(!path->nodes[level]);
305249d0c642SFilipe Manana btrfs_assert_tree_write_locked(path->nodes[level]);
30535f39d397SChris Mason lower = path->nodes[level];
30545f39d397SChris Mason nritems = btrfs_header_nritems(lower);
3055c293498bSStoyan Gaydarov BUG_ON(slot > nritems);
30566ad3cf6dSDavid Sterba BUG_ON(nritems == BTRFS_NODEPTRS_PER_BLOCK(trans->fs_info));
305774123bd7SChris Mason if (slot != nritems) {
3058bf1d3425SDavid Sterba if (level) {
3059f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_move(lower, slot + 1,
3060f3a84ccdSFilipe Manana slot, nritems - slot);
306150b5d1fcSFilipe Manana if (ret < 0) {
306250b5d1fcSFilipe Manana btrfs_abort_transaction(trans, ret);
306350b5d1fcSFilipe Manana return ret;
306450b5d1fcSFilipe Manana }
3065bf1d3425SDavid Sterba }
30665f39d397SChris Mason memmove_extent_buffer(lower,
3067e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(lower, slot + 1),
3068e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(lower, slot),
3069123abc88SChris Mason (nritems - slot) * sizeof(struct btrfs_key_ptr));
307074123bd7SChris Mason }
3071c3e06965SJan Schmidt if (level) {
3072f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(lower, slot,
307333cff222SFilipe Manana BTRFS_MOD_LOG_KEY_ADD);
307450b5d1fcSFilipe Manana if (ret < 0) {
307550b5d1fcSFilipe Manana btrfs_abort_transaction(trans, ret);
307650b5d1fcSFilipe Manana return ret;
307750b5d1fcSFilipe Manana }
3078f3ea38daSJan Schmidt }
30795f39d397SChris Mason btrfs_set_node_key(lower, key, slot);
3080db94535dSChris Mason btrfs_set_node_blockptr(lower, slot, bytenr);
308174493f7aSChris Mason WARN_ON(trans->transid == 0);
308274493f7aSChris Mason btrfs_set_node_ptr_generation(lower, slot, trans->transid);
30835f39d397SChris Mason btrfs_set_header_nritems(lower, nritems + 1);
3084*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, lower);
308550b5d1fcSFilipe Manana
308650b5d1fcSFilipe Manana return 0;
308774123bd7SChris Mason }
308874123bd7SChris Mason
308997571fd0SChris Mason /*
309097571fd0SChris Mason * split the node at the specified level in path in two.
309197571fd0SChris Mason * The path is corrected to point to the appropriate node after the split
309297571fd0SChris Mason *
309397571fd0SChris Mason * Before splitting this tries to make some room in the node by pushing
309497571fd0SChris Mason * left and right, if either one works, it returns right away.
3095aa5d6bedSChris Mason *
3096aa5d6bedSChris Mason * returns 0 on success and < 0 on failure
309797571fd0SChris Mason */
split_node(struct btrfs_trans_handle * trans,struct btrfs_root * root,struct btrfs_path * path,int level)3098e02119d5SChris Mason static noinline int split_node(struct btrfs_trans_handle *trans,
3099e02119d5SChris Mason struct btrfs_root *root,
3100e02119d5SChris Mason struct btrfs_path *path, int level)
3101be0e5c09SChris Mason {
31020b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info;
31035f39d397SChris Mason struct extent_buffer *c;
31045f39d397SChris Mason struct extent_buffer *split;
31055f39d397SChris Mason struct btrfs_disk_key disk_key;
3106be0e5c09SChris Mason int mid;
31075c680ed6SChris Mason int ret;
31087518a238SChris Mason u32 c_nritems;
3109be0e5c09SChris Mason
31105f39d397SChris Mason c = path->nodes[level];
31117bb86316SChris Mason WARN_ON(btrfs_header_generation(c) != trans->transid);
31125f39d397SChris Mason if (c == root->node) {
3113d9abbf1cSJan Schmidt /*
311490f8d62eSJan Schmidt * trying to split the root, lets make a new one
311590f8d62eSJan Schmidt *
3116fdd99c72SLiu Bo * tree mod log: We don't log_removal old root in
311790f8d62eSJan Schmidt * insert_new_root, because that root buffer will be kept as a
311890f8d62eSJan Schmidt * normal node. We are going to log removal of half of the
3119f3a84ccdSFilipe Manana * elements below with btrfs_tree_mod_log_eb_copy(). We're
3120f3a84ccdSFilipe Manana * holding a tree lock on the buffer, which is why we cannot
3121f3a84ccdSFilipe Manana * race with other tree_mod_log users.
3122d9abbf1cSJan Schmidt */
3123fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, level + 1);
31245c680ed6SChris Mason if (ret)
31255c680ed6SChris Mason return ret;
3126b3612421SChris Mason } else {
3127e66f709bSChris Mason ret = push_nodes_for_insert(trans, root, path, level);
31285f39d397SChris Mason c = path->nodes[level];
31295f39d397SChris Mason if (!ret && btrfs_header_nritems(c) <
31300b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3)
3131e66f709bSChris Mason return 0;
313254aa1f4dSChris Mason if (ret < 0)
313354aa1f4dSChris Mason return ret;
31345c680ed6SChris Mason }
3135e66f709bSChris Mason
31365f39d397SChris Mason c_nritems = btrfs_header_nritems(c);
31375d4f98a2SYan Zheng mid = (c_nritems + 1) / 2;
31385d4f98a2SYan Zheng btrfs_node_key(c, &disk_key, mid);
31397bb86316SChris Mason
314079bd3712SFilipe Manana split = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid,
314179bd3712SFilipe Manana &disk_key, level, c->start, 0,
314279bd3712SFilipe Manana BTRFS_NESTING_SPLIT);
31435f39d397SChris Mason if (IS_ERR(split))
31445f39d397SChris Mason return PTR_ERR(split);
314554aa1f4dSChris Mason
31460b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize);
3147bc877d28SNikolay Borisov ASSERT(btrfs_header_level(c) == level);
31485f39d397SChris Mason
3149f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_eb_copy(split, c, 0, mid, c_nritems - mid);
31505de865eeSFilipe David Borba Manana if (ret) {
3151ede600e4SFilipe Manana btrfs_tree_unlock(split);
3152ede600e4SFilipe Manana free_extent_buffer(split);
315366642832SJeff Mahoney btrfs_abort_transaction(trans, ret);
31545de865eeSFilipe David Borba Manana return ret;
31555de865eeSFilipe David Borba Manana }
31565f39d397SChris Mason copy_extent_buffer(split, c,
3157e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(split, 0),
3158e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(c, mid),
3159123abc88SChris Mason (c_nritems - mid) * sizeof(struct btrfs_key_ptr));
31605f39d397SChris Mason btrfs_set_header_nritems(split, c_nritems - mid);
31615f39d397SChris Mason btrfs_set_header_nritems(c, mid);
3162aa5d6bedSChris Mason
3163*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, c);
3164*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, split);
31655f39d397SChris Mason
316650b5d1fcSFilipe Manana ret = insert_ptr(trans, path, &disk_key, split->start,
3167c3e06965SJan Schmidt path->slots[level + 1] + 1, level + 1);
316850b5d1fcSFilipe Manana if (ret < 0) {
316950b5d1fcSFilipe Manana btrfs_tree_unlock(split);
317050b5d1fcSFilipe Manana free_extent_buffer(split);
317150b5d1fcSFilipe Manana return ret;
317250b5d1fcSFilipe Manana }
3173aa5d6bedSChris Mason
31745de08d7dSChris Mason if (path->slots[level] >= mid) {
31755c680ed6SChris Mason path->slots[level] -= mid;
3176925baeddSChris Mason btrfs_tree_unlock(c);
31775f39d397SChris Mason free_extent_buffer(c);
31785f39d397SChris Mason path->nodes[level] = split;
31795c680ed6SChris Mason path->slots[level + 1] += 1;
3180eb60ceacSChris Mason } else {
3181925baeddSChris Mason btrfs_tree_unlock(split);
31825f39d397SChris Mason free_extent_buffer(split);
3183be0e5c09SChris Mason }
3184d5286a92SNikolay Borisov return 0;
3185be0e5c09SChris Mason }
3186be0e5c09SChris Mason
318774123bd7SChris Mason /*
318874123bd7SChris Mason * how many bytes are required to store the items in a leaf. start
318974123bd7SChris Mason * and nr indicate which items in the leaf to check. This totals up the
319074123bd7SChris Mason * space used both by the item structs and the item data
319174123bd7SChris Mason */
leaf_space_used(const struct extent_buffer * l,int start,int nr)31926c75a589SQu Wenruo static int leaf_space_used(const struct extent_buffer *l, int start, int nr)
3193be0e5c09SChris Mason {
3194be0e5c09SChris Mason int data_len;
31955f39d397SChris Mason int nritems = btrfs_header_nritems(l);
3196d4dbff95SChris Mason int end = min(nritems, start + nr) - 1;
3197be0e5c09SChris Mason
3198be0e5c09SChris Mason if (!nr)
3199be0e5c09SChris Mason return 0;
32003212fa14SJosef Bacik data_len = btrfs_item_offset(l, start) + btrfs_item_size(l, start);
32013212fa14SJosef Bacik data_len = data_len - btrfs_item_offset(l, end);
32020783fcfcSChris Mason data_len += sizeof(struct btrfs_item) * nr;
3203d4dbff95SChris Mason WARN_ON(data_len < 0);
3204be0e5c09SChris Mason return data_len;
3205be0e5c09SChris Mason }
3206be0e5c09SChris Mason
320774123bd7SChris Mason /*
3208d4dbff95SChris Mason * The space between the end of the leaf items and
3209d4dbff95SChris Mason * the start of the leaf data. IOW, how much room
3210d4dbff95SChris Mason * the leaf has left for both items and data
3211d4dbff95SChris Mason */
btrfs_leaf_free_space(const struct extent_buffer * leaf)32126c75a589SQu Wenruo int btrfs_leaf_free_space(const struct extent_buffer *leaf)
3213d4dbff95SChris Mason {
3214e902baacSDavid Sterba struct btrfs_fs_info *fs_info = leaf->fs_info;
32155f39d397SChris Mason int nritems = btrfs_header_nritems(leaf);
32165f39d397SChris Mason int ret;
32170b246afaSJeff Mahoney
32180b246afaSJeff Mahoney ret = BTRFS_LEAF_DATA_SIZE(fs_info) - leaf_space_used(leaf, 0, nritems);
32195f39d397SChris Mason if (ret < 0) {
32200b246afaSJeff Mahoney btrfs_crit(fs_info,
3221efe120a0SFrank Holton "leaf free space ret %d, leaf data size %lu, used %d nritems %d",
3222da17066cSJeff Mahoney ret,
32230b246afaSJeff Mahoney (unsigned long) BTRFS_LEAF_DATA_SIZE(fs_info),
32245f39d397SChris Mason leaf_space_used(leaf, 0, nritems), nritems);
32255f39d397SChris Mason }
32265f39d397SChris Mason return ret;
3227d4dbff95SChris Mason }
3228d4dbff95SChris Mason
322999d8f83cSChris Mason /*
323099d8f83cSChris Mason * min slot controls the lowest index we're willing to push to the
323199d8f83cSChris Mason * right. We'll push up to and including min_slot, but no lower
323299d8f83cSChris Mason */
__push_leaf_right(struct btrfs_trans_handle * trans,struct btrfs_path * path,int data_size,int empty,struct extent_buffer * right,int free_space,u32 left_nritems,u32 min_slot)3233ed25dab3SJosef Bacik static noinline int __push_leaf_right(struct btrfs_trans_handle *trans,
3234ed25dab3SJosef Bacik struct btrfs_path *path,
323544871b1bSChris Mason int data_size, int empty,
323644871b1bSChris Mason struct extent_buffer *right,
323799d8f83cSChris Mason int free_space, u32 left_nritems,
323899d8f83cSChris Mason u32 min_slot)
323900ec4c51SChris Mason {
3240f72f0010SDavid Sterba struct btrfs_fs_info *fs_info = right->fs_info;
32415f39d397SChris Mason struct extent_buffer *left = path->nodes[0];
324244871b1bSChris Mason struct extent_buffer *upper = path->nodes[1];
3243cfed81a0SChris Mason struct btrfs_map_token token;
32445f39d397SChris Mason struct btrfs_disk_key disk_key;
324500ec4c51SChris Mason int slot;
324634a38218SChris Mason u32 i;
324700ec4c51SChris Mason int push_space = 0;
324800ec4c51SChris Mason int push_items = 0;
324934a38218SChris Mason u32 nr;
32507518a238SChris Mason u32 right_nritems;
32515f39d397SChris Mason u32 data_end;
3252db94535dSChris Mason u32 this_item_size;
325300ec4c51SChris Mason
325434a38218SChris Mason if (empty)
325534a38218SChris Mason nr = 0;
325634a38218SChris Mason else
325799d8f83cSChris Mason nr = max_t(u32, 1, min_slot);
325834a38218SChris Mason
325931840ae1SZheng Yan if (path->slots[0] >= left_nritems)
326087b29b20SYan Zheng push_space += data_size;
326131840ae1SZheng Yan
326244871b1bSChris Mason slot = path->slots[1];
326334a38218SChris Mason i = left_nritems - 1;
326434a38218SChris Mason while (i >= nr) {
326531840ae1SZheng Yan if (!empty && push_items > 0) {
326631840ae1SZheng Yan if (path->slots[0] > i)
326731840ae1SZheng Yan break;
326831840ae1SZheng Yan if (path->slots[0] == i) {
3269e902baacSDavid Sterba int space = btrfs_leaf_free_space(left);
3270e902baacSDavid Sterba
327131840ae1SZheng Yan if (space + push_space * 2 > free_space)
327231840ae1SZheng Yan break;
327331840ae1SZheng Yan }
327431840ae1SZheng Yan }
327531840ae1SZheng Yan
327600ec4c51SChris Mason if (path->slots[0] == i)
327787b29b20SYan Zheng push_space += data_size;
3278db94535dSChris Mason
32793212fa14SJosef Bacik this_item_size = btrfs_item_size(left, i);
328074794207SJosef Bacik if (this_item_size + sizeof(struct btrfs_item) +
328174794207SJosef Bacik push_space > free_space)
328200ec4c51SChris Mason break;
328331840ae1SZheng Yan
328400ec4c51SChris Mason push_items++;
328574794207SJosef Bacik push_space += this_item_size + sizeof(struct btrfs_item);
328634a38218SChris Mason if (i == 0)
328734a38218SChris Mason break;
328834a38218SChris Mason i--;
3289db94535dSChris Mason }
32905f39d397SChris Mason
3291925baeddSChris Mason if (push_items == 0)
3292925baeddSChris Mason goto out_unlock;
32935f39d397SChris Mason
32946c1500f2SJulia Lawall WARN_ON(!empty && push_items == left_nritems);
32955f39d397SChris Mason
329600ec4c51SChris Mason /* push left to right */
32975f39d397SChris Mason right_nritems = btrfs_header_nritems(right);
329834a38218SChris Mason
3299dc2e724eSJosef Bacik push_space = btrfs_item_data_end(left, left_nritems - push_items);
33008f881e8cSDavid Sterba push_space -= leaf_data_end(left);
33015f39d397SChris Mason
330200ec4c51SChris Mason /* make room in the right data area */
33038f881e8cSDavid Sterba data_end = leaf_data_end(right);
3304637e3b48SJosef Bacik memmove_leaf_data(right, data_end - push_space, data_end,
33050b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - data_end);
33065f39d397SChris Mason
330700ec4c51SChris Mason /* copy from the left data area */
3308637e3b48SJosef Bacik copy_leaf_data(right, left, BTRFS_LEAF_DATA_SIZE(fs_info) - push_space,
3309637e3b48SJosef Bacik leaf_data_end(left), push_space);
33105f39d397SChris Mason
3311637e3b48SJosef Bacik memmove_leaf_items(right, push_items, 0, right_nritems);
33125f39d397SChris Mason
331300ec4c51SChris Mason /* copy the items from left to right */
3314637e3b48SJosef Bacik copy_leaf_items(right, left, 0, left_nritems - push_items, push_items);
331500ec4c51SChris Mason
331600ec4c51SChris Mason /* update the item pointers */
3317c82f823cSDavid Sterba btrfs_init_map_token(&token, right);
33187518a238SChris Mason right_nritems += push_items;
33195f39d397SChris Mason btrfs_set_header_nritems(right, right_nritems);
33200b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info);
33217518a238SChris Mason for (i = 0; i < right_nritems; i++) {
33223212fa14SJosef Bacik push_space -= btrfs_token_item_size(&token, i);
33233212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, push_space);
3324db94535dSChris Mason }
3325db94535dSChris Mason
33267518a238SChris Mason left_nritems -= push_items;
33275f39d397SChris Mason btrfs_set_header_nritems(left, left_nritems);
332800ec4c51SChris Mason
332934a38218SChris Mason if (left_nritems)
3330*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, left);
3331f0486c68SYan, Zheng else
3332190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, left);
3333f0486c68SYan, Zheng
3334*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, right);
3335a429e513SChris Mason
33365f39d397SChris Mason btrfs_item_key(right, &disk_key, 0);
33375f39d397SChris Mason btrfs_set_node_key(upper, &disk_key, slot + 1);
3338*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, upper);
333902217ed2SChris Mason
334000ec4c51SChris Mason /* then fixup the leaf pointer in the path */
33417518a238SChris Mason if (path->slots[0] >= left_nritems) {
33427518a238SChris Mason path->slots[0] -= left_nritems;
3343925baeddSChris Mason if (btrfs_header_nritems(path->nodes[0]) == 0)
3344190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, path->nodes[0]);
3345925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]);
33465f39d397SChris Mason free_extent_buffer(path->nodes[0]);
33475f39d397SChris Mason path->nodes[0] = right;
334800ec4c51SChris Mason path->slots[1] += 1;
334900ec4c51SChris Mason } else {
3350925baeddSChris Mason btrfs_tree_unlock(right);
33515f39d397SChris Mason free_extent_buffer(right);
335200ec4c51SChris Mason }
335300ec4c51SChris Mason return 0;
3354925baeddSChris Mason
3355925baeddSChris Mason out_unlock:
3356925baeddSChris Mason btrfs_tree_unlock(right);
3357925baeddSChris Mason free_extent_buffer(right);
3358925baeddSChris Mason return 1;
335900ec4c51SChris Mason }
3360925baeddSChris Mason
336100ec4c51SChris Mason /*
336244871b1bSChris Mason * push some data in the path leaf to the right, trying to free up at
336374123bd7SChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise
336444871b1bSChris Mason *
336544871b1bSChris Mason * returns 1 if the push failed because the other node didn't have enough
336644871b1bSChris Mason * room, 0 if everything worked out and < 0 if there were major errors.
336799d8f83cSChris Mason *
336899d8f83cSChris Mason * this will push starting from min_slot to the end of the leaf. It won't
336999d8f83cSChris Mason * push any slot lower than min_slot
337074123bd7SChris Mason */
push_leaf_right(struct btrfs_trans_handle * trans,struct btrfs_root * root,struct btrfs_path * path,int min_data_size,int data_size,int empty,u32 min_slot)337144871b1bSChris Mason static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root
337299d8f83cSChris Mason *root, struct btrfs_path *path,
337399d8f83cSChris Mason int min_data_size, int data_size,
337499d8f83cSChris Mason int empty, u32 min_slot)
3375be0e5c09SChris Mason {
337644871b1bSChris Mason struct extent_buffer *left = path->nodes[0];
337744871b1bSChris Mason struct extent_buffer *right;
337844871b1bSChris Mason struct extent_buffer *upper;
337944871b1bSChris Mason int slot;
338044871b1bSChris Mason int free_space;
338144871b1bSChris Mason u32 left_nritems;
338244871b1bSChris Mason int ret;
338344871b1bSChris Mason
338444871b1bSChris Mason if (!path->nodes[1])
338544871b1bSChris Mason return 1;
338644871b1bSChris Mason
338744871b1bSChris Mason slot = path->slots[1];
338844871b1bSChris Mason upper = path->nodes[1];
338944871b1bSChris Mason if (slot >= btrfs_header_nritems(upper) - 1)
339044871b1bSChris Mason return 1;
339144871b1bSChris Mason
339249d0c642SFilipe Manana btrfs_assert_tree_write_locked(path->nodes[1]);
339344871b1bSChris Mason
33944b231ae4SDavid Sterba right = btrfs_read_node_slot(upper, slot + 1);
3395fb770ae4SLiu Bo if (IS_ERR(right))
33969cf14029SJosef Bacik return PTR_ERR(right);
339791ca338dSTsutomu Itoh
3398bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT);
339944871b1bSChris Mason
3400e902baacSDavid Sterba free_space = btrfs_leaf_free_space(right);
340144871b1bSChris Mason if (free_space < data_size)
340244871b1bSChris Mason goto out_unlock;
340344871b1bSChris Mason
340444871b1bSChris Mason ret = btrfs_cow_block(trans, root, right, upper,
3405bf59a5a2SJosef Bacik slot + 1, &right, BTRFS_NESTING_RIGHT_COW);
340644871b1bSChris Mason if (ret)
340744871b1bSChris Mason goto out_unlock;
340844871b1bSChris Mason
340944871b1bSChris Mason left_nritems = btrfs_header_nritems(left);
341044871b1bSChris Mason if (left_nritems == 0)
341144871b1bSChris Mason goto out_unlock;
341244871b1bSChris Mason
3413d16c702fSQu Wenruo if (check_sibling_keys(left, right)) {
3414d16c702fSQu Wenruo ret = -EUCLEAN;
34159ae5afd0SFilipe Manana btrfs_abort_transaction(trans, ret);
3416d16c702fSQu Wenruo btrfs_tree_unlock(right);
3417d16c702fSQu Wenruo free_extent_buffer(right);
3418d16c702fSQu Wenruo return ret;
3419d16c702fSQu Wenruo }
34202ef1fed2SFilipe David Borba Manana if (path->slots[0] == left_nritems && !empty) {
34212ef1fed2SFilipe David Borba Manana /* Key greater than all keys in the leaf, right neighbor has
34222ef1fed2SFilipe David Borba Manana * enough room for it and we're not emptying our leaf to delete
34232ef1fed2SFilipe David Borba Manana * it, therefore use right neighbor to insert the new item and
342452042d8eSAndrea Gelmini * no need to touch/dirty our left leaf. */
34252ef1fed2SFilipe David Borba Manana btrfs_tree_unlock(left);
34262ef1fed2SFilipe David Borba Manana free_extent_buffer(left);
34272ef1fed2SFilipe David Borba Manana path->nodes[0] = right;
34282ef1fed2SFilipe David Borba Manana path->slots[0] = 0;
34292ef1fed2SFilipe David Borba Manana path->slots[1]++;
34302ef1fed2SFilipe David Borba Manana return 0;
34312ef1fed2SFilipe David Borba Manana }
34322ef1fed2SFilipe David Borba Manana
3433ed25dab3SJosef Bacik return __push_leaf_right(trans, path, min_data_size, empty, right,
3434ed25dab3SJosef Bacik free_space, left_nritems, min_slot);
343544871b1bSChris Mason out_unlock:
343644871b1bSChris Mason btrfs_tree_unlock(right);
343744871b1bSChris Mason free_extent_buffer(right);
343844871b1bSChris Mason return 1;
343944871b1bSChris Mason }
344044871b1bSChris Mason
344144871b1bSChris Mason /*
344244871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at
344344871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise
344499d8f83cSChris Mason *
344599d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The
344699d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us do all the
344799d8f83cSChris Mason * items
344844871b1bSChris Mason */
__push_leaf_left(struct btrfs_trans_handle * trans,struct btrfs_path * path,int data_size,int empty,struct extent_buffer * left,int free_space,u32 right_nritems,u32 max_slot)3449ed25dab3SJosef Bacik static noinline int __push_leaf_left(struct btrfs_trans_handle *trans,
3450ed25dab3SJosef Bacik struct btrfs_path *path, int data_size,
345144871b1bSChris Mason int empty, struct extent_buffer *left,
345299d8f83cSChris Mason int free_space, u32 right_nritems,
345399d8f83cSChris Mason u32 max_slot)
345444871b1bSChris Mason {
34558087c193SDavid Sterba struct btrfs_fs_info *fs_info = left->fs_info;
34565f39d397SChris Mason struct btrfs_disk_key disk_key;
34575f39d397SChris Mason struct extent_buffer *right = path->nodes[0];
3458be0e5c09SChris Mason int i;
3459be0e5c09SChris Mason int push_space = 0;
3460be0e5c09SChris Mason int push_items = 0;
34617518a238SChris Mason u32 old_left_nritems;
346234a38218SChris Mason u32 nr;
3463aa5d6bedSChris Mason int ret = 0;
3464db94535dSChris Mason u32 this_item_size;
3465db94535dSChris Mason u32 old_left_item_size;
3466cfed81a0SChris Mason struct btrfs_map_token token;
3467cfed81a0SChris Mason
346834a38218SChris Mason if (empty)
346999d8f83cSChris Mason nr = min(right_nritems, max_slot);
347034a38218SChris Mason else
347199d8f83cSChris Mason nr = min(right_nritems - 1, max_slot);
347234a38218SChris Mason
347334a38218SChris Mason for (i = 0; i < nr; i++) {
347431840ae1SZheng Yan if (!empty && push_items > 0) {
347531840ae1SZheng Yan if (path->slots[0] < i)
347631840ae1SZheng Yan break;
347731840ae1SZheng Yan if (path->slots[0] == i) {
3478e902baacSDavid Sterba int space = btrfs_leaf_free_space(right);
3479e902baacSDavid Sterba
348031840ae1SZheng Yan if (space + push_space * 2 > free_space)
348131840ae1SZheng Yan break;
348231840ae1SZheng Yan }
348331840ae1SZheng Yan }
348431840ae1SZheng Yan
3485be0e5c09SChris Mason if (path->slots[0] == i)
348687b29b20SYan Zheng push_space += data_size;
3487db94535dSChris Mason
34883212fa14SJosef Bacik this_item_size = btrfs_item_size(right, i);
348974794207SJosef Bacik if (this_item_size + sizeof(struct btrfs_item) + push_space >
349074794207SJosef Bacik free_space)
3491be0e5c09SChris Mason break;
3492db94535dSChris Mason
3493be0e5c09SChris Mason push_items++;
349474794207SJosef Bacik push_space += this_item_size + sizeof(struct btrfs_item);
3495be0e5c09SChris Mason }
3496db94535dSChris Mason
3497be0e5c09SChris Mason if (push_items == 0) {
3498925baeddSChris Mason ret = 1;
3499925baeddSChris Mason goto out;
3500be0e5c09SChris Mason }
3501fae7f21cSDulshani Gunawardhana WARN_ON(!empty && push_items == btrfs_header_nritems(right));
35025f39d397SChris Mason
3503be0e5c09SChris Mason /* push data from right to left */
3504637e3b48SJosef Bacik copy_leaf_items(left, right, btrfs_header_nritems(left), 0, push_items);
35055f39d397SChris Mason
35060b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info) -
35073212fa14SJosef Bacik btrfs_item_offset(right, push_items - 1);
35085f39d397SChris Mason
3509637e3b48SJosef Bacik copy_leaf_data(left, right, leaf_data_end(left) - push_space,
3510637e3b48SJosef Bacik btrfs_item_offset(right, push_items - 1), push_space);
35115f39d397SChris Mason old_left_nritems = btrfs_header_nritems(left);
351287b29b20SYan Zheng BUG_ON(old_left_nritems <= 0);
3513eb60ceacSChris Mason
3514c82f823cSDavid Sterba btrfs_init_map_token(&token, left);
35153212fa14SJosef Bacik old_left_item_size = btrfs_item_offset(left, old_left_nritems - 1);
3516be0e5c09SChris Mason for (i = old_left_nritems; i < old_left_nritems + push_items; i++) {
35175f39d397SChris Mason u32 ioff;
3518db94535dSChris Mason
35193212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i);
35203212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i,
3521cc4c13d5SDavid Sterba ioff - (BTRFS_LEAF_DATA_SIZE(fs_info) - old_left_item_size));
3522be0e5c09SChris Mason }
35235f39d397SChris Mason btrfs_set_header_nritems(left, old_left_nritems + push_items);
3524be0e5c09SChris Mason
3525be0e5c09SChris Mason /* fixup right node */
352631b1a2bdSJulia Lawall if (push_items > right_nritems)
352731b1a2bdSJulia Lawall WARN(1, KERN_CRIT "push items %d nr %u\n", push_items,
3528d397712bSChris Mason right_nritems);
352934a38218SChris Mason
353034a38218SChris Mason if (push_items < right_nritems) {
35313212fa14SJosef Bacik push_space = btrfs_item_offset(right, push_items - 1) -
35328f881e8cSDavid Sterba leaf_data_end(right);
3533637e3b48SJosef Bacik memmove_leaf_data(right,
35340b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - push_space,
35358f881e8cSDavid Sterba leaf_data_end(right), push_space);
35365f39d397SChris Mason
3537637e3b48SJosef Bacik memmove_leaf_items(right, 0, push_items,
3538637e3b48SJosef Bacik btrfs_header_nritems(right) - push_items);
353934a38218SChris Mason }
3540c82f823cSDavid Sterba
3541c82f823cSDavid Sterba btrfs_init_map_token(&token, right);
3542eef1c494SYan right_nritems -= push_items;
3543eef1c494SYan btrfs_set_header_nritems(right, right_nritems);
35440b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info);
35455f39d397SChris Mason for (i = 0; i < right_nritems; i++) {
35463212fa14SJosef Bacik push_space = push_space - btrfs_token_item_size(&token, i);
35473212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, push_space);
3548db94535dSChris Mason }
3549eb60ceacSChris Mason
3550*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, left);
355134a38218SChris Mason if (right_nritems)
3552*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, right);
3553f0486c68SYan, Zheng else
3554190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, right);
3555098f59c2SChris Mason
35565f39d397SChris Mason btrfs_item_key(right, &disk_key, 0);
3557*d5e09e38SFilipe Manana fixup_low_keys(trans, path, &disk_key, 1);
3558be0e5c09SChris Mason
3559be0e5c09SChris Mason /* then fixup the leaf pointer in the path */
3560be0e5c09SChris Mason if (path->slots[0] < push_items) {
3561be0e5c09SChris Mason path->slots[0] += old_left_nritems;
3562925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]);
35635f39d397SChris Mason free_extent_buffer(path->nodes[0]);
35645f39d397SChris Mason path->nodes[0] = left;
3565be0e5c09SChris Mason path->slots[1] -= 1;
3566be0e5c09SChris Mason } else {
3567925baeddSChris Mason btrfs_tree_unlock(left);
35685f39d397SChris Mason free_extent_buffer(left);
3569be0e5c09SChris Mason path->slots[0] -= push_items;
3570be0e5c09SChris Mason }
3571eb60ceacSChris Mason BUG_ON(path->slots[0] < 0);
3572aa5d6bedSChris Mason return ret;
3573925baeddSChris Mason out:
3574925baeddSChris Mason btrfs_tree_unlock(left);
3575925baeddSChris Mason free_extent_buffer(left);
3576925baeddSChris Mason return ret;
3577be0e5c09SChris Mason }
3578be0e5c09SChris Mason
357974123bd7SChris Mason /*
358044871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at
358144871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise
358299d8f83cSChris Mason *
358399d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The
358499d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us push all the
358599d8f83cSChris Mason * items
358644871b1bSChris Mason */
push_leaf_left(struct btrfs_trans_handle * trans,struct btrfs_root * root,struct btrfs_path * path,int min_data_size,int data_size,int empty,u32 max_slot)358744871b1bSChris Mason static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root
358899d8f83cSChris Mason *root, struct btrfs_path *path, int min_data_size,
358999d8f83cSChris Mason int data_size, int empty, u32 max_slot)
359044871b1bSChris Mason {
359144871b1bSChris Mason struct extent_buffer *right = path->nodes[0];
359244871b1bSChris Mason struct extent_buffer *left;
359344871b1bSChris Mason int slot;
359444871b1bSChris Mason int free_space;
359544871b1bSChris Mason u32 right_nritems;
359644871b1bSChris Mason int ret = 0;
359744871b1bSChris Mason
359844871b1bSChris Mason slot = path->slots[1];
359944871b1bSChris Mason if (slot == 0)
360044871b1bSChris Mason return 1;
360144871b1bSChris Mason if (!path->nodes[1])
360244871b1bSChris Mason return 1;
360344871b1bSChris Mason
360444871b1bSChris Mason right_nritems = btrfs_header_nritems(right);
360544871b1bSChris Mason if (right_nritems == 0)
360644871b1bSChris Mason return 1;
360744871b1bSChris Mason
360849d0c642SFilipe Manana btrfs_assert_tree_write_locked(path->nodes[1]);
360944871b1bSChris Mason
36104b231ae4SDavid Sterba left = btrfs_read_node_slot(path->nodes[1], slot - 1);
3611fb770ae4SLiu Bo if (IS_ERR(left))
36129cf14029SJosef Bacik return PTR_ERR(left);
361391ca338dSTsutomu Itoh
3614bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT);
361544871b1bSChris Mason
3616e902baacSDavid Sterba free_space = btrfs_leaf_free_space(left);
361744871b1bSChris Mason if (free_space < data_size) {
361844871b1bSChris Mason ret = 1;
361944871b1bSChris Mason goto out;
362044871b1bSChris Mason }
362144871b1bSChris Mason
362244871b1bSChris Mason ret = btrfs_cow_block(trans, root, left,
36239631e4ccSJosef Bacik path->nodes[1], slot - 1, &left,
3624bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW);
362544871b1bSChris Mason if (ret) {
362644871b1bSChris Mason /* we hit -ENOSPC, but it isn't fatal here */
362779787eaaSJeff Mahoney if (ret == -ENOSPC)
362844871b1bSChris Mason ret = 1;
362944871b1bSChris Mason goto out;
363044871b1bSChris Mason }
363144871b1bSChris Mason
3632d16c702fSQu Wenruo if (check_sibling_keys(left, right)) {
3633d16c702fSQu Wenruo ret = -EUCLEAN;
36349ae5afd0SFilipe Manana btrfs_abort_transaction(trans, ret);
3635d16c702fSQu Wenruo goto out;
3636d16c702fSQu Wenruo }
3637ed25dab3SJosef Bacik return __push_leaf_left(trans, path, min_data_size, empty, left,
3638ed25dab3SJosef Bacik free_space, right_nritems, max_slot);
363944871b1bSChris Mason out:
364044871b1bSChris Mason btrfs_tree_unlock(left);
364144871b1bSChris Mason free_extent_buffer(left);
364244871b1bSChris Mason return ret;
364344871b1bSChris Mason }
364444871b1bSChris Mason
364544871b1bSChris Mason /*
364674123bd7SChris Mason * split the path's leaf in two, making sure there is at least data_size
364774123bd7SChris Mason * available for the resulting leaf level of the path.
364874123bd7SChris Mason */
copy_for_split(struct btrfs_trans_handle * trans,struct btrfs_path * path,struct extent_buffer * l,struct extent_buffer * right,int slot,int mid,int nritems)364950b5d1fcSFilipe Manana static noinline int copy_for_split(struct btrfs_trans_handle *trans,
365044871b1bSChris Mason struct btrfs_path *path,
365144871b1bSChris Mason struct extent_buffer *l,
365244871b1bSChris Mason struct extent_buffer *right,
365344871b1bSChris Mason int slot, int mid, int nritems)
3654be0e5c09SChris Mason {
365594f94ad9SDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info;
3656be0e5c09SChris Mason int data_copy_size;
3657be0e5c09SChris Mason int rt_data_off;
3658be0e5c09SChris Mason int i;
365950b5d1fcSFilipe Manana int ret;
3660d4dbff95SChris Mason struct btrfs_disk_key disk_key;
3661cfed81a0SChris Mason struct btrfs_map_token token;
3662cfed81a0SChris Mason
36635f39d397SChris Mason nritems = nritems - mid;
36645f39d397SChris Mason btrfs_set_header_nritems(right, nritems);
3665dc2e724eSJosef Bacik data_copy_size = btrfs_item_data_end(l, mid) - leaf_data_end(l);
36665f39d397SChris Mason
3667637e3b48SJosef Bacik copy_leaf_items(right, l, 0, mid, nritems);
36685f39d397SChris Mason
3669637e3b48SJosef Bacik copy_leaf_data(right, l, BTRFS_LEAF_DATA_SIZE(fs_info) - data_copy_size,
36708f881e8cSDavid Sterba leaf_data_end(l), data_copy_size);
367174123bd7SChris Mason
3672dc2e724eSJosef Bacik rt_data_off = BTRFS_LEAF_DATA_SIZE(fs_info) - btrfs_item_data_end(l, mid);
36735f39d397SChris Mason
3674c82f823cSDavid Sterba btrfs_init_map_token(&token, right);
36755f39d397SChris Mason for (i = 0; i < nritems; i++) {
3676db94535dSChris Mason u32 ioff;
3677db94535dSChris Mason
36783212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i);
36793212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff + rt_data_off);
36800783fcfcSChris Mason }
368174123bd7SChris Mason
36825f39d397SChris Mason btrfs_set_header_nritems(l, mid);
36835f39d397SChris Mason btrfs_item_key(right, &disk_key, 0);
368450b5d1fcSFilipe Manana ret = insert_ptr(trans, path, &disk_key, right->start, path->slots[1] + 1, 1);
368550b5d1fcSFilipe Manana if (ret < 0)
368650b5d1fcSFilipe Manana return ret;
36875f39d397SChris Mason
3688*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, right);
3689*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, l);
3690eb60ceacSChris Mason BUG_ON(path->slots[0] != slot);
36915f39d397SChris Mason
3692be0e5c09SChris Mason if (mid <= slot) {
3693925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]);
36945f39d397SChris Mason free_extent_buffer(path->nodes[0]);
36955f39d397SChris Mason path->nodes[0] = right;
3696be0e5c09SChris Mason path->slots[0] -= mid;
3697be0e5c09SChris Mason path->slots[1] += 1;
3698925baeddSChris Mason } else {
3699925baeddSChris Mason btrfs_tree_unlock(right);
37005f39d397SChris Mason free_extent_buffer(right);
3701925baeddSChris Mason }
37025f39d397SChris Mason
3703eb60ceacSChris Mason BUG_ON(path->slots[0] < 0);
370450b5d1fcSFilipe Manana
370550b5d1fcSFilipe Manana return 0;
370644871b1bSChris Mason }
370744871b1bSChris Mason
370844871b1bSChris Mason /*
370999d8f83cSChris Mason * double splits happen when we need to insert a big item in the middle
371099d8f83cSChris Mason * of a leaf. A double split can leave us with 3 mostly empty leaves:
371199d8f83cSChris Mason * leaf: [ slots 0 - N] [ our target ] [ N + 1 - total in leaf ]
371299d8f83cSChris Mason * A B C
371399d8f83cSChris Mason *
371499d8f83cSChris Mason * We avoid this by trying to push the items on either side of our target
371599d8f83cSChris Mason * into the adjacent leaves. If all goes well we can avoid the double split
371699d8f83cSChris Mason * completely.
371799d8f83cSChris Mason */
push_for_double_split(struct btrfs_trans_handle * trans,struct btrfs_root * root,struct btrfs_path * path,int data_size)371899d8f83cSChris Mason static noinline int push_for_double_split(struct btrfs_trans_handle *trans,
371999d8f83cSChris Mason struct btrfs_root *root,
372099d8f83cSChris Mason struct btrfs_path *path,
372199d8f83cSChris Mason int data_size)
372299d8f83cSChris Mason {
372399d8f83cSChris Mason int ret;
372499d8f83cSChris Mason int progress = 0;
372599d8f83cSChris Mason int slot;
372699d8f83cSChris Mason u32 nritems;
37275a4267caSFilipe David Borba Manana int space_needed = data_size;
372899d8f83cSChris Mason
372999d8f83cSChris Mason slot = path->slots[0];
37305a4267caSFilipe David Borba Manana if (slot < btrfs_header_nritems(path->nodes[0]))
3731e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(path->nodes[0]);
373299d8f83cSChris Mason
373399d8f83cSChris Mason /*
373499d8f83cSChris Mason * try to push all the items after our slot into the
373599d8f83cSChris Mason * right leaf
373699d8f83cSChris Mason */
37375a4267caSFilipe David Borba Manana ret = push_leaf_right(trans, root, path, 1, space_needed, 0, slot);
373899d8f83cSChris Mason if (ret < 0)
373999d8f83cSChris Mason return ret;
374099d8f83cSChris Mason
374199d8f83cSChris Mason if (ret == 0)
374299d8f83cSChris Mason progress++;
374399d8f83cSChris Mason
374499d8f83cSChris Mason nritems = btrfs_header_nritems(path->nodes[0]);
374599d8f83cSChris Mason /*
374699d8f83cSChris Mason * our goal is to get our slot at the start or end of a leaf. If
374799d8f83cSChris Mason * we've done so we're done
374899d8f83cSChris Mason */
374999d8f83cSChris Mason if (path->slots[0] == 0 || path->slots[0] == nritems)
375099d8f83cSChris Mason return 0;
375199d8f83cSChris Mason
3752e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= data_size)
375399d8f83cSChris Mason return 0;
375499d8f83cSChris Mason
375599d8f83cSChris Mason /* try to push all the items before our slot into the next leaf */
375699d8f83cSChris Mason slot = path->slots[0];
3757263d3995SFilipe Manana space_needed = data_size;
3758263d3995SFilipe Manana if (slot > 0)
3759e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(path->nodes[0]);
37605a4267caSFilipe David Borba Manana ret = push_leaf_left(trans, root, path, 1, space_needed, 0, slot);
376199d8f83cSChris Mason if (ret < 0)
376299d8f83cSChris Mason return ret;
376399d8f83cSChris Mason
376499d8f83cSChris Mason if (ret == 0)
376599d8f83cSChris Mason progress++;
376699d8f83cSChris Mason
376799d8f83cSChris Mason if (progress)
376899d8f83cSChris Mason return 0;
376999d8f83cSChris Mason return 1;
377099d8f83cSChris Mason }
377199d8f83cSChris Mason
377299d8f83cSChris Mason /*
377344871b1bSChris Mason * split the path's leaf in two, making sure there is at least data_size
377444871b1bSChris Mason * available for the resulting leaf level of the path.
377544871b1bSChris Mason *
377644871b1bSChris Mason * returns 0 if all went well and < 0 on failure.
377744871b1bSChris Mason */
split_leaf(struct btrfs_trans_handle * trans,struct btrfs_root * root,const struct btrfs_key * ins_key,struct btrfs_path * path,int data_size,int extend)377844871b1bSChris Mason static noinline int split_leaf(struct btrfs_trans_handle *trans,
377944871b1bSChris Mason struct btrfs_root *root,
3780310712b2SOmar Sandoval const struct btrfs_key *ins_key,
378144871b1bSChris Mason struct btrfs_path *path, int data_size,
378244871b1bSChris Mason int extend)
378344871b1bSChris Mason {
37845d4f98a2SYan Zheng struct btrfs_disk_key disk_key;
378544871b1bSChris Mason struct extent_buffer *l;
378644871b1bSChris Mason u32 nritems;
378744871b1bSChris Mason int mid;
378844871b1bSChris Mason int slot;
378944871b1bSChris Mason struct extent_buffer *right;
3790b7a0365eSDaniel Dressler struct btrfs_fs_info *fs_info = root->fs_info;
379144871b1bSChris Mason int ret = 0;
379244871b1bSChris Mason int wret;
37935d4f98a2SYan Zheng int split;
379444871b1bSChris Mason int num_doubles = 0;
379599d8f83cSChris Mason int tried_avoid_double = 0;
379644871b1bSChris Mason
3797a5719521SYan, Zheng l = path->nodes[0];
3798a5719521SYan, Zheng slot = path->slots[0];
37993212fa14SJosef Bacik if (extend && data_size + btrfs_item_size(l, slot) +
38000b246afaSJeff Mahoney sizeof(struct btrfs_item) > BTRFS_LEAF_DATA_SIZE(fs_info))
3801a5719521SYan, Zheng return -EOVERFLOW;
3802a5719521SYan, Zheng
380344871b1bSChris Mason /* first try to make some room by pushing left and right */
380433157e05SLiu Bo if (data_size && path->nodes[1]) {
38055a4267caSFilipe David Borba Manana int space_needed = data_size;
38065a4267caSFilipe David Borba Manana
38075a4267caSFilipe David Borba Manana if (slot < btrfs_header_nritems(l))
3808e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(l);
38095a4267caSFilipe David Borba Manana
38105a4267caSFilipe David Borba Manana wret = push_leaf_right(trans, root, path, space_needed,
38115a4267caSFilipe David Borba Manana space_needed, 0, 0);
381244871b1bSChris Mason if (wret < 0)
381344871b1bSChris Mason return wret;
381444871b1bSChris Mason if (wret) {
3815263d3995SFilipe Manana space_needed = data_size;
3816263d3995SFilipe Manana if (slot > 0)
3817e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(l);
38185a4267caSFilipe David Borba Manana wret = push_leaf_left(trans, root, path, space_needed,
38195a4267caSFilipe David Borba Manana space_needed, 0, (u32)-1);
382044871b1bSChris Mason if (wret < 0)
382144871b1bSChris Mason return wret;
382244871b1bSChris Mason }
382344871b1bSChris Mason l = path->nodes[0];
382444871b1bSChris Mason
382544871b1bSChris Mason /* did the pushes work? */
3826e902baacSDavid Sterba if (btrfs_leaf_free_space(l) >= data_size)
382744871b1bSChris Mason return 0;
382844871b1bSChris Mason }
382944871b1bSChris Mason
383044871b1bSChris Mason if (!path->nodes[1]) {
3831fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, 1);
383244871b1bSChris Mason if (ret)
383344871b1bSChris Mason return ret;
383444871b1bSChris Mason }
383544871b1bSChris Mason again:
38365d4f98a2SYan Zheng split = 1;
383744871b1bSChris Mason l = path->nodes[0];
383844871b1bSChris Mason slot = path->slots[0];
383944871b1bSChris Mason nritems = btrfs_header_nritems(l);
384044871b1bSChris Mason mid = (nritems + 1) / 2;
384144871b1bSChris Mason
38425d4f98a2SYan Zheng if (mid <= slot) {
38435d4f98a2SYan Zheng if (nritems == 1 ||
38445d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + data_size >
38450b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info)) {
38465d4f98a2SYan Zheng if (slot >= nritems) {
38475d4f98a2SYan Zheng split = 0;
38485d4f98a2SYan Zheng } else {
38495d4f98a2SYan Zheng mid = slot;
38505d4f98a2SYan Zheng if (mid != nritems &&
38515d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) +
38520b246afaSJeff Mahoney data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) {
385399d8f83cSChris Mason if (data_size && !tried_avoid_double)
385499d8f83cSChris Mason goto push_for_double;
38555d4f98a2SYan Zheng split = 2;
38565d4f98a2SYan Zheng }
38575d4f98a2SYan Zheng }
38585d4f98a2SYan Zheng }
38595d4f98a2SYan Zheng } else {
38605d4f98a2SYan Zheng if (leaf_space_used(l, 0, mid) + data_size >
38610b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info)) {
38625d4f98a2SYan Zheng if (!extend && data_size && slot == 0) {
38635d4f98a2SYan Zheng split = 0;
38645d4f98a2SYan Zheng } else if ((extend || !data_size) && slot == 0) {
38655d4f98a2SYan Zheng mid = 1;
38665d4f98a2SYan Zheng } else {
38675d4f98a2SYan Zheng mid = slot;
38685d4f98a2SYan Zheng if (mid != nritems &&
38695d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) +
38700b246afaSJeff Mahoney data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) {
387199d8f83cSChris Mason if (data_size && !tried_avoid_double)
387299d8f83cSChris Mason goto push_for_double;
38735d4f98a2SYan Zheng split = 2;
38745d4f98a2SYan Zheng }
38755d4f98a2SYan Zheng }
38765d4f98a2SYan Zheng }
38775d4f98a2SYan Zheng }
38785d4f98a2SYan Zheng
38795d4f98a2SYan Zheng if (split == 0)
38805d4f98a2SYan Zheng btrfs_cpu_key_to_disk(&disk_key, ins_key);
38815d4f98a2SYan Zheng else
38825d4f98a2SYan Zheng btrfs_item_key(l, &disk_key, mid);
38835d4f98a2SYan Zheng
3884ca9d473aSJosef Bacik /*
3885ca9d473aSJosef Bacik * We have to about BTRFS_NESTING_NEW_ROOT here if we've done a double
3886ca9d473aSJosef Bacik * split, because we're only allowed to have MAX_LOCKDEP_SUBCLASSES
3887ca9d473aSJosef Bacik * subclasses, which is 8 at the time of this patch, and we've maxed it
3888ca9d473aSJosef Bacik * out. In the future we could add a
3889ca9d473aSJosef Bacik * BTRFS_NESTING_SPLIT_THE_SPLITTENING if we need to, but for now just
3890ca9d473aSJosef Bacik * use BTRFS_NESTING_NEW_ROOT.
3891ca9d473aSJosef Bacik */
389279bd3712SFilipe Manana right = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid,
389379bd3712SFilipe Manana &disk_key, 0, l->start, 0,
389479bd3712SFilipe Manana num_doubles ? BTRFS_NESTING_NEW_ROOT :
3895ca9d473aSJosef Bacik BTRFS_NESTING_SPLIT);
3896f0486c68SYan, Zheng if (IS_ERR(right))
389744871b1bSChris Mason return PTR_ERR(right);
3898f0486c68SYan, Zheng
38990b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize);
390044871b1bSChris Mason
39015d4f98a2SYan Zheng if (split == 0) {
390244871b1bSChris Mason if (mid <= slot) {
390344871b1bSChris Mason btrfs_set_header_nritems(right, 0);
390450b5d1fcSFilipe Manana ret = insert_ptr(trans, path, &disk_key,
39052ff7e61eSJeff Mahoney right->start, path->slots[1] + 1, 1);
390650b5d1fcSFilipe Manana if (ret < 0) {
390750b5d1fcSFilipe Manana btrfs_tree_unlock(right);
390850b5d1fcSFilipe Manana free_extent_buffer(right);
390950b5d1fcSFilipe Manana return ret;
391050b5d1fcSFilipe Manana }
391144871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]);
391244871b1bSChris Mason free_extent_buffer(path->nodes[0]);
391344871b1bSChris Mason path->nodes[0] = right;
391444871b1bSChris Mason path->slots[0] = 0;
391544871b1bSChris Mason path->slots[1] += 1;
391644871b1bSChris Mason } else {
391744871b1bSChris Mason btrfs_set_header_nritems(right, 0);
391850b5d1fcSFilipe Manana ret = insert_ptr(trans, path, &disk_key,
39192ff7e61eSJeff Mahoney right->start, path->slots[1], 1);
392050b5d1fcSFilipe Manana if (ret < 0) {
392150b5d1fcSFilipe Manana btrfs_tree_unlock(right);
392250b5d1fcSFilipe Manana free_extent_buffer(right);
392350b5d1fcSFilipe Manana return ret;
392450b5d1fcSFilipe Manana }
392544871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]);
392644871b1bSChris Mason free_extent_buffer(path->nodes[0]);
392744871b1bSChris Mason path->nodes[0] = right;
392844871b1bSChris Mason path->slots[0] = 0;
3929143bede5SJeff Mahoney if (path->slots[1] == 0)
3930*d5e09e38SFilipe Manana fixup_low_keys(trans, path, &disk_key, 1);
39315d4f98a2SYan Zheng }
3932196e0249SLiu Bo /*
3933196e0249SLiu Bo * We create a new leaf 'right' for the required ins_len and
3934196e0249SLiu Bo * we'll do btrfs_mark_buffer_dirty() on this leaf after copying
3935196e0249SLiu Bo * the content of ins_len to 'right'.
3936196e0249SLiu Bo */
393744871b1bSChris Mason return ret;
393844871b1bSChris Mason }
393944871b1bSChris Mason
394050b5d1fcSFilipe Manana ret = copy_for_split(trans, path, l, right, slot, mid, nritems);
394150b5d1fcSFilipe Manana if (ret < 0) {
394250b5d1fcSFilipe Manana btrfs_tree_unlock(right);
394350b5d1fcSFilipe Manana free_extent_buffer(right);
394450b5d1fcSFilipe Manana return ret;
394550b5d1fcSFilipe Manana }
394644871b1bSChris Mason
39475d4f98a2SYan Zheng if (split == 2) {
3948cc0c5538SChris Mason BUG_ON(num_doubles != 0);
3949cc0c5538SChris Mason num_doubles++;
3950cc0c5538SChris Mason goto again;
39513326d1b0SChris Mason }
395244871b1bSChris Mason
3953143bede5SJeff Mahoney return 0;
395499d8f83cSChris Mason
395599d8f83cSChris Mason push_for_double:
395699d8f83cSChris Mason push_for_double_split(trans, root, path, data_size);
395799d8f83cSChris Mason tried_avoid_double = 1;
3958e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= data_size)
395999d8f83cSChris Mason return 0;
396099d8f83cSChris Mason goto again;
3961be0e5c09SChris Mason }
3962be0e5c09SChris Mason
setup_leaf_for_split(struct btrfs_trans_handle * trans,struct btrfs_root * root,struct btrfs_path * path,int ins_len)3963ad48fd75SYan, Zheng static noinline int setup_leaf_for_split(struct btrfs_trans_handle *trans,
3964ad48fd75SYan, Zheng struct btrfs_root *root,
3965ad48fd75SYan, Zheng struct btrfs_path *path, int ins_len)
3966ad48fd75SYan, Zheng {
3967ad48fd75SYan, Zheng struct btrfs_key key;
3968ad48fd75SYan, Zheng struct extent_buffer *leaf;
3969ad48fd75SYan, Zheng struct btrfs_file_extent_item *fi;
3970ad48fd75SYan, Zheng u64 extent_len = 0;
3971ad48fd75SYan, Zheng u32 item_size;
3972ad48fd75SYan, Zheng int ret;
3973ad48fd75SYan, Zheng
3974ad48fd75SYan, Zheng leaf = path->nodes[0];
3975ad48fd75SYan, Zheng btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
3976ad48fd75SYan, Zheng
3977ad48fd75SYan, Zheng BUG_ON(key.type != BTRFS_EXTENT_DATA_KEY &&
3978ad48fd75SYan, Zheng key.type != BTRFS_EXTENT_CSUM_KEY);
3979ad48fd75SYan, Zheng
3980e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) >= ins_len)
3981ad48fd75SYan, Zheng return 0;
3982ad48fd75SYan, Zheng
39833212fa14SJosef Bacik item_size = btrfs_item_size(leaf, path->slots[0]);
3984ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) {
3985ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0],
3986ad48fd75SYan, Zheng struct btrfs_file_extent_item);
3987ad48fd75SYan, Zheng extent_len = btrfs_file_extent_num_bytes(leaf, fi);
3988ad48fd75SYan, Zheng }
3989b3b4aa74SDavid Sterba btrfs_release_path(path);
3990ad48fd75SYan, Zheng
3991ad48fd75SYan, Zheng path->keep_locks = 1;
3992ad48fd75SYan, Zheng path->search_for_split = 1;
3993ad48fd75SYan, Zheng ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
3994ad48fd75SYan, Zheng path->search_for_split = 0;
3995a8df6fe6SFilipe Manana if (ret > 0)
3996a8df6fe6SFilipe Manana ret = -EAGAIN;
3997ad48fd75SYan, Zheng if (ret < 0)
3998ad48fd75SYan, Zheng goto err;
3999ad48fd75SYan, Zheng
4000ad48fd75SYan, Zheng ret = -EAGAIN;
4001ad48fd75SYan, Zheng leaf = path->nodes[0];
4002a8df6fe6SFilipe Manana /* if our item isn't there, return now */
40033212fa14SJosef Bacik if (item_size != btrfs_item_size(leaf, path->slots[0]))
4004ad48fd75SYan, Zheng goto err;
4005ad48fd75SYan, Zheng
4006109f6aefSChris Mason /* the leaf has changed, it now has room. return now */
4007e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= ins_len)
4008109f6aefSChris Mason goto err;
4009109f6aefSChris Mason
4010ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) {
4011ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0],
4012ad48fd75SYan, Zheng struct btrfs_file_extent_item);
4013ad48fd75SYan, Zheng if (extent_len != btrfs_file_extent_num_bytes(leaf, fi))
4014ad48fd75SYan, Zheng goto err;
4015ad48fd75SYan, Zheng }
4016ad48fd75SYan, Zheng
4017ad48fd75SYan, Zheng ret = split_leaf(trans, root, &key, path, ins_len, 1);
4018f0486c68SYan, Zheng if (ret)
4019f0486c68SYan, Zheng goto err;
4020ad48fd75SYan, Zheng
4021ad48fd75SYan, Zheng path->keep_locks = 0;
4022ad48fd75SYan, Zheng btrfs_unlock_up_safe(path, 1);
4023ad48fd75SYan, Zheng return 0;
4024ad48fd75SYan, Zheng err:
4025ad48fd75SYan, Zheng path->keep_locks = 0;
4026ad48fd75SYan, Zheng return ret;
4027ad48fd75SYan, Zheng }
4028ad48fd75SYan, Zheng
split_item(struct btrfs_trans_handle * trans,struct btrfs_path * path,const struct btrfs_key * new_key,unsigned long split_offset)4029*d5e09e38SFilipe Manana static noinline int split_item(struct btrfs_trans_handle *trans,
4030*d5e09e38SFilipe Manana struct btrfs_path *path,
4031310712b2SOmar Sandoval const struct btrfs_key *new_key,
4032459931ecSChris Mason unsigned long split_offset)
4033459931ecSChris Mason {
4034459931ecSChris Mason struct extent_buffer *leaf;
4035c91666b1SJosef Bacik int orig_slot, slot;
4036ad48fd75SYan, Zheng char *buf;
4037459931ecSChris Mason u32 nritems;
4038ad48fd75SYan, Zheng u32 item_size;
4039459931ecSChris Mason u32 orig_offset;
4040459931ecSChris Mason struct btrfs_disk_key disk_key;
4041459931ecSChris Mason
4042459931ecSChris Mason leaf = path->nodes[0];
40437569141eSFilipe Manana /*
40447569141eSFilipe Manana * Shouldn't happen because the caller must have previously called
40457569141eSFilipe Manana * setup_leaf_for_split() to make room for the new item in the leaf.
40467569141eSFilipe Manana */
40477569141eSFilipe Manana if (WARN_ON(btrfs_leaf_free_space(leaf) < sizeof(struct btrfs_item)))
40487569141eSFilipe Manana return -ENOSPC;
4049b9473439SChris Mason
4050c91666b1SJosef Bacik orig_slot = path->slots[0];
40513212fa14SJosef Bacik orig_offset = btrfs_item_offset(leaf, path->slots[0]);
40523212fa14SJosef Bacik item_size = btrfs_item_size(leaf, path->slots[0]);
4053459931ecSChris Mason
4054459931ecSChris Mason buf = kmalloc(item_size, GFP_NOFS);
4055ad48fd75SYan, Zheng if (!buf)
4056ad48fd75SYan, Zheng return -ENOMEM;
4057ad48fd75SYan, Zheng
4058459931ecSChris Mason read_extent_buffer(leaf, buf, btrfs_item_ptr_offset(leaf,
4059459931ecSChris Mason path->slots[0]), item_size);
4060ad48fd75SYan, Zheng
4061459931ecSChris Mason slot = path->slots[0] + 1;
4062459931ecSChris Mason nritems = btrfs_header_nritems(leaf);
4063459931ecSChris Mason if (slot != nritems) {
4064459931ecSChris Mason /* shift the items */
4065637e3b48SJosef Bacik memmove_leaf_items(leaf, slot + 1, slot, nritems - slot);
4066459931ecSChris Mason }
4067459931ecSChris Mason
4068459931ecSChris Mason btrfs_cpu_key_to_disk(&disk_key, new_key);
4069459931ecSChris Mason btrfs_set_item_key(leaf, &disk_key, slot);
4070459931ecSChris Mason
40713212fa14SJosef Bacik btrfs_set_item_offset(leaf, slot, orig_offset);
40723212fa14SJosef Bacik btrfs_set_item_size(leaf, slot, item_size - split_offset);
4073459931ecSChris Mason
40743212fa14SJosef Bacik btrfs_set_item_offset(leaf, orig_slot,
4075459931ecSChris Mason orig_offset + item_size - split_offset);
40763212fa14SJosef Bacik btrfs_set_item_size(leaf, orig_slot, split_offset);
4077459931ecSChris Mason
4078459931ecSChris Mason btrfs_set_header_nritems(leaf, nritems + 1);
4079459931ecSChris Mason
4080459931ecSChris Mason /* write the data for the start of the original item */
4081459931ecSChris Mason write_extent_buffer(leaf, buf,
4082459931ecSChris Mason btrfs_item_ptr_offset(leaf, path->slots[0]),
4083459931ecSChris Mason split_offset);
4084459931ecSChris Mason
4085459931ecSChris Mason /* write the data for the new item */
4086459931ecSChris Mason write_extent_buffer(leaf, buf + split_offset,
4087459931ecSChris Mason btrfs_item_ptr_offset(leaf, slot),
4088459931ecSChris Mason item_size - split_offset);
4089*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, leaf);
4090459931ecSChris Mason
4091e902baacSDavid Sterba BUG_ON(btrfs_leaf_free_space(leaf) < 0);
4092459931ecSChris Mason kfree(buf);
4093ad48fd75SYan, Zheng return 0;
4094ad48fd75SYan, Zheng }
4095ad48fd75SYan, Zheng
4096ad48fd75SYan, Zheng /*
4097ad48fd75SYan, Zheng * This function splits a single item into two items,
4098ad48fd75SYan, Zheng * giving 'new_key' to the new item and splitting the
4099ad48fd75SYan, Zheng * old one at split_offset (from the start of the item).
4100ad48fd75SYan, Zheng *
4101ad48fd75SYan, Zheng * The path may be released by this operation. After
4102ad48fd75SYan, Zheng * the split, the path is pointing to the old item. The
4103ad48fd75SYan, Zheng * new item is going to be in the same node as the old one.
4104ad48fd75SYan, Zheng *
4105ad48fd75SYan, Zheng * Note, the item being split must be smaller enough to live alone on
4106ad48fd75SYan, Zheng * a tree block with room for one extra struct btrfs_item
4107ad48fd75SYan, Zheng *
4108ad48fd75SYan, Zheng * This allows us to split the item in place, keeping a lock on the
4109ad48fd75SYan, Zheng * leaf the entire time.
4110ad48fd75SYan, Zheng */
btrfs_split_item(struct btrfs_trans_handle * trans,struct btrfs_root * root,struct btrfs_path * path,const struct btrfs_key * new_key,unsigned long split_offset)4111ad48fd75SYan, Zheng int btrfs_split_item(struct btrfs_trans_handle *trans,
4112ad48fd75SYan, Zheng struct btrfs_root *root,
4113ad48fd75SYan, Zheng struct btrfs_path *path,
4114310712b2SOmar Sandoval const struct btrfs_key *new_key,
4115ad48fd75SYan, Zheng unsigned long split_offset)
4116ad48fd75SYan, Zheng {
4117ad48fd75SYan, Zheng int ret;
4118ad48fd75SYan, Zheng ret = setup_leaf_for_split(trans, root, path,
4119ad48fd75SYan, Zheng sizeof(struct btrfs_item));
4120ad48fd75SYan, Zheng if (ret)
4121459931ecSChris Mason return ret;
4122ad48fd75SYan, Zheng
4123*d5e09e38SFilipe Manana ret = split_item(trans, path, new_key, split_offset);
4124ad48fd75SYan, Zheng return ret;
4125ad48fd75SYan, Zheng }
4126ad48fd75SYan, Zheng
4127ad48fd75SYan, Zheng /*
4128d352ac68SChris Mason * make the item pointed to by the path smaller. new_size indicates
4129d352ac68SChris Mason * how small to make it, and from_end tells us if we just chop bytes
4130d352ac68SChris Mason * off the end of the item or if we shift the item to chop bytes off
4131d352ac68SChris Mason * the front.
4132d352ac68SChris Mason */
btrfs_truncate_item(struct btrfs_trans_handle * trans,struct btrfs_path * path,u32 new_size,int from_end)4133*d5e09e38SFilipe Manana void btrfs_truncate_item(struct btrfs_trans_handle *trans,
4134*d5e09e38SFilipe Manana struct btrfs_path *path, u32 new_size, int from_end)
4135b18c6685SChris Mason {
4136b18c6685SChris Mason int slot;
41375f39d397SChris Mason struct extent_buffer *leaf;
4138b18c6685SChris Mason u32 nritems;
4139b18c6685SChris Mason unsigned int data_end;
4140b18c6685SChris Mason unsigned int old_data_start;
4141b18c6685SChris Mason unsigned int old_size;
4142b18c6685SChris Mason unsigned int size_diff;
4143b18c6685SChris Mason int i;
4144cfed81a0SChris Mason struct btrfs_map_token token;
4145cfed81a0SChris Mason
41465f39d397SChris Mason leaf = path->nodes[0];
4147179e29e4SChris Mason slot = path->slots[0];
4148179e29e4SChris Mason
41493212fa14SJosef Bacik old_size = btrfs_item_size(leaf, slot);
4150179e29e4SChris Mason if (old_size == new_size)
4151143bede5SJeff Mahoney return;
4152b18c6685SChris Mason
41535f39d397SChris Mason nritems = btrfs_header_nritems(leaf);
41548f881e8cSDavid Sterba data_end = leaf_data_end(leaf);
4155b18c6685SChris Mason
41563212fa14SJosef Bacik old_data_start = btrfs_item_offset(leaf, slot);
4157179e29e4SChris Mason
4158b18c6685SChris Mason size_diff = old_size - new_size;
4159b18c6685SChris Mason
4160b18c6685SChris Mason BUG_ON(slot < 0);
4161b18c6685SChris Mason BUG_ON(slot >= nritems);
4162b18c6685SChris Mason
4163b18c6685SChris Mason /*
4164b18c6685SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size
4165b18c6685SChris Mason */
4166b18c6685SChris Mason /* first correct the data pointers */
4167c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf);
4168b18c6685SChris Mason for (i = slot; i < nritems; i++) {
41695f39d397SChris Mason u32 ioff;
4170db94535dSChris Mason
41713212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i);
41723212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff + size_diff);
4173b18c6685SChris Mason }
4174db94535dSChris Mason
4175b18c6685SChris Mason /* shift the data */
4176179e29e4SChris Mason if (from_end) {
4177637e3b48SJosef Bacik memmove_leaf_data(leaf, data_end + size_diff, data_end,
4178637e3b48SJosef Bacik old_data_start + new_size - data_end);
4179179e29e4SChris Mason } else {
4180179e29e4SChris Mason struct btrfs_disk_key disk_key;
4181179e29e4SChris Mason u64 offset;
4182179e29e4SChris Mason
4183179e29e4SChris Mason btrfs_item_key(leaf, &disk_key, slot);
4184179e29e4SChris Mason
4185179e29e4SChris Mason if (btrfs_disk_key_type(&disk_key) == BTRFS_EXTENT_DATA_KEY) {
4186179e29e4SChris Mason unsigned long ptr;
4187179e29e4SChris Mason struct btrfs_file_extent_item *fi;
4188179e29e4SChris Mason
4189179e29e4SChris Mason fi = btrfs_item_ptr(leaf, slot,
4190179e29e4SChris Mason struct btrfs_file_extent_item);
4191179e29e4SChris Mason fi = (struct btrfs_file_extent_item *)(
4192179e29e4SChris Mason (unsigned long)fi - size_diff);
4193179e29e4SChris Mason
4194179e29e4SChris Mason if (btrfs_file_extent_type(leaf, fi) ==
4195179e29e4SChris Mason BTRFS_FILE_EXTENT_INLINE) {
4196179e29e4SChris Mason ptr = btrfs_item_ptr_offset(leaf, slot);
4197179e29e4SChris Mason memmove_extent_buffer(leaf, ptr,
4198179e29e4SChris Mason (unsigned long)fi,
41997ec20afbSDavid Sterba BTRFS_FILE_EXTENT_INLINE_DATA_START);
4200179e29e4SChris Mason }
4201179e29e4SChris Mason }
4202179e29e4SChris Mason
4203637e3b48SJosef Bacik memmove_leaf_data(leaf, data_end + size_diff, data_end,
4204637e3b48SJosef Bacik old_data_start - data_end);
4205179e29e4SChris Mason
4206179e29e4SChris Mason offset = btrfs_disk_key_offset(&disk_key);
4207179e29e4SChris Mason btrfs_set_disk_key_offset(&disk_key, offset + size_diff);
4208179e29e4SChris Mason btrfs_set_item_key(leaf, &disk_key, slot);
4209179e29e4SChris Mason if (slot == 0)
4210*d5e09e38SFilipe Manana fixup_low_keys(trans, path, &disk_key, 1);
4211179e29e4SChris Mason }
42125f39d397SChris Mason
42133212fa14SJosef Bacik btrfs_set_item_size(leaf, slot, new_size);
4214*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, leaf);
4215b18c6685SChris Mason
4216e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) {
4217a4f78750SDavid Sterba btrfs_print_leaf(leaf);
4218b18c6685SChris Mason BUG();
42195f39d397SChris Mason }
4220b18c6685SChris Mason }
4221b18c6685SChris Mason
4222d352ac68SChris Mason /*
42238f69dbd2SStefan Behrens * make the item pointed to by the path bigger, data_size is the added size.
4224d352ac68SChris Mason */
btrfs_extend_item(struct btrfs_trans_handle * trans,struct btrfs_path * path,u32 data_size)4225*d5e09e38SFilipe Manana void btrfs_extend_item(struct btrfs_trans_handle *trans,
4226*d5e09e38SFilipe Manana struct btrfs_path *path, u32 data_size)
42276567e837SChris Mason {
42286567e837SChris Mason int slot;
42295f39d397SChris Mason struct extent_buffer *leaf;
42306567e837SChris Mason u32 nritems;
42316567e837SChris Mason unsigned int data_end;
42326567e837SChris Mason unsigned int old_data;
42336567e837SChris Mason unsigned int old_size;
42346567e837SChris Mason int i;
4235cfed81a0SChris Mason struct btrfs_map_token token;
4236cfed81a0SChris Mason
42375f39d397SChris Mason leaf = path->nodes[0];
42386567e837SChris Mason
42395f39d397SChris Mason nritems = btrfs_header_nritems(leaf);
42408f881e8cSDavid Sterba data_end = leaf_data_end(leaf);
42416567e837SChris Mason
4242e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < data_size) {
4243a4f78750SDavid Sterba btrfs_print_leaf(leaf);
42446567e837SChris Mason BUG();
42455f39d397SChris Mason }
42466567e837SChris Mason slot = path->slots[0];
4247dc2e724eSJosef Bacik old_data = btrfs_item_data_end(leaf, slot);
42486567e837SChris Mason
42496567e837SChris Mason BUG_ON(slot < 0);
42503326d1b0SChris Mason if (slot >= nritems) {
4251a4f78750SDavid Sterba btrfs_print_leaf(leaf);
4252c71dd880SDavid Sterba btrfs_crit(leaf->fs_info, "slot %d too large, nritems %d",
4253d397712bSChris Mason slot, nritems);
4254290342f6SArnd Bergmann BUG();
42553326d1b0SChris Mason }
42566567e837SChris Mason
42576567e837SChris Mason /*
42586567e837SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size
42596567e837SChris Mason */
42606567e837SChris Mason /* first correct the data pointers */
4261c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf);
42626567e837SChris Mason for (i = slot; i < nritems; i++) {
42635f39d397SChris Mason u32 ioff;
4264db94535dSChris Mason
42653212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i);
42663212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff - data_size);
42676567e837SChris Mason }
42685f39d397SChris Mason
42696567e837SChris Mason /* shift the data */
4270637e3b48SJosef Bacik memmove_leaf_data(leaf, data_end - data_size, data_end,
4271637e3b48SJosef Bacik old_data - data_end);
42725f39d397SChris Mason
42736567e837SChris Mason data_end = old_data;
42743212fa14SJosef Bacik old_size = btrfs_item_size(leaf, slot);
42753212fa14SJosef Bacik btrfs_set_item_size(leaf, slot, old_size + data_size);
4276*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, leaf);
42776567e837SChris Mason
4278e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) {
4279a4f78750SDavid Sterba btrfs_print_leaf(leaf);
42806567e837SChris Mason BUG();
42815f39d397SChris Mason }
42826567e837SChris Mason }
42836567e837SChris Mason
428443dd529aSDavid Sterba /*
428543dd529aSDavid Sterba * Make space in the node before inserting one or more items.
4286da9ffb24SNikolay Borisov *
4287*d5e09e38SFilipe Manana * @trans: transaction handle
4288da9ffb24SNikolay Borisov * @root: root we are inserting items to
4289da9ffb24SNikolay Borisov * @path: points to the leaf/slot where we are going to insert new items
4290b7ef5f3aSFilipe Manana * @batch: information about the batch of items to insert
429143dd529aSDavid Sterba *
429243dd529aSDavid Sterba * Main purpose is to save stack depth by doing the bulk of the work in a
429343dd529aSDavid Sterba * function that doesn't call btrfs_search_slot
429474123bd7SChris Mason */
setup_items_for_insert(struct btrfs_trans_handle * trans,struct btrfs_root * root,struct btrfs_path * path,const struct btrfs_item_batch * batch)4295*d5e09e38SFilipe Manana static void setup_items_for_insert(struct btrfs_trans_handle *trans,
4296*d5e09e38SFilipe Manana struct btrfs_root *root, struct btrfs_path *path,
4297b7ef5f3aSFilipe Manana const struct btrfs_item_batch *batch)
4298be0e5c09SChris Mason {
42990b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info;
43009c58309dSChris Mason int i;
43017518a238SChris Mason u32 nritems;
4302be0e5c09SChris Mason unsigned int data_end;
4303e2fa7227SChris Mason struct btrfs_disk_key disk_key;
430444871b1bSChris Mason struct extent_buffer *leaf;
430544871b1bSChris Mason int slot;
4306cfed81a0SChris Mason struct btrfs_map_token token;
4307fc0d82e1SNikolay Borisov u32 total_size;
4308fc0d82e1SNikolay Borisov
4309b7ef5f3aSFilipe Manana /*
4310b7ef5f3aSFilipe Manana * Before anything else, update keys in the parent and other ancestors
4311b7ef5f3aSFilipe Manana * if needed, then release the write locks on them, so that other tasks
4312b7ef5f3aSFilipe Manana * can use them while we modify the leaf.
4313b7ef5f3aSFilipe Manana */
431424cdc847SFilipe Manana if (path->slots[0] == 0) {
4315b7ef5f3aSFilipe Manana btrfs_cpu_key_to_disk(&disk_key, &batch->keys[0]);
4316*d5e09e38SFilipe Manana fixup_low_keys(trans, path, &disk_key, 1);
431724cdc847SFilipe Manana }
431824cdc847SFilipe Manana btrfs_unlock_up_safe(path, 1);
431924cdc847SFilipe Manana
43205f39d397SChris Mason leaf = path->nodes[0];
432144871b1bSChris Mason slot = path->slots[0];
432274123bd7SChris Mason
43235f39d397SChris Mason nritems = btrfs_header_nritems(leaf);
43248f881e8cSDavid Sterba data_end = leaf_data_end(leaf);
4325b7ef5f3aSFilipe Manana total_size = batch->total_data_size + (batch->nr * sizeof(struct btrfs_item));
4326eb60ceacSChris Mason
4327e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < total_size) {
4328a4f78750SDavid Sterba btrfs_print_leaf(leaf);
43290b246afaSJeff Mahoney btrfs_crit(fs_info, "not enough freespace need %u have %d",
4330e902baacSDavid Sterba total_size, btrfs_leaf_free_space(leaf));
4331be0e5c09SChris Mason BUG();
4332d4dbff95SChris Mason }
43335f39d397SChris Mason
4334c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf);
4335be0e5c09SChris Mason if (slot != nritems) {
4336dc2e724eSJosef Bacik unsigned int old_data = btrfs_item_data_end(leaf, slot);
4337be0e5c09SChris Mason
43385f39d397SChris Mason if (old_data < data_end) {
4339a4f78750SDavid Sterba btrfs_print_leaf(leaf);
43407269ddd2SNikolay Borisov btrfs_crit(fs_info,
43417269ddd2SNikolay Borisov "item at slot %d with data offset %u beyond data end of leaf %u",
43425f39d397SChris Mason slot, old_data, data_end);
4343290342f6SArnd Bergmann BUG();
43445f39d397SChris Mason }
4345be0e5c09SChris Mason /*
4346be0e5c09SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size
4347be0e5c09SChris Mason */
4348be0e5c09SChris Mason /* first correct the data pointers */
43490783fcfcSChris Mason for (i = slot; i < nritems; i++) {
43505f39d397SChris Mason u32 ioff;
4351db94535dSChris Mason
43523212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i);
43533212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i,
4354b7ef5f3aSFilipe Manana ioff - batch->total_data_size);
43550783fcfcSChris Mason }
4356be0e5c09SChris Mason /* shift the items */
4357637e3b48SJosef Bacik memmove_leaf_items(leaf, slot + batch->nr, slot, nritems - slot);
4358be0e5c09SChris Mason
4359be0e5c09SChris Mason /* shift the data */
4360637e3b48SJosef Bacik memmove_leaf_data(leaf, data_end - batch->total_data_size,
4361637e3b48SJosef Bacik data_end, old_data - data_end);
4362be0e5c09SChris Mason data_end = old_data;
4363be0e5c09SChris Mason }
43645f39d397SChris Mason
436562e2749eSChris Mason /* setup the item for the new data */
4366b7ef5f3aSFilipe Manana for (i = 0; i < batch->nr; i++) {
4367b7ef5f3aSFilipe Manana btrfs_cpu_key_to_disk(&disk_key, &batch->keys[i]);
43689c58309dSChris Mason btrfs_set_item_key(leaf, &disk_key, slot + i);
4369b7ef5f3aSFilipe Manana data_end -= batch->data_sizes[i];
43703212fa14SJosef Bacik btrfs_set_token_item_offset(&token, slot + i, data_end);
43713212fa14SJosef Bacik btrfs_set_token_item_size(&token, slot + i, batch->data_sizes[i]);
43729c58309dSChris Mason }
437344871b1bSChris Mason
4374b7ef5f3aSFilipe Manana btrfs_set_header_nritems(leaf, nritems + batch->nr);
4375*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, leaf);
4376aa5d6bedSChris Mason
4377e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) {
4378a4f78750SDavid Sterba btrfs_print_leaf(leaf);
4379be0e5c09SChris Mason BUG();
43805f39d397SChris Mason }
438144871b1bSChris Mason }
438244871b1bSChris Mason
438344871b1bSChris Mason /*
4384f0641656SFilipe Manana * Insert a new item into a leaf.
4385f0641656SFilipe Manana *
4386*d5e09e38SFilipe Manana * @trans: Transaction handle.
4387f0641656SFilipe Manana * @root: The root of the btree.
4388f0641656SFilipe Manana * @path: A path pointing to the target leaf and slot.
4389f0641656SFilipe Manana * @key: The key of the new item.
4390f0641656SFilipe Manana * @data_size: The size of the data associated with the new key.
4391f0641656SFilipe Manana */
btrfs_setup_item_for_insert(struct btrfs_trans_handle * trans,struct btrfs_root * root,struct btrfs_path * path,const struct btrfs_key * key,u32 data_size)4392*d5e09e38SFilipe Manana void btrfs_setup_item_for_insert(struct btrfs_trans_handle *trans,
4393*d5e09e38SFilipe Manana struct btrfs_root *root,
4394f0641656SFilipe Manana struct btrfs_path *path,
4395f0641656SFilipe Manana const struct btrfs_key *key,
4396f0641656SFilipe Manana u32 data_size)
4397f0641656SFilipe Manana {
4398f0641656SFilipe Manana struct btrfs_item_batch batch;
4399f0641656SFilipe Manana
4400f0641656SFilipe Manana batch.keys = key;
4401f0641656SFilipe Manana batch.data_sizes = &data_size;
4402f0641656SFilipe Manana batch.total_data_size = data_size;
4403f0641656SFilipe Manana batch.nr = 1;
4404f0641656SFilipe Manana
4405*d5e09e38SFilipe Manana setup_items_for_insert(trans, root, path, &batch);
4406f0641656SFilipe Manana }
4407f0641656SFilipe Manana
4408f0641656SFilipe Manana /*
440944871b1bSChris Mason * Given a key and some data, insert items into the tree.
441044871b1bSChris Mason * This does all the path init required, making room in the tree if needed.
441144871b1bSChris Mason */
btrfs_insert_empty_items(struct btrfs_trans_handle * trans,struct btrfs_root * root,struct btrfs_path * path,const struct btrfs_item_batch * batch)441244871b1bSChris Mason int btrfs_insert_empty_items(struct btrfs_trans_handle *trans,
441344871b1bSChris Mason struct btrfs_root *root,
441444871b1bSChris Mason struct btrfs_path *path,
4415b7ef5f3aSFilipe Manana const struct btrfs_item_batch *batch)
441644871b1bSChris Mason {
441744871b1bSChris Mason int ret = 0;
441844871b1bSChris Mason int slot;
4419b7ef5f3aSFilipe Manana u32 total_size;
442044871b1bSChris Mason
4421b7ef5f3aSFilipe Manana total_size = batch->total_data_size + (batch->nr * sizeof(struct btrfs_item));
4422b7ef5f3aSFilipe Manana ret = btrfs_search_slot(trans, root, &batch->keys[0], path, total_size, 1);
442344871b1bSChris Mason if (ret == 0)
442444871b1bSChris Mason return -EEXIST;
442544871b1bSChris Mason if (ret < 0)
4426143bede5SJeff Mahoney return ret;
442744871b1bSChris Mason
442844871b1bSChris Mason slot = path->slots[0];
442944871b1bSChris Mason BUG_ON(slot < 0);
443044871b1bSChris Mason
4431*d5e09e38SFilipe Manana setup_items_for_insert(trans, root, path, batch);
4432143bede5SJeff Mahoney return 0;
443362e2749eSChris Mason }
443462e2749eSChris Mason
443562e2749eSChris Mason /*
443662e2749eSChris Mason * Given a key and some data, insert an item into the tree.
443762e2749eSChris Mason * This does all the path init required, making room in the tree if needed.
443862e2749eSChris Mason */
btrfs_insert_item(struct btrfs_trans_handle * trans,struct btrfs_root * root,const struct btrfs_key * cpu_key,void * data,u32 data_size)4439310712b2SOmar Sandoval int btrfs_insert_item(struct btrfs_trans_handle *trans, struct btrfs_root *root,
4440310712b2SOmar Sandoval const struct btrfs_key *cpu_key, void *data,
4441310712b2SOmar Sandoval u32 data_size)
444262e2749eSChris Mason {
444362e2749eSChris Mason int ret = 0;
44442c90e5d6SChris Mason struct btrfs_path *path;
44455f39d397SChris Mason struct extent_buffer *leaf;
44465f39d397SChris Mason unsigned long ptr;
444762e2749eSChris Mason
44482c90e5d6SChris Mason path = btrfs_alloc_path();
4449db5b493aSTsutomu Itoh if (!path)
4450db5b493aSTsutomu Itoh return -ENOMEM;
44512c90e5d6SChris Mason ret = btrfs_insert_empty_item(trans, root, path, cpu_key, data_size);
445262e2749eSChris Mason if (!ret) {
44535f39d397SChris Mason leaf = path->nodes[0];
44545f39d397SChris Mason ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);
44555f39d397SChris Mason write_extent_buffer(leaf, data, ptr, data_size);
4456*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, leaf);
445762e2749eSChris Mason }
44582c90e5d6SChris Mason btrfs_free_path(path);
4459aa5d6bedSChris Mason return ret;
4460be0e5c09SChris Mason }
4461be0e5c09SChris Mason
446274123bd7SChris Mason /*
4463f0641656SFilipe Manana * This function duplicates an item, giving 'new_key' to the new item.
4464f0641656SFilipe Manana * It guarantees both items live in the same tree leaf and the new item is
4465f0641656SFilipe Manana * contiguous with the original item.
4466f0641656SFilipe Manana *
4467f0641656SFilipe Manana * This allows us to split a file extent in place, keeping a lock on the leaf
4468f0641656SFilipe Manana * the entire time.
4469f0641656SFilipe Manana */
btrfs_duplicate_item(struct btrfs_trans_handle * trans,struct btrfs_root * root,struct btrfs_path * path,const struct btrfs_key * new_key)4470f0641656SFilipe Manana int btrfs_duplicate_item(struct btrfs_trans_handle *trans,
4471f0641656SFilipe Manana struct btrfs_root *root,
4472f0641656SFilipe Manana struct btrfs_path *path,
4473f0641656SFilipe Manana const struct btrfs_key *new_key)
4474f0641656SFilipe Manana {
4475f0641656SFilipe Manana struct extent_buffer *leaf;
4476f0641656SFilipe Manana int ret;
4477f0641656SFilipe Manana u32 item_size;
4478f0641656SFilipe Manana
4479f0641656SFilipe Manana leaf = path->nodes[0];
44803212fa14SJosef Bacik item_size = btrfs_item_size(leaf, path->slots[0]);
4481f0641656SFilipe Manana ret = setup_leaf_for_split(trans, root, path,
4482f0641656SFilipe Manana item_size + sizeof(struct btrfs_item));
4483f0641656SFilipe Manana if (ret)
4484f0641656SFilipe Manana return ret;
4485f0641656SFilipe Manana
4486f0641656SFilipe Manana path->slots[0]++;
4487*d5e09e38SFilipe Manana btrfs_setup_item_for_insert(trans, root, path, new_key, item_size);
4488f0641656SFilipe Manana leaf = path->nodes[0];
4489f0641656SFilipe Manana memcpy_extent_buffer(leaf,
4490f0641656SFilipe Manana btrfs_item_ptr_offset(leaf, path->slots[0]),
4491f0641656SFilipe Manana btrfs_item_ptr_offset(leaf, path->slots[0] - 1),
4492f0641656SFilipe Manana item_size);
4493f0641656SFilipe Manana return 0;
4494f0641656SFilipe Manana }
4495f0641656SFilipe Manana
4496f0641656SFilipe Manana /*
44975de08d7dSChris Mason * delete the pointer from a given node.
449874123bd7SChris Mason *
4499d352ac68SChris Mason * the tree should have been previously balanced so the deletion does not
4500d352ac68SChris Mason * empty a node.
4501016f9d0bSJosef Bacik *
4502016f9d0bSJosef Bacik * This is exported for use inside btrfs-progs, don't un-export it.
450374123bd7SChris Mason */
btrfs_del_ptr(struct btrfs_trans_handle * trans,struct btrfs_root * root,struct btrfs_path * path,int level,int slot)4504751a2761SFilipe Manana int btrfs_del_ptr(struct btrfs_trans_handle *trans, struct btrfs_root *root,
4505751a2761SFilipe Manana struct btrfs_path *path, int level, int slot)
4506be0e5c09SChris Mason {
45075f39d397SChris Mason struct extent_buffer *parent = path->nodes[level];
45087518a238SChris Mason u32 nritems;
4509f3ea38daSJan Schmidt int ret;
4510be0e5c09SChris Mason
45115f39d397SChris Mason nritems = btrfs_header_nritems(parent);
4512be0e5c09SChris Mason if (slot != nritems - 1) {
4513bf1d3425SDavid Sterba if (level) {
4514f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_move(parent, slot,
4515f3a84ccdSFilipe Manana slot + 1, nritems - slot - 1);
4516751a2761SFilipe Manana if (ret < 0) {
4517751a2761SFilipe Manana btrfs_abort_transaction(trans, ret);
4518751a2761SFilipe Manana return ret;
4519751a2761SFilipe Manana }
4520bf1d3425SDavid Sterba }
45215f39d397SChris Mason memmove_extent_buffer(parent,
4522e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(parent, slot),
4523e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(parent, slot + 1),
4524d6025579SChris Mason sizeof(struct btrfs_key_ptr) *
4525d6025579SChris Mason (nritems - slot - 1));
452657ba86c0SChris Mason } else if (level) {
4527f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, slot,
452833cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REMOVE);
4529751a2761SFilipe Manana if (ret < 0) {
4530751a2761SFilipe Manana btrfs_abort_transaction(trans, ret);
4531751a2761SFilipe Manana return ret;
4532751a2761SFilipe Manana }
4533be0e5c09SChris Mason }
4534f3ea38daSJan Schmidt
45357518a238SChris Mason nritems--;
45365f39d397SChris Mason btrfs_set_header_nritems(parent, nritems);
45377518a238SChris Mason if (nritems == 0 && parent == root->node) {
45385f39d397SChris Mason BUG_ON(btrfs_header_level(root->node) != 1);
4539eb60ceacSChris Mason /* just turn the root into a leaf and break */
45405f39d397SChris Mason btrfs_set_header_level(root->node, 0);
4541bb803951SChris Mason } else if (slot == 0) {
45425f39d397SChris Mason struct btrfs_disk_key disk_key;
45435f39d397SChris Mason
45445f39d397SChris Mason btrfs_node_key(parent, &disk_key, 0);
4545*d5e09e38SFilipe Manana fixup_low_keys(trans, path, &disk_key, level + 1);
4546be0e5c09SChris Mason }
4547*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, parent);
4548751a2761SFilipe Manana return 0;
4549be0e5c09SChris Mason }
4550be0e5c09SChris Mason
455174123bd7SChris Mason /*
4552323ac95bSChris Mason * a helper function to delete the leaf pointed to by path->slots[1] and
45535d4f98a2SYan Zheng * path->nodes[1].
4554323ac95bSChris Mason *
4555323ac95bSChris Mason * This deletes the pointer in path->nodes[1] and frees the leaf
4556323ac95bSChris Mason * block extent. zero is returned if it all worked out, < 0 otherwise.
4557323ac95bSChris Mason *
4558323ac95bSChris Mason * The path must have already been setup for deleting the leaf, including
4559323ac95bSChris Mason * all the proper balancing. path->nodes[1] must be locked.
4560323ac95bSChris Mason */
btrfs_del_leaf(struct btrfs_trans_handle * trans,struct btrfs_root * root,struct btrfs_path * path,struct extent_buffer * leaf)4561751a2761SFilipe Manana static noinline int btrfs_del_leaf(struct btrfs_trans_handle *trans,
4562323ac95bSChris Mason struct btrfs_root *root,
45635d4f98a2SYan Zheng struct btrfs_path *path,
45645d4f98a2SYan Zheng struct extent_buffer *leaf)
4565323ac95bSChris Mason {
4566751a2761SFilipe Manana int ret;
4567751a2761SFilipe Manana
45685d4f98a2SYan Zheng WARN_ON(btrfs_header_generation(leaf) != trans->transid);
4569751a2761SFilipe Manana ret = btrfs_del_ptr(trans, root, path, 1, path->slots[1]);
4570751a2761SFilipe Manana if (ret < 0)
4571751a2761SFilipe Manana return ret;
4572323ac95bSChris Mason
45734d081c41SChris Mason /*
45744d081c41SChris Mason * btrfs_free_extent is expensive, we want to make sure we
45754d081c41SChris Mason * aren't holding any locks when we call it
45764d081c41SChris Mason */
45774d081c41SChris Mason btrfs_unlock_up_safe(path, 0);
45784d081c41SChris Mason
4579f0486c68SYan, Zheng root_sub_used(root, leaf->len);
4580f0486c68SYan, Zheng
458167439dadSDavid Sterba atomic_inc(&leaf->refs);
45827a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), leaf, 0, 1);
45833083ee2eSJosef Bacik free_extent_buffer_stale(leaf);
4584751a2761SFilipe Manana return 0;
4585323ac95bSChris Mason }
4586323ac95bSChris Mason /*
458774123bd7SChris Mason * delete the item at the leaf level in path. If that empties
458874123bd7SChris Mason * the leaf, remove it from the tree
458974123bd7SChris Mason */
btrfs_del_items(struct btrfs_trans_handle * trans,struct btrfs_root * root,struct btrfs_path * path,int slot,int nr)459085e21bacSChris Mason int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root,
459185e21bacSChris Mason struct btrfs_path *path, int slot, int nr)
4592be0e5c09SChris Mason {
45930b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info;
45945f39d397SChris Mason struct extent_buffer *leaf;
4595aa5d6bedSChris Mason int ret = 0;
4596aa5d6bedSChris Mason int wret;
45977518a238SChris Mason u32 nritems;
4598be0e5c09SChris Mason
45995f39d397SChris Mason leaf = path->nodes[0];
46005f39d397SChris Mason nritems = btrfs_header_nritems(leaf);
4601be0e5c09SChris Mason
460285e21bacSChris Mason if (slot + nr != nritems) {
46030cae23b6SFilipe Manana const u32 last_off = btrfs_item_offset(leaf, slot + nr - 1);
46040cae23b6SFilipe Manana const int data_end = leaf_data_end(leaf);
4605c82f823cSDavid Sterba struct btrfs_map_token token;
46060cae23b6SFilipe Manana u32 dsize = 0;
46070cae23b6SFilipe Manana int i;
46080cae23b6SFilipe Manana
46090cae23b6SFilipe Manana for (i = 0; i < nr; i++)
46100cae23b6SFilipe Manana dsize += btrfs_item_size(leaf, slot + i);
46115f39d397SChris Mason
4612637e3b48SJosef Bacik memmove_leaf_data(leaf, data_end + dsize, data_end,
461385e21bacSChris Mason last_off - data_end);
46145f39d397SChris Mason
4615c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf);
461685e21bacSChris Mason for (i = slot + nr; i < nritems; i++) {
46175f39d397SChris Mason u32 ioff;
4618db94535dSChris Mason
46193212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i);
46203212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff + dsize);
46210783fcfcSChris Mason }
4622db94535dSChris Mason
4623637e3b48SJosef Bacik memmove_leaf_items(leaf, slot, slot + nr, nritems - slot - nr);
4624be0e5c09SChris Mason }
462585e21bacSChris Mason btrfs_set_header_nritems(leaf, nritems - nr);
462685e21bacSChris Mason nritems -= nr;
46275f39d397SChris Mason
462874123bd7SChris Mason /* delete the leaf if we've emptied it */
46297518a238SChris Mason if (nritems == 0) {
46305f39d397SChris Mason if (leaf == root->node) {
46315f39d397SChris Mason btrfs_set_header_level(leaf, 0);
46329a8dd150SChris Mason } else {
4633190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, leaf);
4634751a2761SFilipe Manana ret = btrfs_del_leaf(trans, root, path, leaf);
4635751a2761SFilipe Manana if (ret < 0)
4636751a2761SFilipe Manana return ret;
46379a8dd150SChris Mason }
4638be0e5c09SChris Mason } else {
46397518a238SChris Mason int used = leaf_space_used(leaf, 0, nritems);
4640aa5d6bedSChris Mason if (slot == 0) {
46415f39d397SChris Mason struct btrfs_disk_key disk_key;
46425f39d397SChris Mason
46435f39d397SChris Mason btrfs_item_key(leaf, &disk_key, 0);
4644*d5e09e38SFilipe Manana fixup_low_keys(trans, path, &disk_key, 1);
4645aa5d6bedSChris Mason }
4646aa5d6bedSChris Mason
46477c4063d1SFilipe Manana /*
46487c4063d1SFilipe Manana * Try to delete the leaf if it is mostly empty. We do this by
46497c4063d1SFilipe Manana * trying to move all its items into its left and right neighbours.
46507c4063d1SFilipe Manana * If we can't move all the items, then we don't delete it - it's
46517c4063d1SFilipe Manana * not ideal, but future insertions might fill the leaf with more
46527c4063d1SFilipe Manana * items, or items from other leaves might be moved later into our
46537c4063d1SFilipe Manana * leaf due to deletions on those leaves.
46547c4063d1SFilipe Manana */
46550b246afaSJeff Mahoney if (used < BTRFS_LEAF_DATA_SIZE(fs_info) / 3) {
46567c4063d1SFilipe Manana u32 min_push_space;
46577c4063d1SFilipe Manana
4658be0e5c09SChris Mason /* push_leaf_left fixes the path.
4659be0e5c09SChris Mason * make sure the path still points to our leaf
4660016f9d0bSJosef Bacik * for possible call to btrfs_del_ptr below
4661be0e5c09SChris Mason */
46624920c9acSChris Mason slot = path->slots[1];
466367439dadSDavid Sterba atomic_inc(&leaf->refs);
46647c4063d1SFilipe Manana /*
46657c4063d1SFilipe Manana * We want to be able to at least push one item to the
46667c4063d1SFilipe Manana * left neighbour leaf, and that's the first item.
46677c4063d1SFilipe Manana */
46687c4063d1SFilipe Manana min_push_space = sizeof(struct btrfs_item) +
46697c4063d1SFilipe Manana btrfs_item_size(leaf, 0);
46707c4063d1SFilipe Manana wret = push_leaf_left(trans, root, path, 0,
46717c4063d1SFilipe Manana min_push_space, 1, (u32)-1);
467254aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC)
4673aa5d6bedSChris Mason ret = wret;
46745f39d397SChris Mason
46755f39d397SChris Mason if (path->nodes[0] == leaf &&
46765f39d397SChris Mason btrfs_header_nritems(leaf)) {
46777c4063d1SFilipe Manana /*
46787c4063d1SFilipe Manana * If we were not able to push all items from our
46797c4063d1SFilipe Manana * leaf to its left neighbour, then attempt to
46807c4063d1SFilipe Manana * either push all the remaining items to the
46817c4063d1SFilipe Manana * right neighbour or none. There's no advantage
46827c4063d1SFilipe Manana * in pushing only some items, instead of all, as
46837c4063d1SFilipe Manana * it's pointless to end up with a leaf having
46847c4063d1SFilipe Manana * too few items while the neighbours can be full
46857c4063d1SFilipe Manana * or nearly full.
46867c4063d1SFilipe Manana */
46877c4063d1SFilipe Manana nritems = btrfs_header_nritems(leaf);
46887c4063d1SFilipe Manana min_push_space = leaf_space_used(leaf, 0, nritems);
46897c4063d1SFilipe Manana wret = push_leaf_right(trans, root, path, 0,
46907c4063d1SFilipe Manana min_push_space, 1, 0);
469154aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC)
4692aa5d6bedSChris Mason ret = wret;
4693aa5d6bedSChris Mason }
46945f39d397SChris Mason
46955f39d397SChris Mason if (btrfs_header_nritems(leaf) == 0) {
4696323ac95bSChris Mason path->slots[1] = slot;
4697751a2761SFilipe Manana ret = btrfs_del_leaf(trans, root, path, leaf);
4698751a2761SFilipe Manana if (ret < 0)
4699751a2761SFilipe Manana return ret;
47005f39d397SChris Mason free_extent_buffer(leaf);
4701143bede5SJeff Mahoney ret = 0;
47025de08d7dSChris Mason } else {
4703925baeddSChris Mason /* if we're still in the path, make sure
4704925baeddSChris Mason * we're dirty. Otherwise, one of the
4705925baeddSChris Mason * push_leaf functions must have already
4706925baeddSChris Mason * dirtied this buffer
4707925baeddSChris Mason */
4708925baeddSChris Mason if (path->nodes[0] == leaf)
4709*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, leaf);
47105f39d397SChris Mason free_extent_buffer(leaf);
4711be0e5c09SChris Mason }
4712d5719762SChris Mason } else {
4713*d5e09e38SFilipe Manana btrfs_mark_buffer_dirty(trans, leaf);
4714be0e5c09SChris Mason }
47159a8dd150SChris Mason }
4716aa5d6bedSChris Mason return ret;
47179a8dd150SChris Mason }
47189a8dd150SChris Mason
471997571fd0SChris Mason /*
47203f157a2fSChris Mason * A helper function to walk down the tree starting at min_key, and looking
4721de78b51aSEric Sandeen * for nodes or leaves that are have a minimum transaction id.
4722de78b51aSEric Sandeen * This is used by the btree defrag code, and tree logging
47233f157a2fSChris Mason *
47243f157a2fSChris Mason * This does not cow, but it does stuff the starting key it finds back
47253f157a2fSChris Mason * into min_key, so you can call btrfs_search_slot with cow=1 on the
47263f157a2fSChris Mason * key and get a writable path.
47273f157a2fSChris Mason *
47283f157a2fSChris Mason * This honors path->lowest_level to prevent descent past a given level
47293f157a2fSChris Mason * of the tree.
47303f157a2fSChris Mason *
4731d352ac68SChris Mason * min_trans indicates the oldest transaction that you are interested
4732d352ac68SChris Mason * in walking through. Any nodes or leaves older than min_trans are
4733d352ac68SChris Mason * skipped over (without reading them).
4734d352ac68SChris Mason *
47353f157a2fSChris Mason * returns zero if something useful was found, < 0 on error and 1 if there
47363f157a2fSChris Mason * was nothing in the tree that matched the search criteria.
47373f157a2fSChris Mason */
btrfs_search_forward(struct btrfs_root * root,struct btrfs_key * min_key,struct btrfs_path * path,u64 min_trans)47383f157a2fSChris Mason int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key,
4739de78b51aSEric Sandeen struct btrfs_path *path,
47403f157a2fSChris Mason u64 min_trans)
47413f157a2fSChris Mason {
47423f157a2fSChris Mason struct extent_buffer *cur;
47433f157a2fSChris Mason struct btrfs_key found_key;
47443f157a2fSChris Mason int slot;
47459652480bSYan int sret;
47463f157a2fSChris Mason u32 nritems;
47473f157a2fSChris Mason int level;
47483f157a2fSChris Mason int ret = 1;
4749f98de9b9SFilipe Manana int keep_locks = path->keep_locks;
47503f157a2fSChris Mason
4751c922b016SStefan Roesch ASSERT(!path->nowait);
4752f98de9b9SFilipe Manana path->keep_locks = 1;
47533f157a2fSChris Mason again:
4754bd681513SChris Mason cur = btrfs_read_lock_root_node(root);
47553f157a2fSChris Mason level = btrfs_header_level(cur);
4756e02119d5SChris Mason WARN_ON(path->nodes[level]);
47573f157a2fSChris Mason path->nodes[level] = cur;
4758bd681513SChris Mason path->locks[level] = BTRFS_READ_LOCK;
47593f157a2fSChris Mason
47603f157a2fSChris Mason if (btrfs_header_generation(cur) < min_trans) {
47613f157a2fSChris Mason ret = 1;
47623f157a2fSChris Mason goto out;
47633f157a2fSChris Mason }
47643f157a2fSChris Mason while (1) {
47653f157a2fSChris Mason nritems = btrfs_header_nritems(cur);
47663f157a2fSChris Mason level = btrfs_header_level(cur);
4767fdf8d595SAnand Jain sret = btrfs_bin_search(cur, 0, min_key, &slot);
4768cbca7d59SFilipe Manana if (sret < 0) {
4769cbca7d59SFilipe Manana ret = sret;
4770cbca7d59SFilipe Manana goto out;
4771cbca7d59SFilipe Manana }
47723f157a2fSChris Mason
4773323ac95bSChris Mason /* at the lowest level, we're done, setup the path and exit */
4774323ac95bSChris Mason if (level == path->lowest_level) {
4775e02119d5SChris Mason if (slot >= nritems)
4776e02119d5SChris Mason goto find_next_key;
47773f157a2fSChris Mason ret = 0;
47783f157a2fSChris Mason path->slots[level] = slot;
47793f157a2fSChris Mason btrfs_item_key_to_cpu(cur, &found_key, slot);
47803f157a2fSChris Mason goto out;
47813f157a2fSChris Mason }
47829652480bSYan if (sret && slot > 0)
47839652480bSYan slot--;
47843f157a2fSChris Mason /*
4785de78b51aSEric Sandeen * check this node pointer against the min_trans parameters.
4786260db43cSRandy Dunlap * If it is too old, skip to the next one.
47873f157a2fSChris Mason */
47883f157a2fSChris Mason while (slot < nritems) {
47893f157a2fSChris Mason u64 gen;
4790e02119d5SChris Mason
47913f157a2fSChris Mason gen = btrfs_node_ptr_generation(cur, slot);
47923f157a2fSChris Mason if (gen < min_trans) {
47933f157a2fSChris Mason slot++;
47943f157a2fSChris Mason continue;
47953f157a2fSChris Mason }
47963f157a2fSChris Mason break;
47973f157a2fSChris Mason }
4798e02119d5SChris Mason find_next_key:
47993f157a2fSChris Mason /*
48003f157a2fSChris Mason * we didn't find a candidate key in this node, walk forward
48013f157a2fSChris Mason * and find another one
48023f157a2fSChris Mason */
48033f157a2fSChris Mason if (slot >= nritems) {
4804e02119d5SChris Mason path->slots[level] = slot;
4805e02119d5SChris Mason sret = btrfs_find_next_key(root, path, min_key, level,
4806de78b51aSEric Sandeen min_trans);
4807e02119d5SChris Mason if (sret == 0) {
4808b3b4aa74SDavid Sterba btrfs_release_path(path);
48093f157a2fSChris Mason goto again;
48103f157a2fSChris Mason } else {
48113f157a2fSChris Mason goto out;
48123f157a2fSChris Mason }
48133f157a2fSChris Mason }
48143f157a2fSChris Mason /* save our key for returning back */
48153f157a2fSChris Mason btrfs_node_key_to_cpu(cur, &found_key, slot);
48163f157a2fSChris Mason path->slots[level] = slot;
48173f157a2fSChris Mason if (level == path->lowest_level) {
48183f157a2fSChris Mason ret = 0;
48193f157a2fSChris Mason goto out;
48203f157a2fSChris Mason }
48214b231ae4SDavid Sterba cur = btrfs_read_node_slot(cur, slot);
4822fb770ae4SLiu Bo if (IS_ERR(cur)) {
4823fb770ae4SLiu Bo ret = PTR_ERR(cur);
4824fb770ae4SLiu Bo goto out;
4825fb770ae4SLiu Bo }
48263f157a2fSChris Mason
4827bd681513SChris Mason btrfs_tree_read_lock(cur);
4828b4ce94deSChris Mason
4829bd681513SChris Mason path->locks[level - 1] = BTRFS_READ_LOCK;
48303f157a2fSChris Mason path->nodes[level - 1] = cur;
4831f7c79f30SChris Mason unlock_up(path, level, 1, 0, NULL);
48323f157a2fSChris Mason }
48333f157a2fSChris Mason out:
4834f98de9b9SFilipe Manana path->keep_locks = keep_locks;
4835f98de9b9SFilipe Manana if (ret == 0) {
4836f98de9b9SFilipe Manana btrfs_unlock_up_safe(path, path->lowest_level + 1);
4837f98de9b9SFilipe Manana memcpy(min_key, &found_key, sizeof(found_key));
4838f98de9b9SFilipe Manana }
48393f157a2fSChris Mason return ret;
48403f157a2fSChris Mason }
48413f157a2fSChris Mason
48423f157a2fSChris Mason /*
48433f157a2fSChris Mason * this is similar to btrfs_next_leaf, but does not try to preserve
48443f157a2fSChris Mason * and fixup the path. It looks for and returns the next key in the
4845de78b51aSEric Sandeen * tree based on the current path and the min_trans parameters.
48463f157a2fSChris Mason *
48473f157a2fSChris Mason * 0 is returned if another key is found, < 0 if there are any errors
48483f157a2fSChris Mason * and 1 is returned if there are no higher keys in the tree
48493f157a2fSChris Mason *
48503f157a2fSChris Mason * path->keep_locks should be set to 1 on the search made before
48513f157a2fSChris Mason * calling this function.
48523f157a2fSChris Mason */
btrfs_find_next_key(struct btrfs_root * root,struct btrfs_path * path,struct btrfs_key * key,int level,u64 min_trans)4853e7a84565SChris Mason int btrfs_find_next_key(struct btrfs_root *root, struct btrfs_path *path,
4854de78b51aSEric Sandeen struct btrfs_key *key, int level, u64 min_trans)
4855e7a84565SChris Mason {
4856e7a84565SChris Mason int slot;
4857e7a84565SChris Mason struct extent_buffer *c;
4858e7a84565SChris Mason
48596a9fb468SJosef Bacik WARN_ON(!path->keep_locks && !path->skip_locking);
4860e7a84565SChris Mason while (level < BTRFS_MAX_LEVEL) {
4861e7a84565SChris Mason if (!path->nodes[level])
4862e7a84565SChris Mason return 1;
4863e7a84565SChris Mason
4864e7a84565SChris Mason slot = path->slots[level] + 1;
4865e7a84565SChris Mason c = path->nodes[level];
48663f157a2fSChris Mason next:
4867e7a84565SChris Mason if (slot >= btrfs_header_nritems(c)) {
486833c66f43SYan Zheng int ret;
486933c66f43SYan Zheng int orig_lowest;
487033c66f43SYan Zheng struct btrfs_key cur_key;
487133c66f43SYan Zheng if (level + 1 >= BTRFS_MAX_LEVEL ||
487233c66f43SYan Zheng !path->nodes[level + 1])
4873e7a84565SChris Mason return 1;
487433c66f43SYan Zheng
48756a9fb468SJosef Bacik if (path->locks[level + 1] || path->skip_locking) {
487633c66f43SYan Zheng level++;
4877e7a84565SChris Mason continue;
4878e7a84565SChris Mason }
487933c66f43SYan Zheng
488033c66f43SYan Zheng slot = btrfs_header_nritems(c) - 1;
488133c66f43SYan Zheng if (level == 0)
488233c66f43SYan Zheng btrfs_item_key_to_cpu(c, &cur_key, slot);
488333c66f43SYan Zheng else
488433c66f43SYan Zheng btrfs_node_key_to_cpu(c, &cur_key, slot);
488533c66f43SYan Zheng
488633c66f43SYan Zheng orig_lowest = path->lowest_level;
4887b3b4aa74SDavid Sterba btrfs_release_path(path);
488833c66f43SYan Zheng path->lowest_level = level;
488933c66f43SYan Zheng ret = btrfs_search_slot(NULL, root, &cur_key, path,
489033c66f43SYan Zheng 0, 0);
489133c66f43SYan Zheng path->lowest_level = orig_lowest;
489233c66f43SYan Zheng if (ret < 0)
489333c66f43SYan Zheng return ret;
489433c66f43SYan Zheng
489533c66f43SYan Zheng c = path->nodes[level];
489633c66f43SYan Zheng slot = path->slots[level];
489733c66f43SYan Zheng if (ret == 0)
489833c66f43SYan Zheng slot++;
489933c66f43SYan Zheng goto next;
490033c66f43SYan Zheng }
490133c66f43SYan Zheng
4902e7a84565SChris Mason if (level == 0)
4903e7a84565SChris Mason btrfs_item_key_to_cpu(c, key, slot);
49043f157a2fSChris Mason else {
49053f157a2fSChris Mason u64 gen = btrfs_node_ptr_generation(c, slot);
49063f157a2fSChris Mason
49073f157a2fSChris Mason if (gen < min_trans) {
49083f157a2fSChris Mason slot++;
49093f157a2fSChris Mason goto next;
49103f157a2fSChris Mason }
4911e7a84565SChris Mason btrfs_node_key_to_cpu(c, key, slot);
49123f157a2fSChris Mason }
4913e7a84565SChris Mason return 0;
4914e7a84565SChris Mason }
4915e7a84565SChris Mason return 1;
4916e7a84565SChris Mason }
4917e7a84565SChris Mason
btrfs_next_old_leaf(struct btrfs_root * root,struct btrfs_path * path,u64 time_seq)49183d7806ecSJan Schmidt int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path,
49193d7806ecSJan Schmidt u64 time_seq)
49203d7806ecSJan Schmidt {
4921d97e63b6SChris Mason int slot;
49228e73f275SChris Mason int level;
49235f39d397SChris Mason struct extent_buffer *c;
49248e73f275SChris Mason struct extent_buffer *next;
4925d96b3424SFilipe Manana struct btrfs_fs_info *fs_info = root->fs_info;
4926925baeddSChris Mason struct btrfs_key key;
4927d96b3424SFilipe Manana bool need_commit_sem = false;
4928925baeddSChris Mason u32 nritems;
4929925baeddSChris Mason int ret;
49300e46318dSJosef Bacik int i;
4931925baeddSChris Mason
4932bdcdd86cSFilipe Manana /*
4933bdcdd86cSFilipe Manana * The nowait semantics are used only for write paths, where we don't
4934bdcdd86cSFilipe Manana * use the tree mod log and sequence numbers.
4935bdcdd86cSFilipe Manana */
4936bdcdd86cSFilipe Manana if (time_seq)
4937c922b016SStefan Roesch ASSERT(!path->nowait);
4938c922b016SStefan Roesch
4939925baeddSChris Mason nritems = btrfs_header_nritems(path->nodes[0]);
4940d397712bSChris Mason if (nritems == 0)
4941925baeddSChris Mason return 1;
4942925baeddSChris Mason
49438e73f275SChris Mason btrfs_item_key_to_cpu(path->nodes[0], &key, nritems - 1);
49448e73f275SChris Mason again:
49458e73f275SChris Mason level = 1;
49468e73f275SChris Mason next = NULL;
4947b3b4aa74SDavid Sterba btrfs_release_path(path);
49488e73f275SChris Mason
4949a2135011SChris Mason path->keep_locks = 1;
49508e73f275SChris Mason
4951d96b3424SFilipe Manana if (time_seq) {
49523d7806ecSJan Schmidt ret = btrfs_search_old_slot(root, &key, path, time_seq);
4953d96b3424SFilipe Manana } else {
4954d96b3424SFilipe Manana if (path->need_commit_sem) {
4955d96b3424SFilipe Manana path->need_commit_sem = 0;
4956d96b3424SFilipe Manana need_commit_sem = true;
4957bdcdd86cSFilipe Manana if (path->nowait) {
4958bdcdd86cSFilipe Manana if (!down_read_trylock(&fs_info->commit_root_sem)) {
4959bdcdd86cSFilipe Manana ret = -EAGAIN;
4960bdcdd86cSFilipe Manana goto done;
4961bdcdd86cSFilipe Manana }
4962bdcdd86cSFilipe Manana } else {
4963d96b3424SFilipe Manana down_read(&fs_info->commit_root_sem);
4964d96b3424SFilipe Manana }
4965bdcdd86cSFilipe Manana }
4966925baeddSChris Mason ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
4967d96b3424SFilipe Manana }
4968925baeddSChris Mason path->keep_locks = 0;
4969925baeddSChris Mason
4970925baeddSChris Mason if (ret < 0)
4971d96b3424SFilipe Manana goto done;
4972925baeddSChris Mason
4973a2135011SChris Mason nritems = btrfs_header_nritems(path->nodes[0]);
4974168fd7d2SChris Mason /*
4975168fd7d2SChris Mason * by releasing the path above we dropped all our locks. A balance
4976168fd7d2SChris Mason * could have added more items next to the key that used to be
4977168fd7d2SChris Mason * at the very end of the block. So, check again here and
4978168fd7d2SChris Mason * advance the path if there are now more items available.
4979168fd7d2SChris Mason */
4980a2135011SChris Mason if (nritems > 0 && path->slots[0] < nritems - 1) {
4981e457afecSYan Zheng if (ret == 0)
4982168fd7d2SChris Mason path->slots[0]++;
49838e73f275SChris Mason ret = 0;
4984925baeddSChris Mason goto done;
4985925baeddSChris Mason }
49860b43e04fSLiu Bo /*
49870b43e04fSLiu Bo * So the above check misses one case:
49880b43e04fSLiu Bo * - after releasing the path above, someone has removed the item that
49890b43e04fSLiu Bo * used to be at the very end of the block, and balance between leafs
49900b43e04fSLiu Bo * gets another one with bigger key.offset to replace it.
49910b43e04fSLiu Bo *
49920b43e04fSLiu Bo * This one should be returned as well, or we can get leaf corruption
49930b43e04fSLiu Bo * later(esp. in __btrfs_drop_extents()).
49940b43e04fSLiu Bo *
49950b43e04fSLiu Bo * And a bit more explanation about this check,
49960b43e04fSLiu Bo * with ret > 0, the key isn't found, the path points to the slot
49970b43e04fSLiu Bo * where it should be inserted, so the path->slots[0] item must be the
49980b43e04fSLiu Bo * bigger one.
49990b43e04fSLiu Bo */
50000b43e04fSLiu Bo if (nritems > 0 && ret > 0 && path->slots[0] == nritems - 1) {
50010b43e04fSLiu Bo ret = 0;
50020b43e04fSLiu Bo goto done;
50030b43e04fSLiu Bo }
5004d97e63b6SChris Mason
5005234b63a0SChris Mason while (level < BTRFS_MAX_LEVEL) {
50068e73f275SChris Mason if (!path->nodes[level]) {
50078e73f275SChris Mason ret = 1;
50088e73f275SChris Mason goto done;
50098e73f275SChris Mason }
50105f39d397SChris Mason
5011d97e63b6SChris Mason slot = path->slots[level] + 1;
5012d97e63b6SChris Mason c = path->nodes[level];
50135f39d397SChris Mason if (slot >= btrfs_header_nritems(c)) {
5014d97e63b6SChris Mason level++;
50158e73f275SChris Mason if (level == BTRFS_MAX_LEVEL) {
50168e73f275SChris Mason ret = 1;
50178e73f275SChris Mason goto done;
50188e73f275SChris Mason }
5019d97e63b6SChris Mason continue;
5020d97e63b6SChris Mason }
50215f39d397SChris Mason
50220e46318dSJosef Bacik
50230e46318dSJosef Bacik /*
50240e46318dSJosef Bacik * Our current level is where we're going to start from, and to
50250e46318dSJosef Bacik * make sure lockdep doesn't complain we need to drop our locks
50260e46318dSJosef Bacik * and nodes from 0 to our current level.
50270e46318dSJosef Bacik */
50280e46318dSJosef Bacik for (i = 0; i < level; i++) {
50290e46318dSJosef Bacik if (path->locks[level]) {
50300e46318dSJosef Bacik btrfs_tree_read_unlock(path->nodes[i]);
50310e46318dSJosef Bacik path->locks[i] = 0;
50320e46318dSJosef Bacik }
50330e46318dSJosef Bacik free_extent_buffer(path->nodes[i]);
50340e46318dSJosef Bacik path->nodes[i] = NULL;
5035925baeddSChris Mason }
50365f39d397SChris Mason
50378e73f275SChris Mason next = c;
5038d07b8528SLiu Bo ret = read_block_for_search(root, path, &next, level,
5039cda79c54SDavid Sterba slot, &key);
5040bdcdd86cSFilipe Manana if (ret == -EAGAIN && !path->nowait)
50418e73f275SChris Mason goto again;
50425f39d397SChris Mason
504376a05b35SChris Mason if (ret < 0) {
5044b3b4aa74SDavid Sterba btrfs_release_path(path);
504576a05b35SChris Mason goto done;
504676a05b35SChris Mason }
504776a05b35SChris Mason
50485cd57b2cSChris Mason if (!path->skip_locking) {
5049bd681513SChris Mason ret = btrfs_try_tree_read_lock(next);
5050bdcdd86cSFilipe Manana if (!ret && path->nowait) {
5051bdcdd86cSFilipe Manana ret = -EAGAIN;
5052bdcdd86cSFilipe Manana goto done;
5053bdcdd86cSFilipe Manana }
5054d42244a0SJan Schmidt if (!ret && time_seq) {
5055d42244a0SJan Schmidt /*
5056d42244a0SJan Schmidt * If we don't get the lock, we may be racing
5057d42244a0SJan Schmidt * with push_leaf_left, holding that lock while
5058d42244a0SJan Schmidt * itself waiting for the leaf we've currently
5059d42244a0SJan Schmidt * locked. To solve this situation, we give up
5060d42244a0SJan Schmidt * on our lock and cycle.
5061d42244a0SJan Schmidt */
5062cf538830SJan Schmidt free_extent_buffer(next);
5063d42244a0SJan Schmidt btrfs_release_path(path);
5064d42244a0SJan Schmidt cond_resched();
5065d42244a0SJan Schmidt goto again;
5066d42244a0SJan Schmidt }
50670e46318dSJosef Bacik if (!ret)
50680e46318dSJosef Bacik btrfs_tree_read_lock(next);
5069bd681513SChris Mason }
5070d97e63b6SChris Mason break;
5071d97e63b6SChris Mason }
5072d97e63b6SChris Mason path->slots[level] = slot;
5073d97e63b6SChris Mason while (1) {
5074d97e63b6SChris Mason level--;
5075d97e63b6SChris Mason path->nodes[level] = next;
5076d97e63b6SChris Mason path->slots[level] = 0;
5077a74a4b97SChris Mason if (!path->skip_locking)
5078ffeb03cfSJosef Bacik path->locks[level] = BTRFS_READ_LOCK;
5079d97e63b6SChris Mason if (!level)
5080d97e63b6SChris Mason break;
5081b4ce94deSChris Mason
5082d07b8528SLiu Bo ret = read_block_for_search(root, path, &next, level,
5083cda79c54SDavid Sterba 0, &key);
5084bdcdd86cSFilipe Manana if (ret == -EAGAIN && !path->nowait)
50858e73f275SChris Mason goto again;
50868e73f275SChris Mason
508776a05b35SChris Mason if (ret < 0) {
5088b3b4aa74SDavid Sterba btrfs_release_path(path);
508976a05b35SChris Mason goto done;
509076a05b35SChris Mason }
509176a05b35SChris Mason
5092bdcdd86cSFilipe Manana if (!path->skip_locking) {
5093bdcdd86cSFilipe Manana if (path->nowait) {
5094bdcdd86cSFilipe Manana if (!btrfs_try_tree_read_lock(next)) {
5095bdcdd86cSFilipe Manana ret = -EAGAIN;
5096bdcdd86cSFilipe Manana goto done;
5097bdcdd86cSFilipe Manana }
5098bdcdd86cSFilipe Manana } else {
50990e46318dSJosef Bacik btrfs_tree_read_lock(next);
5100d97e63b6SChris Mason }
5101bdcdd86cSFilipe Manana }
5102bdcdd86cSFilipe Manana }
51038e73f275SChris Mason ret = 0;
5104925baeddSChris Mason done:
5105f7c79f30SChris Mason unlock_up(path, 0, 1, 0, NULL);
5106d96b3424SFilipe Manana if (need_commit_sem) {
5107d96b3424SFilipe Manana int ret2;
5108d96b3424SFilipe Manana
5109d96b3424SFilipe Manana path->need_commit_sem = 1;
5110d96b3424SFilipe Manana ret2 = finish_need_commit_sem_search(path);
5111d96b3424SFilipe Manana up_read(&fs_info->commit_root_sem);
5112d96b3424SFilipe Manana if (ret2)
5113d96b3424SFilipe Manana ret = ret2;
5114d96b3424SFilipe Manana }
51158e73f275SChris Mason
51168e73f275SChris Mason return ret;
5117d97e63b6SChris Mason }
51180b86a832SChris Mason
btrfs_next_old_item(struct btrfs_root * root,struct btrfs_path * path,u64 time_seq)5119890d2b1aSJosef Bacik int btrfs_next_old_item(struct btrfs_root *root, struct btrfs_path *path, u64 time_seq)
5120890d2b1aSJosef Bacik {
5121890d2b1aSJosef Bacik path->slots[0]++;
5122890d2b1aSJosef Bacik if (path->slots[0] >= btrfs_header_nritems(path->nodes[0]))
5123890d2b1aSJosef Bacik return btrfs_next_old_leaf(root, path, time_seq);
5124890d2b1aSJosef Bacik return 0;
5125890d2b1aSJosef Bacik }
5126890d2b1aSJosef Bacik
51273f157a2fSChris Mason /*
51283f157a2fSChris Mason * this uses btrfs_prev_leaf to walk backwards in the tree, and keeps
51293f157a2fSChris Mason * searching until it gets past min_objectid or finds an item of 'type'
51303f157a2fSChris Mason *
51313f157a2fSChris Mason * returns 0 if something is found, 1 if nothing was found and < 0 on error
51323f157a2fSChris Mason */
btrfs_previous_item(struct btrfs_root * root,struct btrfs_path * path,u64 min_objectid,int type)51330b86a832SChris Mason int btrfs_previous_item(struct btrfs_root *root,
51340b86a832SChris Mason struct btrfs_path *path, u64 min_objectid,
51350b86a832SChris Mason int type)
51360b86a832SChris Mason {
51370b86a832SChris Mason struct btrfs_key found_key;
51380b86a832SChris Mason struct extent_buffer *leaf;
5139e02119d5SChris Mason u32 nritems;
51400b86a832SChris Mason int ret;
51410b86a832SChris Mason
51420b86a832SChris Mason while (1) {
51430b86a832SChris Mason if (path->slots[0] == 0) {
51440b86a832SChris Mason ret = btrfs_prev_leaf(root, path);
51450b86a832SChris Mason if (ret != 0)
51460b86a832SChris Mason return ret;
51470b86a832SChris Mason } else {
51480b86a832SChris Mason path->slots[0]--;
51490b86a832SChris Mason }
51500b86a832SChris Mason leaf = path->nodes[0];
5151e02119d5SChris Mason nritems = btrfs_header_nritems(leaf);
5152e02119d5SChris Mason if (nritems == 0)
5153e02119d5SChris Mason return 1;
5154e02119d5SChris Mason if (path->slots[0] == nritems)
5155e02119d5SChris Mason path->slots[0]--;
5156e02119d5SChris Mason
51570b86a832SChris Mason btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
5158e02119d5SChris Mason if (found_key.objectid < min_objectid)
5159e02119d5SChris Mason break;
51600a4eefbbSYan Zheng if (found_key.type == type)
51610a4eefbbSYan Zheng return 0;
5162e02119d5SChris Mason if (found_key.objectid == min_objectid &&
5163e02119d5SChris Mason found_key.type < type)
5164e02119d5SChris Mason break;
51650b86a832SChris Mason }
51660b86a832SChris Mason return 1;
51670b86a832SChris Mason }
5168ade2e0b3SWang Shilong
5169ade2e0b3SWang Shilong /*
5170ade2e0b3SWang Shilong * search in extent tree to find a previous Metadata/Data extent item with
5171ade2e0b3SWang Shilong * min objecitd.
5172ade2e0b3SWang Shilong *
5173ade2e0b3SWang Shilong * returns 0 if something is found, 1 if nothing was found and < 0 on error
5174ade2e0b3SWang Shilong */
btrfs_previous_extent_item(struct btrfs_root * root,struct btrfs_path * path,u64 min_objectid)5175ade2e0b3SWang Shilong int btrfs_previous_extent_item(struct btrfs_root *root,
5176ade2e0b3SWang Shilong struct btrfs_path *path, u64 min_objectid)
5177ade2e0b3SWang Shilong {
5178ade2e0b3SWang Shilong struct btrfs_key found_key;
5179ade2e0b3SWang Shilong struct extent_buffer *leaf;
5180ade2e0b3SWang Shilong u32 nritems;
5181ade2e0b3SWang Shilong int ret;
5182ade2e0b3SWang Shilong
5183ade2e0b3SWang Shilong while (1) {
5184ade2e0b3SWang Shilong if (path->slots[0] == 0) {
5185ade2e0b3SWang Shilong ret = btrfs_prev_leaf(root, path);
5186ade2e0b3SWang Shilong if (ret != 0)
5187ade2e0b3SWang Shilong return ret;
5188ade2e0b3SWang Shilong } else {
5189ade2e0b3SWang Shilong path->slots[0]--;
5190ade2e0b3SWang Shilong }
5191ade2e0b3SWang Shilong leaf = path->nodes[0];
5192ade2e0b3SWang Shilong nritems = btrfs_header_nritems(leaf);
5193ade2e0b3SWang Shilong if (nritems == 0)
5194ade2e0b3SWang Shilong return 1;
5195ade2e0b3SWang Shilong if (path->slots[0] == nritems)
5196ade2e0b3SWang Shilong path->slots[0]--;
5197ade2e0b3SWang Shilong
5198ade2e0b3SWang Shilong btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
5199ade2e0b3SWang Shilong if (found_key.objectid < min_objectid)
5200ade2e0b3SWang Shilong break;
5201ade2e0b3SWang Shilong if (found_key.type == BTRFS_EXTENT_ITEM_KEY ||
5202ade2e0b3SWang Shilong found_key.type == BTRFS_METADATA_ITEM_KEY)
5203ade2e0b3SWang Shilong return 0;
5204ade2e0b3SWang Shilong if (found_key.objectid == min_objectid &&
5205ade2e0b3SWang Shilong found_key.type < BTRFS_EXTENT_ITEM_KEY)
5206ade2e0b3SWang Shilong break;
5207ade2e0b3SWang Shilong }
5208ade2e0b3SWang Shilong return 1;
5209ade2e0b3SWang Shilong }
5210226463d7SJosef Bacik
btrfs_ctree_init(void)5211226463d7SJosef Bacik int __init btrfs_ctree_init(void)
5212226463d7SJosef Bacik {
5213226463d7SJosef Bacik btrfs_path_cachep = kmem_cache_create("btrfs_path",
5214226463d7SJosef Bacik sizeof(struct btrfs_path), 0,
5215226463d7SJosef Bacik SLAB_MEM_SPREAD, NULL);
5216226463d7SJosef Bacik if (!btrfs_path_cachep)
5217226463d7SJosef Bacik return -ENOMEM;
5218226463d7SJosef Bacik return 0;
5219226463d7SJosef Bacik }
5220226463d7SJosef Bacik
btrfs_ctree_exit(void)5221226463d7SJosef Bacik void __cold btrfs_ctree_exit(void)
5222226463d7SJosef Bacik {
5223226463d7SJosef Bacik kmem_cache_destroy(btrfs_path_cachep);
5224226463d7SJosef Bacik }
5225