1c1d7c514SDavid Sterba // SPDX-License-Identifier: GPL-2.0 26cbd5570SChris Mason /* 3d352ac68SChris Mason * Copyright (C) 2007,2008 Oracle. All rights reserved. 46cbd5570SChris Mason */ 56cbd5570SChris Mason 6a6b6e75eSChris Mason #include <linux/sched.h> 75a0e3ad6STejun Heo #include <linux/slab.h> 8bd989ba3SJan Schmidt #include <linux/rbtree.h> 9adf02123SDavid Sterba #include <linux/mm.h> 10e41d12f5SChristoph Hellwig #include <linux/error-injection.h> 119b569ea0SJosef Bacik #include "messages.h" 12eb60ceacSChris Mason #include "ctree.h" 13eb60ceacSChris Mason #include "disk-io.h" 147f5c1516SChris Mason #include "transaction.h" 155f39d397SChris Mason #include "print-tree.h" 16925baeddSChris Mason #include "locking.h" 17de37aa51SNikolay Borisov #include "volumes.h" 18f616f5cdSQu Wenruo #include "qgroup.h" 19f3a84ccdSFilipe Manana #include "tree-mod-log.h" 2088c602abSQu Wenruo #include "tree-checker.h" 21ec8eb376SJosef Bacik #include "fs.h" 22ad1ac501SJosef Bacik #include "accessors.h" 23a0231804SJosef Bacik #include "extent-tree.h" 2467707479SJosef Bacik #include "relocation.h" 256bfd0ffaSJosef Bacik #include "file-item.h" 269a8dd150SChris Mason 27226463d7SJosef Bacik static struct kmem_cache *btrfs_path_cachep; 28226463d7SJosef Bacik 29e089f05cSChris Mason static int split_node(struct btrfs_trans_handle *trans, struct btrfs_root 30e089f05cSChris Mason *root, struct btrfs_path *path, int level); 31310712b2SOmar Sandoval static int split_leaf(struct btrfs_trans_handle *trans, struct btrfs_root *root, 32310712b2SOmar Sandoval const struct btrfs_key *ins_key, struct btrfs_path *path, 33310712b2SOmar Sandoval int data_size, int extend); 345f39d397SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 352ff7e61eSJeff Mahoney struct extent_buffer *dst, 36971a1f66SChris Mason struct extent_buffer *src, int empty); 375f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 385f39d397SChris Mason struct extent_buffer *dst_buf, 395f39d397SChris Mason struct extent_buffer *src_buf); 40afe5fea7STsutomu Itoh static void del_ptr(struct btrfs_root *root, struct btrfs_path *path, 41afe5fea7STsutomu Itoh int level, int slot); 42d97e63b6SChris Mason 43af024ed2SJohannes Thumshirn static const struct btrfs_csums { 44af024ed2SJohannes Thumshirn u16 size; 4559a0fcdbSDavid Sterba const char name[10]; 4659a0fcdbSDavid Sterba const char driver[12]; 47af024ed2SJohannes Thumshirn } btrfs_csums[] = { 48af024ed2SJohannes Thumshirn [BTRFS_CSUM_TYPE_CRC32] = { .size = 4, .name = "crc32c" }, 493951e7f0SJohannes Thumshirn [BTRFS_CSUM_TYPE_XXHASH] = { .size = 8, .name = "xxhash64" }, 503831bf00SJohannes Thumshirn [BTRFS_CSUM_TYPE_SHA256] = { .size = 32, .name = "sha256" }, 51352ae07bSDavid Sterba [BTRFS_CSUM_TYPE_BLAKE2] = { .size = 32, .name = "blake2b", 52352ae07bSDavid Sterba .driver = "blake2b-256" }, 53af024ed2SJohannes Thumshirn }; 54af024ed2SJohannes Thumshirn 553a3178c7SJosef Bacik /* 563a3178c7SJosef Bacik * The leaf data grows from end-to-front in the node. this returns the address 573a3178c7SJosef Bacik * of the start of the last item, which is the stop of the leaf data stack. 583a3178c7SJosef Bacik */ 593a3178c7SJosef Bacik static unsigned int leaf_data_end(const struct extent_buffer *leaf) 603a3178c7SJosef Bacik { 613a3178c7SJosef Bacik u32 nr = btrfs_header_nritems(leaf); 623a3178c7SJosef Bacik 633a3178c7SJosef Bacik if (nr == 0) 643a3178c7SJosef Bacik return BTRFS_LEAF_DATA_SIZE(leaf->fs_info); 653a3178c7SJosef Bacik return btrfs_item_offset(leaf, nr - 1); 663a3178c7SJosef Bacik } 673a3178c7SJosef Bacik 68637e3b48SJosef Bacik /* 69637e3b48SJosef Bacik * Move data in a @leaf (using memmove, safe for overlapping ranges). 70637e3b48SJosef Bacik * 71637e3b48SJosef Bacik * @leaf: leaf that we're doing a memmove on 72637e3b48SJosef Bacik * @dst_offset: item data offset we're moving to 73637e3b48SJosef Bacik * @src_offset: item data offset were' moving from 74637e3b48SJosef Bacik * @len: length of the data we're moving 75637e3b48SJosef Bacik * 76637e3b48SJosef Bacik * Wrapper around memmove_extent_buffer() that takes into account the header on 77637e3b48SJosef Bacik * the leaf. The btrfs_item offset's start directly after the header, so we 78637e3b48SJosef Bacik * have to adjust any offsets to account for the header in the leaf. This 79637e3b48SJosef Bacik * handles that math to simplify the callers. 80637e3b48SJosef Bacik */ 81637e3b48SJosef Bacik static inline void memmove_leaf_data(const struct extent_buffer *leaf, 82637e3b48SJosef Bacik unsigned long dst_offset, 83637e3b48SJosef Bacik unsigned long src_offset, 84637e3b48SJosef Bacik unsigned long len) 85637e3b48SJosef Bacik { 868009adf3SJosef Bacik memmove_extent_buffer(leaf, btrfs_item_nr_offset(leaf, 0) + dst_offset, 878009adf3SJosef Bacik btrfs_item_nr_offset(leaf, 0) + src_offset, len); 88637e3b48SJosef Bacik } 89637e3b48SJosef Bacik 90637e3b48SJosef Bacik /* 91637e3b48SJosef Bacik * Copy item data from @src into @dst at the given @offset. 92637e3b48SJosef Bacik * 93637e3b48SJosef Bacik * @dst: destination leaf that we're copying into 94637e3b48SJosef Bacik * @src: source leaf that we're copying from 95637e3b48SJosef Bacik * @dst_offset: item data offset we're copying to 96637e3b48SJosef Bacik * @src_offset: item data offset were' copying from 97637e3b48SJosef Bacik * @len: length of the data we're copying 98637e3b48SJosef Bacik * 99637e3b48SJosef Bacik * Wrapper around copy_extent_buffer() that takes into account the header on 100637e3b48SJosef Bacik * the leaf. The btrfs_item offset's start directly after the header, so we 101637e3b48SJosef Bacik * have to adjust any offsets to account for the header in the leaf. This 102637e3b48SJosef Bacik * handles that math to simplify the callers. 103637e3b48SJosef Bacik */ 104637e3b48SJosef Bacik static inline void copy_leaf_data(const struct extent_buffer *dst, 105637e3b48SJosef Bacik const struct extent_buffer *src, 106637e3b48SJosef Bacik unsigned long dst_offset, 107637e3b48SJosef Bacik unsigned long src_offset, unsigned long len) 108637e3b48SJosef Bacik { 1098009adf3SJosef Bacik copy_extent_buffer(dst, src, btrfs_item_nr_offset(dst, 0) + dst_offset, 1108009adf3SJosef Bacik btrfs_item_nr_offset(src, 0) + src_offset, len); 111637e3b48SJosef Bacik } 112637e3b48SJosef Bacik 113637e3b48SJosef Bacik /* 114637e3b48SJosef Bacik * Move items in a @leaf (using memmove). 115637e3b48SJosef Bacik * 116637e3b48SJosef Bacik * @dst: destination leaf for the items 117637e3b48SJosef Bacik * @dst_item: the item nr we're copying into 118637e3b48SJosef Bacik * @src_item: the item nr we're copying from 119637e3b48SJosef Bacik * @nr_items: the number of items to copy 120637e3b48SJosef Bacik * 121637e3b48SJosef Bacik * Wrapper around memmove_extent_buffer() that does the math to get the 122637e3b48SJosef Bacik * appropriate offsets into the leaf from the item numbers. 123637e3b48SJosef Bacik */ 124637e3b48SJosef Bacik static inline void memmove_leaf_items(const struct extent_buffer *leaf, 125637e3b48SJosef Bacik int dst_item, int src_item, int nr_items) 126637e3b48SJosef Bacik { 127637e3b48SJosef Bacik memmove_extent_buffer(leaf, btrfs_item_nr_offset(leaf, dst_item), 128637e3b48SJosef Bacik btrfs_item_nr_offset(leaf, src_item), 129637e3b48SJosef Bacik nr_items * sizeof(struct btrfs_item)); 130637e3b48SJosef Bacik } 131637e3b48SJosef Bacik 132637e3b48SJosef Bacik /* 133637e3b48SJosef Bacik * Copy items from @src into @dst at the given @offset. 134637e3b48SJosef Bacik * 135637e3b48SJosef Bacik * @dst: destination leaf for the items 136637e3b48SJosef Bacik * @src: source leaf for the items 137637e3b48SJosef Bacik * @dst_item: the item nr we're copying into 138637e3b48SJosef Bacik * @src_item: the item nr we're copying from 139637e3b48SJosef Bacik * @nr_items: the number of items to copy 140637e3b48SJosef Bacik * 141637e3b48SJosef Bacik * Wrapper around copy_extent_buffer() that does the math to get the 142637e3b48SJosef Bacik * appropriate offsets into the leaf from the item numbers. 143637e3b48SJosef Bacik */ 144637e3b48SJosef Bacik static inline void copy_leaf_items(const struct extent_buffer *dst, 145637e3b48SJosef Bacik const struct extent_buffer *src, 146637e3b48SJosef Bacik int dst_item, int src_item, int nr_items) 147637e3b48SJosef Bacik { 148637e3b48SJosef Bacik copy_extent_buffer(dst, src, btrfs_item_nr_offset(dst, dst_item), 149637e3b48SJosef Bacik btrfs_item_nr_offset(src, src_item), 150637e3b48SJosef Bacik nr_items * sizeof(struct btrfs_item)); 151637e3b48SJosef Bacik } 152637e3b48SJosef Bacik 153af024ed2SJohannes Thumshirn int btrfs_super_csum_size(const struct btrfs_super_block *s) 154af024ed2SJohannes Thumshirn { 155af024ed2SJohannes Thumshirn u16 t = btrfs_super_csum_type(s); 156af024ed2SJohannes Thumshirn /* 157af024ed2SJohannes Thumshirn * csum type is validated at mount time 158af024ed2SJohannes Thumshirn */ 159af024ed2SJohannes Thumshirn return btrfs_csums[t].size; 160af024ed2SJohannes Thumshirn } 161af024ed2SJohannes Thumshirn 162af024ed2SJohannes Thumshirn const char *btrfs_super_csum_name(u16 csum_type) 163af024ed2SJohannes Thumshirn { 164af024ed2SJohannes Thumshirn /* csum type is validated at mount time */ 165af024ed2SJohannes Thumshirn return btrfs_csums[csum_type].name; 166af024ed2SJohannes Thumshirn } 167af024ed2SJohannes Thumshirn 168b4e967beSDavid Sterba /* 169b4e967beSDavid Sterba * Return driver name if defined, otherwise the name that's also a valid driver 170b4e967beSDavid Sterba * name 171b4e967beSDavid Sterba */ 172b4e967beSDavid Sterba const char *btrfs_super_csum_driver(u16 csum_type) 173b4e967beSDavid Sterba { 174b4e967beSDavid Sterba /* csum type is validated at mount time */ 17559a0fcdbSDavid Sterba return btrfs_csums[csum_type].driver[0] ? 17659a0fcdbSDavid Sterba btrfs_csums[csum_type].driver : 177b4e967beSDavid Sterba btrfs_csums[csum_type].name; 178b4e967beSDavid Sterba } 179b4e967beSDavid Sterba 180604997b4SDavid Sterba size_t __attribute_const__ btrfs_get_num_csums(void) 181f7cea56cSDavid Sterba { 182f7cea56cSDavid Sterba return ARRAY_SIZE(btrfs_csums); 183f7cea56cSDavid Sterba } 184f7cea56cSDavid Sterba 1852c90e5d6SChris Mason struct btrfs_path *btrfs_alloc_path(void) 1862c90e5d6SChris Mason { 187a4c853afSChenXiaoSong might_sleep(); 188a4c853afSChenXiaoSong 189e2c89907SMasahiro Yamada return kmem_cache_zalloc(btrfs_path_cachep, GFP_NOFS); 1902c90e5d6SChris Mason } 1912c90e5d6SChris Mason 192d352ac68SChris Mason /* this also releases the path */ 1932c90e5d6SChris Mason void btrfs_free_path(struct btrfs_path *p) 1942c90e5d6SChris Mason { 195ff175d57SJesper Juhl if (!p) 196ff175d57SJesper Juhl return; 197b3b4aa74SDavid Sterba btrfs_release_path(p); 1982c90e5d6SChris Mason kmem_cache_free(btrfs_path_cachep, p); 1992c90e5d6SChris Mason } 2002c90e5d6SChris Mason 201d352ac68SChris Mason /* 202d352ac68SChris Mason * path release drops references on the extent buffers in the path 203d352ac68SChris Mason * and it drops any locks held by this path 204d352ac68SChris Mason * 205d352ac68SChris Mason * It is safe to call this on paths that no locks or extent buffers held. 206d352ac68SChris Mason */ 207b3b4aa74SDavid Sterba noinline void btrfs_release_path(struct btrfs_path *p) 208eb60ceacSChris Mason { 209eb60ceacSChris Mason int i; 210a2135011SChris Mason 211234b63a0SChris Mason for (i = 0; i < BTRFS_MAX_LEVEL; i++) { 2123f157a2fSChris Mason p->slots[i] = 0; 213eb60ceacSChris Mason if (!p->nodes[i]) 214925baeddSChris Mason continue; 215925baeddSChris Mason if (p->locks[i]) { 216bd681513SChris Mason btrfs_tree_unlock_rw(p->nodes[i], p->locks[i]); 217925baeddSChris Mason p->locks[i] = 0; 218925baeddSChris Mason } 2195f39d397SChris Mason free_extent_buffer(p->nodes[i]); 2203f157a2fSChris Mason p->nodes[i] = NULL; 221eb60ceacSChris Mason } 222eb60ceacSChris Mason } 223eb60ceacSChris Mason 224d352ac68SChris Mason /* 2258bb808c6SDavid Sterba * We want the transaction abort to print stack trace only for errors where the 2268bb808c6SDavid Sterba * cause could be a bug, eg. due to ENOSPC, and not for common errors that are 2278bb808c6SDavid Sterba * caused by external factors. 2288bb808c6SDavid Sterba */ 2298bb808c6SDavid Sterba bool __cold abort_should_print_stack(int errno) 2308bb808c6SDavid Sterba { 2318bb808c6SDavid Sterba switch (errno) { 2328bb808c6SDavid Sterba case -EIO: 2338bb808c6SDavid Sterba case -EROFS: 2348bb808c6SDavid Sterba case -ENOMEM: 2358bb808c6SDavid Sterba return false; 2368bb808c6SDavid Sterba } 2378bb808c6SDavid Sterba return true; 2388bb808c6SDavid Sterba } 2398bb808c6SDavid Sterba 2408bb808c6SDavid Sterba /* 241d352ac68SChris Mason * safely gets a reference on the root node of a tree. A lock 242d352ac68SChris Mason * is not taken, so a concurrent writer may put a different node 243d352ac68SChris Mason * at the root of the tree. See btrfs_lock_root_node for the 244d352ac68SChris Mason * looping required. 245d352ac68SChris Mason * 246d352ac68SChris Mason * The extent buffer returned by this has a reference taken, so 247d352ac68SChris Mason * it won't disappear. It may stop being the root of the tree 248d352ac68SChris Mason * at any time because there are no locks held. 249d352ac68SChris Mason */ 250925baeddSChris Mason struct extent_buffer *btrfs_root_node(struct btrfs_root *root) 251925baeddSChris Mason { 252925baeddSChris Mason struct extent_buffer *eb; 253240f62c8SChris Mason 2543083ee2eSJosef Bacik while (1) { 255240f62c8SChris Mason rcu_read_lock(); 256240f62c8SChris Mason eb = rcu_dereference(root->node); 2573083ee2eSJosef Bacik 2583083ee2eSJosef Bacik /* 2593083ee2eSJosef Bacik * RCU really hurts here, we could free up the root node because 26001327610SNicholas D Steeves * it was COWed but we may not get the new root node yet so do 2613083ee2eSJosef Bacik * the inc_not_zero dance and if it doesn't work then 2623083ee2eSJosef Bacik * synchronize_rcu and try again. 2633083ee2eSJosef Bacik */ 2643083ee2eSJosef Bacik if (atomic_inc_not_zero(&eb->refs)) { 265240f62c8SChris Mason rcu_read_unlock(); 2663083ee2eSJosef Bacik break; 2673083ee2eSJosef Bacik } 2683083ee2eSJosef Bacik rcu_read_unlock(); 2693083ee2eSJosef Bacik synchronize_rcu(); 2703083ee2eSJosef Bacik } 271925baeddSChris Mason return eb; 272925baeddSChris Mason } 273925baeddSChris Mason 27492a7cc42SQu Wenruo /* 27592a7cc42SQu Wenruo * Cowonly root (not-shareable trees, everything not subvolume or reloc roots), 27692a7cc42SQu Wenruo * just get put onto a simple dirty list. Transaction walks this list to make 27792a7cc42SQu Wenruo * sure they get properly updated on disk. 278d352ac68SChris Mason */ 2790b86a832SChris Mason static void add_root_to_dirty_list(struct btrfs_root *root) 2800b86a832SChris Mason { 2810b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 2820b246afaSJeff Mahoney 283e7070be1SJosef Bacik if (test_bit(BTRFS_ROOT_DIRTY, &root->state) || 284e7070be1SJosef Bacik !test_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state)) 285e7070be1SJosef Bacik return; 286e7070be1SJosef Bacik 2870b246afaSJeff Mahoney spin_lock(&fs_info->trans_lock); 288e7070be1SJosef Bacik if (!test_and_set_bit(BTRFS_ROOT_DIRTY, &root->state)) { 289e7070be1SJosef Bacik /* Want the extent tree to be the last on the list */ 2904fd786e6SMisono Tomohiro if (root->root_key.objectid == BTRFS_EXTENT_TREE_OBJECTID) 291e7070be1SJosef Bacik list_move_tail(&root->dirty_list, 2920b246afaSJeff Mahoney &fs_info->dirty_cowonly_roots); 293e7070be1SJosef Bacik else 294e7070be1SJosef Bacik list_move(&root->dirty_list, 2950b246afaSJeff Mahoney &fs_info->dirty_cowonly_roots); 2960b86a832SChris Mason } 2970b246afaSJeff Mahoney spin_unlock(&fs_info->trans_lock); 2980b86a832SChris Mason } 2990b86a832SChris Mason 300d352ac68SChris Mason /* 301d352ac68SChris Mason * used by snapshot creation to make a copy of a root for a tree with 302d352ac68SChris Mason * a given objectid. The buffer with the new root node is returned in 303d352ac68SChris Mason * cow_ret, and this func returns zero on success or a negative error code. 304d352ac68SChris Mason */ 305be20aa9dSChris Mason int btrfs_copy_root(struct btrfs_trans_handle *trans, 306be20aa9dSChris Mason struct btrfs_root *root, 307be20aa9dSChris Mason struct extent_buffer *buf, 308be20aa9dSChris Mason struct extent_buffer **cow_ret, u64 new_root_objectid) 309be20aa9dSChris Mason { 3100b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 311be20aa9dSChris Mason struct extent_buffer *cow; 312be20aa9dSChris Mason int ret = 0; 313be20aa9dSChris Mason int level; 3145d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 315be20aa9dSChris Mason 31692a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 3170b246afaSJeff Mahoney trans->transid != fs_info->running_transaction->transid); 31892a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 31927cdeb70SMiao Xie trans->transid != root->last_trans); 320be20aa9dSChris Mason 321be20aa9dSChris Mason level = btrfs_header_level(buf); 3225d4f98a2SYan Zheng if (level == 0) 3235d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 3245d4f98a2SYan Zheng else 3255d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 32631840ae1SZheng Yan 3274d75f8a9SDavid Sterba cow = btrfs_alloc_tree_block(trans, root, 0, new_root_objectid, 328cf6f34aaSJosef Bacik &disk_key, level, buf->start, 0, 329cf6f34aaSJosef Bacik BTRFS_NESTING_NEW_ROOT); 3305d4f98a2SYan Zheng if (IS_ERR(cow)) 331be20aa9dSChris Mason return PTR_ERR(cow); 332be20aa9dSChris Mason 33358e8012cSDavid Sterba copy_extent_buffer_full(cow, buf); 334be20aa9dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 335be20aa9dSChris Mason btrfs_set_header_generation(cow, trans->transid); 3365d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 3375d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 3385d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 3395d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 3405d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 3415d4f98a2SYan Zheng else 342be20aa9dSChris Mason btrfs_set_header_owner(cow, new_root_objectid); 343be20aa9dSChris Mason 344de37aa51SNikolay Borisov write_extent_buffer_fsid(cow, fs_info->fs_devices->metadata_uuid); 3452b82032cSYan Zheng 346be20aa9dSChris Mason WARN_ON(btrfs_header_generation(buf) > trans->transid); 3475d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 348e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 3495d4f98a2SYan Zheng else 350e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 351867ed321SJosef Bacik if (ret) { 35272c9925fSFilipe Manana btrfs_tree_unlock(cow); 35372c9925fSFilipe Manana free_extent_buffer(cow); 354867ed321SJosef Bacik btrfs_abort_transaction(trans, ret); 355be20aa9dSChris Mason return ret; 356867ed321SJosef Bacik } 357be20aa9dSChris Mason 358be20aa9dSChris Mason btrfs_mark_buffer_dirty(cow); 359be20aa9dSChris Mason *cow_ret = cow; 360be20aa9dSChris Mason return 0; 361be20aa9dSChris Mason } 362be20aa9dSChris Mason 363d352ac68SChris Mason /* 3645d4f98a2SYan Zheng * check if the tree block can be shared by multiple trees 3655d4f98a2SYan Zheng */ 3665d4f98a2SYan Zheng int btrfs_block_can_be_shared(struct btrfs_root *root, 3675d4f98a2SYan Zheng struct extent_buffer *buf) 3685d4f98a2SYan Zheng { 3695d4f98a2SYan Zheng /* 37092a7cc42SQu Wenruo * Tree blocks not in shareable trees and tree roots are never shared. 37192a7cc42SQu Wenruo * If a block was allocated after the last snapshot and the block was 37292a7cc42SQu Wenruo * not allocated by tree relocation, we know the block is not shared. 3735d4f98a2SYan Zheng */ 37492a7cc42SQu Wenruo if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 3755d4f98a2SYan Zheng buf != root->node && buf != root->commit_root && 3765d4f98a2SYan Zheng (btrfs_header_generation(buf) <= 3775d4f98a2SYan Zheng btrfs_root_last_snapshot(&root->root_item) || 3785d4f98a2SYan Zheng btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC))) 3795d4f98a2SYan Zheng return 1; 380a79865c6SNikolay Borisov 3815d4f98a2SYan Zheng return 0; 3825d4f98a2SYan Zheng } 3835d4f98a2SYan Zheng 3845d4f98a2SYan Zheng static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans, 3855d4f98a2SYan Zheng struct btrfs_root *root, 3865d4f98a2SYan Zheng struct extent_buffer *buf, 387f0486c68SYan, Zheng struct extent_buffer *cow, 388f0486c68SYan, Zheng int *last_ref) 3895d4f98a2SYan Zheng { 3900b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 3915d4f98a2SYan Zheng u64 refs; 3925d4f98a2SYan Zheng u64 owner; 3935d4f98a2SYan Zheng u64 flags; 3945d4f98a2SYan Zheng u64 new_flags = 0; 3955d4f98a2SYan Zheng int ret; 3965d4f98a2SYan Zheng 3975d4f98a2SYan Zheng /* 3985d4f98a2SYan Zheng * Backrefs update rules: 3995d4f98a2SYan Zheng * 4005d4f98a2SYan Zheng * Always use full backrefs for extent pointers in tree block 4015d4f98a2SYan Zheng * allocated by tree relocation. 4025d4f98a2SYan Zheng * 4035d4f98a2SYan Zheng * If a shared tree block is no longer referenced by its owner 4045d4f98a2SYan Zheng * tree (btrfs_header_owner(buf) == root->root_key.objectid), 4055d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 4065d4f98a2SYan Zheng * 4075d4f98a2SYan Zheng * If a tree block is been relocating 4085d4f98a2SYan Zheng * (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID), 4095d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 4105d4f98a2SYan Zheng * The reason for this is some operations (such as drop tree) 4115d4f98a2SYan Zheng * are only allowed for blocks use full backrefs. 4125d4f98a2SYan Zheng */ 4135d4f98a2SYan Zheng 4145d4f98a2SYan Zheng if (btrfs_block_can_be_shared(root, buf)) { 4152ff7e61eSJeff Mahoney ret = btrfs_lookup_extent_info(trans, fs_info, buf->start, 4163173a18fSJosef Bacik btrfs_header_level(buf), 1, 4173173a18fSJosef Bacik &refs, &flags); 418be1a5564SMark Fasheh if (ret) 419be1a5564SMark Fasheh return ret; 420e5df9573SMark Fasheh if (refs == 0) { 421e5df9573SMark Fasheh ret = -EROFS; 4220b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 423e5df9573SMark Fasheh return ret; 424e5df9573SMark Fasheh } 4255d4f98a2SYan Zheng } else { 4265d4f98a2SYan Zheng refs = 1; 4275d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 4285d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 4295d4f98a2SYan Zheng flags = BTRFS_BLOCK_FLAG_FULL_BACKREF; 4305d4f98a2SYan Zheng else 4315d4f98a2SYan Zheng flags = 0; 4325d4f98a2SYan Zheng } 4335d4f98a2SYan Zheng 4345d4f98a2SYan Zheng owner = btrfs_header_owner(buf); 4355d4f98a2SYan Zheng BUG_ON(owner == BTRFS_TREE_RELOC_OBJECTID && 4365d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)); 4375d4f98a2SYan Zheng 4385d4f98a2SYan Zheng if (refs > 1) { 4395d4f98a2SYan Zheng if ((owner == root->root_key.objectid || 4405d4f98a2SYan Zheng root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && 4415d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) { 442e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, buf, 1); 443692826b2SJeff Mahoney if (ret) 444692826b2SJeff Mahoney return ret; 4455d4f98a2SYan Zheng 4465d4f98a2SYan Zheng if (root->root_key.objectid == 4475d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) { 448e339a6b0SJosef Bacik ret = btrfs_dec_ref(trans, root, buf, 0); 449692826b2SJeff Mahoney if (ret) 450692826b2SJeff Mahoney return ret; 451e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 452692826b2SJeff Mahoney if (ret) 453692826b2SJeff Mahoney return ret; 4545d4f98a2SYan Zheng } 4555d4f98a2SYan Zheng new_flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF; 4565d4f98a2SYan Zheng } else { 4575d4f98a2SYan Zheng 4585d4f98a2SYan Zheng if (root->root_key.objectid == 4595d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 460e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 4615d4f98a2SYan Zheng else 462e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 463692826b2SJeff Mahoney if (ret) 464692826b2SJeff Mahoney return ret; 4655d4f98a2SYan Zheng } 4665d4f98a2SYan Zheng if (new_flags != 0) { 467b1c79e09SJosef Bacik int level = btrfs_header_level(buf); 468b1c79e09SJosef Bacik 46942c9d0b5SDavid Sterba ret = btrfs_set_disk_extent_flags(trans, buf, 4702fe6a5a1SDavid Sterba new_flags, level); 471be1a5564SMark Fasheh if (ret) 472be1a5564SMark Fasheh return ret; 4735d4f98a2SYan Zheng } 4745d4f98a2SYan Zheng } else { 4755d4f98a2SYan Zheng if (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) { 4765d4f98a2SYan Zheng if (root->root_key.objectid == 4775d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 478e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 4795d4f98a2SYan Zheng else 480e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 481692826b2SJeff Mahoney if (ret) 482692826b2SJeff Mahoney return ret; 483e339a6b0SJosef Bacik ret = btrfs_dec_ref(trans, root, buf, 1); 484692826b2SJeff Mahoney if (ret) 485692826b2SJeff Mahoney return ret; 4865d4f98a2SYan Zheng } 487190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, buf); 488f0486c68SYan, Zheng *last_ref = 1; 4895d4f98a2SYan Zheng } 4905d4f98a2SYan Zheng return 0; 4915d4f98a2SYan Zheng } 4925d4f98a2SYan Zheng 4935d4f98a2SYan Zheng /* 494d397712bSChris Mason * does the dirty work in cow of a single block. The parent block (if 495d397712bSChris Mason * supplied) is updated to point to the new cow copy. The new buffer is marked 496d397712bSChris Mason * dirty and returned locked. If you modify the block it needs to be marked 497d397712bSChris Mason * dirty again. 498d352ac68SChris Mason * 499d352ac68SChris Mason * search_start -- an allocation hint for the new block 500d352ac68SChris Mason * 501d397712bSChris Mason * empty_size -- a hint that you plan on doing more cow. This is the size in 502d397712bSChris Mason * bytes the allocator should try to find free next to the block it returns. 503d397712bSChris Mason * This is just a hint and may be ignored by the allocator. 504d352ac68SChris Mason */ 505d397712bSChris Mason static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans, 5065f39d397SChris Mason struct btrfs_root *root, 5075f39d397SChris Mason struct extent_buffer *buf, 5085f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 5095f39d397SChris Mason struct extent_buffer **cow_ret, 5109631e4ccSJosef Bacik u64 search_start, u64 empty_size, 5119631e4ccSJosef Bacik enum btrfs_lock_nesting nest) 5126702ed49SChris Mason { 5130b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 5145d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 5155f39d397SChris Mason struct extent_buffer *cow; 516be1a5564SMark Fasheh int level, ret; 517f0486c68SYan, Zheng int last_ref = 0; 518925baeddSChris Mason int unlock_orig = 0; 5190f5053ebSGoldwyn Rodrigues u64 parent_start = 0; 5206702ed49SChris Mason 521925baeddSChris Mason if (*cow_ret == buf) 522925baeddSChris Mason unlock_orig = 1; 523925baeddSChris Mason 52449d0c642SFilipe Manana btrfs_assert_tree_write_locked(buf); 525925baeddSChris Mason 52692a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 5270b246afaSJeff Mahoney trans->transid != fs_info->running_transaction->transid); 52892a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 52927cdeb70SMiao Xie trans->transid != root->last_trans); 5305f39d397SChris Mason 5317bb86316SChris Mason level = btrfs_header_level(buf); 53231840ae1SZheng Yan 5335d4f98a2SYan Zheng if (level == 0) 5345d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 5355d4f98a2SYan Zheng else 5365d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 5375d4f98a2SYan Zheng 5380f5053ebSGoldwyn Rodrigues if ((root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && parent) 5395d4f98a2SYan Zheng parent_start = parent->start; 5405d4f98a2SYan Zheng 54179bd3712SFilipe Manana cow = btrfs_alloc_tree_block(trans, root, parent_start, 54279bd3712SFilipe Manana root->root_key.objectid, &disk_key, level, 54379bd3712SFilipe Manana search_start, empty_size, nest); 5446702ed49SChris Mason if (IS_ERR(cow)) 5456702ed49SChris Mason return PTR_ERR(cow); 5466702ed49SChris Mason 547b4ce94deSChris Mason /* cow is set to blocking by btrfs_init_new_buffer */ 548b4ce94deSChris Mason 54958e8012cSDavid Sterba copy_extent_buffer_full(cow, buf); 550db94535dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 5515f39d397SChris Mason btrfs_set_header_generation(cow, trans->transid); 5525d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 5535d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 5545d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 5555d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) 5565d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 5575d4f98a2SYan Zheng else 5585f39d397SChris Mason btrfs_set_header_owner(cow, root->root_key.objectid); 5596702ed49SChris Mason 560de37aa51SNikolay Borisov write_extent_buffer_fsid(cow, fs_info->fs_devices->metadata_uuid); 5612b82032cSYan Zheng 562be1a5564SMark Fasheh ret = update_ref_for_cow(trans, root, buf, cow, &last_ref); 563b68dc2a9SMark Fasheh if (ret) { 564572c83acSJosef Bacik btrfs_tree_unlock(cow); 565572c83acSJosef Bacik free_extent_buffer(cow); 56666642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 567b68dc2a9SMark Fasheh return ret; 568b68dc2a9SMark Fasheh } 5691a40e23bSZheng Yan 57092a7cc42SQu Wenruo if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) { 57183d4cfd4SJosef Bacik ret = btrfs_reloc_cow_block(trans, root, buf, cow); 57293314e3bSZhaolei if (ret) { 573572c83acSJosef Bacik btrfs_tree_unlock(cow); 574572c83acSJosef Bacik free_extent_buffer(cow); 57566642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 57683d4cfd4SJosef Bacik return ret; 57783d4cfd4SJosef Bacik } 57893314e3bSZhaolei } 5793fd0a558SYan, Zheng 5806702ed49SChris Mason if (buf == root->node) { 581925baeddSChris Mason WARN_ON(parent && parent != buf); 5825d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 5835d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 5845d4f98a2SYan Zheng parent_start = buf->start; 585925baeddSChris Mason 58667439dadSDavid Sterba atomic_inc(&cow->refs); 587406808abSFilipe Manana ret = btrfs_tree_mod_log_insert_root(root->node, cow, true); 588d9d19a01SDavid Sterba BUG_ON(ret < 0); 589240f62c8SChris Mason rcu_assign_pointer(root->node, cow); 590925baeddSChris Mason 5917a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), buf, 5927a163608SFilipe Manana parent_start, last_ref); 5935f39d397SChris Mason free_extent_buffer(buf); 5940b86a832SChris Mason add_root_to_dirty_list(root); 5956702ed49SChris Mason } else { 5965d4f98a2SYan Zheng WARN_ON(trans->transid != btrfs_header_generation(parent)); 597f3a84ccdSFilipe Manana btrfs_tree_mod_log_insert_key(parent, parent_slot, 59833cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 5995f39d397SChris Mason btrfs_set_node_blockptr(parent, parent_slot, 600db94535dSChris Mason cow->start); 60174493f7aSChris Mason btrfs_set_node_ptr_generation(parent, parent_slot, 60274493f7aSChris Mason trans->transid); 6036702ed49SChris Mason btrfs_mark_buffer_dirty(parent); 6045de865eeSFilipe David Borba Manana if (last_ref) { 605f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_free_eb(buf); 6065de865eeSFilipe David Borba Manana if (ret) { 607572c83acSJosef Bacik btrfs_tree_unlock(cow); 608572c83acSJosef Bacik free_extent_buffer(cow); 60966642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 6105de865eeSFilipe David Borba Manana return ret; 6115de865eeSFilipe David Borba Manana } 6125de865eeSFilipe David Borba Manana } 6137a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), buf, 6147a163608SFilipe Manana parent_start, last_ref); 6156702ed49SChris Mason } 616925baeddSChris Mason if (unlock_orig) 617925baeddSChris Mason btrfs_tree_unlock(buf); 6183083ee2eSJosef Bacik free_extent_buffer_stale(buf); 6196702ed49SChris Mason btrfs_mark_buffer_dirty(cow); 6206702ed49SChris Mason *cow_ret = cow; 6216702ed49SChris Mason return 0; 6226702ed49SChris Mason } 6236702ed49SChris Mason 6245d4f98a2SYan Zheng static inline int should_cow_block(struct btrfs_trans_handle *trans, 6255d4f98a2SYan Zheng struct btrfs_root *root, 6265d4f98a2SYan Zheng struct extent_buffer *buf) 6275d4f98a2SYan Zheng { 628f5ee5c9aSJeff Mahoney if (btrfs_is_testing(root->fs_info)) 629faa2dbf0SJosef Bacik return 0; 630fccb84c9SDavid Sterba 631d1980131SDavid Sterba /* Ensure we can see the FORCE_COW bit */ 632d1980131SDavid Sterba smp_mb__before_atomic(); 633f1ebcc74SLiu Bo 634f1ebcc74SLiu Bo /* 635f1ebcc74SLiu Bo * We do not need to cow a block if 636f1ebcc74SLiu Bo * 1) this block is not created or changed in this transaction; 637f1ebcc74SLiu Bo * 2) this block does not belong to TREE_RELOC tree; 638f1ebcc74SLiu Bo * 3) the root is not forced COW. 639f1ebcc74SLiu Bo * 640f1ebcc74SLiu Bo * What is forced COW: 64101327610SNicholas D Steeves * when we create snapshot during committing the transaction, 64252042d8eSAndrea Gelmini * after we've finished copying src root, we must COW the shared 643f1ebcc74SLiu Bo * block to ensure the metadata consistency. 644f1ebcc74SLiu Bo */ 6455d4f98a2SYan Zheng if (btrfs_header_generation(buf) == trans->transid && 6465d4f98a2SYan Zheng !btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN) && 6475d4f98a2SYan Zheng !(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID && 648f1ebcc74SLiu Bo btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)) && 64927cdeb70SMiao Xie !test_bit(BTRFS_ROOT_FORCE_COW, &root->state)) 6505d4f98a2SYan Zheng return 0; 6515d4f98a2SYan Zheng return 1; 6525d4f98a2SYan Zheng } 6535d4f98a2SYan Zheng 654d352ac68SChris Mason /* 655d352ac68SChris Mason * cows a single block, see __btrfs_cow_block for the real work. 65601327610SNicholas D Steeves * This version of it has extra checks so that a block isn't COWed more than 657d352ac68SChris Mason * once per transaction, as long as it hasn't been written yet 658d352ac68SChris Mason */ 659d397712bSChris Mason noinline int btrfs_cow_block(struct btrfs_trans_handle *trans, 6605f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *buf, 6615f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 6629631e4ccSJosef Bacik struct extent_buffer **cow_ret, 6639631e4ccSJosef Bacik enum btrfs_lock_nesting nest) 66402217ed2SChris Mason { 6650b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 6666702ed49SChris Mason u64 search_start; 667f510cfecSChris Mason int ret; 668dc17ff8fSChris Mason 66983354f07SJosef Bacik if (test_bit(BTRFS_ROOT_DELETING, &root->state)) 67083354f07SJosef Bacik btrfs_err(fs_info, 67183354f07SJosef Bacik "COW'ing blocks on a fs root that's being dropped"); 67283354f07SJosef Bacik 6730b246afaSJeff Mahoney if (trans->transaction != fs_info->running_transaction) 67431b1a2bdSJulia Lawall WARN(1, KERN_CRIT "trans %llu running %llu\n", 675c1c9ff7cSGeert Uytterhoeven trans->transid, 6760b246afaSJeff Mahoney fs_info->running_transaction->transid); 67731b1a2bdSJulia Lawall 6780b246afaSJeff Mahoney if (trans->transid != fs_info->generation) 67931b1a2bdSJulia Lawall WARN(1, KERN_CRIT "trans %llu running %llu\n", 6800b246afaSJeff Mahoney trans->transid, fs_info->generation); 681dc17ff8fSChris Mason 6825d4f98a2SYan Zheng if (!should_cow_block(trans, root, buf)) { 68302217ed2SChris Mason *cow_ret = buf; 68402217ed2SChris Mason return 0; 68502217ed2SChris Mason } 686c487685dSChris Mason 687ee22184bSByongho Lee search_start = buf->start & ~((u64)SZ_1G - 1); 688b4ce94deSChris Mason 689f616f5cdSQu Wenruo /* 690f616f5cdSQu Wenruo * Before CoWing this block for later modification, check if it's 691f616f5cdSQu Wenruo * the subtree root and do the delayed subtree trace if needed. 692f616f5cdSQu Wenruo * 693f616f5cdSQu Wenruo * Also We don't care about the error, as it's handled internally. 694f616f5cdSQu Wenruo */ 695f616f5cdSQu Wenruo btrfs_qgroup_trace_subtree_after_cow(trans, root, buf); 696f510cfecSChris Mason ret = __btrfs_cow_block(trans, root, buf, parent, 6979631e4ccSJosef Bacik parent_slot, cow_ret, search_start, 0, nest); 6981abe9b8aSliubo 6991abe9b8aSliubo trace_btrfs_cow_block(root, buf, *cow_ret); 7001abe9b8aSliubo 701f510cfecSChris Mason return ret; 7022c90e5d6SChris Mason } 703f75e2b79SJosef Bacik ALLOW_ERROR_INJECTION(btrfs_cow_block, ERRNO); 7046702ed49SChris Mason 705d352ac68SChris Mason /* 706d352ac68SChris Mason * helper function for defrag to decide if two blocks pointed to by a 707d352ac68SChris Mason * node are actually close by 708d352ac68SChris Mason */ 7096b80053dSChris Mason static int close_blocks(u64 blocknr, u64 other, u32 blocksize) 7106702ed49SChris Mason { 7116b80053dSChris Mason if (blocknr < other && other - (blocknr + blocksize) < 32768) 7126702ed49SChris Mason return 1; 7136b80053dSChris Mason if (blocknr > other && blocknr - (other + blocksize) < 32768) 7146702ed49SChris Mason return 1; 71502217ed2SChris Mason return 0; 71602217ed2SChris Mason } 71702217ed2SChris Mason 718ce6ef5abSDavid Sterba #ifdef __LITTLE_ENDIAN 719ce6ef5abSDavid Sterba 720ce6ef5abSDavid Sterba /* 721ce6ef5abSDavid Sterba * Compare two keys, on little-endian the disk order is same as CPU order and 722ce6ef5abSDavid Sterba * we can avoid the conversion. 723ce6ef5abSDavid Sterba */ 724ce6ef5abSDavid Sterba static int comp_keys(const struct btrfs_disk_key *disk_key, 725ce6ef5abSDavid Sterba const struct btrfs_key *k2) 726ce6ef5abSDavid Sterba { 727ce6ef5abSDavid Sterba const struct btrfs_key *k1 = (const struct btrfs_key *)disk_key; 728ce6ef5abSDavid Sterba 729ce6ef5abSDavid Sterba return btrfs_comp_cpu_keys(k1, k2); 730ce6ef5abSDavid Sterba } 731ce6ef5abSDavid Sterba 732ce6ef5abSDavid Sterba #else 733ce6ef5abSDavid Sterba 734081e9573SChris Mason /* 735081e9573SChris Mason * compare two keys in a memcmp fashion 736081e9573SChris Mason */ 737310712b2SOmar Sandoval static int comp_keys(const struct btrfs_disk_key *disk, 738310712b2SOmar Sandoval const struct btrfs_key *k2) 739081e9573SChris Mason { 740081e9573SChris Mason struct btrfs_key k1; 741081e9573SChris Mason 742081e9573SChris Mason btrfs_disk_key_to_cpu(&k1, disk); 743081e9573SChris Mason 74420736abaSDiego Calleja return btrfs_comp_cpu_keys(&k1, k2); 745081e9573SChris Mason } 746ce6ef5abSDavid Sterba #endif 747081e9573SChris Mason 748f3465ca4SJosef Bacik /* 749f3465ca4SJosef Bacik * same as comp_keys only with two btrfs_key's 750f3465ca4SJosef Bacik */ 751e1f60a65SDavid Sterba int __pure btrfs_comp_cpu_keys(const struct btrfs_key *k1, const struct btrfs_key *k2) 752f3465ca4SJosef Bacik { 753f3465ca4SJosef Bacik if (k1->objectid > k2->objectid) 754f3465ca4SJosef Bacik return 1; 755f3465ca4SJosef Bacik if (k1->objectid < k2->objectid) 756f3465ca4SJosef Bacik return -1; 757f3465ca4SJosef Bacik if (k1->type > k2->type) 758f3465ca4SJosef Bacik return 1; 759f3465ca4SJosef Bacik if (k1->type < k2->type) 760f3465ca4SJosef Bacik return -1; 761f3465ca4SJosef Bacik if (k1->offset > k2->offset) 762f3465ca4SJosef Bacik return 1; 763f3465ca4SJosef Bacik if (k1->offset < k2->offset) 764f3465ca4SJosef Bacik return -1; 765f3465ca4SJosef Bacik return 0; 766f3465ca4SJosef Bacik } 767081e9573SChris Mason 768d352ac68SChris Mason /* 769d352ac68SChris Mason * this is used by the defrag code to go through all the 770d352ac68SChris Mason * leaves pointed to by a node and reallocate them so that 771d352ac68SChris Mason * disk order is close to key order 772d352ac68SChris Mason */ 7736702ed49SChris Mason int btrfs_realloc_node(struct btrfs_trans_handle *trans, 7745f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *parent, 775de78b51aSEric Sandeen int start_slot, u64 *last_ret, 776a6b6e75eSChris Mason struct btrfs_key *progress) 7776702ed49SChris Mason { 7780b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 7796b80053dSChris Mason struct extent_buffer *cur; 7806702ed49SChris Mason u64 blocknr; 781e9d0b13bSChris Mason u64 search_start = *last_ret; 782e9d0b13bSChris Mason u64 last_block = 0; 7836702ed49SChris Mason u64 other; 7846702ed49SChris Mason u32 parent_nritems; 7856702ed49SChris Mason int end_slot; 7866702ed49SChris Mason int i; 7876702ed49SChris Mason int err = 0; 7886b80053dSChris Mason u32 blocksize; 789081e9573SChris Mason int progress_passed = 0; 790081e9573SChris Mason struct btrfs_disk_key disk_key; 7916702ed49SChris Mason 7920b246afaSJeff Mahoney WARN_ON(trans->transaction != fs_info->running_transaction); 7930b246afaSJeff Mahoney WARN_ON(trans->transid != fs_info->generation); 79486479a04SChris Mason 7956b80053dSChris Mason parent_nritems = btrfs_header_nritems(parent); 7960b246afaSJeff Mahoney blocksize = fs_info->nodesize; 7975dfe2be7SFilipe Manana end_slot = parent_nritems - 1; 7986702ed49SChris Mason 7995dfe2be7SFilipe Manana if (parent_nritems <= 1) 8006702ed49SChris Mason return 0; 8016702ed49SChris Mason 8025dfe2be7SFilipe Manana for (i = start_slot; i <= end_slot; i++) { 8036702ed49SChris Mason int close = 1; 804a6b6e75eSChris Mason 805081e9573SChris Mason btrfs_node_key(parent, &disk_key, i); 806081e9573SChris Mason if (!progress_passed && comp_keys(&disk_key, progress) < 0) 807081e9573SChris Mason continue; 808081e9573SChris Mason 809081e9573SChris Mason progress_passed = 1; 8106b80053dSChris Mason blocknr = btrfs_node_blockptr(parent, i); 811e9d0b13bSChris Mason if (last_block == 0) 812e9d0b13bSChris Mason last_block = blocknr; 8135708b959SChris Mason 8146702ed49SChris Mason if (i > 0) { 8156b80053dSChris Mason other = btrfs_node_blockptr(parent, i - 1); 8166b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 8176702ed49SChris Mason } 8185dfe2be7SFilipe Manana if (!close && i < end_slot) { 8196b80053dSChris Mason other = btrfs_node_blockptr(parent, i + 1); 8206b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 8216702ed49SChris Mason } 822e9d0b13bSChris Mason if (close) { 823e9d0b13bSChris Mason last_block = blocknr; 8246702ed49SChris Mason continue; 825e9d0b13bSChris Mason } 8266702ed49SChris Mason 827206983b7SJosef Bacik cur = btrfs_read_node_slot(parent, i); 828206983b7SJosef Bacik if (IS_ERR(cur)) 82964c043deSLiu Bo return PTR_ERR(cur); 830e9d0b13bSChris Mason if (search_start == 0) 8316b80053dSChris Mason search_start = last_block; 832e9d0b13bSChris Mason 833e7a84565SChris Mason btrfs_tree_lock(cur); 8346b80053dSChris Mason err = __btrfs_cow_block(trans, root, cur, parent, i, 835e7a84565SChris Mason &cur, search_start, 8366b80053dSChris Mason min(16 * blocksize, 8379631e4ccSJosef Bacik (end_slot - i) * blocksize), 8389631e4ccSJosef Bacik BTRFS_NESTING_COW); 839252c38f0SYan if (err) { 840e7a84565SChris Mason btrfs_tree_unlock(cur); 8416b80053dSChris Mason free_extent_buffer(cur); 8426702ed49SChris Mason break; 843252c38f0SYan } 844e7a84565SChris Mason search_start = cur->start; 845e7a84565SChris Mason last_block = cur->start; 846f2183bdeSChris Mason *last_ret = search_start; 847e7a84565SChris Mason btrfs_tree_unlock(cur); 848e7a84565SChris Mason free_extent_buffer(cur); 8496702ed49SChris Mason } 8506702ed49SChris Mason return err; 8516702ed49SChris Mason } 8526702ed49SChris Mason 85374123bd7SChris Mason /* 854fb81212cSFilipe Manana * Search for a key in the given extent_buffer. 8555f39d397SChris Mason * 856a724f313SFilipe Manana * The lower boundary for the search is specified by the slot number @first_slot. 857a724f313SFilipe Manana * Use a value of 0 to search over the whole extent buffer. 85874123bd7SChris Mason * 859fb81212cSFilipe Manana * The slot in the extent buffer is returned via @slot. If the key exists in the 860fb81212cSFilipe Manana * extent buffer, then @slot will point to the slot where the key is, otherwise 861fb81212cSFilipe Manana * it points to the slot where you would insert the key. 862fb81212cSFilipe Manana * 863fb81212cSFilipe Manana * Slot may point to the total number of items (i.e. one position beyond the last 864fb81212cSFilipe Manana * key) if the key is bigger than the last key in the extent buffer. 86574123bd7SChris Mason */ 866a724f313SFilipe Manana int btrfs_generic_bin_search(struct extent_buffer *eb, int first_slot, 86767d5e289SMarcos Paulo de Souza const struct btrfs_key *key, int *slot) 868be0e5c09SChris Mason { 869fb81212cSFilipe Manana unsigned long p; 870fb81212cSFilipe Manana int item_size; 871a724f313SFilipe Manana /* 872a724f313SFilipe Manana * Use unsigned types for the low and high slots, so that we get a more 873a724f313SFilipe Manana * efficient division in the search loop below. 874a724f313SFilipe Manana */ 875a724f313SFilipe Manana u32 low = first_slot; 876a724f313SFilipe Manana u32 high = btrfs_header_nritems(eb); 877be0e5c09SChris Mason int ret; 8785cd17f34SDavid Sterba const int key_size = sizeof(struct btrfs_disk_key); 879be0e5c09SChris Mason 880a724f313SFilipe Manana if (unlikely(low > high)) { 8815e24e9afSLiu Bo btrfs_err(eb->fs_info, 882a724f313SFilipe Manana "%s: low (%u) > high (%u) eb %llu owner %llu level %d", 8835e24e9afSLiu Bo __func__, low, high, eb->start, 8845e24e9afSLiu Bo btrfs_header_owner(eb), btrfs_header_level(eb)); 8855e24e9afSLiu Bo return -EINVAL; 8865e24e9afSLiu Bo } 8875e24e9afSLiu Bo 888fb81212cSFilipe Manana if (btrfs_header_level(eb) == 0) { 889fb81212cSFilipe Manana p = offsetof(struct btrfs_leaf, items); 890fb81212cSFilipe Manana item_size = sizeof(struct btrfs_item); 891fb81212cSFilipe Manana } else { 892fb81212cSFilipe Manana p = offsetof(struct btrfs_node, ptrs); 893fb81212cSFilipe Manana item_size = sizeof(struct btrfs_key_ptr); 894fb81212cSFilipe Manana } 895fb81212cSFilipe Manana 896be0e5c09SChris Mason while (low < high) { 8975cd17f34SDavid Sterba unsigned long oip; 8985cd17f34SDavid Sterba unsigned long offset; 8995cd17f34SDavid Sterba struct btrfs_disk_key *tmp; 9005cd17f34SDavid Sterba struct btrfs_disk_key unaligned; 9015cd17f34SDavid Sterba int mid; 9025cd17f34SDavid Sterba 903be0e5c09SChris Mason mid = (low + high) / 2; 9045f39d397SChris Mason offset = p + mid * item_size; 9055cd17f34SDavid Sterba oip = offset_in_page(offset); 9065f39d397SChris Mason 9075cd17f34SDavid Sterba if (oip + key_size <= PAGE_SIZE) { 908884b07d0SQu Wenruo const unsigned long idx = get_eb_page_index(offset); 9095cd17f34SDavid Sterba char *kaddr = page_address(eb->pages[idx]); 910934d375bSChris Mason 911884b07d0SQu Wenruo oip = get_eb_offset_in_page(eb, offset); 9125cd17f34SDavid Sterba tmp = (struct btrfs_disk_key *)(kaddr + oip); 9135cd17f34SDavid Sterba } else { 9145cd17f34SDavid Sterba read_extent_buffer(eb, &unaligned, offset, key_size); 9155f39d397SChris Mason tmp = &unaligned; 916479965d6SChris Mason } 917479965d6SChris Mason 918be0e5c09SChris Mason ret = comp_keys(tmp, key); 919be0e5c09SChris Mason 920be0e5c09SChris Mason if (ret < 0) 921be0e5c09SChris Mason low = mid + 1; 922be0e5c09SChris Mason else if (ret > 0) 923be0e5c09SChris Mason high = mid; 924be0e5c09SChris Mason else { 925be0e5c09SChris Mason *slot = mid; 926be0e5c09SChris Mason return 0; 927be0e5c09SChris Mason } 928be0e5c09SChris Mason } 929be0e5c09SChris Mason *slot = low; 930be0e5c09SChris Mason return 1; 931be0e5c09SChris Mason } 932be0e5c09SChris Mason 933f0486c68SYan, Zheng static void root_add_used(struct btrfs_root *root, u32 size) 934f0486c68SYan, Zheng { 935f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 936f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 937f0486c68SYan, Zheng btrfs_root_used(&root->root_item) + size); 938f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 939f0486c68SYan, Zheng } 940f0486c68SYan, Zheng 941f0486c68SYan, Zheng static void root_sub_used(struct btrfs_root *root, u32 size) 942f0486c68SYan, Zheng { 943f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 944f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 945f0486c68SYan, Zheng btrfs_root_used(&root->root_item) - size); 946f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 947f0486c68SYan, Zheng } 948f0486c68SYan, Zheng 949d352ac68SChris Mason /* given a node and slot number, this reads the blocks it points to. The 950d352ac68SChris Mason * extent buffer is returned with a reference taken (but unlocked). 951d352ac68SChris Mason */ 9524b231ae4SDavid Sterba struct extent_buffer *btrfs_read_node_slot(struct extent_buffer *parent, 9534b231ae4SDavid Sterba int slot) 954bb803951SChris Mason { 955ca7a79adSChris Mason int level = btrfs_header_level(parent); 956789d6a3aSQu Wenruo struct btrfs_tree_parent_check check = { 0 }; 957416bc658SJosef Bacik struct extent_buffer *eb; 958416bc658SJosef Bacik 959fb770ae4SLiu Bo if (slot < 0 || slot >= btrfs_header_nritems(parent)) 960fb770ae4SLiu Bo return ERR_PTR(-ENOENT); 961ca7a79adSChris Mason 962d4694728SJosef Bacik ASSERT(level); 963ca7a79adSChris Mason 964789d6a3aSQu Wenruo check.level = level - 1; 965789d6a3aSQu Wenruo check.transid = btrfs_node_ptr_generation(parent, slot); 966789d6a3aSQu Wenruo check.owner_root = btrfs_header_owner(parent); 967789d6a3aSQu Wenruo check.has_first_key = true; 968789d6a3aSQu Wenruo btrfs_node_key_to_cpu(parent, &check.first_key, slot); 969789d6a3aSQu Wenruo 970d0d20b0fSDavid Sterba eb = read_tree_block(parent->fs_info, btrfs_node_blockptr(parent, slot), 971789d6a3aSQu Wenruo &check); 9724eb150d6SQu Wenruo if (IS_ERR(eb)) 9734eb150d6SQu Wenruo return eb; 9744eb150d6SQu Wenruo if (!extent_buffer_uptodate(eb)) { 975416bc658SJosef Bacik free_extent_buffer(eb); 9764eb150d6SQu Wenruo return ERR_PTR(-EIO); 977416bc658SJosef Bacik } 978416bc658SJosef Bacik 979416bc658SJosef Bacik return eb; 980bb803951SChris Mason } 981bb803951SChris Mason 982d352ac68SChris Mason /* 983d352ac68SChris Mason * node level balancing, used to make sure nodes are in proper order for 984d352ac68SChris Mason * item deletion. We balance from the top down, so we have to make sure 985d352ac68SChris Mason * that a deletion won't leave an node completely empty later on. 986d352ac68SChris Mason */ 987e02119d5SChris Mason static noinline int balance_level(struct btrfs_trans_handle *trans, 98898ed5174SChris Mason struct btrfs_root *root, 98998ed5174SChris Mason struct btrfs_path *path, int level) 990bb803951SChris Mason { 9910b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 9925f39d397SChris Mason struct extent_buffer *right = NULL; 9935f39d397SChris Mason struct extent_buffer *mid; 9945f39d397SChris Mason struct extent_buffer *left = NULL; 9955f39d397SChris Mason struct extent_buffer *parent = NULL; 996bb803951SChris Mason int ret = 0; 997bb803951SChris Mason int wret; 998bb803951SChris Mason int pslot; 999bb803951SChris Mason int orig_slot = path->slots[level]; 100079f95c82SChris Mason u64 orig_ptr; 1001bb803951SChris Mason 100298e6b1ebSLiu Bo ASSERT(level > 0); 1003bb803951SChris Mason 10045f39d397SChris Mason mid = path->nodes[level]; 1005b4ce94deSChris Mason 1006ac5887c8SJosef Bacik WARN_ON(path->locks[level] != BTRFS_WRITE_LOCK); 10077bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 10087bb86316SChris Mason 10091d4f8a0cSChris Mason orig_ptr = btrfs_node_blockptr(mid, orig_slot); 101079f95c82SChris Mason 1011a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 10125f39d397SChris Mason parent = path->nodes[level + 1]; 1013bb803951SChris Mason pslot = path->slots[level + 1]; 1014a05a9bb1SLi Zefan } 1015bb803951SChris Mason 101640689478SChris Mason /* 101740689478SChris Mason * deal with the case where there is only one pointer in the root 101840689478SChris Mason * by promoting the node below to a root 101940689478SChris Mason */ 10205f39d397SChris Mason if (!parent) { 10215f39d397SChris Mason struct extent_buffer *child; 1022bb803951SChris Mason 10235f39d397SChris Mason if (btrfs_header_nritems(mid) != 1) 1024bb803951SChris Mason return 0; 1025bb803951SChris Mason 1026bb803951SChris Mason /* promote the child to a root */ 10274b231ae4SDavid Sterba child = btrfs_read_node_slot(mid, 0); 1028fb770ae4SLiu Bo if (IS_ERR(child)) { 1029fb770ae4SLiu Bo ret = PTR_ERR(child); 10300b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 1031305a26afSMark Fasheh goto enospc; 1032305a26afSMark Fasheh } 1033305a26afSMark Fasheh 1034925baeddSChris Mason btrfs_tree_lock(child); 10359631e4ccSJosef Bacik ret = btrfs_cow_block(trans, root, child, mid, 0, &child, 10369631e4ccSJosef Bacik BTRFS_NESTING_COW); 1037f0486c68SYan, Zheng if (ret) { 1038f0486c68SYan, Zheng btrfs_tree_unlock(child); 1039f0486c68SYan, Zheng free_extent_buffer(child); 1040f0486c68SYan, Zheng goto enospc; 1041f0486c68SYan, Zheng } 10422f375ab9SYan 1043406808abSFilipe Manana ret = btrfs_tree_mod_log_insert_root(root->node, child, true); 1044d9d19a01SDavid Sterba BUG_ON(ret < 0); 1045240f62c8SChris Mason rcu_assign_pointer(root->node, child); 1046925baeddSChris Mason 10470b86a832SChris Mason add_root_to_dirty_list(root); 1048925baeddSChris Mason btrfs_tree_unlock(child); 1049b4ce94deSChris Mason 1050925baeddSChris Mason path->locks[level] = 0; 1051bb803951SChris Mason path->nodes[level] = NULL; 1052190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, mid); 1053925baeddSChris Mason btrfs_tree_unlock(mid); 1054bb803951SChris Mason /* once for the path */ 10555f39d397SChris Mason free_extent_buffer(mid); 1056f0486c68SYan, Zheng 1057f0486c68SYan, Zheng root_sub_used(root, mid->len); 10587a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), mid, 0, 1); 1059bb803951SChris Mason /* once for the root ptr */ 10603083ee2eSJosef Bacik free_extent_buffer_stale(mid); 1061f0486c68SYan, Zheng return 0; 1062bb803951SChris Mason } 10635f39d397SChris Mason if (btrfs_header_nritems(mid) > 10640b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 4) 1065bb803951SChris Mason return 0; 1066bb803951SChris Mason 1067*9cf14029SJosef Bacik if (pslot) { 10684b231ae4SDavid Sterba left = btrfs_read_node_slot(parent, pslot - 1); 1069*9cf14029SJosef Bacik if (IS_ERR(left)) { 1070*9cf14029SJosef Bacik ret = PTR_ERR(left); 1071fb770ae4SLiu Bo left = NULL; 1072*9cf14029SJosef Bacik goto enospc; 1073*9cf14029SJosef Bacik } 1074fb770ae4SLiu Bo 1075bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT); 10765f39d397SChris Mason wret = btrfs_cow_block(trans, root, left, 10779631e4ccSJosef Bacik parent, pslot - 1, &left, 1078bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW); 107954aa1f4dSChris Mason if (wret) { 108054aa1f4dSChris Mason ret = wret; 108154aa1f4dSChris Mason goto enospc; 108254aa1f4dSChris Mason } 10832cc58cf2SChris Mason } 1084fb770ae4SLiu Bo 1085*9cf14029SJosef Bacik if (pslot + 1 < btrfs_header_nritems(parent)) { 10864b231ae4SDavid Sterba right = btrfs_read_node_slot(parent, pslot + 1); 1087*9cf14029SJosef Bacik if (IS_ERR(right)) { 1088*9cf14029SJosef Bacik ret = PTR_ERR(right); 1089fb770ae4SLiu Bo right = NULL; 1090*9cf14029SJosef Bacik goto enospc; 1091*9cf14029SJosef Bacik } 1092fb770ae4SLiu Bo 1093bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT); 10945f39d397SChris Mason wret = btrfs_cow_block(trans, root, right, 10959631e4ccSJosef Bacik parent, pslot + 1, &right, 1096bf59a5a2SJosef Bacik BTRFS_NESTING_RIGHT_COW); 10972cc58cf2SChris Mason if (wret) { 10982cc58cf2SChris Mason ret = wret; 10992cc58cf2SChris Mason goto enospc; 11002cc58cf2SChris Mason } 11012cc58cf2SChris Mason } 11022cc58cf2SChris Mason 11032cc58cf2SChris Mason /* first, try to make some room in the middle buffer */ 11045f39d397SChris Mason if (left) { 11055f39d397SChris Mason orig_slot += btrfs_header_nritems(left); 1106d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 1); 110779f95c82SChris Mason if (wret < 0) 110879f95c82SChris Mason ret = wret; 1109bb803951SChris Mason } 111079f95c82SChris Mason 111179f95c82SChris Mason /* 111279f95c82SChris Mason * then try to empty the right most buffer into the middle 111379f95c82SChris Mason */ 11145f39d397SChris Mason if (right) { 1115d30a668fSDavid Sterba wret = push_node_left(trans, mid, right, 1); 111654aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 111779f95c82SChris Mason ret = wret; 11185f39d397SChris Mason if (btrfs_header_nritems(right) == 0) { 1119190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, right); 1120925baeddSChris Mason btrfs_tree_unlock(right); 1121afe5fea7STsutomu Itoh del_ptr(root, path, level + 1, pslot + 1); 1122f0486c68SYan, Zheng root_sub_used(root, right->len); 11237a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), right, 11247a163608SFilipe Manana 0, 1); 11253083ee2eSJosef Bacik free_extent_buffer_stale(right); 1126f0486c68SYan, Zheng right = NULL; 1127bb803951SChris Mason } else { 11285f39d397SChris Mason struct btrfs_disk_key right_key; 11295f39d397SChris Mason btrfs_node_key(right, &right_key, 0); 1130f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot + 1, 113133cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 11320e82bcfeSDavid Sterba BUG_ON(ret < 0); 11335f39d397SChris Mason btrfs_set_node_key(parent, &right_key, pslot + 1); 11345f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 1135bb803951SChris Mason } 1136bb803951SChris Mason } 11375f39d397SChris Mason if (btrfs_header_nritems(mid) == 1) { 113879f95c82SChris Mason /* 113979f95c82SChris Mason * we're not allowed to leave a node with one item in the 114079f95c82SChris Mason * tree during a delete. A deletion from lower in the tree 114179f95c82SChris Mason * could try to delete the only pointer in this node. 114279f95c82SChris Mason * So, pull some keys from the left. 114379f95c82SChris Mason * There has to be a left pointer at this point because 114479f95c82SChris Mason * otherwise we would have pulled some pointers from the 114579f95c82SChris Mason * right 114679f95c82SChris Mason */ 1147305a26afSMark Fasheh if (!left) { 1148305a26afSMark Fasheh ret = -EROFS; 11490b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 1150305a26afSMark Fasheh goto enospc; 1151305a26afSMark Fasheh } 115255d32ed8SDavid Sterba wret = balance_node_right(trans, mid, left); 115354aa1f4dSChris Mason if (wret < 0) { 115479f95c82SChris Mason ret = wret; 115554aa1f4dSChris Mason goto enospc; 115654aa1f4dSChris Mason } 1157bce4eae9SChris Mason if (wret == 1) { 1158d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 1); 1159bce4eae9SChris Mason if (wret < 0) 1160bce4eae9SChris Mason ret = wret; 1161bce4eae9SChris Mason } 116279f95c82SChris Mason BUG_ON(wret == 1); 116379f95c82SChris Mason } 11645f39d397SChris Mason if (btrfs_header_nritems(mid) == 0) { 1165190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, mid); 1166925baeddSChris Mason btrfs_tree_unlock(mid); 1167afe5fea7STsutomu Itoh del_ptr(root, path, level + 1, pslot); 1168f0486c68SYan, Zheng root_sub_used(root, mid->len); 11697a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), mid, 0, 1); 11703083ee2eSJosef Bacik free_extent_buffer_stale(mid); 1171f0486c68SYan, Zheng mid = NULL; 117279f95c82SChris Mason } else { 117379f95c82SChris Mason /* update the parent key to reflect our changes */ 11745f39d397SChris Mason struct btrfs_disk_key mid_key; 11755f39d397SChris Mason btrfs_node_key(mid, &mid_key, 0); 1176f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot, 117733cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 11780e82bcfeSDavid Sterba BUG_ON(ret < 0); 11795f39d397SChris Mason btrfs_set_node_key(parent, &mid_key, pslot); 11805f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 118179f95c82SChris Mason } 1182bb803951SChris Mason 118379f95c82SChris Mason /* update the path */ 11845f39d397SChris Mason if (left) { 11855f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 118667439dadSDavid Sterba atomic_inc(&left->refs); 1187925baeddSChris Mason /* left was locked after cow */ 11885f39d397SChris Mason path->nodes[level] = left; 1189bb803951SChris Mason path->slots[level + 1] -= 1; 1190bb803951SChris Mason path->slots[level] = orig_slot; 1191925baeddSChris Mason if (mid) { 1192925baeddSChris Mason btrfs_tree_unlock(mid); 11935f39d397SChris Mason free_extent_buffer(mid); 1194925baeddSChris Mason } 1195bb803951SChris Mason } else { 11965f39d397SChris Mason orig_slot -= btrfs_header_nritems(left); 1197bb803951SChris Mason path->slots[level] = orig_slot; 1198bb803951SChris Mason } 1199bb803951SChris Mason } 120079f95c82SChris Mason /* double check we haven't messed things up */ 1201e20d96d6SChris Mason if (orig_ptr != 12025f39d397SChris Mason btrfs_node_blockptr(path->nodes[level], path->slots[level])) 120379f95c82SChris Mason BUG(); 120454aa1f4dSChris Mason enospc: 1205925baeddSChris Mason if (right) { 1206925baeddSChris Mason btrfs_tree_unlock(right); 12075f39d397SChris Mason free_extent_buffer(right); 1208925baeddSChris Mason } 1209925baeddSChris Mason if (left) { 1210925baeddSChris Mason if (path->nodes[level] != left) 1211925baeddSChris Mason btrfs_tree_unlock(left); 12125f39d397SChris Mason free_extent_buffer(left); 1213925baeddSChris Mason } 1214bb803951SChris Mason return ret; 1215bb803951SChris Mason } 1216bb803951SChris Mason 1217d352ac68SChris Mason /* Node balancing for insertion. Here we only split or push nodes around 1218d352ac68SChris Mason * when they are completely full. This is also done top down, so we 1219d352ac68SChris Mason * have to be pessimistic. 1220d352ac68SChris Mason */ 1221d397712bSChris Mason static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans, 1222e66f709bSChris Mason struct btrfs_root *root, 1223e66f709bSChris Mason struct btrfs_path *path, int level) 1224e66f709bSChris Mason { 12250b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 12265f39d397SChris Mason struct extent_buffer *right = NULL; 12275f39d397SChris Mason struct extent_buffer *mid; 12285f39d397SChris Mason struct extent_buffer *left = NULL; 12295f39d397SChris Mason struct extent_buffer *parent = NULL; 1230e66f709bSChris Mason int ret = 0; 1231e66f709bSChris Mason int wret; 1232e66f709bSChris Mason int pslot; 1233e66f709bSChris Mason int orig_slot = path->slots[level]; 1234e66f709bSChris Mason 1235e66f709bSChris Mason if (level == 0) 1236e66f709bSChris Mason return 1; 1237e66f709bSChris Mason 12385f39d397SChris Mason mid = path->nodes[level]; 12397bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 1240e66f709bSChris Mason 1241a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 12425f39d397SChris Mason parent = path->nodes[level + 1]; 1243e66f709bSChris Mason pslot = path->slots[level + 1]; 1244a05a9bb1SLi Zefan } 1245e66f709bSChris Mason 12465f39d397SChris Mason if (!parent) 1247e66f709bSChris Mason return 1; 1248e66f709bSChris Mason 1249*9cf14029SJosef Bacik /* first, try to make some room in the middle buffer */ 1250*9cf14029SJosef Bacik if (pslot) { 1251*9cf14029SJosef Bacik u32 left_nr; 1252*9cf14029SJosef Bacik 12534b231ae4SDavid Sterba left = btrfs_read_node_slot(parent, pslot - 1); 1254fb770ae4SLiu Bo if (IS_ERR(left)) 1255*9cf14029SJosef Bacik return PTR_ERR(left); 1256925baeddSChris Mason 1257bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT); 1258b4ce94deSChris Mason 12595f39d397SChris Mason left_nr = btrfs_header_nritems(left); 12600b246afaSJeff Mahoney if (left_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) { 126133ade1f8SChris Mason wret = 1; 126233ade1f8SChris Mason } else { 12635f39d397SChris Mason ret = btrfs_cow_block(trans, root, left, parent, 12649631e4ccSJosef Bacik pslot - 1, &left, 1265bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW); 126654aa1f4dSChris Mason if (ret) 126754aa1f4dSChris Mason wret = 1; 126854aa1f4dSChris Mason else { 1269d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 0); 127054aa1f4dSChris Mason } 127133ade1f8SChris Mason } 1272e66f709bSChris Mason if (wret < 0) 1273e66f709bSChris Mason ret = wret; 1274e66f709bSChris Mason if (wret == 0) { 12755f39d397SChris Mason struct btrfs_disk_key disk_key; 1276e66f709bSChris Mason orig_slot += left_nr; 12775f39d397SChris Mason btrfs_node_key(mid, &disk_key, 0); 1278f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot, 127933cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 12800e82bcfeSDavid Sterba BUG_ON(ret < 0); 12815f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot); 12825f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 12835f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 12845f39d397SChris Mason path->nodes[level] = left; 1285e66f709bSChris Mason path->slots[level + 1] -= 1; 1286e66f709bSChris Mason path->slots[level] = orig_slot; 1287925baeddSChris Mason btrfs_tree_unlock(mid); 12885f39d397SChris Mason free_extent_buffer(mid); 1289e66f709bSChris Mason } else { 1290e66f709bSChris Mason orig_slot -= 12915f39d397SChris Mason btrfs_header_nritems(left); 1292e66f709bSChris Mason path->slots[level] = orig_slot; 1293925baeddSChris Mason btrfs_tree_unlock(left); 12945f39d397SChris Mason free_extent_buffer(left); 1295e66f709bSChris Mason } 1296e66f709bSChris Mason return 0; 1297e66f709bSChris Mason } 1298925baeddSChris Mason btrfs_tree_unlock(left); 12995f39d397SChris Mason free_extent_buffer(left); 1300e66f709bSChris Mason } 1301e66f709bSChris Mason 1302e66f709bSChris Mason /* 1303e66f709bSChris Mason * then try to empty the right most buffer into the middle 1304e66f709bSChris Mason */ 1305*9cf14029SJosef Bacik if (pslot + 1 < btrfs_header_nritems(parent)) { 130633ade1f8SChris Mason u32 right_nr; 1307b4ce94deSChris Mason 1308*9cf14029SJosef Bacik right = btrfs_read_node_slot(parent, pslot + 1); 1309*9cf14029SJosef Bacik if (IS_ERR(right)) 1310*9cf14029SJosef Bacik return PTR_ERR(right); 1311*9cf14029SJosef Bacik 1312bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT); 1313b4ce94deSChris Mason 13145f39d397SChris Mason right_nr = btrfs_header_nritems(right); 13150b246afaSJeff Mahoney if (right_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) { 131633ade1f8SChris Mason wret = 1; 131733ade1f8SChris Mason } else { 13185f39d397SChris Mason ret = btrfs_cow_block(trans, root, right, 13195f39d397SChris Mason parent, pslot + 1, 1320bf59a5a2SJosef Bacik &right, BTRFS_NESTING_RIGHT_COW); 132154aa1f4dSChris Mason if (ret) 132254aa1f4dSChris Mason wret = 1; 132354aa1f4dSChris Mason else { 132455d32ed8SDavid Sterba wret = balance_node_right(trans, right, mid); 132533ade1f8SChris Mason } 132654aa1f4dSChris Mason } 1327e66f709bSChris Mason if (wret < 0) 1328e66f709bSChris Mason ret = wret; 1329e66f709bSChris Mason if (wret == 0) { 13305f39d397SChris Mason struct btrfs_disk_key disk_key; 13315f39d397SChris Mason 13325f39d397SChris Mason btrfs_node_key(right, &disk_key, 0); 1333f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot + 1, 133433cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 13350e82bcfeSDavid Sterba BUG_ON(ret < 0); 13365f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot + 1); 13375f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 13385f39d397SChris Mason 13395f39d397SChris Mason if (btrfs_header_nritems(mid) <= orig_slot) { 13405f39d397SChris Mason path->nodes[level] = right; 1341e66f709bSChris Mason path->slots[level + 1] += 1; 1342e66f709bSChris Mason path->slots[level] = orig_slot - 13435f39d397SChris Mason btrfs_header_nritems(mid); 1344925baeddSChris Mason btrfs_tree_unlock(mid); 13455f39d397SChris Mason free_extent_buffer(mid); 1346e66f709bSChris Mason } else { 1347925baeddSChris Mason btrfs_tree_unlock(right); 13485f39d397SChris Mason free_extent_buffer(right); 1349e66f709bSChris Mason } 1350e66f709bSChris Mason return 0; 1351e66f709bSChris Mason } 1352925baeddSChris Mason btrfs_tree_unlock(right); 13535f39d397SChris Mason free_extent_buffer(right); 1354e66f709bSChris Mason } 1355e66f709bSChris Mason return 1; 1356e66f709bSChris Mason } 1357e66f709bSChris Mason 135874123bd7SChris Mason /* 1359d352ac68SChris Mason * readahead one full node of leaves, finding things that are close 1360d352ac68SChris Mason * to the block in 'slot', and triggering ra on them. 13613c69faecSChris Mason */ 13622ff7e61eSJeff Mahoney static void reada_for_search(struct btrfs_fs_info *fs_info, 1363e02119d5SChris Mason struct btrfs_path *path, 136401f46658SChris Mason int level, int slot, u64 objectid) 13653c69faecSChris Mason { 13665f39d397SChris Mason struct extent_buffer *node; 136701f46658SChris Mason struct btrfs_disk_key disk_key; 13683c69faecSChris Mason u32 nritems; 13693c69faecSChris Mason u64 search; 1370a7175319SChris Mason u64 target; 13716b80053dSChris Mason u64 nread = 0; 1372ace75066SFilipe Manana u64 nread_max; 13736b80053dSChris Mason u32 nr; 13746b80053dSChris Mason u32 blocksize; 13756b80053dSChris Mason u32 nscan = 0; 1376db94535dSChris Mason 1377ace75066SFilipe Manana if (level != 1 && path->reada != READA_FORWARD_ALWAYS) 13783c69faecSChris Mason return; 13793c69faecSChris Mason 13806702ed49SChris Mason if (!path->nodes[level]) 13816702ed49SChris Mason return; 13826702ed49SChris Mason 13835f39d397SChris Mason node = path->nodes[level]; 1384925baeddSChris Mason 1385ace75066SFilipe Manana /* 1386ace75066SFilipe Manana * Since the time between visiting leaves is much shorter than the time 1387ace75066SFilipe Manana * between visiting nodes, limit read ahead of nodes to 1, to avoid too 1388ace75066SFilipe Manana * much IO at once (possibly random). 1389ace75066SFilipe Manana */ 1390ace75066SFilipe Manana if (path->reada == READA_FORWARD_ALWAYS) { 1391ace75066SFilipe Manana if (level > 1) 1392ace75066SFilipe Manana nread_max = node->fs_info->nodesize; 1393ace75066SFilipe Manana else 1394ace75066SFilipe Manana nread_max = SZ_128K; 1395ace75066SFilipe Manana } else { 1396ace75066SFilipe Manana nread_max = SZ_64K; 1397ace75066SFilipe Manana } 1398ace75066SFilipe Manana 13993c69faecSChris Mason search = btrfs_node_blockptr(node, slot); 14000b246afaSJeff Mahoney blocksize = fs_info->nodesize; 1401069a2e37SFilipe Manana if (path->reada != READA_FORWARD_ALWAYS) { 1402069a2e37SFilipe Manana struct extent_buffer *eb; 1403069a2e37SFilipe Manana 14040b246afaSJeff Mahoney eb = find_extent_buffer(fs_info, search); 14055f39d397SChris Mason if (eb) { 14065f39d397SChris Mason free_extent_buffer(eb); 14073c69faecSChris Mason return; 14083c69faecSChris Mason } 1409069a2e37SFilipe Manana } 14103c69faecSChris Mason 1411a7175319SChris Mason target = search; 14126b80053dSChris Mason 14135f39d397SChris Mason nritems = btrfs_header_nritems(node); 14146b80053dSChris Mason nr = slot; 141525b8b936SJosef Bacik 14163c69faecSChris Mason while (1) { 1417e4058b54SDavid Sterba if (path->reada == READA_BACK) { 14186b80053dSChris Mason if (nr == 0) 14193c69faecSChris Mason break; 14206b80053dSChris Mason nr--; 1421ace75066SFilipe Manana } else if (path->reada == READA_FORWARD || 1422ace75066SFilipe Manana path->reada == READA_FORWARD_ALWAYS) { 14236b80053dSChris Mason nr++; 14246b80053dSChris Mason if (nr >= nritems) 14256b80053dSChris Mason break; 14263c69faecSChris Mason } 1427e4058b54SDavid Sterba if (path->reada == READA_BACK && objectid) { 142801f46658SChris Mason btrfs_node_key(node, &disk_key, nr); 142901f46658SChris Mason if (btrfs_disk_key_objectid(&disk_key) != objectid) 143001f46658SChris Mason break; 143101f46658SChris Mason } 14326b80053dSChris Mason search = btrfs_node_blockptr(node, nr); 1433ace75066SFilipe Manana if (path->reada == READA_FORWARD_ALWAYS || 1434ace75066SFilipe Manana (search <= target && target - search <= 65536) || 1435a7175319SChris Mason (search > target && search - target <= 65536)) { 1436bfb484d9SJosef Bacik btrfs_readahead_node_child(node, nr); 14376b80053dSChris Mason nread += blocksize; 14383c69faecSChris Mason } 14396b80053dSChris Mason nscan++; 1440ace75066SFilipe Manana if (nread > nread_max || nscan > 32) 14416b80053dSChris Mason break; 14423c69faecSChris Mason } 14433c69faecSChris Mason } 1444925baeddSChris Mason 1445bfb484d9SJosef Bacik static noinline void reada_for_balance(struct btrfs_path *path, int level) 1446b4ce94deSChris Mason { 1447bfb484d9SJosef Bacik struct extent_buffer *parent; 1448b4ce94deSChris Mason int slot; 1449b4ce94deSChris Mason int nritems; 1450b4ce94deSChris Mason 14518c594ea8SChris Mason parent = path->nodes[level + 1]; 1452b4ce94deSChris Mason if (!parent) 14530b08851fSJosef Bacik return; 1454b4ce94deSChris Mason 1455b4ce94deSChris Mason nritems = btrfs_header_nritems(parent); 14568c594ea8SChris Mason slot = path->slots[level + 1]; 1457b4ce94deSChris Mason 1458bfb484d9SJosef Bacik if (slot > 0) 1459bfb484d9SJosef Bacik btrfs_readahead_node_child(parent, slot - 1); 1460bfb484d9SJosef Bacik if (slot + 1 < nritems) 1461bfb484d9SJosef Bacik btrfs_readahead_node_child(parent, slot + 1); 1462b4ce94deSChris Mason } 1463b4ce94deSChris Mason 1464b4ce94deSChris Mason 1465b4ce94deSChris Mason /* 1466d397712bSChris Mason * when we walk down the tree, it is usually safe to unlock the higher layers 1467d397712bSChris Mason * in the tree. The exceptions are when our path goes through slot 0, because 1468d397712bSChris Mason * operations on the tree might require changing key pointers higher up in the 1469d397712bSChris Mason * tree. 1470d352ac68SChris Mason * 1471d397712bSChris Mason * callers might also have set path->keep_locks, which tells this code to keep 1472d397712bSChris Mason * the lock if the path points to the last slot in the block. This is part of 1473d397712bSChris Mason * walking through the tree, and selecting the next slot in the higher block. 1474d352ac68SChris Mason * 1475d397712bSChris Mason * lowest_unlock sets the lowest level in the tree we're allowed to unlock. so 1476d397712bSChris Mason * if lowest_unlock is 1, level 0 won't be unlocked 1477d352ac68SChris Mason */ 1478e02119d5SChris Mason static noinline void unlock_up(struct btrfs_path *path, int level, 1479f7c79f30SChris Mason int lowest_unlock, int min_write_lock_level, 1480f7c79f30SChris Mason int *write_lock_level) 1481925baeddSChris Mason { 1482925baeddSChris Mason int i; 1483925baeddSChris Mason int skip_level = level; 1484c1227996SNikolay Borisov bool check_skip = true; 1485925baeddSChris Mason 1486925baeddSChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 1487925baeddSChris Mason if (!path->nodes[i]) 1488925baeddSChris Mason break; 1489925baeddSChris Mason if (!path->locks[i]) 1490925baeddSChris Mason break; 1491c1227996SNikolay Borisov 1492c1227996SNikolay Borisov if (check_skip) { 1493c1227996SNikolay Borisov if (path->slots[i] == 0) { 1494925baeddSChris Mason skip_level = i + 1; 1495925baeddSChris Mason continue; 1496925baeddSChris Mason } 1497c1227996SNikolay Borisov 1498c1227996SNikolay Borisov if (path->keep_locks) { 1499925baeddSChris Mason u32 nritems; 1500c1227996SNikolay Borisov 1501c1227996SNikolay Borisov nritems = btrfs_header_nritems(path->nodes[i]); 1502051e1b9fSChris Mason if (nritems < 1 || path->slots[i] >= nritems - 1) { 1503925baeddSChris Mason skip_level = i + 1; 1504925baeddSChris Mason continue; 1505925baeddSChris Mason } 1506925baeddSChris Mason } 1507c1227996SNikolay Borisov } 1508051e1b9fSChris Mason 1509d80bb3f9SLiu Bo if (i >= lowest_unlock && i > skip_level) { 1510c1227996SNikolay Borisov check_skip = false; 1511c1227996SNikolay Borisov btrfs_tree_unlock_rw(path->nodes[i], path->locks[i]); 1512925baeddSChris Mason path->locks[i] = 0; 1513f7c79f30SChris Mason if (write_lock_level && 1514f7c79f30SChris Mason i > min_write_lock_level && 1515f7c79f30SChris Mason i <= *write_lock_level) { 1516f7c79f30SChris Mason *write_lock_level = i - 1; 1517f7c79f30SChris Mason } 1518925baeddSChris Mason } 1519925baeddSChris Mason } 1520925baeddSChris Mason } 1521925baeddSChris Mason 15223c69faecSChris Mason /* 1523376a21d7SFilipe Manana * Helper function for btrfs_search_slot() and other functions that do a search 1524376a21d7SFilipe Manana * on a btree. The goal is to find a tree block in the cache (the radix tree at 1525376a21d7SFilipe Manana * fs_info->buffer_radix), but if we can't find it, or it's not up to date, read 1526376a21d7SFilipe Manana * its pages from disk. 1527c8c42864SChris Mason * 1528376a21d7SFilipe Manana * Returns -EAGAIN, with the path unlocked, if the caller needs to repeat the 1529376a21d7SFilipe Manana * whole btree search, starting again from the current root node. 1530c8c42864SChris Mason */ 1531c8c42864SChris Mason static int 1532d07b8528SLiu Bo read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, 1533c8c42864SChris Mason struct extent_buffer **eb_ret, int level, int slot, 1534cda79c54SDavid Sterba const struct btrfs_key *key) 1535c8c42864SChris Mason { 15360b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 1537789d6a3aSQu Wenruo struct btrfs_tree_parent_check check = { 0 }; 1538c8c42864SChris Mason u64 blocknr; 1539c8c42864SChris Mason u64 gen; 1540c8c42864SChris Mason struct extent_buffer *tmp; 154176a05b35SChris Mason int ret; 1542581c1760SQu Wenruo int parent_level; 1543b246666eSFilipe Manana bool unlock_up; 1544c8c42864SChris Mason 1545b246666eSFilipe Manana unlock_up = ((level + 1 < BTRFS_MAX_LEVEL) && p->locks[level + 1]); 1546213ff4b7SNikolay Borisov blocknr = btrfs_node_blockptr(*eb_ret, slot); 1547213ff4b7SNikolay Borisov gen = btrfs_node_ptr_generation(*eb_ret, slot); 1548213ff4b7SNikolay Borisov parent_level = btrfs_header_level(*eb_ret); 1549789d6a3aSQu Wenruo btrfs_node_key_to_cpu(*eb_ret, &check.first_key, slot); 1550789d6a3aSQu Wenruo check.has_first_key = true; 1551789d6a3aSQu Wenruo check.level = parent_level - 1; 1552789d6a3aSQu Wenruo check.transid = gen; 1553789d6a3aSQu Wenruo check.owner_root = root->root_key.objectid; 1554c8c42864SChris Mason 1555b246666eSFilipe Manana /* 1556b246666eSFilipe Manana * If we need to read an extent buffer from disk and we are holding locks 1557b246666eSFilipe Manana * on upper level nodes, we unlock all the upper nodes before reading the 1558b246666eSFilipe Manana * extent buffer, and then return -EAGAIN to the caller as it needs to 1559b246666eSFilipe Manana * restart the search. We don't release the lock on the current level 1560b246666eSFilipe Manana * because we need to walk this node to figure out which blocks to read. 1561b246666eSFilipe Manana */ 15620b246afaSJeff Mahoney tmp = find_extent_buffer(fs_info, blocknr); 1563cb44921aSChris Mason if (tmp) { 1564ace75066SFilipe Manana if (p->reada == READA_FORWARD_ALWAYS) 1565ace75066SFilipe Manana reada_for_search(fs_info, p, level, slot, key->objectid); 1566ace75066SFilipe Manana 1567b9fab919SChris Mason /* first we do an atomic uptodate check */ 1568b9fab919SChris Mason if (btrfs_buffer_uptodate(tmp, gen, 1) > 0) { 1569448de471SQu Wenruo /* 1570448de471SQu Wenruo * Do extra check for first_key, eb can be stale due to 1571448de471SQu Wenruo * being cached, read from scrub, or have multiple 1572448de471SQu Wenruo * parents (shared tree blocks). 1573448de471SQu Wenruo */ 1574e064d5e9SDavid Sterba if (btrfs_verify_level_key(tmp, 1575789d6a3aSQu Wenruo parent_level - 1, &check.first_key, gen)) { 1576448de471SQu Wenruo free_extent_buffer(tmp); 1577448de471SQu Wenruo return -EUCLEAN; 1578448de471SQu Wenruo } 1579c8c42864SChris Mason *eb_ret = tmp; 1580c8c42864SChris Mason return 0; 1581c8c42864SChris Mason } 1582bdf7c00eSJosef Bacik 1583857bc13fSJosef Bacik if (p->nowait) { 1584857bc13fSJosef Bacik free_extent_buffer(tmp); 1585857bc13fSJosef Bacik return -EAGAIN; 1586857bc13fSJosef Bacik } 1587857bc13fSJosef Bacik 1588b246666eSFilipe Manana if (unlock_up) 1589b246666eSFilipe Manana btrfs_unlock_up_safe(p, level + 1); 1590b246666eSFilipe Manana 1591b9fab919SChris Mason /* now we're allowed to do a blocking uptodate check */ 1592789d6a3aSQu Wenruo ret = btrfs_read_extent_buffer(tmp, &check); 15939a4ffa1bSQu Wenruo if (ret) { 1594cb44921aSChris Mason free_extent_buffer(tmp); 1595b3b4aa74SDavid Sterba btrfs_release_path(p); 1596cb44921aSChris Mason return -EIO; 1597cb44921aSChris Mason } 159888c602abSQu Wenruo if (btrfs_check_eb_owner(tmp, root->root_key.objectid)) { 159988c602abSQu Wenruo free_extent_buffer(tmp); 160088c602abSQu Wenruo btrfs_release_path(p); 160188c602abSQu Wenruo return -EUCLEAN; 160288c602abSQu Wenruo } 1603b246666eSFilipe Manana 1604b246666eSFilipe Manana if (unlock_up) 1605b246666eSFilipe Manana ret = -EAGAIN; 1606b246666eSFilipe Manana 1607b246666eSFilipe Manana goto out; 1608857bc13fSJosef Bacik } else if (p->nowait) { 1609857bc13fSJosef Bacik return -EAGAIN; 16109a4ffa1bSQu Wenruo } 1611c8c42864SChris Mason 1612b246666eSFilipe Manana if (unlock_up) { 16138c594ea8SChris Mason btrfs_unlock_up_safe(p, level + 1); 16144bb59055SFilipe Manana ret = -EAGAIN; 16154bb59055SFilipe Manana } else { 16164bb59055SFilipe Manana ret = 0; 16174bb59055SFilipe Manana } 16188c594ea8SChris Mason 1619e4058b54SDavid Sterba if (p->reada != READA_NONE) 16202ff7e61eSJeff Mahoney reada_for_search(fs_info, p, level, slot, key->objectid); 1621c8c42864SChris Mason 1622789d6a3aSQu Wenruo tmp = read_tree_block(fs_info, blocknr, &check); 16234eb150d6SQu Wenruo if (IS_ERR(tmp)) { 16244eb150d6SQu Wenruo btrfs_release_path(p); 16254eb150d6SQu Wenruo return PTR_ERR(tmp); 16264eb150d6SQu Wenruo } 162776a05b35SChris Mason /* 162876a05b35SChris Mason * If the read above didn't mark this buffer up to date, 162976a05b35SChris Mason * it will never end up being up to date. Set ret to EIO now 163076a05b35SChris Mason * and give up so that our caller doesn't loop forever 163176a05b35SChris Mason * on our EAGAINs. 163276a05b35SChris Mason */ 1633e6a1d6fdSLiu Bo if (!extent_buffer_uptodate(tmp)) 163476a05b35SChris Mason ret = -EIO; 163502a3307aSLiu Bo 1636b246666eSFilipe Manana out: 16374bb59055SFilipe Manana if (ret == 0) { 16384bb59055SFilipe Manana *eb_ret = tmp; 16394bb59055SFilipe Manana } else { 16404bb59055SFilipe Manana free_extent_buffer(tmp); 164102a3307aSLiu Bo btrfs_release_path(p); 16424bb59055SFilipe Manana } 16434bb59055SFilipe Manana 164476a05b35SChris Mason return ret; 1645c8c42864SChris Mason } 1646c8c42864SChris Mason 1647c8c42864SChris Mason /* 1648c8c42864SChris Mason * helper function for btrfs_search_slot. This does all of the checks 1649c8c42864SChris Mason * for node-level blocks and does any balancing required based on 1650c8c42864SChris Mason * the ins_len. 1651c8c42864SChris Mason * 1652c8c42864SChris Mason * If no extra work was required, zero is returned. If we had to 1653c8c42864SChris Mason * drop the path, -EAGAIN is returned and btrfs_search_slot must 1654c8c42864SChris Mason * start over 1655c8c42864SChris Mason */ 1656c8c42864SChris Mason static int 1657c8c42864SChris Mason setup_nodes_for_search(struct btrfs_trans_handle *trans, 1658c8c42864SChris Mason struct btrfs_root *root, struct btrfs_path *p, 1659bd681513SChris Mason struct extent_buffer *b, int level, int ins_len, 1660bd681513SChris Mason int *write_lock_level) 1661c8c42864SChris Mason { 16620b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 166395b982deSNikolay Borisov int ret = 0; 16640b246afaSJeff Mahoney 1665c8c42864SChris Mason if ((p->search_for_split || ins_len > 0) && btrfs_header_nritems(b) >= 16660b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3) { 1667c8c42864SChris Mason 1668bd681513SChris Mason if (*write_lock_level < level + 1) { 1669bd681513SChris Mason *write_lock_level = level + 1; 1670bd681513SChris Mason btrfs_release_path(p); 167195b982deSNikolay Borisov return -EAGAIN; 1672bd681513SChris Mason } 1673bd681513SChris Mason 1674bfb484d9SJosef Bacik reada_for_balance(p, level); 167595b982deSNikolay Borisov ret = split_node(trans, root, p, level); 1676c8c42864SChris Mason 1677c8c42864SChris Mason b = p->nodes[level]; 1678c8c42864SChris Mason } else if (ins_len < 0 && btrfs_header_nritems(b) < 16790b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 2) { 1680c8c42864SChris Mason 1681bd681513SChris Mason if (*write_lock_level < level + 1) { 1682bd681513SChris Mason *write_lock_level = level + 1; 1683bd681513SChris Mason btrfs_release_path(p); 168495b982deSNikolay Borisov return -EAGAIN; 1685bd681513SChris Mason } 1686bd681513SChris Mason 1687bfb484d9SJosef Bacik reada_for_balance(p, level); 168895b982deSNikolay Borisov ret = balance_level(trans, root, p, level); 168995b982deSNikolay Borisov if (ret) 169095b982deSNikolay Borisov return ret; 1691c8c42864SChris Mason 1692c8c42864SChris Mason b = p->nodes[level]; 1693c8c42864SChris Mason if (!b) { 1694b3b4aa74SDavid Sterba btrfs_release_path(p); 169595b982deSNikolay Borisov return -EAGAIN; 1696c8c42864SChris Mason } 1697c8c42864SChris Mason BUG_ON(btrfs_header_nritems(b) == 1); 1698c8c42864SChris Mason } 1699c8c42864SChris Mason return ret; 1700c8c42864SChris Mason } 1701c8c42864SChris Mason 1702381cf658SDavid Sterba int btrfs_find_item(struct btrfs_root *fs_root, struct btrfs_path *path, 1703e33d5c3dSKelley Nielsen u64 iobjectid, u64 ioff, u8 key_type, 1704e33d5c3dSKelley Nielsen struct btrfs_key *found_key) 1705e33d5c3dSKelley Nielsen { 1706e33d5c3dSKelley Nielsen int ret; 1707e33d5c3dSKelley Nielsen struct btrfs_key key; 1708e33d5c3dSKelley Nielsen struct extent_buffer *eb; 1709381cf658SDavid Sterba 1710381cf658SDavid Sterba ASSERT(path); 17111d4c08e0SDavid Sterba ASSERT(found_key); 1712e33d5c3dSKelley Nielsen 1713e33d5c3dSKelley Nielsen key.type = key_type; 1714e33d5c3dSKelley Nielsen key.objectid = iobjectid; 1715e33d5c3dSKelley Nielsen key.offset = ioff; 1716e33d5c3dSKelley Nielsen 1717e33d5c3dSKelley Nielsen ret = btrfs_search_slot(NULL, fs_root, &key, path, 0, 0); 17181d4c08e0SDavid Sterba if (ret < 0) 1719e33d5c3dSKelley Nielsen return ret; 1720e33d5c3dSKelley Nielsen 1721e33d5c3dSKelley Nielsen eb = path->nodes[0]; 1722e33d5c3dSKelley Nielsen if (ret && path->slots[0] >= btrfs_header_nritems(eb)) { 1723e33d5c3dSKelley Nielsen ret = btrfs_next_leaf(fs_root, path); 1724e33d5c3dSKelley Nielsen if (ret) 1725e33d5c3dSKelley Nielsen return ret; 1726e33d5c3dSKelley Nielsen eb = path->nodes[0]; 1727e33d5c3dSKelley Nielsen } 1728e33d5c3dSKelley Nielsen 1729e33d5c3dSKelley Nielsen btrfs_item_key_to_cpu(eb, found_key, path->slots[0]); 1730e33d5c3dSKelley Nielsen if (found_key->type != key.type || 1731e33d5c3dSKelley Nielsen found_key->objectid != key.objectid) 1732e33d5c3dSKelley Nielsen return 1; 1733e33d5c3dSKelley Nielsen 1734e33d5c3dSKelley Nielsen return 0; 1735e33d5c3dSKelley Nielsen } 1736e33d5c3dSKelley Nielsen 17371fc28d8eSLiu Bo static struct extent_buffer *btrfs_search_slot_get_root(struct btrfs_root *root, 17381fc28d8eSLiu Bo struct btrfs_path *p, 17391fc28d8eSLiu Bo int write_lock_level) 17401fc28d8eSLiu Bo { 17411fc28d8eSLiu Bo struct extent_buffer *b; 1742120de408SJosef Bacik int root_lock = 0; 17431fc28d8eSLiu Bo int level = 0; 17441fc28d8eSLiu Bo 17451fc28d8eSLiu Bo if (p->search_commit_root) { 17461fc28d8eSLiu Bo b = root->commit_root; 174767439dadSDavid Sterba atomic_inc(&b->refs); 17481fc28d8eSLiu Bo level = btrfs_header_level(b); 1749f9ddfd05SLiu Bo /* 1750f9ddfd05SLiu Bo * Ensure that all callers have set skip_locking when 1751f9ddfd05SLiu Bo * p->search_commit_root = 1. 1752f9ddfd05SLiu Bo */ 1753f9ddfd05SLiu Bo ASSERT(p->skip_locking == 1); 17541fc28d8eSLiu Bo 17551fc28d8eSLiu Bo goto out; 17561fc28d8eSLiu Bo } 17571fc28d8eSLiu Bo 17581fc28d8eSLiu Bo if (p->skip_locking) { 17591fc28d8eSLiu Bo b = btrfs_root_node(root); 17601fc28d8eSLiu Bo level = btrfs_header_level(b); 17611fc28d8eSLiu Bo goto out; 17621fc28d8eSLiu Bo } 17631fc28d8eSLiu Bo 1764120de408SJosef Bacik /* We try very hard to do read locks on the root */ 1765120de408SJosef Bacik root_lock = BTRFS_READ_LOCK; 1766120de408SJosef Bacik 17671fc28d8eSLiu Bo /* 1768662c653bSLiu Bo * If the level is set to maximum, we can skip trying to get the read 1769662c653bSLiu Bo * lock. 1770662c653bSLiu Bo */ 1771662c653bSLiu Bo if (write_lock_level < BTRFS_MAX_LEVEL) { 1772662c653bSLiu Bo /* 1773662c653bSLiu Bo * We don't know the level of the root node until we actually 1774662c653bSLiu Bo * have it read locked 17751fc28d8eSLiu Bo */ 1776857bc13fSJosef Bacik if (p->nowait) { 1777857bc13fSJosef Bacik b = btrfs_try_read_lock_root_node(root); 1778857bc13fSJosef Bacik if (IS_ERR(b)) 1779857bc13fSJosef Bacik return b; 1780857bc13fSJosef Bacik } else { 17811bb96598SJosef Bacik b = btrfs_read_lock_root_node(root); 1782857bc13fSJosef Bacik } 17831fc28d8eSLiu Bo level = btrfs_header_level(b); 17841fc28d8eSLiu Bo if (level > write_lock_level) 17851fc28d8eSLiu Bo goto out; 17861fc28d8eSLiu Bo 1787662c653bSLiu Bo /* Whoops, must trade for write lock */ 17881fc28d8eSLiu Bo btrfs_tree_read_unlock(b); 17891fc28d8eSLiu Bo free_extent_buffer(b); 1790662c653bSLiu Bo } 1791662c653bSLiu Bo 17921fc28d8eSLiu Bo b = btrfs_lock_root_node(root); 17931fc28d8eSLiu Bo root_lock = BTRFS_WRITE_LOCK; 17941fc28d8eSLiu Bo 17951fc28d8eSLiu Bo /* The level might have changed, check again */ 17961fc28d8eSLiu Bo level = btrfs_header_level(b); 17971fc28d8eSLiu Bo 17981fc28d8eSLiu Bo out: 1799120de408SJosef Bacik /* 1800120de408SJosef Bacik * The root may have failed to write out at some point, and thus is no 1801120de408SJosef Bacik * longer valid, return an error in this case. 1802120de408SJosef Bacik */ 1803120de408SJosef Bacik if (!extent_buffer_uptodate(b)) { 1804120de408SJosef Bacik if (root_lock) 1805120de408SJosef Bacik btrfs_tree_unlock_rw(b, root_lock); 1806120de408SJosef Bacik free_extent_buffer(b); 1807120de408SJosef Bacik return ERR_PTR(-EIO); 1808120de408SJosef Bacik } 1809120de408SJosef Bacik 18101fc28d8eSLiu Bo p->nodes[level] = b; 18111fc28d8eSLiu Bo if (!p->skip_locking) 18121fc28d8eSLiu Bo p->locks[level] = root_lock; 18131fc28d8eSLiu Bo /* 18141fc28d8eSLiu Bo * Callers are responsible for dropping b's references. 18151fc28d8eSLiu Bo */ 18161fc28d8eSLiu Bo return b; 18171fc28d8eSLiu Bo } 18181fc28d8eSLiu Bo 1819d96b3424SFilipe Manana /* 1820d96b3424SFilipe Manana * Replace the extent buffer at the lowest level of the path with a cloned 1821d96b3424SFilipe Manana * version. The purpose is to be able to use it safely, after releasing the 1822d96b3424SFilipe Manana * commit root semaphore, even if relocation is happening in parallel, the 1823d96b3424SFilipe Manana * transaction used for relocation is committed and the extent buffer is 1824d96b3424SFilipe Manana * reallocated in the next transaction. 1825d96b3424SFilipe Manana * 1826d96b3424SFilipe Manana * This is used in a context where the caller does not prevent transaction 1827d96b3424SFilipe Manana * commits from happening, either by holding a transaction handle or holding 1828d96b3424SFilipe Manana * some lock, while it's doing searches through a commit root. 1829d96b3424SFilipe Manana * At the moment it's only used for send operations. 1830d96b3424SFilipe Manana */ 1831d96b3424SFilipe Manana static int finish_need_commit_sem_search(struct btrfs_path *path) 1832d96b3424SFilipe Manana { 1833d96b3424SFilipe Manana const int i = path->lowest_level; 1834d96b3424SFilipe Manana const int slot = path->slots[i]; 1835d96b3424SFilipe Manana struct extent_buffer *lowest = path->nodes[i]; 1836d96b3424SFilipe Manana struct extent_buffer *clone; 1837d96b3424SFilipe Manana 1838d96b3424SFilipe Manana ASSERT(path->need_commit_sem); 1839d96b3424SFilipe Manana 1840d96b3424SFilipe Manana if (!lowest) 1841d96b3424SFilipe Manana return 0; 1842d96b3424SFilipe Manana 1843d96b3424SFilipe Manana lockdep_assert_held_read(&lowest->fs_info->commit_root_sem); 1844d96b3424SFilipe Manana 1845d96b3424SFilipe Manana clone = btrfs_clone_extent_buffer(lowest); 1846d96b3424SFilipe Manana if (!clone) 1847d96b3424SFilipe Manana return -ENOMEM; 1848d96b3424SFilipe Manana 1849d96b3424SFilipe Manana btrfs_release_path(path); 1850d96b3424SFilipe Manana path->nodes[i] = clone; 1851d96b3424SFilipe Manana path->slots[i] = slot; 1852d96b3424SFilipe Manana 1853d96b3424SFilipe Manana return 0; 1854d96b3424SFilipe Manana } 18551fc28d8eSLiu Bo 1856e2e58d0fSFilipe Manana static inline int search_for_key_slot(struct extent_buffer *eb, 1857e2e58d0fSFilipe Manana int search_low_slot, 1858e2e58d0fSFilipe Manana const struct btrfs_key *key, 1859e2e58d0fSFilipe Manana int prev_cmp, 1860e2e58d0fSFilipe Manana int *slot) 1861e2e58d0fSFilipe Manana { 1862e2e58d0fSFilipe Manana /* 1863e2e58d0fSFilipe Manana * If a previous call to btrfs_bin_search() on a parent node returned an 1864e2e58d0fSFilipe Manana * exact match (prev_cmp == 0), we can safely assume the target key will 1865e2e58d0fSFilipe Manana * always be at slot 0 on lower levels, since each key pointer 1866e2e58d0fSFilipe Manana * (struct btrfs_key_ptr) refers to the lowest key accessible from the 1867e2e58d0fSFilipe Manana * subtree it points to. Thus we can skip searching lower levels. 1868e2e58d0fSFilipe Manana */ 1869e2e58d0fSFilipe Manana if (prev_cmp == 0) { 1870e2e58d0fSFilipe Manana *slot = 0; 1871e2e58d0fSFilipe Manana return 0; 1872e2e58d0fSFilipe Manana } 1873e2e58d0fSFilipe Manana 18747b00dfffSFilipe Manana return btrfs_generic_bin_search(eb, search_low_slot, key, slot); 1875e2e58d0fSFilipe Manana } 1876e2e58d0fSFilipe Manana 1877109324cfSFilipe Manana static int search_leaf(struct btrfs_trans_handle *trans, 1878109324cfSFilipe Manana struct btrfs_root *root, 1879109324cfSFilipe Manana const struct btrfs_key *key, 1880109324cfSFilipe Manana struct btrfs_path *path, 1881109324cfSFilipe Manana int ins_len, 1882109324cfSFilipe Manana int prev_cmp) 1883109324cfSFilipe Manana { 1884109324cfSFilipe Manana struct extent_buffer *leaf = path->nodes[0]; 1885109324cfSFilipe Manana int leaf_free_space = -1; 1886109324cfSFilipe Manana int search_low_slot = 0; 1887109324cfSFilipe Manana int ret; 1888109324cfSFilipe Manana bool do_bin_search = true; 1889109324cfSFilipe Manana 1890109324cfSFilipe Manana /* 1891109324cfSFilipe Manana * If we are doing an insertion, the leaf has enough free space and the 1892109324cfSFilipe Manana * destination slot for the key is not slot 0, then we can unlock our 1893109324cfSFilipe Manana * write lock on the parent, and any other upper nodes, before doing the 1894109324cfSFilipe Manana * binary search on the leaf (with search_for_key_slot()), allowing other 1895109324cfSFilipe Manana * tasks to lock the parent and any other upper nodes. 1896109324cfSFilipe Manana */ 1897109324cfSFilipe Manana if (ins_len > 0) { 1898109324cfSFilipe Manana /* 1899109324cfSFilipe Manana * Cache the leaf free space, since we will need it later and it 1900109324cfSFilipe Manana * will not change until then. 1901109324cfSFilipe Manana */ 1902109324cfSFilipe Manana leaf_free_space = btrfs_leaf_free_space(leaf); 1903109324cfSFilipe Manana 1904109324cfSFilipe Manana /* 1905109324cfSFilipe Manana * !path->locks[1] means we have a single node tree, the leaf is 1906109324cfSFilipe Manana * the root of the tree. 1907109324cfSFilipe Manana */ 1908109324cfSFilipe Manana if (path->locks[1] && leaf_free_space >= ins_len) { 1909109324cfSFilipe Manana struct btrfs_disk_key first_key; 1910109324cfSFilipe Manana 1911109324cfSFilipe Manana ASSERT(btrfs_header_nritems(leaf) > 0); 1912109324cfSFilipe Manana btrfs_item_key(leaf, &first_key, 0); 1913109324cfSFilipe Manana 1914109324cfSFilipe Manana /* 1915109324cfSFilipe Manana * Doing the extra comparison with the first key is cheap, 1916109324cfSFilipe Manana * taking into account that the first key is very likely 1917109324cfSFilipe Manana * already in a cache line because it immediately follows 1918109324cfSFilipe Manana * the extent buffer's header and we have recently accessed 1919109324cfSFilipe Manana * the header's level field. 1920109324cfSFilipe Manana */ 1921109324cfSFilipe Manana ret = comp_keys(&first_key, key); 1922109324cfSFilipe Manana if (ret < 0) { 1923109324cfSFilipe Manana /* 1924109324cfSFilipe Manana * The first key is smaller than the key we want 1925109324cfSFilipe Manana * to insert, so we are safe to unlock all upper 1926109324cfSFilipe Manana * nodes and we have to do the binary search. 1927109324cfSFilipe Manana * 1928109324cfSFilipe Manana * We do use btrfs_unlock_up_safe() and not 1929109324cfSFilipe Manana * unlock_up() because the later does not unlock 1930109324cfSFilipe Manana * nodes with a slot of 0 - we can safely unlock 1931109324cfSFilipe Manana * any node even if its slot is 0 since in this 1932109324cfSFilipe Manana * case the key does not end up at slot 0 of the 1933109324cfSFilipe Manana * leaf and there's no need to split the leaf. 1934109324cfSFilipe Manana */ 1935109324cfSFilipe Manana btrfs_unlock_up_safe(path, 1); 1936109324cfSFilipe Manana search_low_slot = 1; 1937109324cfSFilipe Manana } else { 1938109324cfSFilipe Manana /* 1939109324cfSFilipe Manana * The first key is >= then the key we want to 1940109324cfSFilipe Manana * insert, so we can skip the binary search as 1941109324cfSFilipe Manana * the target key will be at slot 0. 1942109324cfSFilipe Manana * 1943109324cfSFilipe Manana * We can not unlock upper nodes when the key is 1944109324cfSFilipe Manana * less than the first key, because we will need 1945109324cfSFilipe Manana * to update the key at slot 0 of the parent node 1946109324cfSFilipe Manana * and possibly of other upper nodes too. 1947109324cfSFilipe Manana * If the key matches the first key, then we can 1948109324cfSFilipe Manana * unlock all the upper nodes, using 1949109324cfSFilipe Manana * btrfs_unlock_up_safe() instead of unlock_up() 1950109324cfSFilipe Manana * as stated above. 1951109324cfSFilipe Manana */ 1952109324cfSFilipe Manana if (ret == 0) 1953109324cfSFilipe Manana btrfs_unlock_up_safe(path, 1); 1954109324cfSFilipe Manana /* 1955109324cfSFilipe Manana * ret is already 0 or 1, matching the result of 1956109324cfSFilipe Manana * a btrfs_bin_search() call, so there is no need 1957109324cfSFilipe Manana * to adjust it. 1958109324cfSFilipe Manana */ 1959109324cfSFilipe Manana do_bin_search = false; 1960109324cfSFilipe Manana path->slots[0] = 0; 1961109324cfSFilipe Manana } 1962109324cfSFilipe Manana } 1963109324cfSFilipe Manana } 1964109324cfSFilipe Manana 1965109324cfSFilipe Manana if (do_bin_search) { 1966109324cfSFilipe Manana ret = search_for_key_slot(leaf, search_low_slot, key, 1967109324cfSFilipe Manana prev_cmp, &path->slots[0]); 1968109324cfSFilipe Manana if (ret < 0) 1969109324cfSFilipe Manana return ret; 1970109324cfSFilipe Manana } 1971109324cfSFilipe Manana 1972109324cfSFilipe Manana if (ins_len > 0) { 1973109324cfSFilipe Manana /* 1974109324cfSFilipe Manana * Item key already exists. In this case, if we are allowed to 1975109324cfSFilipe Manana * insert the item (for example, in dir_item case, item key 1976109324cfSFilipe Manana * collision is allowed), it will be merged with the original 1977109324cfSFilipe Manana * item. Only the item size grows, no new btrfs item will be 1978109324cfSFilipe Manana * added. If search_for_extension is not set, ins_len already 1979109324cfSFilipe Manana * accounts the size btrfs_item, deduct it here so leaf space 1980109324cfSFilipe Manana * check will be correct. 1981109324cfSFilipe Manana */ 1982109324cfSFilipe Manana if (ret == 0 && !path->search_for_extension) { 1983109324cfSFilipe Manana ASSERT(ins_len >= sizeof(struct btrfs_item)); 1984109324cfSFilipe Manana ins_len -= sizeof(struct btrfs_item); 1985109324cfSFilipe Manana } 1986109324cfSFilipe Manana 1987109324cfSFilipe Manana ASSERT(leaf_free_space >= 0); 1988109324cfSFilipe Manana 1989109324cfSFilipe Manana if (leaf_free_space < ins_len) { 1990109324cfSFilipe Manana int err; 1991109324cfSFilipe Manana 1992109324cfSFilipe Manana err = split_leaf(trans, root, key, path, ins_len, 1993109324cfSFilipe Manana (ret == 0)); 1994bb8e9a60SFilipe Manana ASSERT(err <= 0); 1995bb8e9a60SFilipe Manana if (WARN_ON(err > 0)) 1996bb8e9a60SFilipe Manana err = -EUCLEAN; 1997109324cfSFilipe Manana if (err) 1998109324cfSFilipe Manana ret = err; 1999109324cfSFilipe Manana } 2000109324cfSFilipe Manana } 2001109324cfSFilipe Manana 2002109324cfSFilipe Manana return ret; 2003109324cfSFilipe Manana } 2004109324cfSFilipe Manana 2005c8c42864SChris Mason /* 20064271eceaSNikolay Borisov * btrfs_search_slot - look for a key in a tree and perform necessary 20074271eceaSNikolay Borisov * modifications to preserve tree invariants. 200874123bd7SChris Mason * 20094271eceaSNikolay Borisov * @trans: Handle of transaction, used when modifying the tree 20104271eceaSNikolay Borisov * @p: Holds all btree nodes along the search path 20114271eceaSNikolay Borisov * @root: The root node of the tree 20124271eceaSNikolay Borisov * @key: The key we are looking for 20139a664971Sethanwu * @ins_len: Indicates purpose of search: 20149a664971Sethanwu * >0 for inserts it's size of item inserted (*) 20159a664971Sethanwu * <0 for deletions 20169a664971Sethanwu * 0 for plain searches, not modifying the tree 20179a664971Sethanwu * 20189a664971Sethanwu * (*) If size of item inserted doesn't include 20199a664971Sethanwu * sizeof(struct btrfs_item), then p->search_for_extension must 20209a664971Sethanwu * be set. 20214271eceaSNikolay Borisov * @cow: boolean should CoW operations be performed. Must always be 1 20224271eceaSNikolay Borisov * when modifying the tree. 202397571fd0SChris Mason * 20244271eceaSNikolay Borisov * If @ins_len > 0, nodes and leaves will be split as we walk down the tree. 20254271eceaSNikolay Borisov * If @ins_len < 0, nodes will be merged as we walk down the tree (if possible) 20264271eceaSNikolay Borisov * 20274271eceaSNikolay Borisov * If @key is found, 0 is returned and you can find the item in the leaf level 20284271eceaSNikolay Borisov * of the path (level 0) 20294271eceaSNikolay Borisov * 20304271eceaSNikolay Borisov * If @key isn't found, 1 is returned and the leaf level of the path (level 0) 20314271eceaSNikolay Borisov * points to the slot where it should be inserted 20324271eceaSNikolay Borisov * 20334271eceaSNikolay Borisov * If an error is encountered while searching the tree a negative error number 20344271eceaSNikolay Borisov * is returned 203574123bd7SChris Mason */ 2036310712b2SOmar Sandoval int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root, 2037310712b2SOmar Sandoval const struct btrfs_key *key, struct btrfs_path *p, 2038310712b2SOmar Sandoval int ins_len, int cow) 2039be0e5c09SChris Mason { 2040d96b3424SFilipe Manana struct btrfs_fs_info *fs_info = root->fs_info; 20415f39d397SChris Mason struct extent_buffer *b; 2042be0e5c09SChris Mason int slot; 2043be0e5c09SChris Mason int ret; 204433c66f43SYan Zheng int err; 2045be0e5c09SChris Mason int level; 2046925baeddSChris Mason int lowest_unlock = 1; 2047bd681513SChris Mason /* everything at write_lock_level or lower must be write locked */ 2048bd681513SChris Mason int write_lock_level = 0; 20499f3a7427SChris Mason u8 lowest_level = 0; 2050f7c79f30SChris Mason int min_write_lock_level; 2051d7396f07SFilipe David Borba Manana int prev_cmp; 20529f3a7427SChris Mason 2053a4c853afSChenXiaoSong might_sleep(); 2054a4c853afSChenXiaoSong 20556702ed49SChris Mason lowest_level = p->lowest_level; 2056323ac95bSChris Mason WARN_ON(lowest_level && ins_len > 0); 205722b0ebdaSChris Mason WARN_ON(p->nodes[0] != NULL); 2058eb653de1SFilipe David Borba Manana BUG_ON(!cow && ins_len); 205925179201SJosef Bacik 2060857bc13fSJosef Bacik /* 2061857bc13fSJosef Bacik * For now only allow nowait for read only operations. There's no 2062857bc13fSJosef Bacik * strict reason why we can't, we just only need it for reads so it's 2063857bc13fSJosef Bacik * only implemented for reads. 2064857bc13fSJosef Bacik */ 2065857bc13fSJosef Bacik ASSERT(!p->nowait || !cow); 2066857bc13fSJosef Bacik 2067bd681513SChris Mason if (ins_len < 0) { 2068925baeddSChris Mason lowest_unlock = 2; 206965b51a00SChris Mason 2070bd681513SChris Mason /* when we are removing items, we might have to go up to level 2071bd681513SChris Mason * two as we update tree pointers Make sure we keep write 2072bd681513SChris Mason * for those levels as well 2073bd681513SChris Mason */ 2074bd681513SChris Mason write_lock_level = 2; 2075bd681513SChris Mason } else if (ins_len > 0) { 2076bd681513SChris Mason /* 2077bd681513SChris Mason * for inserting items, make sure we have a write lock on 2078bd681513SChris Mason * level 1 so we can update keys 2079bd681513SChris Mason */ 2080bd681513SChris Mason write_lock_level = 1; 2081bd681513SChris Mason } 2082bd681513SChris Mason 2083bd681513SChris Mason if (!cow) 2084bd681513SChris Mason write_lock_level = -1; 2085bd681513SChris Mason 208609a2a8f9SJosef Bacik if (cow && (p->keep_locks || p->lowest_level)) 2087bd681513SChris Mason write_lock_level = BTRFS_MAX_LEVEL; 2088bd681513SChris Mason 2089f7c79f30SChris Mason min_write_lock_level = write_lock_level; 2090f7c79f30SChris Mason 2091d96b3424SFilipe Manana if (p->need_commit_sem) { 2092d96b3424SFilipe Manana ASSERT(p->search_commit_root); 2093857bc13fSJosef Bacik if (p->nowait) { 2094857bc13fSJosef Bacik if (!down_read_trylock(&fs_info->commit_root_sem)) 2095857bc13fSJosef Bacik return -EAGAIN; 2096857bc13fSJosef Bacik } else { 2097d96b3424SFilipe Manana down_read(&fs_info->commit_root_sem); 2098d96b3424SFilipe Manana } 2099857bc13fSJosef Bacik } 2100d96b3424SFilipe Manana 2101bb803951SChris Mason again: 2102d7396f07SFilipe David Borba Manana prev_cmp = -1; 21031fc28d8eSLiu Bo b = btrfs_search_slot_get_root(root, p, write_lock_level); 2104be6821f8SFilipe Manana if (IS_ERR(b)) { 2105be6821f8SFilipe Manana ret = PTR_ERR(b); 2106be6821f8SFilipe Manana goto done; 2107be6821f8SFilipe Manana } 2108925baeddSChris Mason 2109eb60ceacSChris Mason while (b) { 2110f624d976SQu Wenruo int dec = 0; 2111f624d976SQu Wenruo 21125f39d397SChris Mason level = btrfs_header_level(b); 211365b51a00SChris Mason 211402217ed2SChris Mason if (cow) { 21159ea2c7c9SNikolay Borisov bool last_level = (level == (BTRFS_MAX_LEVEL - 1)); 21169ea2c7c9SNikolay Borisov 2117c8c42864SChris Mason /* 2118c8c42864SChris Mason * if we don't really need to cow this block 2119c8c42864SChris Mason * then we don't want to set the path blocking, 2120c8c42864SChris Mason * so we test it here 2121c8c42864SChris Mason */ 21225963ffcaSJosef Bacik if (!should_cow_block(trans, root, b)) 212365b51a00SChris Mason goto cow_done; 21245d4f98a2SYan Zheng 2125bd681513SChris Mason /* 2126bd681513SChris Mason * must have write locks on this node and the 2127bd681513SChris Mason * parent 2128bd681513SChris Mason */ 21295124e00eSJosef Bacik if (level > write_lock_level || 21305124e00eSJosef Bacik (level + 1 > write_lock_level && 21315124e00eSJosef Bacik level + 1 < BTRFS_MAX_LEVEL && 21325124e00eSJosef Bacik p->nodes[level + 1])) { 2133bd681513SChris Mason write_lock_level = level + 1; 2134bd681513SChris Mason btrfs_release_path(p); 2135bd681513SChris Mason goto again; 2136bd681513SChris Mason } 2137bd681513SChris Mason 21389ea2c7c9SNikolay Borisov if (last_level) 21399ea2c7c9SNikolay Borisov err = btrfs_cow_block(trans, root, b, NULL, 0, 21409631e4ccSJosef Bacik &b, 21419631e4ccSJosef Bacik BTRFS_NESTING_COW); 21429ea2c7c9SNikolay Borisov else 214333c66f43SYan Zheng err = btrfs_cow_block(trans, root, b, 2144e20d96d6SChris Mason p->nodes[level + 1], 21459631e4ccSJosef Bacik p->slots[level + 1], &b, 21469631e4ccSJosef Bacik BTRFS_NESTING_COW); 214733c66f43SYan Zheng if (err) { 214833c66f43SYan Zheng ret = err; 214965b51a00SChris Mason goto done; 215054aa1f4dSChris Mason } 215102217ed2SChris Mason } 215265b51a00SChris Mason cow_done: 2153eb60ceacSChris Mason p->nodes[level] = b; 2154b4ce94deSChris Mason 2155b4ce94deSChris Mason /* 2156b4ce94deSChris Mason * we have a lock on b and as long as we aren't changing 2157b4ce94deSChris Mason * the tree, there is no way to for the items in b to change. 2158b4ce94deSChris Mason * It is safe to drop the lock on our parent before we 2159b4ce94deSChris Mason * go through the expensive btree search on b. 2160b4ce94deSChris Mason * 2161eb653de1SFilipe David Borba Manana * If we're inserting or deleting (ins_len != 0), then we might 2162eb653de1SFilipe David Borba Manana * be changing slot zero, which may require changing the parent. 2163eb653de1SFilipe David Borba Manana * So, we can't drop the lock until after we know which slot 2164eb653de1SFilipe David Borba Manana * we're operating on. 2165b4ce94deSChris Mason */ 2166eb653de1SFilipe David Borba Manana if (!ins_len && !p->keep_locks) { 2167eb653de1SFilipe David Borba Manana int u = level + 1; 2168eb653de1SFilipe David Borba Manana 2169eb653de1SFilipe David Borba Manana if (u < BTRFS_MAX_LEVEL && p->locks[u]) { 2170eb653de1SFilipe David Borba Manana btrfs_tree_unlock_rw(p->nodes[u], p->locks[u]); 2171eb653de1SFilipe David Borba Manana p->locks[u] = 0; 2172eb653de1SFilipe David Borba Manana } 2173eb653de1SFilipe David Borba Manana } 2174b4ce94deSChris Mason 2175e2e58d0fSFilipe Manana if (level == 0) { 2176109324cfSFilipe Manana if (ins_len > 0) 2177e5e1c174SFilipe Manana ASSERT(write_lock_level >= 1); 2178bd681513SChris Mason 2179109324cfSFilipe Manana ret = search_leaf(trans, root, key, p, ins_len, prev_cmp); 2180459931ecSChris Mason if (!p->search_for_split) 2181f7c79f30SChris Mason unlock_up(p, level, lowest_unlock, 21824b6f8e96SLiu Bo min_write_lock_level, NULL); 218365b51a00SChris Mason goto done; 218465b51a00SChris Mason } 2185e2e58d0fSFilipe Manana 2186e2e58d0fSFilipe Manana ret = search_for_key_slot(b, 0, key, prev_cmp, &slot); 2187e2e58d0fSFilipe Manana if (ret < 0) 2188e2e58d0fSFilipe Manana goto done; 2189e2e58d0fSFilipe Manana prev_cmp = ret; 2190e2e58d0fSFilipe Manana 2191f624d976SQu Wenruo if (ret && slot > 0) { 2192f624d976SQu Wenruo dec = 1; 2193f624d976SQu Wenruo slot--; 2194f624d976SQu Wenruo } 2195f624d976SQu Wenruo p->slots[level] = slot; 2196f624d976SQu Wenruo err = setup_nodes_for_search(trans, root, p, b, level, ins_len, 2197f624d976SQu Wenruo &write_lock_level); 2198f624d976SQu Wenruo if (err == -EAGAIN) 2199f624d976SQu Wenruo goto again; 2200f624d976SQu Wenruo if (err) { 2201f624d976SQu Wenruo ret = err; 2202f624d976SQu Wenruo goto done; 2203f624d976SQu Wenruo } 2204f624d976SQu Wenruo b = p->nodes[level]; 2205f624d976SQu Wenruo slot = p->slots[level]; 2206f624d976SQu Wenruo 2207f624d976SQu Wenruo /* 2208f624d976SQu Wenruo * Slot 0 is special, if we change the key we have to update 2209f624d976SQu Wenruo * the parent pointer which means we must have a write lock on 2210f624d976SQu Wenruo * the parent 2211f624d976SQu Wenruo */ 2212f624d976SQu Wenruo if (slot == 0 && ins_len && write_lock_level < level + 1) { 2213f624d976SQu Wenruo write_lock_level = level + 1; 2214f624d976SQu Wenruo btrfs_release_path(p); 2215f624d976SQu Wenruo goto again; 2216f624d976SQu Wenruo } 2217f624d976SQu Wenruo 2218f624d976SQu Wenruo unlock_up(p, level, lowest_unlock, min_write_lock_level, 2219f624d976SQu Wenruo &write_lock_level); 2220f624d976SQu Wenruo 2221f624d976SQu Wenruo if (level == lowest_level) { 2222f624d976SQu Wenruo if (dec) 2223f624d976SQu Wenruo p->slots[level]++; 2224f624d976SQu Wenruo goto done; 2225f624d976SQu Wenruo } 2226f624d976SQu Wenruo 2227f624d976SQu Wenruo err = read_block_for_search(root, p, &b, level, slot, key); 2228f624d976SQu Wenruo if (err == -EAGAIN) 2229f624d976SQu Wenruo goto again; 2230f624d976SQu Wenruo if (err) { 2231f624d976SQu Wenruo ret = err; 2232f624d976SQu Wenruo goto done; 2233f624d976SQu Wenruo } 2234f624d976SQu Wenruo 2235f624d976SQu Wenruo if (!p->skip_locking) { 2236f624d976SQu Wenruo level = btrfs_header_level(b); 2237b40130b2SJosef Bacik 2238b40130b2SJosef Bacik btrfs_maybe_reset_lockdep_class(root, b); 2239b40130b2SJosef Bacik 2240f624d976SQu Wenruo if (level <= write_lock_level) { 2241f624d976SQu Wenruo btrfs_tree_lock(b); 2242f624d976SQu Wenruo p->locks[level] = BTRFS_WRITE_LOCK; 2243f624d976SQu Wenruo } else { 2244857bc13fSJosef Bacik if (p->nowait) { 2245857bc13fSJosef Bacik if (!btrfs_try_tree_read_lock(b)) { 2246857bc13fSJosef Bacik free_extent_buffer(b); 2247857bc13fSJosef Bacik ret = -EAGAIN; 2248857bc13fSJosef Bacik goto done; 2249857bc13fSJosef Bacik } 2250857bc13fSJosef Bacik } else { 2251fe596ca3SJosef Bacik btrfs_tree_read_lock(b); 2252857bc13fSJosef Bacik } 2253f624d976SQu Wenruo p->locks[level] = BTRFS_READ_LOCK; 2254f624d976SQu Wenruo } 2255f624d976SQu Wenruo p->nodes[level] = b; 2256f624d976SQu Wenruo } 225765b51a00SChris Mason } 225865b51a00SChris Mason ret = 1; 225965b51a00SChris Mason done: 22605f5bc6b1SFilipe Manana if (ret < 0 && !p->skip_release_on_error) 2261b3b4aa74SDavid Sterba btrfs_release_path(p); 2262d96b3424SFilipe Manana 2263d96b3424SFilipe Manana if (p->need_commit_sem) { 2264d96b3424SFilipe Manana int ret2; 2265d96b3424SFilipe Manana 2266d96b3424SFilipe Manana ret2 = finish_need_commit_sem_search(p); 2267d96b3424SFilipe Manana up_read(&fs_info->commit_root_sem); 2268d96b3424SFilipe Manana if (ret2) 2269d96b3424SFilipe Manana ret = ret2; 2270d96b3424SFilipe Manana } 2271d96b3424SFilipe Manana 2272be0e5c09SChris Mason return ret; 2273be0e5c09SChris Mason } 2274f75e2b79SJosef Bacik ALLOW_ERROR_INJECTION(btrfs_search_slot, ERRNO); 2275be0e5c09SChris Mason 227674123bd7SChris Mason /* 22775d9e75c4SJan Schmidt * Like btrfs_search_slot, this looks for a key in the given tree. It uses the 22785d9e75c4SJan Schmidt * current state of the tree together with the operations recorded in the tree 22795d9e75c4SJan Schmidt * modification log to search for the key in a previous version of this tree, as 22805d9e75c4SJan Schmidt * denoted by the time_seq parameter. 22815d9e75c4SJan Schmidt * 22825d9e75c4SJan Schmidt * Naturally, there is no support for insert, delete or cow operations. 22835d9e75c4SJan Schmidt * 22845d9e75c4SJan Schmidt * The resulting path and return value will be set up as if we called 22855d9e75c4SJan Schmidt * btrfs_search_slot at that point in time with ins_len and cow both set to 0. 22865d9e75c4SJan Schmidt */ 2287310712b2SOmar Sandoval int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key, 22885d9e75c4SJan Schmidt struct btrfs_path *p, u64 time_seq) 22895d9e75c4SJan Schmidt { 22900b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 22915d9e75c4SJan Schmidt struct extent_buffer *b; 22925d9e75c4SJan Schmidt int slot; 22935d9e75c4SJan Schmidt int ret; 22945d9e75c4SJan Schmidt int err; 22955d9e75c4SJan Schmidt int level; 22965d9e75c4SJan Schmidt int lowest_unlock = 1; 22975d9e75c4SJan Schmidt u8 lowest_level = 0; 22985d9e75c4SJan Schmidt 22995d9e75c4SJan Schmidt lowest_level = p->lowest_level; 23005d9e75c4SJan Schmidt WARN_ON(p->nodes[0] != NULL); 2301c922b016SStefan Roesch ASSERT(!p->nowait); 23025d9e75c4SJan Schmidt 23035d9e75c4SJan Schmidt if (p->search_commit_root) { 23045d9e75c4SJan Schmidt BUG_ON(time_seq); 23055d9e75c4SJan Schmidt return btrfs_search_slot(NULL, root, key, p, 0, 0); 23065d9e75c4SJan Schmidt } 23075d9e75c4SJan Schmidt 23085d9e75c4SJan Schmidt again: 2309f3a84ccdSFilipe Manana b = btrfs_get_old_root(root, time_seq); 2310315bed43SNikolay Borisov if (!b) { 2311315bed43SNikolay Borisov ret = -EIO; 2312315bed43SNikolay Borisov goto done; 2313315bed43SNikolay Borisov } 23145d9e75c4SJan Schmidt level = btrfs_header_level(b); 23155d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 23165d9e75c4SJan Schmidt 23175d9e75c4SJan Schmidt while (b) { 2318abe9339dSQu Wenruo int dec = 0; 2319abe9339dSQu Wenruo 23205d9e75c4SJan Schmidt level = btrfs_header_level(b); 23215d9e75c4SJan Schmidt p->nodes[level] = b; 23225d9e75c4SJan Schmidt 23235d9e75c4SJan Schmidt /* 23245d9e75c4SJan Schmidt * we have a lock on b and as long as we aren't changing 23255d9e75c4SJan Schmidt * the tree, there is no way to for the items in b to change. 23265d9e75c4SJan Schmidt * It is safe to drop the lock on our parent before we 23275d9e75c4SJan Schmidt * go through the expensive btree search on b. 23285d9e75c4SJan Schmidt */ 23295d9e75c4SJan Schmidt btrfs_unlock_up_safe(p, level + 1); 23305d9e75c4SJan Schmidt 2331995e9a16SNikolay Borisov ret = btrfs_bin_search(b, key, &slot); 2332cbca7d59SFilipe Manana if (ret < 0) 2333cbca7d59SFilipe Manana goto done; 23345d9e75c4SJan Schmidt 2335abe9339dSQu Wenruo if (level == 0) { 2336abe9339dSQu Wenruo p->slots[level] = slot; 2337abe9339dSQu Wenruo unlock_up(p, level, lowest_unlock, 0, NULL); 2338abe9339dSQu Wenruo goto done; 2339abe9339dSQu Wenruo } 2340abe9339dSQu Wenruo 23415d9e75c4SJan Schmidt if (ret && slot > 0) { 23425d9e75c4SJan Schmidt dec = 1; 2343abe9339dSQu Wenruo slot--; 23445d9e75c4SJan Schmidt } 23455d9e75c4SJan Schmidt p->slots[level] = slot; 23465d9e75c4SJan Schmidt unlock_up(p, level, lowest_unlock, 0, NULL); 23475d9e75c4SJan Schmidt 23485d9e75c4SJan Schmidt if (level == lowest_level) { 23495d9e75c4SJan Schmidt if (dec) 23505d9e75c4SJan Schmidt p->slots[level]++; 23515d9e75c4SJan Schmidt goto done; 23525d9e75c4SJan Schmidt } 23535d9e75c4SJan Schmidt 2354abe9339dSQu Wenruo err = read_block_for_search(root, p, &b, level, slot, key); 23555d9e75c4SJan Schmidt if (err == -EAGAIN) 23565d9e75c4SJan Schmidt goto again; 23575d9e75c4SJan Schmidt if (err) { 23585d9e75c4SJan Schmidt ret = err; 23595d9e75c4SJan Schmidt goto done; 23605d9e75c4SJan Schmidt } 23615d9e75c4SJan Schmidt 23625d9e75c4SJan Schmidt level = btrfs_header_level(b); 23635d9e75c4SJan Schmidt btrfs_tree_read_lock(b); 2364f3a84ccdSFilipe Manana b = btrfs_tree_mod_log_rewind(fs_info, p, b, time_seq); 2365db7f3436SJosef Bacik if (!b) { 2366db7f3436SJosef Bacik ret = -ENOMEM; 2367db7f3436SJosef Bacik goto done; 2368db7f3436SJosef Bacik } 23695d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 23705d9e75c4SJan Schmidt p->nodes[level] = b; 23715d9e75c4SJan Schmidt } 23725d9e75c4SJan Schmidt ret = 1; 23735d9e75c4SJan Schmidt done: 23745d9e75c4SJan Schmidt if (ret < 0) 23755d9e75c4SJan Schmidt btrfs_release_path(p); 23765d9e75c4SJan Schmidt 23775d9e75c4SJan Schmidt return ret; 23785d9e75c4SJan Schmidt } 23795d9e75c4SJan Schmidt 23805d9e75c4SJan Schmidt /* 23812f38b3e1SArne Jansen * helper to use instead of search slot if no exact match is needed but 23822f38b3e1SArne Jansen * instead the next or previous item should be returned. 23832f38b3e1SArne Jansen * When find_higher is true, the next higher item is returned, the next lower 23842f38b3e1SArne Jansen * otherwise. 23852f38b3e1SArne Jansen * When return_any and find_higher are both true, and no higher item is found, 23862f38b3e1SArne Jansen * return the next lower instead. 23872f38b3e1SArne Jansen * When return_any is true and find_higher is false, and no lower item is found, 23882f38b3e1SArne Jansen * return the next higher instead. 23892f38b3e1SArne Jansen * It returns 0 if any item is found, 1 if none is found (tree empty), and 23902f38b3e1SArne Jansen * < 0 on error 23912f38b3e1SArne Jansen */ 23922f38b3e1SArne Jansen int btrfs_search_slot_for_read(struct btrfs_root *root, 2393310712b2SOmar Sandoval const struct btrfs_key *key, 2394310712b2SOmar Sandoval struct btrfs_path *p, int find_higher, 2395310712b2SOmar Sandoval int return_any) 23962f38b3e1SArne Jansen { 23972f38b3e1SArne Jansen int ret; 23982f38b3e1SArne Jansen struct extent_buffer *leaf; 23992f38b3e1SArne Jansen 24002f38b3e1SArne Jansen again: 24012f38b3e1SArne Jansen ret = btrfs_search_slot(NULL, root, key, p, 0, 0); 24022f38b3e1SArne Jansen if (ret <= 0) 24032f38b3e1SArne Jansen return ret; 24042f38b3e1SArne Jansen /* 24052f38b3e1SArne Jansen * a return value of 1 means the path is at the position where the 24062f38b3e1SArne Jansen * item should be inserted. Normally this is the next bigger item, 24072f38b3e1SArne Jansen * but in case the previous item is the last in a leaf, path points 24082f38b3e1SArne Jansen * to the first free slot in the previous leaf, i.e. at an invalid 24092f38b3e1SArne Jansen * item. 24102f38b3e1SArne Jansen */ 24112f38b3e1SArne Jansen leaf = p->nodes[0]; 24122f38b3e1SArne Jansen 24132f38b3e1SArne Jansen if (find_higher) { 24142f38b3e1SArne Jansen if (p->slots[0] >= btrfs_header_nritems(leaf)) { 24152f38b3e1SArne Jansen ret = btrfs_next_leaf(root, p); 24162f38b3e1SArne Jansen if (ret <= 0) 24172f38b3e1SArne Jansen return ret; 24182f38b3e1SArne Jansen if (!return_any) 24192f38b3e1SArne Jansen return 1; 24202f38b3e1SArne Jansen /* 24212f38b3e1SArne Jansen * no higher item found, return the next 24222f38b3e1SArne Jansen * lower instead 24232f38b3e1SArne Jansen */ 24242f38b3e1SArne Jansen return_any = 0; 24252f38b3e1SArne Jansen find_higher = 0; 24262f38b3e1SArne Jansen btrfs_release_path(p); 24272f38b3e1SArne Jansen goto again; 24282f38b3e1SArne Jansen } 24292f38b3e1SArne Jansen } else { 24302f38b3e1SArne Jansen if (p->slots[0] == 0) { 24312f38b3e1SArne Jansen ret = btrfs_prev_leaf(root, p); 2432e6793769SArne Jansen if (ret < 0) 24332f38b3e1SArne Jansen return ret; 2434e6793769SArne Jansen if (!ret) { 243523c6bf6aSFilipe David Borba Manana leaf = p->nodes[0]; 243623c6bf6aSFilipe David Borba Manana if (p->slots[0] == btrfs_header_nritems(leaf)) 243723c6bf6aSFilipe David Borba Manana p->slots[0]--; 2438e6793769SArne Jansen return 0; 2439e6793769SArne Jansen } 24402f38b3e1SArne Jansen if (!return_any) 24412f38b3e1SArne Jansen return 1; 24422f38b3e1SArne Jansen /* 24432f38b3e1SArne Jansen * no lower item found, return the next 24442f38b3e1SArne Jansen * higher instead 24452f38b3e1SArne Jansen */ 24462f38b3e1SArne Jansen return_any = 0; 24472f38b3e1SArne Jansen find_higher = 1; 24482f38b3e1SArne Jansen btrfs_release_path(p); 24492f38b3e1SArne Jansen goto again; 2450e6793769SArne Jansen } else { 24512f38b3e1SArne Jansen --p->slots[0]; 24522f38b3e1SArne Jansen } 24532f38b3e1SArne Jansen } 24542f38b3e1SArne Jansen return 0; 24552f38b3e1SArne Jansen } 24562f38b3e1SArne Jansen 24572f38b3e1SArne Jansen /* 24580ff40a91SMarcos Paulo de Souza * Execute search and call btrfs_previous_item to traverse backwards if the item 24590ff40a91SMarcos Paulo de Souza * was not found. 24600ff40a91SMarcos Paulo de Souza * 24610ff40a91SMarcos Paulo de Souza * Return 0 if found, 1 if not found and < 0 if error. 24620ff40a91SMarcos Paulo de Souza */ 24630ff40a91SMarcos Paulo de Souza int btrfs_search_backwards(struct btrfs_root *root, struct btrfs_key *key, 24640ff40a91SMarcos Paulo de Souza struct btrfs_path *path) 24650ff40a91SMarcos Paulo de Souza { 24660ff40a91SMarcos Paulo de Souza int ret; 24670ff40a91SMarcos Paulo de Souza 24680ff40a91SMarcos Paulo de Souza ret = btrfs_search_slot(NULL, root, key, path, 0, 0); 24690ff40a91SMarcos Paulo de Souza if (ret > 0) 24700ff40a91SMarcos Paulo de Souza ret = btrfs_previous_item(root, path, key->objectid, key->type); 24710ff40a91SMarcos Paulo de Souza 24720ff40a91SMarcos Paulo de Souza if (ret == 0) 24730ff40a91SMarcos Paulo de Souza btrfs_item_key_to_cpu(path->nodes[0], key, path->slots[0]); 24740ff40a91SMarcos Paulo de Souza 24750ff40a91SMarcos Paulo de Souza return ret; 24760ff40a91SMarcos Paulo de Souza } 24770ff40a91SMarcos Paulo de Souza 247843dd529aSDavid Sterba /* 247962142be3SGabriel Niebler * Search for a valid slot for the given path. 248062142be3SGabriel Niebler * 248162142be3SGabriel Niebler * @root: The root node of the tree. 248262142be3SGabriel Niebler * @key: Will contain a valid item if found. 248362142be3SGabriel Niebler * @path: The starting point to validate the slot. 248462142be3SGabriel Niebler * 248562142be3SGabriel Niebler * Return: 0 if the item is valid 248662142be3SGabriel Niebler * 1 if not found 248762142be3SGabriel Niebler * <0 if error. 248862142be3SGabriel Niebler */ 248962142be3SGabriel Niebler int btrfs_get_next_valid_item(struct btrfs_root *root, struct btrfs_key *key, 249062142be3SGabriel Niebler struct btrfs_path *path) 249162142be3SGabriel Niebler { 249262142be3SGabriel Niebler while (1) { 249362142be3SGabriel Niebler int ret; 249462142be3SGabriel Niebler const int slot = path->slots[0]; 249562142be3SGabriel Niebler const struct extent_buffer *leaf = path->nodes[0]; 249662142be3SGabriel Niebler 249762142be3SGabriel Niebler /* This is where we start walking the path. */ 249862142be3SGabriel Niebler if (slot >= btrfs_header_nritems(leaf)) { 249962142be3SGabriel Niebler /* 250062142be3SGabriel Niebler * If we've reached the last slot in this leaf we need 250162142be3SGabriel Niebler * to go to the next leaf and reset the path. 250262142be3SGabriel Niebler */ 250362142be3SGabriel Niebler ret = btrfs_next_leaf(root, path); 250462142be3SGabriel Niebler if (ret) 250562142be3SGabriel Niebler return ret; 250662142be3SGabriel Niebler continue; 250762142be3SGabriel Niebler } 250862142be3SGabriel Niebler /* Store the found, valid item in @key. */ 250962142be3SGabriel Niebler btrfs_item_key_to_cpu(leaf, key, slot); 251062142be3SGabriel Niebler break; 251162142be3SGabriel Niebler } 251262142be3SGabriel Niebler return 0; 251362142be3SGabriel Niebler } 251462142be3SGabriel Niebler 25150ff40a91SMarcos Paulo de Souza /* 251674123bd7SChris Mason * adjust the pointers going up the tree, starting at level 251774123bd7SChris Mason * making sure the right key of each node is points to 'key'. 251874123bd7SChris Mason * This is used after shifting pointers to the left, so it stops 251974123bd7SChris Mason * fixing up pointers when a given leaf/node is not in slot 0 of the 252074123bd7SChris Mason * higher levels 2521aa5d6bedSChris Mason * 252274123bd7SChris Mason */ 2523b167fa91SNikolay Borisov static void fixup_low_keys(struct btrfs_path *path, 25245f39d397SChris Mason struct btrfs_disk_key *key, int level) 2525be0e5c09SChris Mason { 2526be0e5c09SChris Mason int i; 25275f39d397SChris Mason struct extent_buffer *t; 25280e82bcfeSDavid Sterba int ret; 25295f39d397SChris Mason 2530234b63a0SChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 2531be0e5c09SChris Mason int tslot = path->slots[i]; 25320e82bcfeSDavid Sterba 2533eb60ceacSChris Mason if (!path->nodes[i]) 2534be0e5c09SChris Mason break; 25355f39d397SChris Mason t = path->nodes[i]; 2536f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(t, tslot, 253733cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 25380e82bcfeSDavid Sterba BUG_ON(ret < 0); 25395f39d397SChris Mason btrfs_set_node_key(t, key, tslot); 2540d6025579SChris Mason btrfs_mark_buffer_dirty(path->nodes[i]); 2541be0e5c09SChris Mason if (tslot != 0) 2542be0e5c09SChris Mason break; 2543be0e5c09SChris Mason } 2544be0e5c09SChris Mason } 2545be0e5c09SChris Mason 254674123bd7SChris Mason /* 254731840ae1SZheng Yan * update item key. 254831840ae1SZheng Yan * 254931840ae1SZheng Yan * This function isn't completely safe. It's the caller's responsibility 255031840ae1SZheng Yan * that the new key won't break the order 255131840ae1SZheng Yan */ 2552b7a0365eSDaniel Dressler void btrfs_set_item_key_safe(struct btrfs_fs_info *fs_info, 2553b7a0365eSDaniel Dressler struct btrfs_path *path, 2554310712b2SOmar Sandoval const struct btrfs_key *new_key) 255531840ae1SZheng Yan { 255631840ae1SZheng Yan struct btrfs_disk_key disk_key; 255731840ae1SZheng Yan struct extent_buffer *eb; 255831840ae1SZheng Yan int slot; 255931840ae1SZheng Yan 256031840ae1SZheng Yan eb = path->nodes[0]; 256131840ae1SZheng Yan slot = path->slots[0]; 256231840ae1SZheng Yan if (slot > 0) { 256331840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot - 1); 25647c15d410SQu Wenruo if (unlikely(comp_keys(&disk_key, new_key) >= 0)) { 25657c15d410SQu Wenruo btrfs_crit(fs_info, 25667c15d410SQu Wenruo "slot %u key (%llu %u %llu) new key (%llu %u %llu)", 25677c15d410SQu Wenruo slot, btrfs_disk_key_objectid(&disk_key), 25687c15d410SQu Wenruo btrfs_disk_key_type(&disk_key), 25697c15d410SQu Wenruo btrfs_disk_key_offset(&disk_key), 25707c15d410SQu Wenruo new_key->objectid, new_key->type, 25717c15d410SQu Wenruo new_key->offset); 25727c15d410SQu Wenruo btrfs_print_leaf(eb); 25737c15d410SQu Wenruo BUG(); 25747c15d410SQu Wenruo } 257531840ae1SZheng Yan } 257631840ae1SZheng Yan if (slot < btrfs_header_nritems(eb) - 1) { 257731840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot + 1); 25787c15d410SQu Wenruo if (unlikely(comp_keys(&disk_key, new_key) <= 0)) { 25797c15d410SQu Wenruo btrfs_crit(fs_info, 25807c15d410SQu Wenruo "slot %u key (%llu %u %llu) new key (%llu %u %llu)", 25817c15d410SQu Wenruo slot, btrfs_disk_key_objectid(&disk_key), 25827c15d410SQu Wenruo btrfs_disk_key_type(&disk_key), 25837c15d410SQu Wenruo btrfs_disk_key_offset(&disk_key), 25847c15d410SQu Wenruo new_key->objectid, new_key->type, 25857c15d410SQu Wenruo new_key->offset); 25867c15d410SQu Wenruo btrfs_print_leaf(eb); 25877c15d410SQu Wenruo BUG(); 25887c15d410SQu Wenruo } 258931840ae1SZheng Yan } 259031840ae1SZheng Yan 259131840ae1SZheng Yan btrfs_cpu_key_to_disk(&disk_key, new_key); 259231840ae1SZheng Yan btrfs_set_item_key(eb, &disk_key, slot); 259331840ae1SZheng Yan btrfs_mark_buffer_dirty(eb); 259431840ae1SZheng Yan if (slot == 0) 2595b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 259631840ae1SZheng Yan } 259731840ae1SZheng Yan 259831840ae1SZheng Yan /* 2599d16c702fSQu Wenruo * Check key order of two sibling extent buffers. 2600d16c702fSQu Wenruo * 2601d16c702fSQu Wenruo * Return true if something is wrong. 2602d16c702fSQu Wenruo * Return false if everything is fine. 2603d16c702fSQu Wenruo * 2604d16c702fSQu Wenruo * Tree-checker only works inside one tree block, thus the following 2605d16c702fSQu Wenruo * corruption can not be detected by tree-checker: 2606d16c702fSQu Wenruo * 2607d16c702fSQu Wenruo * Leaf @left | Leaf @right 2608d16c702fSQu Wenruo * -------------------------------------------------------------- 2609d16c702fSQu Wenruo * | 1 | 2 | 3 | 4 | 5 | f6 | | 7 | 8 | 2610d16c702fSQu Wenruo * 2611d16c702fSQu Wenruo * Key f6 in leaf @left itself is valid, but not valid when the next 2612d16c702fSQu Wenruo * key in leaf @right is 7. 2613d16c702fSQu Wenruo * This can only be checked at tree block merge time. 2614d16c702fSQu Wenruo * And since tree checker has ensured all key order in each tree block 2615d16c702fSQu Wenruo * is correct, we only need to bother the last key of @left and the first 2616d16c702fSQu Wenruo * key of @right. 2617d16c702fSQu Wenruo */ 2618d16c702fSQu Wenruo static bool check_sibling_keys(struct extent_buffer *left, 2619d16c702fSQu Wenruo struct extent_buffer *right) 2620d16c702fSQu Wenruo { 2621d16c702fSQu Wenruo struct btrfs_key left_last; 2622d16c702fSQu Wenruo struct btrfs_key right_first; 2623d16c702fSQu Wenruo int level = btrfs_header_level(left); 2624d16c702fSQu Wenruo int nr_left = btrfs_header_nritems(left); 2625d16c702fSQu Wenruo int nr_right = btrfs_header_nritems(right); 2626d16c702fSQu Wenruo 2627d16c702fSQu Wenruo /* No key to check in one of the tree blocks */ 2628d16c702fSQu Wenruo if (!nr_left || !nr_right) 2629d16c702fSQu Wenruo return false; 2630d16c702fSQu Wenruo 2631d16c702fSQu Wenruo if (level) { 2632d16c702fSQu Wenruo btrfs_node_key_to_cpu(left, &left_last, nr_left - 1); 2633d16c702fSQu Wenruo btrfs_node_key_to_cpu(right, &right_first, 0); 2634d16c702fSQu Wenruo } else { 2635d16c702fSQu Wenruo btrfs_item_key_to_cpu(left, &left_last, nr_left - 1); 2636d16c702fSQu Wenruo btrfs_item_key_to_cpu(right, &right_first, 0); 2637d16c702fSQu Wenruo } 2638d16c702fSQu Wenruo 2639d16c702fSQu Wenruo if (btrfs_comp_cpu_keys(&left_last, &right_first) >= 0) { 2640d16c702fSQu Wenruo btrfs_crit(left->fs_info, 2641d16c702fSQu Wenruo "bad key order, sibling blocks, left last (%llu %u %llu) right first (%llu %u %llu)", 2642d16c702fSQu Wenruo left_last.objectid, left_last.type, 2643d16c702fSQu Wenruo left_last.offset, right_first.objectid, 2644d16c702fSQu Wenruo right_first.type, right_first.offset); 2645d16c702fSQu Wenruo return true; 2646d16c702fSQu Wenruo } 2647d16c702fSQu Wenruo return false; 2648d16c702fSQu Wenruo } 2649d16c702fSQu Wenruo 2650d16c702fSQu Wenruo /* 265174123bd7SChris Mason * try to push data from one node into the next node left in the 265279f95c82SChris Mason * tree. 2653aa5d6bedSChris Mason * 2654aa5d6bedSChris Mason * returns 0 if some ptrs were pushed left, < 0 if there was some horrible 2655aa5d6bedSChris Mason * error, and > 0 if there was no room in the left hand block. 265674123bd7SChris Mason */ 265798ed5174SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 26582ff7e61eSJeff Mahoney struct extent_buffer *dst, 2659971a1f66SChris Mason struct extent_buffer *src, int empty) 2660be0e5c09SChris Mason { 2661d30a668fSDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info; 2662be0e5c09SChris Mason int push_items = 0; 2663bb803951SChris Mason int src_nritems; 2664bb803951SChris Mason int dst_nritems; 2665aa5d6bedSChris Mason int ret = 0; 2666be0e5c09SChris Mason 26675f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 26685f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 26690b246afaSJeff Mahoney push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems; 26707bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 26717bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 267254aa1f4dSChris Mason 2673bce4eae9SChris Mason if (!empty && src_nritems <= 8) 2674971a1f66SChris Mason return 1; 2675971a1f66SChris Mason 2676d397712bSChris Mason if (push_items <= 0) 2677be0e5c09SChris Mason return 1; 2678be0e5c09SChris Mason 2679bce4eae9SChris Mason if (empty) { 2680971a1f66SChris Mason push_items = min(src_nritems, push_items); 2681bce4eae9SChris Mason if (push_items < src_nritems) { 2682bce4eae9SChris Mason /* leave at least 8 pointers in the node if 2683bce4eae9SChris Mason * we aren't going to empty it 2684bce4eae9SChris Mason */ 2685bce4eae9SChris Mason if (src_nritems - push_items < 8) { 2686bce4eae9SChris Mason if (push_items <= 8) 2687bce4eae9SChris Mason return 1; 2688bce4eae9SChris Mason push_items -= 8; 2689bce4eae9SChris Mason } 2690bce4eae9SChris Mason } 2691bce4eae9SChris Mason } else 2692bce4eae9SChris Mason push_items = min(src_nritems - 8, push_items); 269379f95c82SChris Mason 2694d16c702fSQu Wenruo /* dst is the left eb, src is the middle eb */ 2695d16c702fSQu Wenruo if (check_sibling_keys(dst, src)) { 2696d16c702fSQu Wenruo ret = -EUCLEAN; 2697d16c702fSQu Wenruo btrfs_abort_transaction(trans, ret); 2698d16c702fSQu Wenruo return ret; 2699d16c702fSQu Wenruo } 2700f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_eb_copy(dst, src, dst_nritems, 0, push_items); 27015de865eeSFilipe David Borba Manana if (ret) { 270266642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 27035de865eeSFilipe David Borba Manana return ret; 27045de865eeSFilipe David Borba Manana } 27055f39d397SChris Mason copy_extent_buffer(dst, src, 2706e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(dst, dst_nritems), 2707e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(src, 0), 2708123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 27095f39d397SChris Mason 2710bb803951SChris Mason if (push_items < src_nritems) { 271157911b8bSJan Schmidt /* 2712f3a84ccdSFilipe Manana * Don't call btrfs_tree_mod_log_insert_move() here, key removal 2713f3a84ccdSFilipe Manana * was already fully logged by btrfs_tree_mod_log_eb_copy() above. 271457911b8bSJan Schmidt */ 2715e23efd8eSJosef Bacik memmove_extent_buffer(src, btrfs_node_key_ptr_offset(src, 0), 2716e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(src, push_items), 2717e2fa7227SChris Mason (src_nritems - push_items) * 2718123abc88SChris Mason sizeof(struct btrfs_key_ptr)); 2719bb803951SChris Mason } 27205f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 27215f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 27225f39d397SChris Mason btrfs_mark_buffer_dirty(src); 27235f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 272431840ae1SZheng Yan 2725bb803951SChris Mason return ret; 2726be0e5c09SChris Mason } 2727be0e5c09SChris Mason 272897571fd0SChris Mason /* 272979f95c82SChris Mason * try to push data from one node into the next node right in the 273079f95c82SChris Mason * tree. 273179f95c82SChris Mason * 273279f95c82SChris Mason * returns 0 if some ptrs were pushed, < 0 if there was some horrible 273379f95c82SChris Mason * error, and > 0 if there was no room in the right hand block. 273479f95c82SChris Mason * 273579f95c82SChris Mason * this will only push up to 1/2 the contents of the left node over 273679f95c82SChris Mason */ 27375f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 27385f39d397SChris Mason struct extent_buffer *dst, 27395f39d397SChris Mason struct extent_buffer *src) 274079f95c82SChris Mason { 274155d32ed8SDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info; 274279f95c82SChris Mason int push_items = 0; 274379f95c82SChris Mason int max_push; 274479f95c82SChris Mason int src_nritems; 274579f95c82SChris Mason int dst_nritems; 274679f95c82SChris Mason int ret = 0; 274779f95c82SChris Mason 27487bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 27497bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 27507bb86316SChris Mason 27515f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 27525f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 27530b246afaSJeff Mahoney push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems; 2754d397712bSChris Mason if (push_items <= 0) 275579f95c82SChris Mason return 1; 2756bce4eae9SChris Mason 2757d397712bSChris Mason if (src_nritems < 4) 2758bce4eae9SChris Mason return 1; 275979f95c82SChris Mason 276079f95c82SChris Mason max_push = src_nritems / 2 + 1; 276179f95c82SChris Mason /* don't try to empty the node */ 2762d397712bSChris Mason if (max_push >= src_nritems) 276379f95c82SChris Mason return 1; 2764252c38f0SYan 276579f95c82SChris Mason if (max_push < push_items) 276679f95c82SChris Mason push_items = max_push; 276779f95c82SChris Mason 2768d16c702fSQu Wenruo /* dst is the right eb, src is the middle eb */ 2769d16c702fSQu Wenruo if (check_sibling_keys(src, dst)) { 2770d16c702fSQu Wenruo ret = -EUCLEAN; 2771d16c702fSQu Wenruo btrfs_abort_transaction(trans, ret); 2772d16c702fSQu Wenruo return ret; 2773d16c702fSQu Wenruo } 2774f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_move(dst, push_items, 0, dst_nritems); 2775bf1d3425SDavid Sterba BUG_ON(ret < 0); 2776e23efd8eSJosef Bacik memmove_extent_buffer(dst, btrfs_node_key_ptr_offset(dst, push_items), 2777e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(dst, 0), 27785f39d397SChris Mason (dst_nritems) * 27795f39d397SChris Mason sizeof(struct btrfs_key_ptr)); 2780d6025579SChris Mason 2781f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_eb_copy(dst, src, 0, src_nritems - push_items, 2782ed874f0dSDavid Sterba push_items); 27835de865eeSFilipe David Borba Manana if (ret) { 278466642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 27855de865eeSFilipe David Borba Manana return ret; 27865de865eeSFilipe David Borba Manana } 27875f39d397SChris Mason copy_extent_buffer(dst, src, 2788e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(dst, 0), 2789e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(src, src_nritems - push_items), 2790123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 279179f95c82SChris Mason 27925f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 27935f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 279479f95c82SChris Mason 27955f39d397SChris Mason btrfs_mark_buffer_dirty(src); 27965f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 279731840ae1SZheng Yan 279879f95c82SChris Mason return ret; 279979f95c82SChris Mason } 280079f95c82SChris Mason 280179f95c82SChris Mason /* 280297571fd0SChris Mason * helper function to insert a new root level in the tree. 280397571fd0SChris Mason * A new node is allocated, and a single item is inserted to 280497571fd0SChris Mason * point to the existing root 2805aa5d6bedSChris Mason * 2806aa5d6bedSChris Mason * returns zero on success or < 0 on failure. 280797571fd0SChris Mason */ 2808d397712bSChris Mason static noinline int insert_new_root(struct btrfs_trans_handle *trans, 28095f39d397SChris Mason struct btrfs_root *root, 2810fdd99c72SLiu Bo struct btrfs_path *path, int level) 281174123bd7SChris Mason { 28120b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 28137bb86316SChris Mason u64 lower_gen; 28145f39d397SChris Mason struct extent_buffer *lower; 28155f39d397SChris Mason struct extent_buffer *c; 2816925baeddSChris Mason struct extent_buffer *old; 28175f39d397SChris Mason struct btrfs_disk_key lower_key; 2818d9d19a01SDavid Sterba int ret; 28195c680ed6SChris Mason 28205c680ed6SChris Mason BUG_ON(path->nodes[level]); 28215c680ed6SChris Mason BUG_ON(path->nodes[level-1] != root->node); 28225c680ed6SChris Mason 28237bb86316SChris Mason lower = path->nodes[level-1]; 28247bb86316SChris Mason if (level == 1) 28257bb86316SChris Mason btrfs_item_key(lower, &lower_key, 0); 28267bb86316SChris Mason else 28277bb86316SChris Mason btrfs_node_key(lower, &lower_key, 0); 28287bb86316SChris Mason 282979bd3712SFilipe Manana c = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 283079bd3712SFilipe Manana &lower_key, level, root->node->start, 0, 2831cf6f34aaSJosef Bacik BTRFS_NESTING_NEW_ROOT); 28325f39d397SChris Mason if (IS_ERR(c)) 28335f39d397SChris Mason return PTR_ERR(c); 2834925baeddSChris Mason 28350b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 2836f0486c68SYan, Zheng 28375f39d397SChris Mason btrfs_set_header_nritems(c, 1); 28385f39d397SChris Mason btrfs_set_node_key(c, &lower_key, 0); 2839db94535dSChris Mason btrfs_set_node_blockptr(c, 0, lower->start); 28407bb86316SChris Mason lower_gen = btrfs_header_generation(lower); 284131840ae1SZheng Yan WARN_ON(lower_gen != trans->transid); 28427bb86316SChris Mason 28437bb86316SChris Mason btrfs_set_node_ptr_generation(c, 0, lower_gen); 28445f39d397SChris Mason 28455f39d397SChris Mason btrfs_mark_buffer_dirty(c); 2846d5719762SChris Mason 2847925baeddSChris Mason old = root->node; 2848406808abSFilipe Manana ret = btrfs_tree_mod_log_insert_root(root->node, c, false); 2849d9d19a01SDavid Sterba BUG_ON(ret < 0); 2850240f62c8SChris Mason rcu_assign_pointer(root->node, c); 2851925baeddSChris Mason 2852925baeddSChris Mason /* the super has an extra ref to root->node */ 2853925baeddSChris Mason free_extent_buffer(old); 2854925baeddSChris Mason 28550b86a832SChris Mason add_root_to_dirty_list(root); 285667439dadSDavid Sterba atomic_inc(&c->refs); 28575f39d397SChris Mason path->nodes[level] = c; 2858ac5887c8SJosef Bacik path->locks[level] = BTRFS_WRITE_LOCK; 285974123bd7SChris Mason path->slots[level] = 0; 286074123bd7SChris Mason return 0; 286174123bd7SChris Mason } 28625c680ed6SChris Mason 28635c680ed6SChris Mason /* 28645c680ed6SChris Mason * worker function to insert a single pointer in a node. 28655c680ed6SChris Mason * the node should have enough room for the pointer already 286697571fd0SChris Mason * 28675c680ed6SChris Mason * slot and level indicate where you want the key to go, and 28685c680ed6SChris Mason * blocknr is the block the key points to. 28695c680ed6SChris Mason */ 2870143bede5SJeff Mahoney static void insert_ptr(struct btrfs_trans_handle *trans, 28716ad3cf6dSDavid Sterba struct btrfs_path *path, 2872143bede5SJeff Mahoney struct btrfs_disk_key *key, u64 bytenr, 2873c3e06965SJan Schmidt int slot, int level) 28745c680ed6SChris Mason { 28755f39d397SChris Mason struct extent_buffer *lower; 28765c680ed6SChris Mason int nritems; 2877f3ea38daSJan Schmidt int ret; 28785c680ed6SChris Mason 28795c680ed6SChris Mason BUG_ON(!path->nodes[level]); 288049d0c642SFilipe Manana btrfs_assert_tree_write_locked(path->nodes[level]); 28815f39d397SChris Mason lower = path->nodes[level]; 28825f39d397SChris Mason nritems = btrfs_header_nritems(lower); 2883c293498bSStoyan Gaydarov BUG_ON(slot > nritems); 28846ad3cf6dSDavid Sterba BUG_ON(nritems == BTRFS_NODEPTRS_PER_BLOCK(trans->fs_info)); 288574123bd7SChris Mason if (slot != nritems) { 2886bf1d3425SDavid Sterba if (level) { 2887f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_move(lower, slot + 1, 2888f3a84ccdSFilipe Manana slot, nritems - slot); 2889bf1d3425SDavid Sterba BUG_ON(ret < 0); 2890bf1d3425SDavid Sterba } 28915f39d397SChris Mason memmove_extent_buffer(lower, 2892e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(lower, slot + 1), 2893e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(lower, slot), 2894123abc88SChris Mason (nritems - slot) * sizeof(struct btrfs_key_ptr)); 289574123bd7SChris Mason } 2896c3e06965SJan Schmidt if (level) { 2897f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(lower, slot, 289833cff222SFilipe Manana BTRFS_MOD_LOG_KEY_ADD); 2899f3ea38daSJan Schmidt BUG_ON(ret < 0); 2900f3ea38daSJan Schmidt } 29015f39d397SChris Mason btrfs_set_node_key(lower, key, slot); 2902db94535dSChris Mason btrfs_set_node_blockptr(lower, slot, bytenr); 290374493f7aSChris Mason WARN_ON(trans->transid == 0); 290474493f7aSChris Mason btrfs_set_node_ptr_generation(lower, slot, trans->transid); 29055f39d397SChris Mason btrfs_set_header_nritems(lower, nritems + 1); 29065f39d397SChris Mason btrfs_mark_buffer_dirty(lower); 290774123bd7SChris Mason } 290874123bd7SChris Mason 290997571fd0SChris Mason /* 291097571fd0SChris Mason * split the node at the specified level in path in two. 291197571fd0SChris Mason * The path is corrected to point to the appropriate node after the split 291297571fd0SChris Mason * 291397571fd0SChris Mason * Before splitting this tries to make some room in the node by pushing 291497571fd0SChris Mason * left and right, if either one works, it returns right away. 2915aa5d6bedSChris Mason * 2916aa5d6bedSChris Mason * returns 0 on success and < 0 on failure 291797571fd0SChris Mason */ 2918e02119d5SChris Mason static noinline int split_node(struct btrfs_trans_handle *trans, 2919e02119d5SChris Mason struct btrfs_root *root, 2920e02119d5SChris Mason struct btrfs_path *path, int level) 2921be0e5c09SChris Mason { 29220b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 29235f39d397SChris Mason struct extent_buffer *c; 29245f39d397SChris Mason struct extent_buffer *split; 29255f39d397SChris Mason struct btrfs_disk_key disk_key; 2926be0e5c09SChris Mason int mid; 29275c680ed6SChris Mason int ret; 29287518a238SChris Mason u32 c_nritems; 2929be0e5c09SChris Mason 29305f39d397SChris Mason c = path->nodes[level]; 29317bb86316SChris Mason WARN_ON(btrfs_header_generation(c) != trans->transid); 29325f39d397SChris Mason if (c == root->node) { 2933d9abbf1cSJan Schmidt /* 293490f8d62eSJan Schmidt * trying to split the root, lets make a new one 293590f8d62eSJan Schmidt * 2936fdd99c72SLiu Bo * tree mod log: We don't log_removal old root in 293790f8d62eSJan Schmidt * insert_new_root, because that root buffer will be kept as a 293890f8d62eSJan Schmidt * normal node. We are going to log removal of half of the 2939f3a84ccdSFilipe Manana * elements below with btrfs_tree_mod_log_eb_copy(). We're 2940f3a84ccdSFilipe Manana * holding a tree lock on the buffer, which is why we cannot 2941f3a84ccdSFilipe Manana * race with other tree_mod_log users. 2942d9abbf1cSJan Schmidt */ 2943fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, level + 1); 29445c680ed6SChris Mason if (ret) 29455c680ed6SChris Mason return ret; 2946b3612421SChris Mason } else { 2947e66f709bSChris Mason ret = push_nodes_for_insert(trans, root, path, level); 29485f39d397SChris Mason c = path->nodes[level]; 29495f39d397SChris Mason if (!ret && btrfs_header_nritems(c) < 29500b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3) 2951e66f709bSChris Mason return 0; 295254aa1f4dSChris Mason if (ret < 0) 295354aa1f4dSChris Mason return ret; 29545c680ed6SChris Mason } 2955e66f709bSChris Mason 29565f39d397SChris Mason c_nritems = btrfs_header_nritems(c); 29575d4f98a2SYan Zheng mid = (c_nritems + 1) / 2; 29585d4f98a2SYan Zheng btrfs_node_key(c, &disk_key, mid); 29597bb86316SChris Mason 296079bd3712SFilipe Manana split = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 296179bd3712SFilipe Manana &disk_key, level, c->start, 0, 296279bd3712SFilipe Manana BTRFS_NESTING_SPLIT); 29635f39d397SChris Mason if (IS_ERR(split)) 29645f39d397SChris Mason return PTR_ERR(split); 296554aa1f4dSChris Mason 29660b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 2967bc877d28SNikolay Borisov ASSERT(btrfs_header_level(c) == level); 29685f39d397SChris Mason 2969f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_eb_copy(split, c, 0, mid, c_nritems - mid); 29705de865eeSFilipe David Borba Manana if (ret) { 297166642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 29725de865eeSFilipe David Borba Manana return ret; 29735de865eeSFilipe David Borba Manana } 29745f39d397SChris Mason copy_extent_buffer(split, c, 2975e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(split, 0), 2976e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(c, mid), 2977123abc88SChris Mason (c_nritems - mid) * sizeof(struct btrfs_key_ptr)); 29785f39d397SChris Mason btrfs_set_header_nritems(split, c_nritems - mid); 29795f39d397SChris Mason btrfs_set_header_nritems(c, mid); 2980aa5d6bedSChris Mason 29815f39d397SChris Mason btrfs_mark_buffer_dirty(c); 29825f39d397SChris Mason btrfs_mark_buffer_dirty(split); 29835f39d397SChris Mason 29846ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, split->start, 2985c3e06965SJan Schmidt path->slots[level + 1] + 1, level + 1); 2986aa5d6bedSChris Mason 29875de08d7dSChris Mason if (path->slots[level] >= mid) { 29885c680ed6SChris Mason path->slots[level] -= mid; 2989925baeddSChris Mason btrfs_tree_unlock(c); 29905f39d397SChris Mason free_extent_buffer(c); 29915f39d397SChris Mason path->nodes[level] = split; 29925c680ed6SChris Mason path->slots[level + 1] += 1; 2993eb60ceacSChris Mason } else { 2994925baeddSChris Mason btrfs_tree_unlock(split); 29955f39d397SChris Mason free_extent_buffer(split); 2996be0e5c09SChris Mason } 2997d5286a92SNikolay Borisov return 0; 2998be0e5c09SChris Mason } 2999be0e5c09SChris Mason 300074123bd7SChris Mason /* 300174123bd7SChris Mason * how many bytes are required to store the items in a leaf. start 300274123bd7SChris Mason * and nr indicate which items in the leaf to check. This totals up the 300374123bd7SChris Mason * space used both by the item structs and the item data 300474123bd7SChris Mason */ 30055f39d397SChris Mason static int leaf_space_used(struct extent_buffer *l, int start, int nr) 3006be0e5c09SChris Mason { 3007be0e5c09SChris Mason int data_len; 30085f39d397SChris Mason int nritems = btrfs_header_nritems(l); 3009d4dbff95SChris Mason int end = min(nritems, start + nr) - 1; 3010be0e5c09SChris Mason 3011be0e5c09SChris Mason if (!nr) 3012be0e5c09SChris Mason return 0; 30133212fa14SJosef Bacik data_len = btrfs_item_offset(l, start) + btrfs_item_size(l, start); 30143212fa14SJosef Bacik data_len = data_len - btrfs_item_offset(l, end); 30150783fcfcSChris Mason data_len += sizeof(struct btrfs_item) * nr; 3016d4dbff95SChris Mason WARN_ON(data_len < 0); 3017be0e5c09SChris Mason return data_len; 3018be0e5c09SChris Mason } 3019be0e5c09SChris Mason 302074123bd7SChris Mason /* 3021d4dbff95SChris Mason * The space between the end of the leaf items and 3022d4dbff95SChris Mason * the start of the leaf data. IOW, how much room 3023d4dbff95SChris Mason * the leaf has left for both items and data 3024d4dbff95SChris Mason */ 3025e902baacSDavid Sterba noinline int btrfs_leaf_free_space(struct extent_buffer *leaf) 3026d4dbff95SChris Mason { 3027e902baacSDavid Sterba struct btrfs_fs_info *fs_info = leaf->fs_info; 30285f39d397SChris Mason int nritems = btrfs_header_nritems(leaf); 30295f39d397SChris Mason int ret; 30300b246afaSJeff Mahoney 30310b246afaSJeff Mahoney ret = BTRFS_LEAF_DATA_SIZE(fs_info) - leaf_space_used(leaf, 0, nritems); 30325f39d397SChris Mason if (ret < 0) { 30330b246afaSJeff Mahoney btrfs_crit(fs_info, 3034efe120a0SFrank Holton "leaf free space ret %d, leaf data size %lu, used %d nritems %d", 3035da17066cSJeff Mahoney ret, 30360b246afaSJeff Mahoney (unsigned long) BTRFS_LEAF_DATA_SIZE(fs_info), 30375f39d397SChris Mason leaf_space_used(leaf, 0, nritems), nritems); 30385f39d397SChris Mason } 30395f39d397SChris Mason return ret; 3040d4dbff95SChris Mason } 3041d4dbff95SChris Mason 304299d8f83cSChris Mason /* 304399d8f83cSChris Mason * min slot controls the lowest index we're willing to push to the 304499d8f83cSChris Mason * right. We'll push up to and including min_slot, but no lower 304599d8f83cSChris Mason */ 3046ed25dab3SJosef Bacik static noinline int __push_leaf_right(struct btrfs_trans_handle *trans, 3047ed25dab3SJosef Bacik struct btrfs_path *path, 304844871b1bSChris Mason int data_size, int empty, 304944871b1bSChris Mason struct extent_buffer *right, 305099d8f83cSChris Mason int free_space, u32 left_nritems, 305199d8f83cSChris Mason u32 min_slot) 305200ec4c51SChris Mason { 3053f72f0010SDavid Sterba struct btrfs_fs_info *fs_info = right->fs_info; 30545f39d397SChris Mason struct extent_buffer *left = path->nodes[0]; 305544871b1bSChris Mason struct extent_buffer *upper = path->nodes[1]; 3056cfed81a0SChris Mason struct btrfs_map_token token; 30575f39d397SChris Mason struct btrfs_disk_key disk_key; 305800ec4c51SChris Mason int slot; 305934a38218SChris Mason u32 i; 306000ec4c51SChris Mason int push_space = 0; 306100ec4c51SChris Mason int push_items = 0; 306234a38218SChris Mason u32 nr; 30637518a238SChris Mason u32 right_nritems; 30645f39d397SChris Mason u32 data_end; 3065db94535dSChris Mason u32 this_item_size; 306600ec4c51SChris Mason 306734a38218SChris Mason if (empty) 306834a38218SChris Mason nr = 0; 306934a38218SChris Mason else 307099d8f83cSChris Mason nr = max_t(u32, 1, min_slot); 307134a38218SChris Mason 307231840ae1SZheng Yan if (path->slots[0] >= left_nritems) 307387b29b20SYan Zheng push_space += data_size; 307431840ae1SZheng Yan 307544871b1bSChris Mason slot = path->slots[1]; 307634a38218SChris Mason i = left_nritems - 1; 307734a38218SChris Mason while (i >= nr) { 307831840ae1SZheng Yan if (!empty && push_items > 0) { 307931840ae1SZheng Yan if (path->slots[0] > i) 308031840ae1SZheng Yan break; 308131840ae1SZheng Yan if (path->slots[0] == i) { 3082e902baacSDavid Sterba int space = btrfs_leaf_free_space(left); 3083e902baacSDavid Sterba 308431840ae1SZheng Yan if (space + push_space * 2 > free_space) 308531840ae1SZheng Yan break; 308631840ae1SZheng Yan } 308731840ae1SZheng Yan } 308831840ae1SZheng Yan 308900ec4c51SChris Mason if (path->slots[0] == i) 309087b29b20SYan Zheng push_space += data_size; 3091db94535dSChris Mason 30923212fa14SJosef Bacik this_item_size = btrfs_item_size(left, i); 309374794207SJosef Bacik if (this_item_size + sizeof(struct btrfs_item) + 309474794207SJosef Bacik push_space > free_space) 309500ec4c51SChris Mason break; 309631840ae1SZheng Yan 309700ec4c51SChris Mason push_items++; 309874794207SJosef Bacik push_space += this_item_size + sizeof(struct btrfs_item); 309934a38218SChris Mason if (i == 0) 310034a38218SChris Mason break; 310134a38218SChris Mason i--; 3102db94535dSChris Mason } 31035f39d397SChris Mason 3104925baeddSChris Mason if (push_items == 0) 3105925baeddSChris Mason goto out_unlock; 31065f39d397SChris Mason 31076c1500f2SJulia Lawall WARN_ON(!empty && push_items == left_nritems); 31085f39d397SChris Mason 310900ec4c51SChris Mason /* push left to right */ 31105f39d397SChris Mason right_nritems = btrfs_header_nritems(right); 311134a38218SChris Mason 3112dc2e724eSJosef Bacik push_space = btrfs_item_data_end(left, left_nritems - push_items); 31138f881e8cSDavid Sterba push_space -= leaf_data_end(left); 31145f39d397SChris Mason 311500ec4c51SChris Mason /* make room in the right data area */ 31168f881e8cSDavid Sterba data_end = leaf_data_end(right); 3117637e3b48SJosef Bacik memmove_leaf_data(right, data_end - push_space, data_end, 31180b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - data_end); 31195f39d397SChris Mason 312000ec4c51SChris Mason /* copy from the left data area */ 3121637e3b48SJosef Bacik copy_leaf_data(right, left, BTRFS_LEAF_DATA_SIZE(fs_info) - push_space, 3122637e3b48SJosef Bacik leaf_data_end(left), push_space); 31235f39d397SChris Mason 3124637e3b48SJosef Bacik memmove_leaf_items(right, push_items, 0, right_nritems); 31255f39d397SChris Mason 312600ec4c51SChris Mason /* copy the items from left to right */ 3127637e3b48SJosef Bacik copy_leaf_items(right, left, 0, left_nritems - push_items, push_items); 312800ec4c51SChris Mason 312900ec4c51SChris Mason /* update the item pointers */ 3130c82f823cSDavid Sterba btrfs_init_map_token(&token, right); 31317518a238SChris Mason right_nritems += push_items; 31325f39d397SChris Mason btrfs_set_header_nritems(right, right_nritems); 31330b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info); 31347518a238SChris Mason for (i = 0; i < right_nritems; i++) { 31353212fa14SJosef Bacik push_space -= btrfs_token_item_size(&token, i); 31363212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, push_space); 3137db94535dSChris Mason } 3138db94535dSChris Mason 31397518a238SChris Mason left_nritems -= push_items; 31405f39d397SChris Mason btrfs_set_header_nritems(left, left_nritems); 314100ec4c51SChris Mason 314234a38218SChris Mason if (left_nritems) 31435f39d397SChris Mason btrfs_mark_buffer_dirty(left); 3144f0486c68SYan, Zheng else 3145190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, left); 3146f0486c68SYan, Zheng 31475f39d397SChris Mason btrfs_mark_buffer_dirty(right); 3148a429e513SChris Mason 31495f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 31505f39d397SChris Mason btrfs_set_node_key(upper, &disk_key, slot + 1); 3151d6025579SChris Mason btrfs_mark_buffer_dirty(upper); 315202217ed2SChris Mason 315300ec4c51SChris Mason /* then fixup the leaf pointer in the path */ 31547518a238SChris Mason if (path->slots[0] >= left_nritems) { 31557518a238SChris Mason path->slots[0] -= left_nritems; 3156925baeddSChris Mason if (btrfs_header_nritems(path->nodes[0]) == 0) 3157190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, path->nodes[0]); 3158925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 31595f39d397SChris Mason free_extent_buffer(path->nodes[0]); 31605f39d397SChris Mason path->nodes[0] = right; 316100ec4c51SChris Mason path->slots[1] += 1; 316200ec4c51SChris Mason } else { 3163925baeddSChris Mason btrfs_tree_unlock(right); 31645f39d397SChris Mason free_extent_buffer(right); 316500ec4c51SChris Mason } 316600ec4c51SChris Mason return 0; 3167925baeddSChris Mason 3168925baeddSChris Mason out_unlock: 3169925baeddSChris Mason btrfs_tree_unlock(right); 3170925baeddSChris Mason free_extent_buffer(right); 3171925baeddSChris Mason return 1; 317200ec4c51SChris Mason } 3173925baeddSChris Mason 317400ec4c51SChris Mason /* 317544871b1bSChris Mason * push some data in the path leaf to the right, trying to free up at 317674123bd7SChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 317744871b1bSChris Mason * 317844871b1bSChris Mason * returns 1 if the push failed because the other node didn't have enough 317944871b1bSChris Mason * room, 0 if everything worked out and < 0 if there were major errors. 318099d8f83cSChris Mason * 318199d8f83cSChris Mason * this will push starting from min_slot to the end of the leaf. It won't 318299d8f83cSChris Mason * push any slot lower than min_slot 318374123bd7SChris Mason */ 318444871b1bSChris Mason static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root 318599d8f83cSChris Mason *root, struct btrfs_path *path, 318699d8f83cSChris Mason int min_data_size, int data_size, 318799d8f83cSChris Mason int empty, u32 min_slot) 3188be0e5c09SChris Mason { 318944871b1bSChris Mason struct extent_buffer *left = path->nodes[0]; 319044871b1bSChris Mason struct extent_buffer *right; 319144871b1bSChris Mason struct extent_buffer *upper; 319244871b1bSChris Mason int slot; 319344871b1bSChris Mason int free_space; 319444871b1bSChris Mason u32 left_nritems; 319544871b1bSChris Mason int ret; 319644871b1bSChris Mason 319744871b1bSChris Mason if (!path->nodes[1]) 319844871b1bSChris Mason return 1; 319944871b1bSChris Mason 320044871b1bSChris Mason slot = path->slots[1]; 320144871b1bSChris Mason upper = path->nodes[1]; 320244871b1bSChris Mason if (slot >= btrfs_header_nritems(upper) - 1) 320344871b1bSChris Mason return 1; 320444871b1bSChris Mason 320549d0c642SFilipe Manana btrfs_assert_tree_write_locked(path->nodes[1]); 320644871b1bSChris Mason 32074b231ae4SDavid Sterba right = btrfs_read_node_slot(upper, slot + 1); 3208fb770ae4SLiu Bo if (IS_ERR(right)) 3209*9cf14029SJosef Bacik return PTR_ERR(right); 321091ca338dSTsutomu Itoh 3211bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT); 321244871b1bSChris Mason 3213e902baacSDavid Sterba free_space = btrfs_leaf_free_space(right); 321444871b1bSChris Mason if (free_space < data_size) 321544871b1bSChris Mason goto out_unlock; 321644871b1bSChris Mason 321744871b1bSChris Mason ret = btrfs_cow_block(trans, root, right, upper, 3218bf59a5a2SJosef Bacik slot + 1, &right, BTRFS_NESTING_RIGHT_COW); 321944871b1bSChris Mason if (ret) 322044871b1bSChris Mason goto out_unlock; 322144871b1bSChris Mason 322244871b1bSChris Mason left_nritems = btrfs_header_nritems(left); 322344871b1bSChris Mason if (left_nritems == 0) 322444871b1bSChris Mason goto out_unlock; 322544871b1bSChris Mason 3226d16c702fSQu Wenruo if (check_sibling_keys(left, right)) { 3227d16c702fSQu Wenruo ret = -EUCLEAN; 3228d16c702fSQu Wenruo btrfs_tree_unlock(right); 3229d16c702fSQu Wenruo free_extent_buffer(right); 3230d16c702fSQu Wenruo return ret; 3231d16c702fSQu Wenruo } 32322ef1fed2SFilipe David Borba Manana if (path->slots[0] == left_nritems && !empty) { 32332ef1fed2SFilipe David Borba Manana /* Key greater than all keys in the leaf, right neighbor has 32342ef1fed2SFilipe David Borba Manana * enough room for it and we're not emptying our leaf to delete 32352ef1fed2SFilipe David Borba Manana * it, therefore use right neighbor to insert the new item and 323652042d8eSAndrea Gelmini * no need to touch/dirty our left leaf. */ 32372ef1fed2SFilipe David Borba Manana btrfs_tree_unlock(left); 32382ef1fed2SFilipe David Borba Manana free_extent_buffer(left); 32392ef1fed2SFilipe David Borba Manana path->nodes[0] = right; 32402ef1fed2SFilipe David Borba Manana path->slots[0] = 0; 32412ef1fed2SFilipe David Borba Manana path->slots[1]++; 32422ef1fed2SFilipe David Borba Manana return 0; 32432ef1fed2SFilipe David Borba Manana } 32442ef1fed2SFilipe David Borba Manana 3245ed25dab3SJosef Bacik return __push_leaf_right(trans, path, min_data_size, empty, right, 3246ed25dab3SJosef Bacik free_space, left_nritems, min_slot); 324744871b1bSChris Mason out_unlock: 324844871b1bSChris Mason btrfs_tree_unlock(right); 324944871b1bSChris Mason free_extent_buffer(right); 325044871b1bSChris Mason return 1; 325144871b1bSChris Mason } 325244871b1bSChris Mason 325344871b1bSChris Mason /* 325444871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 325544871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 325699d8f83cSChris Mason * 325799d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 325899d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us do all the 325999d8f83cSChris Mason * items 326044871b1bSChris Mason */ 3261ed25dab3SJosef Bacik static noinline int __push_leaf_left(struct btrfs_trans_handle *trans, 3262ed25dab3SJosef Bacik struct btrfs_path *path, int data_size, 326344871b1bSChris Mason int empty, struct extent_buffer *left, 326499d8f83cSChris Mason int free_space, u32 right_nritems, 326599d8f83cSChris Mason u32 max_slot) 326644871b1bSChris Mason { 32678087c193SDavid Sterba struct btrfs_fs_info *fs_info = left->fs_info; 32685f39d397SChris Mason struct btrfs_disk_key disk_key; 32695f39d397SChris Mason struct extent_buffer *right = path->nodes[0]; 3270be0e5c09SChris Mason int i; 3271be0e5c09SChris Mason int push_space = 0; 3272be0e5c09SChris Mason int push_items = 0; 32737518a238SChris Mason u32 old_left_nritems; 327434a38218SChris Mason u32 nr; 3275aa5d6bedSChris Mason int ret = 0; 3276db94535dSChris Mason u32 this_item_size; 3277db94535dSChris Mason u32 old_left_item_size; 3278cfed81a0SChris Mason struct btrfs_map_token token; 3279cfed81a0SChris Mason 328034a38218SChris Mason if (empty) 328199d8f83cSChris Mason nr = min(right_nritems, max_slot); 328234a38218SChris Mason else 328399d8f83cSChris Mason nr = min(right_nritems - 1, max_slot); 328434a38218SChris Mason 328534a38218SChris Mason for (i = 0; i < nr; i++) { 328631840ae1SZheng Yan if (!empty && push_items > 0) { 328731840ae1SZheng Yan if (path->slots[0] < i) 328831840ae1SZheng Yan break; 328931840ae1SZheng Yan if (path->slots[0] == i) { 3290e902baacSDavid Sterba int space = btrfs_leaf_free_space(right); 3291e902baacSDavid Sterba 329231840ae1SZheng Yan if (space + push_space * 2 > free_space) 329331840ae1SZheng Yan break; 329431840ae1SZheng Yan } 329531840ae1SZheng Yan } 329631840ae1SZheng Yan 3297be0e5c09SChris Mason if (path->slots[0] == i) 329887b29b20SYan Zheng push_space += data_size; 3299db94535dSChris Mason 33003212fa14SJosef Bacik this_item_size = btrfs_item_size(right, i); 330174794207SJosef Bacik if (this_item_size + sizeof(struct btrfs_item) + push_space > 330274794207SJosef Bacik free_space) 3303be0e5c09SChris Mason break; 3304db94535dSChris Mason 3305be0e5c09SChris Mason push_items++; 330674794207SJosef Bacik push_space += this_item_size + sizeof(struct btrfs_item); 3307be0e5c09SChris Mason } 3308db94535dSChris Mason 3309be0e5c09SChris Mason if (push_items == 0) { 3310925baeddSChris Mason ret = 1; 3311925baeddSChris Mason goto out; 3312be0e5c09SChris Mason } 3313fae7f21cSDulshani Gunawardhana WARN_ON(!empty && push_items == btrfs_header_nritems(right)); 33145f39d397SChris Mason 3315be0e5c09SChris Mason /* push data from right to left */ 3316637e3b48SJosef Bacik copy_leaf_items(left, right, btrfs_header_nritems(left), 0, push_items); 33175f39d397SChris Mason 33180b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info) - 33193212fa14SJosef Bacik btrfs_item_offset(right, push_items - 1); 33205f39d397SChris Mason 3321637e3b48SJosef Bacik copy_leaf_data(left, right, leaf_data_end(left) - push_space, 3322637e3b48SJosef Bacik btrfs_item_offset(right, push_items - 1), push_space); 33235f39d397SChris Mason old_left_nritems = btrfs_header_nritems(left); 332487b29b20SYan Zheng BUG_ON(old_left_nritems <= 0); 3325eb60ceacSChris Mason 3326c82f823cSDavid Sterba btrfs_init_map_token(&token, left); 33273212fa14SJosef Bacik old_left_item_size = btrfs_item_offset(left, old_left_nritems - 1); 3328be0e5c09SChris Mason for (i = old_left_nritems; i < old_left_nritems + push_items; i++) { 33295f39d397SChris Mason u32 ioff; 3330db94535dSChris Mason 33313212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 33323212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, 3333cc4c13d5SDavid Sterba ioff - (BTRFS_LEAF_DATA_SIZE(fs_info) - old_left_item_size)); 3334be0e5c09SChris Mason } 33355f39d397SChris Mason btrfs_set_header_nritems(left, old_left_nritems + push_items); 3336be0e5c09SChris Mason 3337be0e5c09SChris Mason /* fixup right node */ 333831b1a2bdSJulia Lawall if (push_items > right_nritems) 333931b1a2bdSJulia Lawall WARN(1, KERN_CRIT "push items %d nr %u\n", push_items, 3340d397712bSChris Mason right_nritems); 334134a38218SChris Mason 334234a38218SChris Mason if (push_items < right_nritems) { 33433212fa14SJosef Bacik push_space = btrfs_item_offset(right, push_items - 1) - 33448f881e8cSDavid Sterba leaf_data_end(right); 3345637e3b48SJosef Bacik memmove_leaf_data(right, 33460b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - push_space, 33478f881e8cSDavid Sterba leaf_data_end(right), push_space); 33485f39d397SChris Mason 3349637e3b48SJosef Bacik memmove_leaf_items(right, 0, push_items, 3350637e3b48SJosef Bacik btrfs_header_nritems(right) - push_items); 335134a38218SChris Mason } 3352c82f823cSDavid Sterba 3353c82f823cSDavid Sterba btrfs_init_map_token(&token, right); 3354eef1c494SYan right_nritems -= push_items; 3355eef1c494SYan btrfs_set_header_nritems(right, right_nritems); 33560b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info); 33575f39d397SChris Mason for (i = 0; i < right_nritems; i++) { 33583212fa14SJosef Bacik push_space = push_space - btrfs_token_item_size(&token, i); 33593212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, push_space); 3360db94535dSChris Mason } 3361eb60ceacSChris Mason 33625f39d397SChris Mason btrfs_mark_buffer_dirty(left); 336334a38218SChris Mason if (right_nritems) 33645f39d397SChris Mason btrfs_mark_buffer_dirty(right); 3365f0486c68SYan, Zheng else 3366190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, right); 3367098f59c2SChris Mason 33685f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 3369b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 3370be0e5c09SChris Mason 3371be0e5c09SChris Mason /* then fixup the leaf pointer in the path */ 3372be0e5c09SChris Mason if (path->slots[0] < push_items) { 3373be0e5c09SChris Mason path->slots[0] += old_left_nritems; 3374925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 33755f39d397SChris Mason free_extent_buffer(path->nodes[0]); 33765f39d397SChris Mason path->nodes[0] = left; 3377be0e5c09SChris Mason path->slots[1] -= 1; 3378be0e5c09SChris Mason } else { 3379925baeddSChris Mason btrfs_tree_unlock(left); 33805f39d397SChris Mason free_extent_buffer(left); 3381be0e5c09SChris Mason path->slots[0] -= push_items; 3382be0e5c09SChris Mason } 3383eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 3384aa5d6bedSChris Mason return ret; 3385925baeddSChris Mason out: 3386925baeddSChris Mason btrfs_tree_unlock(left); 3387925baeddSChris Mason free_extent_buffer(left); 3388925baeddSChris Mason return ret; 3389be0e5c09SChris Mason } 3390be0e5c09SChris Mason 339174123bd7SChris Mason /* 339244871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 339344871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 339499d8f83cSChris Mason * 339599d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 339699d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us push all the 339799d8f83cSChris Mason * items 339844871b1bSChris Mason */ 339944871b1bSChris Mason static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root 340099d8f83cSChris Mason *root, struct btrfs_path *path, int min_data_size, 340199d8f83cSChris Mason int data_size, int empty, u32 max_slot) 340244871b1bSChris Mason { 340344871b1bSChris Mason struct extent_buffer *right = path->nodes[0]; 340444871b1bSChris Mason struct extent_buffer *left; 340544871b1bSChris Mason int slot; 340644871b1bSChris Mason int free_space; 340744871b1bSChris Mason u32 right_nritems; 340844871b1bSChris Mason int ret = 0; 340944871b1bSChris Mason 341044871b1bSChris Mason slot = path->slots[1]; 341144871b1bSChris Mason if (slot == 0) 341244871b1bSChris Mason return 1; 341344871b1bSChris Mason if (!path->nodes[1]) 341444871b1bSChris Mason return 1; 341544871b1bSChris Mason 341644871b1bSChris Mason right_nritems = btrfs_header_nritems(right); 341744871b1bSChris Mason if (right_nritems == 0) 341844871b1bSChris Mason return 1; 341944871b1bSChris Mason 342049d0c642SFilipe Manana btrfs_assert_tree_write_locked(path->nodes[1]); 342144871b1bSChris Mason 34224b231ae4SDavid Sterba left = btrfs_read_node_slot(path->nodes[1], slot - 1); 3423fb770ae4SLiu Bo if (IS_ERR(left)) 3424*9cf14029SJosef Bacik return PTR_ERR(left); 342591ca338dSTsutomu Itoh 3426bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT); 342744871b1bSChris Mason 3428e902baacSDavid Sterba free_space = btrfs_leaf_free_space(left); 342944871b1bSChris Mason if (free_space < data_size) { 343044871b1bSChris Mason ret = 1; 343144871b1bSChris Mason goto out; 343244871b1bSChris Mason } 343344871b1bSChris Mason 343444871b1bSChris Mason ret = btrfs_cow_block(trans, root, left, 34359631e4ccSJosef Bacik path->nodes[1], slot - 1, &left, 3436bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW); 343744871b1bSChris Mason if (ret) { 343844871b1bSChris Mason /* we hit -ENOSPC, but it isn't fatal here */ 343979787eaaSJeff Mahoney if (ret == -ENOSPC) 344044871b1bSChris Mason ret = 1; 344144871b1bSChris Mason goto out; 344244871b1bSChris Mason } 344344871b1bSChris Mason 3444d16c702fSQu Wenruo if (check_sibling_keys(left, right)) { 3445d16c702fSQu Wenruo ret = -EUCLEAN; 3446d16c702fSQu Wenruo goto out; 3447d16c702fSQu Wenruo } 3448ed25dab3SJosef Bacik return __push_leaf_left(trans, path, min_data_size, empty, left, 3449ed25dab3SJosef Bacik free_space, right_nritems, max_slot); 345044871b1bSChris Mason out: 345144871b1bSChris Mason btrfs_tree_unlock(left); 345244871b1bSChris Mason free_extent_buffer(left); 345344871b1bSChris Mason return ret; 345444871b1bSChris Mason } 345544871b1bSChris Mason 345644871b1bSChris Mason /* 345774123bd7SChris Mason * split the path's leaf in two, making sure there is at least data_size 345874123bd7SChris Mason * available for the resulting leaf level of the path. 345974123bd7SChris Mason */ 3460143bede5SJeff Mahoney static noinline void copy_for_split(struct btrfs_trans_handle *trans, 346144871b1bSChris Mason struct btrfs_path *path, 346244871b1bSChris Mason struct extent_buffer *l, 346344871b1bSChris Mason struct extent_buffer *right, 346444871b1bSChris Mason int slot, int mid, int nritems) 3465be0e5c09SChris Mason { 346694f94ad9SDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info; 3467be0e5c09SChris Mason int data_copy_size; 3468be0e5c09SChris Mason int rt_data_off; 3469be0e5c09SChris Mason int i; 3470d4dbff95SChris Mason struct btrfs_disk_key disk_key; 3471cfed81a0SChris Mason struct btrfs_map_token token; 3472cfed81a0SChris Mason 34735f39d397SChris Mason nritems = nritems - mid; 34745f39d397SChris Mason btrfs_set_header_nritems(right, nritems); 3475dc2e724eSJosef Bacik data_copy_size = btrfs_item_data_end(l, mid) - leaf_data_end(l); 34765f39d397SChris Mason 3477637e3b48SJosef Bacik copy_leaf_items(right, l, 0, mid, nritems); 34785f39d397SChris Mason 3479637e3b48SJosef Bacik copy_leaf_data(right, l, BTRFS_LEAF_DATA_SIZE(fs_info) - data_copy_size, 34808f881e8cSDavid Sterba leaf_data_end(l), data_copy_size); 348174123bd7SChris Mason 3482dc2e724eSJosef Bacik rt_data_off = BTRFS_LEAF_DATA_SIZE(fs_info) - btrfs_item_data_end(l, mid); 34835f39d397SChris Mason 3484c82f823cSDavid Sterba btrfs_init_map_token(&token, right); 34855f39d397SChris Mason for (i = 0; i < nritems; i++) { 3486db94535dSChris Mason u32 ioff; 3487db94535dSChris Mason 34883212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 34893212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff + rt_data_off); 34900783fcfcSChris Mason } 349174123bd7SChris Mason 34925f39d397SChris Mason btrfs_set_header_nritems(l, mid); 34935f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 34946ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, right->start, path->slots[1] + 1, 1); 34955f39d397SChris Mason 34965f39d397SChris Mason btrfs_mark_buffer_dirty(right); 34975f39d397SChris Mason btrfs_mark_buffer_dirty(l); 3498eb60ceacSChris Mason BUG_ON(path->slots[0] != slot); 34995f39d397SChris Mason 3500be0e5c09SChris Mason if (mid <= slot) { 3501925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 35025f39d397SChris Mason free_extent_buffer(path->nodes[0]); 35035f39d397SChris Mason path->nodes[0] = right; 3504be0e5c09SChris Mason path->slots[0] -= mid; 3505be0e5c09SChris Mason path->slots[1] += 1; 3506925baeddSChris Mason } else { 3507925baeddSChris Mason btrfs_tree_unlock(right); 35085f39d397SChris Mason free_extent_buffer(right); 3509925baeddSChris Mason } 35105f39d397SChris Mason 3511eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 351244871b1bSChris Mason } 351344871b1bSChris Mason 351444871b1bSChris Mason /* 351599d8f83cSChris Mason * double splits happen when we need to insert a big item in the middle 351699d8f83cSChris Mason * of a leaf. A double split can leave us with 3 mostly empty leaves: 351799d8f83cSChris Mason * leaf: [ slots 0 - N] [ our target ] [ N + 1 - total in leaf ] 351899d8f83cSChris Mason * A B C 351999d8f83cSChris Mason * 352099d8f83cSChris Mason * We avoid this by trying to push the items on either side of our target 352199d8f83cSChris Mason * into the adjacent leaves. If all goes well we can avoid the double split 352299d8f83cSChris Mason * completely. 352399d8f83cSChris Mason */ 352499d8f83cSChris Mason static noinline int push_for_double_split(struct btrfs_trans_handle *trans, 352599d8f83cSChris Mason struct btrfs_root *root, 352699d8f83cSChris Mason struct btrfs_path *path, 352799d8f83cSChris Mason int data_size) 352899d8f83cSChris Mason { 352999d8f83cSChris Mason int ret; 353099d8f83cSChris Mason int progress = 0; 353199d8f83cSChris Mason int slot; 353299d8f83cSChris Mason u32 nritems; 35335a4267caSFilipe David Borba Manana int space_needed = data_size; 353499d8f83cSChris Mason 353599d8f83cSChris Mason slot = path->slots[0]; 35365a4267caSFilipe David Borba Manana if (slot < btrfs_header_nritems(path->nodes[0])) 3537e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(path->nodes[0]); 353899d8f83cSChris Mason 353999d8f83cSChris Mason /* 354099d8f83cSChris Mason * try to push all the items after our slot into the 354199d8f83cSChris Mason * right leaf 354299d8f83cSChris Mason */ 35435a4267caSFilipe David Borba Manana ret = push_leaf_right(trans, root, path, 1, space_needed, 0, slot); 354499d8f83cSChris Mason if (ret < 0) 354599d8f83cSChris Mason return ret; 354699d8f83cSChris Mason 354799d8f83cSChris Mason if (ret == 0) 354899d8f83cSChris Mason progress++; 354999d8f83cSChris Mason 355099d8f83cSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 355199d8f83cSChris Mason /* 355299d8f83cSChris Mason * our goal is to get our slot at the start or end of a leaf. If 355399d8f83cSChris Mason * we've done so we're done 355499d8f83cSChris Mason */ 355599d8f83cSChris Mason if (path->slots[0] == 0 || path->slots[0] == nritems) 355699d8f83cSChris Mason return 0; 355799d8f83cSChris Mason 3558e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= data_size) 355999d8f83cSChris Mason return 0; 356099d8f83cSChris Mason 356199d8f83cSChris Mason /* try to push all the items before our slot into the next leaf */ 356299d8f83cSChris Mason slot = path->slots[0]; 3563263d3995SFilipe Manana space_needed = data_size; 3564263d3995SFilipe Manana if (slot > 0) 3565e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(path->nodes[0]); 35665a4267caSFilipe David Borba Manana ret = push_leaf_left(trans, root, path, 1, space_needed, 0, slot); 356799d8f83cSChris Mason if (ret < 0) 356899d8f83cSChris Mason return ret; 356999d8f83cSChris Mason 357099d8f83cSChris Mason if (ret == 0) 357199d8f83cSChris Mason progress++; 357299d8f83cSChris Mason 357399d8f83cSChris Mason if (progress) 357499d8f83cSChris Mason return 0; 357599d8f83cSChris Mason return 1; 357699d8f83cSChris Mason } 357799d8f83cSChris Mason 357899d8f83cSChris Mason /* 357944871b1bSChris Mason * split the path's leaf in two, making sure there is at least data_size 358044871b1bSChris Mason * available for the resulting leaf level of the path. 358144871b1bSChris Mason * 358244871b1bSChris Mason * returns 0 if all went well and < 0 on failure. 358344871b1bSChris Mason */ 358444871b1bSChris Mason static noinline int split_leaf(struct btrfs_trans_handle *trans, 358544871b1bSChris Mason struct btrfs_root *root, 3586310712b2SOmar Sandoval const struct btrfs_key *ins_key, 358744871b1bSChris Mason struct btrfs_path *path, int data_size, 358844871b1bSChris Mason int extend) 358944871b1bSChris Mason { 35905d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 359144871b1bSChris Mason struct extent_buffer *l; 359244871b1bSChris Mason u32 nritems; 359344871b1bSChris Mason int mid; 359444871b1bSChris Mason int slot; 359544871b1bSChris Mason struct extent_buffer *right; 3596b7a0365eSDaniel Dressler struct btrfs_fs_info *fs_info = root->fs_info; 359744871b1bSChris Mason int ret = 0; 359844871b1bSChris Mason int wret; 35995d4f98a2SYan Zheng int split; 360044871b1bSChris Mason int num_doubles = 0; 360199d8f83cSChris Mason int tried_avoid_double = 0; 360244871b1bSChris Mason 3603a5719521SYan, Zheng l = path->nodes[0]; 3604a5719521SYan, Zheng slot = path->slots[0]; 36053212fa14SJosef Bacik if (extend && data_size + btrfs_item_size(l, slot) + 36060b246afaSJeff Mahoney sizeof(struct btrfs_item) > BTRFS_LEAF_DATA_SIZE(fs_info)) 3607a5719521SYan, Zheng return -EOVERFLOW; 3608a5719521SYan, Zheng 360944871b1bSChris Mason /* first try to make some room by pushing left and right */ 361033157e05SLiu Bo if (data_size && path->nodes[1]) { 36115a4267caSFilipe David Borba Manana int space_needed = data_size; 36125a4267caSFilipe David Borba Manana 36135a4267caSFilipe David Borba Manana if (slot < btrfs_header_nritems(l)) 3614e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(l); 36155a4267caSFilipe David Borba Manana 36165a4267caSFilipe David Borba Manana wret = push_leaf_right(trans, root, path, space_needed, 36175a4267caSFilipe David Borba Manana space_needed, 0, 0); 361844871b1bSChris Mason if (wret < 0) 361944871b1bSChris Mason return wret; 362044871b1bSChris Mason if (wret) { 3621263d3995SFilipe Manana space_needed = data_size; 3622263d3995SFilipe Manana if (slot > 0) 3623e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(l); 36245a4267caSFilipe David Borba Manana wret = push_leaf_left(trans, root, path, space_needed, 36255a4267caSFilipe David Borba Manana space_needed, 0, (u32)-1); 362644871b1bSChris Mason if (wret < 0) 362744871b1bSChris Mason return wret; 362844871b1bSChris Mason } 362944871b1bSChris Mason l = path->nodes[0]; 363044871b1bSChris Mason 363144871b1bSChris Mason /* did the pushes work? */ 3632e902baacSDavid Sterba if (btrfs_leaf_free_space(l) >= data_size) 363344871b1bSChris Mason return 0; 363444871b1bSChris Mason } 363544871b1bSChris Mason 363644871b1bSChris Mason if (!path->nodes[1]) { 3637fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, 1); 363844871b1bSChris Mason if (ret) 363944871b1bSChris Mason return ret; 364044871b1bSChris Mason } 364144871b1bSChris Mason again: 36425d4f98a2SYan Zheng split = 1; 364344871b1bSChris Mason l = path->nodes[0]; 364444871b1bSChris Mason slot = path->slots[0]; 364544871b1bSChris Mason nritems = btrfs_header_nritems(l); 364644871b1bSChris Mason mid = (nritems + 1) / 2; 364744871b1bSChris Mason 36485d4f98a2SYan Zheng if (mid <= slot) { 36495d4f98a2SYan Zheng if (nritems == 1 || 36505d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + data_size > 36510b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info)) { 36525d4f98a2SYan Zheng if (slot >= nritems) { 36535d4f98a2SYan Zheng split = 0; 36545d4f98a2SYan Zheng } else { 36555d4f98a2SYan Zheng mid = slot; 36565d4f98a2SYan Zheng if (mid != nritems && 36575d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 36580b246afaSJeff Mahoney data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) { 365999d8f83cSChris Mason if (data_size && !tried_avoid_double) 366099d8f83cSChris Mason goto push_for_double; 36615d4f98a2SYan Zheng split = 2; 36625d4f98a2SYan Zheng } 36635d4f98a2SYan Zheng } 36645d4f98a2SYan Zheng } 36655d4f98a2SYan Zheng } else { 36665d4f98a2SYan Zheng if (leaf_space_used(l, 0, mid) + data_size > 36670b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info)) { 36685d4f98a2SYan Zheng if (!extend && data_size && slot == 0) { 36695d4f98a2SYan Zheng split = 0; 36705d4f98a2SYan Zheng } else if ((extend || !data_size) && slot == 0) { 36715d4f98a2SYan Zheng mid = 1; 36725d4f98a2SYan Zheng } else { 36735d4f98a2SYan Zheng mid = slot; 36745d4f98a2SYan Zheng if (mid != nritems && 36755d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 36760b246afaSJeff Mahoney data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) { 367799d8f83cSChris Mason if (data_size && !tried_avoid_double) 367899d8f83cSChris Mason goto push_for_double; 36795d4f98a2SYan Zheng split = 2; 36805d4f98a2SYan Zheng } 36815d4f98a2SYan Zheng } 36825d4f98a2SYan Zheng } 36835d4f98a2SYan Zheng } 36845d4f98a2SYan Zheng 36855d4f98a2SYan Zheng if (split == 0) 36865d4f98a2SYan Zheng btrfs_cpu_key_to_disk(&disk_key, ins_key); 36875d4f98a2SYan Zheng else 36885d4f98a2SYan Zheng btrfs_item_key(l, &disk_key, mid); 36895d4f98a2SYan Zheng 3690ca9d473aSJosef Bacik /* 3691ca9d473aSJosef Bacik * We have to about BTRFS_NESTING_NEW_ROOT here if we've done a double 3692ca9d473aSJosef Bacik * split, because we're only allowed to have MAX_LOCKDEP_SUBCLASSES 3693ca9d473aSJosef Bacik * subclasses, which is 8 at the time of this patch, and we've maxed it 3694ca9d473aSJosef Bacik * out. In the future we could add a 3695ca9d473aSJosef Bacik * BTRFS_NESTING_SPLIT_THE_SPLITTENING if we need to, but for now just 3696ca9d473aSJosef Bacik * use BTRFS_NESTING_NEW_ROOT. 3697ca9d473aSJosef Bacik */ 369879bd3712SFilipe Manana right = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 369979bd3712SFilipe Manana &disk_key, 0, l->start, 0, 370079bd3712SFilipe Manana num_doubles ? BTRFS_NESTING_NEW_ROOT : 3701ca9d473aSJosef Bacik BTRFS_NESTING_SPLIT); 3702f0486c68SYan, Zheng if (IS_ERR(right)) 370344871b1bSChris Mason return PTR_ERR(right); 3704f0486c68SYan, Zheng 37050b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 370644871b1bSChris Mason 37075d4f98a2SYan Zheng if (split == 0) { 370844871b1bSChris Mason if (mid <= slot) { 370944871b1bSChris Mason btrfs_set_header_nritems(right, 0); 37106ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, 37112ff7e61eSJeff Mahoney right->start, path->slots[1] + 1, 1); 371244871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 371344871b1bSChris Mason free_extent_buffer(path->nodes[0]); 371444871b1bSChris Mason path->nodes[0] = right; 371544871b1bSChris Mason path->slots[0] = 0; 371644871b1bSChris Mason path->slots[1] += 1; 371744871b1bSChris Mason } else { 371844871b1bSChris Mason btrfs_set_header_nritems(right, 0); 37196ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, 37202ff7e61eSJeff Mahoney right->start, path->slots[1], 1); 372144871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 372244871b1bSChris Mason free_extent_buffer(path->nodes[0]); 372344871b1bSChris Mason path->nodes[0] = right; 372444871b1bSChris Mason path->slots[0] = 0; 3725143bede5SJeff Mahoney if (path->slots[1] == 0) 3726b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 37275d4f98a2SYan Zheng } 3728196e0249SLiu Bo /* 3729196e0249SLiu Bo * We create a new leaf 'right' for the required ins_len and 3730196e0249SLiu Bo * we'll do btrfs_mark_buffer_dirty() on this leaf after copying 3731196e0249SLiu Bo * the content of ins_len to 'right'. 3732196e0249SLiu Bo */ 373344871b1bSChris Mason return ret; 373444871b1bSChris Mason } 373544871b1bSChris Mason 373694f94ad9SDavid Sterba copy_for_split(trans, path, l, right, slot, mid, nritems); 373744871b1bSChris Mason 37385d4f98a2SYan Zheng if (split == 2) { 3739cc0c5538SChris Mason BUG_ON(num_doubles != 0); 3740cc0c5538SChris Mason num_doubles++; 3741cc0c5538SChris Mason goto again; 37423326d1b0SChris Mason } 374344871b1bSChris Mason 3744143bede5SJeff Mahoney return 0; 374599d8f83cSChris Mason 374699d8f83cSChris Mason push_for_double: 374799d8f83cSChris Mason push_for_double_split(trans, root, path, data_size); 374899d8f83cSChris Mason tried_avoid_double = 1; 3749e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= data_size) 375099d8f83cSChris Mason return 0; 375199d8f83cSChris Mason goto again; 3752be0e5c09SChris Mason } 3753be0e5c09SChris Mason 3754ad48fd75SYan, Zheng static noinline int setup_leaf_for_split(struct btrfs_trans_handle *trans, 3755ad48fd75SYan, Zheng struct btrfs_root *root, 3756ad48fd75SYan, Zheng struct btrfs_path *path, int ins_len) 3757ad48fd75SYan, Zheng { 3758ad48fd75SYan, Zheng struct btrfs_key key; 3759ad48fd75SYan, Zheng struct extent_buffer *leaf; 3760ad48fd75SYan, Zheng struct btrfs_file_extent_item *fi; 3761ad48fd75SYan, Zheng u64 extent_len = 0; 3762ad48fd75SYan, Zheng u32 item_size; 3763ad48fd75SYan, Zheng int ret; 3764ad48fd75SYan, Zheng 3765ad48fd75SYan, Zheng leaf = path->nodes[0]; 3766ad48fd75SYan, Zheng btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 3767ad48fd75SYan, Zheng 3768ad48fd75SYan, Zheng BUG_ON(key.type != BTRFS_EXTENT_DATA_KEY && 3769ad48fd75SYan, Zheng key.type != BTRFS_EXTENT_CSUM_KEY); 3770ad48fd75SYan, Zheng 3771e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) >= ins_len) 3772ad48fd75SYan, Zheng return 0; 3773ad48fd75SYan, Zheng 37743212fa14SJosef Bacik item_size = btrfs_item_size(leaf, path->slots[0]); 3775ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 3776ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 3777ad48fd75SYan, Zheng struct btrfs_file_extent_item); 3778ad48fd75SYan, Zheng extent_len = btrfs_file_extent_num_bytes(leaf, fi); 3779ad48fd75SYan, Zheng } 3780b3b4aa74SDavid Sterba btrfs_release_path(path); 3781ad48fd75SYan, Zheng 3782ad48fd75SYan, Zheng path->keep_locks = 1; 3783ad48fd75SYan, Zheng path->search_for_split = 1; 3784ad48fd75SYan, Zheng ret = btrfs_search_slot(trans, root, &key, path, 0, 1); 3785ad48fd75SYan, Zheng path->search_for_split = 0; 3786a8df6fe6SFilipe Manana if (ret > 0) 3787a8df6fe6SFilipe Manana ret = -EAGAIN; 3788ad48fd75SYan, Zheng if (ret < 0) 3789ad48fd75SYan, Zheng goto err; 3790ad48fd75SYan, Zheng 3791ad48fd75SYan, Zheng ret = -EAGAIN; 3792ad48fd75SYan, Zheng leaf = path->nodes[0]; 3793a8df6fe6SFilipe Manana /* if our item isn't there, return now */ 37943212fa14SJosef Bacik if (item_size != btrfs_item_size(leaf, path->slots[0])) 3795ad48fd75SYan, Zheng goto err; 3796ad48fd75SYan, Zheng 3797109f6aefSChris Mason /* the leaf has changed, it now has room. return now */ 3798e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= ins_len) 3799109f6aefSChris Mason goto err; 3800109f6aefSChris Mason 3801ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 3802ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 3803ad48fd75SYan, Zheng struct btrfs_file_extent_item); 3804ad48fd75SYan, Zheng if (extent_len != btrfs_file_extent_num_bytes(leaf, fi)) 3805ad48fd75SYan, Zheng goto err; 3806ad48fd75SYan, Zheng } 3807ad48fd75SYan, Zheng 3808ad48fd75SYan, Zheng ret = split_leaf(trans, root, &key, path, ins_len, 1); 3809f0486c68SYan, Zheng if (ret) 3810f0486c68SYan, Zheng goto err; 3811ad48fd75SYan, Zheng 3812ad48fd75SYan, Zheng path->keep_locks = 0; 3813ad48fd75SYan, Zheng btrfs_unlock_up_safe(path, 1); 3814ad48fd75SYan, Zheng return 0; 3815ad48fd75SYan, Zheng err: 3816ad48fd75SYan, Zheng path->keep_locks = 0; 3817ad48fd75SYan, Zheng return ret; 3818ad48fd75SYan, Zheng } 3819ad48fd75SYan, Zheng 382025263cd7SDavid Sterba static noinline int split_item(struct btrfs_path *path, 3821310712b2SOmar Sandoval const struct btrfs_key *new_key, 3822459931ecSChris Mason unsigned long split_offset) 3823459931ecSChris Mason { 3824459931ecSChris Mason struct extent_buffer *leaf; 3825c91666b1SJosef Bacik int orig_slot, slot; 3826ad48fd75SYan, Zheng char *buf; 3827459931ecSChris Mason u32 nritems; 3828ad48fd75SYan, Zheng u32 item_size; 3829459931ecSChris Mason u32 orig_offset; 3830459931ecSChris Mason struct btrfs_disk_key disk_key; 3831459931ecSChris Mason 3832459931ecSChris Mason leaf = path->nodes[0]; 3833e902baacSDavid Sterba BUG_ON(btrfs_leaf_free_space(leaf) < sizeof(struct btrfs_item)); 3834b9473439SChris Mason 3835c91666b1SJosef Bacik orig_slot = path->slots[0]; 38363212fa14SJosef Bacik orig_offset = btrfs_item_offset(leaf, path->slots[0]); 38373212fa14SJosef Bacik item_size = btrfs_item_size(leaf, path->slots[0]); 3838459931ecSChris Mason 3839459931ecSChris Mason buf = kmalloc(item_size, GFP_NOFS); 3840ad48fd75SYan, Zheng if (!buf) 3841ad48fd75SYan, Zheng return -ENOMEM; 3842ad48fd75SYan, Zheng 3843459931ecSChris Mason read_extent_buffer(leaf, buf, btrfs_item_ptr_offset(leaf, 3844459931ecSChris Mason path->slots[0]), item_size); 3845ad48fd75SYan, Zheng 3846459931ecSChris Mason slot = path->slots[0] + 1; 3847459931ecSChris Mason nritems = btrfs_header_nritems(leaf); 3848459931ecSChris Mason if (slot != nritems) { 3849459931ecSChris Mason /* shift the items */ 3850637e3b48SJosef Bacik memmove_leaf_items(leaf, slot + 1, slot, nritems - slot); 3851459931ecSChris Mason } 3852459931ecSChris Mason 3853459931ecSChris Mason btrfs_cpu_key_to_disk(&disk_key, new_key); 3854459931ecSChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 3855459931ecSChris Mason 38563212fa14SJosef Bacik btrfs_set_item_offset(leaf, slot, orig_offset); 38573212fa14SJosef Bacik btrfs_set_item_size(leaf, slot, item_size - split_offset); 3858459931ecSChris Mason 38593212fa14SJosef Bacik btrfs_set_item_offset(leaf, orig_slot, 3860459931ecSChris Mason orig_offset + item_size - split_offset); 38613212fa14SJosef Bacik btrfs_set_item_size(leaf, orig_slot, split_offset); 3862459931ecSChris Mason 3863459931ecSChris Mason btrfs_set_header_nritems(leaf, nritems + 1); 3864459931ecSChris Mason 3865459931ecSChris Mason /* write the data for the start of the original item */ 3866459931ecSChris Mason write_extent_buffer(leaf, buf, 3867459931ecSChris Mason btrfs_item_ptr_offset(leaf, path->slots[0]), 3868459931ecSChris Mason split_offset); 3869459931ecSChris Mason 3870459931ecSChris Mason /* write the data for the new item */ 3871459931ecSChris Mason write_extent_buffer(leaf, buf + split_offset, 3872459931ecSChris Mason btrfs_item_ptr_offset(leaf, slot), 3873459931ecSChris Mason item_size - split_offset); 3874459931ecSChris Mason btrfs_mark_buffer_dirty(leaf); 3875459931ecSChris Mason 3876e902baacSDavid Sterba BUG_ON(btrfs_leaf_free_space(leaf) < 0); 3877459931ecSChris Mason kfree(buf); 3878ad48fd75SYan, Zheng return 0; 3879ad48fd75SYan, Zheng } 3880ad48fd75SYan, Zheng 3881ad48fd75SYan, Zheng /* 3882ad48fd75SYan, Zheng * This function splits a single item into two items, 3883ad48fd75SYan, Zheng * giving 'new_key' to the new item and splitting the 3884ad48fd75SYan, Zheng * old one at split_offset (from the start of the item). 3885ad48fd75SYan, Zheng * 3886ad48fd75SYan, Zheng * The path may be released by this operation. After 3887ad48fd75SYan, Zheng * the split, the path is pointing to the old item. The 3888ad48fd75SYan, Zheng * new item is going to be in the same node as the old one. 3889ad48fd75SYan, Zheng * 3890ad48fd75SYan, Zheng * Note, the item being split must be smaller enough to live alone on 3891ad48fd75SYan, Zheng * a tree block with room for one extra struct btrfs_item 3892ad48fd75SYan, Zheng * 3893ad48fd75SYan, Zheng * This allows us to split the item in place, keeping a lock on the 3894ad48fd75SYan, Zheng * leaf the entire time. 3895ad48fd75SYan, Zheng */ 3896ad48fd75SYan, Zheng int btrfs_split_item(struct btrfs_trans_handle *trans, 3897ad48fd75SYan, Zheng struct btrfs_root *root, 3898ad48fd75SYan, Zheng struct btrfs_path *path, 3899310712b2SOmar Sandoval const struct btrfs_key *new_key, 3900ad48fd75SYan, Zheng unsigned long split_offset) 3901ad48fd75SYan, Zheng { 3902ad48fd75SYan, Zheng int ret; 3903ad48fd75SYan, Zheng ret = setup_leaf_for_split(trans, root, path, 3904ad48fd75SYan, Zheng sizeof(struct btrfs_item)); 3905ad48fd75SYan, Zheng if (ret) 3906459931ecSChris Mason return ret; 3907ad48fd75SYan, Zheng 390825263cd7SDavid Sterba ret = split_item(path, new_key, split_offset); 3909ad48fd75SYan, Zheng return ret; 3910ad48fd75SYan, Zheng } 3911ad48fd75SYan, Zheng 3912ad48fd75SYan, Zheng /* 3913d352ac68SChris Mason * make the item pointed to by the path smaller. new_size indicates 3914d352ac68SChris Mason * how small to make it, and from_end tells us if we just chop bytes 3915d352ac68SChris Mason * off the end of the item or if we shift the item to chop bytes off 3916d352ac68SChris Mason * the front. 3917d352ac68SChris Mason */ 391878ac4f9eSDavid Sterba void btrfs_truncate_item(struct btrfs_path *path, u32 new_size, int from_end) 3919b18c6685SChris Mason { 3920b18c6685SChris Mason int slot; 39215f39d397SChris Mason struct extent_buffer *leaf; 3922b18c6685SChris Mason u32 nritems; 3923b18c6685SChris Mason unsigned int data_end; 3924b18c6685SChris Mason unsigned int old_data_start; 3925b18c6685SChris Mason unsigned int old_size; 3926b18c6685SChris Mason unsigned int size_diff; 3927b18c6685SChris Mason int i; 3928cfed81a0SChris Mason struct btrfs_map_token token; 3929cfed81a0SChris Mason 39305f39d397SChris Mason leaf = path->nodes[0]; 3931179e29e4SChris Mason slot = path->slots[0]; 3932179e29e4SChris Mason 39333212fa14SJosef Bacik old_size = btrfs_item_size(leaf, slot); 3934179e29e4SChris Mason if (old_size == new_size) 3935143bede5SJeff Mahoney return; 3936b18c6685SChris Mason 39375f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 39388f881e8cSDavid Sterba data_end = leaf_data_end(leaf); 3939b18c6685SChris Mason 39403212fa14SJosef Bacik old_data_start = btrfs_item_offset(leaf, slot); 3941179e29e4SChris Mason 3942b18c6685SChris Mason size_diff = old_size - new_size; 3943b18c6685SChris Mason 3944b18c6685SChris Mason BUG_ON(slot < 0); 3945b18c6685SChris Mason BUG_ON(slot >= nritems); 3946b18c6685SChris Mason 3947b18c6685SChris Mason /* 3948b18c6685SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 3949b18c6685SChris Mason */ 3950b18c6685SChris Mason /* first correct the data pointers */ 3951c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 3952b18c6685SChris Mason for (i = slot; i < nritems; i++) { 39535f39d397SChris Mason u32 ioff; 3954db94535dSChris Mason 39553212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 39563212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff + size_diff); 3957b18c6685SChris Mason } 3958db94535dSChris Mason 3959b18c6685SChris Mason /* shift the data */ 3960179e29e4SChris Mason if (from_end) { 3961637e3b48SJosef Bacik memmove_leaf_data(leaf, data_end + size_diff, data_end, 3962637e3b48SJosef Bacik old_data_start + new_size - data_end); 3963179e29e4SChris Mason } else { 3964179e29e4SChris Mason struct btrfs_disk_key disk_key; 3965179e29e4SChris Mason u64 offset; 3966179e29e4SChris Mason 3967179e29e4SChris Mason btrfs_item_key(leaf, &disk_key, slot); 3968179e29e4SChris Mason 3969179e29e4SChris Mason if (btrfs_disk_key_type(&disk_key) == BTRFS_EXTENT_DATA_KEY) { 3970179e29e4SChris Mason unsigned long ptr; 3971179e29e4SChris Mason struct btrfs_file_extent_item *fi; 3972179e29e4SChris Mason 3973179e29e4SChris Mason fi = btrfs_item_ptr(leaf, slot, 3974179e29e4SChris Mason struct btrfs_file_extent_item); 3975179e29e4SChris Mason fi = (struct btrfs_file_extent_item *)( 3976179e29e4SChris Mason (unsigned long)fi - size_diff); 3977179e29e4SChris Mason 3978179e29e4SChris Mason if (btrfs_file_extent_type(leaf, fi) == 3979179e29e4SChris Mason BTRFS_FILE_EXTENT_INLINE) { 3980179e29e4SChris Mason ptr = btrfs_item_ptr_offset(leaf, slot); 3981179e29e4SChris Mason memmove_extent_buffer(leaf, ptr, 3982179e29e4SChris Mason (unsigned long)fi, 39837ec20afbSDavid Sterba BTRFS_FILE_EXTENT_INLINE_DATA_START); 3984179e29e4SChris Mason } 3985179e29e4SChris Mason } 3986179e29e4SChris Mason 3987637e3b48SJosef Bacik memmove_leaf_data(leaf, data_end + size_diff, data_end, 3988637e3b48SJosef Bacik old_data_start - data_end); 3989179e29e4SChris Mason 3990179e29e4SChris Mason offset = btrfs_disk_key_offset(&disk_key); 3991179e29e4SChris Mason btrfs_set_disk_key_offset(&disk_key, offset + size_diff); 3992179e29e4SChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 3993179e29e4SChris Mason if (slot == 0) 3994b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 3995179e29e4SChris Mason } 39965f39d397SChris Mason 39973212fa14SJosef Bacik btrfs_set_item_size(leaf, slot, new_size); 39985f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 3999b18c6685SChris Mason 4000e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) { 4001a4f78750SDavid Sterba btrfs_print_leaf(leaf); 4002b18c6685SChris Mason BUG(); 40035f39d397SChris Mason } 4004b18c6685SChris Mason } 4005b18c6685SChris Mason 4006d352ac68SChris Mason /* 40078f69dbd2SStefan Behrens * make the item pointed to by the path bigger, data_size is the added size. 4008d352ac68SChris Mason */ 4009c71dd880SDavid Sterba void btrfs_extend_item(struct btrfs_path *path, u32 data_size) 40106567e837SChris Mason { 40116567e837SChris Mason int slot; 40125f39d397SChris Mason struct extent_buffer *leaf; 40136567e837SChris Mason u32 nritems; 40146567e837SChris Mason unsigned int data_end; 40156567e837SChris Mason unsigned int old_data; 40166567e837SChris Mason unsigned int old_size; 40176567e837SChris Mason int i; 4018cfed81a0SChris Mason struct btrfs_map_token token; 4019cfed81a0SChris Mason 40205f39d397SChris Mason leaf = path->nodes[0]; 40216567e837SChris Mason 40225f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 40238f881e8cSDavid Sterba data_end = leaf_data_end(leaf); 40246567e837SChris Mason 4025e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < data_size) { 4026a4f78750SDavid Sterba btrfs_print_leaf(leaf); 40276567e837SChris Mason BUG(); 40285f39d397SChris Mason } 40296567e837SChris Mason slot = path->slots[0]; 4030dc2e724eSJosef Bacik old_data = btrfs_item_data_end(leaf, slot); 40316567e837SChris Mason 40326567e837SChris Mason BUG_ON(slot < 0); 40333326d1b0SChris Mason if (slot >= nritems) { 4034a4f78750SDavid Sterba btrfs_print_leaf(leaf); 4035c71dd880SDavid Sterba btrfs_crit(leaf->fs_info, "slot %d too large, nritems %d", 4036d397712bSChris Mason slot, nritems); 4037290342f6SArnd Bergmann BUG(); 40383326d1b0SChris Mason } 40396567e837SChris Mason 40406567e837SChris Mason /* 40416567e837SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 40426567e837SChris Mason */ 40436567e837SChris Mason /* first correct the data pointers */ 4044c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 40456567e837SChris Mason for (i = slot; i < nritems; i++) { 40465f39d397SChris Mason u32 ioff; 4047db94535dSChris Mason 40483212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 40493212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff - data_size); 40506567e837SChris Mason } 40515f39d397SChris Mason 40526567e837SChris Mason /* shift the data */ 4053637e3b48SJosef Bacik memmove_leaf_data(leaf, data_end - data_size, data_end, 4054637e3b48SJosef Bacik old_data - data_end); 40555f39d397SChris Mason 40566567e837SChris Mason data_end = old_data; 40573212fa14SJosef Bacik old_size = btrfs_item_size(leaf, slot); 40583212fa14SJosef Bacik btrfs_set_item_size(leaf, slot, old_size + data_size); 40595f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 40606567e837SChris Mason 4061e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) { 4062a4f78750SDavid Sterba btrfs_print_leaf(leaf); 40636567e837SChris Mason BUG(); 40645f39d397SChris Mason } 40656567e837SChris Mason } 40666567e837SChris Mason 406743dd529aSDavid Sterba /* 406843dd529aSDavid Sterba * Make space in the node before inserting one or more items. 4069da9ffb24SNikolay Borisov * 4070da9ffb24SNikolay Borisov * @root: root we are inserting items to 4071da9ffb24SNikolay Borisov * @path: points to the leaf/slot where we are going to insert new items 4072b7ef5f3aSFilipe Manana * @batch: information about the batch of items to insert 407343dd529aSDavid Sterba * 407443dd529aSDavid Sterba * Main purpose is to save stack depth by doing the bulk of the work in a 407543dd529aSDavid Sterba * function that doesn't call btrfs_search_slot 407674123bd7SChris Mason */ 4077f0641656SFilipe Manana static void setup_items_for_insert(struct btrfs_root *root, struct btrfs_path *path, 4078b7ef5f3aSFilipe Manana const struct btrfs_item_batch *batch) 4079be0e5c09SChris Mason { 40800b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 40819c58309dSChris Mason int i; 40827518a238SChris Mason u32 nritems; 4083be0e5c09SChris Mason unsigned int data_end; 4084e2fa7227SChris Mason struct btrfs_disk_key disk_key; 408544871b1bSChris Mason struct extent_buffer *leaf; 408644871b1bSChris Mason int slot; 4087cfed81a0SChris Mason struct btrfs_map_token token; 4088fc0d82e1SNikolay Borisov u32 total_size; 4089fc0d82e1SNikolay Borisov 4090b7ef5f3aSFilipe Manana /* 4091b7ef5f3aSFilipe Manana * Before anything else, update keys in the parent and other ancestors 4092b7ef5f3aSFilipe Manana * if needed, then release the write locks on them, so that other tasks 4093b7ef5f3aSFilipe Manana * can use them while we modify the leaf. 4094b7ef5f3aSFilipe Manana */ 409524cdc847SFilipe Manana if (path->slots[0] == 0) { 4096b7ef5f3aSFilipe Manana btrfs_cpu_key_to_disk(&disk_key, &batch->keys[0]); 4097b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 409824cdc847SFilipe Manana } 409924cdc847SFilipe Manana btrfs_unlock_up_safe(path, 1); 410024cdc847SFilipe Manana 41015f39d397SChris Mason leaf = path->nodes[0]; 410244871b1bSChris Mason slot = path->slots[0]; 410374123bd7SChris Mason 41045f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 41058f881e8cSDavid Sterba data_end = leaf_data_end(leaf); 4106b7ef5f3aSFilipe Manana total_size = batch->total_data_size + (batch->nr * sizeof(struct btrfs_item)); 4107eb60ceacSChris Mason 4108e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < total_size) { 4109a4f78750SDavid Sterba btrfs_print_leaf(leaf); 41100b246afaSJeff Mahoney btrfs_crit(fs_info, "not enough freespace need %u have %d", 4111e902baacSDavid Sterba total_size, btrfs_leaf_free_space(leaf)); 4112be0e5c09SChris Mason BUG(); 4113d4dbff95SChris Mason } 41145f39d397SChris Mason 4115c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 4116be0e5c09SChris Mason if (slot != nritems) { 4117dc2e724eSJosef Bacik unsigned int old_data = btrfs_item_data_end(leaf, slot); 4118be0e5c09SChris Mason 41195f39d397SChris Mason if (old_data < data_end) { 4120a4f78750SDavid Sterba btrfs_print_leaf(leaf); 41217269ddd2SNikolay Borisov btrfs_crit(fs_info, 41227269ddd2SNikolay Borisov "item at slot %d with data offset %u beyond data end of leaf %u", 41235f39d397SChris Mason slot, old_data, data_end); 4124290342f6SArnd Bergmann BUG(); 41255f39d397SChris Mason } 4126be0e5c09SChris Mason /* 4127be0e5c09SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 4128be0e5c09SChris Mason */ 4129be0e5c09SChris Mason /* first correct the data pointers */ 41300783fcfcSChris Mason for (i = slot; i < nritems; i++) { 41315f39d397SChris Mason u32 ioff; 4132db94535dSChris Mason 41333212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 41343212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, 4135b7ef5f3aSFilipe Manana ioff - batch->total_data_size); 41360783fcfcSChris Mason } 4137be0e5c09SChris Mason /* shift the items */ 4138637e3b48SJosef Bacik memmove_leaf_items(leaf, slot + batch->nr, slot, nritems - slot); 4139be0e5c09SChris Mason 4140be0e5c09SChris Mason /* shift the data */ 4141637e3b48SJosef Bacik memmove_leaf_data(leaf, data_end - batch->total_data_size, 4142637e3b48SJosef Bacik data_end, old_data - data_end); 4143be0e5c09SChris Mason data_end = old_data; 4144be0e5c09SChris Mason } 41455f39d397SChris Mason 414662e2749eSChris Mason /* setup the item for the new data */ 4147b7ef5f3aSFilipe Manana for (i = 0; i < batch->nr; i++) { 4148b7ef5f3aSFilipe Manana btrfs_cpu_key_to_disk(&disk_key, &batch->keys[i]); 41499c58309dSChris Mason btrfs_set_item_key(leaf, &disk_key, slot + i); 4150b7ef5f3aSFilipe Manana data_end -= batch->data_sizes[i]; 41513212fa14SJosef Bacik btrfs_set_token_item_offset(&token, slot + i, data_end); 41523212fa14SJosef Bacik btrfs_set_token_item_size(&token, slot + i, batch->data_sizes[i]); 41539c58309dSChris Mason } 415444871b1bSChris Mason 4155b7ef5f3aSFilipe Manana btrfs_set_header_nritems(leaf, nritems + batch->nr); 4156b9473439SChris Mason btrfs_mark_buffer_dirty(leaf); 4157aa5d6bedSChris Mason 4158e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) { 4159a4f78750SDavid Sterba btrfs_print_leaf(leaf); 4160be0e5c09SChris Mason BUG(); 41615f39d397SChris Mason } 416244871b1bSChris Mason } 416344871b1bSChris Mason 416444871b1bSChris Mason /* 4165f0641656SFilipe Manana * Insert a new item into a leaf. 4166f0641656SFilipe Manana * 4167f0641656SFilipe Manana * @root: The root of the btree. 4168f0641656SFilipe Manana * @path: A path pointing to the target leaf and slot. 4169f0641656SFilipe Manana * @key: The key of the new item. 4170f0641656SFilipe Manana * @data_size: The size of the data associated with the new key. 4171f0641656SFilipe Manana */ 4172f0641656SFilipe Manana void btrfs_setup_item_for_insert(struct btrfs_root *root, 4173f0641656SFilipe Manana struct btrfs_path *path, 4174f0641656SFilipe Manana const struct btrfs_key *key, 4175f0641656SFilipe Manana u32 data_size) 4176f0641656SFilipe Manana { 4177f0641656SFilipe Manana struct btrfs_item_batch batch; 4178f0641656SFilipe Manana 4179f0641656SFilipe Manana batch.keys = key; 4180f0641656SFilipe Manana batch.data_sizes = &data_size; 4181f0641656SFilipe Manana batch.total_data_size = data_size; 4182f0641656SFilipe Manana batch.nr = 1; 4183f0641656SFilipe Manana 4184f0641656SFilipe Manana setup_items_for_insert(root, path, &batch); 4185f0641656SFilipe Manana } 4186f0641656SFilipe Manana 4187f0641656SFilipe Manana /* 418844871b1bSChris Mason * Given a key and some data, insert items into the tree. 418944871b1bSChris Mason * This does all the path init required, making room in the tree if needed. 419044871b1bSChris Mason */ 419144871b1bSChris Mason int btrfs_insert_empty_items(struct btrfs_trans_handle *trans, 419244871b1bSChris Mason struct btrfs_root *root, 419344871b1bSChris Mason struct btrfs_path *path, 4194b7ef5f3aSFilipe Manana const struct btrfs_item_batch *batch) 419544871b1bSChris Mason { 419644871b1bSChris Mason int ret = 0; 419744871b1bSChris Mason int slot; 4198b7ef5f3aSFilipe Manana u32 total_size; 419944871b1bSChris Mason 4200b7ef5f3aSFilipe Manana total_size = batch->total_data_size + (batch->nr * sizeof(struct btrfs_item)); 4201b7ef5f3aSFilipe Manana ret = btrfs_search_slot(trans, root, &batch->keys[0], path, total_size, 1); 420244871b1bSChris Mason if (ret == 0) 420344871b1bSChris Mason return -EEXIST; 420444871b1bSChris Mason if (ret < 0) 4205143bede5SJeff Mahoney return ret; 420644871b1bSChris Mason 420744871b1bSChris Mason slot = path->slots[0]; 420844871b1bSChris Mason BUG_ON(slot < 0); 420944871b1bSChris Mason 4210b7ef5f3aSFilipe Manana setup_items_for_insert(root, path, batch); 4211143bede5SJeff Mahoney return 0; 421262e2749eSChris Mason } 421362e2749eSChris Mason 421462e2749eSChris Mason /* 421562e2749eSChris Mason * Given a key and some data, insert an item into the tree. 421662e2749eSChris Mason * This does all the path init required, making room in the tree if needed. 421762e2749eSChris Mason */ 4218310712b2SOmar Sandoval int btrfs_insert_item(struct btrfs_trans_handle *trans, struct btrfs_root *root, 4219310712b2SOmar Sandoval const struct btrfs_key *cpu_key, void *data, 4220310712b2SOmar Sandoval u32 data_size) 422162e2749eSChris Mason { 422262e2749eSChris Mason int ret = 0; 42232c90e5d6SChris Mason struct btrfs_path *path; 42245f39d397SChris Mason struct extent_buffer *leaf; 42255f39d397SChris Mason unsigned long ptr; 422662e2749eSChris Mason 42272c90e5d6SChris Mason path = btrfs_alloc_path(); 4228db5b493aSTsutomu Itoh if (!path) 4229db5b493aSTsutomu Itoh return -ENOMEM; 42302c90e5d6SChris Mason ret = btrfs_insert_empty_item(trans, root, path, cpu_key, data_size); 423162e2749eSChris Mason if (!ret) { 42325f39d397SChris Mason leaf = path->nodes[0]; 42335f39d397SChris Mason ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); 42345f39d397SChris Mason write_extent_buffer(leaf, data, ptr, data_size); 42355f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 423662e2749eSChris Mason } 42372c90e5d6SChris Mason btrfs_free_path(path); 4238aa5d6bedSChris Mason return ret; 4239be0e5c09SChris Mason } 4240be0e5c09SChris Mason 424174123bd7SChris Mason /* 4242f0641656SFilipe Manana * This function duplicates an item, giving 'new_key' to the new item. 4243f0641656SFilipe Manana * It guarantees both items live in the same tree leaf and the new item is 4244f0641656SFilipe Manana * contiguous with the original item. 4245f0641656SFilipe Manana * 4246f0641656SFilipe Manana * This allows us to split a file extent in place, keeping a lock on the leaf 4247f0641656SFilipe Manana * the entire time. 4248f0641656SFilipe Manana */ 4249f0641656SFilipe Manana int btrfs_duplicate_item(struct btrfs_trans_handle *trans, 4250f0641656SFilipe Manana struct btrfs_root *root, 4251f0641656SFilipe Manana struct btrfs_path *path, 4252f0641656SFilipe Manana const struct btrfs_key *new_key) 4253f0641656SFilipe Manana { 4254f0641656SFilipe Manana struct extent_buffer *leaf; 4255f0641656SFilipe Manana int ret; 4256f0641656SFilipe Manana u32 item_size; 4257f0641656SFilipe Manana 4258f0641656SFilipe Manana leaf = path->nodes[0]; 42593212fa14SJosef Bacik item_size = btrfs_item_size(leaf, path->slots[0]); 4260f0641656SFilipe Manana ret = setup_leaf_for_split(trans, root, path, 4261f0641656SFilipe Manana item_size + sizeof(struct btrfs_item)); 4262f0641656SFilipe Manana if (ret) 4263f0641656SFilipe Manana return ret; 4264f0641656SFilipe Manana 4265f0641656SFilipe Manana path->slots[0]++; 4266f0641656SFilipe Manana btrfs_setup_item_for_insert(root, path, new_key, item_size); 4267f0641656SFilipe Manana leaf = path->nodes[0]; 4268f0641656SFilipe Manana memcpy_extent_buffer(leaf, 4269f0641656SFilipe Manana btrfs_item_ptr_offset(leaf, path->slots[0]), 4270f0641656SFilipe Manana btrfs_item_ptr_offset(leaf, path->slots[0] - 1), 4271f0641656SFilipe Manana item_size); 4272f0641656SFilipe Manana return 0; 4273f0641656SFilipe Manana } 4274f0641656SFilipe Manana 4275f0641656SFilipe Manana /* 42765de08d7dSChris Mason * delete the pointer from a given node. 427774123bd7SChris Mason * 4278d352ac68SChris Mason * the tree should have been previously balanced so the deletion does not 4279d352ac68SChris Mason * empty a node. 428074123bd7SChris Mason */ 4281afe5fea7STsutomu Itoh static void del_ptr(struct btrfs_root *root, struct btrfs_path *path, 4282afe5fea7STsutomu Itoh int level, int slot) 4283be0e5c09SChris Mason { 42845f39d397SChris Mason struct extent_buffer *parent = path->nodes[level]; 42857518a238SChris Mason u32 nritems; 4286f3ea38daSJan Schmidt int ret; 4287be0e5c09SChris Mason 42885f39d397SChris Mason nritems = btrfs_header_nritems(parent); 4289be0e5c09SChris Mason if (slot != nritems - 1) { 4290bf1d3425SDavid Sterba if (level) { 4291f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_move(parent, slot, 4292f3a84ccdSFilipe Manana slot + 1, nritems - slot - 1); 4293bf1d3425SDavid Sterba BUG_ON(ret < 0); 4294bf1d3425SDavid Sterba } 42955f39d397SChris Mason memmove_extent_buffer(parent, 4296e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(parent, slot), 4297e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(parent, slot + 1), 4298d6025579SChris Mason sizeof(struct btrfs_key_ptr) * 4299d6025579SChris Mason (nritems - slot - 1)); 430057ba86c0SChris Mason } else if (level) { 4301f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, slot, 430233cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REMOVE); 430357ba86c0SChris Mason BUG_ON(ret < 0); 4304be0e5c09SChris Mason } 4305f3ea38daSJan Schmidt 43067518a238SChris Mason nritems--; 43075f39d397SChris Mason btrfs_set_header_nritems(parent, nritems); 43087518a238SChris Mason if (nritems == 0 && parent == root->node) { 43095f39d397SChris Mason BUG_ON(btrfs_header_level(root->node) != 1); 4310eb60ceacSChris Mason /* just turn the root into a leaf and break */ 43115f39d397SChris Mason btrfs_set_header_level(root->node, 0); 4312bb803951SChris Mason } else if (slot == 0) { 43135f39d397SChris Mason struct btrfs_disk_key disk_key; 43145f39d397SChris Mason 43155f39d397SChris Mason btrfs_node_key(parent, &disk_key, 0); 4316b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, level + 1); 4317be0e5c09SChris Mason } 4318d6025579SChris Mason btrfs_mark_buffer_dirty(parent); 4319be0e5c09SChris Mason } 4320be0e5c09SChris Mason 432174123bd7SChris Mason /* 4322323ac95bSChris Mason * a helper function to delete the leaf pointed to by path->slots[1] and 43235d4f98a2SYan Zheng * path->nodes[1]. 4324323ac95bSChris Mason * 4325323ac95bSChris Mason * This deletes the pointer in path->nodes[1] and frees the leaf 4326323ac95bSChris Mason * block extent. zero is returned if it all worked out, < 0 otherwise. 4327323ac95bSChris Mason * 4328323ac95bSChris Mason * The path must have already been setup for deleting the leaf, including 4329323ac95bSChris Mason * all the proper balancing. path->nodes[1] must be locked. 4330323ac95bSChris Mason */ 4331143bede5SJeff Mahoney static noinline void btrfs_del_leaf(struct btrfs_trans_handle *trans, 4332323ac95bSChris Mason struct btrfs_root *root, 43335d4f98a2SYan Zheng struct btrfs_path *path, 43345d4f98a2SYan Zheng struct extent_buffer *leaf) 4335323ac95bSChris Mason { 43365d4f98a2SYan Zheng WARN_ON(btrfs_header_generation(leaf) != trans->transid); 4337afe5fea7STsutomu Itoh del_ptr(root, path, 1, path->slots[1]); 4338323ac95bSChris Mason 43394d081c41SChris Mason /* 43404d081c41SChris Mason * btrfs_free_extent is expensive, we want to make sure we 43414d081c41SChris Mason * aren't holding any locks when we call it 43424d081c41SChris Mason */ 43434d081c41SChris Mason btrfs_unlock_up_safe(path, 0); 43444d081c41SChris Mason 4345f0486c68SYan, Zheng root_sub_used(root, leaf->len); 4346f0486c68SYan, Zheng 434767439dadSDavid Sterba atomic_inc(&leaf->refs); 43487a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), leaf, 0, 1); 43493083ee2eSJosef Bacik free_extent_buffer_stale(leaf); 4350323ac95bSChris Mason } 4351323ac95bSChris Mason /* 435274123bd7SChris Mason * delete the item at the leaf level in path. If that empties 435374123bd7SChris Mason * the leaf, remove it from the tree 435474123bd7SChris Mason */ 435585e21bacSChris Mason int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root, 435685e21bacSChris Mason struct btrfs_path *path, int slot, int nr) 4357be0e5c09SChris Mason { 43580b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 43595f39d397SChris Mason struct extent_buffer *leaf; 4360aa5d6bedSChris Mason int ret = 0; 4361aa5d6bedSChris Mason int wret; 43627518a238SChris Mason u32 nritems; 4363be0e5c09SChris Mason 43645f39d397SChris Mason leaf = path->nodes[0]; 43655f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 4366be0e5c09SChris Mason 436785e21bacSChris Mason if (slot + nr != nritems) { 43680cae23b6SFilipe Manana const u32 last_off = btrfs_item_offset(leaf, slot + nr - 1); 43690cae23b6SFilipe Manana const int data_end = leaf_data_end(leaf); 4370c82f823cSDavid Sterba struct btrfs_map_token token; 43710cae23b6SFilipe Manana u32 dsize = 0; 43720cae23b6SFilipe Manana int i; 43730cae23b6SFilipe Manana 43740cae23b6SFilipe Manana for (i = 0; i < nr; i++) 43750cae23b6SFilipe Manana dsize += btrfs_item_size(leaf, slot + i); 43765f39d397SChris Mason 4377637e3b48SJosef Bacik memmove_leaf_data(leaf, data_end + dsize, data_end, 437885e21bacSChris Mason last_off - data_end); 43795f39d397SChris Mason 4380c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 438185e21bacSChris Mason for (i = slot + nr; i < nritems; i++) { 43825f39d397SChris Mason u32 ioff; 4383db94535dSChris Mason 43843212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 43853212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff + dsize); 43860783fcfcSChris Mason } 4387db94535dSChris Mason 4388637e3b48SJosef Bacik memmove_leaf_items(leaf, slot, slot + nr, nritems - slot - nr); 4389be0e5c09SChris Mason } 439085e21bacSChris Mason btrfs_set_header_nritems(leaf, nritems - nr); 439185e21bacSChris Mason nritems -= nr; 43925f39d397SChris Mason 439374123bd7SChris Mason /* delete the leaf if we've emptied it */ 43947518a238SChris Mason if (nritems == 0) { 43955f39d397SChris Mason if (leaf == root->node) { 43965f39d397SChris Mason btrfs_set_header_level(leaf, 0); 43979a8dd150SChris Mason } else { 4398190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, leaf); 4399143bede5SJeff Mahoney btrfs_del_leaf(trans, root, path, leaf); 44009a8dd150SChris Mason } 4401be0e5c09SChris Mason } else { 44027518a238SChris Mason int used = leaf_space_used(leaf, 0, nritems); 4403aa5d6bedSChris Mason if (slot == 0) { 44045f39d397SChris Mason struct btrfs_disk_key disk_key; 44055f39d397SChris Mason 44065f39d397SChris Mason btrfs_item_key(leaf, &disk_key, 0); 4407b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 4408aa5d6bedSChris Mason } 4409aa5d6bedSChris Mason 44107c4063d1SFilipe Manana /* 44117c4063d1SFilipe Manana * Try to delete the leaf if it is mostly empty. We do this by 44127c4063d1SFilipe Manana * trying to move all its items into its left and right neighbours. 44137c4063d1SFilipe Manana * If we can't move all the items, then we don't delete it - it's 44147c4063d1SFilipe Manana * not ideal, but future insertions might fill the leaf with more 44157c4063d1SFilipe Manana * items, or items from other leaves might be moved later into our 44167c4063d1SFilipe Manana * leaf due to deletions on those leaves. 44177c4063d1SFilipe Manana */ 44180b246afaSJeff Mahoney if (used < BTRFS_LEAF_DATA_SIZE(fs_info) / 3) { 44197c4063d1SFilipe Manana u32 min_push_space; 44207c4063d1SFilipe Manana 4421be0e5c09SChris Mason /* push_leaf_left fixes the path. 4422be0e5c09SChris Mason * make sure the path still points to our leaf 4423be0e5c09SChris Mason * for possible call to del_ptr below 4424be0e5c09SChris Mason */ 44254920c9acSChris Mason slot = path->slots[1]; 442667439dadSDavid Sterba atomic_inc(&leaf->refs); 44277c4063d1SFilipe Manana /* 44287c4063d1SFilipe Manana * We want to be able to at least push one item to the 44297c4063d1SFilipe Manana * left neighbour leaf, and that's the first item. 44307c4063d1SFilipe Manana */ 44317c4063d1SFilipe Manana min_push_space = sizeof(struct btrfs_item) + 44327c4063d1SFilipe Manana btrfs_item_size(leaf, 0); 44337c4063d1SFilipe Manana wret = push_leaf_left(trans, root, path, 0, 44347c4063d1SFilipe Manana min_push_space, 1, (u32)-1); 443554aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 4436aa5d6bedSChris Mason ret = wret; 44375f39d397SChris Mason 44385f39d397SChris Mason if (path->nodes[0] == leaf && 44395f39d397SChris Mason btrfs_header_nritems(leaf)) { 44407c4063d1SFilipe Manana /* 44417c4063d1SFilipe Manana * If we were not able to push all items from our 44427c4063d1SFilipe Manana * leaf to its left neighbour, then attempt to 44437c4063d1SFilipe Manana * either push all the remaining items to the 44447c4063d1SFilipe Manana * right neighbour or none. There's no advantage 44457c4063d1SFilipe Manana * in pushing only some items, instead of all, as 44467c4063d1SFilipe Manana * it's pointless to end up with a leaf having 44477c4063d1SFilipe Manana * too few items while the neighbours can be full 44487c4063d1SFilipe Manana * or nearly full. 44497c4063d1SFilipe Manana */ 44507c4063d1SFilipe Manana nritems = btrfs_header_nritems(leaf); 44517c4063d1SFilipe Manana min_push_space = leaf_space_used(leaf, 0, nritems); 44527c4063d1SFilipe Manana wret = push_leaf_right(trans, root, path, 0, 44537c4063d1SFilipe Manana min_push_space, 1, 0); 445454aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 4455aa5d6bedSChris Mason ret = wret; 4456aa5d6bedSChris Mason } 44575f39d397SChris Mason 44585f39d397SChris Mason if (btrfs_header_nritems(leaf) == 0) { 4459323ac95bSChris Mason path->slots[1] = slot; 4460143bede5SJeff Mahoney btrfs_del_leaf(trans, root, path, leaf); 44615f39d397SChris Mason free_extent_buffer(leaf); 4462143bede5SJeff Mahoney ret = 0; 44635de08d7dSChris Mason } else { 4464925baeddSChris Mason /* if we're still in the path, make sure 4465925baeddSChris Mason * we're dirty. Otherwise, one of the 4466925baeddSChris Mason * push_leaf functions must have already 4467925baeddSChris Mason * dirtied this buffer 4468925baeddSChris Mason */ 4469925baeddSChris Mason if (path->nodes[0] == leaf) 44705f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 44715f39d397SChris Mason free_extent_buffer(leaf); 4472be0e5c09SChris Mason } 4473d5719762SChris Mason } else { 44745f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 4475be0e5c09SChris Mason } 44769a8dd150SChris Mason } 4477aa5d6bedSChris Mason return ret; 44789a8dd150SChris Mason } 44799a8dd150SChris Mason 448097571fd0SChris Mason /* 4481925baeddSChris Mason * search the tree again to find a leaf with lesser keys 44827bb86316SChris Mason * returns 0 if it found something or 1 if there are no lesser leaves. 44837bb86316SChris Mason * returns < 0 on io errors. 4484d352ac68SChris Mason * 4485d352ac68SChris Mason * This may release the path, and so you may lose any locks held at the 4486d352ac68SChris Mason * time you call it. 44877bb86316SChris Mason */ 448816e7549fSJosef Bacik int btrfs_prev_leaf(struct btrfs_root *root, struct btrfs_path *path) 44897bb86316SChris Mason { 4490925baeddSChris Mason struct btrfs_key key; 4491925baeddSChris Mason struct btrfs_disk_key found_key; 4492925baeddSChris Mason int ret; 44937bb86316SChris Mason 4494925baeddSChris Mason btrfs_item_key_to_cpu(path->nodes[0], &key, 0); 4495925baeddSChris Mason 4496e8b0d724SFilipe David Borba Manana if (key.offset > 0) { 4497925baeddSChris Mason key.offset--; 4498e8b0d724SFilipe David Borba Manana } else if (key.type > 0) { 4499925baeddSChris Mason key.type--; 4500e8b0d724SFilipe David Borba Manana key.offset = (u64)-1; 4501e8b0d724SFilipe David Borba Manana } else if (key.objectid > 0) { 4502925baeddSChris Mason key.objectid--; 4503e8b0d724SFilipe David Borba Manana key.type = (u8)-1; 4504e8b0d724SFilipe David Borba Manana key.offset = (u64)-1; 4505e8b0d724SFilipe David Borba Manana } else { 45067bb86316SChris Mason return 1; 4507e8b0d724SFilipe David Borba Manana } 45087bb86316SChris Mason 4509b3b4aa74SDavid Sterba btrfs_release_path(path); 4510925baeddSChris Mason ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 4511925baeddSChris Mason if (ret < 0) 4512925baeddSChris Mason return ret; 4513925baeddSChris Mason btrfs_item_key(path->nodes[0], &found_key, 0); 4514925baeddSChris Mason ret = comp_keys(&found_key, &key); 4515337c6f68SFilipe Manana /* 4516337c6f68SFilipe Manana * We might have had an item with the previous key in the tree right 4517337c6f68SFilipe Manana * before we released our path. And after we released our path, that 4518337c6f68SFilipe Manana * item might have been pushed to the first slot (0) of the leaf we 4519337c6f68SFilipe Manana * were holding due to a tree balance. Alternatively, an item with the 4520337c6f68SFilipe Manana * previous key can exist as the only element of a leaf (big fat item). 4521337c6f68SFilipe Manana * Therefore account for these 2 cases, so that our callers (like 4522337c6f68SFilipe Manana * btrfs_previous_item) don't miss an existing item with a key matching 4523337c6f68SFilipe Manana * the previous key we computed above. 4524337c6f68SFilipe Manana */ 4525337c6f68SFilipe Manana if (ret <= 0) 45267bb86316SChris Mason return 0; 4527925baeddSChris Mason return 1; 45287bb86316SChris Mason } 45297bb86316SChris Mason 45303f157a2fSChris Mason /* 45313f157a2fSChris Mason * A helper function to walk down the tree starting at min_key, and looking 4532de78b51aSEric Sandeen * for nodes or leaves that are have a minimum transaction id. 4533de78b51aSEric Sandeen * This is used by the btree defrag code, and tree logging 45343f157a2fSChris Mason * 45353f157a2fSChris Mason * This does not cow, but it does stuff the starting key it finds back 45363f157a2fSChris Mason * into min_key, so you can call btrfs_search_slot with cow=1 on the 45373f157a2fSChris Mason * key and get a writable path. 45383f157a2fSChris Mason * 45393f157a2fSChris Mason * This honors path->lowest_level to prevent descent past a given level 45403f157a2fSChris Mason * of the tree. 45413f157a2fSChris Mason * 4542d352ac68SChris Mason * min_trans indicates the oldest transaction that you are interested 4543d352ac68SChris Mason * in walking through. Any nodes or leaves older than min_trans are 4544d352ac68SChris Mason * skipped over (without reading them). 4545d352ac68SChris Mason * 45463f157a2fSChris Mason * returns zero if something useful was found, < 0 on error and 1 if there 45473f157a2fSChris Mason * was nothing in the tree that matched the search criteria. 45483f157a2fSChris Mason */ 45493f157a2fSChris Mason int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key, 4550de78b51aSEric Sandeen struct btrfs_path *path, 45513f157a2fSChris Mason u64 min_trans) 45523f157a2fSChris Mason { 45533f157a2fSChris Mason struct extent_buffer *cur; 45543f157a2fSChris Mason struct btrfs_key found_key; 45553f157a2fSChris Mason int slot; 45569652480bSYan int sret; 45573f157a2fSChris Mason u32 nritems; 45583f157a2fSChris Mason int level; 45593f157a2fSChris Mason int ret = 1; 4560f98de9b9SFilipe Manana int keep_locks = path->keep_locks; 45613f157a2fSChris Mason 4562c922b016SStefan Roesch ASSERT(!path->nowait); 4563f98de9b9SFilipe Manana path->keep_locks = 1; 45643f157a2fSChris Mason again: 4565bd681513SChris Mason cur = btrfs_read_lock_root_node(root); 45663f157a2fSChris Mason level = btrfs_header_level(cur); 4567e02119d5SChris Mason WARN_ON(path->nodes[level]); 45683f157a2fSChris Mason path->nodes[level] = cur; 4569bd681513SChris Mason path->locks[level] = BTRFS_READ_LOCK; 45703f157a2fSChris Mason 45713f157a2fSChris Mason if (btrfs_header_generation(cur) < min_trans) { 45723f157a2fSChris Mason ret = 1; 45733f157a2fSChris Mason goto out; 45743f157a2fSChris Mason } 45753f157a2fSChris Mason while (1) { 45763f157a2fSChris Mason nritems = btrfs_header_nritems(cur); 45773f157a2fSChris Mason level = btrfs_header_level(cur); 4578e3b83361SQu Wenruo sret = btrfs_bin_search(cur, min_key, &slot); 4579cbca7d59SFilipe Manana if (sret < 0) { 4580cbca7d59SFilipe Manana ret = sret; 4581cbca7d59SFilipe Manana goto out; 4582cbca7d59SFilipe Manana } 45833f157a2fSChris Mason 4584323ac95bSChris Mason /* at the lowest level, we're done, setup the path and exit */ 4585323ac95bSChris Mason if (level == path->lowest_level) { 4586e02119d5SChris Mason if (slot >= nritems) 4587e02119d5SChris Mason goto find_next_key; 45883f157a2fSChris Mason ret = 0; 45893f157a2fSChris Mason path->slots[level] = slot; 45903f157a2fSChris Mason btrfs_item_key_to_cpu(cur, &found_key, slot); 45913f157a2fSChris Mason goto out; 45923f157a2fSChris Mason } 45939652480bSYan if (sret && slot > 0) 45949652480bSYan slot--; 45953f157a2fSChris Mason /* 4596de78b51aSEric Sandeen * check this node pointer against the min_trans parameters. 4597260db43cSRandy Dunlap * If it is too old, skip to the next one. 45983f157a2fSChris Mason */ 45993f157a2fSChris Mason while (slot < nritems) { 46003f157a2fSChris Mason u64 gen; 4601e02119d5SChris Mason 46023f157a2fSChris Mason gen = btrfs_node_ptr_generation(cur, slot); 46033f157a2fSChris Mason if (gen < min_trans) { 46043f157a2fSChris Mason slot++; 46053f157a2fSChris Mason continue; 46063f157a2fSChris Mason } 46073f157a2fSChris Mason break; 46083f157a2fSChris Mason } 4609e02119d5SChris Mason find_next_key: 46103f157a2fSChris Mason /* 46113f157a2fSChris Mason * we didn't find a candidate key in this node, walk forward 46123f157a2fSChris Mason * and find another one 46133f157a2fSChris Mason */ 46143f157a2fSChris Mason if (slot >= nritems) { 4615e02119d5SChris Mason path->slots[level] = slot; 4616e02119d5SChris Mason sret = btrfs_find_next_key(root, path, min_key, level, 4617de78b51aSEric Sandeen min_trans); 4618e02119d5SChris Mason if (sret == 0) { 4619b3b4aa74SDavid Sterba btrfs_release_path(path); 46203f157a2fSChris Mason goto again; 46213f157a2fSChris Mason } else { 46223f157a2fSChris Mason goto out; 46233f157a2fSChris Mason } 46243f157a2fSChris Mason } 46253f157a2fSChris Mason /* save our key for returning back */ 46263f157a2fSChris Mason btrfs_node_key_to_cpu(cur, &found_key, slot); 46273f157a2fSChris Mason path->slots[level] = slot; 46283f157a2fSChris Mason if (level == path->lowest_level) { 46293f157a2fSChris Mason ret = 0; 46303f157a2fSChris Mason goto out; 46313f157a2fSChris Mason } 46324b231ae4SDavid Sterba cur = btrfs_read_node_slot(cur, slot); 4633fb770ae4SLiu Bo if (IS_ERR(cur)) { 4634fb770ae4SLiu Bo ret = PTR_ERR(cur); 4635fb770ae4SLiu Bo goto out; 4636fb770ae4SLiu Bo } 46373f157a2fSChris Mason 4638bd681513SChris Mason btrfs_tree_read_lock(cur); 4639b4ce94deSChris Mason 4640bd681513SChris Mason path->locks[level - 1] = BTRFS_READ_LOCK; 46413f157a2fSChris Mason path->nodes[level - 1] = cur; 4642f7c79f30SChris Mason unlock_up(path, level, 1, 0, NULL); 46433f157a2fSChris Mason } 46443f157a2fSChris Mason out: 4645f98de9b9SFilipe Manana path->keep_locks = keep_locks; 4646f98de9b9SFilipe Manana if (ret == 0) { 4647f98de9b9SFilipe Manana btrfs_unlock_up_safe(path, path->lowest_level + 1); 4648f98de9b9SFilipe Manana memcpy(min_key, &found_key, sizeof(found_key)); 4649f98de9b9SFilipe Manana } 46503f157a2fSChris Mason return ret; 46513f157a2fSChris Mason } 46523f157a2fSChris Mason 46533f157a2fSChris Mason /* 46543f157a2fSChris Mason * this is similar to btrfs_next_leaf, but does not try to preserve 46553f157a2fSChris Mason * and fixup the path. It looks for and returns the next key in the 4656de78b51aSEric Sandeen * tree based on the current path and the min_trans parameters. 46573f157a2fSChris Mason * 46583f157a2fSChris Mason * 0 is returned if another key is found, < 0 if there are any errors 46593f157a2fSChris Mason * and 1 is returned if there are no higher keys in the tree 46603f157a2fSChris Mason * 46613f157a2fSChris Mason * path->keep_locks should be set to 1 on the search made before 46623f157a2fSChris Mason * calling this function. 46633f157a2fSChris Mason */ 4664e7a84565SChris Mason int btrfs_find_next_key(struct btrfs_root *root, struct btrfs_path *path, 4665de78b51aSEric Sandeen struct btrfs_key *key, int level, u64 min_trans) 4666e7a84565SChris Mason { 4667e7a84565SChris Mason int slot; 4668e7a84565SChris Mason struct extent_buffer *c; 4669e7a84565SChris Mason 46706a9fb468SJosef Bacik WARN_ON(!path->keep_locks && !path->skip_locking); 4671e7a84565SChris Mason while (level < BTRFS_MAX_LEVEL) { 4672e7a84565SChris Mason if (!path->nodes[level]) 4673e7a84565SChris Mason return 1; 4674e7a84565SChris Mason 4675e7a84565SChris Mason slot = path->slots[level] + 1; 4676e7a84565SChris Mason c = path->nodes[level]; 46773f157a2fSChris Mason next: 4678e7a84565SChris Mason if (slot >= btrfs_header_nritems(c)) { 467933c66f43SYan Zheng int ret; 468033c66f43SYan Zheng int orig_lowest; 468133c66f43SYan Zheng struct btrfs_key cur_key; 468233c66f43SYan Zheng if (level + 1 >= BTRFS_MAX_LEVEL || 468333c66f43SYan Zheng !path->nodes[level + 1]) 4684e7a84565SChris Mason return 1; 468533c66f43SYan Zheng 46866a9fb468SJosef Bacik if (path->locks[level + 1] || path->skip_locking) { 468733c66f43SYan Zheng level++; 4688e7a84565SChris Mason continue; 4689e7a84565SChris Mason } 469033c66f43SYan Zheng 469133c66f43SYan Zheng slot = btrfs_header_nritems(c) - 1; 469233c66f43SYan Zheng if (level == 0) 469333c66f43SYan Zheng btrfs_item_key_to_cpu(c, &cur_key, slot); 469433c66f43SYan Zheng else 469533c66f43SYan Zheng btrfs_node_key_to_cpu(c, &cur_key, slot); 469633c66f43SYan Zheng 469733c66f43SYan Zheng orig_lowest = path->lowest_level; 4698b3b4aa74SDavid Sterba btrfs_release_path(path); 469933c66f43SYan Zheng path->lowest_level = level; 470033c66f43SYan Zheng ret = btrfs_search_slot(NULL, root, &cur_key, path, 470133c66f43SYan Zheng 0, 0); 470233c66f43SYan Zheng path->lowest_level = orig_lowest; 470333c66f43SYan Zheng if (ret < 0) 470433c66f43SYan Zheng return ret; 470533c66f43SYan Zheng 470633c66f43SYan Zheng c = path->nodes[level]; 470733c66f43SYan Zheng slot = path->slots[level]; 470833c66f43SYan Zheng if (ret == 0) 470933c66f43SYan Zheng slot++; 471033c66f43SYan Zheng goto next; 471133c66f43SYan Zheng } 471233c66f43SYan Zheng 4713e7a84565SChris Mason if (level == 0) 4714e7a84565SChris Mason btrfs_item_key_to_cpu(c, key, slot); 47153f157a2fSChris Mason else { 47163f157a2fSChris Mason u64 gen = btrfs_node_ptr_generation(c, slot); 47173f157a2fSChris Mason 47183f157a2fSChris Mason if (gen < min_trans) { 47193f157a2fSChris Mason slot++; 47203f157a2fSChris Mason goto next; 47213f157a2fSChris Mason } 4722e7a84565SChris Mason btrfs_node_key_to_cpu(c, key, slot); 47233f157a2fSChris Mason } 4724e7a84565SChris Mason return 0; 4725e7a84565SChris Mason } 4726e7a84565SChris Mason return 1; 4727e7a84565SChris Mason } 4728e7a84565SChris Mason 47293d7806ecSJan Schmidt int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path, 47303d7806ecSJan Schmidt u64 time_seq) 47313d7806ecSJan Schmidt { 4732d97e63b6SChris Mason int slot; 47338e73f275SChris Mason int level; 47345f39d397SChris Mason struct extent_buffer *c; 47358e73f275SChris Mason struct extent_buffer *next; 4736d96b3424SFilipe Manana struct btrfs_fs_info *fs_info = root->fs_info; 4737925baeddSChris Mason struct btrfs_key key; 4738d96b3424SFilipe Manana bool need_commit_sem = false; 4739925baeddSChris Mason u32 nritems; 4740925baeddSChris Mason int ret; 47410e46318dSJosef Bacik int i; 4742925baeddSChris Mason 4743bdcdd86cSFilipe Manana /* 4744bdcdd86cSFilipe Manana * The nowait semantics are used only for write paths, where we don't 4745bdcdd86cSFilipe Manana * use the tree mod log and sequence numbers. 4746bdcdd86cSFilipe Manana */ 4747bdcdd86cSFilipe Manana if (time_seq) 4748c922b016SStefan Roesch ASSERT(!path->nowait); 4749c922b016SStefan Roesch 4750925baeddSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 4751d397712bSChris Mason if (nritems == 0) 4752925baeddSChris Mason return 1; 4753925baeddSChris Mason 47548e73f275SChris Mason btrfs_item_key_to_cpu(path->nodes[0], &key, nritems - 1); 47558e73f275SChris Mason again: 47568e73f275SChris Mason level = 1; 47578e73f275SChris Mason next = NULL; 4758b3b4aa74SDavid Sterba btrfs_release_path(path); 47598e73f275SChris Mason 4760a2135011SChris Mason path->keep_locks = 1; 47618e73f275SChris Mason 4762d96b3424SFilipe Manana if (time_seq) { 47633d7806ecSJan Schmidt ret = btrfs_search_old_slot(root, &key, path, time_seq); 4764d96b3424SFilipe Manana } else { 4765d96b3424SFilipe Manana if (path->need_commit_sem) { 4766d96b3424SFilipe Manana path->need_commit_sem = 0; 4767d96b3424SFilipe Manana need_commit_sem = true; 4768bdcdd86cSFilipe Manana if (path->nowait) { 4769bdcdd86cSFilipe Manana if (!down_read_trylock(&fs_info->commit_root_sem)) { 4770bdcdd86cSFilipe Manana ret = -EAGAIN; 4771bdcdd86cSFilipe Manana goto done; 4772bdcdd86cSFilipe Manana } 4773bdcdd86cSFilipe Manana } else { 4774d96b3424SFilipe Manana down_read(&fs_info->commit_root_sem); 4775d96b3424SFilipe Manana } 4776bdcdd86cSFilipe Manana } 4777925baeddSChris Mason ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 4778d96b3424SFilipe Manana } 4779925baeddSChris Mason path->keep_locks = 0; 4780925baeddSChris Mason 4781925baeddSChris Mason if (ret < 0) 4782d96b3424SFilipe Manana goto done; 4783925baeddSChris Mason 4784a2135011SChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 4785168fd7d2SChris Mason /* 4786168fd7d2SChris Mason * by releasing the path above we dropped all our locks. A balance 4787168fd7d2SChris Mason * could have added more items next to the key that used to be 4788168fd7d2SChris Mason * at the very end of the block. So, check again here and 4789168fd7d2SChris Mason * advance the path if there are now more items available. 4790168fd7d2SChris Mason */ 4791a2135011SChris Mason if (nritems > 0 && path->slots[0] < nritems - 1) { 4792e457afecSYan Zheng if (ret == 0) 4793168fd7d2SChris Mason path->slots[0]++; 47948e73f275SChris Mason ret = 0; 4795925baeddSChris Mason goto done; 4796925baeddSChris Mason } 47970b43e04fSLiu Bo /* 47980b43e04fSLiu Bo * So the above check misses one case: 47990b43e04fSLiu Bo * - after releasing the path above, someone has removed the item that 48000b43e04fSLiu Bo * used to be at the very end of the block, and balance between leafs 48010b43e04fSLiu Bo * gets another one with bigger key.offset to replace it. 48020b43e04fSLiu Bo * 48030b43e04fSLiu Bo * This one should be returned as well, or we can get leaf corruption 48040b43e04fSLiu Bo * later(esp. in __btrfs_drop_extents()). 48050b43e04fSLiu Bo * 48060b43e04fSLiu Bo * And a bit more explanation about this check, 48070b43e04fSLiu Bo * with ret > 0, the key isn't found, the path points to the slot 48080b43e04fSLiu Bo * where it should be inserted, so the path->slots[0] item must be the 48090b43e04fSLiu Bo * bigger one. 48100b43e04fSLiu Bo */ 48110b43e04fSLiu Bo if (nritems > 0 && ret > 0 && path->slots[0] == nritems - 1) { 48120b43e04fSLiu Bo ret = 0; 48130b43e04fSLiu Bo goto done; 48140b43e04fSLiu Bo } 4815d97e63b6SChris Mason 4816234b63a0SChris Mason while (level < BTRFS_MAX_LEVEL) { 48178e73f275SChris Mason if (!path->nodes[level]) { 48188e73f275SChris Mason ret = 1; 48198e73f275SChris Mason goto done; 48208e73f275SChris Mason } 48215f39d397SChris Mason 4822d97e63b6SChris Mason slot = path->slots[level] + 1; 4823d97e63b6SChris Mason c = path->nodes[level]; 48245f39d397SChris Mason if (slot >= btrfs_header_nritems(c)) { 4825d97e63b6SChris Mason level++; 48268e73f275SChris Mason if (level == BTRFS_MAX_LEVEL) { 48278e73f275SChris Mason ret = 1; 48288e73f275SChris Mason goto done; 48298e73f275SChris Mason } 4830d97e63b6SChris Mason continue; 4831d97e63b6SChris Mason } 48325f39d397SChris Mason 48330e46318dSJosef Bacik 48340e46318dSJosef Bacik /* 48350e46318dSJosef Bacik * Our current level is where we're going to start from, and to 48360e46318dSJosef Bacik * make sure lockdep doesn't complain we need to drop our locks 48370e46318dSJosef Bacik * and nodes from 0 to our current level. 48380e46318dSJosef Bacik */ 48390e46318dSJosef Bacik for (i = 0; i < level; i++) { 48400e46318dSJosef Bacik if (path->locks[level]) { 48410e46318dSJosef Bacik btrfs_tree_read_unlock(path->nodes[i]); 48420e46318dSJosef Bacik path->locks[i] = 0; 48430e46318dSJosef Bacik } 48440e46318dSJosef Bacik free_extent_buffer(path->nodes[i]); 48450e46318dSJosef Bacik path->nodes[i] = NULL; 4846925baeddSChris Mason } 48475f39d397SChris Mason 48488e73f275SChris Mason next = c; 4849d07b8528SLiu Bo ret = read_block_for_search(root, path, &next, level, 4850cda79c54SDavid Sterba slot, &key); 4851bdcdd86cSFilipe Manana if (ret == -EAGAIN && !path->nowait) 48528e73f275SChris Mason goto again; 48535f39d397SChris Mason 485476a05b35SChris Mason if (ret < 0) { 4855b3b4aa74SDavid Sterba btrfs_release_path(path); 485676a05b35SChris Mason goto done; 485776a05b35SChris Mason } 485876a05b35SChris Mason 48595cd57b2cSChris Mason if (!path->skip_locking) { 4860bd681513SChris Mason ret = btrfs_try_tree_read_lock(next); 4861bdcdd86cSFilipe Manana if (!ret && path->nowait) { 4862bdcdd86cSFilipe Manana ret = -EAGAIN; 4863bdcdd86cSFilipe Manana goto done; 4864bdcdd86cSFilipe Manana } 4865d42244a0SJan Schmidt if (!ret && time_seq) { 4866d42244a0SJan Schmidt /* 4867d42244a0SJan Schmidt * If we don't get the lock, we may be racing 4868d42244a0SJan Schmidt * with push_leaf_left, holding that lock while 4869d42244a0SJan Schmidt * itself waiting for the leaf we've currently 4870d42244a0SJan Schmidt * locked. To solve this situation, we give up 4871d42244a0SJan Schmidt * on our lock and cycle. 4872d42244a0SJan Schmidt */ 4873cf538830SJan Schmidt free_extent_buffer(next); 4874d42244a0SJan Schmidt btrfs_release_path(path); 4875d42244a0SJan Schmidt cond_resched(); 4876d42244a0SJan Schmidt goto again; 4877d42244a0SJan Schmidt } 48780e46318dSJosef Bacik if (!ret) 48790e46318dSJosef Bacik btrfs_tree_read_lock(next); 4880bd681513SChris Mason } 4881d97e63b6SChris Mason break; 4882d97e63b6SChris Mason } 4883d97e63b6SChris Mason path->slots[level] = slot; 4884d97e63b6SChris Mason while (1) { 4885d97e63b6SChris Mason level--; 4886d97e63b6SChris Mason path->nodes[level] = next; 4887d97e63b6SChris Mason path->slots[level] = 0; 4888a74a4b97SChris Mason if (!path->skip_locking) 4889ffeb03cfSJosef Bacik path->locks[level] = BTRFS_READ_LOCK; 4890d97e63b6SChris Mason if (!level) 4891d97e63b6SChris Mason break; 4892b4ce94deSChris Mason 4893d07b8528SLiu Bo ret = read_block_for_search(root, path, &next, level, 4894cda79c54SDavid Sterba 0, &key); 4895bdcdd86cSFilipe Manana if (ret == -EAGAIN && !path->nowait) 48968e73f275SChris Mason goto again; 48978e73f275SChris Mason 489876a05b35SChris Mason if (ret < 0) { 4899b3b4aa74SDavid Sterba btrfs_release_path(path); 490076a05b35SChris Mason goto done; 490176a05b35SChris Mason } 490276a05b35SChris Mason 4903bdcdd86cSFilipe Manana if (!path->skip_locking) { 4904bdcdd86cSFilipe Manana if (path->nowait) { 4905bdcdd86cSFilipe Manana if (!btrfs_try_tree_read_lock(next)) { 4906bdcdd86cSFilipe Manana ret = -EAGAIN; 4907bdcdd86cSFilipe Manana goto done; 4908bdcdd86cSFilipe Manana } 4909bdcdd86cSFilipe Manana } else { 49100e46318dSJosef Bacik btrfs_tree_read_lock(next); 4911d97e63b6SChris Mason } 4912bdcdd86cSFilipe Manana } 4913bdcdd86cSFilipe Manana } 49148e73f275SChris Mason ret = 0; 4915925baeddSChris Mason done: 4916f7c79f30SChris Mason unlock_up(path, 0, 1, 0, NULL); 4917d96b3424SFilipe Manana if (need_commit_sem) { 4918d96b3424SFilipe Manana int ret2; 4919d96b3424SFilipe Manana 4920d96b3424SFilipe Manana path->need_commit_sem = 1; 4921d96b3424SFilipe Manana ret2 = finish_need_commit_sem_search(path); 4922d96b3424SFilipe Manana up_read(&fs_info->commit_root_sem); 4923d96b3424SFilipe Manana if (ret2) 4924d96b3424SFilipe Manana ret = ret2; 4925d96b3424SFilipe Manana } 49268e73f275SChris Mason 49278e73f275SChris Mason return ret; 4928d97e63b6SChris Mason } 49290b86a832SChris Mason 4930890d2b1aSJosef Bacik int btrfs_next_old_item(struct btrfs_root *root, struct btrfs_path *path, u64 time_seq) 4931890d2b1aSJosef Bacik { 4932890d2b1aSJosef Bacik path->slots[0]++; 4933890d2b1aSJosef Bacik if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) 4934890d2b1aSJosef Bacik return btrfs_next_old_leaf(root, path, time_seq); 4935890d2b1aSJosef Bacik return 0; 4936890d2b1aSJosef Bacik } 4937890d2b1aSJosef Bacik 49383f157a2fSChris Mason /* 49393f157a2fSChris Mason * this uses btrfs_prev_leaf to walk backwards in the tree, and keeps 49403f157a2fSChris Mason * searching until it gets past min_objectid or finds an item of 'type' 49413f157a2fSChris Mason * 49423f157a2fSChris Mason * returns 0 if something is found, 1 if nothing was found and < 0 on error 49433f157a2fSChris Mason */ 49440b86a832SChris Mason int btrfs_previous_item(struct btrfs_root *root, 49450b86a832SChris Mason struct btrfs_path *path, u64 min_objectid, 49460b86a832SChris Mason int type) 49470b86a832SChris Mason { 49480b86a832SChris Mason struct btrfs_key found_key; 49490b86a832SChris Mason struct extent_buffer *leaf; 4950e02119d5SChris Mason u32 nritems; 49510b86a832SChris Mason int ret; 49520b86a832SChris Mason 49530b86a832SChris Mason while (1) { 49540b86a832SChris Mason if (path->slots[0] == 0) { 49550b86a832SChris Mason ret = btrfs_prev_leaf(root, path); 49560b86a832SChris Mason if (ret != 0) 49570b86a832SChris Mason return ret; 49580b86a832SChris Mason } else { 49590b86a832SChris Mason path->slots[0]--; 49600b86a832SChris Mason } 49610b86a832SChris Mason leaf = path->nodes[0]; 4962e02119d5SChris Mason nritems = btrfs_header_nritems(leaf); 4963e02119d5SChris Mason if (nritems == 0) 4964e02119d5SChris Mason return 1; 4965e02119d5SChris Mason if (path->slots[0] == nritems) 4966e02119d5SChris Mason path->slots[0]--; 4967e02119d5SChris Mason 49680b86a832SChris Mason btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 4969e02119d5SChris Mason if (found_key.objectid < min_objectid) 4970e02119d5SChris Mason break; 49710a4eefbbSYan Zheng if (found_key.type == type) 49720a4eefbbSYan Zheng return 0; 4973e02119d5SChris Mason if (found_key.objectid == min_objectid && 4974e02119d5SChris Mason found_key.type < type) 4975e02119d5SChris Mason break; 49760b86a832SChris Mason } 49770b86a832SChris Mason return 1; 49780b86a832SChris Mason } 4979ade2e0b3SWang Shilong 4980ade2e0b3SWang Shilong /* 4981ade2e0b3SWang Shilong * search in extent tree to find a previous Metadata/Data extent item with 4982ade2e0b3SWang Shilong * min objecitd. 4983ade2e0b3SWang Shilong * 4984ade2e0b3SWang Shilong * returns 0 if something is found, 1 if nothing was found and < 0 on error 4985ade2e0b3SWang Shilong */ 4986ade2e0b3SWang Shilong int btrfs_previous_extent_item(struct btrfs_root *root, 4987ade2e0b3SWang Shilong struct btrfs_path *path, u64 min_objectid) 4988ade2e0b3SWang Shilong { 4989ade2e0b3SWang Shilong struct btrfs_key found_key; 4990ade2e0b3SWang Shilong struct extent_buffer *leaf; 4991ade2e0b3SWang Shilong u32 nritems; 4992ade2e0b3SWang Shilong int ret; 4993ade2e0b3SWang Shilong 4994ade2e0b3SWang Shilong while (1) { 4995ade2e0b3SWang Shilong if (path->slots[0] == 0) { 4996ade2e0b3SWang Shilong ret = btrfs_prev_leaf(root, path); 4997ade2e0b3SWang Shilong if (ret != 0) 4998ade2e0b3SWang Shilong return ret; 4999ade2e0b3SWang Shilong } else { 5000ade2e0b3SWang Shilong path->slots[0]--; 5001ade2e0b3SWang Shilong } 5002ade2e0b3SWang Shilong leaf = path->nodes[0]; 5003ade2e0b3SWang Shilong nritems = btrfs_header_nritems(leaf); 5004ade2e0b3SWang Shilong if (nritems == 0) 5005ade2e0b3SWang Shilong return 1; 5006ade2e0b3SWang Shilong if (path->slots[0] == nritems) 5007ade2e0b3SWang Shilong path->slots[0]--; 5008ade2e0b3SWang Shilong 5009ade2e0b3SWang Shilong btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 5010ade2e0b3SWang Shilong if (found_key.objectid < min_objectid) 5011ade2e0b3SWang Shilong break; 5012ade2e0b3SWang Shilong if (found_key.type == BTRFS_EXTENT_ITEM_KEY || 5013ade2e0b3SWang Shilong found_key.type == BTRFS_METADATA_ITEM_KEY) 5014ade2e0b3SWang Shilong return 0; 5015ade2e0b3SWang Shilong if (found_key.objectid == min_objectid && 5016ade2e0b3SWang Shilong found_key.type < BTRFS_EXTENT_ITEM_KEY) 5017ade2e0b3SWang Shilong break; 5018ade2e0b3SWang Shilong } 5019ade2e0b3SWang Shilong return 1; 5020ade2e0b3SWang Shilong } 5021226463d7SJosef Bacik 5022226463d7SJosef Bacik int __init btrfs_ctree_init(void) 5023226463d7SJosef Bacik { 5024226463d7SJosef Bacik btrfs_path_cachep = kmem_cache_create("btrfs_path", 5025226463d7SJosef Bacik sizeof(struct btrfs_path), 0, 5026226463d7SJosef Bacik SLAB_MEM_SPREAD, NULL); 5027226463d7SJosef Bacik if (!btrfs_path_cachep) 5028226463d7SJosef Bacik return -ENOMEM; 5029226463d7SJosef Bacik return 0; 5030226463d7SJosef Bacik } 5031226463d7SJosef Bacik 5032226463d7SJosef Bacik void __cold btrfs_ctree_exit(void) 5033226463d7SJosef Bacik { 5034226463d7SJosef Bacik kmem_cache_destroy(btrfs_path_cachep); 5035226463d7SJosef Bacik } 5036