1c1d7c514SDavid Sterba // SPDX-License-Identifier: GPL-2.0 26cbd5570SChris Mason /* 3d352ac68SChris Mason * Copyright (C) 2007,2008 Oracle. All rights reserved. 46cbd5570SChris Mason */ 56cbd5570SChris Mason 6a6b6e75eSChris Mason #include <linux/sched.h> 75a0e3ad6STejun Heo #include <linux/slab.h> 8bd989ba3SJan Schmidt #include <linux/rbtree.h> 9adf02123SDavid Sterba #include <linux/mm.h> 10e41d12f5SChristoph Hellwig #include <linux/error-injection.h> 119b569ea0SJosef Bacik #include "messages.h" 12eb60ceacSChris Mason #include "ctree.h" 13eb60ceacSChris Mason #include "disk-io.h" 147f5c1516SChris Mason #include "transaction.h" 155f39d397SChris Mason #include "print-tree.h" 16925baeddSChris Mason #include "locking.h" 17de37aa51SNikolay Borisov #include "volumes.h" 18f616f5cdSQu Wenruo #include "qgroup.h" 19f3a84ccdSFilipe Manana #include "tree-mod-log.h" 2088c602abSQu Wenruo #include "tree-checker.h" 21ec8eb376SJosef Bacik #include "fs.h" 22ad1ac501SJosef Bacik #include "accessors.h" 23a0231804SJosef Bacik #include "extent-tree.h" 2467707479SJosef Bacik #include "relocation.h" 256bfd0ffaSJosef Bacik #include "file-item.h" 269a8dd150SChris Mason 27226463d7SJosef Bacik static struct kmem_cache *btrfs_path_cachep; 28226463d7SJosef Bacik 29e089f05cSChris Mason static int split_node(struct btrfs_trans_handle *trans, struct btrfs_root 30e089f05cSChris Mason *root, struct btrfs_path *path, int level); 31310712b2SOmar Sandoval static int split_leaf(struct btrfs_trans_handle *trans, struct btrfs_root *root, 32310712b2SOmar Sandoval const struct btrfs_key *ins_key, struct btrfs_path *path, 33310712b2SOmar Sandoval int data_size, int extend); 345f39d397SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 352ff7e61eSJeff Mahoney struct extent_buffer *dst, 36971a1f66SChris Mason struct extent_buffer *src, int empty); 375f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 385f39d397SChris Mason struct extent_buffer *dst_buf, 395f39d397SChris Mason struct extent_buffer *src_buf); 40d97e63b6SChris Mason 41af024ed2SJohannes Thumshirn static const struct btrfs_csums { 42af024ed2SJohannes Thumshirn u16 size; 4359a0fcdbSDavid Sterba const char name[10]; 4459a0fcdbSDavid Sterba const char driver[12]; 45af024ed2SJohannes Thumshirn } btrfs_csums[] = { 46af024ed2SJohannes Thumshirn [BTRFS_CSUM_TYPE_CRC32] = { .size = 4, .name = "crc32c" }, 473951e7f0SJohannes Thumshirn [BTRFS_CSUM_TYPE_XXHASH] = { .size = 8, .name = "xxhash64" }, 483831bf00SJohannes Thumshirn [BTRFS_CSUM_TYPE_SHA256] = { .size = 32, .name = "sha256" }, 49352ae07bSDavid Sterba [BTRFS_CSUM_TYPE_BLAKE2] = { .size = 32, .name = "blake2b", 50352ae07bSDavid Sterba .driver = "blake2b-256" }, 51af024ed2SJohannes Thumshirn }; 52af024ed2SJohannes Thumshirn 533a3178c7SJosef Bacik /* 543a3178c7SJosef Bacik * The leaf data grows from end-to-front in the node. this returns the address 553a3178c7SJosef Bacik * of the start of the last item, which is the stop of the leaf data stack. 563a3178c7SJosef Bacik */ 573a3178c7SJosef Bacik static unsigned int leaf_data_end(const struct extent_buffer *leaf) 583a3178c7SJosef Bacik { 593a3178c7SJosef Bacik u32 nr = btrfs_header_nritems(leaf); 603a3178c7SJosef Bacik 613a3178c7SJosef Bacik if (nr == 0) 623a3178c7SJosef Bacik return BTRFS_LEAF_DATA_SIZE(leaf->fs_info); 633a3178c7SJosef Bacik return btrfs_item_offset(leaf, nr - 1); 643a3178c7SJosef Bacik } 653a3178c7SJosef Bacik 66637e3b48SJosef Bacik /* 67637e3b48SJosef Bacik * Move data in a @leaf (using memmove, safe for overlapping ranges). 68637e3b48SJosef Bacik * 69637e3b48SJosef Bacik * @leaf: leaf that we're doing a memmove on 70637e3b48SJosef Bacik * @dst_offset: item data offset we're moving to 71637e3b48SJosef Bacik * @src_offset: item data offset were' moving from 72637e3b48SJosef Bacik * @len: length of the data we're moving 73637e3b48SJosef Bacik * 74637e3b48SJosef Bacik * Wrapper around memmove_extent_buffer() that takes into account the header on 75637e3b48SJosef Bacik * the leaf. The btrfs_item offset's start directly after the header, so we 76637e3b48SJosef Bacik * have to adjust any offsets to account for the header in the leaf. This 77637e3b48SJosef Bacik * handles that math to simplify the callers. 78637e3b48SJosef Bacik */ 79637e3b48SJosef Bacik static inline void memmove_leaf_data(const struct extent_buffer *leaf, 80637e3b48SJosef Bacik unsigned long dst_offset, 81637e3b48SJosef Bacik unsigned long src_offset, 82637e3b48SJosef Bacik unsigned long len) 83637e3b48SJosef Bacik { 848009adf3SJosef Bacik memmove_extent_buffer(leaf, btrfs_item_nr_offset(leaf, 0) + dst_offset, 858009adf3SJosef Bacik btrfs_item_nr_offset(leaf, 0) + src_offset, len); 86637e3b48SJosef Bacik } 87637e3b48SJosef Bacik 88637e3b48SJosef Bacik /* 89637e3b48SJosef Bacik * Copy item data from @src into @dst at the given @offset. 90637e3b48SJosef Bacik * 91637e3b48SJosef Bacik * @dst: destination leaf that we're copying into 92637e3b48SJosef Bacik * @src: source leaf that we're copying from 93637e3b48SJosef Bacik * @dst_offset: item data offset we're copying to 94637e3b48SJosef Bacik * @src_offset: item data offset were' copying from 95637e3b48SJosef Bacik * @len: length of the data we're copying 96637e3b48SJosef Bacik * 97637e3b48SJosef Bacik * Wrapper around copy_extent_buffer() that takes into account the header on 98637e3b48SJosef Bacik * the leaf. The btrfs_item offset's start directly after the header, so we 99637e3b48SJosef Bacik * have to adjust any offsets to account for the header in the leaf. This 100637e3b48SJosef Bacik * handles that math to simplify the callers. 101637e3b48SJosef Bacik */ 102637e3b48SJosef Bacik static inline void copy_leaf_data(const struct extent_buffer *dst, 103637e3b48SJosef Bacik const struct extent_buffer *src, 104637e3b48SJosef Bacik unsigned long dst_offset, 105637e3b48SJosef Bacik unsigned long src_offset, unsigned long len) 106637e3b48SJosef Bacik { 1078009adf3SJosef Bacik copy_extent_buffer(dst, src, btrfs_item_nr_offset(dst, 0) + dst_offset, 1088009adf3SJosef Bacik btrfs_item_nr_offset(src, 0) + src_offset, len); 109637e3b48SJosef Bacik } 110637e3b48SJosef Bacik 111637e3b48SJosef Bacik /* 112637e3b48SJosef Bacik * Move items in a @leaf (using memmove). 113637e3b48SJosef Bacik * 114637e3b48SJosef Bacik * @dst: destination leaf for the items 115637e3b48SJosef Bacik * @dst_item: the item nr we're copying into 116637e3b48SJosef Bacik * @src_item: the item nr we're copying from 117637e3b48SJosef Bacik * @nr_items: the number of items to copy 118637e3b48SJosef Bacik * 119637e3b48SJosef Bacik * Wrapper around memmove_extent_buffer() that does the math to get the 120637e3b48SJosef Bacik * appropriate offsets into the leaf from the item numbers. 121637e3b48SJosef Bacik */ 122637e3b48SJosef Bacik static inline void memmove_leaf_items(const struct extent_buffer *leaf, 123637e3b48SJosef Bacik int dst_item, int src_item, int nr_items) 124637e3b48SJosef Bacik { 125637e3b48SJosef Bacik memmove_extent_buffer(leaf, btrfs_item_nr_offset(leaf, dst_item), 126637e3b48SJosef Bacik btrfs_item_nr_offset(leaf, src_item), 127637e3b48SJosef Bacik nr_items * sizeof(struct btrfs_item)); 128637e3b48SJosef Bacik } 129637e3b48SJosef Bacik 130637e3b48SJosef Bacik /* 131637e3b48SJosef Bacik * Copy items from @src into @dst at the given @offset. 132637e3b48SJosef Bacik * 133637e3b48SJosef Bacik * @dst: destination leaf for the items 134637e3b48SJosef Bacik * @src: source leaf for the items 135637e3b48SJosef Bacik * @dst_item: the item nr we're copying into 136637e3b48SJosef Bacik * @src_item: the item nr we're copying from 137637e3b48SJosef Bacik * @nr_items: the number of items to copy 138637e3b48SJosef Bacik * 139637e3b48SJosef Bacik * Wrapper around copy_extent_buffer() that does the math to get the 140637e3b48SJosef Bacik * appropriate offsets into the leaf from the item numbers. 141637e3b48SJosef Bacik */ 142637e3b48SJosef Bacik static inline void copy_leaf_items(const struct extent_buffer *dst, 143637e3b48SJosef Bacik const struct extent_buffer *src, 144637e3b48SJosef Bacik int dst_item, int src_item, int nr_items) 145637e3b48SJosef Bacik { 146637e3b48SJosef Bacik copy_extent_buffer(dst, src, btrfs_item_nr_offset(dst, dst_item), 147637e3b48SJosef Bacik btrfs_item_nr_offset(src, src_item), 148637e3b48SJosef Bacik nr_items * sizeof(struct btrfs_item)); 149637e3b48SJosef Bacik } 150637e3b48SJosef Bacik 151b3cbfb0dSJosef Bacik /* This exists for btrfs-progs usages. */ 152b3cbfb0dSJosef Bacik u16 btrfs_csum_type_size(u16 type) 153b3cbfb0dSJosef Bacik { 154b3cbfb0dSJosef Bacik return btrfs_csums[type].size; 155b3cbfb0dSJosef Bacik } 156b3cbfb0dSJosef Bacik 157af024ed2SJohannes Thumshirn int btrfs_super_csum_size(const struct btrfs_super_block *s) 158af024ed2SJohannes Thumshirn { 159af024ed2SJohannes Thumshirn u16 t = btrfs_super_csum_type(s); 160af024ed2SJohannes Thumshirn /* 161af024ed2SJohannes Thumshirn * csum type is validated at mount time 162af024ed2SJohannes Thumshirn */ 163b3cbfb0dSJosef Bacik return btrfs_csum_type_size(t); 164af024ed2SJohannes Thumshirn } 165af024ed2SJohannes Thumshirn 166af024ed2SJohannes Thumshirn const char *btrfs_super_csum_name(u16 csum_type) 167af024ed2SJohannes Thumshirn { 168af024ed2SJohannes Thumshirn /* csum type is validated at mount time */ 169af024ed2SJohannes Thumshirn return btrfs_csums[csum_type].name; 170af024ed2SJohannes Thumshirn } 171af024ed2SJohannes Thumshirn 172b4e967beSDavid Sterba /* 173b4e967beSDavid Sterba * Return driver name if defined, otherwise the name that's also a valid driver 174b4e967beSDavid Sterba * name 175b4e967beSDavid Sterba */ 176b4e967beSDavid Sterba const char *btrfs_super_csum_driver(u16 csum_type) 177b4e967beSDavid Sterba { 178b4e967beSDavid Sterba /* csum type is validated at mount time */ 17959a0fcdbSDavid Sterba return btrfs_csums[csum_type].driver[0] ? 18059a0fcdbSDavid Sterba btrfs_csums[csum_type].driver : 181b4e967beSDavid Sterba btrfs_csums[csum_type].name; 182b4e967beSDavid Sterba } 183b4e967beSDavid Sterba 184604997b4SDavid Sterba size_t __attribute_const__ btrfs_get_num_csums(void) 185f7cea56cSDavid Sterba { 186f7cea56cSDavid Sterba return ARRAY_SIZE(btrfs_csums); 187f7cea56cSDavid Sterba } 188f7cea56cSDavid Sterba 1892c90e5d6SChris Mason struct btrfs_path *btrfs_alloc_path(void) 1902c90e5d6SChris Mason { 191a4c853afSChenXiaoSong might_sleep(); 192a4c853afSChenXiaoSong 193e2c89907SMasahiro Yamada return kmem_cache_zalloc(btrfs_path_cachep, GFP_NOFS); 1942c90e5d6SChris Mason } 1952c90e5d6SChris Mason 196d352ac68SChris Mason /* this also releases the path */ 1972c90e5d6SChris Mason void btrfs_free_path(struct btrfs_path *p) 1982c90e5d6SChris Mason { 199ff175d57SJesper Juhl if (!p) 200ff175d57SJesper Juhl return; 201b3b4aa74SDavid Sterba btrfs_release_path(p); 2022c90e5d6SChris Mason kmem_cache_free(btrfs_path_cachep, p); 2032c90e5d6SChris Mason } 2042c90e5d6SChris Mason 205d352ac68SChris Mason /* 206d352ac68SChris Mason * path release drops references on the extent buffers in the path 207d352ac68SChris Mason * and it drops any locks held by this path 208d352ac68SChris Mason * 209d352ac68SChris Mason * It is safe to call this on paths that no locks or extent buffers held. 210d352ac68SChris Mason */ 211b3b4aa74SDavid Sterba noinline void btrfs_release_path(struct btrfs_path *p) 212eb60ceacSChris Mason { 213eb60ceacSChris Mason int i; 214a2135011SChris Mason 215234b63a0SChris Mason for (i = 0; i < BTRFS_MAX_LEVEL; i++) { 2163f157a2fSChris Mason p->slots[i] = 0; 217eb60ceacSChris Mason if (!p->nodes[i]) 218925baeddSChris Mason continue; 219925baeddSChris Mason if (p->locks[i]) { 220bd681513SChris Mason btrfs_tree_unlock_rw(p->nodes[i], p->locks[i]); 221925baeddSChris Mason p->locks[i] = 0; 222925baeddSChris Mason } 2235f39d397SChris Mason free_extent_buffer(p->nodes[i]); 2243f157a2fSChris Mason p->nodes[i] = NULL; 225eb60ceacSChris Mason } 226eb60ceacSChris Mason } 227eb60ceacSChris Mason 228d352ac68SChris Mason /* 2298bb808c6SDavid Sterba * We want the transaction abort to print stack trace only for errors where the 2308bb808c6SDavid Sterba * cause could be a bug, eg. due to ENOSPC, and not for common errors that are 2318bb808c6SDavid Sterba * caused by external factors. 2328bb808c6SDavid Sterba */ 2338bb808c6SDavid Sterba bool __cold abort_should_print_stack(int errno) 2348bb808c6SDavid Sterba { 2358bb808c6SDavid Sterba switch (errno) { 2368bb808c6SDavid Sterba case -EIO: 2378bb808c6SDavid Sterba case -EROFS: 2388bb808c6SDavid Sterba case -ENOMEM: 2398bb808c6SDavid Sterba return false; 2408bb808c6SDavid Sterba } 2418bb808c6SDavid Sterba return true; 2428bb808c6SDavid Sterba } 2438bb808c6SDavid Sterba 2448bb808c6SDavid Sterba /* 245d352ac68SChris Mason * safely gets a reference on the root node of a tree. A lock 246d352ac68SChris Mason * is not taken, so a concurrent writer may put a different node 247d352ac68SChris Mason * at the root of the tree. See btrfs_lock_root_node for the 248d352ac68SChris Mason * looping required. 249d352ac68SChris Mason * 250d352ac68SChris Mason * The extent buffer returned by this has a reference taken, so 251d352ac68SChris Mason * it won't disappear. It may stop being the root of the tree 252d352ac68SChris Mason * at any time because there are no locks held. 253d352ac68SChris Mason */ 254925baeddSChris Mason struct extent_buffer *btrfs_root_node(struct btrfs_root *root) 255925baeddSChris Mason { 256925baeddSChris Mason struct extent_buffer *eb; 257240f62c8SChris Mason 2583083ee2eSJosef Bacik while (1) { 259240f62c8SChris Mason rcu_read_lock(); 260240f62c8SChris Mason eb = rcu_dereference(root->node); 2613083ee2eSJosef Bacik 2623083ee2eSJosef Bacik /* 2633083ee2eSJosef Bacik * RCU really hurts here, we could free up the root node because 26401327610SNicholas D Steeves * it was COWed but we may not get the new root node yet so do 2653083ee2eSJosef Bacik * the inc_not_zero dance and if it doesn't work then 2663083ee2eSJosef Bacik * synchronize_rcu and try again. 2673083ee2eSJosef Bacik */ 2683083ee2eSJosef Bacik if (atomic_inc_not_zero(&eb->refs)) { 269240f62c8SChris Mason rcu_read_unlock(); 2703083ee2eSJosef Bacik break; 2713083ee2eSJosef Bacik } 2723083ee2eSJosef Bacik rcu_read_unlock(); 2733083ee2eSJosef Bacik synchronize_rcu(); 2743083ee2eSJosef Bacik } 275925baeddSChris Mason return eb; 276925baeddSChris Mason } 277925baeddSChris Mason 27892a7cc42SQu Wenruo /* 27992a7cc42SQu Wenruo * Cowonly root (not-shareable trees, everything not subvolume or reloc roots), 28092a7cc42SQu Wenruo * just get put onto a simple dirty list. Transaction walks this list to make 28192a7cc42SQu Wenruo * sure they get properly updated on disk. 282d352ac68SChris Mason */ 2830b86a832SChris Mason static void add_root_to_dirty_list(struct btrfs_root *root) 2840b86a832SChris Mason { 2850b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 2860b246afaSJeff Mahoney 287e7070be1SJosef Bacik if (test_bit(BTRFS_ROOT_DIRTY, &root->state) || 288e7070be1SJosef Bacik !test_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state)) 289e7070be1SJosef Bacik return; 290e7070be1SJosef Bacik 2910b246afaSJeff Mahoney spin_lock(&fs_info->trans_lock); 292e7070be1SJosef Bacik if (!test_and_set_bit(BTRFS_ROOT_DIRTY, &root->state)) { 293e7070be1SJosef Bacik /* Want the extent tree to be the last on the list */ 2944fd786e6SMisono Tomohiro if (root->root_key.objectid == BTRFS_EXTENT_TREE_OBJECTID) 295e7070be1SJosef Bacik list_move_tail(&root->dirty_list, 2960b246afaSJeff Mahoney &fs_info->dirty_cowonly_roots); 297e7070be1SJosef Bacik else 298e7070be1SJosef Bacik list_move(&root->dirty_list, 2990b246afaSJeff Mahoney &fs_info->dirty_cowonly_roots); 3000b86a832SChris Mason } 3010b246afaSJeff Mahoney spin_unlock(&fs_info->trans_lock); 3020b86a832SChris Mason } 3030b86a832SChris Mason 304d352ac68SChris Mason /* 305d352ac68SChris Mason * used by snapshot creation to make a copy of a root for a tree with 306d352ac68SChris Mason * a given objectid. The buffer with the new root node is returned in 307d352ac68SChris Mason * cow_ret, and this func returns zero on success or a negative error code. 308d352ac68SChris Mason */ 309be20aa9dSChris Mason int btrfs_copy_root(struct btrfs_trans_handle *trans, 310be20aa9dSChris Mason struct btrfs_root *root, 311be20aa9dSChris Mason struct extent_buffer *buf, 312be20aa9dSChris Mason struct extent_buffer **cow_ret, u64 new_root_objectid) 313be20aa9dSChris Mason { 3140b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 315be20aa9dSChris Mason struct extent_buffer *cow; 316be20aa9dSChris Mason int ret = 0; 317be20aa9dSChris Mason int level; 3185d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 319be20aa9dSChris Mason 32092a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 3210b246afaSJeff Mahoney trans->transid != fs_info->running_transaction->transid); 32292a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 32327cdeb70SMiao Xie trans->transid != root->last_trans); 324be20aa9dSChris Mason 325be20aa9dSChris Mason level = btrfs_header_level(buf); 3265d4f98a2SYan Zheng if (level == 0) 3275d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 3285d4f98a2SYan Zheng else 3295d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 33031840ae1SZheng Yan 3314d75f8a9SDavid Sterba cow = btrfs_alloc_tree_block(trans, root, 0, new_root_objectid, 332cf6f34aaSJosef Bacik &disk_key, level, buf->start, 0, 333cf6f34aaSJosef Bacik BTRFS_NESTING_NEW_ROOT); 3345d4f98a2SYan Zheng if (IS_ERR(cow)) 335be20aa9dSChris Mason return PTR_ERR(cow); 336be20aa9dSChris Mason 33758e8012cSDavid Sterba copy_extent_buffer_full(cow, buf); 338be20aa9dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 339be20aa9dSChris Mason btrfs_set_header_generation(cow, trans->transid); 3405d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 3415d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 3425d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 3435d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 3445d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 3455d4f98a2SYan Zheng else 346be20aa9dSChris Mason btrfs_set_header_owner(cow, new_root_objectid); 347be20aa9dSChris Mason 348de37aa51SNikolay Borisov write_extent_buffer_fsid(cow, fs_info->fs_devices->metadata_uuid); 3492b82032cSYan Zheng 350be20aa9dSChris Mason WARN_ON(btrfs_header_generation(buf) > trans->transid); 3515d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 352e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 3535d4f98a2SYan Zheng else 354e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 355867ed321SJosef Bacik if (ret) { 35672c9925fSFilipe Manana btrfs_tree_unlock(cow); 35772c9925fSFilipe Manana free_extent_buffer(cow); 358867ed321SJosef Bacik btrfs_abort_transaction(trans, ret); 359be20aa9dSChris Mason return ret; 360867ed321SJosef Bacik } 361be20aa9dSChris Mason 362be20aa9dSChris Mason btrfs_mark_buffer_dirty(cow); 363be20aa9dSChris Mason *cow_ret = cow; 364be20aa9dSChris Mason return 0; 365be20aa9dSChris Mason } 366be20aa9dSChris Mason 367d352ac68SChris Mason /* 3685d4f98a2SYan Zheng * check if the tree block can be shared by multiple trees 3695d4f98a2SYan Zheng */ 3705d4f98a2SYan Zheng int btrfs_block_can_be_shared(struct btrfs_root *root, 3715d4f98a2SYan Zheng struct extent_buffer *buf) 3725d4f98a2SYan Zheng { 3735d4f98a2SYan Zheng /* 37492a7cc42SQu Wenruo * Tree blocks not in shareable trees and tree roots are never shared. 37592a7cc42SQu Wenruo * If a block was allocated after the last snapshot and the block was 37692a7cc42SQu Wenruo * not allocated by tree relocation, we know the block is not shared. 3775d4f98a2SYan Zheng */ 37892a7cc42SQu Wenruo if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 3795d4f98a2SYan Zheng buf != root->node && buf != root->commit_root && 3805d4f98a2SYan Zheng (btrfs_header_generation(buf) <= 3815d4f98a2SYan Zheng btrfs_root_last_snapshot(&root->root_item) || 3825d4f98a2SYan Zheng btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC))) 3835d4f98a2SYan Zheng return 1; 384a79865c6SNikolay Borisov 3855d4f98a2SYan Zheng return 0; 3865d4f98a2SYan Zheng } 3875d4f98a2SYan Zheng 3885d4f98a2SYan Zheng static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans, 3895d4f98a2SYan Zheng struct btrfs_root *root, 3905d4f98a2SYan Zheng struct extent_buffer *buf, 391f0486c68SYan, Zheng struct extent_buffer *cow, 392f0486c68SYan, Zheng int *last_ref) 3935d4f98a2SYan Zheng { 3940b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 3955d4f98a2SYan Zheng u64 refs; 3965d4f98a2SYan Zheng u64 owner; 3975d4f98a2SYan Zheng u64 flags; 3985d4f98a2SYan Zheng u64 new_flags = 0; 3995d4f98a2SYan Zheng int ret; 4005d4f98a2SYan Zheng 4015d4f98a2SYan Zheng /* 4025d4f98a2SYan Zheng * Backrefs update rules: 4035d4f98a2SYan Zheng * 4045d4f98a2SYan Zheng * Always use full backrefs for extent pointers in tree block 4055d4f98a2SYan Zheng * allocated by tree relocation. 4065d4f98a2SYan Zheng * 4075d4f98a2SYan Zheng * If a shared tree block is no longer referenced by its owner 4085d4f98a2SYan Zheng * tree (btrfs_header_owner(buf) == root->root_key.objectid), 4095d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 4105d4f98a2SYan Zheng * 4115d4f98a2SYan Zheng * If a tree block is been relocating 4125d4f98a2SYan Zheng * (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID), 4135d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 4145d4f98a2SYan Zheng * The reason for this is some operations (such as drop tree) 4155d4f98a2SYan Zheng * are only allowed for blocks use full backrefs. 4165d4f98a2SYan Zheng */ 4175d4f98a2SYan Zheng 4185d4f98a2SYan Zheng if (btrfs_block_can_be_shared(root, buf)) { 4192ff7e61eSJeff Mahoney ret = btrfs_lookup_extent_info(trans, fs_info, buf->start, 4203173a18fSJosef Bacik btrfs_header_level(buf), 1, 4213173a18fSJosef Bacik &refs, &flags); 422be1a5564SMark Fasheh if (ret) 423be1a5564SMark Fasheh return ret; 424eced687eSFilipe Manana if (unlikely(refs == 0)) { 425eced687eSFilipe Manana btrfs_crit(fs_info, 426eced687eSFilipe Manana "found 0 references for tree block at bytenr %llu level %d root %llu", 427eced687eSFilipe Manana buf->start, btrfs_header_level(buf), 428eced687eSFilipe Manana btrfs_root_id(root)); 429eced687eSFilipe Manana ret = -EUCLEAN; 430eced687eSFilipe Manana btrfs_abort_transaction(trans, ret); 431e5df9573SMark Fasheh return ret; 432e5df9573SMark Fasheh } 4335d4f98a2SYan Zheng } else { 4345d4f98a2SYan Zheng refs = 1; 4355d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 4365d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 4375d4f98a2SYan Zheng flags = BTRFS_BLOCK_FLAG_FULL_BACKREF; 4385d4f98a2SYan Zheng else 4395d4f98a2SYan Zheng flags = 0; 4405d4f98a2SYan Zheng } 4415d4f98a2SYan Zheng 4425d4f98a2SYan Zheng owner = btrfs_header_owner(buf); 4435d4f98a2SYan Zheng BUG_ON(owner == BTRFS_TREE_RELOC_OBJECTID && 4445d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)); 4455d4f98a2SYan Zheng 4465d4f98a2SYan Zheng if (refs > 1) { 4475d4f98a2SYan Zheng if ((owner == root->root_key.objectid || 4485d4f98a2SYan Zheng root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && 4495d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) { 450e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, buf, 1); 451692826b2SJeff Mahoney if (ret) 452692826b2SJeff Mahoney return ret; 4535d4f98a2SYan Zheng 4545d4f98a2SYan Zheng if (root->root_key.objectid == 4555d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) { 456e339a6b0SJosef Bacik ret = btrfs_dec_ref(trans, root, buf, 0); 457692826b2SJeff Mahoney if (ret) 458692826b2SJeff Mahoney return ret; 459e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 460692826b2SJeff Mahoney if (ret) 461692826b2SJeff Mahoney return ret; 4625d4f98a2SYan Zheng } 4635d4f98a2SYan Zheng new_flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF; 4645d4f98a2SYan Zheng } else { 4655d4f98a2SYan Zheng 4665d4f98a2SYan Zheng if (root->root_key.objectid == 4675d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 468e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 4695d4f98a2SYan Zheng else 470e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 471692826b2SJeff Mahoney if (ret) 472692826b2SJeff Mahoney return ret; 4735d4f98a2SYan Zheng } 4745d4f98a2SYan Zheng if (new_flags != 0) { 4754aec05faSJosef Bacik ret = btrfs_set_disk_extent_flags(trans, buf, new_flags); 476be1a5564SMark Fasheh if (ret) 477be1a5564SMark Fasheh return ret; 4785d4f98a2SYan Zheng } 4795d4f98a2SYan Zheng } else { 4805d4f98a2SYan Zheng if (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) { 4815d4f98a2SYan Zheng if (root->root_key.objectid == 4825d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 483e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 4845d4f98a2SYan Zheng else 485e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 486692826b2SJeff Mahoney if (ret) 487692826b2SJeff Mahoney return ret; 488e339a6b0SJosef Bacik ret = btrfs_dec_ref(trans, root, buf, 1); 489692826b2SJeff Mahoney if (ret) 490692826b2SJeff Mahoney return ret; 4915d4f98a2SYan Zheng } 492190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, buf); 493f0486c68SYan, Zheng *last_ref = 1; 4945d4f98a2SYan Zheng } 4955d4f98a2SYan Zheng return 0; 4965d4f98a2SYan Zheng } 4975d4f98a2SYan Zheng 4985d4f98a2SYan Zheng /* 499d397712bSChris Mason * does the dirty work in cow of a single block. The parent block (if 500d397712bSChris Mason * supplied) is updated to point to the new cow copy. The new buffer is marked 501d397712bSChris Mason * dirty and returned locked. If you modify the block it needs to be marked 502d397712bSChris Mason * dirty again. 503d352ac68SChris Mason * 504d352ac68SChris Mason * search_start -- an allocation hint for the new block 505d352ac68SChris Mason * 506d397712bSChris Mason * empty_size -- a hint that you plan on doing more cow. This is the size in 507d397712bSChris Mason * bytes the allocator should try to find free next to the block it returns. 508d397712bSChris Mason * This is just a hint and may be ignored by the allocator. 509d352ac68SChris Mason */ 510d397712bSChris Mason static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans, 5115f39d397SChris Mason struct btrfs_root *root, 5125f39d397SChris Mason struct extent_buffer *buf, 5135f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 5145f39d397SChris Mason struct extent_buffer **cow_ret, 5159631e4ccSJosef Bacik u64 search_start, u64 empty_size, 5169631e4ccSJosef Bacik enum btrfs_lock_nesting nest) 5176702ed49SChris Mason { 5180b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 5195d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 5205f39d397SChris Mason struct extent_buffer *cow; 521be1a5564SMark Fasheh int level, ret; 522f0486c68SYan, Zheng int last_ref = 0; 523925baeddSChris Mason int unlock_orig = 0; 5240f5053ebSGoldwyn Rodrigues u64 parent_start = 0; 5256702ed49SChris Mason 526925baeddSChris Mason if (*cow_ret == buf) 527925baeddSChris Mason unlock_orig = 1; 528925baeddSChris Mason 52949d0c642SFilipe Manana btrfs_assert_tree_write_locked(buf); 530925baeddSChris Mason 53192a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 5320b246afaSJeff Mahoney trans->transid != fs_info->running_transaction->transid); 53392a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 53427cdeb70SMiao Xie trans->transid != root->last_trans); 5355f39d397SChris Mason 5367bb86316SChris Mason level = btrfs_header_level(buf); 53731840ae1SZheng Yan 5385d4f98a2SYan Zheng if (level == 0) 5395d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 5405d4f98a2SYan Zheng else 5415d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 5425d4f98a2SYan Zheng 5430f5053ebSGoldwyn Rodrigues if ((root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && parent) 5445d4f98a2SYan Zheng parent_start = parent->start; 5455d4f98a2SYan Zheng 54679bd3712SFilipe Manana cow = btrfs_alloc_tree_block(trans, root, parent_start, 54779bd3712SFilipe Manana root->root_key.objectid, &disk_key, level, 54879bd3712SFilipe Manana search_start, empty_size, nest); 5496702ed49SChris Mason if (IS_ERR(cow)) 5506702ed49SChris Mason return PTR_ERR(cow); 5516702ed49SChris Mason 552b4ce94deSChris Mason /* cow is set to blocking by btrfs_init_new_buffer */ 553b4ce94deSChris Mason 55458e8012cSDavid Sterba copy_extent_buffer_full(cow, buf); 555db94535dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 5565f39d397SChris Mason btrfs_set_header_generation(cow, trans->transid); 5575d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 5585d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 5595d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 5605d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) 5615d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 5625d4f98a2SYan Zheng else 5635f39d397SChris Mason btrfs_set_header_owner(cow, root->root_key.objectid); 5646702ed49SChris Mason 565de37aa51SNikolay Borisov write_extent_buffer_fsid(cow, fs_info->fs_devices->metadata_uuid); 5662b82032cSYan Zheng 567be1a5564SMark Fasheh ret = update_ref_for_cow(trans, root, buf, cow, &last_ref); 568b68dc2a9SMark Fasheh if (ret) { 569572c83acSJosef Bacik btrfs_tree_unlock(cow); 570572c83acSJosef Bacik free_extent_buffer(cow); 57166642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 572b68dc2a9SMark Fasheh return ret; 573b68dc2a9SMark Fasheh } 5741a40e23bSZheng Yan 57592a7cc42SQu Wenruo if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) { 57683d4cfd4SJosef Bacik ret = btrfs_reloc_cow_block(trans, root, buf, cow); 57793314e3bSZhaolei if (ret) { 578572c83acSJosef Bacik btrfs_tree_unlock(cow); 579572c83acSJosef Bacik free_extent_buffer(cow); 58066642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 58183d4cfd4SJosef Bacik return ret; 58283d4cfd4SJosef Bacik } 58393314e3bSZhaolei } 5843fd0a558SYan, Zheng 5856702ed49SChris Mason if (buf == root->node) { 586925baeddSChris Mason WARN_ON(parent && parent != buf); 5875d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 5885d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 5895d4f98a2SYan Zheng parent_start = buf->start; 590925baeddSChris Mason 591406808abSFilipe Manana ret = btrfs_tree_mod_log_insert_root(root->node, cow, true); 59240b0a749SFilipe Manana if (ret < 0) { 59340b0a749SFilipe Manana btrfs_tree_unlock(cow); 59440b0a749SFilipe Manana free_extent_buffer(cow); 59540b0a749SFilipe Manana btrfs_abort_transaction(trans, ret); 59640b0a749SFilipe Manana return ret; 59740b0a749SFilipe Manana } 59840b0a749SFilipe Manana atomic_inc(&cow->refs); 599240f62c8SChris Mason rcu_assign_pointer(root->node, cow); 600925baeddSChris Mason 6017a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), buf, 6027a163608SFilipe Manana parent_start, last_ref); 6035f39d397SChris Mason free_extent_buffer(buf); 6040b86a832SChris Mason add_root_to_dirty_list(root); 6056702ed49SChris Mason } else { 6065d4f98a2SYan Zheng WARN_ON(trans->transid != btrfs_header_generation(parent)); 607d09c5152SFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, parent_slot, 60833cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 609d09c5152SFilipe Manana if (ret) { 610d09c5152SFilipe Manana btrfs_tree_unlock(cow); 611d09c5152SFilipe Manana free_extent_buffer(cow); 612d09c5152SFilipe Manana btrfs_abort_transaction(trans, ret); 613d09c5152SFilipe Manana return ret; 614d09c5152SFilipe Manana } 6155f39d397SChris Mason btrfs_set_node_blockptr(parent, parent_slot, 616db94535dSChris Mason cow->start); 61774493f7aSChris Mason btrfs_set_node_ptr_generation(parent, parent_slot, 61874493f7aSChris Mason trans->transid); 6196702ed49SChris Mason btrfs_mark_buffer_dirty(parent); 6205de865eeSFilipe David Borba Manana if (last_ref) { 621f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_free_eb(buf); 6225de865eeSFilipe David Borba Manana if (ret) { 623572c83acSJosef Bacik btrfs_tree_unlock(cow); 624572c83acSJosef Bacik free_extent_buffer(cow); 62566642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 6265de865eeSFilipe David Borba Manana return ret; 6275de865eeSFilipe David Borba Manana } 6285de865eeSFilipe David Borba Manana } 6297a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), buf, 6307a163608SFilipe Manana parent_start, last_ref); 6316702ed49SChris Mason } 632925baeddSChris Mason if (unlock_orig) 633925baeddSChris Mason btrfs_tree_unlock(buf); 6343083ee2eSJosef Bacik free_extent_buffer_stale(buf); 6356702ed49SChris Mason btrfs_mark_buffer_dirty(cow); 6366702ed49SChris Mason *cow_ret = cow; 6376702ed49SChris Mason return 0; 6386702ed49SChris Mason } 6396702ed49SChris Mason 6405d4f98a2SYan Zheng static inline int should_cow_block(struct btrfs_trans_handle *trans, 6415d4f98a2SYan Zheng struct btrfs_root *root, 6425d4f98a2SYan Zheng struct extent_buffer *buf) 6435d4f98a2SYan Zheng { 644f5ee5c9aSJeff Mahoney if (btrfs_is_testing(root->fs_info)) 645faa2dbf0SJosef Bacik return 0; 646fccb84c9SDavid Sterba 647d1980131SDavid Sterba /* Ensure we can see the FORCE_COW bit */ 648d1980131SDavid Sterba smp_mb__before_atomic(); 649f1ebcc74SLiu Bo 650f1ebcc74SLiu Bo /* 651f1ebcc74SLiu Bo * We do not need to cow a block if 652f1ebcc74SLiu Bo * 1) this block is not created or changed in this transaction; 653f1ebcc74SLiu Bo * 2) this block does not belong to TREE_RELOC tree; 654f1ebcc74SLiu Bo * 3) the root is not forced COW. 655f1ebcc74SLiu Bo * 656f1ebcc74SLiu Bo * What is forced COW: 65701327610SNicholas D Steeves * when we create snapshot during committing the transaction, 65852042d8eSAndrea Gelmini * after we've finished copying src root, we must COW the shared 659f1ebcc74SLiu Bo * block to ensure the metadata consistency. 660f1ebcc74SLiu Bo */ 6615d4f98a2SYan Zheng if (btrfs_header_generation(buf) == trans->transid && 6625d4f98a2SYan Zheng !btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN) && 6635d4f98a2SYan Zheng !(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID && 664f1ebcc74SLiu Bo btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)) && 66527cdeb70SMiao Xie !test_bit(BTRFS_ROOT_FORCE_COW, &root->state)) 6665d4f98a2SYan Zheng return 0; 6675d4f98a2SYan Zheng return 1; 6685d4f98a2SYan Zheng } 6695d4f98a2SYan Zheng 670d352ac68SChris Mason /* 671d352ac68SChris Mason * cows a single block, see __btrfs_cow_block for the real work. 67201327610SNicholas D Steeves * This version of it has extra checks so that a block isn't COWed more than 673d352ac68SChris Mason * once per transaction, as long as it hasn't been written yet 674d352ac68SChris Mason */ 675d397712bSChris Mason noinline int btrfs_cow_block(struct btrfs_trans_handle *trans, 6765f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *buf, 6775f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 6789631e4ccSJosef Bacik struct extent_buffer **cow_ret, 6799631e4ccSJosef Bacik enum btrfs_lock_nesting nest) 68002217ed2SChris Mason { 6810b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 6826702ed49SChris Mason u64 search_start; 683f510cfecSChris Mason int ret; 684dc17ff8fSChris Mason 685*a2caab29SFilipe Manana if (unlikely(test_bit(BTRFS_ROOT_DELETING, &root->state))) { 686*a2caab29SFilipe Manana btrfs_abort_transaction(trans, -EUCLEAN); 687*a2caab29SFilipe Manana btrfs_crit(fs_info, 688*a2caab29SFilipe Manana "attempt to COW block %llu on root %llu that is being deleted", 689*a2caab29SFilipe Manana buf->start, btrfs_root_id(root)); 690*a2caab29SFilipe Manana return -EUCLEAN; 691*a2caab29SFilipe Manana } 69283354f07SJosef Bacik 69348774f3bSFilipe Manana /* 69448774f3bSFilipe Manana * COWing must happen through a running transaction, which always 69548774f3bSFilipe Manana * matches the current fs generation (it's a transaction with a state 69648774f3bSFilipe Manana * less than TRANS_STATE_UNBLOCKED). If it doesn't, then turn the fs 69748774f3bSFilipe Manana * into error state to prevent the commit of any transaction. 69848774f3bSFilipe Manana */ 69948774f3bSFilipe Manana if (unlikely(trans->transaction != fs_info->running_transaction || 70048774f3bSFilipe Manana trans->transid != fs_info->generation)) { 70148774f3bSFilipe Manana btrfs_abort_transaction(trans, -EUCLEAN); 70248774f3bSFilipe Manana btrfs_crit(fs_info, 70348774f3bSFilipe Manana "unexpected transaction when attempting to COW block %llu on root %llu, transaction %llu running transaction %llu fs generation %llu", 70448774f3bSFilipe Manana buf->start, btrfs_root_id(root), trans->transid, 70548774f3bSFilipe Manana fs_info->running_transaction->transid, 70648774f3bSFilipe Manana fs_info->generation); 70748774f3bSFilipe Manana return -EUCLEAN; 70848774f3bSFilipe Manana } 709dc17ff8fSChris Mason 7105d4f98a2SYan Zheng if (!should_cow_block(trans, root, buf)) { 71102217ed2SChris Mason *cow_ret = buf; 71202217ed2SChris Mason return 0; 71302217ed2SChris Mason } 714c487685dSChris Mason 715ee22184bSByongho Lee search_start = buf->start & ~((u64)SZ_1G - 1); 716b4ce94deSChris Mason 717f616f5cdSQu Wenruo /* 718f616f5cdSQu Wenruo * Before CoWing this block for later modification, check if it's 719f616f5cdSQu Wenruo * the subtree root and do the delayed subtree trace if needed. 720f616f5cdSQu Wenruo * 721f616f5cdSQu Wenruo * Also We don't care about the error, as it's handled internally. 722f616f5cdSQu Wenruo */ 723f616f5cdSQu Wenruo btrfs_qgroup_trace_subtree_after_cow(trans, root, buf); 724f510cfecSChris Mason ret = __btrfs_cow_block(trans, root, buf, parent, 7259631e4ccSJosef Bacik parent_slot, cow_ret, search_start, 0, nest); 7261abe9b8aSliubo 7271abe9b8aSliubo trace_btrfs_cow_block(root, buf, *cow_ret); 7281abe9b8aSliubo 729f510cfecSChris Mason return ret; 7302c90e5d6SChris Mason } 731f75e2b79SJosef Bacik ALLOW_ERROR_INJECTION(btrfs_cow_block, ERRNO); 7326702ed49SChris Mason 733d352ac68SChris Mason /* 734d352ac68SChris Mason * helper function for defrag to decide if two blocks pointed to by a 735d352ac68SChris Mason * node are actually close by 736d352ac68SChris Mason */ 7376b80053dSChris Mason static int close_blocks(u64 blocknr, u64 other, u32 blocksize) 7386702ed49SChris Mason { 7396b80053dSChris Mason if (blocknr < other && other - (blocknr + blocksize) < 32768) 7406702ed49SChris Mason return 1; 7416b80053dSChris Mason if (blocknr > other && blocknr - (other + blocksize) < 32768) 7426702ed49SChris Mason return 1; 74302217ed2SChris Mason return 0; 74402217ed2SChris Mason } 74502217ed2SChris Mason 746ce6ef5abSDavid Sterba #ifdef __LITTLE_ENDIAN 747ce6ef5abSDavid Sterba 748ce6ef5abSDavid Sterba /* 749ce6ef5abSDavid Sterba * Compare two keys, on little-endian the disk order is same as CPU order and 750ce6ef5abSDavid Sterba * we can avoid the conversion. 751ce6ef5abSDavid Sterba */ 752ce6ef5abSDavid Sterba static int comp_keys(const struct btrfs_disk_key *disk_key, 753ce6ef5abSDavid Sterba const struct btrfs_key *k2) 754ce6ef5abSDavid Sterba { 755ce6ef5abSDavid Sterba const struct btrfs_key *k1 = (const struct btrfs_key *)disk_key; 756ce6ef5abSDavid Sterba 757ce6ef5abSDavid Sterba return btrfs_comp_cpu_keys(k1, k2); 758ce6ef5abSDavid Sterba } 759ce6ef5abSDavid Sterba 760ce6ef5abSDavid Sterba #else 761ce6ef5abSDavid Sterba 762081e9573SChris Mason /* 763081e9573SChris Mason * compare two keys in a memcmp fashion 764081e9573SChris Mason */ 765310712b2SOmar Sandoval static int comp_keys(const struct btrfs_disk_key *disk, 766310712b2SOmar Sandoval const struct btrfs_key *k2) 767081e9573SChris Mason { 768081e9573SChris Mason struct btrfs_key k1; 769081e9573SChris Mason 770081e9573SChris Mason btrfs_disk_key_to_cpu(&k1, disk); 771081e9573SChris Mason 77220736abaSDiego Calleja return btrfs_comp_cpu_keys(&k1, k2); 773081e9573SChris Mason } 774ce6ef5abSDavid Sterba #endif 775081e9573SChris Mason 776f3465ca4SJosef Bacik /* 777f3465ca4SJosef Bacik * same as comp_keys only with two btrfs_key's 778f3465ca4SJosef Bacik */ 779e1f60a65SDavid Sterba int __pure btrfs_comp_cpu_keys(const struct btrfs_key *k1, const struct btrfs_key *k2) 780f3465ca4SJosef Bacik { 781f3465ca4SJosef Bacik if (k1->objectid > k2->objectid) 782f3465ca4SJosef Bacik return 1; 783f3465ca4SJosef Bacik if (k1->objectid < k2->objectid) 784f3465ca4SJosef Bacik return -1; 785f3465ca4SJosef Bacik if (k1->type > k2->type) 786f3465ca4SJosef Bacik return 1; 787f3465ca4SJosef Bacik if (k1->type < k2->type) 788f3465ca4SJosef Bacik return -1; 789f3465ca4SJosef Bacik if (k1->offset > k2->offset) 790f3465ca4SJosef Bacik return 1; 791f3465ca4SJosef Bacik if (k1->offset < k2->offset) 792f3465ca4SJosef Bacik return -1; 793f3465ca4SJosef Bacik return 0; 794f3465ca4SJosef Bacik } 795081e9573SChris Mason 796d352ac68SChris Mason /* 797d352ac68SChris Mason * this is used by the defrag code to go through all the 798d352ac68SChris Mason * leaves pointed to by a node and reallocate them so that 799d352ac68SChris Mason * disk order is close to key order 800d352ac68SChris Mason */ 8016702ed49SChris Mason int btrfs_realloc_node(struct btrfs_trans_handle *trans, 8025f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *parent, 803de78b51aSEric Sandeen int start_slot, u64 *last_ret, 804a6b6e75eSChris Mason struct btrfs_key *progress) 8056702ed49SChris Mason { 8060b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 8076b80053dSChris Mason struct extent_buffer *cur; 8086702ed49SChris Mason u64 blocknr; 809e9d0b13bSChris Mason u64 search_start = *last_ret; 810e9d0b13bSChris Mason u64 last_block = 0; 8116702ed49SChris Mason u64 other; 8126702ed49SChris Mason u32 parent_nritems; 8136702ed49SChris Mason int end_slot; 8146702ed49SChris Mason int i; 8156702ed49SChris Mason int err = 0; 8166b80053dSChris Mason u32 blocksize; 817081e9573SChris Mason int progress_passed = 0; 818081e9573SChris Mason struct btrfs_disk_key disk_key; 8196702ed49SChris Mason 8200b246afaSJeff Mahoney WARN_ON(trans->transaction != fs_info->running_transaction); 8210b246afaSJeff Mahoney WARN_ON(trans->transid != fs_info->generation); 82286479a04SChris Mason 8236b80053dSChris Mason parent_nritems = btrfs_header_nritems(parent); 8240b246afaSJeff Mahoney blocksize = fs_info->nodesize; 8255dfe2be7SFilipe Manana end_slot = parent_nritems - 1; 8266702ed49SChris Mason 8275dfe2be7SFilipe Manana if (parent_nritems <= 1) 8286702ed49SChris Mason return 0; 8296702ed49SChris Mason 8305dfe2be7SFilipe Manana for (i = start_slot; i <= end_slot; i++) { 8316702ed49SChris Mason int close = 1; 832a6b6e75eSChris Mason 833081e9573SChris Mason btrfs_node_key(parent, &disk_key, i); 834081e9573SChris Mason if (!progress_passed && comp_keys(&disk_key, progress) < 0) 835081e9573SChris Mason continue; 836081e9573SChris Mason 837081e9573SChris Mason progress_passed = 1; 8386b80053dSChris Mason blocknr = btrfs_node_blockptr(parent, i); 839e9d0b13bSChris Mason if (last_block == 0) 840e9d0b13bSChris Mason last_block = blocknr; 8415708b959SChris Mason 8426702ed49SChris Mason if (i > 0) { 8436b80053dSChris Mason other = btrfs_node_blockptr(parent, i - 1); 8446b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 8456702ed49SChris Mason } 8465dfe2be7SFilipe Manana if (!close && i < end_slot) { 8476b80053dSChris Mason other = btrfs_node_blockptr(parent, i + 1); 8486b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 8496702ed49SChris Mason } 850e9d0b13bSChris Mason if (close) { 851e9d0b13bSChris Mason last_block = blocknr; 8526702ed49SChris Mason continue; 853e9d0b13bSChris Mason } 8546702ed49SChris Mason 855206983b7SJosef Bacik cur = btrfs_read_node_slot(parent, i); 856206983b7SJosef Bacik if (IS_ERR(cur)) 85764c043deSLiu Bo return PTR_ERR(cur); 858e9d0b13bSChris Mason if (search_start == 0) 8596b80053dSChris Mason search_start = last_block; 860e9d0b13bSChris Mason 861e7a84565SChris Mason btrfs_tree_lock(cur); 8626b80053dSChris Mason err = __btrfs_cow_block(trans, root, cur, parent, i, 863e7a84565SChris Mason &cur, search_start, 8646b80053dSChris Mason min(16 * blocksize, 8659631e4ccSJosef Bacik (end_slot - i) * blocksize), 8669631e4ccSJosef Bacik BTRFS_NESTING_COW); 867252c38f0SYan if (err) { 868e7a84565SChris Mason btrfs_tree_unlock(cur); 8696b80053dSChris Mason free_extent_buffer(cur); 8706702ed49SChris Mason break; 871252c38f0SYan } 872e7a84565SChris Mason search_start = cur->start; 873e7a84565SChris Mason last_block = cur->start; 874f2183bdeSChris Mason *last_ret = search_start; 875e7a84565SChris Mason btrfs_tree_unlock(cur); 876e7a84565SChris Mason free_extent_buffer(cur); 8776702ed49SChris Mason } 8786702ed49SChris Mason return err; 8796702ed49SChris Mason } 8806702ed49SChris Mason 88174123bd7SChris Mason /* 882fb81212cSFilipe Manana * Search for a key in the given extent_buffer. 8835f39d397SChris Mason * 884a724f313SFilipe Manana * The lower boundary for the search is specified by the slot number @first_slot. 885fdf8d595SAnand Jain * Use a value of 0 to search over the whole extent buffer. Works for both 886fdf8d595SAnand Jain * leaves and nodes. 88774123bd7SChris Mason * 888fb81212cSFilipe Manana * The slot in the extent buffer is returned via @slot. If the key exists in the 889fb81212cSFilipe Manana * extent buffer, then @slot will point to the slot where the key is, otherwise 890fb81212cSFilipe Manana * it points to the slot where you would insert the key. 891fb81212cSFilipe Manana * 892fb81212cSFilipe Manana * Slot may point to the total number of items (i.e. one position beyond the last 893fb81212cSFilipe Manana * key) if the key is bigger than the last key in the extent buffer. 89474123bd7SChris Mason */ 895fdf8d595SAnand Jain int btrfs_bin_search(struct extent_buffer *eb, int first_slot, 89667d5e289SMarcos Paulo de Souza const struct btrfs_key *key, int *slot) 897be0e5c09SChris Mason { 898fb81212cSFilipe Manana unsigned long p; 899fb81212cSFilipe Manana int item_size; 900a724f313SFilipe Manana /* 901a724f313SFilipe Manana * Use unsigned types for the low and high slots, so that we get a more 902a724f313SFilipe Manana * efficient division in the search loop below. 903a724f313SFilipe Manana */ 904a724f313SFilipe Manana u32 low = first_slot; 905a724f313SFilipe Manana u32 high = btrfs_header_nritems(eb); 906be0e5c09SChris Mason int ret; 9075cd17f34SDavid Sterba const int key_size = sizeof(struct btrfs_disk_key); 908be0e5c09SChris Mason 909a724f313SFilipe Manana if (unlikely(low > high)) { 9105e24e9afSLiu Bo btrfs_err(eb->fs_info, 911a724f313SFilipe Manana "%s: low (%u) > high (%u) eb %llu owner %llu level %d", 9125e24e9afSLiu Bo __func__, low, high, eb->start, 9135e24e9afSLiu Bo btrfs_header_owner(eb), btrfs_header_level(eb)); 9145e24e9afSLiu Bo return -EINVAL; 9155e24e9afSLiu Bo } 9165e24e9afSLiu Bo 917fb81212cSFilipe Manana if (btrfs_header_level(eb) == 0) { 918fb81212cSFilipe Manana p = offsetof(struct btrfs_leaf, items); 919fb81212cSFilipe Manana item_size = sizeof(struct btrfs_item); 920fb81212cSFilipe Manana } else { 921fb81212cSFilipe Manana p = offsetof(struct btrfs_node, ptrs); 922fb81212cSFilipe Manana item_size = sizeof(struct btrfs_key_ptr); 923fb81212cSFilipe Manana } 924fb81212cSFilipe Manana 925be0e5c09SChris Mason while (low < high) { 9265cd17f34SDavid Sterba unsigned long oip; 9275cd17f34SDavid Sterba unsigned long offset; 9285cd17f34SDavid Sterba struct btrfs_disk_key *tmp; 9295cd17f34SDavid Sterba struct btrfs_disk_key unaligned; 9305cd17f34SDavid Sterba int mid; 9315cd17f34SDavid Sterba 932be0e5c09SChris Mason mid = (low + high) / 2; 9335f39d397SChris Mason offset = p + mid * item_size; 9345cd17f34SDavid Sterba oip = offset_in_page(offset); 9355f39d397SChris Mason 9365cd17f34SDavid Sterba if (oip + key_size <= PAGE_SIZE) { 937884b07d0SQu Wenruo const unsigned long idx = get_eb_page_index(offset); 9385cd17f34SDavid Sterba char *kaddr = page_address(eb->pages[idx]); 939934d375bSChris Mason 940884b07d0SQu Wenruo oip = get_eb_offset_in_page(eb, offset); 9415cd17f34SDavid Sterba tmp = (struct btrfs_disk_key *)(kaddr + oip); 9425cd17f34SDavid Sterba } else { 9435cd17f34SDavid Sterba read_extent_buffer(eb, &unaligned, offset, key_size); 9445f39d397SChris Mason tmp = &unaligned; 945479965d6SChris Mason } 946479965d6SChris Mason 947be0e5c09SChris Mason ret = comp_keys(tmp, key); 948be0e5c09SChris Mason 949be0e5c09SChris Mason if (ret < 0) 950be0e5c09SChris Mason low = mid + 1; 951be0e5c09SChris Mason else if (ret > 0) 952be0e5c09SChris Mason high = mid; 953be0e5c09SChris Mason else { 954be0e5c09SChris Mason *slot = mid; 955be0e5c09SChris Mason return 0; 956be0e5c09SChris Mason } 957be0e5c09SChris Mason } 958be0e5c09SChris Mason *slot = low; 959be0e5c09SChris Mason return 1; 960be0e5c09SChris Mason } 961be0e5c09SChris Mason 962f0486c68SYan, Zheng static void root_add_used(struct btrfs_root *root, u32 size) 963f0486c68SYan, Zheng { 964f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 965f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 966f0486c68SYan, Zheng btrfs_root_used(&root->root_item) + size); 967f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 968f0486c68SYan, Zheng } 969f0486c68SYan, Zheng 970f0486c68SYan, Zheng static void root_sub_used(struct btrfs_root *root, u32 size) 971f0486c68SYan, Zheng { 972f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 973f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 974f0486c68SYan, Zheng btrfs_root_used(&root->root_item) - size); 975f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 976f0486c68SYan, Zheng } 977f0486c68SYan, Zheng 978d352ac68SChris Mason /* given a node and slot number, this reads the blocks it points to. The 979d352ac68SChris Mason * extent buffer is returned with a reference taken (but unlocked). 980d352ac68SChris Mason */ 9814b231ae4SDavid Sterba struct extent_buffer *btrfs_read_node_slot(struct extent_buffer *parent, 9824b231ae4SDavid Sterba int slot) 983bb803951SChris Mason { 984ca7a79adSChris Mason int level = btrfs_header_level(parent); 985789d6a3aSQu Wenruo struct btrfs_tree_parent_check check = { 0 }; 986416bc658SJosef Bacik struct extent_buffer *eb; 987416bc658SJosef Bacik 988fb770ae4SLiu Bo if (slot < 0 || slot >= btrfs_header_nritems(parent)) 989fb770ae4SLiu Bo return ERR_PTR(-ENOENT); 990ca7a79adSChris Mason 991d4694728SJosef Bacik ASSERT(level); 992ca7a79adSChris Mason 993789d6a3aSQu Wenruo check.level = level - 1; 994789d6a3aSQu Wenruo check.transid = btrfs_node_ptr_generation(parent, slot); 995789d6a3aSQu Wenruo check.owner_root = btrfs_header_owner(parent); 996789d6a3aSQu Wenruo check.has_first_key = true; 997789d6a3aSQu Wenruo btrfs_node_key_to_cpu(parent, &check.first_key, slot); 998789d6a3aSQu Wenruo 999d0d20b0fSDavid Sterba eb = read_tree_block(parent->fs_info, btrfs_node_blockptr(parent, slot), 1000789d6a3aSQu Wenruo &check); 10014eb150d6SQu Wenruo if (IS_ERR(eb)) 10024eb150d6SQu Wenruo return eb; 10034eb150d6SQu Wenruo if (!extent_buffer_uptodate(eb)) { 1004416bc658SJosef Bacik free_extent_buffer(eb); 10054eb150d6SQu Wenruo return ERR_PTR(-EIO); 1006416bc658SJosef Bacik } 1007416bc658SJosef Bacik 1008416bc658SJosef Bacik return eb; 1009bb803951SChris Mason } 1010bb803951SChris Mason 1011d352ac68SChris Mason /* 1012d352ac68SChris Mason * node level balancing, used to make sure nodes are in proper order for 1013d352ac68SChris Mason * item deletion. We balance from the top down, so we have to make sure 1014d352ac68SChris Mason * that a deletion won't leave an node completely empty later on. 1015d352ac68SChris Mason */ 1016e02119d5SChris Mason static noinline int balance_level(struct btrfs_trans_handle *trans, 101798ed5174SChris Mason struct btrfs_root *root, 101898ed5174SChris Mason struct btrfs_path *path, int level) 1019bb803951SChris Mason { 10200b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 10215f39d397SChris Mason struct extent_buffer *right = NULL; 10225f39d397SChris Mason struct extent_buffer *mid; 10235f39d397SChris Mason struct extent_buffer *left = NULL; 10245f39d397SChris Mason struct extent_buffer *parent = NULL; 1025bb803951SChris Mason int ret = 0; 1026bb803951SChris Mason int wret; 1027bb803951SChris Mason int pslot; 1028bb803951SChris Mason int orig_slot = path->slots[level]; 102979f95c82SChris Mason u64 orig_ptr; 1030bb803951SChris Mason 103198e6b1ebSLiu Bo ASSERT(level > 0); 1032bb803951SChris Mason 10335f39d397SChris Mason mid = path->nodes[level]; 1034b4ce94deSChris Mason 1035ac5887c8SJosef Bacik WARN_ON(path->locks[level] != BTRFS_WRITE_LOCK); 10367bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 10377bb86316SChris Mason 10381d4f8a0cSChris Mason orig_ptr = btrfs_node_blockptr(mid, orig_slot); 103979f95c82SChris Mason 1040a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 10415f39d397SChris Mason parent = path->nodes[level + 1]; 1042bb803951SChris Mason pslot = path->slots[level + 1]; 1043a05a9bb1SLi Zefan } 1044bb803951SChris Mason 104540689478SChris Mason /* 104640689478SChris Mason * deal with the case where there is only one pointer in the root 104740689478SChris Mason * by promoting the node below to a root 104840689478SChris Mason */ 10495f39d397SChris Mason if (!parent) { 10505f39d397SChris Mason struct extent_buffer *child; 1051bb803951SChris Mason 10525f39d397SChris Mason if (btrfs_header_nritems(mid) != 1) 1053bb803951SChris Mason return 0; 1054bb803951SChris Mason 1055bb803951SChris Mason /* promote the child to a root */ 10564b231ae4SDavid Sterba child = btrfs_read_node_slot(mid, 0); 1057fb770ae4SLiu Bo if (IS_ERR(child)) { 1058fb770ae4SLiu Bo ret = PTR_ERR(child); 1059daefe4d4SFilipe Manana goto out; 1060305a26afSMark Fasheh } 1061305a26afSMark Fasheh 1062925baeddSChris Mason btrfs_tree_lock(child); 10639631e4ccSJosef Bacik ret = btrfs_cow_block(trans, root, child, mid, 0, &child, 10649631e4ccSJosef Bacik BTRFS_NESTING_COW); 1065f0486c68SYan, Zheng if (ret) { 1066f0486c68SYan, Zheng btrfs_tree_unlock(child); 1067f0486c68SYan, Zheng free_extent_buffer(child); 1068daefe4d4SFilipe Manana goto out; 1069f0486c68SYan, Zheng } 10702f375ab9SYan 1071406808abSFilipe Manana ret = btrfs_tree_mod_log_insert_root(root->node, child, true); 107239020d8aSFilipe Manana if (ret < 0) { 107339020d8aSFilipe Manana btrfs_tree_unlock(child); 107439020d8aSFilipe Manana free_extent_buffer(child); 107539020d8aSFilipe Manana btrfs_abort_transaction(trans, ret); 1076daefe4d4SFilipe Manana goto out; 107739020d8aSFilipe Manana } 1078240f62c8SChris Mason rcu_assign_pointer(root->node, child); 1079925baeddSChris Mason 10800b86a832SChris Mason add_root_to_dirty_list(root); 1081925baeddSChris Mason btrfs_tree_unlock(child); 1082b4ce94deSChris Mason 1083925baeddSChris Mason path->locks[level] = 0; 1084bb803951SChris Mason path->nodes[level] = NULL; 1085190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, mid); 1086925baeddSChris Mason btrfs_tree_unlock(mid); 1087bb803951SChris Mason /* once for the path */ 10885f39d397SChris Mason free_extent_buffer(mid); 1089f0486c68SYan, Zheng 1090f0486c68SYan, Zheng root_sub_used(root, mid->len); 10917a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), mid, 0, 1); 1092bb803951SChris Mason /* once for the root ptr */ 10933083ee2eSJosef Bacik free_extent_buffer_stale(mid); 1094f0486c68SYan, Zheng return 0; 1095bb803951SChris Mason } 10965f39d397SChris Mason if (btrfs_header_nritems(mid) > 10970b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 4) 1098bb803951SChris Mason return 0; 1099bb803951SChris Mason 11009cf14029SJosef Bacik if (pslot) { 11014b231ae4SDavid Sterba left = btrfs_read_node_slot(parent, pslot - 1); 11029cf14029SJosef Bacik if (IS_ERR(left)) { 11039cf14029SJosef Bacik ret = PTR_ERR(left); 1104fb770ae4SLiu Bo left = NULL; 1105daefe4d4SFilipe Manana goto out; 11069cf14029SJosef Bacik } 1107fb770ae4SLiu Bo 1108bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT); 11095f39d397SChris Mason wret = btrfs_cow_block(trans, root, left, 11109631e4ccSJosef Bacik parent, pslot - 1, &left, 1111bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW); 111254aa1f4dSChris Mason if (wret) { 111354aa1f4dSChris Mason ret = wret; 1114daefe4d4SFilipe Manana goto out; 111554aa1f4dSChris Mason } 11162cc58cf2SChris Mason } 1117fb770ae4SLiu Bo 11189cf14029SJosef Bacik if (pslot + 1 < btrfs_header_nritems(parent)) { 11194b231ae4SDavid Sterba right = btrfs_read_node_slot(parent, pslot + 1); 11209cf14029SJosef Bacik if (IS_ERR(right)) { 11219cf14029SJosef Bacik ret = PTR_ERR(right); 1122fb770ae4SLiu Bo right = NULL; 1123daefe4d4SFilipe Manana goto out; 11249cf14029SJosef Bacik } 1125fb770ae4SLiu Bo 1126bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT); 11275f39d397SChris Mason wret = btrfs_cow_block(trans, root, right, 11289631e4ccSJosef Bacik parent, pslot + 1, &right, 1129bf59a5a2SJosef Bacik BTRFS_NESTING_RIGHT_COW); 11302cc58cf2SChris Mason if (wret) { 11312cc58cf2SChris Mason ret = wret; 1132daefe4d4SFilipe Manana goto out; 11332cc58cf2SChris Mason } 11342cc58cf2SChris Mason } 11352cc58cf2SChris Mason 11362cc58cf2SChris Mason /* first, try to make some room in the middle buffer */ 11375f39d397SChris Mason if (left) { 11385f39d397SChris Mason orig_slot += btrfs_header_nritems(left); 1139d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 1); 114079f95c82SChris Mason if (wret < 0) 114179f95c82SChris Mason ret = wret; 1142bb803951SChris Mason } 114379f95c82SChris Mason 114479f95c82SChris Mason /* 114579f95c82SChris Mason * then try to empty the right most buffer into the middle 114679f95c82SChris Mason */ 11475f39d397SChris Mason if (right) { 1148d30a668fSDavid Sterba wret = push_node_left(trans, mid, right, 1); 114954aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 115079f95c82SChris Mason ret = wret; 11515f39d397SChris Mason if (btrfs_header_nritems(right) == 0) { 1152190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, right); 1153925baeddSChris Mason btrfs_tree_unlock(right); 1154751a2761SFilipe Manana ret = btrfs_del_ptr(trans, root, path, level + 1, pslot + 1); 1155751a2761SFilipe Manana if (ret < 0) { 1156751a2761SFilipe Manana free_extent_buffer_stale(right); 1157751a2761SFilipe Manana right = NULL; 1158751a2761SFilipe Manana goto out; 1159751a2761SFilipe Manana } 1160f0486c68SYan, Zheng root_sub_used(root, right->len); 11617a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), right, 11627a163608SFilipe Manana 0, 1); 11633083ee2eSJosef Bacik free_extent_buffer_stale(right); 1164f0486c68SYan, Zheng right = NULL; 1165bb803951SChris Mason } else { 11665f39d397SChris Mason struct btrfs_disk_key right_key; 11675f39d397SChris Mason btrfs_node_key(right, &right_key, 0); 1168f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot + 1, 116933cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 117039020d8aSFilipe Manana if (ret < 0) { 117139020d8aSFilipe Manana btrfs_abort_transaction(trans, ret); 1172daefe4d4SFilipe Manana goto out; 117339020d8aSFilipe Manana } 11745f39d397SChris Mason btrfs_set_node_key(parent, &right_key, pslot + 1); 11755f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 1176bb803951SChris Mason } 1177bb803951SChris Mason } 11785f39d397SChris Mason if (btrfs_header_nritems(mid) == 1) { 117979f95c82SChris Mason /* 118079f95c82SChris Mason * we're not allowed to leave a node with one item in the 118179f95c82SChris Mason * tree during a delete. A deletion from lower in the tree 118279f95c82SChris Mason * could try to delete the only pointer in this node. 118379f95c82SChris Mason * So, pull some keys from the left. 118479f95c82SChris Mason * There has to be a left pointer at this point because 118579f95c82SChris Mason * otherwise we would have pulled some pointers from the 118679f95c82SChris Mason * right 118779f95c82SChris Mason */ 1188725026edSFilipe Manana if (unlikely(!left)) { 1189725026edSFilipe Manana btrfs_crit(fs_info, 1190725026edSFilipe Manana "missing left child when middle child only has 1 item, parent bytenr %llu level %d mid bytenr %llu root %llu", 1191725026edSFilipe Manana parent->start, btrfs_header_level(parent), 1192725026edSFilipe Manana mid->start, btrfs_root_id(root)); 1193725026edSFilipe Manana ret = -EUCLEAN; 1194725026edSFilipe Manana btrfs_abort_transaction(trans, ret); 1195daefe4d4SFilipe Manana goto out; 1196305a26afSMark Fasheh } 119755d32ed8SDavid Sterba wret = balance_node_right(trans, mid, left); 119854aa1f4dSChris Mason if (wret < 0) { 119979f95c82SChris Mason ret = wret; 1200daefe4d4SFilipe Manana goto out; 120154aa1f4dSChris Mason } 1202bce4eae9SChris Mason if (wret == 1) { 1203d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 1); 1204bce4eae9SChris Mason if (wret < 0) 1205bce4eae9SChris Mason ret = wret; 1206bce4eae9SChris Mason } 120779f95c82SChris Mason BUG_ON(wret == 1); 120879f95c82SChris Mason } 12095f39d397SChris Mason if (btrfs_header_nritems(mid) == 0) { 1210190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, mid); 1211925baeddSChris Mason btrfs_tree_unlock(mid); 1212751a2761SFilipe Manana ret = btrfs_del_ptr(trans, root, path, level + 1, pslot); 1213751a2761SFilipe Manana if (ret < 0) { 1214751a2761SFilipe Manana free_extent_buffer_stale(mid); 1215751a2761SFilipe Manana mid = NULL; 1216751a2761SFilipe Manana goto out; 1217751a2761SFilipe Manana } 1218f0486c68SYan, Zheng root_sub_used(root, mid->len); 12197a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), mid, 0, 1); 12203083ee2eSJosef Bacik free_extent_buffer_stale(mid); 1221f0486c68SYan, Zheng mid = NULL; 122279f95c82SChris Mason } else { 122379f95c82SChris Mason /* update the parent key to reflect our changes */ 12245f39d397SChris Mason struct btrfs_disk_key mid_key; 12255f39d397SChris Mason btrfs_node_key(mid, &mid_key, 0); 1226f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot, 122733cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 122839020d8aSFilipe Manana if (ret < 0) { 122939020d8aSFilipe Manana btrfs_abort_transaction(trans, ret); 1230daefe4d4SFilipe Manana goto out; 123139020d8aSFilipe Manana } 12325f39d397SChris Mason btrfs_set_node_key(parent, &mid_key, pslot); 12335f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 123479f95c82SChris Mason } 1235bb803951SChris Mason 123679f95c82SChris Mason /* update the path */ 12375f39d397SChris Mason if (left) { 12385f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 123967439dadSDavid Sterba atomic_inc(&left->refs); 1240925baeddSChris Mason /* left was locked after cow */ 12415f39d397SChris Mason path->nodes[level] = left; 1242bb803951SChris Mason path->slots[level + 1] -= 1; 1243bb803951SChris Mason path->slots[level] = orig_slot; 1244925baeddSChris Mason if (mid) { 1245925baeddSChris Mason btrfs_tree_unlock(mid); 12465f39d397SChris Mason free_extent_buffer(mid); 1247925baeddSChris Mason } 1248bb803951SChris Mason } else { 12495f39d397SChris Mason orig_slot -= btrfs_header_nritems(left); 1250bb803951SChris Mason path->slots[level] = orig_slot; 1251bb803951SChris Mason } 1252bb803951SChris Mason } 125379f95c82SChris Mason /* double check we haven't messed things up */ 1254e20d96d6SChris Mason if (orig_ptr != 12555f39d397SChris Mason btrfs_node_blockptr(path->nodes[level], path->slots[level])) 125679f95c82SChris Mason BUG(); 1257daefe4d4SFilipe Manana out: 1258925baeddSChris Mason if (right) { 1259925baeddSChris Mason btrfs_tree_unlock(right); 12605f39d397SChris Mason free_extent_buffer(right); 1261925baeddSChris Mason } 1262925baeddSChris Mason if (left) { 1263925baeddSChris Mason if (path->nodes[level] != left) 1264925baeddSChris Mason btrfs_tree_unlock(left); 12655f39d397SChris Mason free_extent_buffer(left); 1266925baeddSChris Mason } 1267bb803951SChris Mason return ret; 1268bb803951SChris Mason } 1269bb803951SChris Mason 1270d352ac68SChris Mason /* Node balancing for insertion. Here we only split or push nodes around 1271d352ac68SChris Mason * when they are completely full. This is also done top down, so we 1272d352ac68SChris Mason * have to be pessimistic. 1273d352ac68SChris Mason */ 1274d397712bSChris Mason static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans, 1275e66f709bSChris Mason struct btrfs_root *root, 1276e66f709bSChris Mason struct btrfs_path *path, int level) 1277e66f709bSChris Mason { 12780b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 12795f39d397SChris Mason struct extent_buffer *right = NULL; 12805f39d397SChris Mason struct extent_buffer *mid; 12815f39d397SChris Mason struct extent_buffer *left = NULL; 12825f39d397SChris Mason struct extent_buffer *parent = NULL; 1283e66f709bSChris Mason int ret = 0; 1284e66f709bSChris Mason int wret; 1285e66f709bSChris Mason int pslot; 1286e66f709bSChris Mason int orig_slot = path->slots[level]; 1287e66f709bSChris Mason 1288e66f709bSChris Mason if (level == 0) 1289e66f709bSChris Mason return 1; 1290e66f709bSChris Mason 12915f39d397SChris Mason mid = path->nodes[level]; 12927bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 1293e66f709bSChris Mason 1294a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 12955f39d397SChris Mason parent = path->nodes[level + 1]; 1296e66f709bSChris Mason pslot = path->slots[level + 1]; 1297a05a9bb1SLi Zefan } 1298e66f709bSChris Mason 12995f39d397SChris Mason if (!parent) 1300e66f709bSChris Mason return 1; 1301e66f709bSChris Mason 13029cf14029SJosef Bacik /* first, try to make some room in the middle buffer */ 13039cf14029SJosef Bacik if (pslot) { 13049cf14029SJosef Bacik u32 left_nr; 13059cf14029SJosef Bacik 13064b231ae4SDavid Sterba left = btrfs_read_node_slot(parent, pslot - 1); 1307fb770ae4SLiu Bo if (IS_ERR(left)) 13089cf14029SJosef Bacik return PTR_ERR(left); 1309925baeddSChris Mason 1310bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT); 1311b4ce94deSChris Mason 13125f39d397SChris Mason left_nr = btrfs_header_nritems(left); 13130b246afaSJeff Mahoney if (left_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) { 131433ade1f8SChris Mason wret = 1; 131533ade1f8SChris Mason } else { 13165f39d397SChris Mason ret = btrfs_cow_block(trans, root, left, parent, 13179631e4ccSJosef Bacik pslot - 1, &left, 1318bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW); 131954aa1f4dSChris Mason if (ret) 132054aa1f4dSChris Mason wret = 1; 132154aa1f4dSChris Mason else { 1322d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 0); 132354aa1f4dSChris Mason } 132433ade1f8SChris Mason } 1325e66f709bSChris Mason if (wret < 0) 1326e66f709bSChris Mason ret = wret; 1327e66f709bSChris Mason if (wret == 0) { 13285f39d397SChris Mason struct btrfs_disk_key disk_key; 1329e66f709bSChris Mason orig_slot += left_nr; 13305f39d397SChris Mason btrfs_node_key(mid, &disk_key, 0); 1331f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot, 133233cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 133311d6ae03SFilipe Manana if (ret < 0) { 133411d6ae03SFilipe Manana btrfs_tree_unlock(left); 133511d6ae03SFilipe Manana free_extent_buffer(left); 133611d6ae03SFilipe Manana btrfs_abort_transaction(trans, ret); 133711d6ae03SFilipe Manana return ret; 133811d6ae03SFilipe Manana } 13395f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot); 13405f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 13415f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 13425f39d397SChris Mason path->nodes[level] = left; 1343e66f709bSChris Mason path->slots[level + 1] -= 1; 1344e66f709bSChris Mason path->slots[level] = orig_slot; 1345925baeddSChris Mason btrfs_tree_unlock(mid); 13465f39d397SChris Mason free_extent_buffer(mid); 1347e66f709bSChris Mason } else { 1348e66f709bSChris Mason orig_slot -= 13495f39d397SChris Mason btrfs_header_nritems(left); 1350e66f709bSChris Mason path->slots[level] = orig_slot; 1351925baeddSChris Mason btrfs_tree_unlock(left); 13525f39d397SChris Mason free_extent_buffer(left); 1353e66f709bSChris Mason } 1354e66f709bSChris Mason return 0; 1355e66f709bSChris Mason } 1356925baeddSChris Mason btrfs_tree_unlock(left); 13575f39d397SChris Mason free_extent_buffer(left); 1358e66f709bSChris Mason } 1359e66f709bSChris Mason 1360e66f709bSChris Mason /* 1361e66f709bSChris Mason * then try to empty the right most buffer into the middle 1362e66f709bSChris Mason */ 13639cf14029SJosef Bacik if (pslot + 1 < btrfs_header_nritems(parent)) { 136433ade1f8SChris Mason u32 right_nr; 1365b4ce94deSChris Mason 13669cf14029SJosef Bacik right = btrfs_read_node_slot(parent, pslot + 1); 13679cf14029SJosef Bacik if (IS_ERR(right)) 13689cf14029SJosef Bacik return PTR_ERR(right); 13699cf14029SJosef Bacik 1370bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT); 1371b4ce94deSChris Mason 13725f39d397SChris Mason right_nr = btrfs_header_nritems(right); 13730b246afaSJeff Mahoney if (right_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) { 137433ade1f8SChris Mason wret = 1; 137533ade1f8SChris Mason } else { 13765f39d397SChris Mason ret = btrfs_cow_block(trans, root, right, 13775f39d397SChris Mason parent, pslot + 1, 1378bf59a5a2SJosef Bacik &right, BTRFS_NESTING_RIGHT_COW); 137954aa1f4dSChris Mason if (ret) 138054aa1f4dSChris Mason wret = 1; 138154aa1f4dSChris Mason else { 138255d32ed8SDavid Sterba wret = balance_node_right(trans, right, mid); 138333ade1f8SChris Mason } 138454aa1f4dSChris Mason } 1385e66f709bSChris Mason if (wret < 0) 1386e66f709bSChris Mason ret = wret; 1387e66f709bSChris Mason if (wret == 0) { 13885f39d397SChris Mason struct btrfs_disk_key disk_key; 13895f39d397SChris Mason 13905f39d397SChris Mason btrfs_node_key(right, &disk_key, 0); 1391f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot + 1, 139233cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 139311d6ae03SFilipe Manana if (ret < 0) { 139411d6ae03SFilipe Manana btrfs_tree_unlock(right); 139511d6ae03SFilipe Manana free_extent_buffer(right); 139611d6ae03SFilipe Manana btrfs_abort_transaction(trans, ret); 139711d6ae03SFilipe Manana return ret; 139811d6ae03SFilipe Manana } 13995f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot + 1); 14005f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 14015f39d397SChris Mason 14025f39d397SChris Mason if (btrfs_header_nritems(mid) <= orig_slot) { 14035f39d397SChris Mason path->nodes[level] = right; 1404e66f709bSChris Mason path->slots[level + 1] += 1; 1405e66f709bSChris Mason path->slots[level] = orig_slot - 14065f39d397SChris Mason btrfs_header_nritems(mid); 1407925baeddSChris Mason btrfs_tree_unlock(mid); 14085f39d397SChris Mason free_extent_buffer(mid); 1409e66f709bSChris Mason } else { 1410925baeddSChris Mason btrfs_tree_unlock(right); 14115f39d397SChris Mason free_extent_buffer(right); 1412e66f709bSChris Mason } 1413e66f709bSChris Mason return 0; 1414e66f709bSChris Mason } 1415925baeddSChris Mason btrfs_tree_unlock(right); 14165f39d397SChris Mason free_extent_buffer(right); 1417e66f709bSChris Mason } 1418e66f709bSChris Mason return 1; 1419e66f709bSChris Mason } 1420e66f709bSChris Mason 142174123bd7SChris Mason /* 1422d352ac68SChris Mason * readahead one full node of leaves, finding things that are close 1423d352ac68SChris Mason * to the block in 'slot', and triggering ra on them. 14243c69faecSChris Mason */ 14252ff7e61eSJeff Mahoney static void reada_for_search(struct btrfs_fs_info *fs_info, 1426e02119d5SChris Mason struct btrfs_path *path, 142701f46658SChris Mason int level, int slot, u64 objectid) 14283c69faecSChris Mason { 14295f39d397SChris Mason struct extent_buffer *node; 143001f46658SChris Mason struct btrfs_disk_key disk_key; 14313c69faecSChris Mason u32 nritems; 14323c69faecSChris Mason u64 search; 1433a7175319SChris Mason u64 target; 14346b80053dSChris Mason u64 nread = 0; 1435ace75066SFilipe Manana u64 nread_max; 14366b80053dSChris Mason u32 nr; 14376b80053dSChris Mason u32 blocksize; 14386b80053dSChris Mason u32 nscan = 0; 1439db94535dSChris Mason 1440ace75066SFilipe Manana if (level != 1 && path->reada != READA_FORWARD_ALWAYS) 14413c69faecSChris Mason return; 14423c69faecSChris Mason 14436702ed49SChris Mason if (!path->nodes[level]) 14446702ed49SChris Mason return; 14456702ed49SChris Mason 14465f39d397SChris Mason node = path->nodes[level]; 1447925baeddSChris Mason 1448ace75066SFilipe Manana /* 1449ace75066SFilipe Manana * Since the time between visiting leaves is much shorter than the time 1450ace75066SFilipe Manana * between visiting nodes, limit read ahead of nodes to 1, to avoid too 1451ace75066SFilipe Manana * much IO at once (possibly random). 1452ace75066SFilipe Manana */ 1453ace75066SFilipe Manana if (path->reada == READA_FORWARD_ALWAYS) { 1454ace75066SFilipe Manana if (level > 1) 1455ace75066SFilipe Manana nread_max = node->fs_info->nodesize; 1456ace75066SFilipe Manana else 1457ace75066SFilipe Manana nread_max = SZ_128K; 1458ace75066SFilipe Manana } else { 1459ace75066SFilipe Manana nread_max = SZ_64K; 1460ace75066SFilipe Manana } 1461ace75066SFilipe Manana 14623c69faecSChris Mason search = btrfs_node_blockptr(node, slot); 14630b246afaSJeff Mahoney blocksize = fs_info->nodesize; 1464069a2e37SFilipe Manana if (path->reada != READA_FORWARD_ALWAYS) { 1465069a2e37SFilipe Manana struct extent_buffer *eb; 1466069a2e37SFilipe Manana 14670b246afaSJeff Mahoney eb = find_extent_buffer(fs_info, search); 14685f39d397SChris Mason if (eb) { 14695f39d397SChris Mason free_extent_buffer(eb); 14703c69faecSChris Mason return; 14713c69faecSChris Mason } 1472069a2e37SFilipe Manana } 14733c69faecSChris Mason 1474a7175319SChris Mason target = search; 14756b80053dSChris Mason 14765f39d397SChris Mason nritems = btrfs_header_nritems(node); 14776b80053dSChris Mason nr = slot; 147825b8b936SJosef Bacik 14793c69faecSChris Mason while (1) { 1480e4058b54SDavid Sterba if (path->reada == READA_BACK) { 14816b80053dSChris Mason if (nr == 0) 14823c69faecSChris Mason break; 14836b80053dSChris Mason nr--; 1484ace75066SFilipe Manana } else if (path->reada == READA_FORWARD || 1485ace75066SFilipe Manana path->reada == READA_FORWARD_ALWAYS) { 14866b80053dSChris Mason nr++; 14876b80053dSChris Mason if (nr >= nritems) 14886b80053dSChris Mason break; 14893c69faecSChris Mason } 1490e4058b54SDavid Sterba if (path->reada == READA_BACK && objectid) { 149101f46658SChris Mason btrfs_node_key(node, &disk_key, nr); 149201f46658SChris Mason if (btrfs_disk_key_objectid(&disk_key) != objectid) 149301f46658SChris Mason break; 149401f46658SChris Mason } 14956b80053dSChris Mason search = btrfs_node_blockptr(node, nr); 1496ace75066SFilipe Manana if (path->reada == READA_FORWARD_ALWAYS || 1497ace75066SFilipe Manana (search <= target && target - search <= 65536) || 1498a7175319SChris Mason (search > target && search - target <= 65536)) { 1499bfb484d9SJosef Bacik btrfs_readahead_node_child(node, nr); 15006b80053dSChris Mason nread += blocksize; 15013c69faecSChris Mason } 15026b80053dSChris Mason nscan++; 1503ace75066SFilipe Manana if (nread > nread_max || nscan > 32) 15046b80053dSChris Mason break; 15053c69faecSChris Mason } 15063c69faecSChris Mason } 1507925baeddSChris Mason 1508bfb484d9SJosef Bacik static noinline void reada_for_balance(struct btrfs_path *path, int level) 1509b4ce94deSChris Mason { 1510bfb484d9SJosef Bacik struct extent_buffer *parent; 1511b4ce94deSChris Mason int slot; 1512b4ce94deSChris Mason int nritems; 1513b4ce94deSChris Mason 15148c594ea8SChris Mason parent = path->nodes[level + 1]; 1515b4ce94deSChris Mason if (!parent) 15160b08851fSJosef Bacik return; 1517b4ce94deSChris Mason 1518b4ce94deSChris Mason nritems = btrfs_header_nritems(parent); 15198c594ea8SChris Mason slot = path->slots[level + 1]; 1520b4ce94deSChris Mason 1521bfb484d9SJosef Bacik if (slot > 0) 1522bfb484d9SJosef Bacik btrfs_readahead_node_child(parent, slot - 1); 1523bfb484d9SJosef Bacik if (slot + 1 < nritems) 1524bfb484d9SJosef Bacik btrfs_readahead_node_child(parent, slot + 1); 1525b4ce94deSChris Mason } 1526b4ce94deSChris Mason 1527b4ce94deSChris Mason 1528b4ce94deSChris Mason /* 1529d397712bSChris Mason * when we walk down the tree, it is usually safe to unlock the higher layers 1530d397712bSChris Mason * in the tree. The exceptions are when our path goes through slot 0, because 1531d397712bSChris Mason * operations on the tree might require changing key pointers higher up in the 1532d397712bSChris Mason * tree. 1533d352ac68SChris Mason * 1534d397712bSChris Mason * callers might also have set path->keep_locks, which tells this code to keep 1535d397712bSChris Mason * the lock if the path points to the last slot in the block. This is part of 1536d397712bSChris Mason * walking through the tree, and selecting the next slot in the higher block. 1537d352ac68SChris Mason * 1538d397712bSChris Mason * lowest_unlock sets the lowest level in the tree we're allowed to unlock. so 1539d397712bSChris Mason * if lowest_unlock is 1, level 0 won't be unlocked 1540d352ac68SChris Mason */ 1541e02119d5SChris Mason static noinline void unlock_up(struct btrfs_path *path, int level, 1542f7c79f30SChris Mason int lowest_unlock, int min_write_lock_level, 1543f7c79f30SChris Mason int *write_lock_level) 1544925baeddSChris Mason { 1545925baeddSChris Mason int i; 1546925baeddSChris Mason int skip_level = level; 1547c1227996SNikolay Borisov bool check_skip = true; 1548925baeddSChris Mason 1549925baeddSChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 1550925baeddSChris Mason if (!path->nodes[i]) 1551925baeddSChris Mason break; 1552925baeddSChris Mason if (!path->locks[i]) 1553925baeddSChris Mason break; 1554c1227996SNikolay Borisov 1555c1227996SNikolay Borisov if (check_skip) { 1556c1227996SNikolay Borisov if (path->slots[i] == 0) { 1557925baeddSChris Mason skip_level = i + 1; 1558925baeddSChris Mason continue; 1559925baeddSChris Mason } 1560c1227996SNikolay Borisov 1561c1227996SNikolay Borisov if (path->keep_locks) { 1562925baeddSChris Mason u32 nritems; 1563c1227996SNikolay Borisov 1564c1227996SNikolay Borisov nritems = btrfs_header_nritems(path->nodes[i]); 1565051e1b9fSChris Mason if (nritems < 1 || path->slots[i] >= nritems - 1) { 1566925baeddSChris Mason skip_level = i + 1; 1567925baeddSChris Mason continue; 1568925baeddSChris Mason } 1569925baeddSChris Mason } 1570c1227996SNikolay Borisov } 1571051e1b9fSChris Mason 1572d80bb3f9SLiu Bo if (i >= lowest_unlock && i > skip_level) { 1573c1227996SNikolay Borisov check_skip = false; 1574c1227996SNikolay Borisov btrfs_tree_unlock_rw(path->nodes[i], path->locks[i]); 1575925baeddSChris Mason path->locks[i] = 0; 1576f7c79f30SChris Mason if (write_lock_level && 1577f7c79f30SChris Mason i > min_write_lock_level && 1578f7c79f30SChris Mason i <= *write_lock_level) { 1579f7c79f30SChris Mason *write_lock_level = i - 1; 1580f7c79f30SChris Mason } 1581925baeddSChris Mason } 1582925baeddSChris Mason } 1583925baeddSChris Mason } 1584925baeddSChris Mason 15853c69faecSChris Mason /* 1586376a21d7SFilipe Manana * Helper function for btrfs_search_slot() and other functions that do a search 1587376a21d7SFilipe Manana * on a btree. The goal is to find a tree block in the cache (the radix tree at 1588376a21d7SFilipe Manana * fs_info->buffer_radix), but if we can't find it, or it's not up to date, read 1589376a21d7SFilipe Manana * its pages from disk. 1590c8c42864SChris Mason * 1591376a21d7SFilipe Manana * Returns -EAGAIN, with the path unlocked, if the caller needs to repeat the 1592376a21d7SFilipe Manana * whole btree search, starting again from the current root node. 1593c8c42864SChris Mason */ 1594c8c42864SChris Mason static int 1595d07b8528SLiu Bo read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, 1596c8c42864SChris Mason struct extent_buffer **eb_ret, int level, int slot, 1597cda79c54SDavid Sterba const struct btrfs_key *key) 1598c8c42864SChris Mason { 15990b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 1600789d6a3aSQu Wenruo struct btrfs_tree_parent_check check = { 0 }; 1601c8c42864SChris Mason u64 blocknr; 1602c8c42864SChris Mason u64 gen; 1603c8c42864SChris Mason struct extent_buffer *tmp; 160476a05b35SChris Mason int ret; 1605581c1760SQu Wenruo int parent_level; 1606b246666eSFilipe Manana bool unlock_up; 1607c8c42864SChris Mason 1608b246666eSFilipe Manana unlock_up = ((level + 1 < BTRFS_MAX_LEVEL) && p->locks[level + 1]); 1609213ff4b7SNikolay Borisov blocknr = btrfs_node_blockptr(*eb_ret, slot); 1610213ff4b7SNikolay Borisov gen = btrfs_node_ptr_generation(*eb_ret, slot); 1611213ff4b7SNikolay Borisov parent_level = btrfs_header_level(*eb_ret); 1612789d6a3aSQu Wenruo btrfs_node_key_to_cpu(*eb_ret, &check.first_key, slot); 1613789d6a3aSQu Wenruo check.has_first_key = true; 1614789d6a3aSQu Wenruo check.level = parent_level - 1; 1615789d6a3aSQu Wenruo check.transid = gen; 1616789d6a3aSQu Wenruo check.owner_root = root->root_key.objectid; 1617c8c42864SChris Mason 1618b246666eSFilipe Manana /* 1619b246666eSFilipe Manana * If we need to read an extent buffer from disk and we are holding locks 1620b246666eSFilipe Manana * on upper level nodes, we unlock all the upper nodes before reading the 1621b246666eSFilipe Manana * extent buffer, and then return -EAGAIN to the caller as it needs to 1622b246666eSFilipe Manana * restart the search. We don't release the lock on the current level 1623b246666eSFilipe Manana * because we need to walk this node to figure out which blocks to read. 1624b246666eSFilipe Manana */ 16250b246afaSJeff Mahoney tmp = find_extent_buffer(fs_info, blocknr); 1626cb44921aSChris Mason if (tmp) { 1627ace75066SFilipe Manana if (p->reada == READA_FORWARD_ALWAYS) 1628ace75066SFilipe Manana reada_for_search(fs_info, p, level, slot, key->objectid); 1629ace75066SFilipe Manana 1630b9fab919SChris Mason /* first we do an atomic uptodate check */ 1631b9fab919SChris Mason if (btrfs_buffer_uptodate(tmp, gen, 1) > 0) { 1632448de471SQu Wenruo /* 1633448de471SQu Wenruo * Do extra check for first_key, eb can be stale due to 1634448de471SQu Wenruo * being cached, read from scrub, or have multiple 1635448de471SQu Wenruo * parents (shared tree blocks). 1636448de471SQu Wenruo */ 1637e064d5e9SDavid Sterba if (btrfs_verify_level_key(tmp, 1638789d6a3aSQu Wenruo parent_level - 1, &check.first_key, gen)) { 1639448de471SQu Wenruo free_extent_buffer(tmp); 1640448de471SQu Wenruo return -EUCLEAN; 1641448de471SQu Wenruo } 1642c8c42864SChris Mason *eb_ret = tmp; 1643c8c42864SChris Mason return 0; 1644c8c42864SChris Mason } 1645bdf7c00eSJosef Bacik 1646857bc13fSJosef Bacik if (p->nowait) { 1647857bc13fSJosef Bacik free_extent_buffer(tmp); 1648857bc13fSJosef Bacik return -EAGAIN; 1649857bc13fSJosef Bacik } 1650857bc13fSJosef Bacik 1651b246666eSFilipe Manana if (unlock_up) 1652b246666eSFilipe Manana btrfs_unlock_up_safe(p, level + 1); 1653b246666eSFilipe Manana 1654b9fab919SChris Mason /* now we're allowed to do a blocking uptodate check */ 1655789d6a3aSQu Wenruo ret = btrfs_read_extent_buffer(tmp, &check); 16569a4ffa1bSQu Wenruo if (ret) { 1657cb44921aSChris Mason free_extent_buffer(tmp); 1658b3b4aa74SDavid Sterba btrfs_release_path(p); 1659cb44921aSChris Mason return -EIO; 1660cb44921aSChris Mason } 166188c602abSQu Wenruo if (btrfs_check_eb_owner(tmp, root->root_key.objectid)) { 166288c602abSQu Wenruo free_extent_buffer(tmp); 166388c602abSQu Wenruo btrfs_release_path(p); 166488c602abSQu Wenruo return -EUCLEAN; 166588c602abSQu Wenruo } 1666b246666eSFilipe Manana 1667b246666eSFilipe Manana if (unlock_up) 1668b246666eSFilipe Manana ret = -EAGAIN; 1669b246666eSFilipe Manana 1670b246666eSFilipe Manana goto out; 1671857bc13fSJosef Bacik } else if (p->nowait) { 1672857bc13fSJosef Bacik return -EAGAIN; 16739a4ffa1bSQu Wenruo } 1674c8c42864SChris Mason 1675b246666eSFilipe Manana if (unlock_up) { 16768c594ea8SChris Mason btrfs_unlock_up_safe(p, level + 1); 16774bb59055SFilipe Manana ret = -EAGAIN; 16784bb59055SFilipe Manana } else { 16794bb59055SFilipe Manana ret = 0; 16804bb59055SFilipe Manana } 16818c594ea8SChris Mason 1682e4058b54SDavid Sterba if (p->reada != READA_NONE) 16832ff7e61eSJeff Mahoney reada_for_search(fs_info, p, level, slot, key->objectid); 1684c8c42864SChris Mason 1685789d6a3aSQu Wenruo tmp = read_tree_block(fs_info, blocknr, &check); 16864eb150d6SQu Wenruo if (IS_ERR(tmp)) { 16874eb150d6SQu Wenruo btrfs_release_path(p); 16884eb150d6SQu Wenruo return PTR_ERR(tmp); 16894eb150d6SQu Wenruo } 169076a05b35SChris Mason /* 169176a05b35SChris Mason * If the read above didn't mark this buffer up to date, 169276a05b35SChris Mason * it will never end up being up to date. Set ret to EIO now 169376a05b35SChris Mason * and give up so that our caller doesn't loop forever 169476a05b35SChris Mason * on our EAGAINs. 169576a05b35SChris Mason */ 1696e6a1d6fdSLiu Bo if (!extent_buffer_uptodate(tmp)) 169776a05b35SChris Mason ret = -EIO; 169802a3307aSLiu Bo 1699b246666eSFilipe Manana out: 17004bb59055SFilipe Manana if (ret == 0) { 17014bb59055SFilipe Manana *eb_ret = tmp; 17024bb59055SFilipe Manana } else { 17034bb59055SFilipe Manana free_extent_buffer(tmp); 170402a3307aSLiu Bo btrfs_release_path(p); 17054bb59055SFilipe Manana } 17064bb59055SFilipe Manana 170776a05b35SChris Mason return ret; 1708c8c42864SChris Mason } 1709c8c42864SChris Mason 1710c8c42864SChris Mason /* 1711c8c42864SChris Mason * helper function for btrfs_search_slot. This does all of the checks 1712c8c42864SChris Mason * for node-level blocks and does any balancing required based on 1713c8c42864SChris Mason * the ins_len. 1714c8c42864SChris Mason * 1715c8c42864SChris Mason * If no extra work was required, zero is returned. If we had to 1716c8c42864SChris Mason * drop the path, -EAGAIN is returned and btrfs_search_slot must 1717c8c42864SChris Mason * start over 1718c8c42864SChris Mason */ 1719c8c42864SChris Mason static int 1720c8c42864SChris Mason setup_nodes_for_search(struct btrfs_trans_handle *trans, 1721c8c42864SChris Mason struct btrfs_root *root, struct btrfs_path *p, 1722bd681513SChris Mason struct extent_buffer *b, int level, int ins_len, 1723bd681513SChris Mason int *write_lock_level) 1724c8c42864SChris Mason { 17250b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 172695b982deSNikolay Borisov int ret = 0; 17270b246afaSJeff Mahoney 1728c8c42864SChris Mason if ((p->search_for_split || ins_len > 0) && btrfs_header_nritems(b) >= 17290b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3) { 1730c8c42864SChris Mason 1731bd681513SChris Mason if (*write_lock_level < level + 1) { 1732bd681513SChris Mason *write_lock_level = level + 1; 1733bd681513SChris Mason btrfs_release_path(p); 173495b982deSNikolay Borisov return -EAGAIN; 1735bd681513SChris Mason } 1736bd681513SChris Mason 1737bfb484d9SJosef Bacik reada_for_balance(p, level); 173895b982deSNikolay Borisov ret = split_node(trans, root, p, level); 1739c8c42864SChris Mason 1740c8c42864SChris Mason b = p->nodes[level]; 1741c8c42864SChris Mason } else if (ins_len < 0 && btrfs_header_nritems(b) < 17420b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 2) { 1743c8c42864SChris Mason 1744bd681513SChris Mason if (*write_lock_level < level + 1) { 1745bd681513SChris Mason *write_lock_level = level + 1; 1746bd681513SChris Mason btrfs_release_path(p); 174795b982deSNikolay Borisov return -EAGAIN; 1748bd681513SChris Mason } 1749bd681513SChris Mason 1750bfb484d9SJosef Bacik reada_for_balance(p, level); 175195b982deSNikolay Borisov ret = balance_level(trans, root, p, level); 175295b982deSNikolay Borisov if (ret) 175395b982deSNikolay Borisov return ret; 1754c8c42864SChris Mason 1755c8c42864SChris Mason b = p->nodes[level]; 1756c8c42864SChris Mason if (!b) { 1757b3b4aa74SDavid Sterba btrfs_release_path(p); 175895b982deSNikolay Borisov return -EAGAIN; 1759c8c42864SChris Mason } 1760c8c42864SChris Mason BUG_ON(btrfs_header_nritems(b) == 1); 1761c8c42864SChris Mason } 1762c8c42864SChris Mason return ret; 1763c8c42864SChris Mason } 1764c8c42864SChris Mason 1765381cf658SDavid Sterba int btrfs_find_item(struct btrfs_root *fs_root, struct btrfs_path *path, 1766e33d5c3dSKelley Nielsen u64 iobjectid, u64 ioff, u8 key_type, 1767e33d5c3dSKelley Nielsen struct btrfs_key *found_key) 1768e33d5c3dSKelley Nielsen { 1769e33d5c3dSKelley Nielsen int ret; 1770e33d5c3dSKelley Nielsen struct btrfs_key key; 1771e33d5c3dSKelley Nielsen struct extent_buffer *eb; 1772381cf658SDavid Sterba 1773381cf658SDavid Sterba ASSERT(path); 17741d4c08e0SDavid Sterba ASSERT(found_key); 1775e33d5c3dSKelley Nielsen 1776e33d5c3dSKelley Nielsen key.type = key_type; 1777e33d5c3dSKelley Nielsen key.objectid = iobjectid; 1778e33d5c3dSKelley Nielsen key.offset = ioff; 1779e33d5c3dSKelley Nielsen 1780e33d5c3dSKelley Nielsen ret = btrfs_search_slot(NULL, fs_root, &key, path, 0, 0); 17811d4c08e0SDavid Sterba if (ret < 0) 1782e33d5c3dSKelley Nielsen return ret; 1783e33d5c3dSKelley Nielsen 1784e33d5c3dSKelley Nielsen eb = path->nodes[0]; 1785e33d5c3dSKelley Nielsen if (ret && path->slots[0] >= btrfs_header_nritems(eb)) { 1786e33d5c3dSKelley Nielsen ret = btrfs_next_leaf(fs_root, path); 1787e33d5c3dSKelley Nielsen if (ret) 1788e33d5c3dSKelley Nielsen return ret; 1789e33d5c3dSKelley Nielsen eb = path->nodes[0]; 1790e33d5c3dSKelley Nielsen } 1791e33d5c3dSKelley Nielsen 1792e33d5c3dSKelley Nielsen btrfs_item_key_to_cpu(eb, found_key, path->slots[0]); 1793e33d5c3dSKelley Nielsen if (found_key->type != key.type || 1794e33d5c3dSKelley Nielsen found_key->objectid != key.objectid) 1795e33d5c3dSKelley Nielsen return 1; 1796e33d5c3dSKelley Nielsen 1797e33d5c3dSKelley Nielsen return 0; 1798e33d5c3dSKelley Nielsen } 1799e33d5c3dSKelley Nielsen 18001fc28d8eSLiu Bo static struct extent_buffer *btrfs_search_slot_get_root(struct btrfs_root *root, 18011fc28d8eSLiu Bo struct btrfs_path *p, 18021fc28d8eSLiu Bo int write_lock_level) 18031fc28d8eSLiu Bo { 18041fc28d8eSLiu Bo struct extent_buffer *b; 1805120de408SJosef Bacik int root_lock = 0; 18061fc28d8eSLiu Bo int level = 0; 18071fc28d8eSLiu Bo 18081fc28d8eSLiu Bo if (p->search_commit_root) { 18091fc28d8eSLiu Bo b = root->commit_root; 181067439dadSDavid Sterba atomic_inc(&b->refs); 18111fc28d8eSLiu Bo level = btrfs_header_level(b); 1812f9ddfd05SLiu Bo /* 1813f9ddfd05SLiu Bo * Ensure that all callers have set skip_locking when 1814f9ddfd05SLiu Bo * p->search_commit_root = 1. 1815f9ddfd05SLiu Bo */ 1816f9ddfd05SLiu Bo ASSERT(p->skip_locking == 1); 18171fc28d8eSLiu Bo 18181fc28d8eSLiu Bo goto out; 18191fc28d8eSLiu Bo } 18201fc28d8eSLiu Bo 18211fc28d8eSLiu Bo if (p->skip_locking) { 18221fc28d8eSLiu Bo b = btrfs_root_node(root); 18231fc28d8eSLiu Bo level = btrfs_header_level(b); 18241fc28d8eSLiu Bo goto out; 18251fc28d8eSLiu Bo } 18261fc28d8eSLiu Bo 1827120de408SJosef Bacik /* We try very hard to do read locks on the root */ 1828120de408SJosef Bacik root_lock = BTRFS_READ_LOCK; 1829120de408SJosef Bacik 18301fc28d8eSLiu Bo /* 1831662c653bSLiu Bo * If the level is set to maximum, we can skip trying to get the read 1832662c653bSLiu Bo * lock. 1833662c653bSLiu Bo */ 1834662c653bSLiu Bo if (write_lock_level < BTRFS_MAX_LEVEL) { 1835662c653bSLiu Bo /* 1836662c653bSLiu Bo * We don't know the level of the root node until we actually 1837662c653bSLiu Bo * have it read locked 18381fc28d8eSLiu Bo */ 1839857bc13fSJosef Bacik if (p->nowait) { 1840857bc13fSJosef Bacik b = btrfs_try_read_lock_root_node(root); 1841857bc13fSJosef Bacik if (IS_ERR(b)) 1842857bc13fSJosef Bacik return b; 1843857bc13fSJosef Bacik } else { 18441bb96598SJosef Bacik b = btrfs_read_lock_root_node(root); 1845857bc13fSJosef Bacik } 18461fc28d8eSLiu Bo level = btrfs_header_level(b); 18471fc28d8eSLiu Bo if (level > write_lock_level) 18481fc28d8eSLiu Bo goto out; 18491fc28d8eSLiu Bo 1850662c653bSLiu Bo /* Whoops, must trade for write lock */ 18511fc28d8eSLiu Bo btrfs_tree_read_unlock(b); 18521fc28d8eSLiu Bo free_extent_buffer(b); 1853662c653bSLiu Bo } 1854662c653bSLiu Bo 18551fc28d8eSLiu Bo b = btrfs_lock_root_node(root); 18561fc28d8eSLiu Bo root_lock = BTRFS_WRITE_LOCK; 18571fc28d8eSLiu Bo 18581fc28d8eSLiu Bo /* The level might have changed, check again */ 18591fc28d8eSLiu Bo level = btrfs_header_level(b); 18601fc28d8eSLiu Bo 18611fc28d8eSLiu Bo out: 1862120de408SJosef Bacik /* 1863120de408SJosef Bacik * The root may have failed to write out at some point, and thus is no 1864120de408SJosef Bacik * longer valid, return an error in this case. 1865120de408SJosef Bacik */ 1866120de408SJosef Bacik if (!extent_buffer_uptodate(b)) { 1867120de408SJosef Bacik if (root_lock) 1868120de408SJosef Bacik btrfs_tree_unlock_rw(b, root_lock); 1869120de408SJosef Bacik free_extent_buffer(b); 1870120de408SJosef Bacik return ERR_PTR(-EIO); 1871120de408SJosef Bacik } 1872120de408SJosef Bacik 18731fc28d8eSLiu Bo p->nodes[level] = b; 18741fc28d8eSLiu Bo if (!p->skip_locking) 18751fc28d8eSLiu Bo p->locks[level] = root_lock; 18761fc28d8eSLiu Bo /* 18771fc28d8eSLiu Bo * Callers are responsible for dropping b's references. 18781fc28d8eSLiu Bo */ 18791fc28d8eSLiu Bo return b; 18801fc28d8eSLiu Bo } 18811fc28d8eSLiu Bo 1882d96b3424SFilipe Manana /* 1883d96b3424SFilipe Manana * Replace the extent buffer at the lowest level of the path with a cloned 1884d96b3424SFilipe Manana * version. The purpose is to be able to use it safely, after releasing the 1885d96b3424SFilipe Manana * commit root semaphore, even if relocation is happening in parallel, the 1886d96b3424SFilipe Manana * transaction used for relocation is committed and the extent buffer is 1887d96b3424SFilipe Manana * reallocated in the next transaction. 1888d96b3424SFilipe Manana * 1889d96b3424SFilipe Manana * This is used in a context where the caller does not prevent transaction 1890d96b3424SFilipe Manana * commits from happening, either by holding a transaction handle or holding 1891d96b3424SFilipe Manana * some lock, while it's doing searches through a commit root. 1892d96b3424SFilipe Manana * At the moment it's only used for send operations. 1893d96b3424SFilipe Manana */ 1894d96b3424SFilipe Manana static int finish_need_commit_sem_search(struct btrfs_path *path) 1895d96b3424SFilipe Manana { 1896d96b3424SFilipe Manana const int i = path->lowest_level; 1897d96b3424SFilipe Manana const int slot = path->slots[i]; 1898d96b3424SFilipe Manana struct extent_buffer *lowest = path->nodes[i]; 1899d96b3424SFilipe Manana struct extent_buffer *clone; 1900d96b3424SFilipe Manana 1901d96b3424SFilipe Manana ASSERT(path->need_commit_sem); 1902d96b3424SFilipe Manana 1903d96b3424SFilipe Manana if (!lowest) 1904d96b3424SFilipe Manana return 0; 1905d96b3424SFilipe Manana 1906d96b3424SFilipe Manana lockdep_assert_held_read(&lowest->fs_info->commit_root_sem); 1907d96b3424SFilipe Manana 1908d96b3424SFilipe Manana clone = btrfs_clone_extent_buffer(lowest); 1909d96b3424SFilipe Manana if (!clone) 1910d96b3424SFilipe Manana return -ENOMEM; 1911d96b3424SFilipe Manana 1912d96b3424SFilipe Manana btrfs_release_path(path); 1913d96b3424SFilipe Manana path->nodes[i] = clone; 1914d96b3424SFilipe Manana path->slots[i] = slot; 1915d96b3424SFilipe Manana 1916d96b3424SFilipe Manana return 0; 1917d96b3424SFilipe Manana } 19181fc28d8eSLiu Bo 1919e2e58d0fSFilipe Manana static inline int search_for_key_slot(struct extent_buffer *eb, 1920e2e58d0fSFilipe Manana int search_low_slot, 1921e2e58d0fSFilipe Manana const struct btrfs_key *key, 1922e2e58d0fSFilipe Manana int prev_cmp, 1923e2e58d0fSFilipe Manana int *slot) 1924e2e58d0fSFilipe Manana { 1925e2e58d0fSFilipe Manana /* 1926e2e58d0fSFilipe Manana * If a previous call to btrfs_bin_search() on a parent node returned an 1927e2e58d0fSFilipe Manana * exact match (prev_cmp == 0), we can safely assume the target key will 1928e2e58d0fSFilipe Manana * always be at slot 0 on lower levels, since each key pointer 1929e2e58d0fSFilipe Manana * (struct btrfs_key_ptr) refers to the lowest key accessible from the 1930e2e58d0fSFilipe Manana * subtree it points to. Thus we can skip searching lower levels. 1931e2e58d0fSFilipe Manana */ 1932e2e58d0fSFilipe Manana if (prev_cmp == 0) { 1933e2e58d0fSFilipe Manana *slot = 0; 1934e2e58d0fSFilipe Manana return 0; 1935e2e58d0fSFilipe Manana } 1936e2e58d0fSFilipe Manana 1937fdf8d595SAnand Jain return btrfs_bin_search(eb, search_low_slot, key, slot); 1938e2e58d0fSFilipe Manana } 1939e2e58d0fSFilipe Manana 1940109324cfSFilipe Manana static int search_leaf(struct btrfs_trans_handle *trans, 1941109324cfSFilipe Manana struct btrfs_root *root, 1942109324cfSFilipe Manana const struct btrfs_key *key, 1943109324cfSFilipe Manana struct btrfs_path *path, 1944109324cfSFilipe Manana int ins_len, 1945109324cfSFilipe Manana int prev_cmp) 1946109324cfSFilipe Manana { 1947109324cfSFilipe Manana struct extent_buffer *leaf = path->nodes[0]; 1948109324cfSFilipe Manana int leaf_free_space = -1; 1949109324cfSFilipe Manana int search_low_slot = 0; 1950109324cfSFilipe Manana int ret; 1951109324cfSFilipe Manana bool do_bin_search = true; 1952109324cfSFilipe Manana 1953109324cfSFilipe Manana /* 1954109324cfSFilipe Manana * If we are doing an insertion, the leaf has enough free space and the 1955109324cfSFilipe Manana * destination slot for the key is not slot 0, then we can unlock our 1956109324cfSFilipe Manana * write lock on the parent, and any other upper nodes, before doing the 1957109324cfSFilipe Manana * binary search on the leaf (with search_for_key_slot()), allowing other 1958109324cfSFilipe Manana * tasks to lock the parent and any other upper nodes. 1959109324cfSFilipe Manana */ 1960109324cfSFilipe Manana if (ins_len > 0) { 1961109324cfSFilipe Manana /* 1962109324cfSFilipe Manana * Cache the leaf free space, since we will need it later and it 1963109324cfSFilipe Manana * will not change until then. 1964109324cfSFilipe Manana */ 1965109324cfSFilipe Manana leaf_free_space = btrfs_leaf_free_space(leaf); 1966109324cfSFilipe Manana 1967109324cfSFilipe Manana /* 1968109324cfSFilipe Manana * !path->locks[1] means we have a single node tree, the leaf is 1969109324cfSFilipe Manana * the root of the tree. 1970109324cfSFilipe Manana */ 1971109324cfSFilipe Manana if (path->locks[1] && leaf_free_space >= ins_len) { 1972109324cfSFilipe Manana struct btrfs_disk_key first_key; 1973109324cfSFilipe Manana 1974109324cfSFilipe Manana ASSERT(btrfs_header_nritems(leaf) > 0); 1975109324cfSFilipe Manana btrfs_item_key(leaf, &first_key, 0); 1976109324cfSFilipe Manana 1977109324cfSFilipe Manana /* 1978109324cfSFilipe Manana * Doing the extra comparison with the first key is cheap, 1979109324cfSFilipe Manana * taking into account that the first key is very likely 1980109324cfSFilipe Manana * already in a cache line because it immediately follows 1981109324cfSFilipe Manana * the extent buffer's header and we have recently accessed 1982109324cfSFilipe Manana * the header's level field. 1983109324cfSFilipe Manana */ 1984109324cfSFilipe Manana ret = comp_keys(&first_key, key); 1985109324cfSFilipe Manana if (ret < 0) { 1986109324cfSFilipe Manana /* 1987109324cfSFilipe Manana * The first key is smaller than the key we want 1988109324cfSFilipe Manana * to insert, so we are safe to unlock all upper 1989109324cfSFilipe Manana * nodes and we have to do the binary search. 1990109324cfSFilipe Manana * 1991109324cfSFilipe Manana * We do use btrfs_unlock_up_safe() and not 1992109324cfSFilipe Manana * unlock_up() because the later does not unlock 1993109324cfSFilipe Manana * nodes with a slot of 0 - we can safely unlock 1994109324cfSFilipe Manana * any node even if its slot is 0 since in this 1995109324cfSFilipe Manana * case the key does not end up at slot 0 of the 1996109324cfSFilipe Manana * leaf and there's no need to split the leaf. 1997109324cfSFilipe Manana */ 1998109324cfSFilipe Manana btrfs_unlock_up_safe(path, 1); 1999109324cfSFilipe Manana search_low_slot = 1; 2000109324cfSFilipe Manana } else { 2001109324cfSFilipe Manana /* 2002109324cfSFilipe Manana * The first key is >= then the key we want to 2003109324cfSFilipe Manana * insert, so we can skip the binary search as 2004109324cfSFilipe Manana * the target key will be at slot 0. 2005109324cfSFilipe Manana * 2006109324cfSFilipe Manana * We can not unlock upper nodes when the key is 2007109324cfSFilipe Manana * less than the first key, because we will need 2008109324cfSFilipe Manana * to update the key at slot 0 of the parent node 2009109324cfSFilipe Manana * and possibly of other upper nodes too. 2010109324cfSFilipe Manana * If the key matches the first key, then we can 2011109324cfSFilipe Manana * unlock all the upper nodes, using 2012109324cfSFilipe Manana * btrfs_unlock_up_safe() instead of unlock_up() 2013109324cfSFilipe Manana * as stated above. 2014109324cfSFilipe Manana */ 2015109324cfSFilipe Manana if (ret == 0) 2016109324cfSFilipe Manana btrfs_unlock_up_safe(path, 1); 2017109324cfSFilipe Manana /* 2018109324cfSFilipe Manana * ret is already 0 or 1, matching the result of 2019109324cfSFilipe Manana * a btrfs_bin_search() call, so there is no need 2020109324cfSFilipe Manana * to adjust it. 2021109324cfSFilipe Manana */ 2022109324cfSFilipe Manana do_bin_search = false; 2023109324cfSFilipe Manana path->slots[0] = 0; 2024109324cfSFilipe Manana } 2025109324cfSFilipe Manana } 2026109324cfSFilipe Manana } 2027109324cfSFilipe Manana 2028109324cfSFilipe Manana if (do_bin_search) { 2029109324cfSFilipe Manana ret = search_for_key_slot(leaf, search_low_slot, key, 2030109324cfSFilipe Manana prev_cmp, &path->slots[0]); 2031109324cfSFilipe Manana if (ret < 0) 2032109324cfSFilipe Manana return ret; 2033109324cfSFilipe Manana } 2034109324cfSFilipe Manana 2035109324cfSFilipe Manana if (ins_len > 0) { 2036109324cfSFilipe Manana /* 2037109324cfSFilipe Manana * Item key already exists. In this case, if we are allowed to 2038109324cfSFilipe Manana * insert the item (for example, in dir_item case, item key 2039109324cfSFilipe Manana * collision is allowed), it will be merged with the original 2040109324cfSFilipe Manana * item. Only the item size grows, no new btrfs item will be 2041109324cfSFilipe Manana * added. If search_for_extension is not set, ins_len already 2042109324cfSFilipe Manana * accounts the size btrfs_item, deduct it here so leaf space 2043109324cfSFilipe Manana * check will be correct. 2044109324cfSFilipe Manana */ 2045109324cfSFilipe Manana if (ret == 0 && !path->search_for_extension) { 2046109324cfSFilipe Manana ASSERT(ins_len >= sizeof(struct btrfs_item)); 2047109324cfSFilipe Manana ins_len -= sizeof(struct btrfs_item); 2048109324cfSFilipe Manana } 2049109324cfSFilipe Manana 2050109324cfSFilipe Manana ASSERT(leaf_free_space >= 0); 2051109324cfSFilipe Manana 2052109324cfSFilipe Manana if (leaf_free_space < ins_len) { 2053109324cfSFilipe Manana int err; 2054109324cfSFilipe Manana 2055109324cfSFilipe Manana err = split_leaf(trans, root, key, path, ins_len, 2056109324cfSFilipe Manana (ret == 0)); 2057bb8e9a60SFilipe Manana ASSERT(err <= 0); 2058bb8e9a60SFilipe Manana if (WARN_ON(err > 0)) 2059bb8e9a60SFilipe Manana err = -EUCLEAN; 2060109324cfSFilipe Manana if (err) 2061109324cfSFilipe Manana ret = err; 2062109324cfSFilipe Manana } 2063109324cfSFilipe Manana } 2064109324cfSFilipe Manana 2065109324cfSFilipe Manana return ret; 2066109324cfSFilipe Manana } 2067109324cfSFilipe Manana 2068c8c42864SChris Mason /* 20694271eceaSNikolay Borisov * btrfs_search_slot - look for a key in a tree and perform necessary 20704271eceaSNikolay Borisov * modifications to preserve tree invariants. 207174123bd7SChris Mason * 20724271eceaSNikolay Borisov * @trans: Handle of transaction, used when modifying the tree 20734271eceaSNikolay Borisov * @p: Holds all btree nodes along the search path 20744271eceaSNikolay Borisov * @root: The root node of the tree 20754271eceaSNikolay Borisov * @key: The key we are looking for 20769a664971Sethanwu * @ins_len: Indicates purpose of search: 20779a664971Sethanwu * >0 for inserts it's size of item inserted (*) 20789a664971Sethanwu * <0 for deletions 20799a664971Sethanwu * 0 for plain searches, not modifying the tree 20809a664971Sethanwu * 20819a664971Sethanwu * (*) If size of item inserted doesn't include 20829a664971Sethanwu * sizeof(struct btrfs_item), then p->search_for_extension must 20839a664971Sethanwu * be set. 20844271eceaSNikolay Borisov * @cow: boolean should CoW operations be performed. Must always be 1 20854271eceaSNikolay Borisov * when modifying the tree. 208697571fd0SChris Mason * 20874271eceaSNikolay Borisov * If @ins_len > 0, nodes and leaves will be split as we walk down the tree. 20884271eceaSNikolay Borisov * If @ins_len < 0, nodes will be merged as we walk down the tree (if possible) 20894271eceaSNikolay Borisov * 20904271eceaSNikolay Borisov * If @key is found, 0 is returned and you can find the item in the leaf level 20914271eceaSNikolay Borisov * of the path (level 0) 20924271eceaSNikolay Borisov * 20934271eceaSNikolay Borisov * If @key isn't found, 1 is returned and the leaf level of the path (level 0) 20944271eceaSNikolay Borisov * points to the slot where it should be inserted 20954271eceaSNikolay Borisov * 20964271eceaSNikolay Borisov * If an error is encountered while searching the tree a negative error number 20974271eceaSNikolay Borisov * is returned 209874123bd7SChris Mason */ 2099310712b2SOmar Sandoval int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root, 2100310712b2SOmar Sandoval const struct btrfs_key *key, struct btrfs_path *p, 2101310712b2SOmar Sandoval int ins_len, int cow) 2102be0e5c09SChris Mason { 2103d96b3424SFilipe Manana struct btrfs_fs_info *fs_info = root->fs_info; 21045f39d397SChris Mason struct extent_buffer *b; 2105be0e5c09SChris Mason int slot; 2106be0e5c09SChris Mason int ret; 210733c66f43SYan Zheng int err; 2108be0e5c09SChris Mason int level; 2109925baeddSChris Mason int lowest_unlock = 1; 2110bd681513SChris Mason /* everything at write_lock_level or lower must be write locked */ 2111bd681513SChris Mason int write_lock_level = 0; 21129f3a7427SChris Mason u8 lowest_level = 0; 2113f7c79f30SChris Mason int min_write_lock_level; 2114d7396f07SFilipe David Borba Manana int prev_cmp; 21159f3a7427SChris Mason 2116a4c853afSChenXiaoSong might_sleep(); 2117a4c853afSChenXiaoSong 21186702ed49SChris Mason lowest_level = p->lowest_level; 2119323ac95bSChris Mason WARN_ON(lowest_level && ins_len > 0); 212022b0ebdaSChris Mason WARN_ON(p->nodes[0] != NULL); 2121eb653de1SFilipe David Borba Manana BUG_ON(!cow && ins_len); 212225179201SJosef Bacik 2123857bc13fSJosef Bacik /* 2124857bc13fSJosef Bacik * For now only allow nowait for read only operations. There's no 2125857bc13fSJosef Bacik * strict reason why we can't, we just only need it for reads so it's 2126857bc13fSJosef Bacik * only implemented for reads. 2127857bc13fSJosef Bacik */ 2128857bc13fSJosef Bacik ASSERT(!p->nowait || !cow); 2129857bc13fSJosef Bacik 2130bd681513SChris Mason if (ins_len < 0) { 2131925baeddSChris Mason lowest_unlock = 2; 213265b51a00SChris Mason 2133bd681513SChris Mason /* when we are removing items, we might have to go up to level 2134bd681513SChris Mason * two as we update tree pointers Make sure we keep write 2135bd681513SChris Mason * for those levels as well 2136bd681513SChris Mason */ 2137bd681513SChris Mason write_lock_level = 2; 2138bd681513SChris Mason } else if (ins_len > 0) { 2139bd681513SChris Mason /* 2140bd681513SChris Mason * for inserting items, make sure we have a write lock on 2141bd681513SChris Mason * level 1 so we can update keys 2142bd681513SChris Mason */ 2143bd681513SChris Mason write_lock_level = 1; 2144bd681513SChris Mason } 2145bd681513SChris Mason 2146bd681513SChris Mason if (!cow) 2147bd681513SChris Mason write_lock_level = -1; 2148bd681513SChris Mason 214909a2a8f9SJosef Bacik if (cow && (p->keep_locks || p->lowest_level)) 2150bd681513SChris Mason write_lock_level = BTRFS_MAX_LEVEL; 2151bd681513SChris Mason 2152f7c79f30SChris Mason min_write_lock_level = write_lock_level; 2153f7c79f30SChris Mason 2154d96b3424SFilipe Manana if (p->need_commit_sem) { 2155d96b3424SFilipe Manana ASSERT(p->search_commit_root); 2156857bc13fSJosef Bacik if (p->nowait) { 2157857bc13fSJosef Bacik if (!down_read_trylock(&fs_info->commit_root_sem)) 2158857bc13fSJosef Bacik return -EAGAIN; 2159857bc13fSJosef Bacik } else { 2160d96b3424SFilipe Manana down_read(&fs_info->commit_root_sem); 2161d96b3424SFilipe Manana } 2162857bc13fSJosef Bacik } 2163d96b3424SFilipe Manana 2164bb803951SChris Mason again: 2165d7396f07SFilipe David Borba Manana prev_cmp = -1; 21661fc28d8eSLiu Bo b = btrfs_search_slot_get_root(root, p, write_lock_level); 2167be6821f8SFilipe Manana if (IS_ERR(b)) { 2168be6821f8SFilipe Manana ret = PTR_ERR(b); 2169be6821f8SFilipe Manana goto done; 2170be6821f8SFilipe Manana } 2171925baeddSChris Mason 2172eb60ceacSChris Mason while (b) { 2173f624d976SQu Wenruo int dec = 0; 2174f624d976SQu Wenruo 21755f39d397SChris Mason level = btrfs_header_level(b); 217665b51a00SChris Mason 217702217ed2SChris Mason if (cow) { 21789ea2c7c9SNikolay Borisov bool last_level = (level == (BTRFS_MAX_LEVEL - 1)); 21799ea2c7c9SNikolay Borisov 2180c8c42864SChris Mason /* 2181c8c42864SChris Mason * if we don't really need to cow this block 2182c8c42864SChris Mason * then we don't want to set the path blocking, 2183c8c42864SChris Mason * so we test it here 2184c8c42864SChris Mason */ 21855963ffcaSJosef Bacik if (!should_cow_block(trans, root, b)) 218665b51a00SChris Mason goto cow_done; 21875d4f98a2SYan Zheng 2188bd681513SChris Mason /* 2189bd681513SChris Mason * must have write locks on this node and the 2190bd681513SChris Mason * parent 2191bd681513SChris Mason */ 21925124e00eSJosef Bacik if (level > write_lock_level || 21935124e00eSJosef Bacik (level + 1 > write_lock_level && 21945124e00eSJosef Bacik level + 1 < BTRFS_MAX_LEVEL && 21955124e00eSJosef Bacik p->nodes[level + 1])) { 2196bd681513SChris Mason write_lock_level = level + 1; 2197bd681513SChris Mason btrfs_release_path(p); 2198bd681513SChris Mason goto again; 2199bd681513SChris Mason } 2200bd681513SChris Mason 22019ea2c7c9SNikolay Borisov if (last_level) 22029ea2c7c9SNikolay Borisov err = btrfs_cow_block(trans, root, b, NULL, 0, 22039631e4ccSJosef Bacik &b, 22049631e4ccSJosef Bacik BTRFS_NESTING_COW); 22059ea2c7c9SNikolay Borisov else 220633c66f43SYan Zheng err = btrfs_cow_block(trans, root, b, 2207e20d96d6SChris Mason p->nodes[level + 1], 22089631e4ccSJosef Bacik p->slots[level + 1], &b, 22099631e4ccSJosef Bacik BTRFS_NESTING_COW); 221033c66f43SYan Zheng if (err) { 221133c66f43SYan Zheng ret = err; 221265b51a00SChris Mason goto done; 221354aa1f4dSChris Mason } 221402217ed2SChris Mason } 221565b51a00SChris Mason cow_done: 2216eb60ceacSChris Mason p->nodes[level] = b; 2217b4ce94deSChris Mason 2218b4ce94deSChris Mason /* 2219b4ce94deSChris Mason * we have a lock on b and as long as we aren't changing 2220b4ce94deSChris Mason * the tree, there is no way to for the items in b to change. 2221b4ce94deSChris Mason * It is safe to drop the lock on our parent before we 2222b4ce94deSChris Mason * go through the expensive btree search on b. 2223b4ce94deSChris Mason * 2224eb653de1SFilipe David Borba Manana * If we're inserting or deleting (ins_len != 0), then we might 2225eb653de1SFilipe David Borba Manana * be changing slot zero, which may require changing the parent. 2226eb653de1SFilipe David Borba Manana * So, we can't drop the lock until after we know which slot 2227eb653de1SFilipe David Borba Manana * we're operating on. 2228b4ce94deSChris Mason */ 2229eb653de1SFilipe David Borba Manana if (!ins_len && !p->keep_locks) { 2230eb653de1SFilipe David Borba Manana int u = level + 1; 2231eb653de1SFilipe David Borba Manana 2232eb653de1SFilipe David Borba Manana if (u < BTRFS_MAX_LEVEL && p->locks[u]) { 2233eb653de1SFilipe David Borba Manana btrfs_tree_unlock_rw(p->nodes[u], p->locks[u]); 2234eb653de1SFilipe David Borba Manana p->locks[u] = 0; 2235eb653de1SFilipe David Borba Manana } 2236eb653de1SFilipe David Borba Manana } 2237b4ce94deSChris Mason 2238e2e58d0fSFilipe Manana if (level == 0) { 2239109324cfSFilipe Manana if (ins_len > 0) 2240e5e1c174SFilipe Manana ASSERT(write_lock_level >= 1); 2241bd681513SChris Mason 2242109324cfSFilipe Manana ret = search_leaf(trans, root, key, p, ins_len, prev_cmp); 2243459931ecSChris Mason if (!p->search_for_split) 2244f7c79f30SChris Mason unlock_up(p, level, lowest_unlock, 22454b6f8e96SLiu Bo min_write_lock_level, NULL); 224665b51a00SChris Mason goto done; 224765b51a00SChris Mason } 2248e2e58d0fSFilipe Manana 2249e2e58d0fSFilipe Manana ret = search_for_key_slot(b, 0, key, prev_cmp, &slot); 2250e2e58d0fSFilipe Manana if (ret < 0) 2251e2e58d0fSFilipe Manana goto done; 2252e2e58d0fSFilipe Manana prev_cmp = ret; 2253e2e58d0fSFilipe Manana 2254f624d976SQu Wenruo if (ret && slot > 0) { 2255f624d976SQu Wenruo dec = 1; 2256f624d976SQu Wenruo slot--; 2257f624d976SQu Wenruo } 2258f624d976SQu Wenruo p->slots[level] = slot; 2259f624d976SQu Wenruo err = setup_nodes_for_search(trans, root, p, b, level, ins_len, 2260f624d976SQu Wenruo &write_lock_level); 2261f624d976SQu Wenruo if (err == -EAGAIN) 2262f624d976SQu Wenruo goto again; 2263f624d976SQu Wenruo if (err) { 2264f624d976SQu Wenruo ret = err; 2265f624d976SQu Wenruo goto done; 2266f624d976SQu Wenruo } 2267f624d976SQu Wenruo b = p->nodes[level]; 2268f624d976SQu Wenruo slot = p->slots[level]; 2269f624d976SQu Wenruo 2270f624d976SQu Wenruo /* 2271f624d976SQu Wenruo * Slot 0 is special, if we change the key we have to update 2272f624d976SQu Wenruo * the parent pointer which means we must have a write lock on 2273f624d976SQu Wenruo * the parent 2274f624d976SQu Wenruo */ 2275f624d976SQu Wenruo if (slot == 0 && ins_len && write_lock_level < level + 1) { 2276f624d976SQu Wenruo write_lock_level = level + 1; 2277f624d976SQu Wenruo btrfs_release_path(p); 2278f624d976SQu Wenruo goto again; 2279f624d976SQu Wenruo } 2280f624d976SQu Wenruo 2281f624d976SQu Wenruo unlock_up(p, level, lowest_unlock, min_write_lock_level, 2282f624d976SQu Wenruo &write_lock_level); 2283f624d976SQu Wenruo 2284f624d976SQu Wenruo if (level == lowest_level) { 2285f624d976SQu Wenruo if (dec) 2286f624d976SQu Wenruo p->slots[level]++; 2287f624d976SQu Wenruo goto done; 2288f624d976SQu Wenruo } 2289f624d976SQu Wenruo 2290f624d976SQu Wenruo err = read_block_for_search(root, p, &b, level, slot, key); 2291f624d976SQu Wenruo if (err == -EAGAIN) 2292f624d976SQu Wenruo goto again; 2293f624d976SQu Wenruo if (err) { 2294f624d976SQu Wenruo ret = err; 2295f624d976SQu Wenruo goto done; 2296f624d976SQu Wenruo } 2297f624d976SQu Wenruo 2298f624d976SQu Wenruo if (!p->skip_locking) { 2299f624d976SQu Wenruo level = btrfs_header_level(b); 2300b40130b2SJosef Bacik 2301b40130b2SJosef Bacik btrfs_maybe_reset_lockdep_class(root, b); 2302b40130b2SJosef Bacik 2303f624d976SQu Wenruo if (level <= write_lock_level) { 2304f624d976SQu Wenruo btrfs_tree_lock(b); 2305f624d976SQu Wenruo p->locks[level] = BTRFS_WRITE_LOCK; 2306f624d976SQu Wenruo } else { 2307857bc13fSJosef Bacik if (p->nowait) { 2308857bc13fSJosef Bacik if (!btrfs_try_tree_read_lock(b)) { 2309857bc13fSJosef Bacik free_extent_buffer(b); 2310857bc13fSJosef Bacik ret = -EAGAIN; 2311857bc13fSJosef Bacik goto done; 2312857bc13fSJosef Bacik } 2313857bc13fSJosef Bacik } else { 2314fe596ca3SJosef Bacik btrfs_tree_read_lock(b); 2315857bc13fSJosef Bacik } 2316f624d976SQu Wenruo p->locks[level] = BTRFS_READ_LOCK; 2317f624d976SQu Wenruo } 2318f624d976SQu Wenruo p->nodes[level] = b; 2319f624d976SQu Wenruo } 232065b51a00SChris Mason } 232165b51a00SChris Mason ret = 1; 232265b51a00SChris Mason done: 23235f5bc6b1SFilipe Manana if (ret < 0 && !p->skip_release_on_error) 2324b3b4aa74SDavid Sterba btrfs_release_path(p); 2325d96b3424SFilipe Manana 2326d96b3424SFilipe Manana if (p->need_commit_sem) { 2327d96b3424SFilipe Manana int ret2; 2328d96b3424SFilipe Manana 2329d96b3424SFilipe Manana ret2 = finish_need_commit_sem_search(p); 2330d96b3424SFilipe Manana up_read(&fs_info->commit_root_sem); 2331d96b3424SFilipe Manana if (ret2) 2332d96b3424SFilipe Manana ret = ret2; 2333d96b3424SFilipe Manana } 2334d96b3424SFilipe Manana 2335be0e5c09SChris Mason return ret; 2336be0e5c09SChris Mason } 2337f75e2b79SJosef Bacik ALLOW_ERROR_INJECTION(btrfs_search_slot, ERRNO); 2338be0e5c09SChris Mason 233974123bd7SChris Mason /* 23405d9e75c4SJan Schmidt * Like btrfs_search_slot, this looks for a key in the given tree. It uses the 23415d9e75c4SJan Schmidt * current state of the tree together with the operations recorded in the tree 23425d9e75c4SJan Schmidt * modification log to search for the key in a previous version of this tree, as 23435d9e75c4SJan Schmidt * denoted by the time_seq parameter. 23445d9e75c4SJan Schmidt * 23455d9e75c4SJan Schmidt * Naturally, there is no support for insert, delete or cow operations. 23465d9e75c4SJan Schmidt * 23475d9e75c4SJan Schmidt * The resulting path and return value will be set up as if we called 23485d9e75c4SJan Schmidt * btrfs_search_slot at that point in time with ins_len and cow both set to 0. 23495d9e75c4SJan Schmidt */ 2350310712b2SOmar Sandoval int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key, 23515d9e75c4SJan Schmidt struct btrfs_path *p, u64 time_seq) 23525d9e75c4SJan Schmidt { 23530b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 23545d9e75c4SJan Schmidt struct extent_buffer *b; 23555d9e75c4SJan Schmidt int slot; 23565d9e75c4SJan Schmidt int ret; 23575d9e75c4SJan Schmidt int err; 23585d9e75c4SJan Schmidt int level; 23595d9e75c4SJan Schmidt int lowest_unlock = 1; 23605d9e75c4SJan Schmidt u8 lowest_level = 0; 23615d9e75c4SJan Schmidt 23625d9e75c4SJan Schmidt lowest_level = p->lowest_level; 23635d9e75c4SJan Schmidt WARN_ON(p->nodes[0] != NULL); 2364c922b016SStefan Roesch ASSERT(!p->nowait); 23655d9e75c4SJan Schmidt 23665d9e75c4SJan Schmidt if (p->search_commit_root) { 23675d9e75c4SJan Schmidt BUG_ON(time_seq); 23685d9e75c4SJan Schmidt return btrfs_search_slot(NULL, root, key, p, 0, 0); 23695d9e75c4SJan Schmidt } 23705d9e75c4SJan Schmidt 23715d9e75c4SJan Schmidt again: 2372f3a84ccdSFilipe Manana b = btrfs_get_old_root(root, time_seq); 2373315bed43SNikolay Borisov if (!b) { 2374315bed43SNikolay Borisov ret = -EIO; 2375315bed43SNikolay Borisov goto done; 2376315bed43SNikolay Borisov } 23775d9e75c4SJan Schmidt level = btrfs_header_level(b); 23785d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 23795d9e75c4SJan Schmidt 23805d9e75c4SJan Schmidt while (b) { 2381abe9339dSQu Wenruo int dec = 0; 2382abe9339dSQu Wenruo 23835d9e75c4SJan Schmidt level = btrfs_header_level(b); 23845d9e75c4SJan Schmidt p->nodes[level] = b; 23855d9e75c4SJan Schmidt 23865d9e75c4SJan Schmidt /* 23875d9e75c4SJan Schmidt * we have a lock on b and as long as we aren't changing 23885d9e75c4SJan Schmidt * the tree, there is no way to for the items in b to change. 23895d9e75c4SJan Schmidt * It is safe to drop the lock on our parent before we 23905d9e75c4SJan Schmidt * go through the expensive btree search on b. 23915d9e75c4SJan Schmidt */ 23925d9e75c4SJan Schmidt btrfs_unlock_up_safe(p, level + 1); 23935d9e75c4SJan Schmidt 2394fdf8d595SAnand Jain ret = btrfs_bin_search(b, 0, key, &slot); 2395cbca7d59SFilipe Manana if (ret < 0) 2396cbca7d59SFilipe Manana goto done; 23975d9e75c4SJan Schmidt 2398abe9339dSQu Wenruo if (level == 0) { 2399abe9339dSQu Wenruo p->slots[level] = slot; 2400abe9339dSQu Wenruo unlock_up(p, level, lowest_unlock, 0, NULL); 2401abe9339dSQu Wenruo goto done; 2402abe9339dSQu Wenruo } 2403abe9339dSQu Wenruo 24045d9e75c4SJan Schmidt if (ret && slot > 0) { 24055d9e75c4SJan Schmidt dec = 1; 2406abe9339dSQu Wenruo slot--; 24075d9e75c4SJan Schmidt } 24085d9e75c4SJan Schmidt p->slots[level] = slot; 24095d9e75c4SJan Schmidt unlock_up(p, level, lowest_unlock, 0, NULL); 24105d9e75c4SJan Schmidt 24115d9e75c4SJan Schmidt if (level == lowest_level) { 24125d9e75c4SJan Schmidt if (dec) 24135d9e75c4SJan Schmidt p->slots[level]++; 24145d9e75c4SJan Schmidt goto done; 24155d9e75c4SJan Schmidt } 24165d9e75c4SJan Schmidt 2417abe9339dSQu Wenruo err = read_block_for_search(root, p, &b, level, slot, key); 24185d9e75c4SJan Schmidt if (err == -EAGAIN) 24195d9e75c4SJan Schmidt goto again; 24205d9e75c4SJan Schmidt if (err) { 24215d9e75c4SJan Schmidt ret = err; 24225d9e75c4SJan Schmidt goto done; 24235d9e75c4SJan Schmidt } 24245d9e75c4SJan Schmidt 24255d9e75c4SJan Schmidt level = btrfs_header_level(b); 24265d9e75c4SJan Schmidt btrfs_tree_read_lock(b); 2427f3a84ccdSFilipe Manana b = btrfs_tree_mod_log_rewind(fs_info, p, b, time_seq); 2428db7f3436SJosef Bacik if (!b) { 2429db7f3436SJosef Bacik ret = -ENOMEM; 2430db7f3436SJosef Bacik goto done; 2431db7f3436SJosef Bacik } 24325d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 24335d9e75c4SJan Schmidt p->nodes[level] = b; 24345d9e75c4SJan Schmidt } 24355d9e75c4SJan Schmidt ret = 1; 24365d9e75c4SJan Schmidt done: 24375d9e75c4SJan Schmidt if (ret < 0) 24385d9e75c4SJan Schmidt btrfs_release_path(p); 24395d9e75c4SJan Schmidt 24405d9e75c4SJan Schmidt return ret; 24415d9e75c4SJan Schmidt } 24425d9e75c4SJan Schmidt 24435d9e75c4SJan Schmidt /* 2444f469c8bdSFilipe Manana * Search the tree again to find a leaf with smaller keys. 2445f469c8bdSFilipe Manana * Returns 0 if it found something. 2446f469c8bdSFilipe Manana * Returns 1 if there are no smaller keys. 2447f469c8bdSFilipe Manana * Returns < 0 on error. 2448f469c8bdSFilipe Manana * 2449f469c8bdSFilipe Manana * This may release the path, and so you may lose any locks held at the 2450f469c8bdSFilipe Manana * time you call it. 2451f469c8bdSFilipe Manana */ 2452f469c8bdSFilipe Manana static int btrfs_prev_leaf(struct btrfs_root *root, struct btrfs_path *path) 2453f469c8bdSFilipe Manana { 2454f469c8bdSFilipe Manana struct btrfs_key key; 2455f469c8bdSFilipe Manana struct btrfs_key orig_key; 2456f469c8bdSFilipe Manana struct btrfs_disk_key found_key; 2457f469c8bdSFilipe Manana int ret; 2458f469c8bdSFilipe Manana 2459f469c8bdSFilipe Manana btrfs_item_key_to_cpu(path->nodes[0], &key, 0); 2460f469c8bdSFilipe Manana orig_key = key; 2461f469c8bdSFilipe Manana 2462f469c8bdSFilipe Manana if (key.offset > 0) { 2463f469c8bdSFilipe Manana key.offset--; 2464f469c8bdSFilipe Manana } else if (key.type > 0) { 2465f469c8bdSFilipe Manana key.type--; 2466f469c8bdSFilipe Manana key.offset = (u64)-1; 2467f469c8bdSFilipe Manana } else if (key.objectid > 0) { 2468f469c8bdSFilipe Manana key.objectid--; 2469f469c8bdSFilipe Manana key.type = (u8)-1; 2470f469c8bdSFilipe Manana key.offset = (u64)-1; 2471f469c8bdSFilipe Manana } else { 2472f469c8bdSFilipe Manana return 1; 2473f469c8bdSFilipe Manana } 2474f469c8bdSFilipe Manana 2475f469c8bdSFilipe Manana btrfs_release_path(path); 2476f469c8bdSFilipe Manana ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 2477f469c8bdSFilipe Manana if (ret <= 0) 2478f469c8bdSFilipe Manana return ret; 2479f469c8bdSFilipe Manana 2480f469c8bdSFilipe Manana /* 2481f469c8bdSFilipe Manana * Previous key not found. Even if we were at slot 0 of the leaf we had 2482f469c8bdSFilipe Manana * before releasing the path and calling btrfs_search_slot(), we now may 2483f469c8bdSFilipe Manana * be in a slot pointing to the same original key - this can happen if 2484f469c8bdSFilipe Manana * after we released the path, one of more items were moved from a 2485f469c8bdSFilipe Manana * sibling leaf into the front of the leaf we had due to an insertion 2486f469c8bdSFilipe Manana * (see push_leaf_right()). 2487f469c8bdSFilipe Manana * If we hit this case and our slot is > 0 and just decrement the slot 2488f469c8bdSFilipe Manana * so that the caller does not process the same key again, which may or 2489f469c8bdSFilipe Manana * may not break the caller, depending on its logic. 2490f469c8bdSFilipe Manana */ 2491f469c8bdSFilipe Manana if (path->slots[0] < btrfs_header_nritems(path->nodes[0])) { 2492f469c8bdSFilipe Manana btrfs_item_key(path->nodes[0], &found_key, path->slots[0]); 2493f469c8bdSFilipe Manana ret = comp_keys(&found_key, &orig_key); 2494f469c8bdSFilipe Manana if (ret == 0) { 2495f469c8bdSFilipe Manana if (path->slots[0] > 0) { 2496f469c8bdSFilipe Manana path->slots[0]--; 2497f469c8bdSFilipe Manana return 0; 2498f469c8bdSFilipe Manana } 2499f469c8bdSFilipe Manana /* 2500f469c8bdSFilipe Manana * At slot 0, same key as before, it means orig_key is 2501f469c8bdSFilipe Manana * the lowest, leftmost, key in the tree. We're done. 2502f469c8bdSFilipe Manana */ 2503f469c8bdSFilipe Manana return 1; 2504f469c8bdSFilipe Manana } 2505f469c8bdSFilipe Manana } 2506f469c8bdSFilipe Manana 2507f469c8bdSFilipe Manana btrfs_item_key(path->nodes[0], &found_key, 0); 2508f469c8bdSFilipe Manana ret = comp_keys(&found_key, &key); 2509f469c8bdSFilipe Manana /* 2510f469c8bdSFilipe Manana * We might have had an item with the previous key in the tree right 2511f469c8bdSFilipe Manana * before we released our path. And after we released our path, that 2512f469c8bdSFilipe Manana * item might have been pushed to the first slot (0) of the leaf we 2513f469c8bdSFilipe Manana * were holding due to a tree balance. Alternatively, an item with the 2514f469c8bdSFilipe Manana * previous key can exist as the only element of a leaf (big fat item). 2515f469c8bdSFilipe Manana * Therefore account for these 2 cases, so that our callers (like 2516f469c8bdSFilipe Manana * btrfs_previous_item) don't miss an existing item with a key matching 2517f469c8bdSFilipe Manana * the previous key we computed above. 2518f469c8bdSFilipe Manana */ 2519f469c8bdSFilipe Manana if (ret <= 0) 2520f469c8bdSFilipe Manana return 0; 2521f469c8bdSFilipe Manana return 1; 2522f469c8bdSFilipe Manana } 2523f469c8bdSFilipe Manana 2524f469c8bdSFilipe Manana /* 25252f38b3e1SArne Jansen * helper to use instead of search slot if no exact match is needed but 25262f38b3e1SArne Jansen * instead the next or previous item should be returned. 25272f38b3e1SArne Jansen * When find_higher is true, the next higher item is returned, the next lower 25282f38b3e1SArne Jansen * otherwise. 25292f38b3e1SArne Jansen * When return_any and find_higher are both true, and no higher item is found, 25302f38b3e1SArne Jansen * return the next lower instead. 25312f38b3e1SArne Jansen * When return_any is true and find_higher is false, and no lower item is found, 25322f38b3e1SArne Jansen * return the next higher instead. 25332f38b3e1SArne Jansen * It returns 0 if any item is found, 1 if none is found (tree empty), and 25342f38b3e1SArne Jansen * < 0 on error 25352f38b3e1SArne Jansen */ 25362f38b3e1SArne Jansen int btrfs_search_slot_for_read(struct btrfs_root *root, 2537310712b2SOmar Sandoval const struct btrfs_key *key, 2538310712b2SOmar Sandoval struct btrfs_path *p, int find_higher, 2539310712b2SOmar Sandoval int return_any) 25402f38b3e1SArne Jansen { 25412f38b3e1SArne Jansen int ret; 25422f38b3e1SArne Jansen struct extent_buffer *leaf; 25432f38b3e1SArne Jansen 25442f38b3e1SArne Jansen again: 25452f38b3e1SArne Jansen ret = btrfs_search_slot(NULL, root, key, p, 0, 0); 25462f38b3e1SArne Jansen if (ret <= 0) 25472f38b3e1SArne Jansen return ret; 25482f38b3e1SArne Jansen /* 25492f38b3e1SArne Jansen * a return value of 1 means the path is at the position where the 25502f38b3e1SArne Jansen * item should be inserted. Normally this is the next bigger item, 25512f38b3e1SArne Jansen * but in case the previous item is the last in a leaf, path points 25522f38b3e1SArne Jansen * to the first free slot in the previous leaf, i.e. at an invalid 25532f38b3e1SArne Jansen * item. 25542f38b3e1SArne Jansen */ 25552f38b3e1SArne Jansen leaf = p->nodes[0]; 25562f38b3e1SArne Jansen 25572f38b3e1SArne Jansen if (find_higher) { 25582f38b3e1SArne Jansen if (p->slots[0] >= btrfs_header_nritems(leaf)) { 25592f38b3e1SArne Jansen ret = btrfs_next_leaf(root, p); 25602f38b3e1SArne Jansen if (ret <= 0) 25612f38b3e1SArne Jansen return ret; 25622f38b3e1SArne Jansen if (!return_any) 25632f38b3e1SArne Jansen return 1; 25642f38b3e1SArne Jansen /* 25652f38b3e1SArne Jansen * no higher item found, return the next 25662f38b3e1SArne Jansen * lower instead 25672f38b3e1SArne Jansen */ 25682f38b3e1SArne Jansen return_any = 0; 25692f38b3e1SArne Jansen find_higher = 0; 25702f38b3e1SArne Jansen btrfs_release_path(p); 25712f38b3e1SArne Jansen goto again; 25722f38b3e1SArne Jansen } 25732f38b3e1SArne Jansen } else { 25742f38b3e1SArne Jansen if (p->slots[0] == 0) { 25752f38b3e1SArne Jansen ret = btrfs_prev_leaf(root, p); 2576e6793769SArne Jansen if (ret < 0) 25772f38b3e1SArne Jansen return ret; 2578e6793769SArne Jansen if (!ret) { 257923c6bf6aSFilipe David Borba Manana leaf = p->nodes[0]; 258023c6bf6aSFilipe David Borba Manana if (p->slots[0] == btrfs_header_nritems(leaf)) 258123c6bf6aSFilipe David Borba Manana p->slots[0]--; 2582e6793769SArne Jansen return 0; 2583e6793769SArne Jansen } 25842f38b3e1SArne Jansen if (!return_any) 25852f38b3e1SArne Jansen return 1; 25862f38b3e1SArne Jansen /* 25872f38b3e1SArne Jansen * no lower item found, return the next 25882f38b3e1SArne Jansen * higher instead 25892f38b3e1SArne Jansen */ 25902f38b3e1SArne Jansen return_any = 0; 25912f38b3e1SArne Jansen find_higher = 1; 25922f38b3e1SArne Jansen btrfs_release_path(p); 25932f38b3e1SArne Jansen goto again; 2594e6793769SArne Jansen } else { 25952f38b3e1SArne Jansen --p->slots[0]; 25962f38b3e1SArne Jansen } 25972f38b3e1SArne Jansen } 25982f38b3e1SArne Jansen return 0; 25992f38b3e1SArne Jansen } 26002f38b3e1SArne Jansen 26012f38b3e1SArne Jansen /* 26020ff40a91SMarcos Paulo de Souza * Execute search and call btrfs_previous_item to traverse backwards if the item 26030ff40a91SMarcos Paulo de Souza * was not found. 26040ff40a91SMarcos Paulo de Souza * 26050ff40a91SMarcos Paulo de Souza * Return 0 if found, 1 if not found and < 0 if error. 26060ff40a91SMarcos Paulo de Souza */ 26070ff40a91SMarcos Paulo de Souza int btrfs_search_backwards(struct btrfs_root *root, struct btrfs_key *key, 26080ff40a91SMarcos Paulo de Souza struct btrfs_path *path) 26090ff40a91SMarcos Paulo de Souza { 26100ff40a91SMarcos Paulo de Souza int ret; 26110ff40a91SMarcos Paulo de Souza 26120ff40a91SMarcos Paulo de Souza ret = btrfs_search_slot(NULL, root, key, path, 0, 0); 26130ff40a91SMarcos Paulo de Souza if (ret > 0) 26140ff40a91SMarcos Paulo de Souza ret = btrfs_previous_item(root, path, key->objectid, key->type); 26150ff40a91SMarcos Paulo de Souza 26160ff40a91SMarcos Paulo de Souza if (ret == 0) 26170ff40a91SMarcos Paulo de Souza btrfs_item_key_to_cpu(path->nodes[0], key, path->slots[0]); 26180ff40a91SMarcos Paulo de Souza 26190ff40a91SMarcos Paulo de Souza return ret; 26200ff40a91SMarcos Paulo de Souza } 26210ff40a91SMarcos Paulo de Souza 262243dd529aSDavid Sterba /* 262362142be3SGabriel Niebler * Search for a valid slot for the given path. 262462142be3SGabriel Niebler * 262562142be3SGabriel Niebler * @root: The root node of the tree. 262662142be3SGabriel Niebler * @key: Will contain a valid item if found. 262762142be3SGabriel Niebler * @path: The starting point to validate the slot. 262862142be3SGabriel Niebler * 262962142be3SGabriel Niebler * Return: 0 if the item is valid 263062142be3SGabriel Niebler * 1 if not found 263162142be3SGabriel Niebler * <0 if error. 263262142be3SGabriel Niebler */ 263362142be3SGabriel Niebler int btrfs_get_next_valid_item(struct btrfs_root *root, struct btrfs_key *key, 263462142be3SGabriel Niebler struct btrfs_path *path) 263562142be3SGabriel Niebler { 2636524f14bbSFilipe Manana if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) { 263762142be3SGabriel Niebler int ret; 263862142be3SGabriel Niebler 263962142be3SGabriel Niebler ret = btrfs_next_leaf(root, path); 264062142be3SGabriel Niebler if (ret) 264162142be3SGabriel Niebler return ret; 264262142be3SGabriel Niebler } 2643524f14bbSFilipe Manana 2644524f14bbSFilipe Manana btrfs_item_key_to_cpu(path->nodes[0], key, path->slots[0]); 264562142be3SGabriel Niebler return 0; 264662142be3SGabriel Niebler } 264762142be3SGabriel Niebler 26480ff40a91SMarcos Paulo de Souza /* 264974123bd7SChris Mason * adjust the pointers going up the tree, starting at level 265074123bd7SChris Mason * making sure the right key of each node is points to 'key'. 265174123bd7SChris Mason * This is used after shifting pointers to the left, so it stops 265274123bd7SChris Mason * fixing up pointers when a given leaf/node is not in slot 0 of the 265374123bd7SChris Mason * higher levels 2654aa5d6bedSChris Mason * 265574123bd7SChris Mason */ 2656b167fa91SNikolay Borisov static void fixup_low_keys(struct btrfs_path *path, 26575f39d397SChris Mason struct btrfs_disk_key *key, int level) 2658be0e5c09SChris Mason { 2659be0e5c09SChris Mason int i; 26605f39d397SChris Mason struct extent_buffer *t; 26610e82bcfeSDavid Sterba int ret; 26625f39d397SChris Mason 2663234b63a0SChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 2664be0e5c09SChris Mason int tslot = path->slots[i]; 26650e82bcfeSDavid Sterba 2666eb60ceacSChris Mason if (!path->nodes[i]) 2667be0e5c09SChris Mason break; 26685f39d397SChris Mason t = path->nodes[i]; 2669f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(t, tslot, 267033cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 26710e82bcfeSDavid Sterba BUG_ON(ret < 0); 26725f39d397SChris Mason btrfs_set_node_key(t, key, tslot); 2673d6025579SChris Mason btrfs_mark_buffer_dirty(path->nodes[i]); 2674be0e5c09SChris Mason if (tslot != 0) 2675be0e5c09SChris Mason break; 2676be0e5c09SChris Mason } 2677be0e5c09SChris Mason } 2678be0e5c09SChris Mason 267974123bd7SChris Mason /* 268031840ae1SZheng Yan * update item key. 268131840ae1SZheng Yan * 268231840ae1SZheng Yan * This function isn't completely safe. It's the caller's responsibility 268331840ae1SZheng Yan * that the new key won't break the order 268431840ae1SZheng Yan */ 2685b7a0365eSDaniel Dressler void btrfs_set_item_key_safe(struct btrfs_fs_info *fs_info, 2686b7a0365eSDaniel Dressler struct btrfs_path *path, 2687310712b2SOmar Sandoval const struct btrfs_key *new_key) 268831840ae1SZheng Yan { 268931840ae1SZheng Yan struct btrfs_disk_key disk_key; 269031840ae1SZheng Yan struct extent_buffer *eb; 269131840ae1SZheng Yan int slot; 269231840ae1SZheng Yan 269331840ae1SZheng Yan eb = path->nodes[0]; 269431840ae1SZheng Yan slot = path->slots[0]; 269531840ae1SZheng Yan if (slot > 0) { 269631840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot - 1); 26977c15d410SQu Wenruo if (unlikely(comp_keys(&disk_key, new_key) >= 0)) { 2698eee3b811SQu Wenruo btrfs_print_leaf(eb); 26997c15d410SQu Wenruo btrfs_crit(fs_info, 27007c15d410SQu Wenruo "slot %u key (%llu %u %llu) new key (%llu %u %llu)", 27017c15d410SQu Wenruo slot, btrfs_disk_key_objectid(&disk_key), 27027c15d410SQu Wenruo btrfs_disk_key_type(&disk_key), 27037c15d410SQu Wenruo btrfs_disk_key_offset(&disk_key), 27047c15d410SQu Wenruo new_key->objectid, new_key->type, 27057c15d410SQu Wenruo new_key->offset); 27067c15d410SQu Wenruo BUG(); 27077c15d410SQu Wenruo } 270831840ae1SZheng Yan } 270931840ae1SZheng Yan if (slot < btrfs_header_nritems(eb) - 1) { 271031840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot + 1); 27117c15d410SQu Wenruo if (unlikely(comp_keys(&disk_key, new_key) <= 0)) { 2712eee3b811SQu Wenruo btrfs_print_leaf(eb); 27137c15d410SQu Wenruo btrfs_crit(fs_info, 27147c15d410SQu Wenruo "slot %u key (%llu %u %llu) new key (%llu %u %llu)", 27157c15d410SQu Wenruo slot, btrfs_disk_key_objectid(&disk_key), 27167c15d410SQu Wenruo btrfs_disk_key_type(&disk_key), 27177c15d410SQu Wenruo btrfs_disk_key_offset(&disk_key), 27187c15d410SQu Wenruo new_key->objectid, new_key->type, 27197c15d410SQu Wenruo new_key->offset); 27207c15d410SQu Wenruo BUG(); 27217c15d410SQu Wenruo } 272231840ae1SZheng Yan } 272331840ae1SZheng Yan 272431840ae1SZheng Yan btrfs_cpu_key_to_disk(&disk_key, new_key); 272531840ae1SZheng Yan btrfs_set_item_key(eb, &disk_key, slot); 272631840ae1SZheng Yan btrfs_mark_buffer_dirty(eb); 272731840ae1SZheng Yan if (slot == 0) 2728b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 272931840ae1SZheng Yan } 273031840ae1SZheng Yan 273131840ae1SZheng Yan /* 2732d16c702fSQu Wenruo * Check key order of two sibling extent buffers. 2733d16c702fSQu Wenruo * 2734d16c702fSQu Wenruo * Return true if something is wrong. 2735d16c702fSQu Wenruo * Return false if everything is fine. 2736d16c702fSQu Wenruo * 2737d16c702fSQu Wenruo * Tree-checker only works inside one tree block, thus the following 2738d16c702fSQu Wenruo * corruption can not be detected by tree-checker: 2739d16c702fSQu Wenruo * 2740d16c702fSQu Wenruo * Leaf @left | Leaf @right 2741d16c702fSQu Wenruo * -------------------------------------------------------------- 2742d16c702fSQu Wenruo * | 1 | 2 | 3 | 4 | 5 | f6 | | 7 | 8 | 2743d16c702fSQu Wenruo * 2744d16c702fSQu Wenruo * Key f6 in leaf @left itself is valid, but not valid when the next 2745d16c702fSQu Wenruo * key in leaf @right is 7. 2746d16c702fSQu Wenruo * This can only be checked at tree block merge time. 2747d16c702fSQu Wenruo * And since tree checker has ensured all key order in each tree block 2748d16c702fSQu Wenruo * is correct, we only need to bother the last key of @left and the first 2749d16c702fSQu Wenruo * key of @right. 2750d16c702fSQu Wenruo */ 2751d16c702fSQu Wenruo static bool check_sibling_keys(struct extent_buffer *left, 2752d16c702fSQu Wenruo struct extent_buffer *right) 2753d16c702fSQu Wenruo { 2754d16c702fSQu Wenruo struct btrfs_key left_last; 2755d16c702fSQu Wenruo struct btrfs_key right_first; 2756d16c702fSQu Wenruo int level = btrfs_header_level(left); 2757d16c702fSQu Wenruo int nr_left = btrfs_header_nritems(left); 2758d16c702fSQu Wenruo int nr_right = btrfs_header_nritems(right); 2759d16c702fSQu Wenruo 2760d16c702fSQu Wenruo /* No key to check in one of the tree blocks */ 2761d16c702fSQu Wenruo if (!nr_left || !nr_right) 2762d16c702fSQu Wenruo return false; 2763d16c702fSQu Wenruo 2764d16c702fSQu Wenruo if (level) { 2765d16c702fSQu Wenruo btrfs_node_key_to_cpu(left, &left_last, nr_left - 1); 2766d16c702fSQu Wenruo btrfs_node_key_to_cpu(right, &right_first, 0); 2767d16c702fSQu Wenruo } else { 2768d16c702fSQu Wenruo btrfs_item_key_to_cpu(left, &left_last, nr_left - 1); 2769d16c702fSQu Wenruo btrfs_item_key_to_cpu(right, &right_first, 0); 2770d16c702fSQu Wenruo } 2771d16c702fSQu Wenruo 277288ad95b0SFilipe Manana if (unlikely(btrfs_comp_cpu_keys(&left_last, &right_first) >= 0)) { 2773a2cea677SFilipe Manana btrfs_crit(left->fs_info, "left extent buffer:"); 2774a2cea677SFilipe Manana btrfs_print_tree(left, false); 2775a2cea677SFilipe Manana btrfs_crit(left->fs_info, "right extent buffer:"); 2776a2cea677SFilipe Manana btrfs_print_tree(right, false); 2777d16c702fSQu Wenruo btrfs_crit(left->fs_info, 2778d16c702fSQu Wenruo "bad key order, sibling blocks, left last (%llu %u %llu) right first (%llu %u %llu)", 2779d16c702fSQu Wenruo left_last.objectid, left_last.type, 2780d16c702fSQu Wenruo left_last.offset, right_first.objectid, 2781d16c702fSQu Wenruo right_first.type, right_first.offset); 2782d16c702fSQu Wenruo return true; 2783d16c702fSQu Wenruo } 2784d16c702fSQu Wenruo return false; 2785d16c702fSQu Wenruo } 2786d16c702fSQu Wenruo 2787d16c702fSQu Wenruo /* 278874123bd7SChris Mason * try to push data from one node into the next node left in the 278979f95c82SChris Mason * tree. 2790aa5d6bedSChris Mason * 2791aa5d6bedSChris Mason * returns 0 if some ptrs were pushed left, < 0 if there was some horrible 2792aa5d6bedSChris Mason * error, and > 0 if there was no room in the left hand block. 279374123bd7SChris Mason */ 279498ed5174SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 27952ff7e61eSJeff Mahoney struct extent_buffer *dst, 2796971a1f66SChris Mason struct extent_buffer *src, int empty) 2797be0e5c09SChris Mason { 2798d30a668fSDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info; 2799be0e5c09SChris Mason int push_items = 0; 2800bb803951SChris Mason int src_nritems; 2801bb803951SChris Mason int dst_nritems; 2802aa5d6bedSChris Mason int ret = 0; 2803be0e5c09SChris Mason 28045f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 28055f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 28060b246afaSJeff Mahoney push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems; 28077bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 28087bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 280954aa1f4dSChris Mason 2810bce4eae9SChris Mason if (!empty && src_nritems <= 8) 2811971a1f66SChris Mason return 1; 2812971a1f66SChris Mason 2813d397712bSChris Mason if (push_items <= 0) 2814be0e5c09SChris Mason return 1; 2815be0e5c09SChris Mason 2816bce4eae9SChris Mason if (empty) { 2817971a1f66SChris Mason push_items = min(src_nritems, push_items); 2818bce4eae9SChris Mason if (push_items < src_nritems) { 2819bce4eae9SChris Mason /* leave at least 8 pointers in the node if 2820bce4eae9SChris Mason * we aren't going to empty it 2821bce4eae9SChris Mason */ 2822bce4eae9SChris Mason if (src_nritems - push_items < 8) { 2823bce4eae9SChris Mason if (push_items <= 8) 2824bce4eae9SChris Mason return 1; 2825bce4eae9SChris Mason push_items -= 8; 2826bce4eae9SChris Mason } 2827bce4eae9SChris Mason } 2828bce4eae9SChris Mason } else 2829bce4eae9SChris Mason push_items = min(src_nritems - 8, push_items); 283079f95c82SChris Mason 2831d16c702fSQu Wenruo /* dst is the left eb, src is the middle eb */ 2832d16c702fSQu Wenruo if (check_sibling_keys(dst, src)) { 2833d16c702fSQu Wenruo ret = -EUCLEAN; 2834d16c702fSQu Wenruo btrfs_abort_transaction(trans, ret); 2835d16c702fSQu Wenruo return ret; 2836d16c702fSQu Wenruo } 2837f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_eb_copy(dst, src, dst_nritems, 0, push_items); 28385de865eeSFilipe David Borba Manana if (ret) { 283966642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 28405de865eeSFilipe David Borba Manana return ret; 28415de865eeSFilipe David Borba Manana } 28425f39d397SChris Mason copy_extent_buffer(dst, src, 2843e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(dst, dst_nritems), 2844e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(src, 0), 2845123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 28465f39d397SChris Mason 2847bb803951SChris Mason if (push_items < src_nritems) { 284857911b8bSJan Schmidt /* 28495cead542SBoris Burkov * btrfs_tree_mod_log_eb_copy handles logging the move, so we 28505cead542SBoris Burkov * don't need to do an explicit tree mod log operation for it. 285157911b8bSJan Schmidt */ 2852e23efd8eSJosef Bacik memmove_extent_buffer(src, btrfs_node_key_ptr_offset(src, 0), 2853e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(src, push_items), 2854e2fa7227SChris Mason (src_nritems - push_items) * 2855123abc88SChris Mason sizeof(struct btrfs_key_ptr)); 2856bb803951SChris Mason } 28575f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 28585f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 28595f39d397SChris Mason btrfs_mark_buffer_dirty(src); 28605f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 286131840ae1SZheng Yan 2862bb803951SChris Mason return ret; 2863be0e5c09SChris Mason } 2864be0e5c09SChris Mason 286597571fd0SChris Mason /* 286679f95c82SChris Mason * try to push data from one node into the next node right in the 286779f95c82SChris Mason * tree. 286879f95c82SChris Mason * 286979f95c82SChris Mason * returns 0 if some ptrs were pushed, < 0 if there was some horrible 287079f95c82SChris Mason * error, and > 0 if there was no room in the right hand block. 287179f95c82SChris Mason * 287279f95c82SChris Mason * this will only push up to 1/2 the contents of the left node over 287379f95c82SChris Mason */ 28745f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 28755f39d397SChris Mason struct extent_buffer *dst, 28765f39d397SChris Mason struct extent_buffer *src) 287779f95c82SChris Mason { 287855d32ed8SDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info; 287979f95c82SChris Mason int push_items = 0; 288079f95c82SChris Mason int max_push; 288179f95c82SChris Mason int src_nritems; 288279f95c82SChris Mason int dst_nritems; 288379f95c82SChris Mason int ret = 0; 288479f95c82SChris Mason 28857bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 28867bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 28877bb86316SChris Mason 28885f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 28895f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 28900b246afaSJeff Mahoney push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems; 2891d397712bSChris Mason if (push_items <= 0) 289279f95c82SChris Mason return 1; 2893bce4eae9SChris Mason 2894d397712bSChris Mason if (src_nritems < 4) 2895bce4eae9SChris Mason return 1; 289679f95c82SChris Mason 289779f95c82SChris Mason max_push = src_nritems / 2 + 1; 289879f95c82SChris Mason /* don't try to empty the node */ 2899d397712bSChris Mason if (max_push >= src_nritems) 290079f95c82SChris Mason return 1; 2901252c38f0SYan 290279f95c82SChris Mason if (max_push < push_items) 290379f95c82SChris Mason push_items = max_push; 290479f95c82SChris Mason 2905d16c702fSQu Wenruo /* dst is the right eb, src is the middle eb */ 2906d16c702fSQu Wenruo if (check_sibling_keys(src, dst)) { 2907d16c702fSQu Wenruo ret = -EUCLEAN; 2908d16c702fSQu Wenruo btrfs_abort_transaction(trans, ret); 2909d16c702fSQu Wenruo return ret; 2910d16c702fSQu Wenruo } 29115cead542SBoris Burkov 29125cead542SBoris Burkov /* 29135cead542SBoris Burkov * btrfs_tree_mod_log_eb_copy handles logging the move, so we don't 29145cead542SBoris Burkov * need to do an explicit tree mod log operation for it. 29155cead542SBoris Burkov */ 2916e23efd8eSJosef Bacik memmove_extent_buffer(dst, btrfs_node_key_ptr_offset(dst, push_items), 2917e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(dst, 0), 29185f39d397SChris Mason (dst_nritems) * 29195f39d397SChris Mason sizeof(struct btrfs_key_ptr)); 2920d6025579SChris Mason 2921f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_eb_copy(dst, src, 0, src_nritems - push_items, 2922ed874f0dSDavid Sterba push_items); 29235de865eeSFilipe David Borba Manana if (ret) { 292466642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 29255de865eeSFilipe David Borba Manana return ret; 29265de865eeSFilipe David Borba Manana } 29275f39d397SChris Mason copy_extent_buffer(dst, src, 2928e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(dst, 0), 2929e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(src, src_nritems - push_items), 2930123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 293179f95c82SChris Mason 29325f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 29335f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 293479f95c82SChris Mason 29355f39d397SChris Mason btrfs_mark_buffer_dirty(src); 29365f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 293731840ae1SZheng Yan 293879f95c82SChris Mason return ret; 293979f95c82SChris Mason } 294079f95c82SChris Mason 294179f95c82SChris Mason /* 294297571fd0SChris Mason * helper function to insert a new root level in the tree. 294397571fd0SChris Mason * A new node is allocated, and a single item is inserted to 294497571fd0SChris Mason * point to the existing root 2945aa5d6bedSChris Mason * 2946aa5d6bedSChris Mason * returns zero on success or < 0 on failure. 294797571fd0SChris Mason */ 2948d397712bSChris Mason static noinline int insert_new_root(struct btrfs_trans_handle *trans, 29495f39d397SChris Mason struct btrfs_root *root, 2950fdd99c72SLiu Bo struct btrfs_path *path, int level) 295174123bd7SChris Mason { 29520b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 29537bb86316SChris Mason u64 lower_gen; 29545f39d397SChris Mason struct extent_buffer *lower; 29555f39d397SChris Mason struct extent_buffer *c; 2956925baeddSChris Mason struct extent_buffer *old; 29575f39d397SChris Mason struct btrfs_disk_key lower_key; 2958d9d19a01SDavid Sterba int ret; 29595c680ed6SChris Mason 29605c680ed6SChris Mason BUG_ON(path->nodes[level]); 29615c680ed6SChris Mason BUG_ON(path->nodes[level-1] != root->node); 29625c680ed6SChris Mason 29637bb86316SChris Mason lower = path->nodes[level-1]; 29647bb86316SChris Mason if (level == 1) 29657bb86316SChris Mason btrfs_item_key(lower, &lower_key, 0); 29667bb86316SChris Mason else 29677bb86316SChris Mason btrfs_node_key(lower, &lower_key, 0); 29687bb86316SChris Mason 296979bd3712SFilipe Manana c = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 297079bd3712SFilipe Manana &lower_key, level, root->node->start, 0, 2971cf6f34aaSJosef Bacik BTRFS_NESTING_NEW_ROOT); 29725f39d397SChris Mason if (IS_ERR(c)) 29735f39d397SChris Mason return PTR_ERR(c); 2974925baeddSChris Mason 29750b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 2976f0486c68SYan, Zheng 29775f39d397SChris Mason btrfs_set_header_nritems(c, 1); 29785f39d397SChris Mason btrfs_set_node_key(c, &lower_key, 0); 2979db94535dSChris Mason btrfs_set_node_blockptr(c, 0, lower->start); 29807bb86316SChris Mason lower_gen = btrfs_header_generation(lower); 298131840ae1SZheng Yan WARN_ON(lower_gen != trans->transid); 29827bb86316SChris Mason 29837bb86316SChris Mason btrfs_set_node_ptr_generation(c, 0, lower_gen); 29845f39d397SChris Mason 29855f39d397SChris Mason btrfs_mark_buffer_dirty(c); 2986d5719762SChris Mason 2987925baeddSChris Mason old = root->node; 2988406808abSFilipe Manana ret = btrfs_tree_mod_log_insert_root(root->node, c, false); 2989f61aa7baSFilipe Manana if (ret < 0) { 2990f61aa7baSFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), c, 0, 1); 2991f61aa7baSFilipe Manana btrfs_tree_unlock(c); 2992f61aa7baSFilipe Manana free_extent_buffer(c); 2993f61aa7baSFilipe Manana return ret; 2994f61aa7baSFilipe Manana } 2995240f62c8SChris Mason rcu_assign_pointer(root->node, c); 2996925baeddSChris Mason 2997925baeddSChris Mason /* the super has an extra ref to root->node */ 2998925baeddSChris Mason free_extent_buffer(old); 2999925baeddSChris Mason 30000b86a832SChris Mason add_root_to_dirty_list(root); 300167439dadSDavid Sterba atomic_inc(&c->refs); 30025f39d397SChris Mason path->nodes[level] = c; 3003ac5887c8SJosef Bacik path->locks[level] = BTRFS_WRITE_LOCK; 300474123bd7SChris Mason path->slots[level] = 0; 300574123bd7SChris Mason return 0; 300674123bd7SChris Mason } 30075c680ed6SChris Mason 30085c680ed6SChris Mason /* 30095c680ed6SChris Mason * worker function to insert a single pointer in a node. 30105c680ed6SChris Mason * the node should have enough room for the pointer already 301197571fd0SChris Mason * 30125c680ed6SChris Mason * slot and level indicate where you want the key to go, and 30135c680ed6SChris Mason * blocknr is the block the key points to. 30145c680ed6SChris Mason */ 301550b5d1fcSFilipe Manana static int insert_ptr(struct btrfs_trans_handle *trans, 30166ad3cf6dSDavid Sterba struct btrfs_path *path, 3017143bede5SJeff Mahoney struct btrfs_disk_key *key, u64 bytenr, 3018c3e06965SJan Schmidt int slot, int level) 30195c680ed6SChris Mason { 30205f39d397SChris Mason struct extent_buffer *lower; 30215c680ed6SChris Mason int nritems; 3022f3ea38daSJan Schmidt int ret; 30235c680ed6SChris Mason 30245c680ed6SChris Mason BUG_ON(!path->nodes[level]); 302549d0c642SFilipe Manana btrfs_assert_tree_write_locked(path->nodes[level]); 30265f39d397SChris Mason lower = path->nodes[level]; 30275f39d397SChris Mason nritems = btrfs_header_nritems(lower); 3028c293498bSStoyan Gaydarov BUG_ON(slot > nritems); 30296ad3cf6dSDavid Sterba BUG_ON(nritems == BTRFS_NODEPTRS_PER_BLOCK(trans->fs_info)); 303074123bd7SChris Mason if (slot != nritems) { 3031bf1d3425SDavid Sterba if (level) { 3032f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_move(lower, slot + 1, 3033f3a84ccdSFilipe Manana slot, nritems - slot); 303450b5d1fcSFilipe Manana if (ret < 0) { 303550b5d1fcSFilipe Manana btrfs_abort_transaction(trans, ret); 303650b5d1fcSFilipe Manana return ret; 303750b5d1fcSFilipe Manana } 3038bf1d3425SDavid Sterba } 30395f39d397SChris Mason memmove_extent_buffer(lower, 3040e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(lower, slot + 1), 3041e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(lower, slot), 3042123abc88SChris Mason (nritems - slot) * sizeof(struct btrfs_key_ptr)); 304374123bd7SChris Mason } 3044c3e06965SJan Schmidt if (level) { 3045f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(lower, slot, 304633cff222SFilipe Manana BTRFS_MOD_LOG_KEY_ADD); 304750b5d1fcSFilipe Manana if (ret < 0) { 304850b5d1fcSFilipe Manana btrfs_abort_transaction(trans, ret); 304950b5d1fcSFilipe Manana return ret; 305050b5d1fcSFilipe Manana } 3051f3ea38daSJan Schmidt } 30525f39d397SChris Mason btrfs_set_node_key(lower, key, slot); 3053db94535dSChris Mason btrfs_set_node_blockptr(lower, slot, bytenr); 305474493f7aSChris Mason WARN_ON(trans->transid == 0); 305574493f7aSChris Mason btrfs_set_node_ptr_generation(lower, slot, trans->transid); 30565f39d397SChris Mason btrfs_set_header_nritems(lower, nritems + 1); 30575f39d397SChris Mason btrfs_mark_buffer_dirty(lower); 305850b5d1fcSFilipe Manana 305950b5d1fcSFilipe Manana return 0; 306074123bd7SChris Mason } 306174123bd7SChris Mason 306297571fd0SChris Mason /* 306397571fd0SChris Mason * split the node at the specified level in path in two. 306497571fd0SChris Mason * The path is corrected to point to the appropriate node after the split 306597571fd0SChris Mason * 306697571fd0SChris Mason * Before splitting this tries to make some room in the node by pushing 306797571fd0SChris Mason * left and right, if either one works, it returns right away. 3068aa5d6bedSChris Mason * 3069aa5d6bedSChris Mason * returns 0 on success and < 0 on failure 307097571fd0SChris Mason */ 3071e02119d5SChris Mason static noinline int split_node(struct btrfs_trans_handle *trans, 3072e02119d5SChris Mason struct btrfs_root *root, 3073e02119d5SChris Mason struct btrfs_path *path, int level) 3074be0e5c09SChris Mason { 30750b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 30765f39d397SChris Mason struct extent_buffer *c; 30775f39d397SChris Mason struct extent_buffer *split; 30785f39d397SChris Mason struct btrfs_disk_key disk_key; 3079be0e5c09SChris Mason int mid; 30805c680ed6SChris Mason int ret; 30817518a238SChris Mason u32 c_nritems; 3082be0e5c09SChris Mason 30835f39d397SChris Mason c = path->nodes[level]; 30847bb86316SChris Mason WARN_ON(btrfs_header_generation(c) != trans->transid); 30855f39d397SChris Mason if (c == root->node) { 3086d9abbf1cSJan Schmidt /* 308790f8d62eSJan Schmidt * trying to split the root, lets make a new one 308890f8d62eSJan Schmidt * 3089fdd99c72SLiu Bo * tree mod log: We don't log_removal old root in 309090f8d62eSJan Schmidt * insert_new_root, because that root buffer will be kept as a 309190f8d62eSJan Schmidt * normal node. We are going to log removal of half of the 3092f3a84ccdSFilipe Manana * elements below with btrfs_tree_mod_log_eb_copy(). We're 3093f3a84ccdSFilipe Manana * holding a tree lock on the buffer, which is why we cannot 3094f3a84ccdSFilipe Manana * race with other tree_mod_log users. 3095d9abbf1cSJan Schmidt */ 3096fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, level + 1); 30975c680ed6SChris Mason if (ret) 30985c680ed6SChris Mason return ret; 3099b3612421SChris Mason } else { 3100e66f709bSChris Mason ret = push_nodes_for_insert(trans, root, path, level); 31015f39d397SChris Mason c = path->nodes[level]; 31025f39d397SChris Mason if (!ret && btrfs_header_nritems(c) < 31030b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3) 3104e66f709bSChris Mason return 0; 310554aa1f4dSChris Mason if (ret < 0) 310654aa1f4dSChris Mason return ret; 31075c680ed6SChris Mason } 3108e66f709bSChris Mason 31095f39d397SChris Mason c_nritems = btrfs_header_nritems(c); 31105d4f98a2SYan Zheng mid = (c_nritems + 1) / 2; 31115d4f98a2SYan Zheng btrfs_node_key(c, &disk_key, mid); 31127bb86316SChris Mason 311379bd3712SFilipe Manana split = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 311479bd3712SFilipe Manana &disk_key, level, c->start, 0, 311579bd3712SFilipe Manana BTRFS_NESTING_SPLIT); 31165f39d397SChris Mason if (IS_ERR(split)) 31175f39d397SChris Mason return PTR_ERR(split); 311854aa1f4dSChris Mason 31190b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 3120bc877d28SNikolay Borisov ASSERT(btrfs_header_level(c) == level); 31215f39d397SChris Mason 3122f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_eb_copy(split, c, 0, mid, c_nritems - mid); 31235de865eeSFilipe David Borba Manana if (ret) { 3124ede600e4SFilipe Manana btrfs_tree_unlock(split); 3125ede600e4SFilipe Manana free_extent_buffer(split); 312666642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 31275de865eeSFilipe David Borba Manana return ret; 31285de865eeSFilipe David Borba Manana } 31295f39d397SChris Mason copy_extent_buffer(split, c, 3130e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(split, 0), 3131e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(c, mid), 3132123abc88SChris Mason (c_nritems - mid) * sizeof(struct btrfs_key_ptr)); 31335f39d397SChris Mason btrfs_set_header_nritems(split, c_nritems - mid); 31345f39d397SChris Mason btrfs_set_header_nritems(c, mid); 3135aa5d6bedSChris Mason 31365f39d397SChris Mason btrfs_mark_buffer_dirty(c); 31375f39d397SChris Mason btrfs_mark_buffer_dirty(split); 31385f39d397SChris Mason 313950b5d1fcSFilipe Manana ret = insert_ptr(trans, path, &disk_key, split->start, 3140c3e06965SJan Schmidt path->slots[level + 1] + 1, level + 1); 314150b5d1fcSFilipe Manana if (ret < 0) { 314250b5d1fcSFilipe Manana btrfs_tree_unlock(split); 314350b5d1fcSFilipe Manana free_extent_buffer(split); 314450b5d1fcSFilipe Manana return ret; 314550b5d1fcSFilipe Manana } 3146aa5d6bedSChris Mason 31475de08d7dSChris Mason if (path->slots[level] >= mid) { 31485c680ed6SChris Mason path->slots[level] -= mid; 3149925baeddSChris Mason btrfs_tree_unlock(c); 31505f39d397SChris Mason free_extent_buffer(c); 31515f39d397SChris Mason path->nodes[level] = split; 31525c680ed6SChris Mason path->slots[level + 1] += 1; 3153eb60ceacSChris Mason } else { 3154925baeddSChris Mason btrfs_tree_unlock(split); 31555f39d397SChris Mason free_extent_buffer(split); 3156be0e5c09SChris Mason } 3157d5286a92SNikolay Borisov return 0; 3158be0e5c09SChris Mason } 3159be0e5c09SChris Mason 316074123bd7SChris Mason /* 316174123bd7SChris Mason * how many bytes are required to store the items in a leaf. start 316274123bd7SChris Mason * and nr indicate which items in the leaf to check. This totals up the 316374123bd7SChris Mason * space used both by the item structs and the item data 316474123bd7SChris Mason */ 31656c75a589SQu Wenruo static int leaf_space_used(const struct extent_buffer *l, int start, int nr) 3166be0e5c09SChris Mason { 3167be0e5c09SChris Mason int data_len; 31685f39d397SChris Mason int nritems = btrfs_header_nritems(l); 3169d4dbff95SChris Mason int end = min(nritems, start + nr) - 1; 3170be0e5c09SChris Mason 3171be0e5c09SChris Mason if (!nr) 3172be0e5c09SChris Mason return 0; 31733212fa14SJosef Bacik data_len = btrfs_item_offset(l, start) + btrfs_item_size(l, start); 31743212fa14SJosef Bacik data_len = data_len - btrfs_item_offset(l, end); 31750783fcfcSChris Mason data_len += sizeof(struct btrfs_item) * nr; 3176d4dbff95SChris Mason WARN_ON(data_len < 0); 3177be0e5c09SChris Mason return data_len; 3178be0e5c09SChris Mason } 3179be0e5c09SChris Mason 318074123bd7SChris Mason /* 3181d4dbff95SChris Mason * The space between the end of the leaf items and 3182d4dbff95SChris Mason * the start of the leaf data. IOW, how much room 3183d4dbff95SChris Mason * the leaf has left for both items and data 3184d4dbff95SChris Mason */ 31856c75a589SQu Wenruo int btrfs_leaf_free_space(const struct extent_buffer *leaf) 3186d4dbff95SChris Mason { 3187e902baacSDavid Sterba struct btrfs_fs_info *fs_info = leaf->fs_info; 31885f39d397SChris Mason int nritems = btrfs_header_nritems(leaf); 31895f39d397SChris Mason int ret; 31900b246afaSJeff Mahoney 31910b246afaSJeff Mahoney ret = BTRFS_LEAF_DATA_SIZE(fs_info) - leaf_space_used(leaf, 0, nritems); 31925f39d397SChris Mason if (ret < 0) { 31930b246afaSJeff Mahoney btrfs_crit(fs_info, 3194efe120a0SFrank Holton "leaf free space ret %d, leaf data size %lu, used %d nritems %d", 3195da17066cSJeff Mahoney ret, 31960b246afaSJeff Mahoney (unsigned long) BTRFS_LEAF_DATA_SIZE(fs_info), 31975f39d397SChris Mason leaf_space_used(leaf, 0, nritems), nritems); 31985f39d397SChris Mason } 31995f39d397SChris Mason return ret; 3200d4dbff95SChris Mason } 3201d4dbff95SChris Mason 320299d8f83cSChris Mason /* 320399d8f83cSChris Mason * min slot controls the lowest index we're willing to push to the 320499d8f83cSChris Mason * right. We'll push up to and including min_slot, but no lower 320599d8f83cSChris Mason */ 3206ed25dab3SJosef Bacik static noinline int __push_leaf_right(struct btrfs_trans_handle *trans, 3207ed25dab3SJosef Bacik struct btrfs_path *path, 320844871b1bSChris Mason int data_size, int empty, 320944871b1bSChris Mason struct extent_buffer *right, 321099d8f83cSChris Mason int free_space, u32 left_nritems, 321199d8f83cSChris Mason u32 min_slot) 321200ec4c51SChris Mason { 3213f72f0010SDavid Sterba struct btrfs_fs_info *fs_info = right->fs_info; 32145f39d397SChris Mason struct extent_buffer *left = path->nodes[0]; 321544871b1bSChris Mason struct extent_buffer *upper = path->nodes[1]; 3216cfed81a0SChris Mason struct btrfs_map_token token; 32175f39d397SChris Mason struct btrfs_disk_key disk_key; 321800ec4c51SChris Mason int slot; 321934a38218SChris Mason u32 i; 322000ec4c51SChris Mason int push_space = 0; 322100ec4c51SChris Mason int push_items = 0; 322234a38218SChris Mason u32 nr; 32237518a238SChris Mason u32 right_nritems; 32245f39d397SChris Mason u32 data_end; 3225db94535dSChris Mason u32 this_item_size; 322600ec4c51SChris Mason 322734a38218SChris Mason if (empty) 322834a38218SChris Mason nr = 0; 322934a38218SChris Mason else 323099d8f83cSChris Mason nr = max_t(u32, 1, min_slot); 323134a38218SChris Mason 323231840ae1SZheng Yan if (path->slots[0] >= left_nritems) 323387b29b20SYan Zheng push_space += data_size; 323431840ae1SZheng Yan 323544871b1bSChris Mason slot = path->slots[1]; 323634a38218SChris Mason i = left_nritems - 1; 323734a38218SChris Mason while (i >= nr) { 323831840ae1SZheng Yan if (!empty && push_items > 0) { 323931840ae1SZheng Yan if (path->slots[0] > i) 324031840ae1SZheng Yan break; 324131840ae1SZheng Yan if (path->slots[0] == i) { 3242e902baacSDavid Sterba int space = btrfs_leaf_free_space(left); 3243e902baacSDavid Sterba 324431840ae1SZheng Yan if (space + push_space * 2 > free_space) 324531840ae1SZheng Yan break; 324631840ae1SZheng Yan } 324731840ae1SZheng Yan } 324831840ae1SZheng Yan 324900ec4c51SChris Mason if (path->slots[0] == i) 325087b29b20SYan Zheng push_space += data_size; 3251db94535dSChris Mason 32523212fa14SJosef Bacik this_item_size = btrfs_item_size(left, i); 325374794207SJosef Bacik if (this_item_size + sizeof(struct btrfs_item) + 325474794207SJosef Bacik push_space > free_space) 325500ec4c51SChris Mason break; 325631840ae1SZheng Yan 325700ec4c51SChris Mason push_items++; 325874794207SJosef Bacik push_space += this_item_size + sizeof(struct btrfs_item); 325934a38218SChris Mason if (i == 0) 326034a38218SChris Mason break; 326134a38218SChris Mason i--; 3262db94535dSChris Mason } 32635f39d397SChris Mason 3264925baeddSChris Mason if (push_items == 0) 3265925baeddSChris Mason goto out_unlock; 32665f39d397SChris Mason 32676c1500f2SJulia Lawall WARN_ON(!empty && push_items == left_nritems); 32685f39d397SChris Mason 326900ec4c51SChris Mason /* push left to right */ 32705f39d397SChris Mason right_nritems = btrfs_header_nritems(right); 327134a38218SChris Mason 3272dc2e724eSJosef Bacik push_space = btrfs_item_data_end(left, left_nritems - push_items); 32738f881e8cSDavid Sterba push_space -= leaf_data_end(left); 32745f39d397SChris Mason 327500ec4c51SChris Mason /* make room in the right data area */ 32768f881e8cSDavid Sterba data_end = leaf_data_end(right); 3277637e3b48SJosef Bacik memmove_leaf_data(right, data_end - push_space, data_end, 32780b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - data_end); 32795f39d397SChris Mason 328000ec4c51SChris Mason /* copy from the left data area */ 3281637e3b48SJosef Bacik copy_leaf_data(right, left, BTRFS_LEAF_DATA_SIZE(fs_info) - push_space, 3282637e3b48SJosef Bacik leaf_data_end(left), push_space); 32835f39d397SChris Mason 3284637e3b48SJosef Bacik memmove_leaf_items(right, push_items, 0, right_nritems); 32855f39d397SChris Mason 328600ec4c51SChris Mason /* copy the items from left to right */ 3287637e3b48SJosef Bacik copy_leaf_items(right, left, 0, left_nritems - push_items, push_items); 328800ec4c51SChris Mason 328900ec4c51SChris Mason /* update the item pointers */ 3290c82f823cSDavid Sterba btrfs_init_map_token(&token, right); 32917518a238SChris Mason right_nritems += push_items; 32925f39d397SChris Mason btrfs_set_header_nritems(right, right_nritems); 32930b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info); 32947518a238SChris Mason for (i = 0; i < right_nritems; i++) { 32953212fa14SJosef Bacik push_space -= btrfs_token_item_size(&token, i); 32963212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, push_space); 3297db94535dSChris Mason } 3298db94535dSChris Mason 32997518a238SChris Mason left_nritems -= push_items; 33005f39d397SChris Mason btrfs_set_header_nritems(left, left_nritems); 330100ec4c51SChris Mason 330234a38218SChris Mason if (left_nritems) 33035f39d397SChris Mason btrfs_mark_buffer_dirty(left); 3304f0486c68SYan, Zheng else 3305190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, left); 3306f0486c68SYan, Zheng 33075f39d397SChris Mason btrfs_mark_buffer_dirty(right); 3308a429e513SChris Mason 33095f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 33105f39d397SChris Mason btrfs_set_node_key(upper, &disk_key, slot + 1); 3311d6025579SChris Mason btrfs_mark_buffer_dirty(upper); 331202217ed2SChris Mason 331300ec4c51SChris Mason /* then fixup the leaf pointer in the path */ 33147518a238SChris Mason if (path->slots[0] >= left_nritems) { 33157518a238SChris Mason path->slots[0] -= left_nritems; 3316925baeddSChris Mason if (btrfs_header_nritems(path->nodes[0]) == 0) 3317190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, path->nodes[0]); 3318925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 33195f39d397SChris Mason free_extent_buffer(path->nodes[0]); 33205f39d397SChris Mason path->nodes[0] = right; 332100ec4c51SChris Mason path->slots[1] += 1; 332200ec4c51SChris Mason } else { 3323925baeddSChris Mason btrfs_tree_unlock(right); 33245f39d397SChris Mason free_extent_buffer(right); 332500ec4c51SChris Mason } 332600ec4c51SChris Mason return 0; 3327925baeddSChris Mason 3328925baeddSChris Mason out_unlock: 3329925baeddSChris Mason btrfs_tree_unlock(right); 3330925baeddSChris Mason free_extent_buffer(right); 3331925baeddSChris Mason return 1; 333200ec4c51SChris Mason } 3333925baeddSChris Mason 333400ec4c51SChris Mason /* 333544871b1bSChris Mason * push some data in the path leaf to the right, trying to free up at 333674123bd7SChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 333744871b1bSChris Mason * 333844871b1bSChris Mason * returns 1 if the push failed because the other node didn't have enough 333944871b1bSChris Mason * room, 0 if everything worked out and < 0 if there were major errors. 334099d8f83cSChris Mason * 334199d8f83cSChris Mason * this will push starting from min_slot to the end of the leaf. It won't 334299d8f83cSChris Mason * push any slot lower than min_slot 334374123bd7SChris Mason */ 334444871b1bSChris Mason static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root 334599d8f83cSChris Mason *root, struct btrfs_path *path, 334699d8f83cSChris Mason int min_data_size, int data_size, 334799d8f83cSChris Mason int empty, u32 min_slot) 3348be0e5c09SChris Mason { 334944871b1bSChris Mason struct extent_buffer *left = path->nodes[0]; 335044871b1bSChris Mason struct extent_buffer *right; 335144871b1bSChris Mason struct extent_buffer *upper; 335244871b1bSChris Mason int slot; 335344871b1bSChris Mason int free_space; 335444871b1bSChris Mason u32 left_nritems; 335544871b1bSChris Mason int ret; 335644871b1bSChris Mason 335744871b1bSChris Mason if (!path->nodes[1]) 335844871b1bSChris Mason return 1; 335944871b1bSChris Mason 336044871b1bSChris Mason slot = path->slots[1]; 336144871b1bSChris Mason upper = path->nodes[1]; 336244871b1bSChris Mason if (slot >= btrfs_header_nritems(upper) - 1) 336344871b1bSChris Mason return 1; 336444871b1bSChris Mason 336549d0c642SFilipe Manana btrfs_assert_tree_write_locked(path->nodes[1]); 336644871b1bSChris Mason 33674b231ae4SDavid Sterba right = btrfs_read_node_slot(upper, slot + 1); 3368fb770ae4SLiu Bo if (IS_ERR(right)) 33699cf14029SJosef Bacik return PTR_ERR(right); 337091ca338dSTsutomu Itoh 3371bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT); 337244871b1bSChris Mason 3373e902baacSDavid Sterba free_space = btrfs_leaf_free_space(right); 337444871b1bSChris Mason if (free_space < data_size) 337544871b1bSChris Mason goto out_unlock; 337644871b1bSChris Mason 337744871b1bSChris Mason ret = btrfs_cow_block(trans, root, right, upper, 3378bf59a5a2SJosef Bacik slot + 1, &right, BTRFS_NESTING_RIGHT_COW); 337944871b1bSChris Mason if (ret) 338044871b1bSChris Mason goto out_unlock; 338144871b1bSChris Mason 338244871b1bSChris Mason left_nritems = btrfs_header_nritems(left); 338344871b1bSChris Mason if (left_nritems == 0) 338444871b1bSChris Mason goto out_unlock; 338544871b1bSChris Mason 3386d16c702fSQu Wenruo if (check_sibling_keys(left, right)) { 3387d16c702fSQu Wenruo ret = -EUCLEAN; 33889ae5afd0SFilipe Manana btrfs_abort_transaction(trans, ret); 3389d16c702fSQu Wenruo btrfs_tree_unlock(right); 3390d16c702fSQu Wenruo free_extent_buffer(right); 3391d16c702fSQu Wenruo return ret; 3392d16c702fSQu Wenruo } 33932ef1fed2SFilipe David Borba Manana if (path->slots[0] == left_nritems && !empty) { 33942ef1fed2SFilipe David Borba Manana /* Key greater than all keys in the leaf, right neighbor has 33952ef1fed2SFilipe David Borba Manana * enough room for it and we're not emptying our leaf to delete 33962ef1fed2SFilipe David Borba Manana * it, therefore use right neighbor to insert the new item and 339752042d8eSAndrea Gelmini * no need to touch/dirty our left leaf. */ 33982ef1fed2SFilipe David Borba Manana btrfs_tree_unlock(left); 33992ef1fed2SFilipe David Borba Manana free_extent_buffer(left); 34002ef1fed2SFilipe David Borba Manana path->nodes[0] = right; 34012ef1fed2SFilipe David Borba Manana path->slots[0] = 0; 34022ef1fed2SFilipe David Borba Manana path->slots[1]++; 34032ef1fed2SFilipe David Borba Manana return 0; 34042ef1fed2SFilipe David Borba Manana } 34052ef1fed2SFilipe David Borba Manana 3406ed25dab3SJosef Bacik return __push_leaf_right(trans, path, min_data_size, empty, right, 3407ed25dab3SJosef Bacik free_space, left_nritems, min_slot); 340844871b1bSChris Mason out_unlock: 340944871b1bSChris Mason btrfs_tree_unlock(right); 341044871b1bSChris Mason free_extent_buffer(right); 341144871b1bSChris Mason return 1; 341244871b1bSChris Mason } 341344871b1bSChris Mason 341444871b1bSChris Mason /* 341544871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 341644871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 341799d8f83cSChris Mason * 341899d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 341999d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us do all the 342099d8f83cSChris Mason * items 342144871b1bSChris Mason */ 3422ed25dab3SJosef Bacik static noinline int __push_leaf_left(struct btrfs_trans_handle *trans, 3423ed25dab3SJosef Bacik struct btrfs_path *path, int data_size, 342444871b1bSChris Mason int empty, struct extent_buffer *left, 342599d8f83cSChris Mason int free_space, u32 right_nritems, 342699d8f83cSChris Mason u32 max_slot) 342744871b1bSChris Mason { 34288087c193SDavid Sterba struct btrfs_fs_info *fs_info = left->fs_info; 34295f39d397SChris Mason struct btrfs_disk_key disk_key; 34305f39d397SChris Mason struct extent_buffer *right = path->nodes[0]; 3431be0e5c09SChris Mason int i; 3432be0e5c09SChris Mason int push_space = 0; 3433be0e5c09SChris Mason int push_items = 0; 34347518a238SChris Mason u32 old_left_nritems; 343534a38218SChris Mason u32 nr; 3436aa5d6bedSChris Mason int ret = 0; 3437db94535dSChris Mason u32 this_item_size; 3438db94535dSChris Mason u32 old_left_item_size; 3439cfed81a0SChris Mason struct btrfs_map_token token; 3440cfed81a0SChris Mason 344134a38218SChris Mason if (empty) 344299d8f83cSChris Mason nr = min(right_nritems, max_slot); 344334a38218SChris Mason else 344499d8f83cSChris Mason nr = min(right_nritems - 1, max_slot); 344534a38218SChris Mason 344634a38218SChris Mason for (i = 0; i < nr; i++) { 344731840ae1SZheng Yan if (!empty && push_items > 0) { 344831840ae1SZheng Yan if (path->slots[0] < i) 344931840ae1SZheng Yan break; 345031840ae1SZheng Yan if (path->slots[0] == i) { 3451e902baacSDavid Sterba int space = btrfs_leaf_free_space(right); 3452e902baacSDavid Sterba 345331840ae1SZheng Yan if (space + push_space * 2 > free_space) 345431840ae1SZheng Yan break; 345531840ae1SZheng Yan } 345631840ae1SZheng Yan } 345731840ae1SZheng Yan 3458be0e5c09SChris Mason if (path->slots[0] == i) 345987b29b20SYan Zheng push_space += data_size; 3460db94535dSChris Mason 34613212fa14SJosef Bacik this_item_size = btrfs_item_size(right, i); 346274794207SJosef Bacik if (this_item_size + sizeof(struct btrfs_item) + push_space > 346374794207SJosef Bacik free_space) 3464be0e5c09SChris Mason break; 3465db94535dSChris Mason 3466be0e5c09SChris Mason push_items++; 346774794207SJosef Bacik push_space += this_item_size + sizeof(struct btrfs_item); 3468be0e5c09SChris Mason } 3469db94535dSChris Mason 3470be0e5c09SChris Mason if (push_items == 0) { 3471925baeddSChris Mason ret = 1; 3472925baeddSChris Mason goto out; 3473be0e5c09SChris Mason } 3474fae7f21cSDulshani Gunawardhana WARN_ON(!empty && push_items == btrfs_header_nritems(right)); 34755f39d397SChris Mason 3476be0e5c09SChris Mason /* push data from right to left */ 3477637e3b48SJosef Bacik copy_leaf_items(left, right, btrfs_header_nritems(left), 0, push_items); 34785f39d397SChris Mason 34790b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info) - 34803212fa14SJosef Bacik btrfs_item_offset(right, push_items - 1); 34815f39d397SChris Mason 3482637e3b48SJosef Bacik copy_leaf_data(left, right, leaf_data_end(left) - push_space, 3483637e3b48SJosef Bacik btrfs_item_offset(right, push_items - 1), push_space); 34845f39d397SChris Mason old_left_nritems = btrfs_header_nritems(left); 348587b29b20SYan Zheng BUG_ON(old_left_nritems <= 0); 3486eb60ceacSChris Mason 3487c82f823cSDavid Sterba btrfs_init_map_token(&token, left); 34883212fa14SJosef Bacik old_left_item_size = btrfs_item_offset(left, old_left_nritems - 1); 3489be0e5c09SChris Mason for (i = old_left_nritems; i < old_left_nritems + push_items; i++) { 34905f39d397SChris Mason u32 ioff; 3491db94535dSChris Mason 34923212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 34933212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, 3494cc4c13d5SDavid Sterba ioff - (BTRFS_LEAF_DATA_SIZE(fs_info) - old_left_item_size)); 3495be0e5c09SChris Mason } 34965f39d397SChris Mason btrfs_set_header_nritems(left, old_left_nritems + push_items); 3497be0e5c09SChris Mason 3498be0e5c09SChris Mason /* fixup right node */ 349931b1a2bdSJulia Lawall if (push_items > right_nritems) 350031b1a2bdSJulia Lawall WARN(1, KERN_CRIT "push items %d nr %u\n", push_items, 3501d397712bSChris Mason right_nritems); 350234a38218SChris Mason 350334a38218SChris Mason if (push_items < right_nritems) { 35043212fa14SJosef Bacik push_space = btrfs_item_offset(right, push_items - 1) - 35058f881e8cSDavid Sterba leaf_data_end(right); 3506637e3b48SJosef Bacik memmove_leaf_data(right, 35070b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - push_space, 35088f881e8cSDavid Sterba leaf_data_end(right), push_space); 35095f39d397SChris Mason 3510637e3b48SJosef Bacik memmove_leaf_items(right, 0, push_items, 3511637e3b48SJosef Bacik btrfs_header_nritems(right) - push_items); 351234a38218SChris Mason } 3513c82f823cSDavid Sterba 3514c82f823cSDavid Sterba btrfs_init_map_token(&token, right); 3515eef1c494SYan right_nritems -= push_items; 3516eef1c494SYan btrfs_set_header_nritems(right, right_nritems); 35170b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info); 35185f39d397SChris Mason for (i = 0; i < right_nritems; i++) { 35193212fa14SJosef Bacik push_space = push_space - btrfs_token_item_size(&token, i); 35203212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, push_space); 3521db94535dSChris Mason } 3522eb60ceacSChris Mason 35235f39d397SChris Mason btrfs_mark_buffer_dirty(left); 352434a38218SChris Mason if (right_nritems) 35255f39d397SChris Mason btrfs_mark_buffer_dirty(right); 3526f0486c68SYan, Zheng else 3527190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, right); 3528098f59c2SChris Mason 35295f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 3530b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 3531be0e5c09SChris Mason 3532be0e5c09SChris Mason /* then fixup the leaf pointer in the path */ 3533be0e5c09SChris Mason if (path->slots[0] < push_items) { 3534be0e5c09SChris Mason path->slots[0] += old_left_nritems; 3535925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 35365f39d397SChris Mason free_extent_buffer(path->nodes[0]); 35375f39d397SChris Mason path->nodes[0] = left; 3538be0e5c09SChris Mason path->slots[1] -= 1; 3539be0e5c09SChris Mason } else { 3540925baeddSChris Mason btrfs_tree_unlock(left); 35415f39d397SChris Mason free_extent_buffer(left); 3542be0e5c09SChris Mason path->slots[0] -= push_items; 3543be0e5c09SChris Mason } 3544eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 3545aa5d6bedSChris Mason return ret; 3546925baeddSChris Mason out: 3547925baeddSChris Mason btrfs_tree_unlock(left); 3548925baeddSChris Mason free_extent_buffer(left); 3549925baeddSChris Mason return ret; 3550be0e5c09SChris Mason } 3551be0e5c09SChris Mason 355274123bd7SChris Mason /* 355344871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 355444871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 355599d8f83cSChris Mason * 355699d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 355799d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us push all the 355899d8f83cSChris Mason * items 355944871b1bSChris Mason */ 356044871b1bSChris Mason static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root 356199d8f83cSChris Mason *root, struct btrfs_path *path, int min_data_size, 356299d8f83cSChris Mason int data_size, int empty, u32 max_slot) 356344871b1bSChris Mason { 356444871b1bSChris Mason struct extent_buffer *right = path->nodes[0]; 356544871b1bSChris Mason struct extent_buffer *left; 356644871b1bSChris Mason int slot; 356744871b1bSChris Mason int free_space; 356844871b1bSChris Mason u32 right_nritems; 356944871b1bSChris Mason int ret = 0; 357044871b1bSChris Mason 357144871b1bSChris Mason slot = path->slots[1]; 357244871b1bSChris Mason if (slot == 0) 357344871b1bSChris Mason return 1; 357444871b1bSChris Mason if (!path->nodes[1]) 357544871b1bSChris Mason return 1; 357644871b1bSChris Mason 357744871b1bSChris Mason right_nritems = btrfs_header_nritems(right); 357844871b1bSChris Mason if (right_nritems == 0) 357944871b1bSChris Mason return 1; 358044871b1bSChris Mason 358149d0c642SFilipe Manana btrfs_assert_tree_write_locked(path->nodes[1]); 358244871b1bSChris Mason 35834b231ae4SDavid Sterba left = btrfs_read_node_slot(path->nodes[1], slot - 1); 3584fb770ae4SLiu Bo if (IS_ERR(left)) 35859cf14029SJosef Bacik return PTR_ERR(left); 358691ca338dSTsutomu Itoh 3587bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT); 358844871b1bSChris Mason 3589e902baacSDavid Sterba free_space = btrfs_leaf_free_space(left); 359044871b1bSChris Mason if (free_space < data_size) { 359144871b1bSChris Mason ret = 1; 359244871b1bSChris Mason goto out; 359344871b1bSChris Mason } 359444871b1bSChris Mason 359544871b1bSChris Mason ret = btrfs_cow_block(trans, root, left, 35969631e4ccSJosef Bacik path->nodes[1], slot - 1, &left, 3597bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW); 359844871b1bSChris Mason if (ret) { 359944871b1bSChris Mason /* we hit -ENOSPC, but it isn't fatal here */ 360079787eaaSJeff Mahoney if (ret == -ENOSPC) 360144871b1bSChris Mason ret = 1; 360244871b1bSChris Mason goto out; 360344871b1bSChris Mason } 360444871b1bSChris Mason 3605d16c702fSQu Wenruo if (check_sibling_keys(left, right)) { 3606d16c702fSQu Wenruo ret = -EUCLEAN; 36079ae5afd0SFilipe Manana btrfs_abort_transaction(trans, ret); 3608d16c702fSQu Wenruo goto out; 3609d16c702fSQu Wenruo } 3610ed25dab3SJosef Bacik return __push_leaf_left(trans, path, min_data_size, empty, left, 3611ed25dab3SJosef Bacik free_space, right_nritems, max_slot); 361244871b1bSChris Mason out: 361344871b1bSChris Mason btrfs_tree_unlock(left); 361444871b1bSChris Mason free_extent_buffer(left); 361544871b1bSChris Mason return ret; 361644871b1bSChris Mason } 361744871b1bSChris Mason 361844871b1bSChris Mason /* 361974123bd7SChris Mason * split the path's leaf in two, making sure there is at least data_size 362074123bd7SChris Mason * available for the resulting leaf level of the path. 362174123bd7SChris Mason */ 362250b5d1fcSFilipe Manana static noinline int copy_for_split(struct btrfs_trans_handle *trans, 362344871b1bSChris Mason struct btrfs_path *path, 362444871b1bSChris Mason struct extent_buffer *l, 362544871b1bSChris Mason struct extent_buffer *right, 362644871b1bSChris Mason int slot, int mid, int nritems) 3627be0e5c09SChris Mason { 362894f94ad9SDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info; 3629be0e5c09SChris Mason int data_copy_size; 3630be0e5c09SChris Mason int rt_data_off; 3631be0e5c09SChris Mason int i; 363250b5d1fcSFilipe Manana int ret; 3633d4dbff95SChris Mason struct btrfs_disk_key disk_key; 3634cfed81a0SChris Mason struct btrfs_map_token token; 3635cfed81a0SChris Mason 36365f39d397SChris Mason nritems = nritems - mid; 36375f39d397SChris Mason btrfs_set_header_nritems(right, nritems); 3638dc2e724eSJosef Bacik data_copy_size = btrfs_item_data_end(l, mid) - leaf_data_end(l); 36395f39d397SChris Mason 3640637e3b48SJosef Bacik copy_leaf_items(right, l, 0, mid, nritems); 36415f39d397SChris Mason 3642637e3b48SJosef Bacik copy_leaf_data(right, l, BTRFS_LEAF_DATA_SIZE(fs_info) - data_copy_size, 36438f881e8cSDavid Sterba leaf_data_end(l), data_copy_size); 364474123bd7SChris Mason 3645dc2e724eSJosef Bacik rt_data_off = BTRFS_LEAF_DATA_SIZE(fs_info) - btrfs_item_data_end(l, mid); 36465f39d397SChris Mason 3647c82f823cSDavid Sterba btrfs_init_map_token(&token, right); 36485f39d397SChris Mason for (i = 0; i < nritems; i++) { 3649db94535dSChris Mason u32 ioff; 3650db94535dSChris Mason 36513212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 36523212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff + rt_data_off); 36530783fcfcSChris Mason } 365474123bd7SChris Mason 36555f39d397SChris Mason btrfs_set_header_nritems(l, mid); 36565f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 365750b5d1fcSFilipe Manana ret = insert_ptr(trans, path, &disk_key, right->start, path->slots[1] + 1, 1); 365850b5d1fcSFilipe Manana if (ret < 0) 365950b5d1fcSFilipe Manana return ret; 36605f39d397SChris Mason 36615f39d397SChris Mason btrfs_mark_buffer_dirty(right); 36625f39d397SChris Mason btrfs_mark_buffer_dirty(l); 3663eb60ceacSChris Mason BUG_ON(path->slots[0] != slot); 36645f39d397SChris Mason 3665be0e5c09SChris Mason if (mid <= slot) { 3666925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 36675f39d397SChris Mason free_extent_buffer(path->nodes[0]); 36685f39d397SChris Mason path->nodes[0] = right; 3669be0e5c09SChris Mason path->slots[0] -= mid; 3670be0e5c09SChris Mason path->slots[1] += 1; 3671925baeddSChris Mason } else { 3672925baeddSChris Mason btrfs_tree_unlock(right); 36735f39d397SChris Mason free_extent_buffer(right); 3674925baeddSChris Mason } 36755f39d397SChris Mason 3676eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 367750b5d1fcSFilipe Manana 367850b5d1fcSFilipe Manana return 0; 367944871b1bSChris Mason } 368044871b1bSChris Mason 368144871b1bSChris Mason /* 368299d8f83cSChris Mason * double splits happen when we need to insert a big item in the middle 368399d8f83cSChris Mason * of a leaf. A double split can leave us with 3 mostly empty leaves: 368499d8f83cSChris Mason * leaf: [ slots 0 - N] [ our target ] [ N + 1 - total in leaf ] 368599d8f83cSChris Mason * A B C 368699d8f83cSChris Mason * 368799d8f83cSChris Mason * We avoid this by trying to push the items on either side of our target 368899d8f83cSChris Mason * into the adjacent leaves. If all goes well we can avoid the double split 368999d8f83cSChris Mason * completely. 369099d8f83cSChris Mason */ 369199d8f83cSChris Mason static noinline int push_for_double_split(struct btrfs_trans_handle *trans, 369299d8f83cSChris Mason struct btrfs_root *root, 369399d8f83cSChris Mason struct btrfs_path *path, 369499d8f83cSChris Mason int data_size) 369599d8f83cSChris Mason { 369699d8f83cSChris Mason int ret; 369799d8f83cSChris Mason int progress = 0; 369899d8f83cSChris Mason int slot; 369999d8f83cSChris Mason u32 nritems; 37005a4267caSFilipe David Borba Manana int space_needed = data_size; 370199d8f83cSChris Mason 370299d8f83cSChris Mason slot = path->slots[0]; 37035a4267caSFilipe David Borba Manana if (slot < btrfs_header_nritems(path->nodes[0])) 3704e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(path->nodes[0]); 370599d8f83cSChris Mason 370699d8f83cSChris Mason /* 370799d8f83cSChris Mason * try to push all the items after our slot into the 370899d8f83cSChris Mason * right leaf 370999d8f83cSChris Mason */ 37105a4267caSFilipe David Borba Manana ret = push_leaf_right(trans, root, path, 1, space_needed, 0, slot); 371199d8f83cSChris Mason if (ret < 0) 371299d8f83cSChris Mason return ret; 371399d8f83cSChris Mason 371499d8f83cSChris Mason if (ret == 0) 371599d8f83cSChris Mason progress++; 371699d8f83cSChris Mason 371799d8f83cSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 371899d8f83cSChris Mason /* 371999d8f83cSChris Mason * our goal is to get our slot at the start or end of a leaf. If 372099d8f83cSChris Mason * we've done so we're done 372199d8f83cSChris Mason */ 372299d8f83cSChris Mason if (path->slots[0] == 0 || path->slots[0] == nritems) 372399d8f83cSChris Mason return 0; 372499d8f83cSChris Mason 3725e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= data_size) 372699d8f83cSChris Mason return 0; 372799d8f83cSChris Mason 372899d8f83cSChris Mason /* try to push all the items before our slot into the next leaf */ 372999d8f83cSChris Mason slot = path->slots[0]; 3730263d3995SFilipe Manana space_needed = data_size; 3731263d3995SFilipe Manana if (slot > 0) 3732e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(path->nodes[0]); 37335a4267caSFilipe David Borba Manana ret = push_leaf_left(trans, root, path, 1, space_needed, 0, slot); 373499d8f83cSChris Mason if (ret < 0) 373599d8f83cSChris Mason return ret; 373699d8f83cSChris Mason 373799d8f83cSChris Mason if (ret == 0) 373899d8f83cSChris Mason progress++; 373999d8f83cSChris Mason 374099d8f83cSChris Mason if (progress) 374199d8f83cSChris Mason return 0; 374299d8f83cSChris Mason return 1; 374399d8f83cSChris Mason } 374499d8f83cSChris Mason 374599d8f83cSChris Mason /* 374644871b1bSChris Mason * split the path's leaf in two, making sure there is at least data_size 374744871b1bSChris Mason * available for the resulting leaf level of the path. 374844871b1bSChris Mason * 374944871b1bSChris Mason * returns 0 if all went well and < 0 on failure. 375044871b1bSChris Mason */ 375144871b1bSChris Mason static noinline int split_leaf(struct btrfs_trans_handle *trans, 375244871b1bSChris Mason struct btrfs_root *root, 3753310712b2SOmar Sandoval const struct btrfs_key *ins_key, 375444871b1bSChris Mason struct btrfs_path *path, int data_size, 375544871b1bSChris Mason int extend) 375644871b1bSChris Mason { 37575d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 375844871b1bSChris Mason struct extent_buffer *l; 375944871b1bSChris Mason u32 nritems; 376044871b1bSChris Mason int mid; 376144871b1bSChris Mason int slot; 376244871b1bSChris Mason struct extent_buffer *right; 3763b7a0365eSDaniel Dressler struct btrfs_fs_info *fs_info = root->fs_info; 376444871b1bSChris Mason int ret = 0; 376544871b1bSChris Mason int wret; 37665d4f98a2SYan Zheng int split; 376744871b1bSChris Mason int num_doubles = 0; 376899d8f83cSChris Mason int tried_avoid_double = 0; 376944871b1bSChris Mason 3770a5719521SYan, Zheng l = path->nodes[0]; 3771a5719521SYan, Zheng slot = path->slots[0]; 37723212fa14SJosef Bacik if (extend && data_size + btrfs_item_size(l, slot) + 37730b246afaSJeff Mahoney sizeof(struct btrfs_item) > BTRFS_LEAF_DATA_SIZE(fs_info)) 3774a5719521SYan, Zheng return -EOVERFLOW; 3775a5719521SYan, Zheng 377644871b1bSChris Mason /* first try to make some room by pushing left and right */ 377733157e05SLiu Bo if (data_size && path->nodes[1]) { 37785a4267caSFilipe David Borba Manana int space_needed = data_size; 37795a4267caSFilipe David Borba Manana 37805a4267caSFilipe David Borba Manana if (slot < btrfs_header_nritems(l)) 3781e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(l); 37825a4267caSFilipe David Borba Manana 37835a4267caSFilipe David Borba Manana wret = push_leaf_right(trans, root, path, space_needed, 37845a4267caSFilipe David Borba Manana space_needed, 0, 0); 378544871b1bSChris Mason if (wret < 0) 378644871b1bSChris Mason return wret; 378744871b1bSChris Mason if (wret) { 3788263d3995SFilipe Manana space_needed = data_size; 3789263d3995SFilipe Manana if (slot > 0) 3790e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(l); 37915a4267caSFilipe David Borba Manana wret = push_leaf_left(trans, root, path, space_needed, 37925a4267caSFilipe David Borba Manana space_needed, 0, (u32)-1); 379344871b1bSChris Mason if (wret < 0) 379444871b1bSChris Mason return wret; 379544871b1bSChris Mason } 379644871b1bSChris Mason l = path->nodes[0]; 379744871b1bSChris Mason 379844871b1bSChris Mason /* did the pushes work? */ 3799e902baacSDavid Sterba if (btrfs_leaf_free_space(l) >= data_size) 380044871b1bSChris Mason return 0; 380144871b1bSChris Mason } 380244871b1bSChris Mason 380344871b1bSChris Mason if (!path->nodes[1]) { 3804fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, 1); 380544871b1bSChris Mason if (ret) 380644871b1bSChris Mason return ret; 380744871b1bSChris Mason } 380844871b1bSChris Mason again: 38095d4f98a2SYan Zheng split = 1; 381044871b1bSChris Mason l = path->nodes[0]; 381144871b1bSChris Mason slot = path->slots[0]; 381244871b1bSChris Mason nritems = btrfs_header_nritems(l); 381344871b1bSChris Mason mid = (nritems + 1) / 2; 381444871b1bSChris Mason 38155d4f98a2SYan Zheng if (mid <= slot) { 38165d4f98a2SYan Zheng if (nritems == 1 || 38175d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + data_size > 38180b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info)) { 38195d4f98a2SYan Zheng if (slot >= nritems) { 38205d4f98a2SYan Zheng split = 0; 38215d4f98a2SYan Zheng } else { 38225d4f98a2SYan Zheng mid = slot; 38235d4f98a2SYan Zheng if (mid != nritems && 38245d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 38250b246afaSJeff Mahoney data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) { 382699d8f83cSChris Mason if (data_size && !tried_avoid_double) 382799d8f83cSChris Mason goto push_for_double; 38285d4f98a2SYan Zheng split = 2; 38295d4f98a2SYan Zheng } 38305d4f98a2SYan Zheng } 38315d4f98a2SYan Zheng } 38325d4f98a2SYan Zheng } else { 38335d4f98a2SYan Zheng if (leaf_space_used(l, 0, mid) + data_size > 38340b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info)) { 38355d4f98a2SYan Zheng if (!extend && data_size && slot == 0) { 38365d4f98a2SYan Zheng split = 0; 38375d4f98a2SYan Zheng } else if ((extend || !data_size) && slot == 0) { 38385d4f98a2SYan Zheng mid = 1; 38395d4f98a2SYan Zheng } else { 38405d4f98a2SYan Zheng mid = slot; 38415d4f98a2SYan Zheng if (mid != nritems && 38425d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 38430b246afaSJeff Mahoney data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) { 384499d8f83cSChris Mason if (data_size && !tried_avoid_double) 384599d8f83cSChris Mason goto push_for_double; 38465d4f98a2SYan Zheng split = 2; 38475d4f98a2SYan Zheng } 38485d4f98a2SYan Zheng } 38495d4f98a2SYan Zheng } 38505d4f98a2SYan Zheng } 38515d4f98a2SYan Zheng 38525d4f98a2SYan Zheng if (split == 0) 38535d4f98a2SYan Zheng btrfs_cpu_key_to_disk(&disk_key, ins_key); 38545d4f98a2SYan Zheng else 38555d4f98a2SYan Zheng btrfs_item_key(l, &disk_key, mid); 38565d4f98a2SYan Zheng 3857ca9d473aSJosef Bacik /* 3858ca9d473aSJosef Bacik * We have to about BTRFS_NESTING_NEW_ROOT here if we've done a double 3859ca9d473aSJosef Bacik * split, because we're only allowed to have MAX_LOCKDEP_SUBCLASSES 3860ca9d473aSJosef Bacik * subclasses, which is 8 at the time of this patch, and we've maxed it 3861ca9d473aSJosef Bacik * out. In the future we could add a 3862ca9d473aSJosef Bacik * BTRFS_NESTING_SPLIT_THE_SPLITTENING if we need to, but for now just 3863ca9d473aSJosef Bacik * use BTRFS_NESTING_NEW_ROOT. 3864ca9d473aSJosef Bacik */ 386579bd3712SFilipe Manana right = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 386679bd3712SFilipe Manana &disk_key, 0, l->start, 0, 386779bd3712SFilipe Manana num_doubles ? BTRFS_NESTING_NEW_ROOT : 3868ca9d473aSJosef Bacik BTRFS_NESTING_SPLIT); 3869f0486c68SYan, Zheng if (IS_ERR(right)) 387044871b1bSChris Mason return PTR_ERR(right); 3871f0486c68SYan, Zheng 38720b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 387344871b1bSChris Mason 38745d4f98a2SYan Zheng if (split == 0) { 387544871b1bSChris Mason if (mid <= slot) { 387644871b1bSChris Mason btrfs_set_header_nritems(right, 0); 387750b5d1fcSFilipe Manana ret = insert_ptr(trans, path, &disk_key, 38782ff7e61eSJeff Mahoney right->start, path->slots[1] + 1, 1); 387950b5d1fcSFilipe Manana if (ret < 0) { 388050b5d1fcSFilipe Manana btrfs_tree_unlock(right); 388150b5d1fcSFilipe Manana free_extent_buffer(right); 388250b5d1fcSFilipe Manana return ret; 388350b5d1fcSFilipe Manana } 388444871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 388544871b1bSChris Mason free_extent_buffer(path->nodes[0]); 388644871b1bSChris Mason path->nodes[0] = right; 388744871b1bSChris Mason path->slots[0] = 0; 388844871b1bSChris Mason path->slots[1] += 1; 388944871b1bSChris Mason } else { 389044871b1bSChris Mason btrfs_set_header_nritems(right, 0); 389150b5d1fcSFilipe Manana ret = insert_ptr(trans, path, &disk_key, 38922ff7e61eSJeff Mahoney right->start, path->slots[1], 1); 389350b5d1fcSFilipe Manana if (ret < 0) { 389450b5d1fcSFilipe Manana btrfs_tree_unlock(right); 389550b5d1fcSFilipe Manana free_extent_buffer(right); 389650b5d1fcSFilipe Manana return ret; 389750b5d1fcSFilipe Manana } 389844871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 389944871b1bSChris Mason free_extent_buffer(path->nodes[0]); 390044871b1bSChris Mason path->nodes[0] = right; 390144871b1bSChris Mason path->slots[0] = 0; 3902143bede5SJeff Mahoney if (path->slots[1] == 0) 3903b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 39045d4f98a2SYan Zheng } 3905196e0249SLiu Bo /* 3906196e0249SLiu Bo * We create a new leaf 'right' for the required ins_len and 3907196e0249SLiu Bo * we'll do btrfs_mark_buffer_dirty() on this leaf after copying 3908196e0249SLiu Bo * the content of ins_len to 'right'. 3909196e0249SLiu Bo */ 391044871b1bSChris Mason return ret; 391144871b1bSChris Mason } 391244871b1bSChris Mason 391350b5d1fcSFilipe Manana ret = copy_for_split(trans, path, l, right, slot, mid, nritems); 391450b5d1fcSFilipe Manana if (ret < 0) { 391550b5d1fcSFilipe Manana btrfs_tree_unlock(right); 391650b5d1fcSFilipe Manana free_extent_buffer(right); 391750b5d1fcSFilipe Manana return ret; 391850b5d1fcSFilipe Manana } 391944871b1bSChris Mason 39205d4f98a2SYan Zheng if (split == 2) { 3921cc0c5538SChris Mason BUG_ON(num_doubles != 0); 3922cc0c5538SChris Mason num_doubles++; 3923cc0c5538SChris Mason goto again; 39243326d1b0SChris Mason } 392544871b1bSChris Mason 3926143bede5SJeff Mahoney return 0; 392799d8f83cSChris Mason 392899d8f83cSChris Mason push_for_double: 392999d8f83cSChris Mason push_for_double_split(trans, root, path, data_size); 393099d8f83cSChris Mason tried_avoid_double = 1; 3931e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= data_size) 393299d8f83cSChris Mason return 0; 393399d8f83cSChris Mason goto again; 3934be0e5c09SChris Mason } 3935be0e5c09SChris Mason 3936ad48fd75SYan, Zheng static noinline int setup_leaf_for_split(struct btrfs_trans_handle *trans, 3937ad48fd75SYan, Zheng struct btrfs_root *root, 3938ad48fd75SYan, Zheng struct btrfs_path *path, int ins_len) 3939ad48fd75SYan, Zheng { 3940ad48fd75SYan, Zheng struct btrfs_key key; 3941ad48fd75SYan, Zheng struct extent_buffer *leaf; 3942ad48fd75SYan, Zheng struct btrfs_file_extent_item *fi; 3943ad48fd75SYan, Zheng u64 extent_len = 0; 3944ad48fd75SYan, Zheng u32 item_size; 3945ad48fd75SYan, Zheng int ret; 3946ad48fd75SYan, Zheng 3947ad48fd75SYan, Zheng leaf = path->nodes[0]; 3948ad48fd75SYan, Zheng btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 3949ad48fd75SYan, Zheng 3950ad48fd75SYan, Zheng BUG_ON(key.type != BTRFS_EXTENT_DATA_KEY && 3951ad48fd75SYan, Zheng key.type != BTRFS_EXTENT_CSUM_KEY); 3952ad48fd75SYan, Zheng 3953e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) >= ins_len) 3954ad48fd75SYan, Zheng return 0; 3955ad48fd75SYan, Zheng 39563212fa14SJosef Bacik item_size = btrfs_item_size(leaf, path->slots[0]); 3957ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 3958ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 3959ad48fd75SYan, Zheng struct btrfs_file_extent_item); 3960ad48fd75SYan, Zheng extent_len = btrfs_file_extent_num_bytes(leaf, fi); 3961ad48fd75SYan, Zheng } 3962b3b4aa74SDavid Sterba btrfs_release_path(path); 3963ad48fd75SYan, Zheng 3964ad48fd75SYan, Zheng path->keep_locks = 1; 3965ad48fd75SYan, Zheng path->search_for_split = 1; 3966ad48fd75SYan, Zheng ret = btrfs_search_slot(trans, root, &key, path, 0, 1); 3967ad48fd75SYan, Zheng path->search_for_split = 0; 3968a8df6fe6SFilipe Manana if (ret > 0) 3969a8df6fe6SFilipe Manana ret = -EAGAIN; 3970ad48fd75SYan, Zheng if (ret < 0) 3971ad48fd75SYan, Zheng goto err; 3972ad48fd75SYan, Zheng 3973ad48fd75SYan, Zheng ret = -EAGAIN; 3974ad48fd75SYan, Zheng leaf = path->nodes[0]; 3975a8df6fe6SFilipe Manana /* if our item isn't there, return now */ 39763212fa14SJosef Bacik if (item_size != btrfs_item_size(leaf, path->slots[0])) 3977ad48fd75SYan, Zheng goto err; 3978ad48fd75SYan, Zheng 3979109f6aefSChris Mason /* the leaf has changed, it now has room. return now */ 3980e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= ins_len) 3981109f6aefSChris Mason goto err; 3982109f6aefSChris Mason 3983ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 3984ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 3985ad48fd75SYan, Zheng struct btrfs_file_extent_item); 3986ad48fd75SYan, Zheng if (extent_len != btrfs_file_extent_num_bytes(leaf, fi)) 3987ad48fd75SYan, Zheng goto err; 3988ad48fd75SYan, Zheng } 3989ad48fd75SYan, Zheng 3990ad48fd75SYan, Zheng ret = split_leaf(trans, root, &key, path, ins_len, 1); 3991f0486c68SYan, Zheng if (ret) 3992f0486c68SYan, Zheng goto err; 3993ad48fd75SYan, Zheng 3994ad48fd75SYan, Zheng path->keep_locks = 0; 3995ad48fd75SYan, Zheng btrfs_unlock_up_safe(path, 1); 3996ad48fd75SYan, Zheng return 0; 3997ad48fd75SYan, Zheng err: 3998ad48fd75SYan, Zheng path->keep_locks = 0; 3999ad48fd75SYan, Zheng return ret; 4000ad48fd75SYan, Zheng } 4001ad48fd75SYan, Zheng 400225263cd7SDavid Sterba static noinline int split_item(struct btrfs_path *path, 4003310712b2SOmar Sandoval const struct btrfs_key *new_key, 4004459931ecSChris Mason unsigned long split_offset) 4005459931ecSChris Mason { 4006459931ecSChris Mason struct extent_buffer *leaf; 4007c91666b1SJosef Bacik int orig_slot, slot; 4008ad48fd75SYan, Zheng char *buf; 4009459931ecSChris Mason u32 nritems; 4010ad48fd75SYan, Zheng u32 item_size; 4011459931ecSChris Mason u32 orig_offset; 4012459931ecSChris Mason struct btrfs_disk_key disk_key; 4013459931ecSChris Mason 4014459931ecSChris Mason leaf = path->nodes[0]; 40157569141eSFilipe Manana /* 40167569141eSFilipe Manana * Shouldn't happen because the caller must have previously called 40177569141eSFilipe Manana * setup_leaf_for_split() to make room for the new item in the leaf. 40187569141eSFilipe Manana */ 40197569141eSFilipe Manana if (WARN_ON(btrfs_leaf_free_space(leaf) < sizeof(struct btrfs_item))) 40207569141eSFilipe Manana return -ENOSPC; 4021b9473439SChris Mason 4022c91666b1SJosef Bacik orig_slot = path->slots[0]; 40233212fa14SJosef Bacik orig_offset = btrfs_item_offset(leaf, path->slots[0]); 40243212fa14SJosef Bacik item_size = btrfs_item_size(leaf, path->slots[0]); 4025459931ecSChris Mason 4026459931ecSChris Mason buf = kmalloc(item_size, GFP_NOFS); 4027ad48fd75SYan, Zheng if (!buf) 4028ad48fd75SYan, Zheng return -ENOMEM; 4029ad48fd75SYan, Zheng 4030459931ecSChris Mason read_extent_buffer(leaf, buf, btrfs_item_ptr_offset(leaf, 4031459931ecSChris Mason path->slots[0]), item_size); 4032ad48fd75SYan, Zheng 4033459931ecSChris Mason slot = path->slots[0] + 1; 4034459931ecSChris Mason nritems = btrfs_header_nritems(leaf); 4035459931ecSChris Mason if (slot != nritems) { 4036459931ecSChris Mason /* shift the items */ 4037637e3b48SJosef Bacik memmove_leaf_items(leaf, slot + 1, slot, nritems - slot); 4038459931ecSChris Mason } 4039459931ecSChris Mason 4040459931ecSChris Mason btrfs_cpu_key_to_disk(&disk_key, new_key); 4041459931ecSChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 4042459931ecSChris Mason 40433212fa14SJosef Bacik btrfs_set_item_offset(leaf, slot, orig_offset); 40443212fa14SJosef Bacik btrfs_set_item_size(leaf, slot, item_size - split_offset); 4045459931ecSChris Mason 40463212fa14SJosef Bacik btrfs_set_item_offset(leaf, orig_slot, 4047459931ecSChris Mason orig_offset + item_size - split_offset); 40483212fa14SJosef Bacik btrfs_set_item_size(leaf, orig_slot, split_offset); 4049459931ecSChris Mason 4050459931ecSChris Mason btrfs_set_header_nritems(leaf, nritems + 1); 4051459931ecSChris Mason 4052459931ecSChris Mason /* write the data for the start of the original item */ 4053459931ecSChris Mason write_extent_buffer(leaf, buf, 4054459931ecSChris Mason btrfs_item_ptr_offset(leaf, path->slots[0]), 4055459931ecSChris Mason split_offset); 4056459931ecSChris Mason 4057459931ecSChris Mason /* write the data for the new item */ 4058459931ecSChris Mason write_extent_buffer(leaf, buf + split_offset, 4059459931ecSChris Mason btrfs_item_ptr_offset(leaf, slot), 4060459931ecSChris Mason item_size - split_offset); 4061459931ecSChris Mason btrfs_mark_buffer_dirty(leaf); 4062459931ecSChris Mason 4063e902baacSDavid Sterba BUG_ON(btrfs_leaf_free_space(leaf) < 0); 4064459931ecSChris Mason kfree(buf); 4065ad48fd75SYan, Zheng return 0; 4066ad48fd75SYan, Zheng } 4067ad48fd75SYan, Zheng 4068ad48fd75SYan, Zheng /* 4069ad48fd75SYan, Zheng * This function splits a single item into two items, 4070ad48fd75SYan, Zheng * giving 'new_key' to the new item and splitting the 4071ad48fd75SYan, Zheng * old one at split_offset (from the start of the item). 4072ad48fd75SYan, Zheng * 4073ad48fd75SYan, Zheng * The path may be released by this operation. After 4074ad48fd75SYan, Zheng * the split, the path is pointing to the old item. The 4075ad48fd75SYan, Zheng * new item is going to be in the same node as the old one. 4076ad48fd75SYan, Zheng * 4077ad48fd75SYan, Zheng * Note, the item being split must be smaller enough to live alone on 4078ad48fd75SYan, Zheng * a tree block with room for one extra struct btrfs_item 4079ad48fd75SYan, Zheng * 4080ad48fd75SYan, Zheng * This allows us to split the item in place, keeping a lock on the 4081ad48fd75SYan, Zheng * leaf the entire time. 4082ad48fd75SYan, Zheng */ 4083ad48fd75SYan, Zheng int btrfs_split_item(struct btrfs_trans_handle *trans, 4084ad48fd75SYan, Zheng struct btrfs_root *root, 4085ad48fd75SYan, Zheng struct btrfs_path *path, 4086310712b2SOmar Sandoval const struct btrfs_key *new_key, 4087ad48fd75SYan, Zheng unsigned long split_offset) 4088ad48fd75SYan, Zheng { 4089ad48fd75SYan, Zheng int ret; 4090ad48fd75SYan, Zheng ret = setup_leaf_for_split(trans, root, path, 4091ad48fd75SYan, Zheng sizeof(struct btrfs_item)); 4092ad48fd75SYan, Zheng if (ret) 4093459931ecSChris Mason return ret; 4094ad48fd75SYan, Zheng 409525263cd7SDavid Sterba ret = split_item(path, new_key, split_offset); 4096ad48fd75SYan, Zheng return ret; 4097ad48fd75SYan, Zheng } 4098ad48fd75SYan, Zheng 4099ad48fd75SYan, Zheng /* 4100d352ac68SChris Mason * make the item pointed to by the path smaller. new_size indicates 4101d352ac68SChris Mason * how small to make it, and from_end tells us if we just chop bytes 4102d352ac68SChris Mason * off the end of the item or if we shift the item to chop bytes off 4103d352ac68SChris Mason * the front. 4104d352ac68SChris Mason */ 410578ac4f9eSDavid Sterba void btrfs_truncate_item(struct btrfs_path *path, u32 new_size, int from_end) 4106b18c6685SChris Mason { 4107b18c6685SChris Mason int slot; 41085f39d397SChris Mason struct extent_buffer *leaf; 4109b18c6685SChris Mason u32 nritems; 4110b18c6685SChris Mason unsigned int data_end; 4111b18c6685SChris Mason unsigned int old_data_start; 4112b18c6685SChris Mason unsigned int old_size; 4113b18c6685SChris Mason unsigned int size_diff; 4114b18c6685SChris Mason int i; 4115cfed81a0SChris Mason struct btrfs_map_token token; 4116cfed81a0SChris Mason 41175f39d397SChris Mason leaf = path->nodes[0]; 4118179e29e4SChris Mason slot = path->slots[0]; 4119179e29e4SChris Mason 41203212fa14SJosef Bacik old_size = btrfs_item_size(leaf, slot); 4121179e29e4SChris Mason if (old_size == new_size) 4122143bede5SJeff Mahoney return; 4123b18c6685SChris Mason 41245f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 41258f881e8cSDavid Sterba data_end = leaf_data_end(leaf); 4126b18c6685SChris Mason 41273212fa14SJosef Bacik old_data_start = btrfs_item_offset(leaf, slot); 4128179e29e4SChris Mason 4129b18c6685SChris Mason size_diff = old_size - new_size; 4130b18c6685SChris Mason 4131b18c6685SChris Mason BUG_ON(slot < 0); 4132b18c6685SChris Mason BUG_ON(slot >= nritems); 4133b18c6685SChris Mason 4134b18c6685SChris Mason /* 4135b18c6685SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 4136b18c6685SChris Mason */ 4137b18c6685SChris Mason /* first correct the data pointers */ 4138c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 4139b18c6685SChris Mason for (i = slot; i < nritems; i++) { 41405f39d397SChris Mason u32 ioff; 4141db94535dSChris Mason 41423212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 41433212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff + size_diff); 4144b18c6685SChris Mason } 4145db94535dSChris Mason 4146b18c6685SChris Mason /* shift the data */ 4147179e29e4SChris Mason if (from_end) { 4148637e3b48SJosef Bacik memmove_leaf_data(leaf, data_end + size_diff, data_end, 4149637e3b48SJosef Bacik old_data_start + new_size - data_end); 4150179e29e4SChris Mason } else { 4151179e29e4SChris Mason struct btrfs_disk_key disk_key; 4152179e29e4SChris Mason u64 offset; 4153179e29e4SChris Mason 4154179e29e4SChris Mason btrfs_item_key(leaf, &disk_key, slot); 4155179e29e4SChris Mason 4156179e29e4SChris Mason if (btrfs_disk_key_type(&disk_key) == BTRFS_EXTENT_DATA_KEY) { 4157179e29e4SChris Mason unsigned long ptr; 4158179e29e4SChris Mason struct btrfs_file_extent_item *fi; 4159179e29e4SChris Mason 4160179e29e4SChris Mason fi = btrfs_item_ptr(leaf, slot, 4161179e29e4SChris Mason struct btrfs_file_extent_item); 4162179e29e4SChris Mason fi = (struct btrfs_file_extent_item *)( 4163179e29e4SChris Mason (unsigned long)fi - size_diff); 4164179e29e4SChris Mason 4165179e29e4SChris Mason if (btrfs_file_extent_type(leaf, fi) == 4166179e29e4SChris Mason BTRFS_FILE_EXTENT_INLINE) { 4167179e29e4SChris Mason ptr = btrfs_item_ptr_offset(leaf, slot); 4168179e29e4SChris Mason memmove_extent_buffer(leaf, ptr, 4169179e29e4SChris Mason (unsigned long)fi, 41707ec20afbSDavid Sterba BTRFS_FILE_EXTENT_INLINE_DATA_START); 4171179e29e4SChris Mason } 4172179e29e4SChris Mason } 4173179e29e4SChris Mason 4174637e3b48SJosef Bacik memmove_leaf_data(leaf, data_end + size_diff, data_end, 4175637e3b48SJosef Bacik old_data_start - data_end); 4176179e29e4SChris Mason 4177179e29e4SChris Mason offset = btrfs_disk_key_offset(&disk_key); 4178179e29e4SChris Mason btrfs_set_disk_key_offset(&disk_key, offset + size_diff); 4179179e29e4SChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 4180179e29e4SChris Mason if (slot == 0) 4181b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 4182179e29e4SChris Mason } 41835f39d397SChris Mason 41843212fa14SJosef Bacik btrfs_set_item_size(leaf, slot, new_size); 41855f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 4186b18c6685SChris Mason 4187e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) { 4188a4f78750SDavid Sterba btrfs_print_leaf(leaf); 4189b18c6685SChris Mason BUG(); 41905f39d397SChris Mason } 4191b18c6685SChris Mason } 4192b18c6685SChris Mason 4193d352ac68SChris Mason /* 41948f69dbd2SStefan Behrens * make the item pointed to by the path bigger, data_size is the added size. 4195d352ac68SChris Mason */ 4196c71dd880SDavid Sterba void btrfs_extend_item(struct btrfs_path *path, u32 data_size) 41976567e837SChris Mason { 41986567e837SChris Mason int slot; 41995f39d397SChris Mason struct extent_buffer *leaf; 42006567e837SChris Mason u32 nritems; 42016567e837SChris Mason unsigned int data_end; 42026567e837SChris Mason unsigned int old_data; 42036567e837SChris Mason unsigned int old_size; 42046567e837SChris Mason int i; 4205cfed81a0SChris Mason struct btrfs_map_token token; 4206cfed81a0SChris Mason 42075f39d397SChris Mason leaf = path->nodes[0]; 42086567e837SChris Mason 42095f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 42108f881e8cSDavid Sterba data_end = leaf_data_end(leaf); 42116567e837SChris Mason 4212e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < data_size) { 4213a4f78750SDavid Sterba btrfs_print_leaf(leaf); 42146567e837SChris Mason BUG(); 42155f39d397SChris Mason } 42166567e837SChris Mason slot = path->slots[0]; 4217dc2e724eSJosef Bacik old_data = btrfs_item_data_end(leaf, slot); 42186567e837SChris Mason 42196567e837SChris Mason BUG_ON(slot < 0); 42203326d1b0SChris Mason if (slot >= nritems) { 4221a4f78750SDavid Sterba btrfs_print_leaf(leaf); 4222c71dd880SDavid Sterba btrfs_crit(leaf->fs_info, "slot %d too large, nritems %d", 4223d397712bSChris Mason slot, nritems); 4224290342f6SArnd Bergmann BUG(); 42253326d1b0SChris Mason } 42266567e837SChris Mason 42276567e837SChris Mason /* 42286567e837SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 42296567e837SChris Mason */ 42306567e837SChris Mason /* first correct the data pointers */ 4231c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 42326567e837SChris Mason for (i = slot; i < nritems; i++) { 42335f39d397SChris Mason u32 ioff; 4234db94535dSChris Mason 42353212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 42363212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff - data_size); 42376567e837SChris Mason } 42385f39d397SChris Mason 42396567e837SChris Mason /* shift the data */ 4240637e3b48SJosef Bacik memmove_leaf_data(leaf, data_end - data_size, data_end, 4241637e3b48SJosef Bacik old_data - data_end); 42425f39d397SChris Mason 42436567e837SChris Mason data_end = old_data; 42443212fa14SJosef Bacik old_size = btrfs_item_size(leaf, slot); 42453212fa14SJosef Bacik btrfs_set_item_size(leaf, slot, old_size + data_size); 42465f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 42476567e837SChris Mason 4248e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) { 4249a4f78750SDavid Sterba btrfs_print_leaf(leaf); 42506567e837SChris Mason BUG(); 42515f39d397SChris Mason } 42526567e837SChris Mason } 42536567e837SChris Mason 425443dd529aSDavid Sterba /* 425543dd529aSDavid Sterba * Make space in the node before inserting one or more items. 4256da9ffb24SNikolay Borisov * 4257da9ffb24SNikolay Borisov * @root: root we are inserting items to 4258da9ffb24SNikolay Borisov * @path: points to the leaf/slot where we are going to insert new items 4259b7ef5f3aSFilipe Manana * @batch: information about the batch of items to insert 426043dd529aSDavid Sterba * 426143dd529aSDavid Sterba * Main purpose is to save stack depth by doing the bulk of the work in a 426243dd529aSDavid Sterba * function that doesn't call btrfs_search_slot 426374123bd7SChris Mason */ 4264f0641656SFilipe Manana static void setup_items_for_insert(struct btrfs_root *root, struct btrfs_path *path, 4265b7ef5f3aSFilipe Manana const struct btrfs_item_batch *batch) 4266be0e5c09SChris Mason { 42670b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 42689c58309dSChris Mason int i; 42697518a238SChris Mason u32 nritems; 4270be0e5c09SChris Mason unsigned int data_end; 4271e2fa7227SChris Mason struct btrfs_disk_key disk_key; 427244871b1bSChris Mason struct extent_buffer *leaf; 427344871b1bSChris Mason int slot; 4274cfed81a0SChris Mason struct btrfs_map_token token; 4275fc0d82e1SNikolay Borisov u32 total_size; 4276fc0d82e1SNikolay Borisov 4277b7ef5f3aSFilipe Manana /* 4278b7ef5f3aSFilipe Manana * Before anything else, update keys in the parent and other ancestors 4279b7ef5f3aSFilipe Manana * if needed, then release the write locks on them, so that other tasks 4280b7ef5f3aSFilipe Manana * can use them while we modify the leaf. 4281b7ef5f3aSFilipe Manana */ 428224cdc847SFilipe Manana if (path->slots[0] == 0) { 4283b7ef5f3aSFilipe Manana btrfs_cpu_key_to_disk(&disk_key, &batch->keys[0]); 4284b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 428524cdc847SFilipe Manana } 428624cdc847SFilipe Manana btrfs_unlock_up_safe(path, 1); 428724cdc847SFilipe Manana 42885f39d397SChris Mason leaf = path->nodes[0]; 428944871b1bSChris Mason slot = path->slots[0]; 429074123bd7SChris Mason 42915f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 42928f881e8cSDavid Sterba data_end = leaf_data_end(leaf); 4293b7ef5f3aSFilipe Manana total_size = batch->total_data_size + (batch->nr * sizeof(struct btrfs_item)); 4294eb60ceacSChris Mason 4295e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < total_size) { 4296a4f78750SDavid Sterba btrfs_print_leaf(leaf); 42970b246afaSJeff Mahoney btrfs_crit(fs_info, "not enough freespace need %u have %d", 4298e902baacSDavid Sterba total_size, btrfs_leaf_free_space(leaf)); 4299be0e5c09SChris Mason BUG(); 4300d4dbff95SChris Mason } 43015f39d397SChris Mason 4302c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 4303be0e5c09SChris Mason if (slot != nritems) { 4304dc2e724eSJosef Bacik unsigned int old_data = btrfs_item_data_end(leaf, slot); 4305be0e5c09SChris Mason 43065f39d397SChris Mason if (old_data < data_end) { 4307a4f78750SDavid Sterba btrfs_print_leaf(leaf); 43087269ddd2SNikolay Borisov btrfs_crit(fs_info, 43097269ddd2SNikolay Borisov "item at slot %d with data offset %u beyond data end of leaf %u", 43105f39d397SChris Mason slot, old_data, data_end); 4311290342f6SArnd Bergmann BUG(); 43125f39d397SChris Mason } 4313be0e5c09SChris Mason /* 4314be0e5c09SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 4315be0e5c09SChris Mason */ 4316be0e5c09SChris Mason /* first correct the data pointers */ 43170783fcfcSChris Mason for (i = slot; i < nritems; i++) { 43185f39d397SChris Mason u32 ioff; 4319db94535dSChris Mason 43203212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 43213212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, 4322b7ef5f3aSFilipe Manana ioff - batch->total_data_size); 43230783fcfcSChris Mason } 4324be0e5c09SChris Mason /* shift the items */ 4325637e3b48SJosef Bacik memmove_leaf_items(leaf, slot + batch->nr, slot, nritems - slot); 4326be0e5c09SChris Mason 4327be0e5c09SChris Mason /* shift the data */ 4328637e3b48SJosef Bacik memmove_leaf_data(leaf, data_end - batch->total_data_size, 4329637e3b48SJosef Bacik data_end, old_data - data_end); 4330be0e5c09SChris Mason data_end = old_data; 4331be0e5c09SChris Mason } 43325f39d397SChris Mason 433362e2749eSChris Mason /* setup the item for the new data */ 4334b7ef5f3aSFilipe Manana for (i = 0; i < batch->nr; i++) { 4335b7ef5f3aSFilipe Manana btrfs_cpu_key_to_disk(&disk_key, &batch->keys[i]); 43369c58309dSChris Mason btrfs_set_item_key(leaf, &disk_key, slot + i); 4337b7ef5f3aSFilipe Manana data_end -= batch->data_sizes[i]; 43383212fa14SJosef Bacik btrfs_set_token_item_offset(&token, slot + i, data_end); 43393212fa14SJosef Bacik btrfs_set_token_item_size(&token, slot + i, batch->data_sizes[i]); 43409c58309dSChris Mason } 434144871b1bSChris Mason 4342b7ef5f3aSFilipe Manana btrfs_set_header_nritems(leaf, nritems + batch->nr); 4343b9473439SChris Mason btrfs_mark_buffer_dirty(leaf); 4344aa5d6bedSChris Mason 4345e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) { 4346a4f78750SDavid Sterba btrfs_print_leaf(leaf); 4347be0e5c09SChris Mason BUG(); 43485f39d397SChris Mason } 434944871b1bSChris Mason } 435044871b1bSChris Mason 435144871b1bSChris Mason /* 4352f0641656SFilipe Manana * Insert a new item into a leaf. 4353f0641656SFilipe Manana * 4354f0641656SFilipe Manana * @root: The root of the btree. 4355f0641656SFilipe Manana * @path: A path pointing to the target leaf and slot. 4356f0641656SFilipe Manana * @key: The key of the new item. 4357f0641656SFilipe Manana * @data_size: The size of the data associated with the new key. 4358f0641656SFilipe Manana */ 4359f0641656SFilipe Manana void btrfs_setup_item_for_insert(struct btrfs_root *root, 4360f0641656SFilipe Manana struct btrfs_path *path, 4361f0641656SFilipe Manana const struct btrfs_key *key, 4362f0641656SFilipe Manana u32 data_size) 4363f0641656SFilipe Manana { 4364f0641656SFilipe Manana struct btrfs_item_batch batch; 4365f0641656SFilipe Manana 4366f0641656SFilipe Manana batch.keys = key; 4367f0641656SFilipe Manana batch.data_sizes = &data_size; 4368f0641656SFilipe Manana batch.total_data_size = data_size; 4369f0641656SFilipe Manana batch.nr = 1; 4370f0641656SFilipe Manana 4371f0641656SFilipe Manana setup_items_for_insert(root, path, &batch); 4372f0641656SFilipe Manana } 4373f0641656SFilipe Manana 4374f0641656SFilipe Manana /* 437544871b1bSChris Mason * Given a key and some data, insert items into the tree. 437644871b1bSChris Mason * This does all the path init required, making room in the tree if needed. 437744871b1bSChris Mason */ 437844871b1bSChris Mason int btrfs_insert_empty_items(struct btrfs_trans_handle *trans, 437944871b1bSChris Mason struct btrfs_root *root, 438044871b1bSChris Mason struct btrfs_path *path, 4381b7ef5f3aSFilipe Manana const struct btrfs_item_batch *batch) 438244871b1bSChris Mason { 438344871b1bSChris Mason int ret = 0; 438444871b1bSChris Mason int slot; 4385b7ef5f3aSFilipe Manana u32 total_size; 438644871b1bSChris Mason 4387b7ef5f3aSFilipe Manana total_size = batch->total_data_size + (batch->nr * sizeof(struct btrfs_item)); 4388b7ef5f3aSFilipe Manana ret = btrfs_search_slot(trans, root, &batch->keys[0], path, total_size, 1); 438944871b1bSChris Mason if (ret == 0) 439044871b1bSChris Mason return -EEXIST; 439144871b1bSChris Mason if (ret < 0) 4392143bede5SJeff Mahoney return ret; 439344871b1bSChris Mason 439444871b1bSChris Mason slot = path->slots[0]; 439544871b1bSChris Mason BUG_ON(slot < 0); 439644871b1bSChris Mason 4397b7ef5f3aSFilipe Manana setup_items_for_insert(root, path, batch); 4398143bede5SJeff Mahoney return 0; 439962e2749eSChris Mason } 440062e2749eSChris Mason 440162e2749eSChris Mason /* 440262e2749eSChris Mason * Given a key and some data, insert an item into the tree. 440362e2749eSChris Mason * This does all the path init required, making room in the tree if needed. 440462e2749eSChris Mason */ 4405310712b2SOmar Sandoval int btrfs_insert_item(struct btrfs_trans_handle *trans, struct btrfs_root *root, 4406310712b2SOmar Sandoval const struct btrfs_key *cpu_key, void *data, 4407310712b2SOmar Sandoval u32 data_size) 440862e2749eSChris Mason { 440962e2749eSChris Mason int ret = 0; 44102c90e5d6SChris Mason struct btrfs_path *path; 44115f39d397SChris Mason struct extent_buffer *leaf; 44125f39d397SChris Mason unsigned long ptr; 441362e2749eSChris Mason 44142c90e5d6SChris Mason path = btrfs_alloc_path(); 4415db5b493aSTsutomu Itoh if (!path) 4416db5b493aSTsutomu Itoh return -ENOMEM; 44172c90e5d6SChris Mason ret = btrfs_insert_empty_item(trans, root, path, cpu_key, data_size); 441862e2749eSChris Mason if (!ret) { 44195f39d397SChris Mason leaf = path->nodes[0]; 44205f39d397SChris Mason ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); 44215f39d397SChris Mason write_extent_buffer(leaf, data, ptr, data_size); 44225f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 442362e2749eSChris Mason } 44242c90e5d6SChris Mason btrfs_free_path(path); 4425aa5d6bedSChris Mason return ret; 4426be0e5c09SChris Mason } 4427be0e5c09SChris Mason 442874123bd7SChris Mason /* 4429f0641656SFilipe Manana * This function duplicates an item, giving 'new_key' to the new item. 4430f0641656SFilipe Manana * It guarantees both items live in the same tree leaf and the new item is 4431f0641656SFilipe Manana * contiguous with the original item. 4432f0641656SFilipe Manana * 4433f0641656SFilipe Manana * This allows us to split a file extent in place, keeping a lock on the leaf 4434f0641656SFilipe Manana * the entire time. 4435f0641656SFilipe Manana */ 4436f0641656SFilipe Manana int btrfs_duplicate_item(struct btrfs_trans_handle *trans, 4437f0641656SFilipe Manana struct btrfs_root *root, 4438f0641656SFilipe Manana struct btrfs_path *path, 4439f0641656SFilipe Manana const struct btrfs_key *new_key) 4440f0641656SFilipe Manana { 4441f0641656SFilipe Manana struct extent_buffer *leaf; 4442f0641656SFilipe Manana int ret; 4443f0641656SFilipe Manana u32 item_size; 4444f0641656SFilipe Manana 4445f0641656SFilipe Manana leaf = path->nodes[0]; 44463212fa14SJosef Bacik item_size = btrfs_item_size(leaf, path->slots[0]); 4447f0641656SFilipe Manana ret = setup_leaf_for_split(trans, root, path, 4448f0641656SFilipe Manana item_size + sizeof(struct btrfs_item)); 4449f0641656SFilipe Manana if (ret) 4450f0641656SFilipe Manana return ret; 4451f0641656SFilipe Manana 4452f0641656SFilipe Manana path->slots[0]++; 4453f0641656SFilipe Manana btrfs_setup_item_for_insert(root, path, new_key, item_size); 4454f0641656SFilipe Manana leaf = path->nodes[0]; 4455f0641656SFilipe Manana memcpy_extent_buffer(leaf, 4456f0641656SFilipe Manana btrfs_item_ptr_offset(leaf, path->slots[0]), 4457f0641656SFilipe Manana btrfs_item_ptr_offset(leaf, path->slots[0] - 1), 4458f0641656SFilipe Manana item_size); 4459f0641656SFilipe Manana return 0; 4460f0641656SFilipe Manana } 4461f0641656SFilipe Manana 4462f0641656SFilipe Manana /* 44635de08d7dSChris Mason * delete the pointer from a given node. 446474123bd7SChris Mason * 4465d352ac68SChris Mason * the tree should have been previously balanced so the deletion does not 4466d352ac68SChris Mason * empty a node. 4467016f9d0bSJosef Bacik * 4468016f9d0bSJosef Bacik * This is exported for use inside btrfs-progs, don't un-export it. 446974123bd7SChris Mason */ 4470751a2761SFilipe Manana int btrfs_del_ptr(struct btrfs_trans_handle *trans, struct btrfs_root *root, 4471751a2761SFilipe Manana struct btrfs_path *path, int level, int slot) 4472be0e5c09SChris Mason { 44735f39d397SChris Mason struct extent_buffer *parent = path->nodes[level]; 44747518a238SChris Mason u32 nritems; 4475f3ea38daSJan Schmidt int ret; 4476be0e5c09SChris Mason 44775f39d397SChris Mason nritems = btrfs_header_nritems(parent); 4478be0e5c09SChris Mason if (slot != nritems - 1) { 4479bf1d3425SDavid Sterba if (level) { 4480f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_move(parent, slot, 4481f3a84ccdSFilipe Manana slot + 1, nritems - slot - 1); 4482751a2761SFilipe Manana if (ret < 0) { 4483751a2761SFilipe Manana btrfs_abort_transaction(trans, ret); 4484751a2761SFilipe Manana return ret; 4485751a2761SFilipe Manana } 4486bf1d3425SDavid Sterba } 44875f39d397SChris Mason memmove_extent_buffer(parent, 4488e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(parent, slot), 4489e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(parent, slot + 1), 4490d6025579SChris Mason sizeof(struct btrfs_key_ptr) * 4491d6025579SChris Mason (nritems - slot - 1)); 449257ba86c0SChris Mason } else if (level) { 4493f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, slot, 449433cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REMOVE); 4495751a2761SFilipe Manana if (ret < 0) { 4496751a2761SFilipe Manana btrfs_abort_transaction(trans, ret); 4497751a2761SFilipe Manana return ret; 4498751a2761SFilipe Manana } 4499be0e5c09SChris Mason } 4500f3ea38daSJan Schmidt 45017518a238SChris Mason nritems--; 45025f39d397SChris Mason btrfs_set_header_nritems(parent, nritems); 45037518a238SChris Mason if (nritems == 0 && parent == root->node) { 45045f39d397SChris Mason BUG_ON(btrfs_header_level(root->node) != 1); 4505eb60ceacSChris Mason /* just turn the root into a leaf and break */ 45065f39d397SChris Mason btrfs_set_header_level(root->node, 0); 4507bb803951SChris Mason } else if (slot == 0) { 45085f39d397SChris Mason struct btrfs_disk_key disk_key; 45095f39d397SChris Mason 45105f39d397SChris Mason btrfs_node_key(parent, &disk_key, 0); 4511b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, level + 1); 4512be0e5c09SChris Mason } 4513d6025579SChris Mason btrfs_mark_buffer_dirty(parent); 4514751a2761SFilipe Manana return 0; 4515be0e5c09SChris Mason } 4516be0e5c09SChris Mason 451774123bd7SChris Mason /* 4518323ac95bSChris Mason * a helper function to delete the leaf pointed to by path->slots[1] and 45195d4f98a2SYan Zheng * path->nodes[1]. 4520323ac95bSChris Mason * 4521323ac95bSChris Mason * This deletes the pointer in path->nodes[1] and frees the leaf 4522323ac95bSChris Mason * block extent. zero is returned if it all worked out, < 0 otherwise. 4523323ac95bSChris Mason * 4524323ac95bSChris Mason * The path must have already been setup for deleting the leaf, including 4525323ac95bSChris Mason * all the proper balancing. path->nodes[1] must be locked. 4526323ac95bSChris Mason */ 4527751a2761SFilipe Manana static noinline int btrfs_del_leaf(struct btrfs_trans_handle *trans, 4528323ac95bSChris Mason struct btrfs_root *root, 45295d4f98a2SYan Zheng struct btrfs_path *path, 45305d4f98a2SYan Zheng struct extent_buffer *leaf) 4531323ac95bSChris Mason { 4532751a2761SFilipe Manana int ret; 4533751a2761SFilipe Manana 45345d4f98a2SYan Zheng WARN_ON(btrfs_header_generation(leaf) != trans->transid); 4535751a2761SFilipe Manana ret = btrfs_del_ptr(trans, root, path, 1, path->slots[1]); 4536751a2761SFilipe Manana if (ret < 0) 4537751a2761SFilipe Manana return ret; 4538323ac95bSChris Mason 45394d081c41SChris Mason /* 45404d081c41SChris Mason * btrfs_free_extent is expensive, we want to make sure we 45414d081c41SChris Mason * aren't holding any locks when we call it 45424d081c41SChris Mason */ 45434d081c41SChris Mason btrfs_unlock_up_safe(path, 0); 45444d081c41SChris Mason 4545f0486c68SYan, Zheng root_sub_used(root, leaf->len); 4546f0486c68SYan, Zheng 454767439dadSDavid Sterba atomic_inc(&leaf->refs); 45487a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), leaf, 0, 1); 45493083ee2eSJosef Bacik free_extent_buffer_stale(leaf); 4550751a2761SFilipe Manana return 0; 4551323ac95bSChris Mason } 4552323ac95bSChris Mason /* 455374123bd7SChris Mason * delete the item at the leaf level in path. If that empties 455474123bd7SChris Mason * the leaf, remove it from the tree 455574123bd7SChris Mason */ 455685e21bacSChris Mason int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root, 455785e21bacSChris Mason struct btrfs_path *path, int slot, int nr) 4558be0e5c09SChris Mason { 45590b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 45605f39d397SChris Mason struct extent_buffer *leaf; 4561aa5d6bedSChris Mason int ret = 0; 4562aa5d6bedSChris Mason int wret; 45637518a238SChris Mason u32 nritems; 4564be0e5c09SChris Mason 45655f39d397SChris Mason leaf = path->nodes[0]; 45665f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 4567be0e5c09SChris Mason 456885e21bacSChris Mason if (slot + nr != nritems) { 45690cae23b6SFilipe Manana const u32 last_off = btrfs_item_offset(leaf, slot + nr - 1); 45700cae23b6SFilipe Manana const int data_end = leaf_data_end(leaf); 4571c82f823cSDavid Sterba struct btrfs_map_token token; 45720cae23b6SFilipe Manana u32 dsize = 0; 45730cae23b6SFilipe Manana int i; 45740cae23b6SFilipe Manana 45750cae23b6SFilipe Manana for (i = 0; i < nr; i++) 45760cae23b6SFilipe Manana dsize += btrfs_item_size(leaf, slot + i); 45775f39d397SChris Mason 4578637e3b48SJosef Bacik memmove_leaf_data(leaf, data_end + dsize, data_end, 457985e21bacSChris Mason last_off - data_end); 45805f39d397SChris Mason 4581c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 458285e21bacSChris Mason for (i = slot + nr; i < nritems; i++) { 45835f39d397SChris Mason u32 ioff; 4584db94535dSChris Mason 45853212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 45863212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff + dsize); 45870783fcfcSChris Mason } 4588db94535dSChris Mason 4589637e3b48SJosef Bacik memmove_leaf_items(leaf, slot, slot + nr, nritems - slot - nr); 4590be0e5c09SChris Mason } 459185e21bacSChris Mason btrfs_set_header_nritems(leaf, nritems - nr); 459285e21bacSChris Mason nritems -= nr; 45935f39d397SChris Mason 459474123bd7SChris Mason /* delete the leaf if we've emptied it */ 45957518a238SChris Mason if (nritems == 0) { 45965f39d397SChris Mason if (leaf == root->node) { 45975f39d397SChris Mason btrfs_set_header_level(leaf, 0); 45989a8dd150SChris Mason } else { 4599190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, leaf); 4600751a2761SFilipe Manana ret = btrfs_del_leaf(trans, root, path, leaf); 4601751a2761SFilipe Manana if (ret < 0) 4602751a2761SFilipe Manana return ret; 46039a8dd150SChris Mason } 4604be0e5c09SChris Mason } else { 46057518a238SChris Mason int used = leaf_space_used(leaf, 0, nritems); 4606aa5d6bedSChris Mason if (slot == 0) { 46075f39d397SChris Mason struct btrfs_disk_key disk_key; 46085f39d397SChris Mason 46095f39d397SChris Mason btrfs_item_key(leaf, &disk_key, 0); 4610b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 4611aa5d6bedSChris Mason } 4612aa5d6bedSChris Mason 46137c4063d1SFilipe Manana /* 46147c4063d1SFilipe Manana * Try to delete the leaf if it is mostly empty. We do this by 46157c4063d1SFilipe Manana * trying to move all its items into its left and right neighbours. 46167c4063d1SFilipe Manana * If we can't move all the items, then we don't delete it - it's 46177c4063d1SFilipe Manana * not ideal, but future insertions might fill the leaf with more 46187c4063d1SFilipe Manana * items, or items from other leaves might be moved later into our 46197c4063d1SFilipe Manana * leaf due to deletions on those leaves. 46207c4063d1SFilipe Manana */ 46210b246afaSJeff Mahoney if (used < BTRFS_LEAF_DATA_SIZE(fs_info) / 3) { 46227c4063d1SFilipe Manana u32 min_push_space; 46237c4063d1SFilipe Manana 4624be0e5c09SChris Mason /* push_leaf_left fixes the path. 4625be0e5c09SChris Mason * make sure the path still points to our leaf 4626016f9d0bSJosef Bacik * for possible call to btrfs_del_ptr below 4627be0e5c09SChris Mason */ 46284920c9acSChris Mason slot = path->slots[1]; 462967439dadSDavid Sterba atomic_inc(&leaf->refs); 46307c4063d1SFilipe Manana /* 46317c4063d1SFilipe Manana * We want to be able to at least push one item to the 46327c4063d1SFilipe Manana * left neighbour leaf, and that's the first item. 46337c4063d1SFilipe Manana */ 46347c4063d1SFilipe Manana min_push_space = sizeof(struct btrfs_item) + 46357c4063d1SFilipe Manana btrfs_item_size(leaf, 0); 46367c4063d1SFilipe Manana wret = push_leaf_left(trans, root, path, 0, 46377c4063d1SFilipe Manana min_push_space, 1, (u32)-1); 463854aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 4639aa5d6bedSChris Mason ret = wret; 46405f39d397SChris Mason 46415f39d397SChris Mason if (path->nodes[0] == leaf && 46425f39d397SChris Mason btrfs_header_nritems(leaf)) { 46437c4063d1SFilipe Manana /* 46447c4063d1SFilipe Manana * If we were not able to push all items from our 46457c4063d1SFilipe Manana * leaf to its left neighbour, then attempt to 46467c4063d1SFilipe Manana * either push all the remaining items to the 46477c4063d1SFilipe Manana * right neighbour or none. There's no advantage 46487c4063d1SFilipe Manana * in pushing only some items, instead of all, as 46497c4063d1SFilipe Manana * it's pointless to end up with a leaf having 46507c4063d1SFilipe Manana * too few items while the neighbours can be full 46517c4063d1SFilipe Manana * or nearly full. 46527c4063d1SFilipe Manana */ 46537c4063d1SFilipe Manana nritems = btrfs_header_nritems(leaf); 46547c4063d1SFilipe Manana min_push_space = leaf_space_used(leaf, 0, nritems); 46557c4063d1SFilipe Manana wret = push_leaf_right(trans, root, path, 0, 46567c4063d1SFilipe Manana min_push_space, 1, 0); 465754aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 4658aa5d6bedSChris Mason ret = wret; 4659aa5d6bedSChris Mason } 46605f39d397SChris Mason 46615f39d397SChris Mason if (btrfs_header_nritems(leaf) == 0) { 4662323ac95bSChris Mason path->slots[1] = slot; 4663751a2761SFilipe Manana ret = btrfs_del_leaf(trans, root, path, leaf); 4664751a2761SFilipe Manana if (ret < 0) 4665751a2761SFilipe Manana return ret; 46665f39d397SChris Mason free_extent_buffer(leaf); 4667143bede5SJeff Mahoney ret = 0; 46685de08d7dSChris Mason } else { 4669925baeddSChris Mason /* if we're still in the path, make sure 4670925baeddSChris Mason * we're dirty. Otherwise, one of the 4671925baeddSChris Mason * push_leaf functions must have already 4672925baeddSChris Mason * dirtied this buffer 4673925baeddSChris Mason */ 4674925baeddSChris Mason if (path->nodes[0] == leaf) 46755f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 46765f39d397SChris Mason free_extent_buffer(leaf); 4677be0e5c09SChris Mason } 4678d5719762SChris Mason } else { 46795f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 4680be0e5c09SChris Mason } 46819a8dd150SChris Mason } 4682aa5d6bedSChris Mason return ret; 46839a8dd150SChris Mason } 46849a8dd150SChris Mason 468597571fd0SChris Mason /* 46863f157a2fSChris Mason * A helper function to walk down the tree starting at min_key, and looking 4687de78b51aSEric Sandeen * for nodes or leaves that are have a minimum transaction id. 4688de78b51aSEric Sandeen * This is used by the btree defrag code, and tree logging 46893f157a2fSChris Mason * 46903f157a2fSChris Mason * This does not cow, but it does stuff the starting key it finds back 46913f157a2fSChris Mason * into min_key, so you can call btrfs_search_slot with cow=1 on the 46923f157a2fSChris Mason * key and get a writable path. 46933f157a2fSChris Mason * 46943f157a2fSChris Mason * This honors path->lowest_level to prevent descent past a given level 46953f157a2fSChris Mason * of the tree. 46963f157a2fSChris Mason * 4697d352ac68SChris Mason * min_trans indicates the oldest transaction that you are interested 4698d352ac68SChris Mason * in walking through. Any nodes or leaves older than min_trans are 4699d352ac68SChris Mason * skipped over (without reading them). 4700d352ac68SChris Mason * 47013f157a2fSChris Mason * returns zero if something useful was found, < 0 on error and 1 if there 47023f157a2fSChris Mason * was nothing in the tree that matched the search criteria. 47033f157a2fSChris Mason */ 47043f157a2fSChris Mason int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key, 4705de78b51aSEric Sandeen struct btrfs_path *path, 47063f157a2fSChris Mason u64 min_trans) 47073f157a2fSChris Mason { 47083f157a2fSChris Mason struct extent_buffer *cur; 47093f157a2fSChris Mason struct btrfs_key found_key; 47103f157a2fSChris Mason int slot; 47119652480bSYan int sret; 47123f157a2fSChris Mason u32 nritems; 47133f157a2fSChris Mason int level; 47143f157a2fSChris Mason int ret = 1; 4715f98de9b9SFilipe Manana int keep_locks = path->keep_locks; 47163f157a2fSChris Mason 4717c922b016SStefan Roesch ASSERT(!path->nowait); 4718f98de9b9SFilipe Manana path->keep_locks = 1; 47193f157a2fSChris Mason again: 4720bd681513SChris Mason cur = btrfs_read_lock_root_node(root); 47213f157a2fSChris Mason level = btrfs_header_level(cur); 4722e02119d5SChris Mason WARN_ON(path->nodes[level]); 47233f157a2fSChris Mason path->nodes[level] = cur; 4724bd681513SChris Mason path->locks[level] = BTRFS_READ_LOCK; 47253f157a2fSChris Mason 47263f157a2fSChris Mason if (btrfs_header_generation(cur) < min_trans) { 47273f157a2fSChris Mason ret = 1; 47283f157a2fSChris Mason goto out; 47293f157a2fSChris Mason } 47303f157a2fSChris Mason while (1) { 47313f157a2fSChris Mason nritems = btrfs_header_nritems(cur); 47323f157a2fSChris Mason level = btrfs_header_level(cur); 4733fdf8d595SAnand Jain sret = btrfs_bin_search(cur, 0, min_key, &slot); 4734cbca7d59SFilipe Manana if (sret < 0) { 4735cbca7d59SFilipe Manana ret = sret; 4736cbca7d59SFilipe Manana goto out; 4737cbca7d59SFilipe Manana } 47383f157a2fSChris Mason 4739323ac95bSChris Mason /* at the lowest level, we're done, setup the path and exit */ 4740323ac95bSChris Mason if (level == path->lowest_level) { 4741e02119d5SChris Mason if (slot >= nritems) 4742e02119d5SChris Mason goto find_next_key; 47433f157a2fSChris Mason ret = 0; 47443f157a2fSChris Mason path->slots[level] = slot; 47453f157a2fSChris Mason btrfs_item_key_to_cpu(cur, &found_key, slot); 47463f157a2fSChris Mason goto out; 47473f157a2fSChris Mason } 47489652480bSYan if (sret && slot > 0) 47499652480bSYan slot--; 47503f157a2fSChris Mason /* 4751de78b51aSEric Sandeen * check this node pointer against the min_trans parameters. 4752260db43cSRandy Dunlap * If it is too old, skip to the next one. 47533f157a2fSChris Mason */ 47543f157a2fSChris Mason while (slot < nritems) { 47553f157a2fSChris Mason u64 gen; 4756e02119d5SChris Mason 47573f157a2fSChris Mason gen = btrfs_node_ptr_generation(cur, slot); 47583f157a2fSChris Mason if (gen < min_trans) { 47593f157a2fSChris Mason slot++; 47603f157a2fSChris Mason continue; 47613f157a2fSChris Mason } 47623f157a2fSChris Mason break; 47633f157a2fSChris Mason } 4764e02119d5SChris Mason find_next_key: 47653f157a2fSChris Mason /* 47663f157a2fSChris Mason * we didn't find a candidate key in this node, walk forward 47673f157a2fSChris Mason * and find another one 47683f157a2fSChris Mason */ 47693f157a2fSChris Mason if (slot >= nritems) { 4770e02119d5SChris Mason path->slots[level] = slot; 4771e02119d5SChris Mason sret = btrfs_find_next_key(root, path, min_key, level, 4772de78b51aSEric Sandeen min_trans); 4773e02119d5SChris Mason if (sret == 0) { 4774b3b4aa74SDavid Sterba btrfs_release_path(path); 47753f157a2fSChris Mason goto again; 47763f157a2fSChris Mason } else { 47773f157a2fSChris Mason goto out; 47783f157a2fSChris Mason } 47793f157a2fSChris Mason } 47803f157a2fSChris Mason /* save our key for returning back */ 47813f157a2fSChris Mason btrfs_node_key_to_cpu(cur, &found_key, slot); 47823f157a2fSChris Mason path->slots[level] = slot; 47833f157a2fSChris Mason if (level == path->lowest_level) { 47843f157a2fSChris Mason ret = 0; 47853f157a2fSChris Mason goto out; 47863f157a2fSChris Mason } 47874b231ae4SDavid Sterba cur = btrfs_read_node_slot(cur, slot); 4788fb770ae4SLiu Bo if (IS_ERR(cur)) { 4789fb770ae4SLiu Bo ret = PTR_ERR(cur); 4790fb770ae4SLiu Bo goto out; 4791fb770ae4SLiu Bo } 47923f157a2fSChris Mason 4793bd681513SChris Mason btrfs_tree_read_lock(cur); 4794b4ce94deSChris Mason 4795bd681513SChris Mason path->locks[level - 1] = BTRFS_READ_LOCK; 47963f157a2fSChris Mason path->nodes[level - 1] = cur; 4797f7c79f30SChris Mason unlock_up(path, level, 1, 0, NULL); 47983f157a2fSChris Mason } 47993f157a2fSChris Mason out: 4800f98de9b9SFilipe Manana path->keep_locks = keep_locks; 4801f98de9b9SFilipe Manana if (ret == 0) { 4802f98de9b9SFilipe Manana btrfs_unlock_up_safe(path, path->lowest_level + 1); 4803f98de9b9SFilipe Manana memcpy(min_key, &found_key, sizeof(found_key)); 4804f98de9b9SFilipe Manana } 48053f157a2fSChris Mason return ret; 48063f157a2fSChris Mason } 48073f157a2fSChris Mason 48083f157a2fSChris Mason /* 48093f157a2fSChris Mason * this is similar to btrfs_next_leaf, but does not try to preserve 48103f157a2fSChris Mason * and fixup the path. It looks for and returns the next key in the 4811de78b51aSEric Sandeen * tree based on the current path and the min_trans parameters. 48123f157a2fSChris Mason * 48133f157a2fSChris Mason * 0 is returned if another key is found, < 0 if there are any errors 48143f157a2fSChris Mason * and 1 is returned if there are no higher keys in the tree 48153f157a2fSChris Mason * 48163f157a2fSChris Mason * path->keep_locks should be set to 1 on the search made before 48173f157a2fSChris Mason * calling this function. 48183f157a2fSChris Mason */ 4819e7a84565SChris Mason int btrfs_find_next_key(struct btrfs_root *root, struct btrfs_path *path, 4820de78b51aSEric Sandeen struct btrfs_key *key, int level, u64 min_trans) 4821e7a84565SChris Mason { 4822e7a84565SChris Mason int slot; 4823e7a84565SChris Mason struct extent_buffer *c; 4824e7a84565SChris Mason 48256a9fb468SJosef Bacik WARN_ON(!path->keep_locks && !path->skip_locking); 4826e7a84565SChris Mason while (level < BTRFS_MAX_LEVEL) { 4827e7a84565SChris Mason if (!path->nodes[level]) 4828e7a84565SChris Mason return 1; 4829e7a84565SChris Mason 4830e7a84565SChris Mason slot = path->slots[level] + 1; 4831e7a84565SChris Mason c = path->nodes[level]; 48323f157a2fSChris Mason next: 4833e7a84565SChris Mason if (slot >= btrfs_header_nritems(c)) { 483433c66f43SYan Zheng int ret; 483533c66f43SYan Zheng int orig_lowest; 483633c66f43SYan Zheng struct btrfs_key cur_key; 483733c66f43SYan Zheng if (level + 1 >= BTRFS_MAX_LEVEL || 483833c66f43SYan Zheng !path->nodes[level + 1]) 4839e7a84565SChris Mason return 1; 484033c66f43SYan Zheng 48416a9fb468SJosef Bacik if (path->locks[level + 1] || path->skip_locking) { 484233c66f43SYan Zheng level++; 4843e7a84565SChris Mason continue; 4844e7a84565SChris Mason } 484533c66f43SYan Zheng 484633c66f43SYan Zheng slot = btrfs_header_nritems(c) - 1; 484733c66f43SYan Zheng if (level == 0) 484833c66f43SYan Zheng btrfs_item_key_to_cpu(c, &cur_key, slot); 484933c66f43SYan Zheng else 485033c66f43SYan Zheng btrfs_node_key_to_cpu(c, &cur_key, slot); 485133c66f43SYan Zheng 485233c66f43SYan Zheng orig_lowest = path->lowest_level; 4853b3b4aa74SDavid Sterba btrfs_release_path(path); 485433c66f43SYan Zheng path->lowest_level = level; 485533c66f43SYan Zheng ret = btrfs_search_slot(NULL, root, &cur_key, path, 485633c66f43SYan Zheng 0, 0); 485733c66f43SYan Zheng path->lowest_level = orig_lowest; 485833c66f43SYan Zheng if (ret < 0) 485933c66f43SYan Zheng return ret; 486033c66f43SYan Zheng 486133c66f43SYan Zheng c = path->nodes[level]; 486233c66f43SYan Zheng slot = path->slots[level]; 486333c66f43SYan Zheng if (ret == 0) 486433c66f43SYan Zheng slot++; 486533c66f43SYan Zheng goto next; 486633c66f43SYan Zheng } 486733c66f43SYan Zheng 4868e7a84565SChris Mason if (level == 0) 4869e7a84565SChris Mason btrfs_item_key_to_cpu(c, key, slot); 48703f157a2fSChris Mason else { 48713f157a2fSChris Mason u64 gen = btrfs_node_ptr_generation(c, slot); 48723f157a2fSChris Mason 48733f157a2fSChris Mason if (gen < min_trans) { 48743f157a2fSChris Mason slot++; 48753f157a2fSChris Mason goto next; 48763f157a2fSChris Mason } 4877e7a84565SChris Mason btrfs_node_key_to_cpu(c, key, slot); 48783f157a2fSChris Mason } 4879e7a84565SChris Mason return 0; 4880e7a84565SChris Mason } 4881e7a84565SChris Mason return 1; 4882e7a84565SChris Mason } 4883e7a84565SChris Mason 48843d7806ecSJan Schmidt int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path, 48853d7806ecSJan Schmidt u64 time_seq) 48863d7806ecSJan Schmidt { 4887d97e63b6SChris Mason int slot; 48888e73f275SChris Mason int level; 48895f39d397SChris Mason struct extent_buffer *c; 48908e73f275SChris Mason struct extent_buffer *next; 4891d96b3424SFilipe Manana struct btrfs_fs_info *fs_info = root->fs_info; 4892925baeddSChris Mason struct btrfs_key key; 4893d96b3424SFilipe Manana bool need_commit_sem = false; 4894925baeddSChris Mason u32 nritems; 4895925baeddSChris Mason int ret; 48960e46318dSJosef Bacik int i; 4897925baeddSChris Mason 4898bdcdd86cSFilipe Manana /* 4899bdcdd86cSFilipe Manana * The nowait semantics are used only for write paths, where we don't 4900bdcdd86cSFilipe Manana * use the tree mod log and sequence numbers. 4901bdcdd86cSFilipe Manana */ 4902bdcdd86cSFilipe Manana if (time_seq) 4903c922b016SStefan Roesch ASSERT(!path->nowait); 4904c922b016SStefan Roesch 4905925baeddSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 4906d397712bSChris Mason if (nritems == 0) 4907925baeddSChris Mason return 1; 4908925baeddSChris Mason 49098e73f275SChris Mason btrfs_item_key_to_cpu(path->nodes[0], &key, nritems - 1); 49108e73f275SChris Mason again: 49118e73f275SChris Mason level = 1; 49128e73f275SChris Mason next = NULL; 4913b3b4aa74SDavid Sterba btrfs_release_path(path); 49148e73f275SChris Mason 4915a2135011SChris Mason path->keep_locks = 1; 49168e73f275SChris Mason 4917d96b3424SFilipe Manana if (time_seq) { 49183d7806ecSJan Schmidt ret = btrfs_search_old_slot(root, &key, path, time_seq); 4919d96b3424SFilipe Manana } else { 4920d96b3424SFilipe Manana if (path->need_commit_sem) { 4921d96b3424SFilipe Manana path->need_commit_sem = 0; 4922d96b3424SFilipe Manana need_commit_sem = true; 4923bdcdd86cSFilipe Manana if (path->nowait) { 4924bdcdd86cSFilipe Manana if (!down_read_trylock(&fs_info->commit_root_sem)) { 4925bdcdd86cSFilipe Manana ret = -EAGAIN; 4926bdcdd86cSFilipe Manana goto done; 4927bdcdd86cSFilipe Manana } 4928bdcdd86cSFilipe Manana } else { 4929d96b3424SFilipe Manana down_read(&fs_info->commit_root_sem); 4930d96b3424SFilipe Manana } 4931bdcdd86cSFilipe Manana } 4932925baeddSChris Mason ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 4933d96b3424SFilipe Manana } 4934925baeddSChris Mason path->keep_locks = 0; 4935925baeddSChris Mason 4936925baeddSChris Mason if (ret < 0) 4937d96b3424SFilipe Manana goto done; 4938925baeddSChris Mason 4939a2135011SChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 4940168fd7d2SChris Mason /* 4941168fd7d2SChris Mason * by releasing the path above we dropped all our locks. A balance 4942168fd7d2SChris Mason * could have added more items next to the key that used to be 4943168fd7d2SChris Mason * at the very end of the block. So, check again here and 4944168fd7d2SChris Mason * advance the path if there are now more items available. 4945168fd7d2SChris Mason */ 4946a2135011SChris Mason if (nritems > 0 && path->slots[0] < nritems - 1) { 4947e457afecSYan Zheng if (ret == 0) 4948168fd7d2SChris Mason path->slots[0]++; 49498e73f275SChris Mason ret = 0; 4950925baeddSChris Mason goto done; 4951925baeddSChris Mason } 49520b43e04fSLiu Bo /* 49530b43e04fSLiu Bo * So the above check misses one case: 49540b43e04fSLiu Bo * - after releasing the path above, someone has removed the item that 49550b43e04fSLiu Bo * used to be at the very end of the block, and balance between leafs 49560b43e04fSLiu Bo * gets another one with bigger key.offset to replace it. 49570b43e04fSLiu Bo * 49580b43e04fSLiu Bo * This one should be returned as well, or we can get leaf corruption 49590b43e04fSLiu Bo * later(esp. in __btrfs_drop_extents()). 49600b43e04fSLiu Bo * 49610b43e04fSLiu Bo * And a bit more explanation about this check, 49620b43e04fSLiu Bo * with ret > 0, the key isn't found, the path points to the slot 49630b43e04fSLiu Bo * where it should be inserted, so the path->slots[0] item must be the 49640b43e04fSLiu Bo * bigger one. 49650b43e04fSLiu Bo */ 49660b43e04fSLiu Bo if (nritems > 0 && ret > 0 && path->slots[0] == nritems - 1) { 49670b43e04fSLiu Bo ret = 0; 49680b43e04fSLiu Bo goto done; 49690b43e04fSLiu Bo } 4970d97e63b6SChris Mason 4971234b63a0SChris Mason while (level < BTRFS_MAX_LEVEL) { 49728e73f275SChris Mason if (!path->nodes[level]) { 49738e73f275SChris Mason ret = 1; 49748e73f275SChris Mason goto done; 49758e73f275SChris Mason } 49765f39d397SChris Mason 4977d97e63b6SChris Mason slot = path->slots[level] + 1; 4978d97e63b6SChris Mason c = path->nodes[level]; 49795f39d397SChris Mason if (slot >= btrfs_header_nritems(c)) { 4980d97e63b6SChris Mason level++; 49818e73f275SChris Mason if (level == BTRFS_MAX_LEVEL) { 49828e73f275SChris Mason ret = 1; 49838e73f275SChris Mason goto done; 49848e73f275SChris Mason } 4985d97e63b6SChris Mason continue; 4986d97e63b6SChris Mason } 49875f39d397SChris Mason 49880e46318dSJosef Bacik 49890e46318dSJosef Bacik /* 49900e46318dSJosef Bacik * Our current level is where we're going to start from, and to 49910e46318dSJosef Bacik * make sure lockdep doesn't complain we need to drop our locks 49920e46318dSJosef Bacik * and nodes from 0 to our current level. 49930e46318dSJosef Bacik */ 49940e46318dSJosef Bacik for (i = 0; i < level; i++) { 49950e46318dSJosef Bacik if (path->locks[level]) { 49960e46318dSJosef Bacik btrfs_tree_read_unlock(path->nodes[i]); 49970e46318dSJosef Bacik path->locks[i] = 0; 49980e46318dSJosef Bacik } 49990e46318dSJosef Bacik free_extent_buffer(path->nodes[i]); 50000e46318dSJosef Bacik path->nodes[i] = NULL; 5001925baeddSChris Mason } 50025f39d397SChris Mason 50038e73f275SChris Mason next = c; 5004d07b8528SLiu Bo ret = read_block_for_search(root, path, &next, level, 5005cda79c54SDavid Sterba slot, &key); 5006bdcdd86cSFilipe Manana if (ret == -EAGAIN && !path->nowait) 50078e73f275SChris Mason goto again; 50085f39d397SChris Mason 500976a05b35SChris Mason if (ret < 0) { 5010b3b4aa74SDavid Sterba btrfs_release_path(path); 501176a05b35SChris Mason goto done; 501276a05b35SChris Mason } 501376a05b35SChris Mason 50145cd57b2cSChris Mason if (!path->skip_locking) { 5015bd681513SChris Mason ret = btrfs_try_tree_read_lock(next); 5016bdcdd86cSFilipe Manana if (!ret && path->nowait) { 5017bdcdd86cSFilipe Manana ret = -EAGAIN; 5018bdcdd86cSFilipe Manana goto done; 5019bdcdd86cSFilipe Manana } 5020d42244a0SJan Schmidt if (!ret && time_seq) { 5021d42244a0SJan Schmidt /* 5022d42244a0SJan Schmidt * If we don't get the lock, we may be racing 5023d42244a0SJan Schmidt * with push_leaf_left, holding that lock while 5024d42244a0SJan Schmidt * itself waiting for the leaf we've currently 5025d42244a0SJan Schmidt * locked. To solve this situation, we give up 5026d42244a0SJan Schmidt * on our lock and cycle. 5027d42244a0SJan Schmidt */ 5028cf538830SJan Schmidt free_extent_buffer(next); 5029d42244a0SJan Schmidt btrfs_release_path(path); 5030d42244a0SJan Schmidt cond_resched(); 5031d42244a0SJan Schmidt goto again; 5032d42244a0SJan Schmidt } 50330e46318dSJosef Bacik if (!ret) 50340e46318dSJosef Bacik btrfs_tree_read_lock(next); 5035bd681513SChris Mason } 5036d97e63b6SChris Mason break; 5037d97e63b6SChris Mason } 5038d97e63b6SChris Mason path->slots[level] = slot; 5039d97e63b6SChris Mason while (1) { 5040d97e63b6SChris Mason level--; 5041d97e63b6SChris Mason path->nodes[level] = next; 5042d97e63b6SChris Mason path->slots[level] = 0; 5043a74a4b97SChris Mason if (!path->skip_locking) 5044ffeb03cfSJosef Bacik path->locks[level] = BTRFS_READ_LOCK; 5045d97e63b6SChris Mason if (!level) 5046d97e63b6SChris Mason break; 5047b4ce94deSChris Mason 5048d07b8528SLiu Bo ret = read_block_for_search(root, path, &next, level, 5049cda79c54SDavid Sterba 0, &key); 5050bdcdd86cSFilipe Manana if (ret == -EAGAIN && !path->nowait) 50518e73f275SChris Mason goto again; 50528e73f275SChris Mason 505376a05b35SChris Mason if (ret < 0) { 5054b3b4aa74SDavid Sterba btrfs_release_path(path); 505576a05b35SChris Mason goto done; 505676a05b35SChris Mason } 505776a05b35SChris Mason 5058bdcdd86cSFilipe Manana if (!path->skip_locking) { 5059bdcdd86cSFilipe Manana if (path->nowait) { 5060bdcdd86cSFilipe Manana if (!btrfs_try_tree_read_lock(next)) { 5061bdcdd86cSFilipe Manana ret = -EAGAIN; 5062bdcdd86cSFilipe Manana goto done; 5063bdcdd86cSFilipe Manana } 5064bdcdd86cSFilipe Manana } else { 50650e46318dSJosef Bacik btrfs_tree_read_lock(next); 5066d97e63b6SChris Mason } 5067bdcdd86cSFilipe Manana } 5068bdcdd86cSFilipe Manana } 50698e73f275SChris Mason ret = 0; 5070925baeddSChris Mason done: 5071f7c79f30SChris Mason unlock_up(path, 0, 1, 0, NULL); 5072d96b3424SFilipe Manana if (need_commit_sem) { 5073d96b3424SFilipe Manana int ret2; 5074d96b3424SFilipe Manana 5075d96b3424SFilipe Manana path->need_commit_sem = 1; 5076d96b3424SFilipe Manana ret2 = finish_need_commit_sem_search(path); 5077d96b3424SFilipe Manana up_read(&fs_info->commit_root_sem); 5078d96b3424SFilipe Manana if (ret2) 5079d96b3424SFilipe Manana ret = ret2; 5080d96b3424SFilipe Manana } 50818e73f275SChris Mason 50828e73f275SChris Mason return ret; 5083d97e63b6SChris Mason } 50840b86a832SChris Mason 5085890d2b1aSJosef Bacik int btrfs_next_old_item(struct btrfs_root *root, struct btrfs_path *path, u64 time_seq) 5086890d2b1aSJosef Bacik { 5087890d2b1aSJosef Bacik path->slots[0]++; 5088890d2b1aSJosef Bacik if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) 5089890d2b1aSJosef Bacik return btrfs_next_old_leaf(root, path, time_seq); 5090890d2b1aSJosef Bacik return 0; 5091890d2b1aSJosef Bacik } 5092890d2b1aSJosef Bacik 50933f157a2fSChris Mason /* 50943f157a2fSChris Mason * this uses btrfs_prev_leaf to walk backwards in the tree, and keeps 50953f157a2fSChris Mason * searching until it gets past min_objectid or finds an item of 'type' 50963f157a2fSChris Mason * 50973f157a2fSChris Mason * returns 0 if something is found, 1 if nothing was found and < 0 on error 50983f157a2fSChris Mason */ 50990b86a832SChris Mason int btrfs_previous_item(struct btrfs_root *root, 51000b86a832SChris Mason struct btrfs_path *path, u64 min_objectid, 51010b86a832SChris Mason int type) 51020b86a832SChris Mason { 51030b86a832SChris Mason struct btrfs_key found_key; 51040b86a832SChris Mason struct extent_buffer *leaf; 5105e02119d5SChris Mason u32 nritems; 51060b86a832SChris Mason int ret; 51070b86a832SChris Mason 51080b86a832SChris Mason while (1) { 51090b86a832SChris Mason if (path->slots[0] == 0) { 51100b86a832SChris Mason ret = btrfs_prev_leaf(root, path); 51110b86a832SChris Mason if (ret != 0) 51120b86a832SChris Mason return ret; 51130b86a832SChris Mason } else { 51140b86a832SChris Mason path->slots[0]--; 51150b86a832SChris Mason } 51160b86a832SChris Mason leaf = path->nodes[0]; 5117e02119d5SChris Mason nritems = btrfs_header_nritems(leaf); 5118e02119d5SChris Mason if (nritems == 0) 5119e02119d5SChris Mason return 1; 5120e02119d5SChris Mason if (path->slots[0] == nritems) 5121e02119d5SChris Mason path->slots[0]--; 5122e02119d5SChris Mason 51230b86a832SChris Mason btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 5124e02119d5SChris Mason if (found_key.objectid < min_objectid) 5125e02119d5SChris Mason break; 51260a4eefbbSYan Zheng if (found_key.type == type) 51270a4eefbbSYan Zheng return 0; 5128e02119d5SChris Mason if (found_key.objectid == min_objectid && 5129e02119d5SChris Mason found_key.type < type) 5130e02119d5SChris Mason break; 51310b86a832SChris Mason } 51320b86a832SChris Mason return 1; 51330b86a832SChris Mason } 5134ade2e0b3SWang Shilong 5135ade2e0b3SWang Shilong /* 5136ade2e0b3SWang Shilong * search in extent tree to find a previous Metadata/Data extent item with 5137ade2e0b3SWang Shilong * min objecitd. 5138ade2e0b3SWang Shilong * 5139ade2e0b3SWang Shilong * returns 0 if something is found, 1 if nothing was found and < 0 on error 5140ade2e0b3SWang Shilong */ 5141ade2e0b3SWang Shilong int btrfs_previous_extent_item(struct btrfs_root *root, 5142ade2e0b3SWang Shilong struct btrfs_path *path, u64 min_objectid) 5143ade2e0b3SWang Shilong { 5144ade2e0b3SWang Shilong struct btrfs_key found_key; 5145ade2e0b3SWang Shilong struct extent_buffer *leaf; 5146ade2e0b3SWang Shilong u32 nritems; 5147ade2e0b3SWang Shilong int ret; 5148ade2e0b3SWang Shilong 5149ade2e0b3SWang Shilong while (1) { 5150ade2e0b3SWang Shilong if (path->slots[0] == 0) { 5151ade2e0b3SWang Shilong ret = btrfs_prev_leaf(root, path); 5152ade2e0b3SWang Shilong if (ret != 0) 5153ade2e0b3SWang Shilong return ret; 5154ade2e0b3SWang Shilong } else { 5155ade2e0b3SWang Shilong path->slots[0]--; 5156ade2e0b3SWang Shilong } 5157ade2e0b3SWang Shilong leaf = path->nodes[0]; 5158ade2e0b3SWang Shilong nritems = btrfs_header_nritems(leaf); 5159ade2e0b3SWang Shilong if (nritems == 0) 5160ade2e0b3SWang Shilong return 1; 5161ade2e0b3SWang Shilong if (path->slots[0] == nritems) 5162ade2e0b3SWang Shilong path->slots[0]--; 5163ade2e0b3SWang Shilong 5164ade2e0b3SWang Shilong btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 5165ade2e0b3SWang Shilong if (found_key.objectid < min_objectid) 5166ade2e0b3SWang Shilong break; 5167ade2e0b3SWang Shilong if (found_key.type == BTRFS_EXTENT_ITEM_KEY || 5168ade2e0b3SWang Shilong found_key.type == BTRFS_METADATA_ITEM_KEY) 5169ade2e0b3SWang Shilong return 0; 5170ade2e0b3SWang Shilong if (found_key.objectid == min_objectid && 5171ade2e0b3SWang Shilong found_key.type < BTRFS_EXTENT_ITEM_KEY) 5172ade2e0b3SWang Shilong break; 5173ade2e0b3SWang Shilong } 5174ade2e0b3SWang Shilong return 1; 5175ade2e0b3SWang Shilong } 5176226463d7SJosef Bacik 5177226463d7SJosef Bacik int __init btrfs_ctree_init(void) 5178226463d7SJosef Bacik { 5179226463d7SJosef Bacik btrfs_path_cachep = kmem_cache_create("btrfs_path", 5180226463d7SJosef Bacik sizeof(struct btrfs_path), 0, 5181226463d7SJosef Bacik SLAB_MEM_SPREAD, NULL); 5182226463d7SJosef Bacik if (!btrfs_path_cachep) 5183226463d7SJosef Bacik return -ENOMEM; 5184226463d7SJosef Bacik return 0; 5185226463d7SJosef Bacik } 5186226463d7SJosef Bacik 5187226463d7SJosef Bacik void __cold btrfs_ctree_exit(void) 5188226463d7SJosef Bacik { 5189226463d7SJosef Bacik kmem_cache_destroy(btrfs_path_cachep); 5190226463d7SJosef Bacik } 5191