1c1d7c514SDavid Sterba // SPDX-License-Identifier: GPL-2.0 26cbd5570SChris Mason /* 3d352ac68SChris Mason * Copyright (C) 2007,2008 Oracle. All rights reserved. 46cbd5570SChris Mason */ 56cbd5570SChris Mason 6a6b6e75eSChris Mason #include <linux/sched.h> 75a0e3ad6STejun Heo #include <linux/slab.h> 8bd989ba3SJan Schmidt #include <linux/rbtree.h> 9adf02123SDavid Sterba #include <linux/mm.h> 10e41d12f5SChristoph Hellwig #include <linux/error-injection.h> 119b569ea0SJosef Bacik #include "messages.h" 12eb60ceacSChris Mason #include "ctree.h" 13eb60ceacSChris Mason #include "disk-io.h" 147f5c1516SChris Mason #include "transaction.h" 155f39d397SChris Mason #include "print-tree.h" 16925baeddSChris Mason #include "locking.h" 17de37aa51SNikolay Borisov #include "volumes.h" 18f616f5cdSQu Wenruo #include "qgroup.h" 19f3a84ccdSFilipe Manana #include "tree-mod-log.h" 2088c602abSQu Wenruo #include "tree-checker.h" 21ec8eb376SJosef Bacik #include "fs.h" 22ad1ac501SJosef Bacik #include "accessors.h" 23a0231804SJosef Bacik #include "extent-tree.h" 2467707479SJosef Bacik #include "relocation.h" 256bfd0ffaSJosef Bacik #include "file-item.h" 269a8dd150SChris Mason 27226463d7SJosef Bacik static struct kmem_cache *btrfs_path_cachep; 28226463d7SJosef Bacik 29e089f05cSChris Mason static int split_node(struct btrfs_trans_handle *trans, struct btrfs_root 30e089f05cSChris Mason *root, struct btrfs_path *path, int level); 31310712b2SOmar Sandoval static int split_leaf(struct btrfs_trans_handle *trans, struct btrfs_root *root, 32310712b2SOmar Sandoval const struct btrfs_key *ins_key, struct btrfs_path *path, 33310712b2SOmar Sandoval int data_size, int extend); 345f39d397SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 352ff7e61eSJeff Mahoney struct extent_buffer *dst, 36971a1f66SChris Mason struct extent_buffer *src, int empty); 375f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 385f39d397SChris Mason struct extent_buffer *dst_buf, 395f39d397SChris Mason struct extent_buffer *src_buf); 40d97e63b6SChris Mason 41af024ed2SJohannes Thumshirn static const struct btrfs_csums { 42af024ed2SJohannes Thumshirn u16 size; 4359a0fcdbSDavid Sterba const char name[10]; 4459a0fcdbSDavid Sterba const char driver[12]; 45af024ed2SJohannes Thumshirn } btrfs_csums[] = { 46af024ed2SJohannes Thumshirn [BTRFS_CSUM_TYPE_CRC32] = { .size = 4, .name = "crc32c" }, 473951e7f0SJohannes Thumshirn [BTRFS_CSUM_TYPE_XXHASH] = { .size = 8, .name = "xxhash64" }, 483831bf00SJohannes Thumshirn [BTRFS_CSUM_TYPE_SHA256] = { .size = 32, .name = "sha256" }, 49352ae07bSDavid Sterba [BTRFS_CSUM_TYPE_BLAKE2] = { .size = 32, .name = "blake2b", 50352ae07bSDavid Sterba .driver = "blake2b-256" }, 51af024ed2SJohannes Thumshirn }; 52af024ed2SJohannes Thumshirn 533a3178c7SJosef Bacik /* 543a3178c7SJosef Bacik * The leaf data grows from end-to-front in the node. this returns the address 553a3178c7SJosef Bacik * of the start of the last item, which is the stop of the leaf data stack. 563a3178c7SJosef Bacik */ 573a3178c7SJosef Bacik static unsigned int leaf_data_end(const struct extent_buffer *leaf) 583a3178c7SJosef Bacik { 593a3178c7SJosef Bacik u32 nr = btrfs_header_nritems(leaf); 603a3178c7SJosef Bacik 613a3178c7SJosef Bacik if (nr == 0) 623a3178c7SJosef Bacik return BTRFS_LEAF_DATA_SIZE(leaf->fs_info); 633a3178c7SJosef Bacik return btrfs_item_offset(leaf, nr - 1); 643a3178c7SJosef Bacik } 653a3178c7SJosef Bacik 66637e3b48SJosef Bacik /* 67637e3b48SJosef Bacik * Move data in a @leaf (using memmove, safe for overlapping ranges). 68637e3b48SJosef Bacik * 69637e3b48SJosef Bacik * @leaf: leaf that we're doing a memmove on 70637e3b48SJosef Bacik * @dst_offset: item data offset we're moving to 71637e3b48SJosef Bacik * @src_offset: item data offset were' moving from 72637e3b48SJosef Bacik * @len: length of the data we're moving 73637e3b48SJosef Bacik * 74637e3b48SJosef Bacik * Wrapper around memmove_extent_buffer() that takes into account the header on 75637e3b48SJosef Bacik * the leaf. The btrfs_item offset's start directly after the header, so we 76637e3b48SJosef Bacik * have to adjust any offsets to account for the header in the leaf. This 77637e3b48SJosef Bacik * handles that math to simplify the callers. 78637e3b48SJosef Bacik */ 79637e3b48SJosef Bacik static inline void memmove_leaf_data(const struct extent_buffer *leaf, 80637e3b48SJosef Bacik unsigned long dst_offset, 81637e3b48SJosef Bacik unsigned long src_offset, 82637e3b48SJosef Bacik unsigned long len) 83637e3b48SJosef Bacik { 848009adf3SJosef Bacik memmove_extent_buffer(leaf, btrfs_item_nr_offset(leaf, 0) + dst_offset, 858009adf3SJosef Bacik btrfs_item_nr_offset(leaf, 0) + src_offset, len); 86637e3b48SJosef Bacik } 87637e3b48SJosef Bacik 88637e3b48SJosef Bacik /* 89637e3b48SJosef Bacik * Copy item data from @src into @dst at the given @offset. 90637e3b48SJosef Bacik * 91637e3b48SJosef Bacik * @dst: destination leaf that we're copying into 92637e3b48SJosef Bacik * @src: source leaf that we're copying from 93637e3b48SJosef Bacik * @dst_offset: item data offset we're copying to 94637e3b48SJosef Bacik * @src_offset: item data offset were' copying from 95637e3b48SJosef Bacik * @len: length of the data we're copying 96637e3b48SJosef Bacik * 97637e3b48SJosef Bacik * Wrapper around copy_extent_buffer() that takes into account the header on 98637e3b48SJosef Bacik * the leaf. The btrfs_item offset's start directly after the header, so we 99637e3b48SJosef Bacik * have to adjust any offsets to account for the header in the leaf. This 100637e3b48SJosef Bacik * handles that math to simplify the callers. 101637e3b48SJosef Bacik */ 102637e3b48SJosef Bacik static inline void copy_leaf_data(const struct extent_buffer *dst, 103637e3b48SJosef Bacik const struct extent_buffer *src, 104637e3b48SJosef Bacik unsigned long dst_offset, 105637e3b48SJosef Bacik unsigned long src_offset, unsigned long len) 106637e3b48SJosef Bacik { 1078009adf3SJosef Bacik copy_extent_buffer(dst, src, btrfs_item_nr_offset(dst, 0) + dst_offset, 1088009adf3SJosef Bacik btrfs_item_nr_offset(src, 0) + src_offset, len); 109637e3b48SJosef Bacik } 110637e3b48SJosef Bacik 111637e3b48SJosef Bacik /* 112637e3b48SJosef Bacik * Move items in a @leaf (using memmove). 113637e3b48SJosef Bacik * 114637e3b48SJosef Bacik * @dst: destination leaf for the items 115637e3b48SJosef Bacik * @dst_item: the item nr we're copying into 116637e3b48SJosef Bacik * @src_item: the item nr we're copying from 117637e3b48SJosef Bacik * @nr_items: the number of items to copy 118637e3b48SJosef Bacik * 119637e3b48SJosef Bacik * Wrapper around memmove_extent_buffer() that does the math to get the 120637e3b48SJosef Bacik * appropriate offsets into the leaf from the item numbers. 121637e3b48SJosef Bacik */ 122637e3b48SJosef Bacik static inline void memmove_leaf_items(const struct extent_buffer *leaf, 123637e3b48SJosef Bacik int dst_item, int src_item, int nr_items) 124637e3b48SJosef Bacik { 125637e3b48SJosef Bacik memmove_extent_buffer(leaf, btrfs_item_nr_offset(leaf, dst_item), 126637e3b48SJosef Bacik btrfs_item_nr_offset(leaf, src_item), 127637e3b48SJosef Bacik nr_items * sizeof(struct btrfs_item)); 128637e3b48SJosef Bacik } 129637e3b48SJosef Bacik 130637e3b48SJosef Bacik /* 131637e3b48SJosef Bacik * Copy items from @src into @dst at the given @offset. 132637e3b48SJosef Bacik * 133637e3b48SJosef Bacik * @dst: destination leaf for the items 134637e3b48SJosef Bacik * @src: source leaf for the items 135637e3b48SJosef Bacik * @dst_item: the item nr we're copying into 136637e3b48SJosef Bacik * @src_item: the item nr we're copying from 137637e3b48SJosef Bacik * @nr_items: the number of items to copy 138637e3b48SJosef Bacik * 139637e3b48SJosef Bacik * Wrapper around copy_extent_buffer() that does the math to get the 140637e3b48SJosef Bacik * appropriate offsets into the leaf from the item numbers. 141637e3b48SJosef Bacik */ 142637e3b48SJosef Bacik static inline void copy_leaf_items(const struct extent_buffer *dst, 143637e3b48SJosef Bacik const struct extent_buffer *src, 144637e3b48SJosef Bacik int dst_item, int src_item, int nr_items) 145637e3b48SJosef Bacik { 146637e3b48SJosef Bacik copy_extent_buffer(dst, src, btrfs_item_nr_offset(dst, dst_item), 147637e3b48SJosef Bacik btrfs_item_nr_offset(src, src_item), 148637e3b48SJosef Bacik nr_items * sizeof(struct btrfs_item)); 149637e3b48SJosef Bacik } 150637e3b48SJosef Bacik 151b3cbfb0dSJosef Bacik /* This exists for btrfs-progs usages. */ 152b3cbfb0dSJosef Bacik u16 btrfs_csum_type_size(u16 type) 153b3cbfb0dSJosef Bacik { 154b3cbfb0dSJosef Bacik return btrfs_csums[type].size; 155b3cbfb0dSJosef Bacik } 156b3cbfb0dSJosef Bacik 157af024ed2SJohannes Thumshirn int btrfs_super_csum_size(const struct btrfs_super_block *s) 158af024ed2SJohannes Thumshirn { 159af024ed2SJohannes Thumshirn u16 t = btrfs_super_csum_type(s); 160af024ed2SJohannes Thumshirn /* 161af024ed2SJohannes Thumshirn * csum type is validated at mount time 162af024ed2SJohannes Thumshirn */ 163b3cbfb0dSJosef Bacik return btrfs_csum_type_size(t); 164af024ed2SJohannes Thumshirn } 165af024ed2SJohannes Thumshirn 166af024ed2SJohannes Thumshirn const char *btrfs_super_csum_name(u16 csum_type) 167af024ed2SJohannes Thumshirn { 168af024ed2SJohannes Thumshirn /* csum type is validated at mount time */ 169af024ed2SJohannes Thumshirn return btrfs_csums[csum_type].name; 170af024ed2SJohannes Thumshirn } 171af024ed2SJohannes Thumshirn 172b4e967beSDavid Sterba /* 173b4e967beSDavid Sterba * Return driver name if defined, otherwise the name that's also a valid driver 174b4e967beSDavid Sterba * name 175b4e967beSDavid Sterba */ 176b4e967beSDavid Sterba const char *btrfs_super_csum_driver(u16 csum_type) 177b4e967beSDavid Sterba { 178b4e967beSDavid Sterba /* csum type is validated at mount time */ 17959a0fcdbSDavid Sterba return btrfs_csums[csum_type].driver[0] ? 18059a0fcdbSDavid Sterba btrfs_csums[csum_type].driver : 181b4e967beSDavid Sterba btrfs_csums[csum_type].name; 182b4e967beSDavid Sterba } 183b4e967beSDavid Sterba 184604997b4SDavid Sterba size_t __attribute_const__ btrfs_get_num_csums(void) 185f7cea56cSDavid Sterba { 186f7cea56cSDavid Sterba return ARRAY_SIZE(btrfs_csums); 187f7cea56cSDavid Sterba } 188f7cea56cSDavid Sterba 1892c90e5d6SChris Mason struct btrfs_path *btrfs_alloc_path(void) 1902c90e5d6SChris Mason { 191a4c853afSChenXiaoSong might_sleep(); 192a4c853afSChenXiaoSong 193e2c89907SMasahiro Yamada return kmem_cache_zalloc(btrfs_path_cachep, GFP_NOFS); 1942c90e5d6SChris Mason } 1952c90e5d6SChris Mason 196d352ac68SChris Mason /* this also releases the path */ 1972c90e5d6SChris Mason void btrfs_free_path(struct btrfs_path *p) 1982c90e5d6SChris Mason { 199ff175d57SJesper Juhl if (!p) 200ff175d57SJesper Juhl return; 201b3b4aa74SDavid Sterba btrfs_release_path(p); 2022c90e5d6SChris Mason kmem_cache_free(btrfs_path_cachep, p); 2032c90e5d6SChris Mason } 2042c90e5d6SChris Mason 205d352ac68SChris Mason /* 206d352ac68SChris Mason * path release drops references on the extent buffers in the path 207d352ac68SChris Mason * and it drops any locks held by this path 208d352ac68SChris Mason * 209d352ac68SChris Mason * It is safe to call this on paths that no locks or extent buffers held. 210d352ac68SChris Mason */ 211b3b4aa74SDavid Sterba noinline void btrfs_release_path(struct btrfs_path *p) 212eb60ceacSChris Mason { 213eb60ceacSChris Mason int i; 214a2135011SChris Mason 215234b63a0SChris Mason for (i = 0; i < BTRFS_MAX_LEVEL; i++) { 2163f157a2fSChris Mason p->slots[i] = 0; 217eb60ceacSChris Mason if (!p->nodes[i]) 218925baeddSChris Mason continue; 219925baeddSChris Mason if (p->locks[i]) { 220bd681513SChris Mason btrfs_tree_unlock_rw(p->nodes[i], p->locks[i]); 221925baeddSChris Mason p->locks[i] = 0; 222925baeddSChris Mason } 2235f39d397SChris Mason free_extent_buffer(p->nodes[i]); 2243f157a2fSChris Mason p->nodes[i] = NULL; 225eb60ceacSChris Mason } 226eb60ceacSChris Mason } 227eb60ceacSChris Mason 228d352ac68SChris Mason /* 2298bb808c6SDavid Sterba * We want the transaction abort to print stack trace only for errors where the 2308bb808c6SDavid Sterba * cause could be a bug, eg. due to ENOSPC, and not for common errors that are 2318bb808c6SDavid Sterba * caused by external factors. 2328bb808c6SDavid Sterba */ 2338bb808c6SDavid Sterba bool __cold abort_should_print_stack(int errno) 2348bb808c6SDavid Sterba { 2358bb808c6SDavid Sterba switch (errno) { 2368bb808c6SDavid Sterba case -EIO: 2378bb808c6SDavid Sterba case -EROFS: 2388bb808c6SDavid Sterba case -ENOMEM: 2398bb808c6SDavid Sterba return false; 2408bb808c6SDavid Sterba } 2418bb808c6SDavid Sterba return true; 2428bb808c6SDavid Sterba } 2438bb808c6SDavid Sterba 2448bb808c6SDavid Sterba /* 245d352ac68SChris Mason * safely gets a reference on the root node of a tree. A lock 246d352ac68SChris Mason * is not taken, so a concurrent writer may put a different node 247d352ac68SChris Mason * at the root of the tree. See btrfs_lock_root_node for the 248d352ac68SChris Mason * looping required. 249d352ac68SChris Mason * 250d352ac68SChris Mason * The extent buffer returned by this has a reference taken, so 251d352ac68SChris Mason * it won't disappear. It may stop being the root of the tree 252d352ac68SChris Mason * at any time because there are no locks held. 253d352ac68SChris Mason */ 254925baeddSChris Mason struct extent_buffer *btrfs_root_node(struct btrfs_root *root) 255925baeddSChris Mason { 256925baeddSChris Mason struct extent_buffer *eb; 257240f62c8SChris Mason 2583083ee2eSJosef Bacik while (1) { 259240f62c8SChris Mason rcu_read_lock(); 260240f62c8SChris Mason eb = rcu_dereference(root->node); 2613083ee2eSJosef Bacik 2623083ee2eSJosef Bacik /* 2633083ee2eSJosef Bacik * RCU really hurts here, we could free up the root node because 26401327610SNicholas D Steeves * it was COWed but we may not get the new root node yet so do 2653083ee2eSJosef Bacik * the inc_not_zero dance and if it doesn't work then 2663083ee2eSJosef Bacik * synchronize_rcu and try again. 2673083ee2eSJosef Bacik */ 2683083ee2eSJosef Bacik if (atomic_inc_not_zero(&eb->refs)) { 269240f62c8SChris Mason rcu_read_unlock(); 2703083ee2eSJosef Bacik break; 2713083ee2eSJosef Bacik } 2723083ee2eSJosef Bacik rcu_read_unlock(); 2733083ee2eSJosef Bacik synchronize_rcu(); 2743083ee2eSJosef Bacik } 275925baeddSChris Mason return eb; 276925baeddSChris Mason } 277925baeddSChris Mason 27892a7cc42SQu Wenruo /* 27992a7cc42SQu Wenruo * Cowonly root (not-shareable trees, everything not subvolume or reloc roots), 28092a7cc42SQu Wenruo * just get put onto a simple dirty list. Transaction walks this list to make 28192a7cc42SQu Wenruo * sure they get properly updated on disk. 282d352ac68SChris Mason */ 2830b86a832SChris Mason static void add_root_to_dirty_list(struct btrfs_root *root) 2840b86a832SChris Mason { 2850b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 2860b246afaSJeff Mahoney 287e7070be1SJosef Bacik if (test_bit(BTRFS_ROOT_DIRTY, &root->state) || 288e7070be1SJosef Bacik !test_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state)) 289e7070be1SJosef Bacik return; 290e7070be1SJosef Bacik 2910b246afaSJeff Mahoney spin_lock(&fs_info->trans_lock); 292e7070be1SJosef Bacik if (!test_and_set_bit(BTRFS_ROOT_DIRTY, &root->state)) { 293e7070be1SJosef Bacik /* Want the extent tree to be the last on the list */ 2944fd786e6SMisono Tomohiro if (root->root_key.objectid == BTRFS_EXTENT_TREE_OBJECTID) 295e7070be1SJosef Bacik list_move_tail(&root->dirty_list, 2960b246afaSJeff Mahoney &fs_info->dirty_cowonly_roots); 297e7070be1SJosef Bacik else 298e7070be1SJosef Bacik list_move(&root->dirty_list, 2990b246afaSJeff Mahoney &fs_info->dirty_cowonly_roots); 3000b86a832SChris Mason } 3010b246afaSJeff Mahoney spin_unlock(&fs_info->trans_lock); 3020b86a832SChris Mason } 3030b86a832SChris Mason 304d352ac68SChris Mason /* 305d352ac68SChris Mason * used by snapshot creation to make a copy of a root for a tree with 306d352ac68SChris Mason * a given objectid. The buffer with the new root node is returned in 307d352ac68SChris Mason * cow_ret, and this func returns zero on success or a negative error code. 308d352ac68SChris Mason */ 309be20aa9dSChris Mason int btrfs_copy_root(struct btrfs_trans_handle *trans, 310be20aa9dSChris Mason struct btrfs_root *root, 311be20aa9dSChris Mason struct extent_buffer *buf, 312be20aa9dSChris Mason struct extent_buffer **cow_ret, u64 new_root_objectid) 313be20aa9dSChris Mason { 3140b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 315be20aa9dSChris Mason struct extent_buffer *cow; 316be20aa9dSChris Mason int ret = 0; 317be20aa9dSChris Mason int level; 3185d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 319be20aa9dSChris Mason 32092a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 3210b246afaSJeff Mahoney trans->transid != fs_info->running_transaction->transid); 32292a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 32327cdeb70SMiao Xie trans->transid != root->last_trans); 324be20aa9dSChris Mason 325be20aa9dSChris Mason level = btrfs_header_level(buf); 3265d4f98a2SYan Zheng if (level == 0) 3275d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 3285d4f98a2SYan Zheng else 3295d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 33031840ae1SZheng Yan 3314d75f8a9SDavid Sterba cow = btrfs_alloc_tree_block(trans, root, 0, new_root_objectid, 332cf6f34aaSJosef Bacik &disk_key, level, buf->start, 0, 333cf6f34aaSJosef Bacik BTRFS_NESTING_NEW_ROOT); 3345d4f98a2SYan Zheng if (IS_ERR(cow)) 335be20aa9dSChris Mason return PTR_ERR(cow); 336be20aa9dSChris Mason 33758e8012cSDavid Sterba copy_extent_buffer_full(cow, buf); 338be20aa9dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 339be20aa9dSChris Mason btrfs_set_header_generation(cow, trans->transid); 3405d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 3415d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 3425d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 3435d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 3445d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 3455d4f98a2SYan Zheng else 346be20aa9dSChris Mason btrfs_set_header_owner(cow, new_root_objectid); 347be20aa9dSChris Mason 348de37aa51SNikolay Borisov write_extent_buffer_fsid(cow, fs_info->fs_devices->metadata_uuid); 3492b82032cSYan Zheng 350be20aa9dSChris Mason WARN_ON(btrfs_header_generation(buf) > trans->transid); 3515d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 352e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 3535d4f98a2SYan Zheng else 354e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 355867ed321SJosef Bacik if (ret) { 35672c9925fSFilipe Manana btrfs_tree_unlock(cow); 35772c9925fSFilipe Manana free_extent_buffer(cow); 358867ed321SJosef Bacik btrfs_abort_transaction(trans, ret); 359be20aa9dSChris Mason return ret; 360867ed321SJosef Bacik } 361be20aa9dSChris Mason 362be20aa9dSChris Mason btrfs_mark_buffer_dirty(cow); 363be20aa9dSChris Mason *cow_ret = cow; 364be20aa9dSChris Mason return 0; 365be20aa9dSChris Mason } 366be20aa9dSChris Mason 367d352ac68SChris Mason /* 3685d4f98a2SYan Zheng * check if the tree block can be shared by multiple trees 3695d4f98a2SYan Zheng */ 3705d4f98a2SYan Zheng int btrfs_block_can_be_shared(struct btrfs_root *root, 3715d4f98a2SYan Zheng struct extent_buffer *buf) 3725d4f98a2SYan Zheng { 3735d4f98a2SYan Zheng /* 37492a7cc42SQu Wenruo * Tree blocks not in shareable trees and tree roots are never shared. 37592a7cc42SQu Wenruo * If a block was allocated after the last snapshot and the block was 37692a7cc42SQu Wenruo * not allocated by tree relocation, we know the block is not shared. 3775d4f98a2SYan Zheng */ 37892a7cc42SQu Wenruo if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 3795d4f98a2SYan Zheng buf != root->node && buf != root->commit_root && 3805d4f98a2SYan Zheng (btrfs_header_generation(buf) <= 3815d4f98a2SYan Zheng btrfs_root_last_snapshot(&root->root_item) || 3825d4f98a2SYan Zheng btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC))) 3835d4f98a2SYan Zheng return 1; 384a79865c6SNikolay Borisov 3855d4f98a2SYan Zheng return 0; 3865d4f98a2SYan Zheng } 3875d4f98a2SYan Zheng 3885d4f98a2SYan Zheng static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans, 3895d4f98a2SYan Zheng struct btrfs_root *root, 3905d4f98a2SYan Zheng struct extent_buffer *buf, 391f0486c68SYan, Zheng struct extent_buffer *cow, 392f0486c68SYan, Zheng int *last_ref) 3935d4f98a2SYan Zheng { 3940b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 3955d4f98a2SYan Zheng u64 refs; 3965d4f98a2SYan Zheng u64 owner; 3975d4f98a2SYan Zheng u64 flags; 3985d4f98a2SYan Zheng u64 new_flags = 0; 3995d4f98a2SYan Zheng int ret; 4005d4f98a2SYan Zheng 4015d4f98a2SYan Zheng /* 4025d4f98a2SYan Zheng * Backrefs update rules: 4035d4f98a2SYan Zheng * 4045d4f98a2SYan Zheng * Always use full backrefs for extent pointers in tree block 4055d4f98a2SYan Zheng * allocated by tree relocation. 4065d4f98a2SYan Zheng * 4075d4f98a2SYan Zheng * If a shared tree block is no longer referenced by its owner 4085d4f98a2SYan Zheng * tree (btrfs_header_owner(buf) == root->root_key.objectid), 4095d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 4105d4f98a2SYan Zheng * 4115d4f98a2SYan Zheng * If a tree block is been relocating 4125d4f98a2SYan Zheng * (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID), 4135d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 4145d4f98a2SYan Zheng * The reason for this is some operations (such as drop tree) 4155d4f98a2SYan Zheng * are only allowed for blocks use full backrefs. 4165d4f98a2SYan Zheng */ 4175d4f98a2SYan Zheng 4185d4f98a2SYan Zheng if (btrfs_block_can_be_shared(root, buf)) { 4192ff7e61eSJeff Mahoney ret = btrfs_lookup_extent_info(trans, fs_info, buf->start, 4203173a18fSJosef Bacik btrfs_header_level(buf), 1, 4213173a18fSJosef Bacik &refs, &flags); 422be1a5564SMark Fasheh if (ret) 423be1a5564SMark Fasheh return ret; 424e5df9573SMark Fasheh if (refs == 0) { 425e5df9573SMark Fasheh ret = -EROFS; 4260b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 427e5df9573SMark Fasheh return ret; 428e5df9573SMark Fasheh } 4295d4f98a2SYan Zheng } else { 4305d4f98a2SYan Zheng refs = 1; 4315d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 4325d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 4335d4f98a2SYan Zheng flags = BTRFS_BLOCK_FLAG_FULL_BACKREF; 4345d4f98a2SYan Zheng else 4355d4f98a2SYan Zheng flags = 0; 4365d4f98a2SYan Zheng } 4375d4f98a2SYan Zheng 4385d4f98a2SYan Zheng owner = btrfs_header_owner(buf); 4395d4f98a2SYan Zheng BUG_ON(owner == BTRFS_TREE_RELOC_OBJECTID && 4405d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)); 4415d4f98a2SYan Zheng 4425d4f98a2SYan Zheng if (refs > 1) { 4435d4f98a2SYan Zheng if ((owner == root->root_key.objectid || 4445d4f98a2SYan Zheng root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && 4455d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) { 446e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, buf, 1); 447692826b2SJeff Mahoney if (ret) 448692826b2SJeff Mahoney return ret; 4495d4f98a2SYan Zheng 4505d4f98a2SYan Zheng if (root->root_key.objectid == 4515d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) { 452e339a6b0SJosef Bacik ret = btrfs_dec_ref(trans, root, buf, 0); 453692826b2SJeff Mahoney if (ret) 454692826b2SJeff Mahoney return ret; 455e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 456692826b2SJeff Mahoney if (ret) 457692826b2SJeff Mahoney return ret; 4585d4f98a2SYan Zheng } 4595d4f98a2SYan Zheng new_flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF; 4605d4f98a2SYan Zheng } else { 4615d4f98a2SYan Zheng 4625d4f98a2SYan Zheng if (root->root_key.objectid == 4635d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 464e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 4655d4f98a2SYan Zheng else 466e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 467692826b2SJeff Mahoney if (ret) 468692826b2SJeff Mahoney return ret; 4695d4f98a2SYan Zheng } 4705d4f98a2SYan Zheng if (new_flags != 0) { 4714aec05faSJosef Bacik ret = btrfs_set_disk_extent_flags(trans, buf, new_flags); 472be1a5564SMark Fasheh if (ret) 473be1a5564SMark Fasheh return ret; 4745d4f98a2SYan Zheng } 4755d4f98a2SYan Zheng } else { 4765d4f98a2SYan Zheng if (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) { 4775d4f98a2SYan Zheng if (root->root_key.objectid == 4785d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 479e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 4805d4f98a2SYan Zheng else 481e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 482692826b2SJeff Mahoney if (ret) 483692826b2SJeff Mahoney return ret; 484e339a6b0SJosef Bacik ret = btrfs_dec_ref(trans, root, buf, 1); 485692826b2SJeff Mahoney if (ret) 486692826b2SJeff Mahoney return ret; 4875d4f98a2SYan Zheng } 488190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, buf); 489f0486c68SYan, Zheng *last_ref = 1; 4905d4f98a2SYan Zheng } 4915d4f98a2SYan Zheng return 0; 4925d4f98a2SYan Zheng } 4935d4f98a2SYan Zheng 4945d4f98a2SYan Zheng /* 495d397712bSChris Mason * does the dirty work in cow of a single block. The parent block (if 496d397712bSChris Mason * supplied) is updated to point to the new cow copy. The new buffer is marked 497d397712bSChris Mason * dirty and returned locked. If you modify the block it needs to be marked 498d397712bSChris Mason * dirty again. 499d352ac68SChris Mason * 500d352ac68SChris Mason * search_start -- an allocation hint for the new block 501d352ac68SChris Mason * 502d397712bSChris Mason * empty_size -- a hint that you plan on doing more cow. This is the size in 503d397712bSChris Mason * bytes the allocator should try to find free next to the block it returns. 504d397712bSChris Mason * This is just a hint and may be ignored by the allocator. 505d352ac68SChris Mason */ 506d397712bSChris Mason static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans, 5075f39d397SChris Mason struct btrfs_root *root, 5085f39d397SChris Mason struct extent_buffer *buf, 5095f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 5105f39d397SChris Mason struct extent_buffer **cow_ret, 5119631e4ccSJosef Bacik u64 search_start, u64 empty_size, 5129631e4ccSJosef Bacik enum btrfs_lock_nesting nest) 5136702ed49SChris Mason { 5140b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 5155d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 5165f39d397SChris Mason struct extent_buffer *cow; 517be1a5564SMark Fasheh int level, ret; 518f0486c68SYan, Zheng int last_ref = 0; 519925baeddSChris Mason int unlock_orig = 0; 5200f5053ebSGoldwyn Rodrigues u64 parent_start = 0; 5216702ed49SChris Mason 522925baeddSChris Mason if (*cow_ret == buf) 523925baeddSChris Mason unlock_orig = 1; 524925baeddSChris Mason 52549d0c642SFilipe Manana btrfs_assert_tree_write_locked(buf); 526925baeddSChris Mason 52792a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 5280b246afaSJeff Mahoney trans->transid != fs_info->running_transaction->transid); 52992a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 53027cdeb70SMiao Xie trans->transid != root->last_trans); 5315f39d397SChris Mason 5327bb86316SChris Mason level = btrfs_header_level(buf); 53331840ae1SZheng Yan 5345d4f98a2SYan Zheng if (level == 0) 5355d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 5365d4f98a2SYan Zheng else 5375d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 5385d4f98a2SYan Zheng 5390f5053ebSGoldwyn Rodrigues if ((root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && parent) 5405d4f98a2SYan Zheng parent_start = parent->start; 5415d4f98a2SYan Zheng 54279bd3712SFilipe Manana cow = btrfs_alloc_tree_block(trans, root, parent_start, 54379bd3712SFilipe Manana root->root_key.objectid, &disk_key, level, 54479bd3712SFilipe Manana search_start, empty_size, nest); 5456702ed49SChris Mason if (IS_ERR(cow)) 5466702ed49SChris Mason return PTR_ERR(cow); 5476702ed49SChris Mason 548b4ce94deSChris Mason /* cow is set to blocking by btrfs_init_new_buffer */ 549b4ce94deSChris Mason 55058e8012cSDavid Sterba copy_extent_buffer_full(cow, buf); 551db94535dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 5525f39d397SChris Mason btrfs_set_header_generation(cow, trans->transid); 5535d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 5545d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 5555d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 5565d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) 5575d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 5585d4f98a2SYan Zheng else 5595f39d397SChris Mason btrfs_set_header_owner(cow, root->root_key.objectid); 5606702ed49SChris Mason 561de37aa51SNikolay Borisov write_extent_buffer_fsid(cow, fs_info->fs_devices->metadata_uuid); 5622b82032cSYan Zheng 563be1a5564SMark Fasheh ret = update_ref_for_cow(trans, root, buf, cow, &last_ref); 564b68dc2a9SMark Fasheh if (ret) { 565572c83acSJosef Bacik btrfs_tree_unlock(cow); 566572c83acSJosef Bacik free_extent_buffer(cow); 56766642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 568b68dc2a9SMark Fasheh return ret; 569b68dc2a9SMark Fasheh } 5701a40e23bSZheng Yan 57192a7cc42SQu Wenruo if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) { 57283d4cfd4SJosef Bacik ret = btrfs_reloc_cow_block(trans, root, buf, cow); 57393314e3bSZhaolei if (ret) { 574572c83acSJosef Bacik btrfs_tree_unlock(cow); 575572c83acSJosef Bacik free_extent_buffer(cow); 57666642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 57783d4cfd4SJosef Bacik return ret; 57883d4cfd4SJosef Bacik } 57993314e3bSZhaolei } 5803fd0a558SYan, Zheng 5816702ed49SChris Mason if (buf == root->node) { 582925baeddSChris Mason WARN_ON(parent && parent != buf); 5835d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 5845d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 5855d4f98a2SYan Zheng parent_start = buf->start; 586925baeddSChris Mason 587406808abSFilipe Manana ret = btrfs_tree_mod_log_insert_root(root->node, cow, true); 588*40b0a749SFilipe Manana if (ret < 0) { 589*40b0a749SFilipe Manana btrfs_tree_unlock(cow); 590*40b0a749SFilipe Manana free_extent_buffer(cow); 591*40b0a749SFilipe Manana btrfs_abort_transaction(trans, ret); 592*40b0a749SFilipe Manana return ret; 593*40b0a749SFilipe Manana } 594*40b0a749SFilipe Manana atomic_inc(&cow->refs); 595240f62c8SChris Mason rcu_assign_pointer(root->node, cow); 596925baeddSChris Mason 5977a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), buf, 5987a163608SFilipe Manana parent_start, last_ref); 5995f39d397SChris Mason free_extent_buffer(buf); 6000b86a832SChris Mason add_root_to_dirty_list(root); 6016702ed49SChris Mason } else { 6025d4f98a2SYan Zheng WARN_ON(trans->transid != btrfs_header_generation(parent)); 603d09c5152SFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, parent_slot, 60433cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 605d09c5152SFilipe Manana if (ret) { 606d09c5152SFilipe Manana btrfs_tree_unlock(cow); 607d09c5152SFilipe Manana free_extent_buffer(cow); 608d09c5152SFilipe Manana btrfs_abort_transaction(trans, ret); 609d09c5152SFilipe Manana return ret; 610d09c5152SFilipe Manana } 6115f39d397SChris Mason btrfs_set_node_blockptr(parent, parent_slot, 612db94535dSChris Mason cow->start); 61374493f7aSChris Mason btrfs_set_node_ptr_generation(parent, parent_slot, 61474493f7aSChris Mason trans->transid); 6156702ed49SChris Mason btrfs_mark_buffer_dirty(parent); 6165de865eeSFilipe David Borba Manana if (last_ref) { 617f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_free_eb(buf); 6185de865eeSFilipe David Borba Manana if (ret) { 619572c83acSJosef Bacik btrfs_tree_unlock(cow); 620572c83acSJosef Bacik free_extent_buffer(cow); 62166642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 6225de865eeSFilipe David Borba Manana return ret; 6235de865eeSFilipe David Borba Manana } 6245de865eeSFilipe David Borba Manana } 6257a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), buf, 6267a163608SFilipe Manana parent_start, last_ref); 6276702ed49SChris Mason } 628925baeddSChris Mason if (unlock_orig) 629925baeddSChris Mason btrfs_tree_unlock(buf); 6303083ee2eSJosef Bacik free_extent_buffer_stale(buf); 6316702ed49SChris Mason btrfs_mark_buffer_dirty(cow); 6326702ed49SChris Mason *cow_ret = cow; 6336702ed49SChris Mason return 0; 6346702ed49SChris Mason } 6356702ed49SChris Mason 6365d4f98a2SYan Zheng static inline int should_cow_block(struct btrfs_trans_handle *trans, 6375d4f98a2SYan Zheng struct btrfs_root *root, 6385d4f98a2SYan Zheng struct extent_buffer *buf) 6395d4f98a2SYan Zheng { 640f5ee5c9aSJeff Mahoney if (btrfs_is_testing(root->fs_info)) 641faa2dbf0SJosef Bacik return 0; 642fccb84c9SDavid Sterba 643d1980131SDavid Sterba /* Ensure we can see the FORCE_COW bit */ 644d1980131SDavid Sterba smp_mb__before_atomic(); 645f1ebcc74SLiu Bo 646f1ebcc74SLiu Bo /* 647f1ebcc74SLiu Bo * We do not need to cow a block if 648f1ebcc74SLiu Bo * 1) this block is not created or changed in this transaction; 649f1ebcc74SLiu Bo * 2) this block does not belong to TREE_RELOC tree; 650f1ebcc74SLiu Bo * 3) the root is not forced COW. 651f1ebcc74SLiu Bo * 652f1ebcc74SLiu Bo * What is forced COW: 65301327610SNicholas D Steeves * when we create snapshot during committing the transaction, 65452042d8eSAndrea Gelmini * after we've finished copying src root, we must COW the shared 655f1ebcc74SLiu Bo * block to ensure the metadata consistency. 656f1ebcc74SLiu Bo */ 6575d4f98a2SYan Zheng if (btrfs_header_generation(buf) == trans->transid && 6585d4f98a2SYan Zheng !btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN) && 6595d4f98a2SYan Zheng !(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID && 660f1ebcc74SLiu Bo btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)) && 66127cdeb70SMiao Xie !test_bit(BTRFS_ROOT_FORCE_COW, &root->state)) 6625d4f98a2SYan Zheng return 0; 6635d4f98a2SYan Zheng return 1; 6645d4f98a2SYan Zheng } 6655d4f98a2SYan Zheng 666d352ac68SChris Mason /* 667d352ac68SChris Mason * cows a single block, see __btrfs_cow_block for the real work. 66801327610SNicholas D Steeves * This version of it has extra checks so that a block isn't COWed more than 669d352ac68SChris Mason * once per transaction, as long as it hasn't been written yet 670d352ac68SChris Mason */ 671d397712bSChris Mason noinline int btrfs_cow_block(struct btrfs_trans_handle *trans, 6725f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *buf, 6735f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 6749631e4ccSJosef Bacik struct extent_buffer **cow_ret, 6759631e4ccSJosef Bacik enum btrfs_lock_nesting nest) 67602217ed2SChris Mason { 6770b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 6786702ed49SChris Mason u64 search_start; 679f510cfecSChris Mason int ret; 680dc17ff8fSChris Mason 68183354f07SJosef Bacik if (test_bit(BTRFS_ROOT_DELETING, &root->state)) 68283354f07SJosef Bacik btrfs_err(fs_info, 68383354f07SJosef Bacik "COW'ing blocks on a fs root that's being dropped"); 68483354f07SJosef Bacik 6850b246afaSJeff Mahoney if (trans->transaction != fs_info->running_transaction) 68631b1a2bdSJulia Lawall WARN(1, KERN_CRIT "trans %llu running %llu\n", 687c1c9ff7cSGeert Uytterhoeven trans->transid, 6880b246afaSJeff Mahoney fs_info->running_transaction->transid); 68931b1a2bdSJulia Lawall 6900b246afaSJeff Mahoney if (trans->transid != fs_info->generation) 69131b1a2bdSJulia Lawall WARN(1, KERN_CRIT "trans %llu running %llu\n", 6920b246afaSJeff Mahoney trans->transid, fs_info->generation); 693dc17ff8fSChris Mason 6945d4f98a2SYan Zheng if (!should_cow_block(trans, root, buf)) { 69502217ed2SChris Mason *cow_ret = buf; 69602217ed2SChris Mason return 0; 69702217ed2SChris Mason } 698c487685dSChris Mason 699ee22184bSByongho Lee search_start = buf->start & ~((u64)SZ_1G - 1); 700b4ce94deSChris Mason 701f616f5cdSQu Wenruo /* 702f616f5cdSQu Wenruo * Before CoWing this block for later modification, check if it's 703f616f5cdSQu Wenruo * the subtree root and do the delayed subtree trace if needed. 704f616f5cdSQu Wenruo * 705f616f5cdSQu Wenruo * Also We don't care about the error, as it's handled internally. 706f616f5cdSQu Wenruo */ 707f616f5cdSQu Wenruo btrfs_qgroup_trace_subtree_after_cow(trans, root, buf); 708f510cfecSChris Mason ret = __btrfs_cow_block(trans, root, buf, parent, 7099631e4ccSJosef Bacik parent_slot, cow_ret, search_start, 0, nest); 7101abe9b8aSliubo 7111abe9b8aSliubo trace_btrfs_cow_block(root, buf, *cow_ret); 7121abe9b8aSliubo 713f510cfecSChris Mason return ret; 7142c90e5d6SChris Mason } 715f75e2b79SJosef Bacik ALLOW_ERROR_INJECTION(btrfs_cow_block, ERRNO); 7166702ed49SChris Mason 717d352ac68SChris Mason /* 718d352ac68SChris Mason * helper function for defrag to decide if two blocks pointed to by a 719d352ac68SChris Mason * node are actually close by 720d352ac68SChris Mason */ 7216b80053dSChris Mason static int close_blocks(u64 blocknr, u64 other, u32 blocksize) 7226702ed49SChris Mason { 7236b80053dSChris Mason if (blocknr < other && other - (blocknr + blocksize) < 32768) 7246702ed49SChris Mason return 1; 7256b80053dSChris Mason if (blocknr > other && blocknr - (other + blocksize) < 32768) 7266702ed49SChris Mason return 1; 72702217ed2SChris Mason return 0; 72802217ed2SChris Mason } 72902217ed2SChris Mason 730ce6ef5abSDavid Sterba #ifdef __LITTLE_ENDIAN 731ce6ef5abSDavid Sterba 732ce6ef5abSDavid Sterba /* 733ce6ef5abSDavid Sterba * Compare two keys, on little-endian the disk order is same as CPU order and 734ce6ef5abSDavid Sterba * we can avoid the conversion. 735ce6ef5abSDavid Sterba */ 736ce6ef5abSDavid Sterba static int comp_keys(const struct btrfs_disk_key *disk_key, 737ce6ef5abSDavid Sterba const struct btrfs_key *k2) 738ce6ef5abSDavid Sterba { 739ce6ef5abSDavid Sterba const struct btrfs_key *k1 = (const struct btrfs_key *)disk_key; 740ce6ef5abSDavid Sterba 741ce6ef5abSDavid Sterba return btrfs_comp_cpu_keys(k1, k2); 742ce6ef5abSDavid Sterba } 743ce6ef5abSDavid Sterba 744ce6ef5abSDavid Sterba #else 745ce6ef5abSDavid Sterba 746081e9573SChris Mason /* 747081e9573SChris Mason * compare two keys in a memcmp fashion 748081e9573SChris Mason */ 749310712b2SOmar Sandoval static int comp_keys(const struct btrfs_disk_key *disk, 750310712b2SOmar Sandoval const struct btrfs_key *k2) 751081e9573SChris Mason { 752081e9573SChris Mason struct btrfs_key k1; 753081e9573SChris Mason 754081e9573SChris Mason btrfs_disk_key_to_cpu(&k1, disk); 755081e9573SChris Mason 75620736abaSDiego Calleja return btrfs_comp_cpu_keys(&k1, k2); 757081e9573SChris Mason } 758ce6ef5abSDavid Sterba #endif 759081e9573SChris Mason 760f3465ca4SJosef Bacik /* 761f3465ca4SJosef Bacik * same as comp_keys only with two btrfs_key's 762f3465ca4SJosef Bacik */ 763e1f60a65SDavid Sterba int __pure btrfs_comp_cpu_keys(const struct btrfs_key *k1, const struct btrfs_key *k2) 764f3465ca4SJosef Bacik { 765f3465ca4SJosef Bacik if (k1->objectid > k2->objectid) 766f3465ca4SJosef Bacik return 1; 767f3465ca4SJosef Bacik if (k1->objectid < k2->objectid) 768f3465ca4SJosef Bacik return -1; 769f3465ca4SJosef Bacik if (k1->type > k2->type) 770f3465ca4SJosef Bacik return 1; 771f3465ca4SJosef Bacik if (k1->type < k2->type) 772f3465ca4SJosef Bacik return -1; 773f3465ca4SJosef Bacik if (k1->offset > k2->offset) 774f3465ca4SJosef Bacik return 1; 775f3465ca4SJosef Bacik if (k1->offset < k2->offset) 776f3465ca4SJosef Bacik return -1; 777f3465ca4SJosef Bacik return 0; 778f3465ca4SJosef Bacik } 779081e9573SChris Mason 780d352ac68SChris Mason /* 781d352ac68SChris Mason * this is used by the defrag code to go through all the 782d352ac68SChris Mason * leaves pointed to by a node and reallocate them so that 783d352ac68SChris Mason * disk order is close to key order 784d352ac68SChris Mason */ 7856702ed49SChris Mason int btrfs_realloc_node(struct btrfs_trans_handle *trans, 7865f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *parent, 787de78b51aSEric Sandeen int start_slot, u64 *last_ret, 788a6b6e75eSChris Mason struct btrfs_key *progress) 7896702ed49SChris Mason { 7900b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 7916b80053dSChris Mason struct extent_buffer *cur; 7926702ed49SChris Mason u64 blocknr; 793e9d0b13bSChris Mason u64 search_start = *last_ret; 794e9d0b13bSChris Mason u64 last_block = 0; 7956702ed49SChris Mason u64 other; 7966702ed49SChris Mason u32 parent_nritems; 7976702ed49SChris Mason int end_slot; 7986702ed49SChris Mason int i; 7996702ed49SChris Mason int err = 0; 8006b80053dSChris Mason u32 blocksize; 801081e9573SChris Mason int progress_passed = 0; 802081e9573SChris Mason struct btrfs_disk_key disk_key; 8036702ed49SChris Mason 8040b246afaSJeff Mahoney WARN_ON(trans->transaction != fs_info->running_transaction); 8050b246afaSJeff Mahoney WARN_ON(trans->transid != fs_info->generation); 80686479a04SChris Mason 8076b80053dSChris Mason parent_nritems = btrfs_header_nritems(parent); 8080b246afaSJeff Mahoney blocksize = fs_info->nodesize; 8095dfe2be7SFilipe Manana end_slot = parent_nritems - 1; 8106702ed49SChris Mason 8115dfe2be7SFilipe Manana if (parent_nritems <= 1) 8126702ed49SChris Mason return 0; 8136702ed49SChris Mason 8145dfe2be7SFilipe Manana for (i = start_slot; i <= end_slot; i++) { 8156702ed49SChris Mason int close = 1; 816a6b6e75eSChris Mason 817081e9573SChris Mason btrfs_node_key(parent, &disk_key, i); 818081e9573SChris Mason if (!progress_passed && comp_keys(&disk_key, progress) < 0) 819081e9573SChris Mason continue; 820081e9573SChris Mason 821081e9573SChris Mason progress_passed = 1; 8226b80053dSChris Mason blocknr = btrfs_node_blockptr(parent, i); 823e9d0b13bSChris Mason if (last_block == 0) 824e9d0b13bSChris Mason last_block = blocknr; 8255708b959SChris Mason 8266702ed49SChris Mason if (i > 0) { 8276b80053dSChris Mason other = btrfs_node_blockptr(parent, i - 1); 8286b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 8296702ed49SChris Mason } 8305dfe2be7SFilipe Manana if (!close && i < end_slot) { 8316b80053dSChris Mason other = btrfs_node_blockptr(parent, i + 1); 8326b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 8336702ed49SChris Mason } 834e9d0b13bSChris Mason if (close) { 835e9d0b13bSChris Mason last_block = blocknr; 8366702ed49SChris Mason continue; 837e9d0b13bSChris Mason } 8386702ed49SChris Mason 839206983b7SJosef Bacik cur = btrfs_read_node_slot(parent, i); 840206983b7SJosef Bacik if (IS_ERR(cur)) 84164c043deSLiu Bo return PTR_ERR(cur); 842e9d0b13bSChris Mason if (search_start == 0) 8436b80053dSChris Mason search_start = last_block; 844e9d0b13bSChris Mason 845e7a84565SChris Mason btrfs_tree_lock(cur); 8466b80053dSChris Mason err = __btrfs_cow_block(trans, root, cur, parent, i, 847e7a84565SChris Mason &cur, search_start, 8486b80053dSChris Mason min(16 * blocksize, 8499631e4ccSJosef Bacik (end_slot - i) * blocksize), 8509631e4ccSJosef Bacik BTRFS_NESTING_COW); 851252c38f0SYan if (err) { 852e7a84565SChris Mason btrfs_tree_unlock(cur); 8536b80053dSChris Mason free_extent_buffer(cur); 8546702ed49SChris Mason break; 855252c38f0SYan } 856e7a84565SChris Mason search_start = cur->start; 857e7a84565SChris Mason last_block = cur->start; 858f2183bdeSChris Mason *last_ret = search_start; 859e7a84565SChris Mason btrfs_tree_unlock(cur); 860e7a84565SChris Mason free_extent_buffer(cur); 8616702ed49SChris Mason } 8626702ed49SChris Mason return err; 8636702ed49SChris Mason } 8646702ed49SChris Mason 86574123bd7SChris Mason /* 866fb81212cSFilipe Manana * Search for a key in the given extent_buffer. 8675f39d397SChris Mason * 868a724f313SFilipe Manana * The lower boundary for the search is specified by the slot number @first_slot. 869fdf8d595SAnand Jain * Use a value of 0 to search over the whole extent buffer. Works for both 870fdf8d595SAnand Jain * leaves and nodes. 87174123bd7SChris Mason * 872fb81212cSFilipe Manana * The slot in the extent buffer is returned via @slot. If the key exists in the 873fb81212cSFilipe Manana * extent buffer, then @slot will point to the slot where the key is, otherwise 874fb81212cSFilipe Manana * it points to the slot where you would insert the key. 875fb81212cSFilipe Manana * 876fb81212cSFilipe Manana * Slot may point to the total number of items (i.e. one position beyond the last 877fb81212cSFilipe Manana * key) if the key is bigger than the last key in the extent buffer. 87874123bd7SChris Mason */ 879fdf8d595SAnand Jain int btrfs_bin_search(struct extent_buffer *eb, int first_slot, 88067d5e289SMarcos Paulo de Souza const struct btrfs_key *key, int *slot) 881be0e5c09SChris Mason { 882fb81212cSFilipe Manana unsigned long p; 883fb81212cSFilipe Manana int item_size; 884a724f313SFilipe Manana /* 885a724f313SFilipe Manana * Use unsigned types for the low and high slots, so that we get a more 886a724f313SFilipe Manana * efficient division in the search loop below. 887a724f313SFilipe Manana */ 888a724f313SFilipe Manana u32 low = first_slot; 889a724f313SFilipe Manana u32 high = btrfs_header_nritems(eb); 890be0e5c09SChris Mason int ret; 8915cd17f34SDavid Sterba const int key_size = sizeof(struct btrfs_disk_key); 892be0e5c09SChris Mason 893a724f313SFilipe Manana if (unlikely(low > high)) { 8945e24e9afSLiu Bo btrfs_err(eb->fs_info, 895a724f313SFilipe Manana "%s: low (%u) > high (%u) eb %llu owner %llu level %d", 8965e24e9afSLiu Bo __func__, low, high, eb->start, 8975e24e9afSLiu Bo btrfs_header_owner(eb), btrfs_header_level(eb)); 8985e24e9afSLiu Bo return -EINVAL; 8995e24e9afSLiu Bo } 9005e24e9afSLiu Bo 901fb81212cSFilipe Manana if (btrfs_header_level(eb) == 0) { 902fb81212cSFilipe Manana p = offsetof(struct btrfs_leaf, items); 903fb81212cSFilipe Manana item_size = sizeof(struct btrfs_item); 904fb81212cSFilipe Manana } else { 905fb81212cSFilipe Manana p = offsetof(struct btrfs_node, ptrs); 906fb81212cSFilipe Manana item_size = sizeof(struct btrfs_key_ptr); 907fb81212cSFilipe Manana } 908fb81212cSFilipe Manana 909be0e5c09SChris Mason while (low < high) { 9105cd17f34SDavid Sterba unsigned long oip; 9115cd17f34SDavid Sterba unsigned long offset; 9125cd17f34SDavid Sterba struct btrfs_disk_key *tmp; 9135cd17f34SDavid Sterba struct btrfs_disk_key unaligned; 9145cd17f34SDavid Sterba int mid; 9155cd17f34SDavid Sterba 916be0e5c09SChris Mason mid = (low + high) / 2; 9175f39d397SChris Mason offset = p + mid * item_size; 9185cd17f34SDavid Sterba oip = offset_in_page(offset); 9195f39d397SChris Mason 9205cd17f34SDavid Sterba if (oip + key_size <= PAGE_SIZE) { 921884b07d0SQu Wenruo const unsigned long idx = get_eb_page_index(offset); 9225cd17f34SDavid Sterba char *kaddr = page_address(eb->pages[idx]); 923934d375bSChris Mason 924884b07d0SQu Wenruo oip = get_eb_offset_in_page(eb, offset); 9255cd17f34SDavid Sterba tmp = (struct btrfs_disk_key *)(kaddr + oip); 9265cd17f34SDavid Sterba } else { 9275cd17f34SDavid Sterba read_extent_buffer(eb, &unaligned, offset, key_size); 9285f39d397SChris Mason tmp = &unaligned; 929479965d6SChris Mason } 930479965d6SChris Mason 931be0e5c09SChris Mason ret = comp_keys(tmp, key); 932be0e5c09SChris Mason 933be0e5c09SChris Mason if (ret < 0) 934be0e5c09SChris Mason low = mid + 1; 935be0e5c09SChris Mason else if (ret > 0) 936be0e5c09SChris Mason high = mid; 937be0e5c09SChris Mason else { 938be0e5c09SChris Mason *slot = mid; 939be0e5c09SChris Mason return 0; 940be0e5c09SChris Mason } 941be0e5c09SChris Mason } 942be0e5c09SChris Mason *slot = low; 943be0e5c09SChris Mason return 1; 944be0e5c09SChris Mason } 945be0e5c09SChris Mason 946f0486c68SYan, Zheng static void root_add_used(struct btrfs_root *root, u32 size) 947f0486c68SYan, Zheng { 948f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 949f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 950f0486c68SYan, Zheng btrfs_root_used(&root->root_item) + size); 951f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 952f0486c68SYan, Zheng } 953f0486c68SYan, Zheng 954f0486c68SYan, Zheng static void root_sub_used(struct btrfs_root *root, u32 size) 955f0486c68SYan, Zheng { 956f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 957f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 958f0486c68SYan, Zheng btrfs_root_used(&root->root_item) - size); 959f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 960f0486c68SYan, Zheng } 961f0486c68SYan, Zheng 962d352ac68SChris Mason /* given a node and slot number, this reads the blocks it points to. The 963d352ac68SChris Mason * extent buffer is returned with a reference taken (but unlocked). 964d352ac68SChris Mason */ 9654b231ae4SDavid Sterba struct extent_buffer *btrfs_read_node_slot(struct extent_buffer *parent, 9664b231ae4SDavid Sterba int slot) 967bb803951SChris Mason { 968ca7a79adSChris Mason int level = btrfs_header_level(parent); 969789d6a3aSQu Wenruo struct btrfs_tree_parent_check check = { 0 }; 970416bc658SJosef Bacik struct extent_buffer *eb; 971416bc658SJosef Bacik 972fb770ae4SLiu Bo if (slot < 0 || slot >= btrfs_header_nritems(parent)) 973fb770ae4SLiu Bo return ERR_PTR(-ENOENT); 974ca7a79adSChris Mason 975d4694728SJosef Bacik ASSERT(level); 976ca7a79adSChris Mason 977789d6a3aSQu Wenruo check.level = level - 1; 978789d6a3aSQu Wenruo check.transid = btrfs_node_ptr_generation(parent, slot); 979789d6a3aSQu Wenruo check.owner_root = btrfs_header_owner(parent); 980789d6a3aSQu Wenruo check.has_first_key = true; 981789d6a3aSQu Wenruo btrfs_node_key_to_cpu(parent, &check.first_key, slot); 982789d6a3aSQu Wenruo 983d0d20b0fSDavid Sterba eb = read_tree_block(parent->fs_info, btrfs_node_blockptr(parent, slot), 984789d6a3aSQu Wenruo &check); 9854eb150d6SQu Wenruo if (IS_ERR(eb)) 9864eb150d6SQu Wenruo return eb; 9874eb150d6SQu Wenruo if (!extent_buffer_uptodate(eb)) { 988416bc658SJosef Bacik free_extent_buffer(eb); 9894eb150d6SQu Wenruo return ERR_PTR(-EIO); 990416bc658SJosef Bacik } 991416bc658SJosef Bacik 992416bc658SJosef Bacik return eb; 993bb803951SChris Mason } 994bb803951SChris Mason 995d352ac68SChris Mason /* 996d352ac68SChris Mason * node level balancing, used to make sure nodes are in proper order for 997d352ac68SChris Mason * item deletion. We balance from the top down, so we have to make sure 998d352ac68SChris Mason * that a deletion won't leave an node completely empty later on. 999d352ac68SChris Mason */ 1000e02119d5SChris Mason static noinline int balance_level(struct btrfs_trans_handle *trans, 100198ed5174SChris Mason struct btrfs_root *root, 100298ed5174SChris Mason struct btrfs_path *path, int level) 1003bb803951SChris Mason { 10040b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 10055f39d397SChris Mason struct extent_buffer *right = NULL; 10065f39d397SChris Mason struct extent_buffer *mid; 10075f39d397SChris Mason struct extent_buffer *left = NULL; 10085f39d397SChris Mason struct extent_buffer *parent = NULL; 1009bb803951SChris Mason int ret = 0; 1010bb803951SChris Mason int wret; 1011bb803951SChris Mason int pslot; 1012bb803951SChris Mason int orig_slot = path->slots[level]; 101379f95c82SChris Mason u64 orig_ptr; 1014bb803951SChris Mason 101598e6b1ebSLiu Bo ASSERT(level > 0); 1016bb803951SChris Mason 10175f39d397SChris Mason mid = path->nodes[level]; 1018b4ce94deSChris Mason 1019ac5887c8SJosef Bacik WARN_ON(path->locks[level] != BTRFS_WRITE_LOCK); 10207bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 10217bb86316SChris Mason 10221d4f8a0cSChris Mason orig_ptr = btrfs_node_blockptr(mid, orig_slot); 102379f95c82SChris Mason 1024a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 10255f39d397SChris Mason parent = path->nodes[level + 1]; 1026bb803951SChris Mason pslot = path->slots[level + 1]; 1027a05a9bb1SLi Zefan } 1028bb803951SChris Mason 102940689478SChris Mason /* 103040689478SChris Mason * deal with the case where there is only one pointer in the root 103140689478SChris Mason * by promoting the node below to a root 103240689478SChris Mason */ 10335f39d397SChris Mason if (!parent) { 10345f39d397SChris Mason struct extent_buffer *child; 1035bb803951SChris Mason 10365f39d397SChris Mason if (btrfs_header_nritems(mid) != 1) 1037bb803951SChris Mason return 0; 1038bb803951SChris Mason 1039bb803951SChris Mason /* promote the child to a root */ 10404b231ae4SDavid Sterba child = btrfs_read_node_slot(mid, 0); 1041fb770ae4SLiu Bo if (IS_ERR(child)) { 1042fb770ae4SLiu Bo ret = PTR_ERR(child); 10430b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 1044305a26afSMark Fasheh goto enospc; 1045305a26afSMark Fasheh } 1046305a26afSMark Fasheh 1047925baeddSChris Mason btrfs_tree_lock(child); 10489631e4ccSJosef Bacik ret = btrfs_cow_block(trans, root, child, mid, 0, &child, 10499631e4ccSJosef Bacik BTRFS_NESTING_COW); 1050f0486c68SYan, Zheng if (ret) { 1051f0486c68SYan, Zheng btrfs_tree_unlock(child); 1052f0486c68SYan, Zheng free_extent_buffer(child); 1053f0486c68SYan, Zheng goto enospc; 1054f0486c68SYan, Zheng } 10552f375ab9SYan 1056406808abSFilipe Manana ret = btrfs_tree_mod_log_insert_root(root->node, child, true); 1057d9d19a01SDavid Sterba BUG_ON(ret < 0); 1058240f62c8SChris Mason rcu_assign_pointer(root->node, child); 1059925baeddSChris Mason 10600b86a832SChris Mason add_root_to_dirty_list(root); 1061925baeddSChris Mason btrfs_tree_unlock(child); 1062b4ce94deSChris Mason 1063925baeddSChris Mason path->locks[level] = 0; 1064bb803951SChris Mason path->nodes[level] = NULL; 1065190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, mid); 1066925baeddSChris Mason btrfs_tree_unlock(mid); 1067bb803951SChris Mason /* once for the path */ 10685f39d397SChris Mason free_extent_buffer(mid); 1069f0486c68SYan, Zheng 1070f0486c68SYan, Zheng root_sub_used(root, mid->len); 10717a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), mid, 0, 1); 1072bb803951SChris Mason /* once for the root ptr */ 10733083ee2eSJosef Bacik free_extent_buffer_stale(mid); 1074f0486c68SYan, Zheng return 0; 1075bb803951SChris Mason } 10765f39d397SChris Mason if (btrfs_header_nritems(mid) > 10770b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 4) 1078bb803951SChris Mason return 0; 1079bb803951SChris Mason 10809cf14029SJosef Bacik if (pslot) { 10814b231ae4SDavid Sterba left = btrfs_read_node_slot(parent, pslot - 1); 10829cf14029SJosef Bacik if (IS_ERR(left)) { 10839cf14029SJosef Bacik ret = PTR_ERR(left); 1084fb770ae4SLiu Bo left = NULL; 10859cf14029SJosef Bacik goto enospc; 10869cf14029SJosef Bacik } 1087fb770ae4SLiu Bo 1088bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT); 10895f39d397SChris Mason wret = btrfs_cow_block(trans, root, left, 10909631e4ccSJosef Bacik parent, pslot - 1, &left, 1091bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW); 109254aa1f4dSChris Mason if (wret) { 109354aa1f4dSChris Mason ret = wret; 109454aa1f4dSChris Mason goto enospc; 109554aa1f4dSChris Mason } 10962cc58cf2SChris Mason } 1097fb770ae4SLiu Bo 10989cf14029SJosef Bacik if (pslot + 1 < btrfs_header_nritems(parent)) { 10994b231ae4SDavid Sterba right = btrfs_read_node_slot(parent, pslot + 1); 11009cf14029SJosef Bacik if (IS_ERR(right)) { 11019cf14029SJosef Bacik ret = PTR_ERR(right); 1102fb770ae4SLiu Bo right = NULL; 11039cf14029SJosef Bacik goto enospc; 11049cf14029SJosef Bacik } 1105fb770ae4SLiu Bo 1106bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT); 11075f39d397SChris Mason wret = btrfs_cow_block(trans, root, right, 11089631e4ccSJosef Bacik parent, pslot + 1, &right, 1109bf59a5a2SJosef Bacik BTRFS_NESTING_RIGHT_COW); 11102cc58cf2SChris Mason if (wret) { 11112cc58cf2SChris Mason ret = wret; 11122cc58cf2SChris Mason goto enospc; 11132cc58cf2SChris Mason } 11142cc58cf2SChris Mason } 11152cc58cf2SChris Mason 11162cc58cf2SChris Mason /* first, try to make some room in the middle buffer */ 11175f39d397SChris Mason if (left) { 11185f39d397SChris Mason orig_slot += btrfs_header_nritems(left); 1119d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 1); 112079f95c82SChris Mason if (wret < 0) 112179f95c82SChris Mason ret = wret; 1122bb803951SChris Mason } 112379f95c82SChris Mason 112479f95c82SChris Mason /* 112579f95c82SChris Mason * then try to empty the right most buffer into the middle 112679f95c82SChris Mason */ 11275f39d397SChris Mason if (right) { 1128d30a668fSDavid Sterba wret = push_node_left(trans, mid, right, 1); 112954aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 113079f95c82SChris Mason ret = wret; 11315f39d397SChris Mason if (btrfs_header_nritems(right) == 0) { 1132190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, right); 1133925baeddSChris Mason btrfs_tree_unlock(right); 1134016f9d0bSJosef Bacik btrfs_del_ptr(root, path, level + 1, pslot + 1); 1135f0486c68SYan, Zheng root_sub_used(root, right->len); 11367a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), right, 11377a163608SFilipe Manana 0, 1); 11383083ee2eSJosef Bacik free_extent_buffer_stale(right); 1139f0486c68SYan, Zheng right = NULL; 1140bb803951SChris Mason } else { 11415f39d397SChris Mason struct btrfs_disk_key right_key; 11425f39d397SChris Mason btrfs_node_key(right, &right_key, 0); 1143f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot + 1, 114433cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 11450e82bcfeSDavid Sterba BUG_ON(ret < 0); 11465f39d397SChris Mason btrfs_set_node_key(parent, &right_key, pslot + 1); 11475f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 1148bb803951SChris Mason } 1149bb803951SChris Mason } 11505f39d397SChris Mason if (btrfs_header_nritems(mid) == 1) { 115179f95c82SChris Mason /* 115279f95c82SChris Mason * we're not allowed to leave a node with one item in the 115379f95c82SChris Mason * tree during a delete. A deletion from lower in the tree 115479f95c82SChris Mason * could try to delete the only pointer in this node. 115579f95c82SChris Mason * So, pull some keys from the left. 115679f95c82SChris Mason * There has to be a left pointer at this point because 115779f95c82SChris Mason * otherwise we would have pulled some pointers from the 115879f95c82SChris Mason * right 115979f95c82SChris Mason */ 1160305a26afSMark Fasheh if (!left) { 1161305a26afSMark Fasheh ret = -EROFS; 11620b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 1163305a26afSMark Fasheh goto enospc; 1164305a26afSMark Fasheh } 116555d32ed8SDavid Sterba wret = balance_node_right(trans, mid, left); 116654aa1f4dSChris Mason if (wret < 0) { 116779f95c82SChris Mason ret = wret; 116854aa1f4dSChris Mason goto enospc; 116954aa1f4dSChris Mason } 1170bce4eae9SChris Mason if (wret == 1) { 1171d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 1); 1172bce4eae9SChris Mason if (wret < 0) 1173bce4eae9SChris Mason ret = wret; 1174bce4eae9SChris Mason } 117579f95c82SChris Mason BUG_ON(wret == 1); 117679f95c82SChris Mason } 11775f39d397SChris Mason if (btrfs_header_nritems(mid) == 0) { 1178190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, mid); 1179925baeddSChris Mason btrfs_tree_unlock(mid); 1180016f9d0bSJosef Bacik btrfs_del_ptr(root, path, level + 1, pslot); 1181f0486c68SYan, Zheng root_sub_used(root, mid->len); 11827a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), mid, 0, 1); 11833083ee2eSJosef Bacik free_extent_buffer_stale(mid); 1184f0486c68SYan, Zheng mid = NULL; 118579f95c82SChris Mason } else { 118679f95c82SChris Mason /* update the parent key to reflect our changes */ 11875f39d397SChris Mason struct btrfs_disk_key mid_key; 11885f39d397SChris Mason btrfs_node_key(mid, &mid_key, 0); 1189f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot, 119033cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 11910e82bcfeSDavid Sterba BUG_ON(ret < 0); 11925f39d397SChris Mason btrfs_set_node_key(parent, &mid_key, pslot); 11935f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 119479f95c82SChris Mason } 1195bb803951SChris Mason 119679f95c82SChris Mason /* update the path */ 11975f39d397SChris Mason if (left) { 11985f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 119967439dadSDavid Sterba atomic_inc(&left->refs); 1200925baeddSChris Mason /* left was locked after cow */ 12015f39d397SChris Mason path->nodes[level] = left; 1202bb803951SChris Mason path->slots[level + 1] -= 1; 1203bb803951SChris Mason path->slots[level] = orig_slot; 1204925baeddSChris Mason if (mid) { 1205925baeddSChris Mason btrfs_tree_unlock(mid); 12065f39d397SChris Mason free_extent_buffer(mid); 1207925baeddSChris Mason } 1208bb803951SChris Mason } else { 12095f39d397SChris Mason orig_slot -= btrfs_header_nritems(left); 1210bb803951SChris Mason path->slots[level] = orig_slot; 1211bb803951SChris Mason } 1212bb803951SChris Mason } 121379f95c82SChris Mason /* double check we haven't messed things up */ 1214e20d96d6SChris Mason if (orig_ptr != 12155f39d397SChris Mason btrfs_node_blockptr(path->nodes[level], path->slots[level])) 121679f95c82SChris Mason BUG(); 121754aa1f4dSChris Mason enospc: 1218925baeddSChris Mason if (right) { 1219925baeddSChris Mason btrfs_tree_unlock(right); 12205f39d397SChris Mason free_extent_buffer(right); 1221925baeddSChris Mason } 1222925baeddSChris Mason if (left) { 1223925baeddSChris Mason if (path->nodes[level] != left) 1224925baeddSChris Mason btrfs_tree_unlock(left); 12255f39d397SChris Mason free_extent_buffer(left); 1226925baeddSChris Mason } 1227bb803951SChris Mason return ret; 1228bb803951SChris Mason } 1229bb803951SChris Mason 1230d352ac68SChris Mason /* Node balancing for insertion. Here we only split or push nodes around 1231d352ac68SChris Mason * when they are completely full. This is also done top down, so we 1232d352ac68SChris Mason * have to be pessimistic. 1233d352ac68SChris Mason */ 1234d397712bSChris Mason static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans, 1235e66f709bSChris Mason struct btrfs_root *root, 1236e66f709bSChris Mason struct btrfs_path *path, int level) 1237e66f709bSChris Mason { 12380b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 12395f39d397SChris Mason struct extent_buffer *right = NULL; 12405f39d397SChris Mason struct extent_buffer *mid; 12415f39d397SChris Mason struct extent_buffer *left = NULL; 12425f39d397SChris Mason struct extent_buffer *parent = NULL; 1243e66f709bSChris Mason int ret = 0; 1244e66f709bSChris Mason int wret; 1245e66f709bSChris Mason int pslot; 1246e66f709bSChris Mason int orig_slot = path->slots[level]; 1247e66f709bSChris Mason 1248e66f709bSChris Mason if (level == 0) 1249e66f709bSChris Mason return 1; 1250e66f709bSChris Mason 12515f39d397SChris Mason mid = path->nodes[level]; 12527bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 1253e66f709bSChris Mason 1254a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 12555f39d397SChris Mason parent = path->nodes[level + 1]; 1256e66f709bSChris Mason pslot = path->slots[level + 1]; 1257a05a9bb1SLi Zefan } 1258e66f709bSChris Mason 12595f39d397SChris Mason if (!parent) 1260e66f709bSChris Mason return 1; 1261e66f709bSChris Mason 12629cf14029SJosef Bacik /* first, try to make some room in the middle buffer */ 12639cf14029SJosef Bacik if (pslot) { 12649cf14029SJosef Bacik u32 left_nr; 12659cf14029SJosef Bacik 12664b231ae4SDavid Sterba left = btrfs_read_node_slot(parent, pslot - 1); 1267fb770ae4SLiu Bo if (IS_ERR(left)) 12689cf14029SJosef Bacik return PTR_ERR(left); 1269925baeddSChris Mason 1270bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT); 1271b4ce94deSChris Mason 12725f39d397SChris Mason left_nr = btrfs_header_nritems(left); 12730b246afaSJeff Mahoney if (left_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) { 127433ade1f8SChris Mason wret = 1; 127533ade1f8SChris Mason } else { 12765f39d397SChris Mason ret = btrfs_cow_block(trans, root, left, parent, 12779631e4ccSJosef Bacik pslot - 1, &left, 1278bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW); 127954aa1f4dSChris Mason if (ret) 128054aa1f4dSChris Mason wret = 1; 128154aa1f4dSChris Mason else { 1282d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 0); 128354aa1f4dSChris Mason } 128433ade1f8SChris Mason } 1285e66f709bSChris Mason if (wret < 0) 1286e66f709bSChris Mason ret = wret; 1287e66f709bSChris Mason if (wret == 0) { 12885f39d397SChris Mason struct btrfs_disk_key disk_key; 1289e66f709bSChris Mason orig_slot += left_nr; 12905f39d397SChris Mason btrfs_node_key(mid, &disk_key, 0); 1291f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot, 129233cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 12930e82bcfeSDavid Sterba BUG_ON(ret < 0); 12945f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot); 12955f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 12965f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 12975f39d397SChris Mason path->nodes[level] = left; 1298e66f709bSChris Mason path->slots[level + 1] -= 1; 1299e66f709bSChris Mason path->slots[level] = orig_slot; 1300925baeddSChris Mason btrfs_tree_unlock(mid); 13015f39d397SChris Mason free_extent_buffer(mid); 1302e66f709bSChris Mason } else { 1303e66f709bSChris Mason orig_slot -= 13045f39d397SChris Mason btrfs_header_nritems(left); 1305e66f709bSChris Mason path->slots[level] = orig_slot; 1306925baeddSChris Mason btrfs_tree_unlock(left); 13075f39d397SChris Mason free_extent_buffer(left); 1308e66f709bSChris Mason } 1309e66f709bSChris Mason return 0; 1310e66f709bSChris Mason } 1311925baeddSChris Mason btrfs_tree_unlock(left); 13125f39d397SChris Mason free_extent_buffer(left); 1313e66f709bSChris Mason } 1314e66f709bSChris Mason 1315e66f709bSChris Mason /* 1316e66f709bSChris Mason * then try to empty the right most buffer into the middle 1317e66f709bSChris Mason */ 13189cf14029SJosef Bacik if (pslot + 1 < btrfs_header_nritems(parent)) { 131933ade1f8SChris Mason u32 right_nr; 1320b4ce94deSChris Mason 13219cf14029SJosef Bacik right = btrfs_read_node_slot(parent, pslot + 1); 13229cf14029SJosef Bacik if (IS_ERR(right)) 13239cf14029SJosef Bacik return PTR_ERR(right); 13249cf14029SJosef Bacik 1325bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT); 1326b4ce94deSChris Mason 13275f39d397SChris Mason right_nr = btrfs_header_nritems(right); 13280b246afaSJeff Mahoney if (right_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) { 132933ade1f8SChris Mason wret = 1; 133033ade1f8SChris Mason } else { 13315f39d397SChris Mason ret = btrfs_cow_block(trans, root, right, 13325f39d397SChris Mason parent, pslot + 1, 1333bf59a5a2SJosef Bacik &right, BTRFS_NESTING_RIGHT_COW); 133454aa1f4dSChris Mason if (ret) 133554aa1f4dSChris Mason wret = 1; 133654aa1f4dSChris Mason else { 133755d32ed8SDavid Sterba wret = balance_node_right(trans, right, mid); 133833ade1f8SChris Mason } 133954aa1f4dSChris Mason } 1340e66f709bSChris Mason if (wret < 0) 1341e66f709bSChris Mason ret = wret; 1342e66f709bSChris Mason if (wret == 0) { 13435f39d397SChris Mason struct btrfs_disk_key disk_key; 13445f39d397SChris Mason 13455f39d397SChris Mason btrfs_node_key(right, &disk_key, 0); 1346f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot + 1, 134733cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 13480e82bcfeSDavid Sterba BUG_ON(ret < 0); 13495f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot + 1); 13505f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 13515f39d397SChris Mason 13525f39d397SChris Mason if (btrfs_header_nritems(mid) <= orig_slot) { 13535f39d397SChris Mason path->nodes[level] = right; 1354e66f709bSChris Mason path->slots[level + 1] += 1; 1355e66f709bSChris Mason path->slots[level] = orig_slot - 13565f39d397SChris Mason btrfs_header_nritems(mid); 1357925baeddSChris Mason btrfs_tree_unlock(mid); 13585f39d397SChris Mason free_extent_buffer(mid); 1359e66f709bSChris Mason } else { 1360925baeddSChris Mason btrfs_tree_unlock(right); 13615f39d397SChris Mason free_extent_buffer(right); 1362e66f709bSChris Mason } 1363e66f709bSChris Mason return 0; 1364e66f709bSChris Mason } 1365925baeddSChris Mason btrfs_tree_unlock(right); 13665f39d397SChris Mason free_extent_buffer(right); 1367e66f709bSChris Mason } 1368e66f709bSChris Mason return 1; 1369e66f709bSChris Mason } 1370e66f709bSChris Mason 137174123bd7SChris Mason /* 1372d352ac68SChris Mason * readahead one full node of leaves, finding things that are close 1373d352ac68SChris Mason * to the block in 'slot', and triggering ra on them. 13743c69faecSChris Mason */ 13752ff7e61eSJeff Mahoney static void reada_for_search(struct btrfs_fs_info *fs_info, 1376e02119d5SChris Mason struct btrfs_path *path, 137701f46658SChris Mason int level, int slot, u64 objectid) 13783c69faecSChris Mason { 13795f39d397SChris Mason struct extent_buffer *node; 138001f46658SChris Mason struct btrfs_disk_key disk_key; 13813c69faecSChris Mason u32 nritems; 13823c69faecSChris Mason u64 search; 1383a7175319SChris Mason u64 target; 13846b80053dSChris Mason u64 nread = 0; 1385ace75066SFilipe Manana u64 nread_max; 13866b80053dSChris Mason u32 nr; 13876b80053dSChris Mason u32 blocksize; 13886b80053dSChris Mason u32 nscan = 0; 1389db94535dSChris Mason 1390ace75066SFilipe Manana if (level != 1 && path->reada != READA_FORWARD_ALWAYS) 13913c69faecSChris Mason return; 13923c69faecSChris Mason 13936702ed49SChris Mason if (!path->nodes[level]) 13946702ed49SChris Mason return; 13956702ed49SChris Mason 13965f39d397SChris Mason node = path->nodes[level]; 1397925baeddSChris Mason 1398ace75066SFilipe Manana /* 1399ace75066SFilipe Manana * Since the time between visiting leaves is much shorter than the time 1400ace75066SFilipe Manana * between visiting nodes, limit read ahead of nodes to 1, to avoid too 1401ace75066SFilipe Manana * much IO at once (possibly random). 1402ace75066SFilipe Manana */ 1403ace75066SFilipe Manana if (path->reada == READA_FORWARD_ALWAYS) { 1404ace75066SFilipe Manana if (level > 1) 1405ace75066SFilipe Manana nread_max = node->fs_info->nodesize; 1406ace75066SFilipe Manana else 1407ace75066SFilipe Manana nread_max = SZ_128K; 1408ace75066SFilipe Manana } else { 1409ace75066SFilipe Manana nread_max = SZ_64K; 1410ace75066SFilipe Manana } 1411ace75066SFilipe Manana 14123c69faecSChris Mason search = btrfs_node_blockptr(node, slot); 14130b246afaSJeff Mahoney blocksize = fs_info->nodesize; 1414069a2e37SFilipe Manana if (path->reada != READA_FORWARD_ALWAYS) { 1415069a2e37SFilipe Manana struct extent_buffer *eb; 1416069a2e37SFilipe Manana 14170b246afaSJeff Mahoney eb = find_extent_buffer(fs_info, search); 14185f39d397SChris Mason if (eb) { 14195f39d397SChris Mason free_extent_buffer(eb); 14203c69faecSChris Mason return; 14213c69faecSChris Mason } 1422069a2e37SFilipe Manana } 14233c69faecSChris Mason 1424a7175319SChris Mason target = search; 14256b80053dSChris Mason 14265f39d397SChris Mason nritems = btrfs_header_nritems(node); 14276b80053dSChris Mason nr = slot; 142825b8b936SJosef Bacik 14293c69faecSChris Mason while (1) { 1430e4058b54SDavid Sterba if (path->reada == READA_BACK) { 14316b80053dSChris Mason if (nr == 0) 14323c69faecSChris Mason break; 14336b80053dSChris Mason nr--; 1434ace75066SFilipe Manana } else if (path->reada == READA_FORWARD || 1435ace75066SFilipe Manana path->reada == READA_FORWARD_ALWAYS) { 14366b80053dSChris Mason nr++; 14376b80053dSChris Mason if (nr >= nritems) 14386b80053dSChris Mason break; 14393c69faecSChris Mason } 1440e4058b54SDavid Sterba if (path->reada == READA_BACK && objectid) { 144101f46658SChris Mason btrfs_node_key(node, &disk_key, nr); 144201f46658SChris Mason if (btrfs_disk_key_objectid(&disk_key) != objectid) 144301f46658SChris Mason break; 144401f46658SChris Mason } 14456b80053dSChris Mason search = btrfs_node_blockptr(node, nr); 1446ace75066SFilipe Manana if (path->reada == READA_FORWARD_ALWAYS || 1447ace75066SFilipe Manana (search <= target && target - search <= 65536) || 1448a7175319SChris Mason (search > target && search - target <= 65536)) { 1449bfb484d9SJosef Bacik btrfs_readahead_node_child(node, nr); 14506b80053dSChris Mason nread += blocksize; 14513c69faecSChris Mason } 14526b80053dSChris Mason nscan++; 1453ace75066SFilipe Manana if (nread > nread_max || nscan > 32) 14546b80053dSChris Mason break; 14553c69faecSChris Mason } 14563c69faecSChris Mason } 1457925baeddSChris Mason 1458bfb484d9SJosef Bacik static noinline void reada_for_balance(struct btrfs_path *path, int level) 1459b4ce94deSChris Mason { 1460bfb484d9SJosef Bacik struct extent_buffer *parent; 1461b4ce94deSChris Mason int slot; 1462b4ce94deSChris Mason int nritems; 1463b4ce94deSChris Mason 14648c594ea8SChris Mason parent = path->nodes[level + 1]; 1465b4ce94deSChris Mason if (!parent) 14660b08851fSJosef Bacik return; 1467b4ce94deSChris Mason 1468b4ce94deSChris Mason nritems = btrfs_header_nritems(parent); 14698c594ea8SChris Mason slot = path->slots[level + 1]; 1470b4ce94deSChris Mason 1471bfb484d9SJosef Bacik if (slot > 0) 1472bfb484d9SJosef Bacik btrfs_readahead_node_child(parent, slot - 1); 1473bfb484d9SJosef Bacik if (slot + 1 < nritems) 1474bfb484d9SJosef Bacik btrfs_readahead_node_child(parent, slot + 1); 1475b4ce94deSChris Mason } 1476b4ce94deSChris Mason 1477b4ce94deSChris Mason 1478b4ce94deSChris Mason /* 1479d397712bSChris Mason * when we walk down the tree, it is usually safe to unlock the higher layers 1480d397712bSChris Mason * in the tree. The exceptions are when our path goes through slot 0, because 1481d397712bSChris Mason * operations on the tree might require changing key pointers higher up in the 1482d397712bSChris Mason * tree. 1483d352ac68SChris Mason * 1484d397712bSChris Mason * callers might also have set path->keep_locks, which tells this code to keep 1485d397712bSChris Mason * the lock if the path points to the last slot in the block. This is part of 1486d397712bSChris Mason * walking through the tree, and selecting the next slot in the higher block. 1487d352ac68SChris Mason * 1488d397712bSChris Mason * lowest_unlock sets the lowest level in the tree we're allowed to unlock. so 1489d397712bSChris Mason * if lowest_unlock is 1, level 0 won't be unlocked 1490d352ac68SChris Mason */ 1491e02119d5SChris Mason static noinline void unlock_up(struct btrfs_path *path, int level, 1492f7c79f30SChris Mason int lowest_unlock, int min_write_lock_level, 1493f7c79f30SChris Mason int *write_lock_level) 1494925baeddSChris Mason { 1495925baeddSChris Mason int i; 1496925baeddSChris Mason int skip_level = level; 1497c1227996SNikolay Borisov bool check_skip = true; 1498925baeddSChris Mason 1499925baeddSChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 1500925baeddSChris Mason if (!path->nodes[i]) 1501925baeddSChris Mason break; 1502925baeddSChris Mason if (!path->locks[i]) 1503925baeddSChris Mason break; 1504c1227996SNikolay Borisov 1505c1227996SNikolay Borisov if (check_skip) { 1506c1227996SNikolay Borisov if (path->slots[i] == 0) { 1507925baeddSChris Mason skip_level = i + 1; 1508925baeddSChris Mason continue; 1509925baeddSChris Mason } 1510c1227996SNikolay Borisov 1511c1227996SNikolay Borisov if (path->keep_locks) { 1512925baeddSChris Mason u32 nritems; 1513c1227996SNikolay Borisov 1514c1227996SNikolay Borisov nritems = btrfs_header_nritems(path->nodes[i]); 1515051e1b9fSChris Mason if (nritems < 1 || path->slots[i] >= nritems - 1) { 1516925baeddSChris Mason skip_level = i + 1; 1517925baeddSChris Mason continue; 1518925baeddSChris Mason } 1519925baeddSChris Mason } 1520c1227996SNikolay Borisov } 1521051e1b9fSChris Mason 1522d80bb3f9SLiu Bo if (i >= lowest_unlock && i > skip_level) { 1523c1227996SNikolay Borisov check_skip = false; 1524c1227996SNikolay Borisov btrfs_tree_unlock_rw(path->nodes[i], path->locks[i]); 1525925baeddSChris Mason path->locks[i] = 0; 1526f7c79f30SChris Mason if (write_lock_level && 1527f7c79f30SChris Mason i > min_write_lock_level && 1528f7c79f30SChris Mason i <= *write_lock_level) { 1529f7c79f30SChris Mason *write_lock_level = i - 1; 1530f7c79f30SChris Mason } 1531925baeddSChris Mason } 1532925baeddSChris Mason } 1533925baeddSChris Mason } 1534925baeddSChris Mason 15353c69faecSChris Mason /* 1536376a21d7SFilipe Manana * Helper function for btrfs_search_slot() and other functions that do a search 1537376a21d7SFilipe Manana * on a btree. The goal is to find a tree block in the cache (the radix tree at 1538376a21d7SFilipe Manana * fs_info->buffer_radix), but if we can't find it, or it's not up to date, read 1539376a21d7SFilipe Manana * its pages from disk. 1540c8c42864SChris Mason * 1541376a21d7SFilipe Manana * Returns -EAGAIN, with the path unlocked, if the caller needs to repeat the 1542376a21d7SFilipe Manana * whole btree search, starting again from the current root node. 1543c8c42864SChris Mason */ 1544c8c42864SChris Mason static int 1545d07b8528SLiu Bo read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, 1546c8c42864SChris Mason struct extent_buffer **eb_ret, int level, int slot, 1547cda79c54SDavid Sterba const struct btrfs_key *key) 1548c8c42864SChris Mason { 15490b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 1550789d6a3aSQu Wenruo struct btrfs_tree_parent_check check = { 0 }; 1551c8c42864SChris Mason u64 blocknr; 1552c8c42864SChris Mason u64 gen; 1553c8c42864SChris Mason struct extent_buffer *tmp; 155476a05b35SChris Mason int ret; 1555581c1760SQu Wenruo int parent_level; 1556b246666eSFilipe Manana bool unlock_up; 1557c8c42864SChris Mason 1558b246666eSFilipe Manana unlock_up = ((level + 1 < BTRFS_MAX_LEVEL) && p->locks[level + 1]); 1559213ff4b7SNikolay Borisov blocknr = btrfs_node_blockptr(*eb_ret, slot); 1560213ff4b7SNikolay Borisov gen = btrfs_node_ptr_generation(*eb_ret, slot); 1561213ff4b7SNikolay Borisov parent_level = btrfs_header_level(*eb_ret); 1562789d6a3aSQu Wenruo btrfs_node_key_to_cpu(*eb_ret, &check.first_key, slot); 1563789d6a3aSQu Wenruo check.has_first_key = true; 1564789d6a3aSQu Wenruo check.level = parent_level - 1; 1565789d6a3aSQu Wenruo check.transid = gen; 1566789d6a3aSQu Wenruo check.owner_root = root->root_key.objectid; 1567c8c42864SChris Mason 1568b246666eSFilipe Manana /* 1569b246666eSFilipe Manana * If we need to read an extent buffer from disk and we are holding locks 1570b246666eSFilipe Manana * on upper level nodes, we unlock all the upper nodes before reading the 1571b246666eSFilipe Manana * extent buffer, and then return -EAGAIN to the caller as it needs to 1572b246666eSFilipe Manana * restart the search. We don't release the lock on the current level 1573b246666eSFilipe Manana * because we need to walk this node to figure out which blocks to read. 1574b246666eSFilipe Manana */ 15750b246afaSJeff Mahoney tmp = find_extent_buffer(fs_info, blocknr); 1576cb44921aSChris Mason if (tmp) { 1577ace75066SFilipe Manana if (p->reada == READA_FORWARD_ALWAYS) 1578ace75066SFilipe Manana reada_for_search(fs_info, p, level, slot, key->objectid); 1579ace75066SFilipe Manana 1580b9fab919SChris Mason /* first we do an atomic uptodate check */ 1581b9fab919SChris Mason if (btrfs_buffer_uptodate(tmp, gen, 1) > 0) { 1582448de471SQu Wenruo /* 1583448de471SQu Wenruo * Do extra check for first_key, eb can be stale due to 1584448de471SQu Wenruo * being cached, read from scrub, or have multiple 1585448de471SQu Wenruo * parents (shared tree blocks). 1586448de471SQu Wenruo */ 1587e064d5e9SDavid Sterba if (btrfs_verify_level_key(tmp, 1588789d6a3aSQu Wenruo parent_level - 1, &check.first_key, gen)) { 1589448de471SQu Wenruo free_extent_buffer(tmp); 1590448de471SQu Wenruo return -EUCLEAN; 1591448de471SQu Wenruo } 1592c8c42864SChris Mason *eb_ret = tmp; 1593c8c42864SChris Mason return 0; 1594c8c42864SChris Mason } 1595bdf7c00eSJosef Bacik 1596857bc13fSJosef Bacik if (p->nowait) { 1597857bc13fSJosef Bacik free_extent_buffer(tmp); 1598857bc13fSJosef Bacik return -EAGAIN; 1599857bc13fSJosef Bacik } 1600857bc13fSJosef Bacik 1601b246666eSFilipe Manana if (unlock_up) 1602b246666eSFilipe Manana btrfs_unlock_up_safe(p, level + 1); 1603b246666eSFilipe Manana 1604b9fab919SChris Mason /* now we're allowed to do a blocking uptodate check */ 1605789d6a3aSQu Wenruo ret = btrfs_read_extent_buffer(tmp, &check); 16069a4ffa1bSQu Wenruo if (ret) { 1607cb44921aSChris Mason free_extent_buffer(tmp); 1608b3b4aa74SDavid Sterba btrfs_release_path(p); 1609cb44921aSChris Mason return -EIO; 1610cb44921aSChris Mason } 161188c602abSQu Wenruo if (btrfs_check_eb_owner(tmp, root->root_key.objectid)) { 161288c602abSQu Wenruo free_extent_buffer(tmp); 161388c602abSQu Wenruo btrfs_release_path(p); 161488c602abSQu Wenruo return -EUCLEAN; 161588c602abSQu Wenruo } 1616b246666eSFilipe Manana 1617b246666eSFilipe Manana if (unlock_up) 1618b246666eSFilipe Manana ret = -EAGAIN; 1619b246666eSFilipe Manana 1620b246666eSFilipe Manana goto out; 1621857bc13fSJosef Bacik } else if (p->nowait) { 1622857bc13fSJosef Bacik return -EAGAIN; 16239a4ffa1bSQu Wenruo } 1624c8c42864SChris Mason 1625b246666eSFilipe Manana if (unlock_up) { 16268c594ea8SChris Mason btrfs_unlock_up_safe(p, level + 1); 16274bb59055SFilipe Manana ret = -EAGAIN; 16284bb59055SFilipe Manana } else { 16294bb59055SFilipe Manana ret = 0; 16304bb59055SFilipe Manana } 16318c594ea8SChris Mason 1632e4058b54SDavid Sterba if (p->reada != READA_NONE) 16332ff7e61eSJeff Mahoney reada_for_search(fs_info, p, level, slot, key->objectid); 1634c8c42864SChris Mason 1635789d6a3aSQu Wenruo tmp = read_tree_block(fs_info, blocknr, &check); 16364eb150d6SQu Wenruo if (IS_ERR(tmp)) { 16374eb150d6SQu Wenruo btrfs_release_path(p); 16384eb150d6SQu Wenruo return PTR_ERR(tmp); 16394eb150d6SQu Wenruo } 164076a05b35SChris Mason /* 164176a05b35SChris Mason * If the read above didn't mark this buffer up to date, 164276a05b35SChris Mason * it will never end up being up to date. Set ret to EIO now 164376a05b35SChris Mason * and give up so that our caller doesn't loop forever 164476a05b35SChris Mason * on our EAGAINs. 164576a05b35SChris Mason */ 1646e6a1d6fdSLiu Bo if (!extent_buffer_uptodate(tmp)) 164776a05b35SChris Mason ret = -EIO; 164802a3307aSLiu Bo 1649b246666eSFilipe Manana out: 16504bb59055SFilipe Manana if (ret == 0) { 16514bb59055SFilipe Manana *eb_ret = tmp; 16524bb59055SFilipe Manana } else { 16534bb59055SFilipe Manana free_extent_buffer(tmp); 165402a3307aSLiu Bo btrfs_release_path(p); 16554bb59055SFilipe Manana } 16564bb59055SFilipe Manana 165776a05b35SChris Mason return ret; 1658c8c42864SChris Mason } 1659c8c42864SChris Mason 1660c8c42864SChris Mason /* 1661c8c42864SChris Mason * helper function for btrfs_search_slot. This does all of the checks 1662c8c42864SChris Mason * for node-level blocks and does any balancing required based on 1663c8c42864SChris Mason * the ins_len. 1664c8c42864SChris Mason * 1665c8c42864SChris Mason * If no extra work was required, zero is returned. If we had to 1666c8c42864SChris Mason * drop the path, -EAGAIN is returned and btrfs_search_slot must 1667c8c42864SChris Mason * start over 1668c8c42864SChris Mason */ 1669c8c42864SChris Mason static int 1670c8c42864SChris Mason setup_nodes_for_search(struct btrfs_trans_handle *trans, 1671c8c42864SChris Mason struct btrfs_root *root, struct btrfs_path *p, 1672bd681513SChris Mason struct extent_buffer *b, int level, int ins_len, 1673bd681513SChris Mason int *write_lock_level) 1674c8c42864SChris Mason { 16750b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 167695b982deSNikolay Borisov int ret = 0; 16770b246afaSJeff Mahoney 1678c8c42864SChris Mason if ((p->search_for_split || ins_len > 0) && btrfs_header_nritems(b) >= 16790b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3) { 1680c8c42864SChris Mason 1681bd681513SChris Mason if (*write_lock_level < level + 1) { 1682bd681513SChris Mason *write_lock_level = level + 1; 1683bd681513SChris Mason btrfs_release_path(p); 168495b982deSNikolay Borisov return -EAGAIN; 1685bd681513SChris Mason } 1686bd681513SChris Mason 1687bfb484d9SJosef Bacik reada_for_balance(p, level); 168895b982deSNikolay Borisov ret = split_node(trans, root, p, level); 1689c8c42864SChris Mason 1690c8c42864SChris Mason b = p->nodes[level]; 1691c8c42864SChris Mason } else if (ins_len < 0 && btrfs_header_nritems(b) < 16920b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 2) { 1693c8c42864SChris Mason 1694bd681513SChris Mason if (*write_lock_level < level + 1) { 1695bd681513SChris Mason *write_lock_level = level + 1; 1696bd681513SChris Mason btrfs_release_path(p); 169795b982deSNikolay Borisov return -EAGAIN; 1698bd681513SChris Mason } 1699bd681513SChris Mason 1700bfb484d9SJosef Bacik reada_for_balance(p, level); 170195b982deSNikolay Borisov ret = balance_level(trans, root, p, level); 170295b982deSNikolay Borisov if (ret) 170395b982deSNikolay Borisov return ret; 1704c8c42864SChris Mason 1705c8c42864SChris Mason b = p->nodes[level]; 1706c8c42864SChris Mason if (!b) { 1707b3b4aa74SDavid Sterba btrfs_release_path(p); 170895b982deSNikolay Borisov return -EAGAIN; 1709c8c42864SChris Mason } 1710c8c42864SChris Mason BUG_ON(btrfs_header_nritems(b) == 1); 1711c8c42864SChris Mason } 1712c8c42864SChris Mason return ret; 1713c8c42864SChris Mason } 1714c8c42864SChris Mason 1715381cf658SDavid Sterba int btrfs_find_item(struct btrfs_root *fs_root, struct btrfs_path *path, 1716e33d5c3dSKelley Nielsen u64 iobjectid, u64 ioff, u8 key_type, 1717e33d5c3dSKelley Nielsen struct btrfs_key *found_key) 1718e33d5c3dSKelley Nielsen { 1719e33d5c3dSKelley Nielsen int ret; 1720e33d5c3dSKelley Nielsen struct btrfs_key key; 1721e33d5c3dSKelley Nielsen struct extent_buffer *eb; 1722381cf658SDavid Sterba 1723381cf658SDavid Sterba ASSERT(path); 17241d4c08e0SDavid Sterba ASSERT(found_key); 1725e33d5c3dSKelley Nielsen 1726e33d5c3dSKelley Nielsen key.type = key_type; 1727e33d5c3dSKelley Nielsen key.objectid = iobjectid; 1728e33d5c3dSKelley Nielsen key.offset = ioff; 1729e33d5c3dSKelley Nielsen 1730e33d5c3dSKelley Nielsen ret = btrfs_search_slot(NULL, fs_root, &key, path, 0, 0); 17311d4c08e0SDavid Sterba if (ret < 0) 1732e33d5c3dSKelley Nielsen return ret; 1733e33d5c3dSKelley Nielsen 1734e33d5c3dSKelley Nielsen eb = path->nodes[0]; 1735e33d5c3dSKelley Nielsen if (ret && path->slots[0] >= btrfs_header_nritems(eb)) { 1736e33d5c3dSKelley Nielsen ret = btrfs_next_leaf(fs_root, path); 1737e33d5c3dSKelley Nielsen if (ret) 1738e33d5c3dSKelley Nielsen return ret; 1739e33d5c3dSKelley Nielsen eb = path->nodes[0]; 1740e33d5c3dSKelley Nielsen } 1741e33d5c3dSKelley Nielsen 1742e33d5c3dSKelley Nielsen btrfs_item_key_to_cpu(eb, found_key, path->slots[0]); 1743e33d5c3dSKelley Nielsen if (found_key->type != key.type || 1744e33d5c3dSKelley Nielsen found_key->objectid != key.objectid) 1745e33d5c3dSKelley Nielsen return 1; 1746e33d5c3dSKelley Nielsen 1747e33d5c3dSKelley Nielsen return 0; 1748e33d5c3dSKelley Nielsen } 1749e33d5c3dSKelley Nielsen 17501fc28d8eSLiu Bo static struct extent_buffer *btrfs_search_slot_get_root(struct btrfs_root *root, 17511fc28d8eSLiu Bo struct btrfs_path *p, 17521fc28d8eSLiu Bo int write_lock_level) 17531fc28d8eSLiu Bo { 17541fc28d8eSLiu Bo struct extent_buffer *b; 1755120de408SJosef Bacik int root_lock = 0; 17561fc28d8eSLiu Bo int level = 0; 17571fc28d8eSLiu Bo 17581fc28d8eSLiu Bo if (p->search_commit_root) { 17591fc28d8eSLiu Bo b = root->commit_root; 176067439dadSDavid Sterba atomic_inc(&b->refs); 17611fc28d8eSLiu Bo level = btrfs_header_level(b); 1762f9ddfd05SLiu Bo /* 1763f9ddfd05SLiu Bo * Ensure that all callers have set skip_locking when 1764f9ddfd05SLiu Bo * p->search_commit_root = 1. 1765f9ddfd05SLiu Bo */ 1766f9ddfd05SLiu Bo ASSERT(p->skip_locking == 1); 17671fc28d8eSLiu Bo 17681fc28d8eSLiu Bo goto out; 17691fc28d8eSLiu Bo } 17701fc28d8eSLiu Bo 17711fc28d8eSLiu Bo if (p->skip_locking) { 17721fc28d8eSLiu Bo b = btrfs_root_node(root); 17731fc28d8eSLiu Bo level = btrfs_header_level(b); 17741fc28d8eSLiu Bo goto out; 17751fc28d8eSLiu Bo } 17761fc28d8eSLiu Bo 1777120de408SJosef Bacik /* We try very hard to do read locks on the root */ 1778120de408SJosef Bacik root_lock = BTRFS_READ_LOCK; 1779120de408SJosef Bacik 17801fc28d8eSLiu Bo /* 1781662c653bSLiu Bo * If the level is set to maximum, we can skip trying to get the read 1782662c653bSLiu Bo * lock. 1783662c653bSLiu Bo */ 1784662c653bSLiu Bo if (write_lock_level < BTRFS_MAX_LEVEL) { 1785662c653bSLiu Bo /* 1786662c653bSLiu Bo * We don't know the level of the root node until we actually 1787662c653bSLiu Bo * have it read locked 17881fc28d8eSLiu Bo */ 1789857bc13fSJosef Bacik if (p->nowait) { 1790857bc13fSJosef Bacik b = btrfs_try_read_lock_root_node(root); 1791857bc13fSJosef Bacik if (IS_ERR(b)) 1792857bc13fSJosef Bacik return b; 1793857bc13fSJosef Bacik } else { 17941bb96598SJosef Bacik b = btrfs_read_lock_root_node(root); 1795857bc13fSJosef Bacik } 17961fc28d8eSLiu Bo level = btrfs_header_level(b); 17971fc28d8eSLiu Bo if (level > write_lock_level) 17981fc28d8eSLiu Bo goto out; 17991fc28d8eSLiu Bo 1800662c653bSLiu Bo /* Whoops, must trade for write lock */ 18011fc28d8eSLiu Bo btrfs_tree_read_unlock(b); 18021fc28d8eSLiu Bo free_extent_buffer(b); 1803662c653bSLiu Bo } 1804662c653bSLiu Bo 18051fc28d8eSLiu Bo b = btrfs_lock_root_node(root); 18061fc28d8eSLiu Bo root_lock = BTRFS_WRITE_LOCK; 18071fc28d8eSLiu Bo 18081fc28d8eSLiu Bo /* The level might have changed, check again */ 18091fc28d8eSLiu Bo level = btrfs_header_level(b); 18101fc28d8eSLiu Bo 18111fc28d8eSLiu Bo out: 1812120de408SJosef Bacik /* 1813120de408SJosef Bacik * The root may have failed to write out at some point, and thus is no 1814120de408SJosef Bacik * longer valid, return an error in this case. 1815120de408SJosef Bacik */ 1816120de408SJosef Bacik if (!extent_buffer_uptodate(b)) { 1817120de408SJosef Bacik if (root_lock) 1818120de408SJosef Bacik btrfs_tree_unlock_rw(b, root_lock); 1819120de408SJosef Bacik free_extent_buffer(b); 1820120de408SJosef Bacik return ERR_PTR(-EIO); 1821120de408SJosef Bacik } 1822120de408SJosef Bacik 18231fc28d8eSLiu Bo p->nodes[level] = b; 18241fc28d8eSLiu Bo if (!p->skip_locking) 18251fc28d8eSLiu Bo p->locks[level] = root_lock; 18261fc28d8eSLiu Bo /* 18271fc28d8eSLiu Bo * Callers are responsible for dropping b's references. 18281fc28d8eSLiu Bo */ 18291fc28d8eSLiu Bo return b; 18301fc28d8eSLiu Bo } 18311fc28d8eSLiu Bo 1832d96b3424SFilipe Manana /* 1833d96b3424SFilipe Manana * Replace the extent buffer at the lowest level of the path with a cloned 1834d96b3424SFilipe Manana * version. The purpose is to be able to use it safely, after releasing the 1835d96b3424SFilipe Manana * commit root semaphore, even if relocation is happening in parallel, the 1836d96b3424SFilipe Manana * transaction used for relocation is committed and the extent buffer is 1837d96b3424SFilipe Manana * reallocated in the next transaction. 1838d96b3424SFilipe Manana * 1839d96b3424SFilipe Manana * This is used in a context where the caller does not prevent transaction 1840d96b3424SFilipe Manana * commits from happening, either by holding a transaction handle or holding 1841d96b3424SFilipe Manana * some lock, while it's doing searches through a commit root. 1842d96b3424SFilipe Manana * At the moment it's only used for send operations. 1843d96b3424SFilipe Manana */ 1844d96b3424SFilipe Manana static int finish_need_commit_sem_search(struct btrfs_path *path) 1845d96b3424SFilipe Manana { 1846d96b3424SFilipe Manana const int i = path->lowest_level; 1847d96b3424SFilipe Manana const int slot = path->slots[i]; 1848d96b3424SFilipe Manana struct extent_buffer *lowest = path->nodes[i]; 1849d96b3424SFilipe Manana struct extent_buffer *clone; 1850d96b3424SFilipe Manana 1851d96b3424SFilipe Manana ASSERT(path->need_commit_sem); 1852d96b3424SFilipe Manana 1853d96b3424SFilipe Manana if (!lowest) 1854d96b3424SFilipe Manana return 0; 1855d96b3424SFilipe Manana 1856d96b3424SFilipe Manana lockdep_assert_held_read(&lowest->fs_info->commit_root_sem); 1857d96b3424SFilipe Manana 1858d96b3424SFilipe Manana clone = btrfs_clone_extent_buffer(lowest); 1859d96b3424SFilipe Manana if (!clone) 1860d96b3424SFilipe Manana return -ENOMEM; 1861d96b3424SFilipe Manana 1862d96b3424SFilipe Manana btrfs_release_path(path); 1863d96b3424SFilipe Manana path->nodes[i] = clone; 1864d96b3424SFilipe Manana path->slots[i] = slot; 1865d96b3424SFilipe Manana 1866d96b3424SFilipe Manana return 0; 1867d96b3424SFilipe Manana } 18681fc28d8eSLiu Bo 1869e2e58d0fSFilipe Manana static inline int search_for_key_slot(struct extent_buffer *eb, 1870e2e58d0fSFilipe Manana int search_low_slot, 1871e2e58d0fSFilipe Manana const struct btrfs_key *key, 1872e2e58d0fSFilipe Manana int prev_cmp, 1873e2e58d0fSFilipe Manana int *slot) 1874e2e58d0fSFilipe Manana { 1875e2e58d0fSFilipe Manana /* 1876e2e58d0fSFilipe Manana * If a previous call to btrfs_bin_search() on a parent node returned an 1877e2e58d0fSFilipe Manana * exact match (prev_cmp == 0), we can safely assume the target key will 1878e2e58d0fSFilipe Manana * always be at slot 0 on lower levels, since each key pointer 1879e2e58d0fSFilipe Manana * (struct btrfs_key_ptr) refers to the lowest key accessible from the 1880e2e58d0fSFilipe Manana * subtree it points to. Thus we can skip searching lower levels. 1881e2e58d0fSFilipe Manana */ 1882e2e58d0fSFilipe Manana if (prev_cmp == 0) { 1883e2e58d0fSFilipe Manana *slot = 0; 1884e2e58d0fSFilipe Manana return 0; 1885e2e58d0fSFilipe Manana } 1886e2e58d0fSFilipe Manana 1887fdf8d595SAnand Jain return btrfs_bin_search(eb, search_low_slot, key, slot); 1888e2e58d0fSFilipe Manana } 1889e2e58d0fSFilipe Manana 1890109324cfSFilipe Manana static int search_leaf(struct btrfs_trans_handle *trans, 1891109324cfSFilipe Manana struct btrfs_root *root, 1892109324cfSFilipe Manana const struct btrfs_key *key, 1893109324cfSFilipe Manana struct btrfs_path *path, 1894109324cfSFilipe Manana int ins_len, 1895109324cfSFilipe Manana int prev_cmp) 1896109324cfSFilipe Manana { 1897109324cfSFilipe Manana struct extent_buffer *leaf = path->nodes[0]; 1898109324cfSFilipe Manana int leaf_free_space = -1; 1899109324cfSFilipe Manana int search_low_slot = 0; 1900109324cfSFilipe Manana int ret; 1901109324cfSFilipe Manana bool do_bin_search = true; 1902109324cfSFilipe Manana 1903109324cfSFilipe Manana /* 1904109324cfSFilipe Manana * If we are doing an insertion, the leaf has enough free space and the 1905109324cfSFilipe Manana * destination slot for the key is not slot 0, then we can unlock our 1906109324cfSFilipe Manana * write lock on the parent, and any other upper nodes, before doing the 1907109324cfSFilipe Manana * binary search on the leaf (with search_for_key_slot()), allowing other 1908109324cfSFilipe Manana * tasks to lock the parent and any other upper nodes. 1909109324cfSFilipe Manana */ 1910109324cfSFilipe Manana if (ins_len > 0) { 1911109324cfSFilipe Manana /* 1912109324cfSFilipe Manana * Cache the leaf free space, since we will need it later and it 1913109324cfSFilipe Manana * will not change until then. 1914109324cfSFilipe Manana */ 1915109324cfSFilipe Manana leaf_free_space = btrfs_leaf_free_space(leaf); 1916109324cfSFilipe Manana 1917109324cfSFilipe Manana /* 1918109324cfSFilipe Manana * !path->locks[1] means we have a single node tree, the leaf is 1919109324cfSFilipe Manana * the root of the tree. 1920109324cfSFilipe Manana */ 1921109324cfSFilipe Manana if (path->locks[1] && leaf_free_space >= ins_len) { 1922109324cfSFilipe Manana struct btrfs_disk_key first_key; 1923109324cfSFilipe Manana 1924109324cfSFilipe Manana ASSERT(btrfs_header_nritems(leaf) > 0); 1925109324cfSFilipe Manana btrfs_item_key(leaf, &first_key, 0); 1926109324cfSFilipe Manana 1927109324cfSFilipe Manana /* 1928109324cfSFilipe Manana * Doing the extra comparison with the first key is cheap, 1929109324cfSFilipe Manana * taking into account that the first key is very likely 1930109324cfSFilipe Manana * already in a cache line because it immediately follows 1931109324cfSFilipe Manana * the extent buffer's header and we have recently accessed 1932109324cfSFilipe Manana * the header's level field. 1933109324cfSFilipe Manana */ 1934109324cfSFilipe Manana ret = comp_keys(&first_key, key); 1935109324cfSFilipe Manana if (ret < 0) { 1936109324cfSFilipe Manana /* 1937109324cfSFilipe Manana * The first key is smaller than the key we want 1938109324cfSFilipe Manana * to insert, so we are safe to unlock all upper 1939109324cfSFilipe Manana * nodes and we have to do the binary search. 1940109324cfSFilipe Manana * 1941109324cfSFilipe Manana * We do use btrfs_unlock_up_safe() and not 1942109324cfSFilipe Manana * unlock_up() because the later does not unlock 1943109324cfSFilipe Manana * nodes with a slot of 0 - we can safely unlock 1944109324cfSFilipe Manana * any node even if its slot is 0 since in this 1945109324cfSFilipe Manana * case the key does not end up at slot 0 of the 1946109324cfSFilipe Manana * leaf and there's no need to split the leaf. 1947109324cfSFilipe Manana */ 1948109324cfSFilipe Manana btrfs_unlock_up_safe(path, 1); 1949109324cfSFilipe Manana search_low_slot = 1; 1950109324cfSFilipe Manana } else { 1951109324cfSFilipe Manana /* 1952109324cfSFilipe Manana * The first key is >= then the key we want to 1953109324cfSFilipe Manana * insert, so we can skip the binary search as 1954109324cfSFilipe Manana * the target key will be at slot 0. 1955109324cfSFilipe Manana * 1956109324cfSFilipe Manana * We can not unlock upper nodes when the key is 1957109324cfSFilipe Manana * less than the first key, because we will need 1958109324cfSFilipe Manana * to update the key at slot 0 of the parent node 1959109324cfSFilipe Manana * and possibly of other upper nodes too. 1960109324cfSFilipe Manana * If the key matches the first key, then we can 1961109324cfSFilipe Manana * unlock all the upper nodes, using 1962109324cfSFilipe Manana * btrfs_unlock_up_safe() instead of unlock_up() 1963109324cfSFilipe Manana * as stated above. 1964109324cfSFilipe Manana */ 1965109324cfSFilipe Manana if (ret == 0) 1966109324cfSFilipe Manana btrfs_unlock_up_safe(path, 1); 1967109324cfSFilipe Manana /* 1968109324cfSFilipe Manana * ret is already 0 or 1, matching the result of 1969109324cfSFilipe Manana * a btrfs_bin_search() call, so there is no need 1970109324cfSFilipe Manana * to adjust it. 1971109324cfSFilipe Manana */ 1972109324cfSFilipe Manana do_bin_search = false; 1973109324cfSFilipe Manana path->slots[0] = 0; 1974109324cfSFilipe Manana } 1975109324cfSFilipe Manana } 1976109324cfSFilipe Manana } 1977109324cfSFilipe Manana 1978109324cfSFilipe Manana if (do_bin_search) { 1979109324cfSFilipe Manana ret = search_for_key_slot(leaf, search_low_slot, key, 1980109324cfSFilipe Manana prev_cmp, &path->slots[0]); 1981109324cfSFilipe Manana if (ret < 0) 1982109324cfSFilipe Manana return ret; 1983109324cfSFilipe Manana } 1984109324cfSFilipe Manana 1985109324cfSFilipe Manana if (ins_len > 0) { 1986109324cfSFilipe Manana /* 1987109324cfSFilipe Manana * Item key already exists. In this case, if we are allowed to 1988109324cfSFilipe Manana * insert the item (for example, in dir_item case, item key 1989109324cfSFilipe Manana * collision is allowed), it will be merged with the original 1990109324cfSFilipe Manana * item. Only the item size grows, no new btrfs item will be 1991109324cfSFilipe Manana * added. If search_for_extension is not set, ins_len already 1992109324cfSFilipe Manana * accounts the size btrfs_item, deduct it here so leaf space 1993109324cfSFilipe Manana * check will be correct. 1994109324cfSFilipe Manana */ 1995109324cfSFilipe Manana if (ret == 0 && !path->search_for_extension) { 1996109324cfSFilipe Manana ASSERT(ins_len >= sizeof(struct btrfs_item)); 1997109324cfSFilipe Manana ins_len -= sizeof(struct btrfs_item); 1998109324cfSFilipe Manana } 1999109324cfSFilipe Manana 2000109324cfSFilipe Manana ASSERT(leaf_free_space >= 0); 2001109324cfSFilipe Manana 2002109324cfSFilipe Manana if (leaf_free_space < ins_len) { 2003109324cfSFilipe Manana int err; 2004109324cfSFilipe Manana 2005109324cfSFilipe Manana err = split_leaf(trans, root, key, path, ins_len, 2006109324cfSFilipe Manana (ret == 0)); 2007bb8e9a60SFilipe Manana ASSERT(err <= 0); 2008bb8e9a60SFilipe Manana if (WARN_ON(err > 0)) 2009bb8e9a60SFilipe Manana err = -EUCLEAN; 2010109324cfSFilipe Manana if (err) 2011109324cfSFilipe Manana ret = err; 2012109324cfSFilipe Manana } 2013109324cfSFilipe Manana } 2014109324cfSFilipe Manana 2015109324cfSFilipe Manana return ret; 2016109324cfSFilipe Manana } 2017109324cfSFilipe Manana 2018c8c42864SChris Mason /* 20194271eceaSNikolay Borisov * btrfs_search_slot - look for a key in a tree and perform necessary 20204271eceaSNikolay Borisov * modifications to preserve tree invariants. 202174123bd7SChris Mason * 20224271eceaSNikolay Borisov * @trans: Handle of transaction, used when modifying the tree 20234271eceaSNikolay Borisov * @p: Holds all btree nodes along the search path 20244271eceaSNikolay Borisov * @root: The root node of the tree 20254271eceaSNikolay Borisov * @key: The key we are looking for 20269a664971Sethanwu * @ins_len: Indicates purpose of search: 20279a664971Sethanwu * >0 for inserts it's size of item inserted (*) 20289a664971Sethanwu * <0 for deletions 20299a664971Sethanwu * 0 for plain searches, not modifying the tree 20309a664971Sethanwu * 20319a664971Sethanwu * (*) If size of item inserted doesn't include 20329a664971Sethanwu * sizeof(struct btrfs_item), then p->search_for_extension must 20339a664971Sethanwu * be set. 20344271eceaSNikolay Borisov * @cow: boolean should CoW operations be performed. Must always be 1 20354271eceaSNikolay Borisov * when modifying the tree. 203697571fd0SChris Mason * 20374271eceaSNikolay Borisov * If @ins_len > 0, nodes and leaves will be split as we walk down the tree. 20384271eceaSNikolay Borisov * If @ins_len < 0, nodes will be merged as we walk down the tree (if possible) 20394271eceaSNikolay Borisov * 20404271eceaSNikolay Borisov * If @key is found, 0 is returned and you can find the item in the leaf level 20414271eceaSNikolay Borisov * of the path (level 0) 20424271eceaSNikolay Borisov * 20434271eceaSNikolay Borisov * If @key isn't found, 1 is returned and the leaf level of the path (level 0) 20444271eceaSNikolay Borisov * points to the slot where it should be inserted 20454271eceaSNikolay Borisov * 20464271eceaSNikolay Borisov * If an error is encountered while searching the tree a negative error number 20474271eceaSNikolay Borisov * is returned 204874123bd7SChris Mason */ 2049310712b2SOmar Sandoval int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root, 2050310712b2SOmar Sandoval const struct btrfs_key *key, struct btrfs_path *p, 2051310712b2SOmar Sandoval int ins_len, int cow) 2052be0e5c09SChris Mason { 2053d96b3424SFilipe Manana struct btrfs_fs_info *fs_info = root->fs_info; 20545f39d397SChris Mason struct extent_buffer *b; 2055be0e5c09SChris Mason int slot; 2056be0e5c09SChris Mason int ret; 205733c66f43SYan Zheng int err; 2058be0e5c09SChris Mason int level; 2059925baeddSChris Mason int lowest_unlock = 1; 2060bd681513SChris Mason /* everything at write_lock_level or lower must be write locked */ 2061bd681513SChris Mason int write_lock_level = 0; 20629f3a7427SChris Mason u8 lowest_level = 0; 2063f7c79f30SChris Mason int min_write_lock_level; 2064d7396f07SFilipe David Borba Manana int prev_cmp; 20659f3a7427SChris Mason 2066a4c853afSChenXiaoSong might_sleep(); 2067a4c853afSChenXiaoSong 20686702ed49SChris Mason lowest_level = p->lowest_level; 2069323ac95bSChris Mason WARN_ON(lowest_level && ins_len > 0); 207022b0ebdaSChris Mason WARN_ON(p->nodes[0] != NULL); 2071eb653de1SFilipe David Borba Manana BUG_ON(!cow && ins_len); 207225179201SJosef Bacik 2073857bc13fSJosef Bacik /* 2074857bc13fSJosef Bacik * For now only allow nowait for read only operations. There's no 2075857bc13fSJosef Bacik * strict reason why we can't, we just only need it for reads so it's 2076857bc13fSJosef Bacik * only implemented for reads. 2077857bc13fSJosef Bacik */ 2078857bc13fSJosef Bacik ASSERT(!p->nowait || !cow); 2079857bc13fSJosef Bacik 2080bd681513SChris Mason if (ins_len < 0) { 2081925baeddSChris Mason lowest_unlock = 2; 208265b51a00SChris Mason 2083bd681513SChris Mason /* when we are removing items, we might have to go up to level 2084bd681513SChris Mason * two as we update tree pointers Make sure we keep write 2085bd681513SChris Mason * for those levels as well 2086bd681513SChris Mason */ 2087bd681513SChris Mason write_lock_level = 2; 2088bd681513SChris Mason } else if (ins_len > 0) { 2089bd681513SChris Mason /* 2090bd681513SChris Mason * for inserting items, make sure we have a write lock on 2091bd681513SChris Mason * level 1 so we can update keys 2092bd681513SChris Mason */ 2093bd681513SChris Mason write_lock_level = 1; 2094bd681513SChris Mason } 2095bd681513SChris Mason 2096bd681513SChris Mason if (!cow) 2097bd681513SChris Mason write_lock_level = -1; 2098bd681513SChris Mason 209909a2a8f9SJosef Bacik if (cow && (p->keep_locks || p->lowest_level)) 2100bd681513SChris Mason write_lock_level = BTRFS_MAX_LEVEL; 2101bd681513SChris Mason 2102f7c79f30SChris Mason min_write_lock_level = write_lock_level; 2103f7c79f30SChris Mason 2104d96b3424SFilipe Manana if (p->need_commit_sem) { 2105d96b3424SFilipe Manana ASSERT(p->search_commit_root); 2106857bc13fSJosef Bacik if (p->nowait) { 2107857bc13fSJosef Bacik if (!down_read_trylock(&fs_info->commit_root_sem)) 2108857bc13fSJosef Bacik return -EAGAIN; 2109857bc13fSJosef Bacik } else { 2110d96b3424SFilipe Manana down_read(&fs_info->commit_root_sem); 2111d96b3424SFilipe Manana } 2112857bc13fSJosef Bacik } 2113d96b3424SFilipe Manana 2114bb803951SChris Mason again: 2115d7396f07SFilipe David Borba Manana prev_cmp = -1; 21161fc28d8eSLiu Bo b = btrfs_search_slot_get_root(root, p, write_lock_level); 2117be6821f8SFilipe Manana if (IS_ERR(b)) { 2118be6821f8SFilipe Manana ret = PTR_ERR(b); 2119be6821f8SFilipe Manana goto done; 2120be6821f8SFilipe Manana } 2121925baeddSChris Mason 2122eb60ceacSChris Mason while (b) { 2123f624d976SQu Wenruo int dec = 0; 2124f624d976SQu Wenruo 21255f39d397SChris Mason level = btrfs_header_level(b); 212665b51a00SChris Mason 212702217ed2SChris Mason if (cow) { 21289ea2c7c9SNikolay Borisov bool last_level = (level == (BTRFS_MAX_LEVEL - 1)); 21299ea2c7c9SNikolay Borisov 2130c8c42864SChris Mason /* 2131c8c42864SChris Mason * if we don't really need to cow this block 2132c8c42864SChris Mason * then we don't want to set the path blocking, 2133c8c42864SChris Mason * so we test it here 2134c8c42864SChris Mason */ 21355963ffcaSJosef Bacik if (!should_cow_block(trans, root, b)) 213665b51a00SChris Mason goto cow_done; 21375d4f98a2SYan Zheng 2138bd681513SChris Mason /* 2139bd681513SChris Mason * must have write locks on this node and the 2140bd681513SChris Mason * parent 2141bd681513SChris Mason */ 21425124e00eSJosef Bacik if (level > write_lock_level || 21435124e00eSJosef Bacik (level + 1 > write_lock_level && 21445124e00eSJosef Bacik level + 1 < BTRFS_MAX_LEVEL && 21455124e00eSJosef Bacik p->nodes[level + 1])) { 2146bd681513SChris Mason write_lock_level = level + 1; 2147bd681513SChris Mason btrfs_release_path(p); 2148bd681513SChris Mason goto again; 2149bd681513SChris Mason } 2150bd681513SChris Mason 21519ea2c7c9SNikolay Borisov if (last_level) 21529ea2c7c9SNikolay Borisov err = btrfs_cow_block(trans, root, b, NULL, 0, 21539631e4ccSJosef Bacik &b, 21549631e4ccSJosef Bacik BTRFS_NESTING_COW); 21559ea2c7c9SNikolay Borisov else 215633c66f43SYan Zheng err = btrfs_cow_block(trans, root, b, 2157e20d96d6SChris Mason p->nodes[level + 1], 21589631e4ccSJosef Bacik p->slots[level + 1], &b, 21599631e4ccSJosef Bacik BTRFS_NESTING_COW); 216033c66f43SYan Zheng if (err) { 216133c66f43SYan Zheng ret = err; 216265b51a00SChris Mason goto done; 216354aa1f4dSChris Mason } 216402217ed2SChris Mason } 216565b51a00SChris Mason cow_done: 2166eb60ceacSChris Mason p->nodes[level] = b; 2167b4ce94deSChris Mason 2168b4ce94deSChris Mason /* 2169b4ce94deSChris Mason * we have a lock on b and as long as we aren't changing 2170b4ce94deSChris Mason * the tree, there is no way to for the items in b to change. 2171b4ce94deSChris Mason * It is safe to drop the lock on our parent before we 2172b4ce94deSChris Mason * go through the expensive btree search on b. 2173b4ce94deSChris Mason * 2174eb653de1SFilipe David Borba Manana * If we're inserting or deleting (ins_len != 0), then we might 2175eb653de1SFilipe David Borba Manana * be changing slot zero, which may require changing the parent. 2176eb653de1SFilipe David Borba Manana * So, we can't drop the lock until after we know which slot 2177eb653de1SFilipe David Borba Manana * we're operating on. 2178b4ce94deSChris Mason */ 2179eb653de1SFilipe David Borba Manana if (!ins_len && !p->keep_locks) { 2180eb653de1SFilipe David Borba Manana int u = level + 1; 2181eb653de1SFilipe David Borba Manana 2182eb653de1SFilipe David Borba Manana if (u < BTRFS_MAX_LEVEL && p->locks[u]) { 2183eb653de1SFilipe David Borba Manana btrfs_tree_unlock_rw(p->nodes[u], p->locks[u]); 2184eb653de1SFilipe David Borba Manana p->locks[u] = 0; 2185eb653de1SFilipe David Borba Manana } 2186eb653de1SFilipe David Borba Manana } 2187b4ce94deSChris Mason 2188e2e58d0fSFilipe Manana if (level == 0) { 2189109324cfSFilipe Manana if (ins_len > 0) 2190e5e1c174SFilipe Manana ASSERT(write_lock_level >= 1); 2191bd681513SChris Mason 2192109324cfSFilipe Manana ret = search_leaf(trans, root, key, p, ins_len, prev_cmp); 2193459931ecSChris Mason if (!p->search_for_split) 2194f7c79f30SChris Mason unlock_up(p, level, lowest_unlock, 21954b6f8e96SLiu Bo min_write_lock_level, NULL); 219665b51a00SChris Mason goto done; 219765b51a00SChris Mason } 2198e2e58d0fSFilipe Manana 2199e2e58d0fSFilipe Manana ret = search_for_key_slot(b, 0, key, prev_cmp, &slot); 2200e2e58d0fSFilipe Manana if (ret < 0) 2201e2e58d0fSFilipe Manana goto done; 2202e2e58d0fSFilipe Manana prev_cmp = ret; 2203e2e58d0fSFilipe Manana 2204f624d976SQu Wenruo if (ret && slot > 0) { 2205f624d976SQu Wenruo dec = 1; 2206f624d976SQu Wenruo slot--; 2207f624d976SQu Wenruo } 2208f624d976SQu Wenruo p->slots[level] = slot; 2209f624d976SQu Wenruo err = setup_nodes_for_search(trans, root, p, b, level, ins_len, 2210f624d976SQu Wenruo &write_lock_level); 2211f624d976SQu Wenruo if (err == -EAGAIN) 2212f624d976SQu Wenruo goto again; 2213f624d976SQu Wenruo if (err) { 2214f624d976SQu Wenruo ret = err; 2215f624d976SQu Wenruo goto done; 2216f624d976SQu Wenruo } 2217f624d976SQu Wenruo b = p->nodes[level]; 2218f624d976SQu Wenruo slot = p->slots[level]; 2219f624d976SQu Wenruo 2220f624d976SQu Wenruo /* 2221f624d976SQu Wenruo * Slot 0 is special, if we change the key we have to update 2222f624d976SQu Wenruo * the parent pointer which means we must have a write lock on 2223f624d976SQu Wenruo * the parent 2224f624d976SQu Wenruo */ 2225f624d976SQu Wenruo if (slot == 0 && ins_len && write_lock_level < level + 1) { 2226f624d976SQu Wenruo write_lock_level = level + 1; 2227f624d976SQu Wenruo btrfs_release_path(p); 2228f624d976SQu Wenruo goto again; 2229f624d976SQu Wenruo } 2230f624d976SQu Wenruo 2231f624d976SQu Wenruo unlock_up(p, level, lowest_unlock, min_write_lock_level, 2232f624d976SQu Wenruo &write_lock_level); 2233f624d976SQu Wenruo 2234f624d976SQu Wenruo if (level == lowest_level) { 2235f624d976SQu Wenruo if (dec) 2236f624d976SQu Wenruo p->slots[level]++; 2237f624d976SQu Wenruo goto done; 2238f624d976SQu Wenruo } 2239f624d976SQu Wenruo 2240f624d976SQu Wenruo err = read_block_for_search(root, p, &b, level, slot, key); 2241f624d976SQu Wenruo if (err == -EAGAIN) 2242f624d976SQu Wenruo goto again; 2243f624d976SQu Wenruo if (err) { 2244f624d976SQu Wenruo ret = err; 2245f624d976SQu Wenruo goto done; 2246f624d976SQu Wenruo } 2247f624d976SQu Wenruo 2248f624d976SQu Wenruo if (!p->skip_locking) { 2249f624d976SQu Wenruo level = btrfs_header_level(b); 2250b40130b2SJosef Bacik 2251b40130b2SJosef Bacik btrfs_maybe_reset_lockdep_class(root, b); 2252b40130b2SJosef Bacik 2253f624d976SQu Wenruo if (level <= write_lock_level) { 2254f624d976SQu Wenruo btrfs_tree_lock(b); 2255f624d976SQu Wenruo p->locks[level] = BTRFS_WRITE_LOCK; 2256f624d976SQu Wenruo } else { 2257857bc13fSJosef Bacik if (p->nowait) { 2258857bc13fSJosef Bacik if (!btrfs_try_tree_read_lock(b)) { 2259857bc13fSJosef Bacik free_extent_buffer(b); 2260857bc13fSJosef Bacik ret = -EAGAIN; 2261857bc13fSJosef Bacik goto done; 2262857bc13fSJosef Bacik } 2263857bc13fSJosef Bacik } else { 2264fe596ca3SJosef Bacik btrfs_tree_read_lock(b); 2265857bc13fSJosef Bacik } 2266f624d976SQu Wenruo p->locks[level] = BTRFS_READ_LOCK; 2267f624d976SQu Wenruo } 2268f624d976SQu Wenruo p->nodes[level] = b; 2269f624d976SQu Wenruo } 227065b51a00SChris Mason } 227165b51a00SChris Mason ret = 1; 227265b51a00SChris Mason done: 22735f5bc6b1SFilipe Manana if (ret < 0 && !p->skip_release_on_error) 2274b3b4aa74SDavid Sterba btrfs_release_path(p); 2275d96b3424SFilipe Manana 2276d96b3424SFilipe Manana if (p->need_commit_sem) { 2277d96b3424SFilipe Manana int ret2; 2278d96b3424SFilipe Manana 2279d96b3424SFilipe Manana ret2 = finish_need_commit_sem_search(p); 2280d96b3424SFilipe Manana up_read(&fs_info->commit_root_sem); 2281d96b3424SFilipe Manana if (ret2) 2282d96b3424SFilipe Manana ret = ret2; 2283d96b3424SFilipe Manana } 2284d96b3424SFilipe Manana 2285be0e5c09SChris Mason return ret; 2286be0e5c09SChris Mason } 2287f75e2b79SJosef Bacik ALLOW_ERROR_INJECTION(btrfs_search_slot, ERRNO); 2288be0e5c09SChris Mason 228974123bd7SChris Mason /* 22905d9e75c4SJan Schmidt * Like btrfs_search_slot, this looks for a key in the given tree. It uses the 22915d9e75c4SJan Schmidt * current state of the tree together with the operations recorded in the tree 22925d9e75c4SJan Schmidt * modification log to search for the key in a previous version of this tree, as 22935d9e75c4SJan Schmidt * denoted by the time_seq parameter. 22945d9e75c4SJan Schmidt * 22955d9e75c4SJan Schmidt * Naturally, there is no support for insert, delete or cow operations. 22965d9e75c4SJan Schmidt * 22975d9e75c4SJan Schmidt * The resulting path and return value will be set up as if we called 22985d9e75c4SJan Schmidt * btrfs_search_slot at that point in time with ins_len and cow both set to 0. 22995d9e75c4SJan Schmidt */ 2300310712b2SOmar Sandoval int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key, 23015d9e75c4SJan Schmidt struct btrfs_path *p, u64 time_seq) 23025d9e75c4SJan Schmidt { 23030b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 23045d9e75c4SJan Schmidt struct extent_buffer *b; 23055d9e75c4SJan Schmidt int slot; 23065d9e75c4SJan Schmidt int ret; 23075d9e75c4SJan Schmidt int err; 23085d9e75c4SJan Schmidt int level; 23095d9e75c4SJan Schmidt int lowest_unlock = 1; 23105d9e75c4SJan Schmidt u8 lowest_level = 0; 23115d9e75c4SJan Schmidt 23125d9e75c4SJan Schmidt lowest_level = p->lowest_level; 23135d9e75c4SJan Schmidt WARN_ON(p->nodes[0] != NULL); 2314c922b016SStefan Roesch ASSERT(!p->nowait); 23155d9e75c4SJan Schmidt 23165d9e75c4SJan Schmidt if (p->search_commit_root) { 23175d9e75c4SJan Schmidt BUG_ON(time_seq); 23185d9e75c4SJan Schmidt return btrfs_search_slot(NULL, root, key, p, 0, 0); 23195d9e75c4SJan Schmidt } 23205d9e75c4SJan Schmidt 23215d9e75c4SJan Schmidt again: 2322f3a84ccdSFilipe Manana b = btrfs_get_old_root(root, time_seq); 2323315bed43SNikolay Borisov if (!b) { 2324315bed43SNikolay Borisov ret = -EIO; 2325315bed43SNikolay Borisov goto done; 2326315bed43SNikolay Borisov } 23275d9e75c4SJan Schmidt level = btrfs_header_level(b); 23285d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 23295d9e75c4SJan Schmidt 23305d9e75c4SJan Schmidt while (b) { 2331abe9339dSQu Wenruo int dec = 0; 2332abe9339dSQu Wenruo 23335d9e75c4SJan Schmidt level = btrfs_header_level(b); 23345d9e75c4SJan Schmidt p->nodes[level] = b; 23355d9e75c4SJan Schmidt 23365d9e75c4SJan Schmidt /* 23375d9e75c4SJan Schmidt * we have a lock on b and as long as we aren't changing 23385d9e75c4SJan Schmidt * the tree, there is no way to for the items in b to change. 23395d9e75c4SJan Schmidt * It is safe to drop the lock on our parent before we 23405d9e75c4SJan Schmidt * go through the expensive btree search on b. 23415d9e75c4SJan Schmidt */ 23425d9e75c4SJan Schmidt btrfs_unlock_up_safe(p, level + 1); 23435d9e75c4SJan Schmidt 2344fdf8d595SAnand Jain ret = btrfs_bin_search(b, 0, key, &slot); 2345cbca7d59SFilipe Manana if (ret < 0) 2346cbca7d59SFilipe Manana goto done; 23475d9e75c4SJan Schmidt 2348abe9339dSQu Wenruo if (level == 0) { 2349abe9339dSQu Wenruo p->slots[level] = slot; 2350abe9339dSQu Wenruo unlock_up(p, level, lowest_unlock, 0, NULL); 2351abe9339dSQu Wenruo goto done; 2352abe9339dSQu Wenruo } 2353abe9339dSQu Wenruo 23545d9e75c4SJan Schmidt if (ret && slot > 0) { 23555d9e75c4SJan Schmidt dec = 1; 2356abe9339dSQu Wenruo slot--; 23575d9e75c4SJan Schmidt } 23585d9e75c4SJan Schmidt p->slots[level] = slot; 23595d9e75c4SJan Schmidt unlock_up(p, level, lowest_unlock, 0, NULL); 23605d9e75c4SJan Schmidt 23615d9e75c4SJan Schmidt if (level == lowest_level) { 23625d9e75c4SJan Schmidt if (dec) 23635d9e75c4SJan Schmidt p->slots[level]++; 23645d9e75c4SJan Schmidt goto done; 23655d9e75c4SJan Schmidt } 23665d9e75c4SJan Schmidt 2367abe9339dSQu Wenruo err = read_block_for_search(root, p, &b, level, slot, key); 23685d9e75c4SJan Schmidt if (err == -EAGAIN) 23695d9e75c4SJan Schmidt goto again; 23705d9e75c4SJan Schmidt if (err) { 23715d9e75c4SJan Schmidt ret = err; 23725d9e75c4SJan Schmidt goto done; 23735d9e75c4SJan Schmidt } 23745d9e75c4SJan Schmidt 23755d9e75c4SJan Schmidt level = btrfs_header_level(b); 23765d9e75c4SJan Schmidt btrfs_tree_read_lock(b); 2377f3a84ccdSFilipe Manana b = btrfs_tree_mod_log_rewind(fs_info, p, b, time_seq); 2378db7f3436SJosef Bacik if (!b) { 2379db7f3436SJosef Bacik ret = -ENOMEM; 2380db7f3436SJosef Bacik goto done; 2381db7f3436SJosef Bacik } 23825d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 23835d9e75c4SJan Schmidt p->nodes[level] = b; 23845d9e75c4SJan Schmidt } 23855d9e75c4SJan Schmidt ret = 1; 23865d9e75c4SJan Schmidt done: 23875d9e75c4SJan Schmidt if (ret < 0) 23885d9e75c4SJan Schmidt btrfs_release_path(p); 23895d9e75c4SJan Schmidt 23905d9e75c4SJan Schmidt return ret; 23915d9e75c4SJan Schmidt } 23925d9e75c4SJan Schmidt 23935d9e75c4SJan Schmidt /* 2394f469c8bdSFilipe Manana * Search the tree again to find a leaf with smaller keys. 2395f469c8bdSFilipe Manana * Returns 0 if it found something. 2396f469c8bdSFilipe Manana * Returns 1 if there are no smaller keys. 2397f469c8bdSFilipe Manana * Returns < 0 on error. 2398f469c8bdSFilipe Manana * 2399f469c8bdSFilipe Manana * This may release the path, and so you may lose any locks held at the 2400f469c8bdSFilipe Manana * time you call it. 2401f469c8bdSFilipe Manana */ 2402f469c8bdSFilipe Manana static int btrfs_prev_leaf(struct btrfs_root *root, struct btrfs_path *path) 2403f469c8bdSFilipe Manana { 2404f469c8bdSFilipe Manana struct btrfs_key key; 2405f469c8bdSFilipe Manana struct btrfs_key orig_key; 2406f469c8bdSFilipe Manana struct btrfs_disk_key found_key; 2407f469c8bdSFilipe Manana int ret; 2408f469c8bdSFilipe Manana 2409f469c8bdSFilipe Manana btrfs_item_key_to_cpu(path->nodes[0], &key, 0); 2410f469c8bdSFilipe Manana orig_key = key; 2411f469c8bdSFilipe Manana 2412f469c8bdSFilipe Manana if (key.offset > 0) { 2413f469c8bdSFilipe Manana key.offset--; 2414f469c8bdSFilipe Manana } else if (key.type > 0) { 2415f469c8bdSFilipe Manana key.type--; 2416f469c8bdSFilipe Manana key.offset = (u64)-1; 2417f469c8bdSFilipe Manana } else if (key.objectid > 0) { 2418f469c8bdSFilipe Manana key.objectid--; 2419f469c8bdSFilipe Manana key.type = (u8)-1; 2420f469c8bdSFilipe Manana key.offset = (u64)-1; 2421f469c8bdSFilipe Manana } else { 2422f469c8bdSFilipe Manana return 1; 2423f469c8bdSFilipe Manana } 2424f469c8bdSFilipe Manana 2425f469c8bdSFilipe Manana btrfs_release_path(path); 2426f469c8bdSFilipe Manana ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 2427f469c8bdSFilipe Manana if (ret <= 0) 2428f469c8bdSFilipe Manana return ret; 2429f469c8bdSFilipe Manana 2430f469c8bdSFilipe Manana /* 2431f469c8bdSFilipe Manana * Previous key not found. Even if we were at slot 0 of the leaf we had 2432f469c8bdSFilipe Manana * before releasing the path and calling btrfs_search_slot(), we now may 2433f469c8bdSFilipe Manana * be in a slot pointing to the same original key - this can happen if 2434f469c8bdSFilipe Manana * after we released the path, one of more items were moved from a 2435f469c8bdSFilipe Manana * sibling leaf into the front of the leaf we had due to an insertion 2436f469c8bdSFilipe Manana * (see push_leaf_right()). 2437f469c8bdSFilipe Manana * If we hit this case and our slot is > 0 and just decrement the slot 2438f469c8bdSFilipe Manana * so that the caller does not process the same key again, which may or 2439f469c8bdSFilipe Manana * may not break the caller, depending on its logic. 2440f469c8bdSFilipe Manana */ 2441f469c8bdSFilipe Manana if (path->slots[0] < btrfs_header_nritems(path->nodes[0])) { 2442f469c8bdSFilipe Manana btrfs_item_key(path->nodes[0], &found_key, path->slots[0]); 2443f469c8bdSFilipe Manana ret = comp_keys(&found_key, &orig_key); 2444f469c8bdSFilipe Manana if (ret == 0) { 2445f469c8bdSFilipe Manana if (path->slots[0] > 0) { 2446f469c8bdSFilipe Manana path->slots[0]--; 2447f469c8bdSFilipe Manana return 0; 2448f469c8bdSFilipe Manana } 2449f469c8bdSFilipe Manana /* 2450f469c8bdSFilipe Manana * At slot 0, same key as before, it means orig_key is 2451f469c8bdSFilipe Manana * the lowest, leftmost, key in the tree. We're done. 2452f469c8bdSFilipe Manana */ 2453f469c8bdSFilipe Manana return 1; 2454f469c8bdSFilipe Manana } 2455f469c8bdSFilipe Manana } 2456f469c8bdSFilipe Manana 2457f469c8bdSFilipe Manana btrfs_item_key(path->nodes[0], &found_key, 0); 2458f469c8bdSFilipe Manana ret = comp_keys(&found_key, &key); 2459f469c8bdSFilipe Manana /* 2460f469c8bdSFilipe Manana * We might have had an item with the previous key in the tree right 2461f469c8bdSFilipe Manana * before we released our path. And after we released our path, that 2462f469c8bdSFilipe Manana * item might have been pushed to the first slot (0) of the leaf we 2463f469c8bdSFilipe Manana * were holding due to a tree balance. Alternatively, an item with the 2464f469c8bdSFilipe Manana * previous key can exist as the only element of a leaf (big fat item). 2465f469c8bdSFilipe Manana * Therefore account for these 2 cases, so that our callers (like 2466f469c8bdSFilipe Manana * btrfs_previous_item) don't miss an existing item with a key matching 2467f469c8bdSFilipe Manana * the previous key we computed above. 2468f469c8bdSFilipe Manana */ 2469f469c8bdSFilipe Manana if (ret <= 0) 2470f469c8bdSFilipe Manana return 0; 2471f469c8bdSFilipe Manana return 1; 2472f469c8bdSFilipe Manana } 2473f469c8bdSFilipe Manana 2474f469c8bdSFilipe Manana /* 24752f38b3e1SArne Jansen * helper to use instead of search slot if no exact match is needed but 24762f38b3e1SArne Jansen * instead the next or previous item should be returned. 24772f38b3e1SArne Jansen * When find_higher is true, the next higher item is returned, the next lower 24782f38b3e1SArne Jansen * otherwise. 24792f38b3e1SArne Jansen * When return_any and find_higher are both true, and no higher item is found, 24802f38b3e1SArne Jansen * return the next lower instead. 24812f38b3e1SArne Jansen * When return_any is true and find_higher is false, and no lower item is found, 24822f38b3e1SArne Jansen * return the next higher instead. 24832f38b3e1SArne Jansen * It returns 0 if any item is found, 1 if none is found (tree empty), and 24842f38b3e1SArne Jansen * < 0 on error 24852f38b3e1SArne Jansen */ 24862f38b3e1SArne Jansen int btrfs_search_slot_for_read(struct btrfs_root *root, 2487310712b2SOmar Sandoval const struct btrfs_key *key, 2488310712b2SOmar Sandoval struct btrfs_path *p, int find_higher, 2489310712b2SOmar Sandoval int return_any) 24902f38b3e1SArne Jansen { 24912f38b3e1SArne Jansen int ret; 24922f38b3e1SArne Jansen struct extent_buffer *leaf; 24932f38b3e1SArne Jansen 24942f38b3e1SArne Jansen again: 24952f38b3e1SArne Jansen ret = btrfs_search_slot(NULL, root, key, p, 0, 0); 24962f38b3e1SArne Jansen if (ret <= 0) 24972f38b3e1SArne Jansen return ret; 24982f38b3e1SArne Jansen /* 24992f38b3e1SArne Jansen * a return value of 1 means the path is at the position where the 25002f38b3e1SArne Jansen * item should be inserted. Normally this is the next bigger item, 25012f38b3e1SArne Jansen * but in case the previous item is the last in a leaf, path points 25022f38b3e1SArne Jansen * to the first free slot in the previous leaf, i.e. at an invalid 25032f38b3e1SArne Jansen * item. 25042f38b3e1SArne Jansen */ 25052f38b3e1SArne Jansen leaf = p->nodes[0]; 25062f38b3e1SArne Jansen 25072f38b3e1SArne Jansen if (find_higher) { 25082f38b3e1SArne Jansen if (p->slots[0] >= btrfs_header_nritems(leaf)) { 25092f38b3e1SArne Jansen ret = btrfs_next_leaf(root, p); 25102f38b3e1SArne Jansen if (ret <= 0) 25112f38b3e1SArne Jansen return ret; 25122f38b3e1SArne Jansen if (!return_any) 25132f38b3e1SArne Jansen return 1; 25142f38b3e1SArne Jansen /* 25152f38b3e1SArne Jansen * no higher item found, return the next 25162f38b3e1SArne Jansen * lower instead 25172f38b3e1SArne Jansen */ 25182f38b3e1SArne Jansen return_any = 0; 25192f38b3e1SArne Jansen find_higher = 0; 25202f38b3e1SArne Jansen btrfs_release_path(p); 25212f38b3e1SArne Jansen goto again; 25222f38b3e1SArne Jansen } 25232f38b3e1SArne Jansen } else { 25242f38b3e1SArne Jansen if (p->slots[0] == 0) { 25252f38b3e1SArne Jansen ret = btrfs_prev_leaf(root, p); 2526e6793769SArne Jansen if (ret < 0) 25272f38b3e1SArne Jansen return ret; 2528e6793769SArne Jansen if (!ret) { 252923c6bf6aSFilipe David Borba Manana leaf = p->nodes[0]; 253023c6bf6aSFilipe David Borba Manana if (p->slots[0] == btrfs_header_nritems(leaf)) 253123c6bf6aSFilipe David Borba Manana p->slots[0]--; 2532e6793769SArne Jansen return 0; 2533e6793769SArne Jansen } 25342f38b3e1SArne Jansen if (!return_any) 25352f38b3e1SArne Jansen return 1; 25362f38b3e1SArne Jansen /* 25372f38b3e1SArne Jansen * no lower item found, return the next 25382f38b3e1SArne Jansen * higher instead 25392f38b3e1SArne Jansen */ 25402f38b3e1SArne Jansen return_any = 0; 25412f38b3e1SArne Jansen find_higher = 1; 25422f38b3e1SArne Jansen btrfs_release_path(p); 25432f38b3e1SArne Jansen goto again; 2544e6793769SArne Jansen } else { 25452f38b3e1SArne Jansen --p->slots[0]; 25462f38b3e1SArne Jansen } 25472f38b3e1SArne Jansen } 25482f38b3e1SArne Jansen return 0; 25492f38b3e1SArne Jansen } 25502f38b3e1SArne Jansen 25512f38b3e1SArne Jansen /* 25520ff40a91SMarcos Paulo de Souza * Execute search and call btrfs_previous_item to traverse backwards if the item 25530ff40a91SMarcos Paulo de Souza * was not found. 25540ff40a91SMarcos Paulo de Souza * 25550ff40a91SMarcos Paulo de Souza * Return 0 if found, 1 if not found and < 0 if error. 25560ff40a91SMarcos Paulo de Souza */ 25570ff40a91SMarcos Paulo de Souza int btrfs_search_backwards(struct btrfs_root *root, struct btrfs_key *key, 25580ff40a91SMarcos Paulo de Souza struct btrfs_path *path) 25590ff40a91SMarcos Paulo de Souza { 25600ff40a91SMarcos Paulo de Souza int ret; 25610ff40a91SMarcos Paulo de Souza 25620ff40a91SMarcos Paulo de Souza ret = btrfs_search_slot(NULL, root, key, path, 0, 0); 25630ff40a91SMarcos Paulo de Souza if (ret > 0) 25640ff40a91SMarcos Paulo de Souza ret = btrfs_previous_item(root, path, key->objectid, key->type); 25650ff40a91SMarcos Paulo de Souza 25660ff40a91SMarcos Paulo de Souza if (ret == 0) 25670ff40a91SMarcos Paulo de Souza btrfs_item_key_to_cpu(path->nodes[0], key, path->slots[0]); 25680ff40a91SMarcos Paulo de Souza 25690ff40a91SMarcos Paulo de Souza return ret; 25700ff40a91SMarcos Paulo de Souza } 25710ff40a91SMarcos Paulo de Souza 257243dd529aSDavid Sterba /* 257362142be3SGabriel Niebler * Search for a valid slot for the given path. 257462142be3SGabriel Niebler * 257562142be3SGabriel Niebler * @root: The root node of the tree. 257662142be3SGabriel Niebler * @key: Will contain a valid item if found. 257762142be3SGabriel Niebler * @path: The starting point to validate the slot. 257862142be3SGabriel Niebler * 257962142be3SGabriel Niebler * Return: 0 if the item is valid 258062142be3SGabriel Niebler * 1 if not found 258162142be3SGabriel Niebler * <0 if error. 258262142be3SGabriel Niebler */ 258362142be3SGabriel Niebler int btrfs_get_next_valid_item(struct btrfs_root *root, struct btrfs_key *key, 258462142be3SGabriel Niebler struct btrfs_path *path) 258562142be3SGabriel Niebler { 2586524f14bbSFilipe Manana if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) { 258762142be3SGabriel Niebler int ret; 258862142be3SGabriel Niebler 258962142be3SGabriel Niebler ret = btrfs_next_leaf(root, path); 259062142be3SGabriel Niebler if (ret) 259162142be3SGabriel Niebler return ret; 259262142be3SGabriel Niebler } 2593524f14bbSFilipe Manana 2594524f14bbSFilipe Manana btrfs_item_key_to_cpu(path->nodes[0], key, path->slots[0]); 259562142be3SGabriel Niebler return 0; 259662142be3SGabriel Niebler } 259762142be3SGabriel Niebler 25980ff40a91SMarcos Paulo de Souza /* 259974123bd7SChris Mason * adjust the pointers going up the tree, starting at level 260074123bd7SChris Mason * making sure the right key of each node is points to 'key'. 260174123bd7SChris Mason * This is used after shifting pointers to the left, so it stops 260274123bd7SChris Mason * fixing up pointers when a given leaf/node is not in slot 0 of the 260374123bd7SChris Mason * higher levels 2604aa5d6bedSChris Mason * 260574123bd7SChris Mason */ 2606b167fa91SNikolay Borisov static void fixup_low_keys(struct btrfs_path *path, 26075f39d397SChris Mason struct btrfs_disk_key *key, int level) 2608be0e5c09SChris Mason { 2609be0e5c09SChris Mason int i; 26105f39d397SChris Mason struct extent_buffer *t; 26110e82bcfeSDavid Sterba int ret; 26125f39d397SChris Mason 2613234b63a0SChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 2614be0e5c09SChris Mason int tslot = path->slots[i]; 26150e82bcfeSDavid Sterba 2616eb60ceacSChris Mason if (!path->nodes[i]) 2617be0e5c09SChris Mason break; 26185f39d397SChris Mason t = path->nodes[i]; 2619f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(t, tslot, 262033cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 26210e82bcfeSDavid Sterba BUG_ON(ret < 0); 26225f39d397SChris Mason btrfs_set_node_key(t, key, tslot); 2623d6025579SChris Mason btrfs_mark_buffer_dirty(path->nodes[i]); 2624be0e5c09SChris Mason if (tslot != 0) 2625be0e5c09SChris Mason break; 2626be0e5c09SChris Mason } 2627be0e5c09SChris Mason } 2628be0e5c09SChris Mason 262974123bd7SChris Mason /* 263031840ae1SZheng Yan * update item key. 263131840ae1SZheng Yan * 263231840ae1SZheng Yan * This function isn't completely safe. It's the caller's responsibility 263331840ae1SZheng Yan * that the new key won't break the order 263431840ae1SZheng Yan */ 2635b7a0365eSDaniel Dressler void btrfs_set_item_key_safe(struct btrfs_fs_info *fs_info, 2636b7a0365eSDaniel Dressler struct btrfs_path *path, 2637310712b2SOmar Sandoval const struct btrfs_key *new_key) 263831840ae1SZheng Yan { 263931840ae1SZheng Yan struct btrfs_disk_key disk_key; 264031840ae1SZheng Yan struct extent_buffer *eb; 264131840ae1SZheng Yan int slot; 264231840ae1SZheng Yan 264331840ae1SZheng Yan eb = path->nodes[0]; 264431840ae1SZheng Yan slot = path->slots[0]; 264531840ae1SZheng Yan if (slot > 0) { 264631840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot - 1); 26477c15d410SQu Wenruo if (unlikely(comp_keys(&disk_key, new_key) >= 0)) { 2648eee3b811SQu Wenruo btrfs_print_leaf(eb); 26497c15d410SQu Wenruo btrfs_crit(fs_info, 26507c15d410SQu Wenruo "slot %u key (%llu %u %llu) new key (%llu %u %llu)", 26517c15d410SQu Wenruo slot, btrfs_disk_key_objectid(&disk_key), 26527c15d410SQu Wenruo btrfs_disk_key_type(&disk_key), 26537c15d410SQu Wenruo btrfs_disk_key_offset(&disk_key), 26547c15d410SQu Wenruo new_key->objectid, new_key->type, 26557c15d410SQu Wenruo new_key->offset); 26567c15d410SQu Wenruo BUG(); 26577c15d410SQu Wenruo } 265831840ae1SZheng Yan } 265931840ae1SZheng Yan if (slot < btrfs_header_nritems(eb) - 1) { 266031840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot + 1); 26617c15d410SQu Wenruo if (unlikely(comp_keys(&disk_key, new_key) <= 0)) { 2662eee3b811SQu Wenruo btrfs_print_leaf(eb); 26637c15d410SQu Wenruo btrfs_crit(fs_info, 26647c15d410SQu Wenruo "slot %u key (%llu %u %llu) new key (%llu %u %llu)", 26657c15d410SQu Wenruo slot, btrfs_disk_key_objectid(&disk_key), 26667c15d410SQu Wenruo btrfs_disk_key_type(&disk_key), 26677c15d410SQu Wenruo btrfs_disk_key_offset(&disk_key), 26687c15d410SQu Wenruo new_key->objectid, new_key->type, 26697c15d410SQu Wenruo new_key->offset); 26707c15d410SQu Wenruo BUG(); 26717c15d410SQu Wenruo } 267231840ae1SZheng Yan } 267331840ae1SZheng Yan 267431840ae1SZheng Yan btrfs_cpu_key_to_disk(&disk_key, new_key); 267531840ae1SZheng Yan btrfs_set_item_key(eb, &disk_key, slot); 267631840ae1SZheng Yan btrfs_mark_buffer_dirty(eb); 267731840ae1SZheng Yan if (slot == 0) 2678b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 267931840ae1SZheng Yan } 268031840ae1SZheng Yan 268131840ae1SZheng Yan /* 2682d16c702fSQu Wenruo * Check key order of two sibling extent buffers. 2683d16c702fSQu Wenruo * 2684d16c702fSQu Wenruo * Return true if something is wrong. 2685d16c702fSQu Wenruo * Return false if everything is fine. 2686d16c702fSQu Wenruo * 2687d16c702fSQu Wenruo * Tree-checker only works inside one tree block, thus the following 2688d16c702fSQu Wenruo * corruption can not be detected by tree-checker: 2689d16c702fSQu Wenruo * 2690d16c702fSQu Wenruo * Leaf @left | Leaf @right 2691d16c702fSQu Wenruo * -------------------------------------------------------------- 2692d16c702fSQu Wenruo * | 1 | 2 | 3 | 4 | 5 | f6 | | 7 | 8 | 2693d16c702fSQu Wenruo * 2694d16c702fSQu Wenruo * Key f6 in leaf @left itself is valid, but not valid when the next 2695d16c702fSQu Wenruo * key in leaf @right is 7. 2696d16c702fSQu Wenruo * This can only be checked at tree block merge time. 2697d16c702fSQu Wenruo * And since tree checker has ensured all key order in each tree block 2698d16c702fSQu Wenruo * is correct, we only need to bother the last key of @left and the first 2699d16c702fSQu Wenruo * key of @right. 2700d16c702fSQu Wenruo */ 2701d16c702fSQu Wenruo static bool check_sibling_keys(struct extent_buffer *left, 2702d16c702fSQu Wenruo struct extent_buffer *right) 2703d16c702fSQu Wenruo { 2704d16c702fSQu Wenruo struct btrfs_key left_last; 2705d16c702fSQu Wenruo struct btrfs_key right_first; 2706d16c702fSQu Wenruo int level = btrfs_header_level(left); 2707d16c702fSQu Wenruo int nr_left = btrfs_header_nritems(left); 2708d16c702fSQu Wenruo int nr_right = btrfs_header_nritems(right); 2709d16c702fSQu Wenruo 2710d16c702fSQu Wenruo /* No key to check in one of the tree blocks */ 2711d16c702fSQu Wenruo if (!nr_left || !nr_right) 2712d16c702fSQu Wenruo return false; 2713d16c702fSQu Wenruo 2714d16c702fSQu Wenruo if (level) { 2715d16c702fSQu Wenruo btrfs_node_key_to_cpu(left, &left_last, nr_left - 1); 2716d16c702fSQu Wenruo btrfs_node_key_to_cpu(right, &right_first, 0); 2717d16c702fSQu Wenruo } else { 2718d16c702fSQu Wenruo btrfs_item_key_to_cpu(left, &left_last, nr_left - 1); 2719d16c702fSQu Wenruo btrfs_item_key_to_cpu(right, &right_first, 0); 2720d16c702fSQu Wenruo } 2721d16c702fSQu Wenruo 272288ad95b0SFilipe Manana if (unlikely(btrfs_comp_cpu_keys(&left_last, &right_first) >= 0)) { 2723a2cea677SFilipe Manana btrfs_crit(left->fs_info, "left extent buffer:"); 2724a2cea677SFilipe Manana btrfs_print_tree(left, false); 2725a2cea677SFilipe Manana btrfs_crit(left->fs_info, "right extent buffer:"); 2726a2cea677SFilipe Manana btrfs_print_tree(right, false); 2727d16c702fSQu Wenruo btrfs_crit(left->fs_info, 2728d16c702fSQu Wenruo "bad key order, sibling blocks, left last (%llu %u %llu) right first (%llu %u %llu)", 2729d16c702fSQu Wenruo left_last.objectid, left_last.type, 2730d16c702fSQu Wenruo left_last.offset, right_first.objectid, 2731d16c702fSQu Wenruo right_first.type, right_first.offset); 2732d16c702fSQu Wenruo return true; 2733d16c702fSQu Wenruo } 2734d16c702fSQu Wenruo return false; 2735d16c702fSQu Wenruo } 2736d16c702fSQu Wenruo 2737d16c702fSQu Wenruo /* 273874123bd7SChris Mason * try to push data from one node into the next node left in the 273979f95c82SChris Mason * tree. 2740aa5d6bedSChris Mason * 2741aa5d6bedSChris Mason * returns 0 if some ptrs were pushed left, < 0 if there was some horrible 2742aa5d6bedSChris Mason * error, and > 0 if there was no room in the left hand block. 274374123bd7SChris Mason */ 274498ed5174SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 27452ff7e61eSJeff Mahoney struct extent_buffer *dst, 2746971a1f66SChris Mason struct extent_buffer *src, int empty) 2747be0e5c09SChris Mason { 2748d30a668fSDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info; 2749be0e5c09SChris Mason int push_items = 0; 2750bb803951SChris Mason int src_nritems; 2751bb803951SChris Mason int dst_nritems; 2752aa5d6bedSChris Mason int ret = 0; 2753be0e5c09SChris Mason 27545f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 27555f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 27560b246afaSJeff Mahoney push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems; 27577bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 27587bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 275954aa1f4dSChris Mason 2760bce4eae9SChris Mason if (!empty && src_nritems <= 8) 2761971a1f66SChris Mason return 1; 2762971a1f66SChris Mason 2763d397712bSChris Mason if (push_items <= 0) 2764be0e5c09SChris Mason return 1; 2765be0e5c09SChris Mason 2766bce4eae9SChris Mason if (empty) { 2767971a1f66SChris Mason push_items = min(src_nritems, push_items); 2768bce4eae9SChris Mason if (push_items < src_nritems) { 2769bce4eae9SChris Mason /* leave at least 8 pointers in the node if 2770bce4eae9SChris Mason * we aren't going to empty it 2771bce4eae9SChris Mason */ 2772bce4eae9SChris Mason if (src_nritems - push_items < 8) { 2773bce4eae9SChris Mason if (push_items <= 8) 2774bce4eae9SChris Mason return 1; 2775bce4eae9SChris Mason push_items -= 8; 2776bce4eae9SChris Mason } 2777bce4eae9SChris Mason } 2778bce4eae9SChris Mason } else 2779bce4eae9SChris Mason push_items = min(src_nritems - 8, push_items); 278079f95c82SChris Mason 2781d16c702fSQu Wenruo /* dst is the left eb, src is the middle eb */ 2782d16c702fSQu Wenruo if (check_sibling_keys(dst, src)) { 2783d16c702fSQu Wenruo ret = -EUCLEAN; 2784d16c702fSQu Wenruo btrfs_abort_transaction(trans, ret); 2785d16c702fSQu Wenruo return ret; 2786d16c702fSQu Wenruo } 2787f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_eb_copy(dst, src, dst_nritems, 0, push_items); 27885de865eeSFilipe David Borba Manana if (ret) { 278966642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 27905de865eeSFilipe David Borba Manana return ret; 27915de865eeSFilipe David Borba Manana } 27925f39d397SChris Mason copy_extent_buffer(dst, src, 2793e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(dst, dst_nritems), 2794e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(src, 0), 2795123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 27965f39d397SChris Mason 2797bb803951SChris Mason if (push_items < src_nritems) { 279857911b8bSJan Schmidt /* 27995cead542SBoris Burkov * btrfs_tree_mod_log_eb_copy handles logging the move, so we 28005cead542SBoris Burkov * don't need to do an explicit tree mod log operation for it. 280157911b8bSJan Schmidt */ 2802e23efd8eSJosef Bacik memmove_extent_buffer(src, btrfs_node_key_ptr_offset(src, 0), 2803e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(src, push_items), 2804e2fa7227SChris Mason (src_nritems - push_items) * 2805123abc88SChris Mason sizeof(struct btrfs_key_ptr)); 2806bb803951SChris Mason } 28075f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 28085f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 28095f39d397SChris Mason btrfs_mark_buffer_dirty(src); 28105f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 281131840ae1SZheng Yan 2812bb803951SChris Mason return ret; 2813be0e5c09SChris Mason } 2814be0e5c09SChris Mason 281597571fd0SChris Mason /* 281679f95c82SChris Mason * try to push data from one node into the next node right in the 281779f95c82SChris Mason * tree. 281879f95c82SChris Mason * 281979f95c82SChris Mason * returns 0 if some ptrs were pushed, < 0 if there was some horrible 282079f95c82SChris Mason * error, and > 0 if there was no room in the right hand block. 282179f95c82SChris Mason * 282279f95c82SChris Mason * this will only push up to 1/2 the contents of the left node over 282379f95c82SChris Mason */ 28245f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 28255f39d397SChris Mason struct extent_buffer *dst, 28265f39d397SChris Mason struct extent_buffer *src) 282779f95c82SChris Mason { 282855d32ed8SDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info; 282979f95c82SChris Mason int push_items = 0; 283079f95c82SChris Mason int max_push; 283179f95c82SChris Mason int src_nritems; 283279f95c82SChris Mason int dst_nritems; 283379f95c82SChris Mason int ret = 0; 283479f95c82SChris Mason 28357bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 28367bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 28377bb86316SChris Mason 28385f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 28395f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 28400b246afaSJeff Mahoney push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems; 2841d397712bSChris Mason if (push_items <= 0) 284279f95c82SChris Mason return 1; 2843bce4eae9SChris Mason 2844d397712bSChris Mason if (src_nritems < 4) 2845bce4eae9SChris Mason return 1; 284679f95c82SChris Mason 284779f95c82SChris Mason max_push = src_nritems / 2 + 1; 284879f95c82SChris Mason /* don't try to empty the node */ 2849d397712bSChris Mason if (max_push >= src_nritems) 285079f95c82SChris Mason return 1; 2851252c38f0SYan 285279f95c82SChris Mason if (max_push < push_items) 285379f95c82SChris Mason push_items = max_push; 285479f95c82SChris Mason 2855d16c702fSQu Wenruo /* dst is the right eb, src is the middle eb */ 2856d16c702fSQu Wenruo if (check_sibling_keys(src, dst)) { 2857d16c702fSQu Wenruo ret = -EUCLEAN; 2858d16c702fSQu Wenruo btrfs_abort_transaction(trans, ret); 2859d16c702fSQu Wenruo return ret; 2860d16c702fSQu Wenruo } 28615cead542SBoris Burkov 28625cead542SBoris Burkov /* 28635cead542SBoris Burkov * btrfs_tree_mod_log_eb_copy handles logging the move, so we don't 28645cead542SBoris Burkov * need to do an explicit tree mod log operation for it. 28655cead542SBoris Burkov */ 2866e23efd8eSJosef Bacik memmove_extent_buffer(dst, btrfs_node_key_ptr_offset(dst, push_items), 2867e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(dst, 0), 28685f39d397SChris Mason (dst_nritems) * 28695f39d397SChris Mason sizeof(struct btrfs_key_ptr)); 2870d6025579SChris Mason 2871f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_eb_copy(dst, src, 0, src_nritems - push_items, 2872ed874f0dSDavid Sterba push_items); 28735de865eeSFilipe David Borba Manana if (ret) { 287466642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 28755de865eeSFilipe David Borba Manana return ret; 28765de865eeSFilipe David Borba Manana } 28775f39d397SChris Mason copy_extent_buffer(dst, src, 2878e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(dst, 0), 2879e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(src, src_nritems - push_items), 2880123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 288179f95c82SChris Mason 28825f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 28835f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 288479f95c82SChris Mason 28855f39d397SChris Mason btrfs_mark_buffer_dirty(src); 28865f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 288731840ae1SZheng Yan 288879f95c82SChris Mason return ret; 288979f95c82SChris Mason } 289079f95c82SChris Mason 289179f95c82SChris Mason /* 289297571fd0SChris Mason * helper function to insert a new root level in the tree. 289397571fd0SChris Mason * A new node is allocated, and a single item is inserted to 289497571fd0SChris Mason * point to the existing root 2895aa5d6bedSChris Mason * 2896aa5d6bedSChris Mason * returns zero on success or < 0 on failure. 289797571fd0SChris Mason */ 2898d397712bSChris Mason static noinline int insert_new_root(struct btrfs_trans_handle *trans, 28995f39d397SChris Mason struct btrfs_root *root, 2900fdd99c72SLiu Bo struct btrfs_path *path, int level) 290174123bd7SChris Mason { 29020b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 29037bb86316SChris Mason u64 lower_gen; 29045f39d397SChris Mason struct extent_buffer *lower; 29055f39d397SChris Mason struct extent_buffer *c; 2906925baeddSChris Mason struct extent_buffer *old; 29075f39d397SChris Mason struct btrfs_disk_key lower_key; 2908d9d19a01SDavid Sterba int ret; 29095c680ed6SChris Mason 29105c680ed6SChris Mason BUG_ON(path->nodes[level]); 29115c680ed6SChris Mason BUG_ON(path->nodes[level-1] != root->node); 29125c680ed6SChris Mason 29137bb86316SChris Mason lower = path->nodes[level-1]; 29147bb86316SChris Mason if (level == 1) 29157bb86316SChris Mason btrfs_item_key(lower, &lower_key, 0); 29167bb86316SChris Mason else 29177bb86316SChris Mason btrfs_node_key(lower, &lower_key, 0); 29187bb86316SChris Mason 291979bd3712SFilipe Manana c = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 292079bd3712SFilipe Manana &lower_key, level, root->node->start, 0, 2921cf6f34aaSJosef Bacik BTRFS_NESTING_NEW_ROOT); 29225f39d397SChris Mason if (IS_ERR(c)) 29235f39d397SChris Mason return PTR_ERR(c); 2924925baeddSChris Mason 29250b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 2926f0486c68SYan, Zheng 29275f39d397SChris Mason btrfs_set_header_nritems(c, 1); 29285f39d397SChris Mason btrfs_set_node_key(c, &lower_key, 0); 2929db94535dSChris Mason btrfs_set_node_blockptr(c, 0, lower->start); 29307bb86316SChris Mason lower_gen = btrfs_header_generation(lower); 293131840ae1SZheng Yan WARN_ON(lower_gen != trans->transid); 29327bb86316SChris Mason 29337bb86316SChris Mason btrfs_set_node_ptr_generation(c, 0, lower_gen); 29345f39d397SChris Mason 29355f39d397SChris Mason btrfs_mark_buffer_dirty(c); 2936d5719762SChris Mason 2937925baeddSChris Mason old = root->node; 2938406808abSFilipe Manana ret = btrfs_tree_mod_log_insert_root(root->node, c, false); 2939d9d19a01SDavid Sterba BUG_ON(ret < 0); 2940240f62c8SChris Mason rcu_assign_pointer(root->node, c); 2941925baeddSChris Mason 2942925baeddSChris Mason /* the super has an extra ref to root->node */ 2943925baeddSChris Mason free_extent_buffer(old); 2944925baeddSChris Mason 29450b86a832SChris Mason add_root_to_dirty_list(root); 294667439dadSDavid Sterba atomic_inc(&c->refs); 29475f39d397SChris Mason path->nodes[level] = c; 2948ac5887c8SJosef Bacik path->locks[level] = BTRFS_WRITE_LOCK; 294974123bd7SChris Mason path->slots[level] = 0; 295074123bd7SChris Mason return 0; 295174123bd7SChris Mason } 29525c680ed6SChris Mason 29535c680ed6SChris Mason /* 29545c680ed6SChris Mason * worker function to insert a single pointer in a node. 29555c680ed6SChris Mason * the node should have enough room for the pointer already 295697571fd0SChris Mason * 29575c680ed6SChris Mason * slot and level indicate where you want the key to go, and 29585c680ed6SChris Mason * blocknr is the block the key points to. 29595c680ed6SChris Mason */ 2960143bede5SJeff Mahoney static void insert_ptr(struct btrfs_trans_handle *trans, 29616ad3cf6dSDavid Sterba struct btrfs_path *path, 2962143bede5SJeff Mahoney struct btrfs_disk_key *key, u64 bytenr, 2963c3e06965SJan Schmidt int slot, int level) 29645c680ed6SChris Mason { 29655f39d397SChris Mason struct extent_buffer *lower; 29665c680ed6SChris Mason int nritems; 2967f3ea38daSJan Schmidt int ret; 29685c680ed6SChris Mason 29695c680ed6SChris Mason BUG_ON(!path->nodes[level]); 297049d0c642SFilipe Manana btrfs_assert_tree_write_locked(path->nodes[level]); 29715f39d397SChris Mason lower = path->nodes[level]; 29725f39d397SChris Mason nritems = btrfs_header_nritems(lower); 2973c293498bSStoyan Gaydarov BUG_ON(slot > nritems); 29746ad3cf6dSDavid Sterba BUG_ON(nritems == BTRFS_NODEPTRS_PER_BLOCK(trans->fs_info)); 297574123bd7SChris Mason if (slot != nritems) { 2976bf1d3425SDavid Sterba if (level) { 2977f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_move(lower, slot + 1, 2978f3a84ccdSFilipe Manana slot, nritems - slot); 2979bf1d3425SDavid Sterba BUG_ON(ret < 0); 2980bf1d3425SDavid Sterba } 29815f39d397SChris Mason memmove_extent_buffer(lower, 2982e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(lower, slot + 1), 2983e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(lower, slot), 2984123abc88SChris Mason (nritems - slot) * sizeof(struct btrfs_key_ptr)); 298574123bd7SChris Mason } 2986c3e06965SJan Schmidt if (level) { 2987f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(lower, slot, 298833cff222SFilipe Manana BTRFS_MOD_LOG_KEY_ADD); 2989f3ea38daSJan Schmidt BUG_ON(ret < 0); 2990f3ea38daSJan Schmidt } 29915f39d397SChris Mason btrfs_set_node_key(lower, key, slot); 2992db94535dSChris Mason btrfs_set_node_blockptr(lower, slot, bytenr); 299374493f7aSChris Mason WARN_ON(trans->transid == 0); 299474493f7aSChris Mason btrfs_set_node_ptr_generation(lower, slot, trans->transid); 29955f39d397SChris Mason btrfs_set_header_nritems(lower, nritems + 1); 29965f39d397SChris Mason btrfs_mark_buffer_dirty(lower); 299774123bd7SChris Mason } 299874123bd7SChris Mason 299997571fd0SChris Mason /* 300097571fd0SChris Mason * split the node at the specified level in path in two. 300197571fd0SChris Mason * The path is corrected to point to the appropriate node after the split 300297571fd0SChris Mason * 300397571fd0SChris Mason * Before splitting this tries to make some room in the node by pushing 300497571fd0SChris Mason * left and right, if either one works, it returns right away. 3005aa5d6bedSChris Mason * 3006aa5d6bedSChris Mason * returns 0 on success and < 0 on failure 300797571fd0SChris Mason */ 3008e02119d5SChris Mason static noinline int split_node(struct btrfs_trans_handle *trans, 3009e02119d5SChris Mason struct btrfs_root *root, 3010e02119d5SChris Mason struct btrfs_path *path, int level) 3011be0e5c09SChris Mason { 30120b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 30135f39d397SChris Mason struct extent_buffer *c; 30145f39d397SChris Mason struct extent_buffer *split; 30155f39d397SChris Mason struct btrfs_disk_key disk_key; 3016be0e5c09SChris Mason int mid; 30175c680ed6SChris Mason int ret; 30187518a238SChris Mason u32 c_nritems; 3019be0e5c09SChris Mason 30205f39d397SChris Mason c = path->nodes[level]; 30217bb86316SChris Mason WARN_ON(btrfs_header_generation(c) != trans->transid); 30225f39d397SChris Mason if (c == root->node) { 3023d9abbf1cSJan Schmidt /* 302490f8d62eSJan Schmidt * trying to split the root, lets make a new one 302590f8d62eSJan Schmidt * 3026fdd99c72SLiu Bo * tree mod log: We don't log_removal old root in 302790f8d62eSJan Schmidt * insert_new_root, because that root buffer will be kept as a 302890f8d62eSJan Schmidt * normal node. We are going to log removal of half of the 3029f3a84ccdSFilipe Manana * elements below with btrfs_tree_mod_log_eb_copy(). We're 3030f3a84ccdSFilipe Manana * holding a tree lock on the buffer, which is why we cannot 3031f3a84ccdSFilipe Manana * race with other tree_mod_log users. 3032d9abbf1cSJan Schmidt */ 3033fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, level + 1); 30345c680ed6SChris Mason if (ret) 30355c680ed6SChris Mason return ret; 3036b3612421SChris Mason } else { 3037e66f709bSChris Mason ret = push_nodes_for_insert(trans, root, path, level); 30385f39d397SChris Mason c = path->nodes[level]; 30395f39d397SChris Mason if (!ret && btrfs_header_nritems(c) < 30400b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3) 3041e66f709bSChris Mason return 0; 304254aa1f4dSChris Mason if (ret < 0) 304354aa1f4dSChris Mason return ret; 30445c680ed6SChris Mason } 3045e66f709bSChris Mason 30465f39d397SChris Mason c_nritems = btrfs_header_nritems(c); 30475d4f98a2SYan Zheng mid = (c_nritems + 1) / 2; 30485d4f98a2SYan Zheng btrfs_node_key(c, &disk_key, mid); 30497bb86316SChris Mason 305079bd3712SFilipe Manana split = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 305179bd3712SFilipe Manana &disk_key, level, c->start, 0, 305279bd3712SFilipe Manana BTRFS_NESTING_SPLIT); 30535f39d397SChris Mason if (IS_ERR(split)) 30545f39d397SChris Mason return PTR_ERR(split); 305554aa1f4dSChris Mason 30560b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 3057bc877d28SNikolay Borisov ASSERT(btrfs_header_level(c) == level); 30585f39d397SChris Mason 3059f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_eb_copy(split, c, 0, mid, c_nritems - mid); 30605de865eeSFilipe David Borba Manana if (ret) { 3061ede600e4SFilipe Manana btrfs_tree_unlock(split); 3062ede600e4SFilipe Manana free_extent_buffer(split); 306366642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 30645de865eeSFilipe David Borba Manana return ret; 30655de865eeSFilipe David Borba Manana } 30665f39d397SChris Mason copy_extent_buffer(split, c, 3067e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(split, 0), 3068e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(c, mid), 3069123abc88SChris Mason (c_nritems - mid) * sizeof(struct btrfs_key_ptr)); 30705f39d397SChris Mason btrfs_set_header_nritems(split, c_nritems - mid); 30715f39d397SChris Mason btrfs_set_header_nritems(c, mid); 3072aa5d6bedSChris Mason 30735f39d397SChris Mason btrfs_mark_buffer_dirty(c); 30745f39d397SChris Mason btrfs_mark_buffer_dirty(split); 30755f39d397SChris Mason 30766ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, split->start, 3077c3e06965SJan Schmidt path->slots[level + 1] + 1, level + 1); 3078aa5d6bedSChris Mason 30795de08d7dSChris Mason if (path->slots[level] >= mid) { 30805c680ed6SChris Mason path->slots[level] -= mid; 3081925baeddSChris Mason btrfs_tree_unlock(c); 30825f39d397SChris Mason free_extent_buffer(c); 30835f39d397SChris Mason path->nodes[level] = split; 30845c680ed6SChris Mason path->slots[level + 1] += 1; 3085eb60ceacSChris Mason } else { 3086925baeddSChris Mason btrfs_tree_unlock(split); 30875f39d397SChris Mason free_extent_buffer(split); 3088be0e5c09SChris Mason } 3089d5286a92SNikolay Borisov return 0; 3090be0e5c09SChris Mason } 3091be0e5c09SChris Mason 309274123bd7SChris Mason /* 309374123bd7SChris Mason * how many bytes are required to store the items in a leaf. start 309474123bd7SChris Mason * and nr indicate which items in the leaf to check. This totals up the 309574123bd7SChris Mason * space used both by the item structs and the item data 309674123bd7SChris Mason */ 30976c75a589SQu Wenruo static int leaf_space_used(const struct extent_buffer *l, int start, int nr) 3098be0e5c09SChris Mason { 3099be0e5c09SChris Mason int data_len; 31005f39d397SChris Mason int nritems = btrfs_header_nritems(l); 3101d4dbff95SChris Mason int end = min(nritems, start + nr) - 1; 3102be0e5c09SChris Mason 3103be0e5c09SChris Mason if (!nr) 3104be0e5c09SChris Mason return 0; 31053212fa14SJosef Bacik data_len = btrfs_item_offset(l, start) + btrfs_item_size(l, start); 31063212fa14SJosef Bacik data_len = data_len - btrfs_item_offset(l, end); 31070783fcfcSChris Mason data_len += sizeof(struct btrfs_item) * nr; 3108d4dbff95SChris Mason WARN_ON(data_len < 0); 3109be0e5c09SChris Mason return data_len; 3110be0e5c09SChris Mason } 3111be0e5c09SChris Mason 311274123bd7SChris Mason /* 3113d4dbff95SChris Mason * The space between the end of the leaf items and 3114d4dbff95SChris Mason * the start of the leaf data. IOW, how much room 3115d4dbff95SChris Mason * the leaf has left for both items and data 3116d4dbff95SChris Mason */ 31176c75a589SQu Wenruo int btrfs_leaf_free_space(const struct extent_buffer *leaf) 3118d4dbff95SChris Mason { 3119e902baacSDavid Sterba struct btrfs_fs_info *fs_info = leaf->fs_info; 31205f39d397SChris Mason int nritems = btrfs_header_nritems(leaf); 31215f39d397SChris Mason int ret; 31220b246afaSJeff Mahoney 31230b246afaSJeff Mahoney ret = BTRFS_LEAF_DATA_SIZE(fs_info) - leaf_space_used(leaf, 0, nritems); 31245f39d397SChris Mason if (ret < 0) { 31250b246afaSJeff Mahoney btrfs_crit(fs_info, 3126efe120a0SFrank Holton "leaf free space ret %d, leaf data size %lu, used %d nritems %d", 3127da17066cSJeff Mahoney ret, 31280b246afaSJeff Mahoney (unsigned long) BTRFS_LEAF_DATA_SIZE(fs_info), 31295f39d397SChris Mason leaf_space_used(leaf, 0, nritems), nritems); 31305f39d397SChris Mason } 31315f39d397SChris Mason return ret; 3132d4dbff95SChris Mason } 3133d4dbff95SChris Mason 313499d8f83cSChris Mason /* 313599d8f83cSChris Mason * min slot controls the lowest index we're willing to push to the 313699d8f83cSChris Mason * right. We'll push up to and including min_slot, but no lower 313799d8f83cSChris Mason */ 3138ed25dab3SJosef Bacik static noinline int __push_leaf_right(struct btrfs_trans_handle *trans, 3139ed25dab3SJosef Bacik struct btrfs_path *path, 314044871b1bSChris Mason int data_size, int empty, 314144871b1bSChris Mason struct extent_buffer *right, 314299d8f83cSChris Mason int free_space, u32 left_nritems, 314399d8f83cSChris Mason u32 min_slot) 314400ec4c51SChris Mason { 3145f72f0010SDavid Sterba struct btrfs_fs_info *fs_info = right->fs_info; 31465f39d397SChris Mason struct extent_buffer *left = path->nodes[0]; 314744871b1bSChris Mason struct extent_buffer *upper = path->nodes[1]; 3148cfed81a0SChris Mason struct btrfs_map_token token; 31495f39d397SChris Mason struct btrfs_disk_key disk_key; 315000ec4c51SChris Mason int slot; 315134a38218SChris Mason u32 i; 315200ec4c51SChris Mason int push_space = 0; 315300ec4c51SChris Mason int push_items = 0; 315434a38218SChris Mason u32 nr; 31557518a238SChris Mason u32 right_nritems; 31565f39d397SChris Mason u32 data_end; 3157db94535dSChris Mason u32 this_item_size; 315800ec4c51SChris Mason 315934a38218SChris Mason if (empty) 316034a38218SChris Mason nr = 0; 316134a38218SChris Mason else 316299d8f83cSChris Mason nr = max_t(u32, 1, min_slot); 316334a38218SChris Mason 316431840ae1SZheng Yan if (path->slots[0] >= left_nritems) 316587b29b20SYan Zheng push_space += data_size; 316631840ae1SZheng Yan 316744871b1bSChris Mason slot = path->slots[1]; 316834a38218SChris Mason i = left_nritems - 1; 316934a38218SChris Mason while (i >= nr) { 317031840ae1SZheng Yan if (!empty && push_items > 0) { 317131840ae1SZheng Yan if (path->slots[0] > i) 317231840ae1SZheng Yan break; 317331840ae1SZheng Yan if (path->slots[0] == i) { 3174e902baacSDavid Sterba int space = btrfs_leaf_free_space(left); 3175e902baacSDavid Sterba 317631840ae1SZheng Yan if (space + push_space * 2 > free_space) 317731840ae1SZheng Yan break; 317831840ae1SZheng Yan } 317931840ae1SZheng Yan } 318031840ae1SZheng Yan 318100ec4c51SChris Mason if (path->slots[0] == i) 318287b29b20SYan Zheng push_space += data_size; 3183db94535dSChris Mason 31843212fa14SJosef Bacik this_item_size = btrfs_item_size(left, i); 318574794207SJosef Bacik if (this_item_size + sizeof(struct btrfs_item) + 318674794207SJosef Bacik push_space > free_space) 318700ec4c51SChris Mason break; 318831840ae1SZheng Yan 318900ec4c51SChris Mason push_items++; 319074794207SJosef Bacik push_space += this_item_size + sizeof(struct btrfs_item); 319134a38218SChris Mason if (i == 0) 319234a38218SChris Mason break; 319334a38218SChris Mason i--; 3194db94535dSChris Mason } 31955f39d397SChris Mason 3196925baeddSChris Mason if (push_items == 0) 3197925baeddSChris Mason goto out_unlock; 31985f39d397SChris Mason 31996c1500f2SJulia Lawall WARN_ON(!empty && push_items == left_nritems); 32005f39d397SChris Mason 320100ec4c51SChris Mason /* push left to right */ 32025f39d397SChris Mason right_nritems = btrfs_header_nritems(right); 320334a38218SChris Mason 3204dc2e724eSJosef Bacik push_space = btrfs_item_data_end(left, left_nritems - push_items); 32058f881e8cSDavid Sterba push_space -= leaf_data_end(left); 32065f39d397SChris Mason 320700ec4c51SChris Mason /* make room in the right data area */ 32088f881e8cSDavid Sterba data_end = leaf_data_end(right); 3209637e3b48SJosef Bacik memmove_leaf_data(right, data_end - push_space, data_end, 32100b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - data_end); 32115f39d397SChris Mason 321200ec4c51SChris Mason /* copy from the left data area */ 3213637e3b48SJosef Bacik copy_leaf_data(right, left, BTRFS_LEAF_DATA_SIZE(fs_info) - push_space, 3214637e3b48SJosef Bacik leaf_data_end(left), push_space); 32155f39d397SChris Mason 3216637e3b48SJosef Bacik memmove_leaf_items(right, push_items, 0, right_nritems); 32175f39d397SChris Mason 321800ec4c51SChris Mason /* copy the items from left to right */ 3219637e3b48SJosef Bacik copy_leaf_items(right, left, 0, left_nritems - push_items, push_items); 322000ec4c51SChris Mason 322100ec4c51SChris Mason /* update the item pointers */ 3222c82f823cSDavid Sterba btrfs_init_map_token(&token, right); 32237518a238SChris Mason right_nritems += push_items; 32245f39d397SChris Mason btrfs_set_header_nritems(right, right_nritems); 32250b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info); 32267518a238SChris Mason for (i = 0; i < right_nritems; i++) { 32273212fa14SJosef Bacik push_space -= btrfs_token_item_size(&token, i); 32283212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, push_space); 3229db94535dSChris Mason } 3230db94535dSChris Mason 32317518a238SChris Mason left_nritems -= push_items; 32325f39d397SChris Mason btrfs_set_header_nritems(left, left_nritems); 323300ec4c51SChris Mason 323434a38218SChris Mason if (left_nritems) 32355f39d397SChris Mason btrfs_mark_buffer_dirty(left); 3236f0486c68SYan, Zheng else 3237190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, left); 3238f0486c68SYan, Zheng 32395f39d397SChris Mason btrfs_mark_buffer_dirty(right); 3240a429e513SChris Mason 32415f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 32425f39d397SChris Mason btrfs_set_node_key(upper, &disk_key, slot + 1); 3243d6025579SChris Mason btrfs_mark_buffer_dirty(upper); 324402217ed2SChris Mason 324500ec4c51SChris Mason /* then fixup the leaf pointer in the path */ 32467518a238SChris Mason if (path->slots[0] >= left_nritems) { 32477518a238SChris Mason path->slots[0] -= left_nritems; 3248925baeddSChris Mason if (btrfs_header_nritems(path->nodes[0]) == 0) 3249190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, path->nodes[0]); 3250925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 32515f39d397SChris Mason free_extent_buffer(path->nodes[0]); 32525f39d397SChris Mason path->nodes[0] = right; 325300ec4c51SChris Mason path->slots[1] += 1; 325400ec4c51SChris Mason } else { 3255925baeddSChris Mason btrfs_tree_unlock(right); 32565f39d397SChris Mason free_extent_buffer(right); 325700ec4c51SChris Mason } 325800ec4c51SChris Mason return 0; 3259925baeddSChris Mason 3260925baeddSChris Mason out_unlock: 3261925baeddSChris Mason btrfs_tree_unlock(right); 3262925baeddSChris Mason free_extent_buffer(right); 3263925baeddSChris Mason return 1; 326400ec4c51SChris Mason } 3265925baeddSChris Mason 326600ec4c51SChris Mason /* 326744871b1bSChris Mason * push some data in the path leaf to the right, trying to free up at 326874123bd7SChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 326944871b1bSChris Mason * 327044871b1bSChris Mason * returns 1 if the push failed because the other node didn't have enough 327144871b1bSChris Mason * room, 0 if everything worked out and < 0 if there were major errors. 327299d8f83cSChris Mason * 327399d8f83cSChris Mason * this will push starting from min_slot to the end of the leaf. It won't 327499d8f83cSChris Mason * push any slot lower than min_slot 327574123bd7SChris Mason */ 327644871b1bSChris Mason static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root 327799d8f83cSChris Mason *root, struct btrfs_path *path, 327899d8f83cSChris Mason int min_data_size, int data_size, 327999d8f83cSChris Mason int empty, u32 min_slot) 3280be0e5c09SChris Mason { 328144871b1bSChris Mason struct extent_buffer *left = path->nodes[0]; 328244871b1bSChris Mason struct extent_buffer *right; 328344871b1bSChris Mason struct extent_buffer *upper; 328444871b1bSChris Mason int slot; 328544871b1bSChris Mason int free_space; 328644871b1bSChris Mason u32 left_nritems; 328744871b1bSChris Mason int ret; 328844871b1bSChris Mason 328944871b1bSChris Mason if (!path->nodes[1]) 329044871b1bSChris Mason return 1; 329144871b1bSChris Mason 329244871b1bSChris Mason slot = path->slots[1]; 329344871b1bSChris Mason upper = path->nodes[1]; 329444871b1bSChris Mason if (slot >= btrfs_header_nritems(upper) - 1) 329544871b1bSChris Mason return 1; 329644871b1bSChris Mason 329749d0c642SFilipe Manana btrfs_assert_tree_write_locked(path->nodes[1]); 329844871b1bSChris Mason 32994b231ae4SDavid Sterba right = btrfs_read_node_slot(upper, slot + 1); 3300fb770ae4SLiu Bo if (IS_ERR(right)) 33019cf14029SJosef Bacik return PTR_ERR(right); 330291ca338dSTsutomu Itoh 3303bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT); 330444871b1bSChris Mason 3305e902baacSDavid Sterba free_space = btrfs_leaf_free_space(right); 330644871b1bSChris Mason if (free_space < data_size) 330744871b1bSChris Mason goto out_unlock; 330844871b1bSChris Mason 330944871b1bSChris Mason ret = btrfs_cow_block(trans, root, right, upper, 3310bf59a5a2SJosef Bacik slot + 1, &right, BTRFS_NESTING_RIGHT_COW); 331144871b1bSChris Mason if (ret) 331244871b1bSChris Mason goto out_unlock; 331344871b1bSChris Mason 331444871b1bSChris Mason left_nritems = btrfs_header_nritems(left); 331544871b1bSChris Mason if (left_nritems == 0) 331644871b1bSChris Mason goto out_unlock; 331744871b1bSChris Mason 3318d16c702fSQu Wenruo if (check_sibling_keys(left, right)) { 3319d16c702fSQu Wenruo ret = -EUCLEAN; 33209ae5afd0SFilipe Manana btrfs_abort_transaction(trans, ret); 3321d16c702fSQu Wenruo btrfs_tree_unlock(right); 3322d16c702fSQu Wenruo free_extent_buffer(right); 3323d16c702fSQu Wenruo return ret; 3324d16c702fSQu Wenruo } 33252ef1fed2SFilipe David Borba Manana if (path->slots[0] == left_nritems && !empty) { 33262ef1fed2SFilipe David Borba Manana /* Key greater than all keys in the leaf, right neighbor has 33272ef1fed2SFilipe David Borba Manana * enough room for it and we're not emptying our leaf to delete 33282ef1fed2SFilipe David Borba Manana * it, therefore use right neighbor to insert the new item and 332952042d8eSAndrea Gelmini * no need to touch/dirty our left leaf. */ 33302ef1fed2SFilipe David Borba Manana btrfs_tree_unlock(left); 33312ef1fed2SFilipe David Borba Manana free_extent_buffer(left); 33322ef1fed2SFilipe David Borba Manana path->nodes[0] = right; 33332ef1fed2SFilipe David Borba Manana path->slots[0] = 0; 33342ef1fed2SFilipe David Borba Manana path->slots[1]++; 33352ef1fed2SFilipe David Borba Manana return 0; 33362ef1fed2SFilipe David Borba Manana } 33372ef1fed2SFilipe David Borba Manana 3338ed25dab3SJosef Bacik return __push_leaf_right(trans, path, min_data_size, empty, right, 3339ed25dab3SJosef Bacik free_space, left_nritems, min_slot); 334044871b1bSChris Mason out_unlock: 334144871b1bSChris Mason btrfs_tree_unlock(right); 334244871b1bSChris Mason free_extent_buffer(right); 334344871b1bSChris Mason return 1; 334444871b1bSChris Mason } 334544871b1bSChris Mason 334644871b1bSChris Mason /* 334744871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 334844871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 334999d8f83cSChris Mason * 335099d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 335199d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us do all the 335299d8f83cSChris Mason * items 335344871b1bSChris Mason */ 3354ed25dab3SJosef Bacik static noinline int __push_leaf_left(struct btrfs_trans_handle *trans, 3355ed25dab3SJosef Bacik struct btrfs_path *path, int data_size, 335644871b1bSChris Mason int empty, struct extent_buffer *left, 335799d8f83cSChris Mason int free_space, u32 right_nritems, 335899d8f83cSChris Mason u32 max_slot) 335944871b1bSChris Mason { 33608087c193SDavid Sterba struct btrfs_fs_info *fs_info = left->fs_info; 33615f39d397SChris Mason struct btrfs_disk_key disk_key; 33625f39d397SChris Mason struct extent_buffer *right = path->nodes[0]; 3363be0e5c09SChris Mason int i; 3364be0e5c09SChris Mason int push_space = 0; 3365be0e5c09SChris Mason int push_items = 0; 33667518a238SChris Mason u32 old_left_nritems; 336734a38218SChris Mason u32 nr; 3368aa5d6bedSChris Mason int ret = 0; 3369db94535dSChris Mason u32 this_item_size; 3370db94535dSChris Mason u32 old_left_item_size; 3371cfed81a0SChris Mason struct btrfs_map_token token; 3372cfed81a0SChris Mason 337334a38218SChris Mason if (empty) 337499d8f83cSChris Mason nr = min(right_nritems, max_slot); 337534a38218SChris Mason else 337699d8f83cSChris Mason nr = min(right_nritems - 1, max_slot); 337734a38218SChris Mason 337834a38218SChris Mason for (i = 0; i < nr; i++) { 337931840ae1SZheng Yan if (!empty && push_items > 0) { 338031840ae1SZheng Yan if (path->slots[0] < i) 338131840ae1SZheng Yan break; 338231840ae1SZheng Yan if (path->slots[0] == i) { 3383e902baacSDavid Sterba int space = btrfs_leaf_free_space(right); 3384e902baacSDavid Sterba 338531840ae1SZheng Yan if (space + push_space * 2 > free_space) 338631840ae1SZheng Yan break; 338731840ae1SZheng Yan } 338831840ae1SZheng Yan } 338931840ae1SZheng Yan 3390be0e5c09SChris Mason if (path->slots[0] == i) 339187b29b20SYan Zheng push_space += data_size; 3392db94535dSChris Mason 33933212fa14SJosef Bacik this_item_size = btrfs_item_size(right, i); 339474794207SJosef Bacik if (this_item_size + sizeof(struct btrfs_item) + push_space > 339574794207SJosef Bacik free_space) 3396be0e5c09SChris Mason break; 3397db94535dSChris Mason 3398be0e5c09SChris Mason push_items++; 339974794207SJosef Bacik push_space += this_item_size + sizeof(struct btrfs_item); 3400be0e5c09SChris Mason } 3401db94535dSChris Mason 3402be0e5c09SChris Mason if (push_items == 0) { 3403925baeddSChris Mason ret = 1; 3404925baeddSChris Mason goto out; 3405be0e5c09SChris Mason } 3406fae7f21cSDulshani Gunawardhana WARN_ON(!empty && push_items == btrfs_header_nritems(right)); 34075f39d397SChris Mason 3408be0e5c09SChris Mason /* push data from right to left */ 3409637e3b48SJosef Bacik copy_leaf_items(left, right, btrfs_header_nritems(left), 0, push_items); 34105f39d397SChris Mason 34110b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info) - 34123212fa14SJosef Bacik btrfs_item_offset(right, push_items - 1); 34135f39d397SChris Mason 3414637e3b48SJosef Bacik copy_leaf_data(left, right, leaf_data_end(left) - push_space, 3415637e3b48SJosef Bacik btrfs_item_offset(right, push_items - 1), push_space); 34165f39d397SChris Mason old_left_nritems = btrfs_header_nritems(left); 341787b29b20SYan Zheng BUG_ON(old_left_nritems <= 0); 3418eb60ceacSChris Mason 3419c82f823cSDavid Sterba btrfs_init_map_token(&token, left); 34203212fa14SJosef Bacik old_left_item_size = btrfs_item_offset(left, old_left_nritems - 1); 3421be0e5c09SChris Mason for (i = old_left_nritems; i < old_left_nritems + push_items; i++) { 34225f39d397SChris Mason u32 ioff; 3423db94535dSChris Mason 34243212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 34253212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, 3426cc4c13d5SDavid Sterba ioff - (BTRFS_LEAF_DATA_SIZE(fs_info) - old_left_item_size)); 3427be0e5c09SChris Mason } 34285f39d397SChris Mason btrfs_set_header_nritems(left, old_left_nritems + push_items); 3429be0e5c09SChris Mason 3430be0e5c09SChris Mason /* fixup right node */ 343131b1a2bdSJulia Lawall if (push_items > right_nritems) 343231b1a2bdSJulia Lawall WARN(1, KERN_CRIT "push items %d nr %u\n", push_items, 3433d397712bSChris Mason right_nritems); 343434a38218SChris Mason 343534a38218SChris Mason if (push_items < right_nritems) { 34363212fa14SJosef Bacik push_space = btrfs_item_offset(right, push_items - 1) - 34378f881e8cSDavid Sterba leaf_data_end(right); 3438637e3b48SJosef Bacik memmove_leaf_data(right, 34390b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - push_space, 34408f881e8cSDavid Sterba leaf_data_end(right), push_space); 34415f39d397SChris Mason 3442637e3b48SJosef Bacik memmove_leaf_items(right, 0, push_items, 3443637e3b48SJosef Bacik btrfs_header_nritems(right) - push_items); 344434a38218SChris Mason } 3445c82f823cSDavid Sterba 3446c82f823cSDavid Sterba btrfs_init_map_token(&token, right); 3447eef1c494SYan right_nritems -= push_items; 3448eef1c494SYan btrfs_set_header_nritems(right, right_nritems); 34490b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info); 34505f39d397SChris Mason for (i = 0; i < right_nritems; i++) { 34513212fa14SJosef Bacik push_space = push_space - btrfs_token_item_size(&token, i); 34523212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, push_space); 3453db94535dSChris Mason } 3454eb60ceacSChris Mason 34555f39d397SChris Mason btrfs_mark_buffer_dirty(left); 345634a38218SChris Mason if (right_nritems) 34575f39d397SChris Mason btrfs_mark_buffer_dirty(right); 3458f0486c68SYan, Zheng else 3459190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, right); 3460098f59c2SChris Mason 34615f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 3462b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 3463be0e5c09SChris Mason 3464be0e5c09SChris Mason /* then fixup the leaf pointer in the path */ 3465be0e5c09SChris Mason if (path->slots[0] < push_items) { 3466be0e5c09SChris Mason path->slots[0] += old_left_nritems; 3467925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 34685f39d397SChris Mason free_extent_buffer(path->nodes[0]); 34695f39d397SChris Mason path->nodes[0] = left; 3470be0e5c09SChris Mason path->slots[1] -= 1; 3471be0e5c09SChris Mason } else { 3472925baeddSChris Mason btrfs_tree_unlock(left); 34735f39d397SChris Mason free_extent_buffer(left); 3474be0e5c09SChris Mason path->slots[0] -= push_items; 3475be0e5c09SChris Mason } 3476eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 3477aa5d6bedSChris Mason return ret; 3478925baeddSChris Mason out: 3479925baeddSChris Mason btrfs_tree_unlock(left); 3480925baeddSChris Mason free_extent_buffer(left); 3481925baeddSChris Mason return ret; 3482be0e5c09SChris Mason } 3483be0e5c09SChris Mason 348474123bd7SChris Mason /* 348544871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 348644871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 348799d8f83cSChris Mason * 348899d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 348999d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us push all the 349099d8f83cSChris Mason * items 349144871b1bSChris Mason */ 349244871b1bSChris Mason static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root 349399d8f83cSChris Mason *root, struct btrfs_path *path, int min_data_size, 349499d8f83cSChris Mason int data_size, int empty, u32 max_slot) 349544871b1bSChris Mason { 349644871b1bSChris Mason struct extent_buffer *right = path->nodes[0]; 349744871b1bSChris Mason struct extent_buffer *left; 349844871b1bSChris Mason int slot; 349944871b1bSChris Mason int free_space; 350044871b1bSChris Mason u32 right_nritems; 350144871b1bSChris Mason int ret = 0; 350244871b1bSChris Mason 350344871b1bSChris Mason slot = path->slots[1]; 350444871b1bSChris Mason if (slot == 0) 350544871b1bSChris Mason return 1; 350644871b1bSChris Mason if (!path->nodes[1]) 350744871b1bSChris Mason return 1; 350844871b1bSChris Mason 350944871b1bSChris Mason right_nritems = btrfs_header_nritems(right); 351044871b1bSChris Mason if (right_nritems == 0) 351144871b1bSChris Mason return 1; 351244871b1bSChris Mason 351349d0c642SFilipe Manana btrfs_assert_tree_write_locked(path->nodes[1]); 351444871b1bSChris Mason 35154b231ae4SDavid Sterba left = btrfs_read_node_slot(path->nodes[1], slot - 1); 3516fb770ae4SLiu Bo if (IS_ERR(left)) 35179cf14029SJosef Bacik return PTR_ERR(left); 351891ca338dSTsutomu Itoh 3519bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT); 352044871b1bSChris Mason 3521e902baacSDavid Sterba free_space = btrfs_leaf_free_space(left); 352244871b1bSChris Mason if (free_space < data_size) { 352344871b1bSChris Mason ret = 1; 352444871b1bSChris Mason goto out; 352544871b1bSChris Mason } 352644871b1bSChris Mason 352744871b1bSChris Mason ret = btrfs_cow_block(trans, root, left, 35289631e4ccSJosef Bacik path->nodes[1], slot - 1, &left, 3529bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW); 353044871b1bSChris Mason if (ret) { 353144871b1bSChris Mason /* we hit -ENOSPC, but it isn't fatal here */ 353279787eaaSJeff Mahoney if (ret == -ENOSPC) 353344871b1bSChris Mason ret = 1; 353444871b1bSChris Mason goto out; 353544871b1bSChris Mason } 353644871b1bSChris Mason 3537d16c702fSQu Wenruo if (check_sibling_keys(left, right)) { 3538d16c702fSQu Wenruo ret = -EUCLEAN; 35399ae5afd0SFilipe Manana btrfs_abort_transaction(trans, ret); 3540d16c702fSQu Wenruo goto out; 3541d16c702fSQu Wenruo } 3542ed25dab3SJosef Bacik return __push_leaf_left(trans, path, min_data_size, empty, left, 3543ed25dab3SJosef Bacik free_space, right_nritems, max_slot); 354444871b1bSChris Mason out: 354544871b1bSChris Mason btrfs_tree_unlock(left); 354644871b1bSChris Mason free_extent_buffer(left); 354744871b1bSChris Mason return ret; 354844871b1bSChris Mason } 354944871b1bSChris Mason 355044871b1bSChris Mason /* 355174123bd7SChris Mason * split the path's leaf in two, making sure there is at least data_size 355274123bd7SChris Mason * available for the resulting leaf level of the path. 355374123bd7SChris Mason */ 3554143bede5SJeff Mahoney static noinline void copy_for_split(struct btrfs_trans_handle *trans, 355544871b1bSChris Mason struct btrfs_path *path, 355644871b1bSChris Mason struct extent_buffer *l, 355744871b1bSChris Mason struct extent_buffer *right, 355844871b1bSChris Mason int slot, int mid, int nritems) 3559be0e5c09SChris Mason { 356094f94ad9SDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info; 3561be0e5c09SChris Mason int data_copy_size; 3562be0e5c09SChris Mason int rt_data_off; 3563be0e5c09SChris Mason int i; 3564d4dbff95SChris Mason struct btrfs_disk_key disk_key; 3565cfed81a0SChris Mason struct btrfs_map_token token; 3566cfed81a0SChris Mason 35675f39d397SChris Mason nritems = nritems - mid; 35685f39d397SChris Mason btrfs_set_header_nritems(right, nritems); 3569dc2e724eSJosef Bacik data_copy_size = btrfs_item_data_end(l, mid) - leaf_data_end(l); 35705f39d397SChris Mason 3571637e3b48SJosef Bacik copy_leaf_items(right, l, 0, mid, nritems); 35725f39d397SChris Mason 3573637e3b48SJosef Bacik copy_leaf_data(right, l, BTRFS_LEAF_DATA_SIZE(fs_info) - data_copy_size, 35748f881e8cSDavid Sterba leaf_data_end(l), data_copy_size); 357574123bd7SChris Mason 3576dc2e724eSJosef Bacik rt_data_off = BTRFS_LEAF_DATA_SIZE(fs_info) - btrfs_item_data_end(l, mid); 35775f39d397SChris Mason 3578c82f823cSDavid Sterba btrfs_init_map_token(&token, right); 35795f39d397SChris Mason for (i = 0; i < nritems; i++) { 3580db94535dSChris Mason u32 ioff; 3581db94535dSChris Mason 35823212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 35833212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff + rt_data_off); 35840783fcfcSChris Mason } 358574123bd7SChris Mason 35865f39d397SChris Mason btrfs_set_header_nritems(l, mid); 35875f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 35886ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, right->start, path->slots[1] + 1, 1); 35895f39d397SChris Mason 35905f39d397SChris Mason btrfs_mark_buffer_dirty(right); 35915f39d397SChris Mason btrfs_mark_buffer_dirty(l); 3592eb60ceacSChris Mason BUG_ON(path->slots[0] != slot); 35935f39d397SChris Mason 3594be0e5c09SChris Mason if (mid <= slot) { 3595925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 35965f39d397SChris Mason free_extent_buffer(path->nodes[0]); 35975f39d397SChris Mason path->nodes[0] = right; 3598be0e5c09SChris Mason path->slots[0] -= mid; 3599be0e5c09SChris Mason path->slots[1] += 1; 3600925baeddSChris Mason } else { 3601925baeddSChris Mason btrfs_tree_unlock(right); 36025f39d397SChris Mason free_extent_buffer(right); 3603925baeddSChris Mason } 36045f39d397SChris Mason 3605eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 360644871b1bSChris Mason } 360744871b1bSChris Mason 360844871b1bSChris Mason /* 360999d8f83cSChris Mason * double splits happen when we need to insert a big item in the middle 361099d8f83cSChris Mason * of a leaf. A double split can leave us with 3 mostly empty leaves: 361199d8f83cSChris Mason * leaf: [ slots 0 - N] [ our target ] [ N + 1 - total in leaf ] 361299d8f83cSChris Mason * A B C 361399d8f83cSChris Mason * 361499d8f83cSChris Mason * We avoid this by trying to push the items on either side of our target 361599d8f83cSChris Mason * into the adjacent leaves. If all goes well we can avoid the double split 361699d8f83cSChris Mason * completely. 361799d8f83cSChris Mason */ 361899d8f83cSChris Mason static noinline int push_for_double_split(struct btrfs_trans_handle *trans, 361999d8f83cSChris Mason struct btrfs_root *root, 362099d8f83cSChris Mason struct btrfs_path *path, 362199d8f83cSChris Mason int data_size) 362299d8f83cSChris Mason { 362399d8f83cSChris Mason int ret; 362499d8f83cSChris Mason int progress = 0; 362599d8f83cSChris Mason int slot; 362699d8f83cSChris Mason u32 nritems; 36275a4267caSFilipe David Borba Manana int space_needed = data_size; 362899d8f83cSChris Mason 362999d8f83cSChris Mason slot = path->slots[0]; 36305a4267caSFilipe David Borba Manana if (slot < btrfs_header_nritems(path->nodes[0])) 3631e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(path->nodes[0]); 363299d8f83cSChris Mason 363399d8f83cSChris Mason /* 363499d8f83cSChris Mason * try to push all the items after our slot into the 363599d8f83cSChris Mason * right leaf 363699d8f83cSChris Mason */ 36375a4267caSFilipe David Borba Manana ret = push_leaf_right(trans, root, path, 1, space_needed, 0, slot); 363899d8f83cSChris Mason if (ret < 0) 363999d8f83cSChris Mason return ret; 364099d8f83cSChris Mason 364199d8f83cSChris Mason if (ret == 0) 364299d8f83cSChris Mason progress++; 364399d8f83cSChris Mason 364499d8f83cSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 364599d8f83cSChris Mason /* 364699d8f83cSChris Mason * our goal is to get our slot at the start or end of a leaf. If 364799d8f83cSChris Mason * we've done so we're done 364899d8f83cSChris Mason */ 364999d8f83cSChris Mason if (path->slots[0] == 0 || path->slots[0] == nritems) 365099d8f83cSChris Mason return 0; 365199d8f83cSChris Mason 3652e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= data_size) 365399d8f83cSChris Mason return 0; 365499d8f83cSChris Mason 365599d8f83cSChris Mason /* try to push all the items before our slot into the next leaf */ 365699d8f83cSChris Mason slot = path->slots[0]; 3657263d3995SFilipe Manana space_needed = data_size; 3658263d3995SFilipe Manana if (slot > 0) 3659e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(path->nodes[0]); 36605a4267caSFilipe David Borba Manana ret = push_leaf_left(trans, root, path, 1, space_needed, 0, slot); 366199d8f83cSChris Mason if (ret < 0) 366299d8f83cSChris Mason return ret; 366399d8f83cSChris Mason 366499d8f83cSChris Mason if (ret == 0) 366599d8f83cSChris Mason progress++; 366699d8f83cSChris Mason 366799d8f83cSChris Mason if (progress) 366899d8f83cSChris Mason return 0; 366999d8f83cSChris Mason return 1; 367099d8f83cSChris Mason } 367199d8f83cSChris Mason 367299d8f83cSChris Mason /* 367344871b1bSChris Mason * split the path's leaf in two, making sure there is at least data_size 367444871b1bSChris Mason * available for the resulting leaf level of the path. 367544871b1bSChris Mason * 367644871b1bSChris Mason * returns 0 if all went well and < 0 on failure. 367744871b1bSChris Mason */ 367844871b1bSChris Mason static noinline int split_leaf(struct btrfs_trans_handle *trans, 367944871b1bSChris Mason struct btrfs_root *root, 3680310712b2SOmar Sandoval const struct btrfs_key *ins_key, 368144871b1bSChris Mason struct btrfs_path *path, int data_size, 368244871b1bSChris Mason int extend) 368344871b1bSChris Mason { 36845d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 368544871b1bSChris Mason struct extent_buffer *l; 368644871b1bSChris Mason u32 nritems; 368744871b1bSChris Mason int mid; 368844871b1bSChris Mason int slot; 368944871b1bSChris Mason struct extent_buffer *right; 3690b7a0365eSDaniel Dressler struct btrfs_fs_info *fs_info = root->fs_info; 369144871b1bSChris Mason int ret = 0; 369244871b1bSChris Mason int wret; 36935d4f98a2SYan Zheng int split; 369444871b1bSChris Mason int num_doubles = 0; 369599d8f83cSChris Mason int tried_avoid_double = 0; 369644871b1bSChris Mason 3697a5719521SYan, Zheng l = path->nodes[0]; 3698a5719521SYan, Zheng slot = path->slots[0]; 36993212fa14SJosef Bacik if (extend && data_size + btrfs_item_size(l, slot) + 37000b246afaSJeff Mahoney sizeof(struct btrfs_item) > BTRFS_LEAF_DATA_SIZE(fs_info)) 3701a5719521SYan, Zheng return -EOVERFLOW; 3702a5719521SYan, Zheng 370344871b1bSChris Mason /* first try to make some room by pushing left and right */ 370433157e05SLiu Bo if (data_size && path->nodes[1]) { 37055a4267caSFilipe David Borba Manana int space_needed = data_size; 37065a4267caSFilipe David Borba Manana 37075a4267caSFilipe David Borba Manana if (slot < btrfs_header_nritems(l)) 3708e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(l); 37095a4267caSFilipe David Borba Manana 37105a4267caSFilipe David Borba Manana wret = push_leaf_right(trans, root, path, space_needed, 37115a4267caSFilipe David Borba Manana space_needed, 0, 0); 371244871b1bSChris Mason if (wret < 0) 371344871b1bSChris Mason return wret; 371444871b1bSChris Mason if (wret) { 3715263d3995SFilipe Manana space_needed = data_size; 3716263d3995SFilipe Manana if (slot > 0) 3717e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(l); 37185a4267caSFilipe David Borba Manana wret = push_leaf_left(trans, root, path, space_needed, 37195a4267caSFilipe David Borba Manana space_needed, 0, (u32)-1); 372044871b1bSChris Mason if (wret < 0) 372144871b1bSChris Mason return wret; 372244871b1bSChris Mason } 372344871b1bSChris Mason l = path->nodes[0]; 372444871b1bSChris Mason 372544871b1bSChris Mason /* did the pushes work? */ 3726e902baacSDavid Sterba if (btrfs_leaf_free_space(l) >= data_size) 372744871b1bSChris Mason return 0; 372844871b1bSChris Mason } 372944871b1bSChris Mason 373044871b1bSChris Mason if (!path->nodes[1]) { 3731fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, 1); 373244871b1bSChris Mason if (ret) 373344871b1bSChris Mason return ret; 373444871b1bSChris Mason } 373544871b1bSChris Mason again: 37365d4f98a2SYan Zheng split = 1; 373744871b1bSChris Mason l = path->nodes[0]; 373844871b1bSChris Mason slot = path->slots[0]; 373944871b1bSChris Mason nritems = btrfs_header_nritems(l); 374044871b1bSChris Mason mid = (nritems + 1) / 2; 374144871b1bSChris Mason 37425d4f98a2SYan Zheng if (mid <= slot) { 37435d4f98a2SYan Zheng if (nritems == 1 || 37445d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + data_size > 37450b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info)) { 37465d4f98a2SYan Zheng if (slot >= nritems) { 37475d4f98a2SYan Zheng split = 0; 37485d4f98a2SYan Zheng } else { 37495d4f98a2SYan Zheng mid = slot; 37505d4f98a2SYan Zheng if (mid != nritems && 37515d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 37520b246afaSJeff Mahoney data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) { 375399d8f83cSChris Mason if (data_size && !tried_avoid_double) 375499d8f83cSChris Mason goto push_for_double; 37555d4f98a2SYan Zheng split = 2; 37565d4f98a2SYan Zheng } 37575d4f98a2SYan Zheng } 37585d4f98a2SYan Zheng } 37595d4f98a2SYan Zheng } else { 37605d4f98a2SYan Zheng if (leaf_space_used(l, 0, mid) + data_size > 37610b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info)) { 37625d4f98a2SYan Zheng if (!extend && data_size && slot == 0) { 37635d4f98a2SYan Zheng split = 0; 37645d4f98a2SYan Zheng } else if ((extend || !data_size) && slot == 0) { 37655d4f98a2SYan Zheng mid = 1; 37665d4f98a2SYan Zheng } else { 37675d4f98a2SYan Zheng mid = slot; 37685d4f98a2SYan Zheng if (mid != nritems && 37695d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 37700b246afaSJeff Mahoney data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) { 377199d8f83cSChris Mason if (data_size && !tried_avoid_double) 377299d8f83cSChris Mason goto push_for_double; 37735d4f98a2SYan Zheng split = 2; 37745d4f98a2SYan Zheng } 37755d4f98a2SYan Zheng } 37765d4f98a2SYan Zheng } 37775d4f98a2SYan Zheng } 37785d4f98a2SYan Zheng 37795d4f98a2SYan Zheng if (split == 0) 37805d4f98a2SYan Zheng btrfs_cpu_key_to_disk(&disk_key, ins_key); 37815d4f98a2SYan Zheng else 37825d4f98a2SYan Zheng btrfs_item_key(l, &disk_key, mid); 37835d4f98a2SYan Zheng 3784ca9d473aSJosef Bacik /* 3785ca9d473aSJosef Bacik * We have to about BTRFS_NESTING_NEW_ROOT here if we've done a double 3786ca9d473aSJosef Bacik * split, because we're only allowed to have MAX_LOCKDEP_SUBCLASSES 3787ca9d473aSJosef Bacik * subclasses, which is 8 at the time of this patch, and we've maxed it 3788ca9d473aSJosef Bacik * out. In the future we could add a 3789ca9d473aSJosef Bacik * BTRFS_NESTING_SPLIT_THE_SPLITTENING if we need to, but for now just 3790ca9d473aSJosef Bacik * use BTRFS_NESTING_NEW_ROOT. 3791ca9d473aSJosef Bacik */ 379279bd3712SFilipe Manana right = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 379379bd3712SFilipe Manana &disk_key, 0, l->start, 0, 379479bd3712SFilipe Manana num_doubles ? BTRFS_NESTING_NEW_ROOT : 3795ca9d473aSJosef Bacik BTRFS_NESTING_SPLIT); 3796f0486c68SYan, Zheng if (IS_ERR(right)) 379744871b1bSChris Mason return PTR_ERR(right); 3798f0486c68SYan, Zheng 37990b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 380044871b1bSChris Mason 38015d4f98a2SYan Zheng if (split == 0) { 380244871b1bSChris Mason if (mid <= slot) { 380344871b1bSChris Mason btrfs_set_header_nritems(right, 0); 38046ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, 38052ff7e61eSJeff Mahoney right->start, path->slots[1] + 1, 1); 380644871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 380744871b1bSChris Mason free_extent_buffer(path->nodes[0]); 380844871b1bSChris Mason path->nodes[0] = right; 380944871b1bSChris Mason path->slots[0] = 0; 381044871b1bSChris Mason path->slots[1] += 1; 381144871b1bSChris Mason } else { 381244871b1bSChris Mason btrfs_set_header_nritems(right, 0); 38136ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, 38142ff7e61eSJeff Mahoney right->start, path->slots[1], 1); 381544871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 381644871b1bSChris Mason free_extent_buffer(path->nodes[0]); 381744871b1bSChris Mason path->nodes[0] = right; 381844871b1bSChris Mason path->slots[0] = 0; 3819143bede5SJeff Mahoney if (path->slots[1] == 0) 3820b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 38215d4f98a2SYan Zheng } 3822196e0249SLiu Bo /* 3823196e0249SLiu Bo * We create a new leaf 'right' for the required ins_len and 3824196e0249SLiu Bo * we'll do btrfs_mark_buffer_dirty() on this leaf after copying 3825196e0249SLiu Bo * the content of ins_len to 'right'. 3826196e0249SLiu Bo */ 382744871b1bSChris Mason return ret; 382844871b1bSChris Mason } 382944871b1bSChris Mason 383094f94ad9SDavid Sterba copy_for_split(trans, path, l, right, slot, mid, nritems); 383144871b1bSChris Mason 38325d4f98a2SYan Zheng if (split == 2) { 3833cc0c5538SChris Mason BUG_ON(num_doubles != 0); 3834cc0c5538SChris Mason num_doubles++; 3835cc0c5538SChris Mason goto again; 38363326d1b0SChris Mason } 383744871b1bSChris Mason 3838143bede5SJeff Mahoney return 0; 383999d8f83cSChris Mason 384099d8f83cSChris Mason push_for_double: 384199d8f83cSChris Mason push_for_double_split(trans, root, path, data_size); 384299d8f83cSChris Mason tried_avoid_double = 1; 3843e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= data_size) 384499d8f83cSChris Mason return 0; 384599d8f83cSChris Mason goto again; 3846be0e5c09SChris Mason } 3847be0e5c09SChris Mason 3848ad48fd75SYan, Zheng static noinline int setup_leaf_for_split(struct btrfs_trans_handle *trans, 3849ad48fd75SYan, Zheng struct btrfs_root *root, 3850ad48fd75SYan, Zheng struct btrfs_path *path, int ins_len) 3851ad48fd75SYan, Zheng { 3852ad48fd75SYan, Zheng struct btrfs_key key; 3853ad48fd75SYan, Zheng struct extent_buffer *leaf; 3854ad48fd75SYan, Zheng struct btrfs_file_extent_item *fi; 3855ad48fd75SYan, Zheng u64 extent_len = 0; 3856ad48fd75SYan, Zheng u32 item_size; 3857ad48fd75SYan, Zheng int ret; 3858ad48fd75SYan, Zheng 3859ad48fd75SYan, Zheng leaf = path->nodes[0]; 3860ad48fd75SYan, Zheng btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 3861ad48fd75SYan, Zheng 3862ad48fd75SYan, Zheng BUG_ON(key.type != BTRFS_EXTENT_DATA_KEY && 3863ad48fd75SYan, Zheng key.type != BTRFS_EXTENT_CSUM_KEY); 3864ad48fd75SYan, Zheng 3865e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) >= ins_len) 3866ad48fd75SYan, Zheng return 0; 3867ad48fd75SYan, Zheng 38683212fa14SJosef Bacik item_size = btrfs_item_size(leaf, path->slots[0]); 3869ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 3870ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 3871ad48fd75SYan, Zheng struct btrfs_file_extent_item); 3872ad48fd75SYan, Zheng extent_len = btrfs_file_extent_num_bytes(leaf, fi); 3873ad48fd75SYan, Zheng } 3874b3b4aa74SDavid Sterba btrfs_release_path(path); 3875ad48fd75SYan, Zheng 3876ad48fd75SYan, Zheng path->keep_locks = 1; 3877ad48fd75SYan, Zheng path->search_for_split = 1; 3878ad48fd75SYan, Zheng ret = btrfs_search_slot(trans, root, &key, path, 0, 1); 3879ad48fd75SYan, Zheng path->search_for_split = 0; 3880a8df6fe6SFilipe Manana if (ret > 0) 3881a8df6fe6SFilipe Manana ret = -EAGAIN; 3882ad48fd75SYan, Zheng if (ret < 0) 3883ad48fd75SYan, Zheng goto err; 3884ad48fd75SYan, Zheng 3885ad48fd75SYan, Zheng ret = -EAGAIN; 3886ad48fd75SYan, Zheng leaf = path->nodes[0]; 3887a8df6fe6SFilipe Manana /* if our item isn't there, return now */ 38883212fa14SJosef Bacik if (item_size != btrfs_item_size(leaf, path->slots[0])) 3889ad48fd75SYan, Zheng goto err; 3890ad48fd75SYan, Zheng 3891109f6aefSChris Mason /* the leaf has changed, it now has room. return now */ 3892e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= ins_len) 3893109f6aefSChris Mason goto err; 3894109f6aefSChris Mason 3895ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 3896ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 3897ad48fd75SYan, Zheng struct btrfs_file_extent_item); 3898ad48fd75SYan, Zheng if (extent_len != btrfs_file_extent_num_bytes(leaf, fi)) 3899ad48fd75SYan, Zheng goto err; 3900ad48fd75SYan, Zheng } 3901ad48fd75SYan, Zheng 3902ad48fd75SYan, Zheng ret = split_leaf(trans, root, &key, path, ins_len, 1); 3903f0486c68SYan, Zheng if (ret) 3904f0486c68SYan, Zheng goto err; 3905ad48fd75SYan, Zheng 3906ad48fd75SYan, Zheng path->keep_locks = 0; 3907ad48fd75SYan, Zheng btrfs_unlock_up_safe(path, 1); 3908ad48fd75SYan, Zheng return 0; 3909ad48fd75SYan, Zheng err: 3910ad48fd75SYan, Zheng path->keep_locks = 0; 3911ad48fd75SYan, Zheng return ret; 3912ad48fd75SYan, Zheng } 3913ad48fd75SYan, Zheng 391425263cd7SDavid Sterba static noinline int split_item(struct btrfs_path *path, 3915310712b2SOmar Sandoval const struct btrfs_key *new_key, 3916459931ecSChris Mason unsigned long split_offset) 3917459931ecSChris Mason { 3918459931ecSChris Mason struct extent_buffer *leaf; 3919c91666b1SJosef Bacik int orig_slot, slot; 3920ad48fd75SYan, Zheng char *buf; 3921459931ecSChris Mason u32 nritems; 3922ad48fd75SYan, Zheng u32 item_size; 3923459931ecSChris Mason u32 orig_offset; 3924459931ecSChris Mason struct btrfs_disk_key disk_key; 3925459931ecSChris Mason 3926459931ecSChris Mason leaf = path->nodes[0]; 3927e902baacSDavid Sterba BUG_ON(btrfs_leaf_free_space(leaf) < sizeof(struct btrfs_item)); 3928b9473439SChris Mason 3929c91666b1SJosef Bacik orig_slot = path->slots[0]; 39303212fa14SJosef Bacik orig_offset = btrfs_item_offset(leaf, path->slots[0]); 39313212fa14SJosef Bacik item_size = btrfs_item_size(leaf, path->slots[0]); 3932459931ecSChris Mason 3933459931ecSChris Mason buf = kmalloc(item_size, GFP_NOFS); 3934ad48fd75SYan, Zheng if (!buf) 3935ad48fd75SYan, Zheng return -ENOMEM; 3936ad48fd75SYan, Zheng 3937459931ecSChris Mason read_extent_buffer(leaf, buf, btrfs_item_ptr_offset(leaf, 3938459931ecSChris Mason path->slots[0]), item_size); 3939ad48fd75SYan, Zheng 3940459931ecSChris Mason slot = path->slots[0] + 1; 3941459931ecSChris Mason nritems = btrfs_header_nritems(leaf); 3942459931ecSChris Mason if (slot != nritems) { 3943459931ecSChris Mason /* shift the items */ 3944637e3b48SJosef Bacik memmove_leaf_items(leaf, slot + 1, slot, nritems - slot); 3945459931ecSChris Mason } 3946459931ecSChris Mason 3947459931ecSChris Mason btrfs_cpu_key_to_disk(&disk_key, new_key); 3948459931ecSChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 3949459931ecSChris Mason 39503212fa14SJosef Bacik btrfs_set_item_offset(leaf, slot, orig_offset); 39513212fa14SJosef Bacik btrfs_set_item_size(leaf, slot, item_size - split_offset); 3952459931ecSChris Mason 39533212fa14SJosef Bacik btrfs_set_item_offset(leaf, orig_slot, 3954459931ecSChris Mason orig_offset + item_size - split_offset); 39553212fa14SJosef Bacik btrfs_set_item_size(leaf, orig_slot, split_offset); 3956459931ecSChris Mason 3957459931ecSChris Mason btrfs_set_header_nritems(leaf, nritems + 1); 3958459931ecSChris Mason 3959459931ecSChris Mason /* write the data for the start of the original item */ 3960459931ecSChris Mason write_extent_buffer(leaf, buf, 3961459931ecSChris Mason btrfs_item_ptr_offset(leaf, path->slots[0]), 3962459931ecSChris Mason split_offset); 3963459931ecSChris Mason 3964459931ecSChris Mason /* write the data for the new item */ 3965459931ecSChris Mason write_extent_buffer(leaf, buf + split_offset, 3966459931ecSChris Mason btrfs_item_ptr_offset(leaf, slot), 3967459931ecSChris Mason item_size - split_offset); 3968459931ecSChris Mason btrfs_mark_buffer_dirty(leaf); 3969459931ecSChris Mason 3970e902baacSDavid Sterba BUG_ON(btrfs_leaf_free_space(leaf) < 0); 3971459931ecSChris Mason kfree(buf); 3972ad48fd75SYan, Zheng return 0; 3973ad48fd75SYan, Zheng } 3974ad48fd75SYan, Zheng 3975ad48fd75SYan, Zheng /* 3976ad48fd75SYan, Zheng * This function splits a single item into two items, 3977ad48fd75SYan, Zheng * giving 'new_key' to the new item and splitting the 3978ad48fd75SYan, Zheng * old one at split_offset (from the start of the item). 3979ad48fd75SYan, Zheng * 3980ad48fd75SYan, Zheng * The path may be released by this operation. After 3981ad48fd75SYan, Zheng * the split, the path is pointing to the old item. The 3982ad48fd75SYan, Zheng * new item is going to be in the same node as the old one. 3983ad48fd75SYan, Zheng * 3984ad48fd75SYan, Zheng * Note, the item being split must be smaller enough to live alone on 3985ad48fd75SYan, Zheng * a tree block with room for one extra struct btrfs_item 3986ad48fd75SYan, Zheng * 3987ad48fd75SYan, Zheng * This allows us to split the item in place, keeping a lock on the 3988ad48fd75SYan, Zheng * leaf the entire time. 3989ad48fd75SYan, Zheng */ 3990ad48fd75SYan, Zheng int btrfs_split_item(struct btrfs_trans_handle *trans, 3991ad48fd75SYan, Zheng struct btrfs_root *root, 3992ad48fd75SYan, Zheng struct btrfs_path *path, 3993310712b2SOmar Sandoval const struct btrfs_key *new_key, 3994ad48fd75SYan, Zheng unsigned long split_offset) 3995ad48fd75SYan, Zheng { 3996ad48fd75SYan, Zheng int ret; 3997ad48fd75SYan, Zheng ret = setup_leaf_for_split(trans, root, path, 3998ad48fd75SYan, Zheng sizeof(struct btrfs_item)); 3999ad48fd75SYan, Zheng if (ret) 4000459931ecSChris Mason return ret; 4001ad48fd75SYan, Zheng 400225263cd7SDavid Sterba ret = split_item(path, new_key, split_offset); 4003ad48fd75SYan, Zheng return ret; 4004ad48fd75SYan, Zheng } 4005ad48fd75SYan, Zheng 4006ad48fd75SYan, Zheng /* 4007d352ac68SChris Mason * make the item pointed to by the path smaller. new_size indicates 4008d352ac68SChris Mason * how small to make it, and from_end tells us if we just chop bytes 4009d352ac68SChris Mason * off the end of the item or if we shift the item to chop bytes off 4010d352ac68SChris Mason * the front. 4011d352ac68SChris Mason */ 401278ac4f9eSDavid Sterba void btrfs_truncate_item(struct btrfs_path *path, u32 new_size, int from_end) 4013b18c6685SChris Mason { 4014b18c6685SChris Mason int slot; 40155f39d397SChris Mason struct extent_buffer *leaf; 4016b18c6685SChris Mason u32 nritems; 4017b18c6685SChris Mason unsigned int data_end; 4018b18c6685SChris Mason unsigned int old_data_start; 4019b18c6685SChris Mason unsigned int old_size; 4020b18c6685SChris Mason unsigned int size_diff; 4021b18c6685SChris Mason int i; 4022cfed81a0SChris Mason struct btrfs_map_token token; 4023cfed81a0SChris Mason 40245f39d397SChris Mason leaf = path->nodes[0]; 4025179e29e4SChris Mason slot = path->slots[0]; 4026179e29e4SChris Mason 40273212fa14SJosef Bacik old_size = btrfs_item_size(leaf, slot); 4028179e29e4SChris Mason if (old_size == new_size) 4029143bede5SJeff Mahoney return; 4030b18c6685SChris Mason 40315f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 40328f881e8cSDavid Sterba data_end = leaf_data_end(leaf); 4033b18c6685SChris Mason 40343212fa14SJosef Bacik old_data_start = btrfs_item_offset(leaf, slot); 4035179e29e4SChris Mason 4036b18c6685SChris Mason size_diff = old_size - new_size; 4037b18c6685SChris Mason 4038b18c6685SChris Mason BUG_ON(slot < 0); 4039b18c6685SChris Mason BUG_ON(slot >= nritems); 4040b18c6685SChris Mason 4041b18c6685SChris Mason /* 4042b18c6685SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 4043b18c6685SChris Mason */ 4044b18c6685SChris Mason /* first correct the data pointers */ 4045c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 4046b18c6685SChris Mason for (i = slot; i < nritems; i++) { 40475f39d397SChris Mason u32 ioff; 4048db94535dSChris Mason 40493212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 40503212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff + size_diff); 4051b18c6685SChris Mason } 4052db94535dSChris Mason 4053b18c6685SChris Mason /* shift the data */ 4054179e29e4SChris Mason if (from_end) { 4055637e3b48SJosef Bacik memmove_leaf_data(leaf, data_end + size_diff, data_end, 4056637e3b48SJosef Bacik old_data_start + new_size - data_end); 4057179e29e4SChris Mason } else { 4058179e29e4SChris Mason struct btrfs_disk_key disk_key; 4059179e29e4SChris Mason u64 offset; 4060179e29e4SChris Mason 4061179e29e4SChris Mason btrfs_item_key(leaf, &disk_key, slot); 4062179e29e4SChris Mason 4063179e29e4SChris Mason if (btrfs_disk_key_type(&disk_key) == BTRFS_EXTENT_DATA_KEY) { 4064179e29e4SChris Mason unsigned long ptr; 4065179e29e4SChris Mason struct btrfs_file_extent_item *fi; 4066179e29e4SChris Mason 4067179e29e4SChris Mason fi = btrfs_item_ptr(leaf, slot, 4068179e29e4SChris Mason struct btrfs_file_extent_item); 4069179e29e4SChris Mason fi = (struct btrfs_file_extent_item *)( 4070179e29e4SChris Mason (unsigned long)fi - size_diff); 4071179e29e4SChris Mason 4072179e29e4SChris Mason if (btrfs_file_extent_type(leaf, fi) == 4073179e29e4SChris Mason BTRFS_FILE_EXTENT_INLINE) { 4074179e29e4SChris Mason ptr = btrfs_item_ptr_offset(leaf, slot); 4075179e29e4SChris Mason memmove_extent_buffer(leaf, ptr, 4076179e29e4SChris Mason (unsigned long)fi, 40777ec20afbSDavid Sterba BTRFS_FILE_EXTENT_INLINE_DATA_START); 4078179e29e4SChris Mason } 4079179e29e4SChris Mason } 4080179e29e4SChris Mason 4081637e3b48SJosef Bacik memmove_leaf_data(leaf, data_end + size_diff, data_end, 4082637e3b48SJosef Bacik old_data_start - data_end); 4083179e29e4SChris Mason 4084179e29e4SChris Mason offset = btrfs_disk_key_offset(&disk_key); 4085179e29e4SChris Mason btrfs_set_disk_key_offset(&disk_key, offset + size_diff); 4086179e29e4SChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 4087179e29e4SChris Mason if (slot == 0) 4088b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 4089179e29e4SChris Mason } 40905f39d397SChris Mason 40913212fa14SJosef Bacik btrfs_set_item_size(leaf, slot, new_size); 40925f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 4093b18c6685SChris Mason 4094e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) { 4095a4f78750SDavid Sterba btrfs_print_leaf(leaf); 4096b18c6685SChris Mason BUG(); 40975f39d397SChris Mason } 4098b18c6685SChris Mason } 4099b18c6685SChris Mason 4100d352ac68SChris Mason /* 41018f69dbd2SStefan Behrens * make the item pointed to by the path bigger, data_size is the added size. 4102d352ac68SChris Mason */ 4103c71dd880SDavid Sterba void btrfs_extend_item(struct btrfs_path *path, u32 data_size) 41046567e837SChris Mason { 41056567e837SChris Mason int slot; 41065f39d397SChris Mason struct extent_buffer *leaf; 41076567e837SChris Mason u32 nritems; 41086567e837SChris Mason unsigned int data_end; 41096567e837SChris Mason unsigned int old_data; 41106567e837SChris Mason unsigned int old_size; 41116567e837SChris Mason int i; 4112cfed81a0SChris Mason struct btrfs_map_token token; 4113cfed81a0SChris Mason 41145f39d397SChris Mason leaf = path->nodes[0]; 41156567e837SChris Mason 41165f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 41178f881e8cSDavid Sterba data_end = leaf_data_end(leaf); 41186567e837SChris Mason 4119e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < data_size) { 4120a4f78750SDavid Sterba btrfs_print_leaf(leaf); 41216567e837SChris Mason BUG(); 41225f39d397SChris Mason } 41236567e837SChris Mason slot = path->slots[0]; 4124dc2e724eSJosef Bacik old_data = btrfs_item_data_end(leaf, slot); 41256567e837SChris Mason 41266567e837SChris Mason BUG_ON(slot < 0); 41273326d1b0SChris Mason if (slot >= nritems) { 4128a4f78750SDavid Sterba btrfs_print_leaf(leaf); 4129c71dd880SDavid Sterba btrfs_crit(leaf->fs_info, "slot %d too large, nritems %d", 4130d397712bSChris Mason slot, nritems); 4131290342f6SArnd Bergmann BUG(); 41323326d1b0SChris Mason } 41336567e837SChris Mason 41346567e837SChris Mason /* 41356567e837SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 41366567e837SChris Mason */ 41376567e837SChris Mason /* first correct the data pointers */ 4138c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 41396567e837SChris Mason for (i = slot; i < nritems; i++) { 41405f39d397SChris Mason u32 ioff; 4141db94535dSChris Mason 41423212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 41433212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff - data_size); 41446567e837SChris Mason } 41455f39d397SChris Mason 41466567e837SChris Mason /* shift the data */ 4147637e3b48SJosef Bacik memmove_leaf_data(leaf, data_end - data_size, data_end, 4148637e3b48SJosef Bacik old_data - data_end); 41495f39d397SChris Mason 41506567e837SChris Mason data_end = old_data; 41513212fa14SJosef Bacik old_size = btrfs_item_size(leaf, slot); 41523212fa14SJosef Bacik btrfs_set_item_size(leaf, slot, old_size + data_size); 41535f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 41546567e837SChris Mason 4155e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) { 4156a4f78750SDavid Sterba btrfs_print_leaf(leaf); 41576567e837SChris Mason BUG(); 41585f39d397SChris Mason } 41596567e837SChris Mason } 41606567e837SChris Mason 416143dd529aSDavid Sterba /* 416243dd529aSDavid Sterba * Make space in the node before inserting one or more items. 4163da9ffb24SNikolay Borisov * 4164da9ffb24SNikolay Borisov * @root: root we are inserting items to 4165da9ffb24SNikolay Borisov * @path: points to the leaf/slot where we are going to insert new items 4166b7ef5f3aSFilipe Manana * @batch: information about the batch of items to insert 416743dd529aSDavid Sterba * 416843dd529aSDavid Sterba * Main purpose is to save stack depth by doing the bulk of the work in a 416943dd529aSDavid Sterba * function that doesn't call btrfs_search_slot 417074123bd7SChris Mason */ 4171f0641656SFilipe Manana static void setup_items_for_insert(struct btrfs_root *root, struct btrfs_path *path, 4172b7ef5f3aSFilipe Manana const struct btrfs_item_batch *batch) 4173be0e5c09SChris Mason { 41740b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 41759c58309dSChris Mason int i; 41767518a238SChris Mason u32 nritems; 4177be0e5c09SChris Mason unsigned int data_end; 4178e2fa7227SChris Mason struct btrfs_disk_key disk_key; 417944871b1bSChris Mason struct extent_buffer *leaf; 418044871b1bSChris Mason int slot; 4181cfed81a0SChris Mason struct btrfs_map_token token; 4182fc0d82e1SNikolay Borisov u32 total_size; 4183fc0d82e1SNikolay Borisov 4184b7ef5f3aSFilipe Manana /* 4185b7ef5f3aSFilipe Manana * Before anything else, update keys in the parent and other ancestors 4186b7ef5f3aSFilipe Manana * if needed, then release the write locks on them, so that other tasks 4187b7ef5f3aSFilipe Manana * can use them while we modify the leaf. 4188b7ef5f3aSFilipe Manana */ 418924cdc847SFilipe Manana if (path->slots[0] == 0) { 4190b7ef5f3aSFilipe Manana btrfs_cpu_key_to_disk(&disk_key, &batch->keys[0]); 4191b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 419224cdc847SFilipe Manana } 419324cdc847SFilipe Manana btrfs_unlock_up_safe(path, 1); 419424cdc847SFilipe Manana 41955f39d397SChris Mason leaf = path->nodes[0]; 419644871b1bSChris Mason slot = path->slots[0]; 419774123bd7SChris Mason 41985f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 41998f881e8cSDavid Sterba data_end = leaf_data_end(leaf); 4200b7ef5f3aSFilipe Manana total_size = batch->total_data_size + (batch->nr * sizeof(struct btrfs_item)); 4201eb60ceacSChris Mason 4202e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < total_size) { 4203a4f78750SDavid Sterba btrfs_print_leaf(leaf); 42040b246afaSJeff Mahoney btrfs_crit(fs_info, "not enough freespace need %u have %d", 4205e902baacSDavid Sterba total_size, btrfs_leaf_free_space(leaf)); 4206be0e5c09SChris Mason BUG(); 4207d4dbff95SChris Mason } 42085f39d397SChris Mason 4209c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 4210be0e5c09SChris Mason if (slot != nritems) { 4211dc2e724eSJosef Bacik unsigned int old_data = btrfs_item_data_end(leaf, slot); 4212be0e5c09SChris Mason 42135f39d397SChris Mason if (old_data < data_end) { 4214a4f78750SDavid Sterba btrfs_print_leaf(leaf); 42157269ddd2SNikolay Borisov btrfs_crit(fs_info, 42167269ddd2SNikolay Borisov "item at slot %d with data offset %u beyond data end of leaf %u", 42175f39d397SChris Mason slot, old_data, data_end); 4218290342f6SArnd Bergmann BUG(); 42195f39d397SChris Mason } 4220be0e5c09SChris Mason /* 4221be0e5c09SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 4222be0e5c09SChris Mason */ 4223be0e5c09SChris Mason /* first correct the data pointers */ 42240783fcfcSChris Mason for (i = slot; i < nritems; i++) { 42255f39d397SChris Mason u32 ioff; 4226db94535dSChris Mason 42273212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 42283212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, 4229b7ef5f3aSFilipe Manana ioff - batch->total_data_size); 42300783fcfcSChris Mason } 4231be0e5c09SChris Mason /* shift the items */ 4232637e3b48SJosef Bacik memmove_leaf_items(leaf, slot + batch->nr, slot, nritems - slot); 4233be0e5c09SChris Mason 4234be0e5c09SChris Mason /* shift the data */ 4235637e3b48SJosef Bacik memmove_leaf_data(leaf, data_end - batch->total_data_size, 4236637e3b48SJosef Bacik data_end, old_data - data_end); 4237be0e5c09SChris Mason data_end = old_data; 4238be0e5c09SChris Mason } 42395f39d397SChris Mason 424062e2749eSChris Mason /* setup the item for the new data */ 4241b7ef5f3aSFilipe Manana for (i = 0; i < batch->nr; i++) { 4242b7ef5f3aSFilipe Manana btrfs_cpu_key_to_disk(&disk_key, &batch->keys[i]); 42439c58309dSChris Mason btrfs_set_item_key(leaf, &disk_key, slot + i); 4244b7ef5f3aSFilipe Manana data_end -= batch->data_sizes[i]; 42453212fa14SJosef Bacik btrfs_set_token_item_offset(&token, slot + i, data_end); 42463212fa14SJosef Bacik btrfs_set_token_item_size(&token, slot + i, batch->data_sizes[i]); 42479c58309dSChris Mason } 424844871b1bSChris Mason 4249b7ef5f3aSFilipe Manana btrfs_set_header_nritems(leaf, nritems + batch->nr); 4250b9473439SChris Mason btrfs_mark_buffer_dirty(leaf); 4251aa5d6bedSChris Mason 4252e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) { 4253a4f78750SDavid Sterba btrfs_print_leaf(leaf); 4254be0e5c09SChris Mason BUG(); 42555f39d397SChris Mason } 425644871b1bSChris Mason } 425744871b1bSChris Mason 425844871b1bSChris Mason /* 4259f0641656SFilipe Manana * Insert a new item into a leaf. 4260f0641656SFilipe Manana * 4261f0641656SFilipe Manana * @root: The root of the btree. 4262f0641656SFilipe Manana * @path: A path pointing to the target leaf and slot. 4263f0641656SFilipe Manana * @key: The key of the new item. 4264f0641656SFilipe Manana * @data_size: The size of the data associated with the new key. 4265f0641656SFilipe Manana */ 4266f0641656SFilipe Manana void btrfs_setup_item_for_insert(struct btrfs_root *root, 4267f0641656SFilipe Manana struct btrfs_path *path, 4268f0641656SFilipe Manana const struct btrfs_key *key, 4269f0641656SFilipe Manana u32 data_size) 4270f0641656SFilipe Manana { 4271f0641656SFilipe Manana struct btrfs_item_batch batch; 4272f0641656SFilipe Manana 4273f0641656SFilipe Manana batch.keys = key; 4274f0641656SFilipe Manana batch.data_sizes = &data_size; 4275f0641656SFilipe Manana batch.total_data_size = data_size; 4276f0641656SFilipe Manana batch.nr = 1; 4277f0641656SFilipe Manana 4278f0641656SFilipe Manana setup_items_for_insert(root, path, &batch); 4279f0641656SFilipe Manana } 4280f0641656SFilipe Manana 4281f0641656SFilipe Manana /* 428244871b1bSChris Mason * Given a key and some data, insert items into the tree. 428344871b1bSChris Mason * This does all the path init required, making room in the tree if needed. 428444871b1bSChris Mason */ 428544871b1bSChris Mason int btrfs_insert_empty_items(struct btrfs_trans_handle *trans, 428644871b1bSChris Mason struct btrfs_root *root, 428744871b1bSChris Mason struct btrfs_path *path, 4288b7ef5f3aSFilipe Manana const struct btrfs_item_batch *batch) 428944871b1bSChris Mason { 429044871b1bSChris Mason int ret = 0; 429144871b1bSChris Mason int slot; 4292b7ef5f3aSFilipe Manana u32 total_size; 429344871b1bSChris Mason 4294b7ef5f3aSFilipe Manana total_size = batch->total_data_size + (batch->nr * sizeof(struct btrfs_item)); 4295b7ef5f3aSFilipe Manana ret = btrfs_search_slot(trans, root, &batch->keys[0], path, total_size, 1); 429644871b1bSChris Mason if (ret == 0) 429744871b1bSChris Mason return -EEXIST; 429844871b1bSChris Mason if (ret < 0) 4299143bede5SJeff Mahoney return ret; 430044871b1bSChris Mason 430144871b1bSChris Mason slot = path->slots[0]; 430244871b1bSChris Mason BUG_ON(slot < 0); 430344871b1bSChris Mason 4304b7ef5f3aSFilipe Manana setup_items_for_insert(root, path, batch); 4305143bede5SJeff Mahoney return 0; 430662e2749eSChris Mason } 430762e2749eSChris Mason 430862e2749eSChris Mason /* 430962e2749eSChris Mason * Given a key and some data, insert an item into the tree. 431062e2749eSChris Mason * This does all the path init required, making room in the tree if needed. 431162e2749eSChris Mason */ 4312310712b2SOmar Sandoval int btrfs_insert_item(struct btrfs_trans_handle *trans, struct btrfs_root *root, 4313310712b2SOmar Sandoval const struct btrfs_key *cpu_key, void *data, 4314310712b2SOmar Sandoval u32 data_size) 431562e2749eSChris Mason { 431662e2749eSChris Mason int ret = 0; 43172c90e5d6SChris Mason struct btrfs_path *path; 43185f39d397SChris Mason struct extent_buffer *leaf; 43195f39d397SChris Mason unsigned long ptr; 432062e2749eSChris Mason 43212c90e5d6SChris Mason path = btrfs_alloc_path(); 4322db5b493aSTsutomu Itoh if (!path) 4323db5b493aSTsutomu Itoh return -ENOMEM; 43242c90e5d6SChris Mason ret = btrfs_insert_empty_item(trans, root, path, cpu_key, data_size); 432562e2749eSChris Mason if (!ret) { 43265f39d397SChris Mason leaf = path->nodes[0]; 43275f39d397SChris Mason ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); 43285f39d397SChris Mason write_extent_buffer(leaf, data, ptr, data_size); 43295f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 433062e2749eSChris Mason } 43312c90e5d6SChris Mason btrfs_free_path(path); 4332aa5d6bedSChris Mason return ret; 4333be0e5c09SChris Mason } 4334be0e5c09SChris Mason 433574123bd7SChris Mason /* 4336f0641656SFilipe Manana * This function duplicates an item, giving 'new_key' to the new item. 4337f0641656SFilipe Manana * It guarantees both items live in the same tree leaf and the new item is 4338f0641656SFilipe Manana * contiguous with the original item. 4339f0641656SFilipe Manana * 4340f0641656SFilipe Manana * This allows us to split a file extent in place, keeping a lock on the leaf 4341f0641656SFilipe Manana * the entire time. 4342f0641656SFilipe Manana */ 4343f0641656SFilipe Manana int btrfs_duplicate_item(struct btrfs_trans_handle *trans, 4344f0641656SFilipe Manana struct btrfs_root *root, 4345f0641656SFilipe Manana struct btrfs_path *path, 4346f0641656SFilipe Manana const struct btrfs_key *new_key) 4347f0641656SFilipe Manana { 4348f0641656SFilipe Manana struct extent_buffer *leaf; 4349f0641656SFilipe Manana int ret; 4350f0641656SFilipe Manana u32 item_size; 4351f0641656SFilipe Manana 4352f0641656SFilipe Manana leaf = path->nodes[0]; 43533212fa14SJosef Bacik item_size = btrfs_item_size(leaf, path->slots[0]); 4354f0641656SFilipe Manana ret = setup_leaf_for_split(trans, root, path, 4355f0641656SFilipe Manana item_size + sizeof(struct btrfs_item)); 4356f0641656SFilipe Manana if (ret) 4357f0641656SFilipe Manana return ret; 4358f0641656SFilipe Manana 4359f0641656SFilipe Manana path->slots[0]++; 4360f0641656SFilipe Manana btrfs_setup_item_for_insert(root, path, new_key, item_size); 4361f0641656SFilipe Manana leaf = path->nodes[0]; 4362f0641656SFilipe Manana memcpy_extent_buffer(leaf, 4363f0641656SFilipe Manana btrfs_item_ptr_offset(leaf, path->slots[0]), 4364f0641656SFilipe Manana btrfs_item_ptr_offset(leaf, path->slots[0] - 1), 4365f0641656SFilipe Manana item_size); 4366f0641656SFilipe Manana return 0; 4367f0641656SFilipe Manana } 4368f0641656SFilipe Manana 4369f0641656SFilipe Manana /* 43705de08d7dSChris Mason * delete the pointer from a given node. 437174123bd7SChris Mason * 4372d352ac68SChris Mason * the tree should have been previously balanced so the deletion does not 4373d352ac68SChris Mason * empty a node. 4374016f9d0bSJosef Bacik * 4375016f9d0bSJosef Bacik * This is exported for use inside btrfs-progs, don't un-export it. 437674123bd7SChris Mason */ 4377016f9d0bSJosef Bacik void btrfs_del_ptr(struct btrfs_root *root, struct btrfs_path *path, int level, 4378016f9d0bSJosef Bacik int slot) 4379be0e5c09SChris Mason { 43805f39d397SChris Mason struct extent_buffer *parent = path->nodes[level]; 43817518a238SChris Mason u32 nritems; 4382f3ea38daSJan Schmidt int ret; 4383be0e5c09SChris Mason 43845f39d397SChris Mason nritems = btrfs_header_nritems(parent); 4385be0e5c09SChris Mason if (slot != nritems - 1) { 4386bf1d3425SDavid Sterba if (level) { 4387f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_move(parent, slot, 4388f3a84ccdSFilipe Manana slot + 1, nritems - slot - 1); 4389bf1d3425SDavid Sterba BUG_ON(ret < 0); 4390bf1d3425SDavid Sterba } 43915f39d397SChris Mason memmove_extent_buffer(parent, 4392e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(parent, slot), 4393e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(parent, slot + 1), 4394d6025579SChris Mason sizeof(struct btrfs_key_ptr) * 4395d6025579SChris Mason (nritems - slot - 1)); 439657ba86c0SChris Mason } else if (level) { 4397f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, slot, 439833cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REMOVE); 439957ba86c0SChris Mason BUG_ON(ret < 0); 4400be0e5c09SChris Mason } 4401f3ea38daSJan Schmidt 44027518a238SChris Mason nritems--; 44035f39d397SChris Mason btrfs_set_header_nritems(parent, nritems); 44047518a238SChris Mason if (nritems == 0 && parent == root->node) { 44055f39d397SChris Mason BUG_ON(btrfs_header_level(root->node) != 1); 4406eb60ceacSChris Mason /* just turn the root into a leaf and break */ 44075f39d397SChris Mason btrfs_set_header_level(root->node, 0); 4408bb803951SChris Mason } else if (slot == 0) { 44095f39d397SChris Mason struct btrfs_disk_key disk_key; 44105f39d397SChris Mason 44115f39d397SChris Mason btrfs_node_key(parent, &disk_key, 0); 4412b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, level + 1); 4413be0e5c09SChris Mason } 4414d6025579SChris Mason btrfs_mark_buffer_dirty(parent); 4415be0e5c09SChris Mason } 4416be0e5c09SChris Mason 441774123bd7SChris Mason /* 4418323ac95bSChris Mason * a helper function to delete the leaf pointed to by path->slots[1] and 44195d4f98a2SYan Zheng * path->nodes[1]. 4420323ac95bSChris Mason * 4421323ac95bSChris Mason * This deletes the pointer in path->nodes[1] and frees the leaf 4422323ac95bSChris Mason * block extent. zero is returned if it all worked out, < 0 otherwise. 4423323ac95bSChris Mason * 4424323ac95bSChris Mason * The path must have already been setup for deleting the leaf, including 4425323ac95bSChris Mason * all the proper balancing. path->nodes[1] must be locked. 4426323ac95bSChris Mason */ 4427143bede5SJeff Mahoney static noinline void btrfs_del_leaf(struct btrfs_trans_handle *trans, 4428323ac95bSChris Mason struct btrfs_root *root, 44295d4f98a2SYan Zheng struct btrfs_path *path, 44305d4f98a2SYan Zheng struct extent_buffer *leaf) 4431323ac95bSChris Mason { 44325d4f98a2SYan Zheng WARN_ON(btrfs_header_generation(leaf) != trans->transid); 4433016f9d0bSJosef Bacik btrfs_del_ptr(root, path, 1, path->slots[1]); 4434323ac95bSChris Mason 44354d081c41SChris Mason /* 44364d081c41SChris Mason * btrfs_free_extent is expensive, we want to make sure we 44374d081c41SChris Mason * aren't holding any locks when we call it 44384d081c41SChris Mason */ 44394d081c41SChris Mason btrfs_unlock_up_safe(path, 0); 44404d081c41SChris Mason 4441f0486c68SYan, Zheng root_sub_used(root, leaf->len); 4442f0486c68SYan, Zheng 444367439dadSDavid Sterba atomic_inc(&leaf->refs); 44447a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), leaf, 0, 1); 44453083ee2eSJosef Bacik free_extent_buffer_stale(leaf); 4446323ac95bSChris Mason } 4447323ac95bSChris Mason /* 444874123bd7SChris Mason * delete the item at the leaf level in path. If that empties 444974123bd7SChris Mason * the leaf, remove it from the tree 445074123bd7SChris Mason */ 445185e21bacSChris Mason int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root, 445285e21bacSChris Mason struct btrfs_path *path, int slot, int nr) 4453be0e5c09SChris Mason { 44540b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 44555f39d397SChris Mason struct extent_buffer *leaf; 4456aa5d6bedSChris Mason int ret = 0; 4457aa5d6bedSChris Mason int wret; 44587518a238SChris Mason u32 nritems; 4459be0e5c09SChris Mason 44605f39d397SChris Mason leaf = path->nodes[0]; 44615f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 4462be0e5c09SChris Mason 446385e21bacSChris Mason if (slot + nr != nritems) { 44640cae23b6SFilipe Manana const u32 last_off = btrfs_item_offset(leaf, slot + nr - 1); 44650cae23b6SFilipe Manana const int data_end = leaf_data_end(leaf); 4466c82f823cSDavid Sterba struct btrfs_map_token token; 44670cae23b6SFilipe Manana u32 dsize = 0; 44680cae23b6SFilipe Manana int i; 44690cae23b6SFilipe Manana 44700cae23b6SFilipe Manana for (i = 0; i < nr; i++) 44710cae23b6SFilipe Manana dsize += btrfs_item_size(leaf, slot + i); 44725f39d397SChris Mason 4473637e3b48SJosef Bacik memmove_leaf_data(leaf, data_end + dsize, data_end, 447485e21bacSChris Mason last_off - data_end); 44755f39d397SChris Mason 4476c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 447785e21bacSChris Mason for (i = slot + nr; i < nritems; i++) { 44785f39d397SChris Mason u32 ioff; 4479db94535dSChris Mason 44803212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 44813212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff + dsize); 44820783fcfcSChris Mason } 4483db94535dSChris Mason 4484637e3b48SJosef Bacik memmove_leaf_items(leaf, slot, slot + nr, nritems - slot - nr); 4485be0e5c09SChris Mason } 448685e21bacSChris Mason btrfs_set_header_nritems(leaf, nritems - nr); 448785e21bacSChris Mason nritems -= nr; 44885f39d397SChris Mason 448974123bd7SChris Mason /* delete the leaf if we've emptied it */ 44907518a238SChris Mason if (nritems == 0) { 44915f39d397SChris Mason if (leaf == root->node) { 44925f39d397SChris Mason btrfs_set_header_level(leaf, 0); 44939a8dd150SChris Mason } else { 4494190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, leaf); 4495143bede5SJeff Mahoney btrfs_del_leaf(trans, root, path, leaf); 44969a8dd150SChris Mason } 4497be0e5c09SChris Mason } else { 44987518a238SChris Mason int used = leaf_space_used(leaf, 0, nritems); 4499aa5d6bedSChris Mason if (slot == 0) { 45005f39d397SChris Mason struct btrfs_disk_key disk_key; 45015f39d397SChris Mason 45025f39d397SChris Mason btrfs_item_key(leaf, &disk_key, 0); 4503b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 4504aa5d6bedSChris Mason } 4505aa5d6bedSChris Mason 45067c4063d1SFilipe Manana /* 45077c4063d1SFilipe Manana * Try to delete the leaf if it is mostly empty. We do this by 45087c4063d1SFilipe Manana * trying to move all its items into its left and right neighbours. 45097c4063d1SFilipe Manana * If we can't move all the items, then we don't delete it - it's 45107c4063d1SFilipe Manana * not ideal, but future insertions might fill the leaf with more 45117c4063d1SFilipe Manana * items, or items from other leaves might be moved later into our 45127c4063d1SFilipe Manana * leaf due to deletions on those leaves. 45137c4063d1SFilipe Manana */ 45140b246afaSJeff Mahoney if (used < BTRFS_LEAF_DATA_SIZE(fs_info) / 3) { 45157c4063d1SFilipe Manana u32 min_push_space; 45167c4063d1SFilipe Manana 4517be0e5c09SChris Mason /* push_leaf_left fixes the path. 4518be0e5c09SChris Mason * make sure the path still points to our leaf 4519016f9d0bSJosef Bacik * for possible call to btrfs_del_ptr below 4520be0e5c09SChris Mason */ 45214920c9acSChris Mason slot = path->slots[1]; 452267439dadSDavid Sterba atomic_inc(&leaf->refs); 45237c4063d1SFilipe Manana /* 45247c4063d1SFilipe Manana * We want to be able to at least push one item to the 45257c4063d1SFilipe Manana * left neighbour leaf, and that's the first item. 45267c4063d1SFilipe Manana */ 45277c4063d1SFilipe Manana min_push_space = sizeof(struct btrfs_item) + 45287c4063d1SFilipe Manana btrfs_item_size(leaf, 0); 45297c4063d1SFilipe Manana wret = push_leaf_left(trans, root, path, 0, 45307c4063d1SFilipe Manana min_push_space, 1, (u32)-1); 453154aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 4532aa5d6bedSChris Mason ret = wret; 45335f39d397SChris Mason 45345f39d397SChris Mason if (path->nodes[0] == leaf && 45355f39d397SChris Mason btrfs_header_nritems(leaf)) { 45367c4063d1SFilipe Manana /* 45377c4063d1SFilipe Manana * If we were not able to push all items from our 45387c4063d1SFilipe Manana * leaf to its left neighbour, then attempt to 45397c4063d1SFilipe Manana * either push all the remaining items to the 45407c4063d1SFilipe Manana * right neighbour or none. There's no advantage 45417c4063d1SFilipe Manana * in pushing only some items, instead of all, as 45427c4063d1SFilipe Manana * it's pointless to end up with a leaf having 45437c4063d1SFilipe Manana * too few items while the neighbours can be full 45447c4063d1SFilipe Manana * or nearly full. 45457c4063d1SFilipe Manana */ 45467c4063d1SFilipe Manana nritems = btrfs_header_nritems(leaf); 45477c4063d1SFilipe Manana min_push_space = leaf_space_used(leaf, 0, nritems); 45487c4063d1SFilipe Manana wret = push_leaf_right(trans, root, path, 0, 45497c4063d1SFilipe Manana min_push_space, 1, 0); 455054aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 4551aa5d6bedSChris Mason ret = wret; 4552aa5d6bedSChris Mason } 45535f39d397SChris Mason 45545f39d397SChris Mason if (btrfs_header_nritems(leaf) == 0) { 4555323ac95bSChris Mason path->slots[1] = slot; 4556143bede5SJeff Mahoney btrfs_del_leaf(trans, root, path, leaf); 45575f39d397SChris Mason free_extent_buffer(leaf); 4558143bede5SJeff Mahoney ret = 0; 45595de08d7dSChris Mason } else { 4560925baeddSChris Mason /* if we're still in the path, make sure 4561925baeddSChris Mason * we're dirty. Otherwise, one of the 4562925baeddSChris Mason * push_leaf functions must have already 4563925baeddSChris Mason * dirtied this buffer 4564925baeddSChris Mason */ 4565925baeddSChris Mason if (path->nodes[0] == leaf) 45665f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 45675f39d397SChris Mason free_extent_buffer(leaf); 4568be0e5c09SChris Mason } 4569d5719762SChris Mason } else { 45705f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 4571be0e5c09SChris Mason } 45729a8dd150SChris Mason } 4573aa5d6bedSChris Mason return ret; 45749a8dd150SChris Mason } 45759a8dd150SChris Mason 457697571fd0SChris Mason /* 45773f157a2fSChris Mason * A helper function to walk down the tree starting at min_key, and looking 4578de78b51aSEric Sandeen * for nodes or leaves that are have a minimum transaction id. 4579de78b51aSEric Sandeen * This is used by the btree defrag code, and tree logging 45803f157a2fSChris Mason * 45813f157a2fSChris Mason * This does not cow, but it does stuff the starting key it finds back 45823f157a2fSChris Mason * into min_key, so you can call btrfs_search_slot with cow=1 on the 45833f157a2fSChris Mason * key and get a writable path. 45843f157a2fSChris Mason * 45853f157a2fSChris Mason * This honors path->lowest_level to prevent descent past a given level 45863f157a2fSChris Mason * of the tree. 45873f157a2fSChris Mason * 4588d352ac68SChris Mason * min_trans indicates the oldest transaction that you are interested 4589d352ac68SChris Mason * in walking through. Any nodes or leaves older than min_trans are 4590d352ac68SChris Mason * skipped over (without reading them). 4591d352ac68SChris Mason * 45923f157a2fSChris Mason * returns zero if something useful was found, < 0 on error and 1 if there 45933f157a2fSChris Mason * was nothing in the tree that matched the search criteria. 45943f157a2fSChris Mason */ 45953f157a2fSChris Mason int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key, 4596de78b51aSEric Sandeen struct btrfs_path *path, 45973f157a2fSChris Mason u64 min_trans) 45983f157a2fSChris Mason { 45993f157a2fSChris Mason struct extent_buffer *cur; 46003f157a2fSChris Mason struct btrfs_key found_key; 46013f157a2fSChris Mason int slot; 46029652480bSYan int sret; 46033f157a2fSChris Mason u32 nritems; 46043f157a2fSChris Mason int level; 46053f157a2fSChris Mason int ret = 1; 4606f98de9b9SFilipe Manana int keep_locks = path->keep_locks; 46073f157a2fSChris Mason 4608c922b016SStefan Roesch ASSERT(!path->nowait); 4609f98de9b9SFilipe Manana path->keep_locks = 1; 46103f157a2fSChris Mason again: 4611bd681513SChris Mason cur = btrfs_read_lock_root_node(root); 46123f157a2fSChris Mason level = btrfs_header_level(cur); 4613e02119d5SChris Mason WARN_ON(path->nodes[level]); 46143f157a2fSChris Mason path->nodes[level] = cur; 4615bd681513SChris Mason path->locks[level] = BTRFS_READ_LOCK; 46163f157a2fSChris Mason 46173f157a2fSChris Mason if (btrfs_header_generation(cur) < min_trans) { 46183f157a2fSChris Mason ret = 1; 46193f157a2fSChris Mason goto out; 46203f157a2fSChris Mason } 46213f157a2fSChris Mason while (1) { 46223f157a2fSChris Mason nritems = btrfs_header_nritems(cur); 46233f157a2fSChris Mason level = btrfs_header_level(cur); 4624fdf8d595SAnand Jain sret = btrfs_bin_search(cur, 0, min_key, &slot); 4625cbca7d59SFilipe Manana if (sret < 0) { 4626cbca7d59SFilipe Manana ret = sret; 4627cbca7d59SFilipe Manana goto out; 4628cbca7d59SFilipe Manana } 46293f157a2fSChris Mason 4630323ac95bSChris Mason /* at the lowest level, we're done, setup the path and exit */ 4631323ac95bSChris Mason if (level == path->lowest_level) { 4632e02119d5SChris Mason if (slot >= nritems) 4633e02119d5SChris Mason goto find_next_key; 46343f157a2fSChris Mason ret = 0; 46353f157a2fSChris Mason path->slots[level] = slot; 46363f157a2fSChris Mason btrfs_item_key_to_cpu(cur, &found_key, slot); 46373f157a2fSChris Mason goto out; 46383f157a2fSChris Mason } 46399652480bSYan if (sret && slot > 0) 46409652480bSYan slot--; 46413f157a2fSChris Mason /* 4642de78b51aSEric Sandeen * check this node pointer against the min_trans parameters. 4643260db43cSRandy Dunlap * If it is too old, skip to the next one. 46443f157a2fSChris Mason */ 46453f157a2fSChris Mason while (slot < nritems) { 46463f157a2fSChris Mason u64 gen; 4647e02119d5SChris Mason 46483f157a2fSChris Mason gen = btrfs_node_ptr_generation(cur, slot); 46493f157a2fSChris Mason if (gen < min_trans) { 46503f157a2fSChris Mason slot++; 46513f157a2fSChris Mason continue; 46523f157a2fSChris Mason } 46533f157a2fSChris Mason break; 46543f157a2fSChris Mason } 4655e02119d5SChris Mason find_next_key: 46563f157a2fSChris Mason /* 46573f157a2fSChris Mason * we didn't find a candidate key in this node, walk forward 46583f157a2fSChris Mason * and find another one 46593f157a2fSChris Mason */ 46603f157a2fSChris Mason if (slot >= nritems) { 4661e02119d5SChris Mason path->slots[level] = slot; 4662e02119d5SChris Mason sret = btrfs_find_next_key(root, path, min_key, level, 4663de78b51aSEric Sandeen min_trans); 4664e02119d5SChris Mason if (sret == 0) { 4665b3b4aa74SDavid Sterba btrfs_release_path(path); 46663f157a2fSChris Mason goto again; 46673f157a2fSChris Mason } else { 46683f157a2fSChris Mason goto out; 46693f157a2fSChris Mason } 46703f157a2fSChris Mason } 46713f157a2fSChris Mason /* save our key for returning back */ 46723f157a2fSChris Mason btrfs_node_key_to_cpu(cur, &found_key, slot); 46733f157a2fSChris Mason path->slots[level] = slot; 46743f157a2fSChris Mason if (level == path->lowest_level) { 46753f157a2fSChris Mason ret = 0; 46763f157a2fSChris Mason goto out; 46773f157a2fSChris Mason } 46784b231ae4SDavid Sterba cur = btrfs_read_node_slot(cur, slot); 4679fb770ae4SLiu Bo if (IS_ERR(cur)) { 4680fb770ae4SLiu Bo ret = PTR_ERR(cur); 4681fb770ae4SLiu Bo goto out; 4682fb770ae4SLiu Bo } 46833f157a2fSChris Mason 4684bd681513SChris Mason btrfs_tree_read_lock(cur); 4685b4ce94deSChris Mason 4686bd681513SChris Mason path->locks[level - 1] = BTRFS_READ_LOCK; 46873f157a2fSChris Mason path->nodes[level - 1] = cur; 4688f7c79f30SChris Mason unlock_up(path, level, 1, 0, NULL); 46893f157a2fSChris Mason } 46903f157a2fSChris Mason out: 4691f98de9b9SFilipe Manana path->keep_locks = keep_locks; 4692f98de9b9SFilipe Manana if (ret == 0) { 4693f98de9b9SFilipe Manana btrfs_unlock_up_safe(path, path->lowest_level + 1); 4694f98de9b9SFilipe Manana memcpy(min_key, &found_key, sizeof(found_key)); 4695f98de9b9SFilipe Manana } 46963f157a2fSChris Mason return ret; 46973f157a2fSChris Mason } 46983f157a2fSChris Mason 46993f157a2fSChris Mason /* 47003f157a2fSChris Mason * this is similar to btrfs_next_leaf, but does not try to preserve 47013f157a2fSChris Mason * and fixup the path. It looks for and returns the next key in the 4702de78b51aSEric Sandeen * tree based on the current path and the min_trans parameters. 47033f157a2fSChris Mason * 47043f157a2fSChris Mason * 0 is returned if another key is found, < 0 if there are any errors 47053f157a2fSChris Mason * and 1 is returned if there are no higher keys in the tree 47063f157a2fSChris Mason * 47073f157a2fSChris Mason * path->keep_locks should be set to 1 on the search made before 47083f157a2fSChris Mason * calling this function. 47093f157a2fSChris Mason */ 4710e7a84565SChris Mason int btrfs_find_next_key(struct btrfs_root *root, struct btrfs_path *path, 4711de78b51aSEric Sandeen struct btrfs_key *key, int level, u64 min_trans) 4712e7a84565SChris Mason { 4713e7a84565SChris Mason int slot; 4714e7a84565SChris Mason struct extent_buffer *c; 4715e7a84565SChris Mason 47166a9fb468SJosef Bacik WARN_ON(!path->keep_locks && !path->skip_locking); 4717e7a84565SChris Mason while (level < BTRFS_MAX_LEVEL) { 4718e7a84565SChris Mason if (!path->nodes[level]) 4719e7a84565SChris Mason return 1; 4720e7a84565SChris Mason 4721e7a84565SChris Mason slot = path->slots[level] + 1; 4722e7a84565SChris Mason c = path->nodes[level]; 47233f157a2fSChris Mason next: 4724e7a84565SChris Mason if (slot >= btrfs_header_nritems(c)) { 472533c66f43SYan Zheng int ret; 472633c66f43SYan Zheng int orig_lowest; 472733c66f43SYan Zheng struct btrfs_key cur_key; 472833c66f43SYan Zheng if (level + 1 >= BTRFS_MAX_LEVEL || 472933c66f43SYan Zheng !path->nodes[level + 1]) 4730e7a84565SChris Mason return 1; 473133c66f43SYan Zheng 47326a9fb468SJosef Bacik if (path->locks[level + 1] || path->skip_locking) { 473333c66f43SYan Zheng level++; 4734e7a84565SChris Mason continue; 4735e7a84565SChris Mason } 473633c66f43SYan Zheng 473733c66f43SYan Zheng slot = btrfs_header_nritems(c) - 1; 473833c66f43SYan Zheng if (level == 0) 473933c66f43SYan Zheng btrfs_item_key_to_cpu(c, &cur_key, slot); 474033c66f43SYan Zheng else 474133c66f43SYan Zheng btrfs_node_key_to_cpu(c, &cur_key, slot); 474233c66f43SYan Zheng 474333c66f43SYan Zheng orig_lowest = path->lowest_level; 4744b3b4aa74SDavid Sterba btrfs_release_path(path); 474533c66f43SYan Zheng path->lowest_level = level; 474633c66f43SYan Zheng ret = btrfs_search_slot(NULL, root, &cur_key, path, 474733c66f43SYan Zheng 0, 0); 474833c66f43SYan Zheng path->lowest_level = orig_lowest; 474933c66f43SYan Zheng if (ret < 0) 475033c66f43SYan Zheng return ret; 475133c66f43SYan Zheng 475233c66f43SYan Zheng c = path->nodes[level]; 475333c66f43SYan Zheng slot = path->slots[level]; 475433c66f43SYan Zheng if (ret == 0) 475533c66f43SYan Zheng slot++; 475633c66f43SYan Zheng goto next; 475733c66f43SYan Zheng } 475833c66f43SYan Zheng 4759e7a84565SChris Mason if (level == 0) 4760e7a84565SChris Mason btrfs_item_key_to_cpu(c, key, slot); 47613f157a2fSChris Mason else { 47623f157a2fSChris Mason u64 gen = btrfs_node_ptr_generation(c, slot); 47633f157a2fSChris Mason 47643f157a2fSChris Mason if (gen < min_trans) { 47653f157a2fSChris Mason slot++; 47663f157a2fSChris Mason goto next; 47673f157a2fSChris Mason } 4768e7a84565SChris Mason btrfs_node_key_to_cpu(c, key, slot); 47693f157a2fSChris Mason } 4770e7a84565SChris Mason return 0; 4771e7a84565SChris Mason } 4772e7a84565SChris Mason return 1; 4773e7a84565SChris Mason } 4774e7a84565SChris Mason 47753d7806ecSJan Schmidt int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path, 47763d7806ecSJan Schmidt u64 time_seq) 47773d7806ecSJan Schmidt { 4778d97e63b6SChris Mason int slot; 47798e73f275SChris Mason int level; 47805f39d397SChris Mason struct extent_buffer *c; 47818e73f275SChris Mason struct extent_buffer *next; 4782d96b3424SFilipe Manana struct btrfs_fs_info *fs_info = root->fs_info; 4783925baeddSChris Mason struct btrfs_key key; 4784d96b3424SFilipe Manana bool need_commit_sem = false; 4785925baeddSChris Mason u32 nritems; 4786925baeddSChris Mason int ret; 47870e46318dSJosef Bacik int i; 4788925baeddSChris Mason 4789bdcdd86cSFilipe Manana /* 4790bdcdd86cSFilipe Manana * The nowait semantics are used only for write paths, where we don't 4791bdcdd86cSFilipe Manana * use the tree mod log and sequence numbers. 4792bdcdd86cSFilipe Manana */ 4793bdcdd86cSFilipe Manana if (time_seq) 4794c922b016SStefan Roesch ASSERT(!path->nowait); 4795c922b016SStefan Roesch 4796925baeddSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 4797d397712bSChris Mason if (nritems == 0) 4798925baeddSChris Mason return 1; 4799925baeddSChris Mason 48008e73f275SChris Mason btrfs_item_key_to_cpu(path->nodes[0], &key, nritems - 1); 48018e73f275SChris Mason again: 48028e73f275SChris Mason level = 1; 48038e73f275SChris Mason next = NULL; 4804b3b4aa74SDavid Sterba btrfs_release_path(path); 48058e73f275SChris Mason 4806a2135011SChris Mason path->keep_locks = 1; 48078e73f275SChris Mason 4808d96b3424SFilipe Manana if (time_seq) { 48093d7806ecSJan Schmidt ret = btrfs_search_old_slot(root, &key, path, time_seq); 4810d96b3424SFilipe Manana } else { 4811d96b3424SFilipe Manana if (path->need_commit_sem) { 4812d96b3424SFilipe Manana path->need_commit_sem = 0; 4813d96b3424SFilipe Manana need_commit_sem = true; 4814bdcdd86cSFilipe Manana if (path->nowait) { 4815bdcdd86cSFilipe Manana if (!down_read_trylock(&fs_info->commit_root_sem)) { 4816bdcdd86cSFilipe Manana ret = -EAGAIN; 4817bdcdd86cSFilipe Manana goto done; 4818bdcdd86cSFilipe Manana } 4819bdcdd86cSFilipe Manana } else { 4820d96b3424SFilipe Manana down_read(&fs_info->commit_root_sem); 4821d96b3424SFilipe Manana } 4822bdcdd86cSFilipe Manana } 4823925baeddSChris Mason ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 4824d96b3424SFilipe Manana } 4825925baeddSChris Mason path->keep_locks = 0; 4826925baeddSChris Mason 4827925baeddSChris Mason if (ret < 0) 4828d96b3424SFilipe Manana goto done; 4829925baeddSChris Mason 4830a2135011SChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 4831168fd7d2SChris Mason /* 4832168fd7d2SChris Mason * by releasing the path above we dropped all our locks. A balance 4833168fd7d2SChris Mason * could have added more items next to the key that used to be 4834168fd7d2SChris Mason * at the very end of the block. So, check again here and 4835168fd7d2SChris Mason * advance the path if there are now more items available. 4836168fd7d2SChris Mason */ 4837a2135011SChris Mason if (nritems > 0 && path->slots[0] < nritems - 1) { 4838e457afecSYan Zheng if (ret == 0) 4839168fd7d2SChris Mason path->slots[0]++; 48408e73f275SChris Mason ret = 0; 4841925baeddSChris Mason goto done; 4842925baeddSChris Mason } 48430b43e04fSLiu Bo /* 48440b43e04fSLiu Bo * So the above check misses one case: 48450b43e04fSLiu Bo * - after releasing the path above, someone has removed the item that 48460b43e04fSLiu Bo * used to be at the very end of the block, and balance between leafs 48470b43e04fSLiu Bo * gets another one with bigger key.offset to replace it. 48480b43e04fSLiu Bo * 48490b43e04fSLiu Bo * This one should be returned as well, or we can get leaf corruption 48500b43e04fSLiu Bo * later(esp. in __btrfs_drop_extents()). 48510b43e04fSLiu Bo * 48520b43e04fSLiu Bo * And a bit more explanation about this check, 48530b43e04fSLiu Bo * with ret > 0, the key isn't found, the path points to the slot 48540b43e04fSLiu Bo * where it should be inserted, so the path->slots[0] item must be the 48550b43e04fSLiu Bo * bigger one. 48560b43e04fSLiu Bo */ 48570b43e04fSLiu Bo if (nritems > 0 && ret > 0 && path->slots[0] == nritems - 1) { 48580b43e04fSLiu Bo ret = 0; 48590b43e04fSLiu Bo goto done; 48600b43e04fSLiu Bo } 4861d97e63b6SChris Mason 4862234b63a0SChris Mason while (level < BTRFS_MAX_LEVEL) { 48638e73f275SChris Mason if (!path->nodes[level]) { 48648e73f275SChris Mason ret = 1; 48658e73f275SChris Mason goto done; 48668e73f275SChris Mason } 48675f39d397SChris Mason 4868d97e63b6SChris Mason slot = path->slots[level] + 1; 4869d97e63b6SChris Mason c = path->nodes[level]; 48705f39d397SChris Mason if (slot >= btrfs_header_nritems(c)) { 4871d97e63b6SChris Mason level++; 48728e73f275SChris Mason if (level == BTRFS_MAX_LEVEL) { 48738e73f275SChris Mason ret = 1; 48748e73f275SChris Mason goto done; 48758e73f275SChris Mason } 4876d97e63b6SChris Mason continue; 4877d97e63b6SChris Mason } 48785f39d397SChris Mason 48790e46318dSJosef Bacik 48800e46318dSJosef Bacik /* 48810e46318dSJosef Bacik * Our current level is where we're going to start from, and to 48820e46318dSJosef Bacik * make sure lockdep doesn't complain we need to drop our locks 48830e46318dSJosef Bacik * and nodes from 0 to our current level. 48840e46318dSJosef Bacik */ 48850e46318dSJosef Bacik for (i = 0; i < level; i++) { 48860e46318dSJosef Bacik if (path->locks[level]) { 48870e46318dSJosef Bacik btrfs_tree_read_unlock(path->nodes[i]); 48880e46318dSJosef Bacik path->locks[i] = 0; 48890e46318dSJosef Bacik } 48900e46318dSJosef Bacik free_extent_buffer(path->nodes[i]); 48910e46318dSJosef Bacik path->nodes[i] = NULL; 4892925baeddSChris Mason } 48935f39d397SChris Mason 48948e73f275SChris Mason next = c; 4895d07b8528SLiu Bo ret = read_block_for_search(root, path, &next, level, 4896cda79c54SDavid Sterba slot, &key); 4897bdcdd86cSFilipe Manana if (ret == -EAGAIN && !path->nowait) 48988e73f275SChris Mason goto again; 48995f39d397SChris Mason 490076a05b35SChris Mason if (ret < 0) { 4901b3b4aa74SDavid Sterba btrfs_release_path(path); 490276a05b35SChris Mason goto done; 490376a05b35SChris Mason } 490476a05b35SChris Mason 49055cd57b2cSChris Mason if (!path->skip_locking) { 4906bd681513SChris Mason ret = btrfs_try_tree_read_lock(next); 4907bdcdd86cSFilipe Manana if (!ret && path->nowait) { 4908bdcdd86cSFilipe Manana ret = -EAGAIN; 4909bdcdd86cSFilipe Manana goto done; 4910bdcdd86cSFilipe Manana } 4911d42244a0SJan Schmidt if (!ret && time_seq) { 4912d42244a0SJan Schmidt /* 4913d42244a0SJan Schmidt * If we don't get the lock, we may be racing 4914d42244a0SJan Schmidt * with push_leaf_left, holding that lock while 4915d42244a0SJan Schmidt * itself waiting for the leaf we've currently 4916d42244a0SJan Schmidt * locked. To solve this situation, we give up 4917d42244a0SJan Schmidt * on our lock and cycle. 4918d42244a0SJan Schmidt */ 4919cf538830SJan Schmidt free_extent_buffer(next); 4920d42244a0SJan Schmidt btrfs_release_path(path); 4921d42244a0SJan Schmidt cond_resched(); 4922d42244a0SJan Schmidt goto again; 4923d42244a0SJan Schmidt } 49240e46318dSJosef Bacik if (!ret) 49250e46318dSJosef Bacik btrfs_tree_read_lock(next); 4926bd681513SChris Mason } 4927d97e63b6SChris Mason break; 4928d97e63b6SChris Mason } 4929d97e63b6SChris Mason path->slots[level] = slot; 4930d97e63b6SChris Mason while (1) { 4931d97e63b6SChris Mason level--; 4932d97e63b6SChris Mason path->nodes[level] = next; 4933d97e63b6SChris Mason path->slots[level] = 0; 4934a74a4b97SChris Mason if (!path->skip_locking) 4935ffeb03cfSJosef Bacik path->locks[level] = BTRFS_READ_LOCK; 4936d97e63b6SChris Mason if (!level) 4937d97e63b6SChris Mason break; 4938b4ce94deSChris Mason 4939d07b8528SLiu Bo ret = read_block_for_search(root, path, &next, level, 4940cda79c54SDavid Sterba 0, &key); 4941bdcdd86cSFilipe Manana if (ret == -EAGAIN && !path->nowait) 49428e73f275SChris Mason goto again; 49438e73f275SChris Mason 494476a05b35SChris Mason if (ret < 0) { 4945b3b4aa74SDavid Sterba btrfs_release_path(path); 494676a05b35SChris Mason goto done; 494776a05b35SChris Mason } 494876a05b35SChris Mason 4949bdcdd86cSFilipe Manana if (!path->skip_locking) { 4950bdcdd86cSFilipe Manana if (path->nowait) { 4951bdcdd86cSFilipe Manana if (!btrfs_try_tree_read_lock(next)) { 4952bdcdd86cSFilipe Manana ret = -EAGAIN; 4953bdcdd86cSFilipe Manana goto done; 4954bdcdd86cSFilipe Manana } 4955bdcdd86cSFilipe Manana } else { 49560e46318dSJosef Bacik btrfs_tree_read_lock(next); 4957d97e63b6SChris Mason } 4958bdcdd86cSFilipe Manana } 4959bdcdd86cSFilipe Manana } 49608e73f275SChris Mason ret = 0; 4961925baeddSChris Mason done: 4962f7c79f30SChris Mason unlock_up(path, 0, 1, 0, NULL); 4963d96b3424SFilipe Manana if (need_commit_sem) { 4964d96b3424SFilipe Manana int ret2; 4965d96b3424SFilipe Manana 4966d96b3424SFilipe Manana path->need_commit_sem = 1; 4967d96b3424SFilipe Manana ret2 = finish_need_commit_sem_search(path); 4968d96b3424SFilipe Manana up_read(&fs_info->commit_root_sem); 4969d96b3424SFilipe Manana if (ret2) 4970d96b3424SFilipe Manana ret = ret2; 4971d96b3424SFilipe Manana } 49728e73f275SChris Mason 49738e73f275SChris Mason return ret; 4974d97e63b6SChris Mason } 49750b86a832SChris Mason 4976890d2b1aSJosef Bacik int btrfs_next_old_item(struct btrfs_root *root, struct btrfs_path *path, u64 time_seq) 4977890d2b1aSJosef Bacik { 4978890d2b1aSJosef Bacik path->slots[0]++; 4979890d2b1aSJosef Bacik if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) 4980890d2b1aSJosef Bacik return btrfs_next_old_leaf(root, path, time_seq); 4981890d2b1aSJosef Bacik return 0; 4982890d2b1aSJosef Bacik } 4983890d2b1aSJosef Bacik 49843f157a2fSChris Mason /* 49853f157a2fSChris Mason * this uses btrfs_prev_leaf to walk backwards in the tree, and keeps 49863f157a2fSChris Mason * searching until it gets past min_objectid or finds an item of 'type' 49873f157a2fSChris Mason * 49883f157a2fSChris Mason * returns 0 if something is found, 1 if nothing was found and < 0 on error 49893f157a2fSChris Mason */ 49900b86a832SChris Mason int btrfs_previous_item(struct btrfs_root *root, 49910b86a832SChris Mason struct btrfs_path *path, u64 min_objectid, 49920b86a832SChris Mason int type) 49930b86a832SChris Mason { 49940b86a832SChris Mason struct btrfs_key found_key; 49950b86a832SChris Mason struct extent_buffer *leaf; 4996e02119d5SChris Mason u32 nritems; 49970b86a832SChris Mason int ret; 49980b86a832SChris Mason 49990b86a832SChris Mason while (1) { 50000b86a832SChris Mason if (path->slots[0] == 0) { 50010b86a832SChris Mason ret = btrfs_prev_leaf(root, path); 50020b86a832SChris Mason if (ret != 0) 50030b86a832SChris Mason return ret; 50040b86a832SChris Mason } else { 50050b86a832SChris Mason path->slots[0]--; 50060b86a832SChris Mason } 50070b86a832SChris Mason leaf = path->nodes[0]; 5008e02119d5SChris Mason nritems = btrfs_header_nritems(leaf); 5009e02119d5SChris Mason if (nritems == 0) 5010e02119d5SChris Mason return 1; 5011e02119d5SChris Mason if (path->slots[0] == nritems) 5012e02119d5SChris Mason path->slots[0]--; 5013e02119d5SChris Mason 50140b86a832SChris Mason btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 5015e02119d5SChris Mason if (found_key.objectid < min_objectid) 5016e02119d5SChris Mason break; 50170a4eefbbSYan Zheng if (found_key.type == type) 50180a4eefbbSYan Zheng return 0; 5019e02119d5SChris Mason if (found_key.objectid == min_objectid && 5020e02119d5SChris Mason found_key.type < type) 5021e02119d5SChris Mason break; 50220b86a832SChris Mason } 50230b86a832SChris Mason return 1; 50240b86a832SChris Mason } 5025ade2e0b3SWang Shilong 5026ade2e0b3SWang Shilong /* 5027ade2e0b3SWang Shilong * search in extent tree to find a previous Metadata/Data extent item with 5028ade2e0b3SWang Shilong * min objecitd. 5029ade2e0b3SWang Shilong * 5030ade2e0b3SWang Shilong * returns 0 if something is found, 1 if nothing was found and < 0 on error 5031ade2e0b3SWang Shilong */ 5032ade2e0b3SWang Shilong int btrfs_previous_extent_item(struct btrfs_root *root, 5033ade2e0b3SWang Shilong struct btrfs_path *path, u64 min_objectid) 5034ade2e0b3SWang Shilong { 5035ade2e0b3SWang Shilong struct btrfs_key found_key; 5036ade2e0b3SWang Shilong struct extent_buffer *leaf; 5037ade2e0b3SWang Shilong u32 nritems; 5038ade2e0b3SWang Shilong int ret; 5039ade2e0b3SWang Shilong 5040ade2e0b3SWang Shilong while (1) { 5041ade2e0b3SWang Shilong if (path->slots[0] == 0) { 5042ade2e0b3SWang Shilong ret = btrfs_prev_leaf(root, path); 5043ade2e0b3SWang Shilong if (ret != 0) 5044ade2e0b3SWang Shilong return ret; 5045ade2e0b3SWang Shilong } else { 5046ade2e0b3SWang Shilong path->slots[0]--; 5047ade2e0b3SWang Shilong } 5048ade2e0b3SWang Shilong leaf = path->nodes[0]; 5049ade2e0b3SWang Shilong nritems = btrfs_header_nritems(leaf); 5050ade2e0b3SWang Shilong if (nritems == 0) 5051ade2e0b3SWang Shilong return 1; 5052ade2e0b3SWang Shilong if (path->slots[0] == nritems) 5053ade2e0b3SWang Shilong path->slots[0]--; 5054ade2e0b3SWang Shilong 5055ade2e0b3SWang Shilong btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 5056ade2e0b3SWang Shilong if (found_key.objectid < min_objectid) 5057ade2e0b3SWang Shilong break; 5058ade2e0b3SWang Shilong if (found_key.type == BTRFS_EXTENT_ITEM_KEY || 5059ade2e0b3SWang Shilong found_key.type == BTRFS_METADATA_ITEM_KEY) 5060ade2e0b3SWang Shilong return 0; 5061ade2e0b3SWang Shilong if (found_key.objectid == min_objectid && 5062ade2e0b3SWang Shilong found_key.type < BTRFS_EXTENT_ITEM_KEY) 5063ade2e0b3SWang Shilong break; 5064ade2e0b3SWang Shilong } 5065ade2e0b3SWang Shilong return 1; 5066ade2e0b3SWang Shilong } 5067226463d7SJosef Bacik 5068226463d7SJosef Bacik int __init btrfs_ctree_init(void) 5069226463d7SJosef Bacik { 5070226463d7SJosef Bacik btrfs_path_cachep = kmem_cache_create("btrfs_path", 5071226463d7SJosef Bacik sizeof(struct btrfs_path), 0, 5072226463d7SJosef Bacik SLAB_MEM_SPREAD, NULL); 5073226463d7SJosef Bacik if (!btrfs_path_cachep) 5074226463d7SJosef Bacik return -ENOMEM; 5075226463d7SJosef Bacik return 0; 5076226463d7SJosef Bacik } 5077226463d7SJosef Bacik 5078226463d7SJosef Bacik void __cold btrfs_ctree_exit(void) 5079226463d7SJosef Bacik { 5080226463d7SJosef Bacik kmem_cache_destroy(btrfs_path_cachep); 5081226463d7SJosef Bacik } 5082