1c1d7c514SDavid Sterba // SPDX-License-Identifier: GPL-2.0 26cbd5570SChris Mason /* 3d352ac68SChris Mason * Copyright (C) 2007,2008 Oracle. All rights reserved. 46cbd5570SChris Mason */ 56cbd5570SChris Mason 6a6b6e75eSChris Mason #include <linux/sched.h> 75a0e3ad6STejun Heo #include <linux/slab.h> 8bd989ba3SJan Schmidt #include <linux/rbtree.h> 9adf02123SDavid Sterba #include <linux/mm.h> 10e41d12f5SChristoph Hellwig #include <linux/error-injection.h> 11eb60ceacSChris Mason #include "ctree.h" 12eb60ceacSChris Mason #include "disk-io.h" 137f5c1516SChris Mason #include "transaction.h" 145f39d397SChris Mason #include "print-tree.h" 15925baeddSChris Mason #include "locking.h" 16de37aa51SNikolay Borisov #include "volumes.h" 17f616f5cdSQu Wenruo #include "qgroup.h" 18f3a84ccdSFilipe Manana #include "tree-mod-log.h" 1988c602abSQu Wenruo #include "tree-checker.h" 209a8dd150SChris Mason 21e089f05cSChris Mason static int split_node(struct btrfs_trans_handle *trans, struct btrfs_root 22e089f05cSChris Mason *root, struct btrfs_path *path, int level); 23310712b2SOmar Sandoval static int split_leaf(struct btrfs_trans_handle *trans, struct btrfs_root *root, 24310712b2SOmar Sandoval const struct btrfs_key *ins_key, struct btrfs_path *path, 25310712b2SOmar Sandoval int data_size, int extend); 265f39d397SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 272ff7e61eSJeff Mahoney struct extent_buffer *dst, 28971a1f66SChris Mason struct extent_buffer *src, int empty); 295f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 305f39d397SChris Mason struct extent_buffer *dst_buf, 315f39d397SChris Mason struct extent_buffer *src_buf); 32afe5fea7STsutomu Itoh static void del_ptr(struct btrfs_root *root, struct btrfs_path *path, 33afe5fea7STsutomu Itoh int level, int slot); 34d97e63b6SChris Mason 35af024ed2SJohannes Thumshirn static const struct btrfs_csums { 36af024ed2SJohannes Thumshirn u16 size; 3759a0fcdbSDavid Sterba const char name[10]; 3859a0fcdbSDavid Sterba const char driver[12]; 39af024ed2SJohannes Thumshirn } btrfs_csums[] = { 40af024ed2SJohannes Thumshirn [BTRFS_CSUM_TYPE_CRC32] = { .size = 4, .name = "crc32c" }, 413951e7f0SJohannes Thumshirn [BTRFS_CSUM_TYPE_XXHASH] = { .size = 8, .name = "xxhash64" }, 423831bf00SJohannes Thumshirn [BTRFS_CSUM_TYPE_SHA256] = { .size = 32, .name = "sha256" }, 43352ae07bSDavid Sterba [BTRFS_CSUM_TYPE_BLAKE2] = { .size = 32, .name = "blake2b", 44352ae07bSDavid Sterba .driver = "blake2b-256" }, 45af024ed2SJohannes Thumshirn }; 46af024ed2SJohannes Thumshirn 47af024ed2SJohannes Thumshirn int btrfs_super_csum_size(const struct btrfs_super_block *s) 48af024ed2SJohannes Thumshirn { 49af024ed2SJohannes Thumshirn u16 t = btrfs_super_csum_type(s); 50af024ed2SJohannes Thumshirn /* 51af024ed2SJohannes Thumshirn * csum type is validated at mount time 52af024ed2SJohannes Thumshirn */ 53af024ed2SJohannes Thumshirn return btrfs_csums[t].size; 54af024ed2SJohannes Thumshirn } 55af024ed2SJohannes Thumshirn 56af024ed2SJohannes Thumshirn const char *btrfs_super_csum_name(u16 csum_type) 57af024ed2SJohannes Thumshirn { 58af024ed2SJohannes Thumshirn /* csum type is validated at mount time */ 59af024ed2SJohannes Thumshirn return btrfs_csums[csum_type].name; 60af024ed2SJohannes Thumshirn } 61af024ed2SJohannes Thumshirn 62b4e967beSDavid Sterba /* 63b4e967beSDavid Sterba * Return driver name if defined, otherwise the name that's also a valid driver 64b4e967beSDavid Sterba * name 65b4e967beSDavid Sterba */ 66b4e967beSDavid Sterba const char *btrfs_super_csum_driver(u16 csum_type) 67b4e967beSDavid Sterba { 68b4e967beSDavid Sterba /* csum type is validated at mount time */ 6959a0fcdbSDavid Sterba return btrfs_csums[csum_type].driver[0] ? 7059a0fcdbSDavid Sterba btrfs_csums[csum_type].driver : 71b4e967beSDavid Sterba btrfs_csums[csum_type].name; 72b4e967beSDavid Sterba } 73b4e967beSDavid Sterba 74604997b4SDavid Sterba size_t __attribute_const__ btrfs_get_num_csums(void) 75f7cea56cSDavid Sterba { 76f7cea56cSDavid Sterba return ARRAY_SIZE(btrfs_csums); 77f7cea56cSDavid Sterba } 78f7cea56cSDavid Sterba 792c90e5d6SChris Mason struct btrfs_path *btrfs_alloc_path(void) 802c90e5d6SChris Mason { 81e2c89907SMasahiro Yamada return kmem_cache_zalloc(btrfs_path_cachep, GFP_NOFS); 822c90e5d6SChris Mason } 832c90e5d6SChris Mason 84d352ac68SChris Mason /* this also releases the path */ 852c90e5d6SChris Mason void btrfs_free_path(struct btrfs_path *p) 862c90e5d6SChris Mason { 87ff175d57SJesper Juhl if (!p) 88ff175d57SJesper Juhl return; 89b3b4aa74SDavid Sterba btrfs_release_path(p); 902c90e5d6SChris Mason kmem_cache_free(btrfs_path_cachep, p); 912c90e5d6SChris Mason } 922c90e5d6SChris Mason 93d352ac68SChris Mason /* 94d352ac68SChris Mason * path release drops references on the extent buffers in the path 95d352ac68SChris Mason * and it drops any locks held by this path 96d352ac68SChris Mason * 97d352ac68SChris Mason * It is safe to call this on paths that no locks or extent buffers held. 98d352ac68SChris Mason */ 99b3b4aa74SDavid Sterba noinline void btrfs_release_path(struct btrfs_path *p) 100eb60ceacSChris Mason { 101eb60ceacSChris Mason int i; 102a2135011SChris Mason 103234b63a0SChris Mason for (i = 0; i < BTRFS_MAX_LEVEL; i++) { 1043f157a2fSChris Mason p->slots[i] = 0; 105eb60ceacSChris Mason if (!p->nodes[i]) 106925baeddSChris Mason continue; 107925baeddSChris Mason if (p->locks[i]) { 108bd681513SChris Mason btrfs_tree_unlock_rw(p->nodes[i], p->locks[i]); 109925baeddSChris Mason p->locks[i] = 0; 110925baeddSChris Mason } 1115f39d397SChris Mason free_extent_buffer(p->nodes[i]); 1123f157a2fSChris Mason p->nodes[i] = NULL; 113eb60ceacSChris Mason } 114eb60ceacSChris Mason } 115eb60ceacSChris Mason 116d352ac68SChris Mason /* 117d352ac68SChris Mason * safely gets a reference on the root node of a tree. A lock 118d352ac68SChris Mason * is not taken, so a concurrent writer may put a different node 119d352ac68SChris Mason * at the root of the tree. See btrfs_lock_root_node for the 120d352ac68SChris Mason * looping required. 121d352ac68SChris Mason * 122d352ac68SChris Mason * The extent buffer returned by this has a reference taken, so 123d352ac68SChris Mason * it won't disappear. It may stop being the root of the tree 124d352ac68SChris Mason * at any time because there are no locks held. 125d352ac68SChris Mason */ 126925baeddSChris Mason struct extent_buffer *btrfs_root_node(struct btrfs_root *root) 127925baeddSChris Mason { 128925baeddSChris Mason struct extent_buffer *eb; 129240f62c8SChris Mason 1303083ee2eSJosef Bacik while (1) { 131240f62c8SChris Mason rcu_read_lock(); 132240f62c8SChris Mason eb = rcu_dereference(root->node); 1333083ee2eSJosef Bacik 1343083ee2eSJosef Bacik /* 1353083ee2eSJosef Bacik * RCU really hurts here, we could free up the root node because 13601327610SNicholas D Steeves * it was COWed but we may not get the new root node yet so do 1373083ee2eSJosef Bacik * the inc_not_zero dance and if it doesn't work then 1383083ee2eSJosef Bacik * synchronize_rcu and try again. 1393083ee2eSJosef Bacik */ 1403083ee2eSJosef Bacik if (atomic_inc_not_zero(&eb->refs)) { 141240f62c8SChris Mason rcu_read_unlock(); 1423083ee2eSJosef Bacik break; 1433083ee2eSJosef Bacik } 1443083ee2eSJosef Bacik rcu_read_unlock(); 1453083ee2eSJosef Bacik synchronize_rcu(); 1463083ee2eSJosef Bacik } 147925baeddSChris Mason return eb; 148925baeddSChris Mason } 149925baeddSChris Mason 15092a7cc42SQu Wenruo /* 15192a7cc42SQu Wenruo * Cowonly root (not-shareable trees, everything not subvolume or reloc roots), 15292a7cc42SQu Wenruo * just get put onto a simple dirty list. Transaction walks this list to make 15392a7cc42SQu Wenruo * sure they get properly updated on disk. 154d352ac68SChris Mason */ 1550b86a832SChris Mason static void add_root_to_dirty_list(struct btrfs_root *root) 1560b86a832SChris Mason { 1570b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 1580b246afaSJeff Mahoney 159e7070be1SJosef Bacik if (test_bit(BTRFS_ROOT_DIRTY, &root->state) || 160e7070be1SJosef Bacik !test_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state)) 161e7070be1SJosef Bacik return; 162e7070be1SJosef Bacik 1630b246afaSJeff Mahoney spin_lock(&fs_info->trans_lock); 164e7070be1SJosef Bacik if (!test_and_set_bit(BTRFS_ROOT_DIRTY, &root->state)) { 165e7070be1SJosef Bacik /* Want the extent tree to be the last on the list */ 1664fd786e6SMisono Tomohiro if (root->root_key.objectid == BTRFS_EXTENT_TREE_OBJECTID) 167e7070be1SJosef Bacik list_move_tail(&root->dirty_list, 1680b246afaSJeff Mahoney &fs_info->dirty_cowonly_roots); 169e7070be1SJosef Bacik else 170e7070be1SJosef Bacik list_move(&root->dirty_list, 1710b246afaSJeff Mahoney &fs_info->dirty_cowonly_roots); 1720b86a832SChris Mason } 1730b246afaSJeff Mahoney spin_unlock(&fs_info->trans_lock); 1740b86a832SChris Mason } 1750b86a832SChris Mason 176d352ac68SChris Mason /* 177d352ac68SChris Mason * used by snapshot creation to make a copy of a root for a tree with 178d352ac68SChris Mason * a given objectid. The buffer with the new root node is returned in 179d352ac68SChris Mason * cow_ret, and this func returns zero on success or a negative error code. 180d352ac68SChris Mason */ 181be20aa9dSChris Mason int btrfs_copy_root(struct btrfs_trans_handle *trans, 182be20aa9dSChris Mason struct btrfs_root *root, 183be20aa9dSChris Mason struct extent_buffer *buf, 184be20aa9dSChris Mason struct extent_buffer **cow_ret, u64 new_root_objectid) 185be20aa9dSChris Mason { 1860b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 187be20aa9dSChris Mason struct extent_buffer *cow; 188be20aa9dSChris Mason int ret = 0; 189be20aa9dSChris Mason int level; 1905d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 191be20aa9dSChris Mason 19292a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 1930b246afaSJeff Mahoney trans->transid != fs_info->running_transaction->transid); 19492a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 19527cdeb70SMiao Xie trans->transid != root->last_trans); 196be20aa9dSChris Mason 197be20aa9dSChris Mason level = btrfs_header_level(buf); 1985d4f98a2SYan Zheng if (level == 0) 1995d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 2005d4f98a2SYan Zheng else 2015d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 20231840ae1SZheng Yan 2034d75f8a9SDavid Sterba cow = btrfs_alloc_tree_block(trans, root, 0, new_root_objectid, 204cf6f34aaSJosef Bacik &disk_key, level, buf->start, 0, 205cf6f34aaSJosef Bacik BTRFS_NESTING_NEW_ROOT); 2065d4f98a2SYan Zheng if (IS_ERR(cow)) 207be20aa9dSChris Mason return PTR_ERR(cow); 208be20aa9dSChris Mason 20958e8012cSDavid Sterba copy_extent_buffer_full(cow, buf); 210be20aa9dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 211be20aa9dSChris Mason btrfs_set_header_generation(cow, trans->transid); 2125d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 2135d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 2145d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 2155d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 2165d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 2175d4f98a2SYan Zheng else 218be20aa9dSChris Mason btrfs_set_header_owner(cow, new_root_objectid); 219be20aa9dSChris Mason 220de37aa51SNikolay Borisov write_extent_buffer_fsid(cow, fs_info->fs_devices->metadata_uuid); 2212b82032cSYan Zheng 222be20aa9dSChris Mason WARN_ON(btrfs_header_generation(buf) > trans->transid); 2235d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 224e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 2255d4f98a2SYan Zheng else 226e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 227867ed321SJosef Bacik if (ret) { 22872c9925fSFilipe Manana btrfs_tree_unlock(cow); 22972c9925fSFilipe Manana free_extent_buffer(cow); 230867ed321SJosef Bacik btrfs_abort_transaction(trans, ret); 231be20aa9dSChris Mason return ret; 232867ed321SJosef Bacik } 233be20aa9dSChris Mason 234be20aa9dSChris Mason btrfs_mark_buffer_dirty(cow); 235be20aa9dSChris Mason *cow_ret = cow; 236be20aa9dSChris Mason return 0; 237be20aa9dSChris Mason } 238be20aa9dSChris Mason 239d352ac68SChris Mason /* 2405d4f98a2SYan Zheng * check if the tree block can be shared by multiple trees 2415d4f98a2SYan Zheng */ 2425d4f98a2SYan Zheng int btrfs_block_can_be_shared(struct btrfs_root *root, 2435d4f98a2SYan Zheng struct extent_buffer *buf) 2445d4f98a2SYan Zheng { 2455d4f98a2SYan Zheng /* 24692a7cc42SQu Wenruo * Tree blocks not in shareable trees and tree roots are never shared. 24792a7cc42SQu Wenruo * If a block was allocated after the last snapshot and the block was 24892a7cc42SQu Wenruo * not allocated by tree relocation, we know the block is not shared. 2495d4f98a2SYan Zheng */ 25092a7cc42SQu Wenruo if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 2515d4f98a2SYan Zheng buf != root->node && buf != root->commit_root && 2525d4f98a2SYan Zheng (btrfs_header_generation(buf) <= 2535d4f98a2SYan Zheng btrfs_root_last_snapshot(&root->root_item) || 2545d4f98a2SYan Zheng btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC))) 2555d4f98a2SYan Zheng return 1; 256a79865c6SNikolay Borisov 2575d4f98a2SYan Zheng return 0; 2585d4f98a2SYan Zheng } 2595d4f98a2SYan Zheng 2605d4f98a2SYan Zheng static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans, 2615d4f98a2SYan Zheng struct btrfs_root *root, 2625d4f98a2SYan Zheng struct extent_buffer *buf, 263f0486c68SYan, Zheng struct extent_buffer *cow, 264f0486c68SYan, Zheng int *last_ref) 2655d4f98a2SYan Zheng { 2660b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 2675d4f98a2SYan Zheng u64 refs; 2685d4f98a2SYan Zheng u64 owner; 2695d4f98a2SYan Zheng u64 flags; 2705d4f98a2SYan Zheng u64 new_flags = 0; 2715d4f98a2SYan Zheng int ret; 2725d4f98a2SYan Zheng 2735d4f98a2SYan Zheng /* 2745d4f98a2SYan Zheng * Backrefs update rules: 2755d4f98a2SYan Zheng * 2765d4f98a2SYan Zheng * Always use full backrefs for extent pointers in tree block 2775d4f98a2SYan Zheng * allocated by tree relocation. 2785d4f98a2SYan Zheng * 2795d4f98a2SYan Zheng * If a shared tree block is no longer referenced by its owner 2805d4f98a2SYan Zheng * tree (btrfs_header_owner(buf) == root->root_key.objectid), 2815d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 2825d4f98a2SYan Zheng * 2835d4f98a2SYan Zheng * If a tree block is been relocating 2845d4f98a2SYan Zheng * (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID), 2855d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 2865d4f98a2SYan Zheng * The reason for this is some operations (such as drop tree) 2875d4f98a2SYan Zheng * are only allowed for blocks use full backrefs. 2885d4f98a2SYan Zheng */ 2895d4f98a2SYan Zheng 2905d4f98a2SYan Zheng if (btrfs_block_can_be_shared(root, buf)) { 2912ff7e61eSJeff Mahoney ret = btrfs_lookup_extent_info(trans, fs_info, buf->start, 2923173a18fSJosef Bacik btrfs_header_level(buf), 1, 2933173a18fSJosef Bacik &refs, &flags); 294be1a5564SMark Fasheh if (ret) 295be1a5564SMark Fasheh return ret; 296e5df9573SMark Fasheh if (refs == 0) { 297e5df9573SMark Fasheh ret = -EROFS; 2980b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 299e5df9573SMark Fasheh return ret; 300e5df9573SMark Fasheh } 3015d4f98a2SYan Zheng } else { 3025d4f98a2SYan Zheng refs = 1; 3035d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 3045d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 3055d4f98a2SYan Zheng flags = BTRFS_BLOCK_FLAG_FULL_BACKREF; 3065d4f98a2SYan Zheng else 3075d4f98a2SYan Zheng flags = 0; 3085d4f98a2SYan Zheng } 3095d4f98a2SYan Zheng 3105d4f98a2SYan Zheng owner = btrfs_header_owner(buf); 3115d4f98a2SYan Zheng BUG_ON(owner == BTRFS_TREE_RELOC_OBJECTID && 3125d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)); 3135d4f98a2SYan Zheng 3145d4f98a2SYan Zheng if (refs > 1) { 3155d4f98a2SYan Zheng if ((owner == root->root_key.objectid || 3165d4f98a2SYan Zheng root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && 3175d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) { 318e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, buf, 1); 319692826b2SJeff Mahoney if (ret) 320692826b2SJeff Mahoney return ret; 3215d4f98a2SYan Zheng 3225d4f98a2SYan Zheng if (root->root_key.objectid == 3235d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) { 324e339a6b0SJosef Bacik ret = btrfs_dec_ref(trans, root, buf, 0); 325692826b2SJeff Mahoney if (ret) 326692826b2SJeff Mahoney return ret; 327e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 328692826b2SJeff Mahoney if (ret) 329692826b2SJeff Mahoney return ret; 3305d4f98a2SYan Zheng } 3315d4f98a2SYan Zheng new_flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF; 3325d4f98a2SYan Zheng } else { 3335d4f98a2SYan Zheng 3345d4f98a2SYan Zheng if (root->root_key.objectid == 3355d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 336e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 3375d4f98a2SYan Zheng else 338e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 339692826b2SJeff Mahoney if (ret) 340692826b2SJeff Mahoney return ret; 3415d4f98a2SYan Zheng } 3425d4f98a2SYan Zheng if (new_flags != 0) { 343b1c79e09SJosef Bacik int level = btrfs_header_level(buf); 344b1c79e09SJosef Bacik 34542c9d0b5SDavid Sterba ret = btrfs_set_disk_extent_flags(trans, buf, 3462fe6a5a1SDavid Sterba new_flags, level); 347be1a5564SMark Fasheh if (ret) 348be1a5564SMark Fasheh return ret; 3495d4f98a2SYan Zheng } 3505d4f98a2SYan Zheng } else { 3515d4f98a2SYan Zheng if (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) { 3525d4f98a2SYan Zheng if (root->root_key.objectid == 3535d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 354e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 3555d4f98a2SYan Zheng else 356e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 357692826b2SJeff Mahoney if (ret) 358692826b2SJeff Mahoney return ret; 359e339a6b0SJosef Bacik ret = btrfs_dec_ref(trans, root, buf, 1); 360692826b2SJeff Mahoney if (ret) 361692826b2SJeff Mahoney return ret; 3625d4f98a2SYan Zheng } 3636a884d7dSDavid Sterba btrfs_clean_tree_block(buf); 364f0486c68SYan, Zheng *last_ref = 1; 3655d4f98a2SYan Zheng } 3665d4f98a2SYan Zheng return 0; 3675d4f98a2SYan Zheng } 3685d4f98a2SYan Zheng 3695d4f98a2SYan Zheng /* 370d397712bSChris Mason * does the dirty work in cow of a single block. The parent block (if 371d397712bSChris Mason * supplied) is updated to point to the new cow copy. The new buffer is marked 372d397712bSChris Mason * dirty and returned locked. If you modify the block it needs to be marked 373d397712bSChris Mason * dirty again. 374d352ac68SChris Mason * 375d352ac68SChris Mason * search_start -- an allocation hint for the new block 376d352ac68SChris Mason * 377d397712bSChris Mason * empty_size -- a hint that you plan on doing more cow. This is the size in 378d397712bSChris Mason * bytes the allocator should try to find free next to the block it returns. 379d397712bSChris Mason * This is just a hint and may be ignored by the allocator. 380d352ac68SChris Mason */ 381d397712bSChris Mason static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans, 3825f39d397SChris Mason struct btrfs_root *root, 3835f39d397SChris Mason struct extent_buffer *buf, 3845f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 3855f39d397SChris Mason struct extent_buffer **cow_ret, 3869631e4ccSJosef Bacik u64 search_start, u64 empty_size, 3879631e4ccSJosef Bacik enum btrfs_lock_nesting nest) 3886702ed49SChris Mason { 3890b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 3905d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 3915f39d397SChris Mason struct extent_buffer *cow; 392be1a5564SMark Fasheh int level, ret; 393f0486c68SYan, Zheng int last_ref = 0; 394925baeddSChris Mason int unlock_orig = 0; 3950f5053ebSGoldwyn Rodrigues u64 parent_start = 0; 3966702ed49SChris Mason 397925baeddSChris Mason if (*cow_ret == buf) 398925baeddSChris Mason unlock_orig = 1; 399925baeddSChris Mason 40049d0c642SFilipe Manana btrfs_assert_tree_write_locked(buf); 401925baeddSChris Mason 40292a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 4030b246afaSJeff Mahoney trans->transid != fs_info->running_transaction->transid); 40492a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 40527cdeb70SMiao Xie trans->transid != root->last_trans); 4065f39d397SChris Mason 4077bb86316SChris Mason level = btrfs_header_level(buf); 40831840ae1SZheng Yan 4095d4f98a2SYan Zheng if (level == 0) 4105d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 4115d4f98a2SYan Zheng else 4125d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 4135d4f98a2SYan Zheng 4140f5053ebSGoldwyn Rodrigues if ((root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && parent) 4155d4f98a2SYan Zheng parent_start = parent->start; 4165d4f98a2SYan Zheng 41779bd3712SFilipe Manana cow = btrfs_alloc_tree_block(trans, root, parent_start, 41879bd3712SFilipe Manana root->root_key.objectid, &disk_key, level, 41979bd3712SFilipe Manana search_start, empty_size, nest); 4206702ed49SChris Mason if (IS_ERR(cow)) 4216702ed49SChris Mason return PTR_ERR(cow); 4226702ed49SChris Mason 423b4ce94deSChris Mason /* cow is set to blocking by btrfs_init_new_buffer */ 424b4ce94deSChris Mason 42558e8012cSDavid Sterba copy_extent_buffer_full(cow, buf); 426db94535dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 4275f39d397SChris Mason btrfs_set_header_generation(cow, trans->transid); 4285d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 4295d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 4305d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 4315d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) 4325d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 4335d4f98a2SYan Zheng else 4345f39d397SChris Mason btrfs_set_header_owner(cow, root->root_key.objectid); 4356702ed49SChris Mason 436de37aa51SNikolay Borisov write_extent_buffer_fsid(cow, fs_info->fs_devices->metadata_uuid); 4372b82032cSYan Zheng 438be1a5564SMark Fasheh ret = update_ref_for_cow(trans, root, buf, cow, &last_ref); 439b68dc2a9SMark Fasheh if (ret) { 440572c83acSJosef Bacik btrfs_tree_unlock(cow); 441572c83acSJosef Bacik free_extent_buffer(cow); 44266642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 443b68dc2a9SMark Fasheh return ret; 444b68dc2a9SMark Fasheh } 4451a40e23bSZheng Yan 44692a7cc42SQu Wenruo if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) { 44783d4cfd4SJosef Bacik ret = btrfs_reloc_cow_block(trans, root, buf, cow); 44893314e3bSZhaolei if (ret) { 449572c83acSJosef Bacik btrfs_tree_unlock(cow); 450572c83acSJosef Bacik free_extent_buffer(cow); 45166642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 45283d4cfd4SJosef Bacik return ret; 45383d4cfd4SJosef Bacik } 45493314e3bSZhaolei } 4553fd0a558SYan, Zheng 4566702ed49SChris Mason if (buf == root->node) { 457925baeddSChris Mason WARN_ON(parent && parent != buf); 4585d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 4595d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 4605d4f98a2SYan Zheng parent_start = buf->start; 461925baeddSChris Mason 46267439dadSDavid Sterba atomic_inc(&cow->refs); 463406808abSFilipe Manana ret = btrfs_tree_mod_log_insert_root(root->node, cow, true); 464d9d19a01SDavid Sterba BUG_ON(ret < 0); 465240f62c8SChris Mason rcu_assign_pointer(root->node, cow); 466925baeddSChris Mason 4677a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), buf, 4687a163608SFilipe Manana parent_start, last_ref); 4695f39d397SChris Mason free_extent_buffer(buf); 4700b86a832SChris Mason add_root_to_dirty_list(root); 4716702ed49SChris Mason } else { 4725d4f98a2SYan Zheng WARN_ON(trans->transid != btrfs_header_generation(parent)); 473f3a84ccdSFilipe Manana btrfs_tree_mod_log_insert_key(parent, parent_slot, 474f3a84ccdSFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE, GFP_NOFS); 4755f39d397SChris Mason btrfs_set_node_blockptr(parent, parent_slot, 476db94535dSChris Mason cow->start); 47774493f7aSChris Mason btrfs_set_node_ptr_generation(parent, parent_slot, 47874493f7aSChris Mason trans->transid); 4796702ed49SChris Mason btrfs_mark_buffer_dirty(parent); 4805de865eeSFilipe David Borba Manana if (last_ref) { 481f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_free_eb(buf); 4825de865eeSFilipe David Borba Manana if (ret) { 483572c83acSJosef Bacik btrfs_tree_unlock(cow); 484572c83acSJosef Bacik free_extent_buffer(cow); 48566642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 4865de865eeSFilipe David Borba Manana return ret; 4875de865eeSFilipe David Borba Manana } 4885de865eeSFilipe David Borba Manana } 4897a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), buf, 4907a163608SFilipe Manana parent_start, last_ref); 4916702ed49SChris Mason } 492925baeddSChris Mason if (unlock_orig) 493925baeddSChris Mason btrfs_tree_unlock(buf); 4943083ee2eSJosef Bacik free_extent_buffer_stale(buf); 4956702ed49SChris Mason btrfs_mark_buffer_dirty(cow); 4966702ed49SChris Mason *cow_ret = cow; 4976702ed49SChris Mason return 0; 4986702ed49SChris Mason } 4996702ed49SChris Mason 5005d4f98a2SYan Zheng static inline int should_cow_block(struct btrfs_trans_handle *trans, 5015d4f98a2SYan Zheng struct btrfs_root *root, 5025d4f98a2SYan Zheng struct extent_buffer *buf) 5035d4f98a2SYan Zheng { 504f5ee5c9aSJeff Mahoney if (btrfs_is_testing(root->fs_info)) 505faa2dbf0SJosef Bacik return 0; 506fccb84c9SDavid Sterba 507d1980131SDavid Sterba /* Ensure we can see the FORCE_COW bit */ 508d1980131SDavid Sterba smp_mb__before_atomic(); 509f1ebcc74SLiu Bo 510f1ebcc74SLiu Bo /* 511f1ebcc74SLiu Bo * We do not need to cow a block if 512f1ebcc74SLiu Bo * 1) this block is not created or changed in this transaction; 513f1ebcc74SLiu Bo * 2) this block does not belong to TREE_RELOC tree; 514f1ebcc74SLiu Bo * 3) the root is not forced COW. 515f1ebcc74SLiu Bo * 516f1ebcc74SLiu Bo * What is forced COW: 51701327610SNicholas D Steeves * when we create snapshot during committing the transaction, 51852042d8eSAndrea Gelmini * after we've finished copying src root, we must COW the shared 519f1ebcc74SLiu Bo * block to ensure the metadata consistency. 520f1ebcc74SLiu Bo */ 5215d4f98a2SYan Zheng if (btrfs_header_generation(buf) == trans->transid && 5225d4f98a2SYan Zheng !btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN) && 5235d4f98a2SYan Zheng !(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID && 524f1ebcc74SLiu Bo btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)) && 52527cdeb70SMiao Xie !test_bit(BTRFS_ROOT_FORCE_COW, &root->state)) 5265d4f98a2SYan Zheng return 0; 5275d4f98a2SYan Zheng return 1; 5285d4f98a2SYan Zheng } 5295d4f98a2SYan Zheng 530d352ac68SChris Mason /* 531d352ac68SChris Mason * cows a single block, see __btrfs_cow_block for the real work. 53201327610SNicholas D Steeves * This version of it has extra checks so that a block isn't COWed more than 533d352ac68SChris Mason * once per transaction, as long as it hasn't been written yet 534d352ac68SChris Mason */ 535d397712bSChris Mason noinline int btrfs_cow_block(struct btrfs_trans_handle *trans, 5365f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *buf, 5375f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 5389631e4ccSJosef Bacik struct extent_buffer **cow_ret, 5399631e4ccSJosef Bacik enum btrfs_lock_nesting nest) 54002217ed2SChris Mason { 5410b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 5426702ed49SChris Mason u64 search_start; 543f510cfecSChris Mason int ret; 544dc17ff8fSChris Mason 54583354f07SJosef Bacik if (test_bit(BTRFS_ROOT_DELETING, &root->state)) 54683354f07SJosef Bacik btrfs_err(fs_info, 54783354f07SJosef Bacik "COW'ing blocks on a fs root that's being dropped"); 54883354f07SJosef Bacik 5490b246afaSJeff Mahoney if (trans->transaction != fs_info->running_transaction) 55031b1a2bdSJulia Lawall WARN(1, KERN_CRIT "trans %llu running %llu\n", 551c1c9ff7cSGeert Uytterhoeven trans->transid, 5520b246afaSJeff Mahoney fs_info->running_transaction->transid); 55331b1a2bdSJulia Lawall 5540b246afaSJeff Mahoney if (trans->transid != fs_info->generation) 55531b1a2bdSJulia Lawall WARN(1, KERN_CRIT "trans %llu running %llu\n", 5560b246afaSJeff Mahoney trans->transid, fs_info->generation); 557dc17ff8fSChris Mason 5585d4f98a2SYan Zheng if (!should_cow_block(trans, root, buf)) { 55902217ed2SChris Mason *cow_ret = buf; 56002217ed2SChris Mason return 0; 56102217ed2SChris Mason } 562c487685dSChris Mason 563ee22184bSByongho Lee search_start = buf->start & ~((u64)SZ_1G - 1); 564b4ce94deSChris Mason 565f616f5cdSQu Wenruo /* 566f616f5cdSQu Wenruo * Before CoWing this block for later modification, check if it's 567f616f5cdSQu Wenruo * the subtree root and do the delayed subtree trace if needed. 568f616f5cdSQu Wenruo * 569f616f5cdSQu Wenruo * Also We don't care about the error, as it's handled internally. 570f616f5cdSQu Wenruo */ 571f616f5cdSQu Wenruo btrfs_qgroup_trace_subtree_after_cow(trans, root, buf); 572f510cfecSChris Mason ret = __btrfs_cow_block(trans, root, buf, parent, 5739631e4ccSJosef Bacik parent_slot, cow_ret, search_start, 0, nest); 5741abe9b8aSliubo 5751abe9b8aSliubo trace_btrfs_cow_block(root, buf, *cow_ret); 5761abe9b8aSliubo 577f510cfecSChris Mason return ret; 5782c90e5d6SChris Mason } 579f75e2b79SJosef Bacik ALLOW_ERROR_INJECTION(btrfs_cow_block, ERRNO); 5806702ed49SChris Mason 581d352ac68SChris Mason /* 582d352ac68SChris Mason * helper function for defrag to decide if two blocks pointed to by a 583d352ac68SChris Mason * node are actually close by 584d352ac68SChris Mason */ 5856b80053dSChris Mason static int close_blocks(u64 blocknr, u64 other, u32 blocksize) 5866702ed49SChris Mason { 5876b80053dSChris Mason if (blocknr < other && other - (blocknr + blocksize) < 32768) 5886702ed49SChris Mason return 1; 5896b80053dSChris Mason if (blocknr > other && blocknr - (other + blocksize) < 32768) 5906702ed49SChris Mason return 1; 59102217ed2SChris Mason return 0; 59202217ed2SChris Mason } 59302217ed2SChris Mason 594ce6ef5abSDavid Sterba #ifdef __LITTLE_ENDIAN 595ce6ef5abSDavid Sterba 596ce6ef5abSDavid Sterba /* 597ce6ef5abSDavid Sterba * Compare two keys, on little-endian the disk order is same as CPU order and 598ce6ef5abSDavid Sterba * we can avoid the conversion. 599ce6ef5abSDavid Sterba */ 600ce6ef5abSDavid Sterba static int comp_keys(const struct btrfs_disk_key *disk_key, 601ce6ef5abSDavid Sterba const struct btrfs_key *k2) 602ce6ef5abSDavid Sterba { 603ce6ef5abSDavid Sterba const struct btrfs_key *k1 = (const struct btrfs_key *)disk_key; 604ce6ef5abSDavid Sterba 605ce6ef5abSDavid Sterba return btrfs_comp_cpu_keys(k1, k2); 606ce6ef5abSDavid Sterba } 607ce6ef5abSDavid Sterba 608ce6ef5abSDavid Sterba #else 609ce6ef5abSDavid Sterba 610081e9573SChris Mason /* 611081e9573SChris Mason * compare two keys in a memcmp fashion 612081e9573SChris Mason */ 613310712b2SOmar Sandoval static int comp_keys(const struct btrfs_disk_key *disk, 614310712b2SOmar Sandoval const struct btrfs_key *k2) 615081e9573SChris Mason { 616081e9573SChris Mason struct btrfs_key k1; 617081e9573SChris Mason 618081e9573SChris Mason btrfs_disk_key_to_cpu(&k1, disk); 619081e9573SChris Mason 62020736abaSDiego Calleja return btrfs_comp_cpu_keys(&k1, k2); 621081e9573SChris Mason } 622ce6ef5abSDavid Sterba #endif 623081e9573SChris Mason 624f3465ca4SJosef Bacik /* 625f3465ca4SJosef Bacik * same as comp_keys only with two btrfs_key's 626f3465ca4SJosef Bacik */ 627e1f60a65SDavid Sterba int __pure btrfs_comp_cpu_keys(const struct btrfs_key *k1, const struct btrfs_key *k2) 628f3465ca4SJosef Bacik { 629f3465ca4SJosef Bacik if (k1->objectid > k2->objectid) 630f3465ca4SJosef Bacik return 1; 631f3465ca4SJosef Bacik if (k1->objectid < k2->objectid) 632f3465ca4SJosef Bacik return -1; 633f3465ca4SJosef Bacik if (k1->type > k2->type) 634f3465ca4SJosef Bacik return 1; 635f3465ca4SJosef Bacik if (k1->type < k2->type) 636f3465ca4SJosef Bacik return -1; 637f3465ca4SJosef Bacik if (k1->offset > k2->offset) 638f3465ca4SJosef Bacik return 1; 639f3465ca4SJosef Bacik if (k1->offset < k2->offset) 640f3465ca4SJosef Bacik return -1; 641f3465ca4SJosef Bacik return 0; 642f3465ca4SJosef Bacik } 643081e9573SChris Mason 644d352ac68SChris Mason /* 645d352ac68SChris Mason * this is used by the defrag code to go through all the 646d352ac68SChris Mason * leaves pointed to by a node and reallocate them so that 647d352ac68SChris Mason * disk order is close to key order 648d352ac68SChris Mason */ 6496702ed49SChris Mason int btrfs_realloc_node(struct btrfs_trans_handle *trans, 6505f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *parent, 651de78b51aSEric Sandeen int start_slot, u64 *last_ret, 652a6b6e75eSChris Mason struct btrfs_key *progress) 6536702ed49SChris Mason { 6540b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 6556b80053dSChris Mason struct extent_buffer *cur; 6566702ed49SChris Mason u64 blocknr; 657e9d0b13bSChris Mason u64 search_start = *last_ret; 658e9d0b13bSChris Mason u64 last_block = 0; 6596702ed49SChris Mason u64 other; 6606702ed49SChris Mason u32 parent_nritems; 6616702ed49SChris Mason int end_slot; 6626702ed49SChris Mason int i; 6636702ed49SChris Mason int err = 0; 6646b80053dSChris Mason u32 blocksize; 665081e9573SChris Mason int progress_passed = 0; 666081e9573SChris Mason struct btrfs_disk_key disk_key; 6676702ed49SChris Mason 6680b246afaSJeff Mahoney WARN_ON(trans->transaction != fs_info->running_transaction); 6690b246afaSJeff Mahoney WARN_ON(trans->transid != fs_info->generation); 67086479a04SChris Mason 6716b80053dSChris Mason parent_nritems = btrfs_header_nritems(parent); 6720b246afaSJeff Mahoney blocksize = fs_info->nodesize; 6735dfe2be7SFilipe Manana end_slot = parent_nritems - 1; 6746702ed49SChris Mason 6755dfe2be7SFilipe Manana if (parent_nritems <= 1) 6766702ed49SChris Mason return 0; 6776702ed49SChris Mason 6785dfe2be7SFilipe Manana for (i = start_slot; i <= end_slot; i++) { 6796702ed49SChris Mason int close = 1; 680a6b6e75eSChris Mason 681081e9573SChris Mason btrfs_node_key(parent, &disk_key, i); 682081e9573SChris Mason if (!progress_passed && comp_keys(&disk_key, progress) < 0) 683081e9573SChris Mason continue; 684081e9573SChris Mason 685081e9573SChris Mason progress_passed = 1; 6866b80053dSChris Mason blocknr = btrfs_node_blockptr(parent, i); 687e9d0b13bSChris Mason if (last_block == 0) 688e9d0b13bSChris Mason last_block = blocknr; 6895708b959SChris Mason 6906702ed49SChris Mason if (i > 0) { 6916b80053dSChris Mason other = btrfs_node_blockptr(parent, i - 1); 6926b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 6936702ed49SChris Mason } 6945dfe2be7SFilipe Manana if (!close && i < end_slot) { 6956b80053dSChris Mason other = btrfs_node_blockptr(parent, i + 1); 6966b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 6976702ed49SChris Mason } 698e9d0b13bSChris Mason if (close) { 699e9d0b13bSChris Mason last_block = blocknr; 7006702ed49SChris Mason continue; 701e9d0b13bSChris Mason } 7026702ed49SChris Mason 703206983b7SJosef Bacik cur = btrfs_read_node_slot(parent, i); 704206983b7SJosef Bacik if (IS_ERR(cur)) 70564c043deSLiu Bo return PTR_ERR(cur); 706e9d0b13bSChris Mason if (search_start == 0) 7076b80053dSChris Mason search_start = last_block; 708e9d0b13bSChris Mason 709e7a84565SChris Mason btrfs_tree_lock(cur); 7106b80053dSChris Mason err = __btrfs_cow_block(trans, root, cur, parent, i, 711e7a84565SChris Mason &cur, search_start, 7126b80053dSChris Mason min(16 * blocksize, 7139631e4ccSJosef Bacik (end_slot - i) * blocksize), 7149631e4ccSJosef Bacik BTRFS_NESTING_COW); 715252c38f0SYan if (err) { 716e7a84565SChris Mason btrfs_tree_unlock(cur); 7176b80053dSChris Mason free_extent_buffer(cur); 7186702ed49SChris Mason break; 719252c38f0SYan } 720e7a84565SChris Mason search_start = cur->start; 721e7a84565SChris Mason last_block = cur->start; 722f2183bdeSChris Mason *last_ret = search_start; 723e7a84565SChris Mason btrfs_tree_unlock(cur); 724e7a84565SChris Mason free_extent_buffer(cur); 7256702ed49SChris Mason } 7266702ed49SChris Mason return err; 7276702ed49SChris Mason } 7286702ed49SChris Mason 72974123bd7SChris Mason /* 730fb81212cSFilipe Manana * Search for a key in the given extent_buffer. 7315f39d397SChris Mason * 732fb81212cSFilipe Manana * The lower boundary for the search is specified by the slot number @low. Use a 733fb81212cSFilipe Manana * value of 0 to search over the whole extent buffer. 73474123bd7SChris Mason * 735fb81212cSFilipe Manana * The slot in the extent buffer is returned via @slot. If the key exists in the 736fb81212cSFilipe Manana * extent buffer, then @slot will point to the slot where the key is, otherwise 737fb81212cSFilipe Manana * it points to the slot where you would insert the key. 738fb81212cSFilipe Manana * 739fb81212cSFilipe Manana * Slot may point to the total number of items (i.e. one position beyond the last 740fb81212cSFilipe Manana * key) if the key is bigger than the last key in the extent buffer. 74174123bd7SChris Mason */ 742fb81212cSFilipe Manana static noinline int generic_bin_search(struct extent_buffer *eb, int low, 74367d5e289SMarcos Paulo de Souza const struct btrfs_key *key, int *slot) 744be0e5c09SChris Mason { 745fb81212cSFilipe Manana unsigned long p; 746fb81212cSFilipe Manana int item_size; 74767d5e289SMarcos Paulo de Souza int high = btrfs_header_nritems(eb); 748be0e5c09SChris Mason int ret; 7495cd17f34SDavid Sterba const int key_size = sizeof(struct btrfs_disk_key); 750be0e5c09SChris Mason 7515e24e9afSLiu Bo if (low > high) { 7525e24e9afSLiu Bo btrfs_err(eb->fs_info, 7535e24e9afSLiu Bo "%s: low (%d) > high (%d) eb %llu owner %llu level %d", 7545e24e9afSLiu Bo __func__, low, high, eb->start, 7555e24e9afSLiu Bo btrfs_header_owner(eb), btrfs_header_level(eb)); 7565e24e9afSLiu Bo return -EINVAL; 7575e24e9afSLiu Bo } 7585e24e9afSLiu Bo 759fb81212cSFilipe Manana if (btrfs_header_level(eb) == 0) { 760fb81212cSFilipe Manana p = offsetof(struct btrfs_leaf, items); 761fb81212cSFilipe Manana item_size = sizeof(struct btrfs_item); 762fb81212cSFilipe Manana } else { 763fb81212cSFilipe Manana p = offsetof(struct btrfs_node, ptrs); 764fb81212cSFilipe Manana item_size = sizeof(struct btrfs_key_ptr); 765fb81212cSFilipe Manana } 766fb81212cSFilipe Manana 767be0e5c09SChris Mason while (low < high) { 7685cd17f34SDavid Sterba unsigned long oip; 7695cd17f34SDavid Sterba unsigned long offset; 7705cd17f34SDavid Sterba struct btrfs_disk_key *tmp; 7715cd17f34SDavid Sterba struct btrfs_disk_key unaligned; 7725cd17f34SDavid Sterba int mid; 7735cd17f34SDavid Sterba 774be0e5c09SChris Mason mid = (low + high) / 2; 7755f39d397SChris Mason offset = p + mid * item_size; 7765cd17f34SDavid Sterba oip = offset_in_page(offset); 7775f39d397SChris Mason 7785cd17f34SDavid Sterba if (oip + key_size <= PAGE_SIZE) { 779884b07d0SQu Wenruo const unsigned long idx = get_eb_page_index(offset); 7805cd17f34SDavid Sterba char *kaddr = page_address(eb->pages[idx]); 781934d375bSChris Mason 782884b07d0SQu Wenruo oip = get_eb_offset_in_page(eb, offset); 7835cd17f34SDavid Sterba tmp = (struct btrfs_disk_key *)(kaddr + oip); 7845cd17f34SDavid Sterba } else { 7855cd17f34SDavid Sterba read_extent_buffer(eb, &unaligned, offset, key_size); 7865f39d397SChris Mason tmp = &unaligned; 787479965d6SChris Mason } 788479965d6SChris Mason 789be0e5c09SChris Mason ret = comp_keys(tmp, key); 790be0e5c09SChris Mason 791be0e5c09SChris Mason if (ret < 0) 792be0e5c09SChris Mason low = mid + 1; 793be0e5c09SChris Mason else if (ret > 0) 794be0e5c09SChris Mason high = mid; 795be0e5c09SChris Mason else { 796be0e5c09SChris Mason *slot = mid; 797be0e5c09SChris Mason return 0; 798be0e5c09SChris Mason } 799be0e5c09SChris Mason } 800be0e5c09SChris Mason *slot = low; 801be0e5c09SChris Mason return 1; 802be0e5c09SChris Mason } 803be0e5c09SChris Mason 80497571fd0SChris Mason /* 805fb81212cSFilipe Manana * Simple binary search on an extent buffer. Works for both leaves and nodes, and 806fb81212cSFilipe Manana * always searches over the whole range of keys (slot 0 to slot 'nritems - 1'). 80797571fd0SChris Mason */ 808a74b35ecSNikolay Borisov int btrfs_bin_search(struct extent_buffer *eb, const struct btrfs_key *key, 809e3b83361SQu Wenruo int *slot) 810be0e5c09SChris Mason { 811fb81212cSFilipe Manana return generic_bin_search(eb, 0, key, slot); 812be0e5c09SChris Mason } 813be0e5c09SChris Mason 814f0486c68SYan, Zheng static void root_add_used(struct btrfs_root *root, u32 size) 815f0486c68SYan, Zheng { 816f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 817f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 818f0486c68SYan, Zheng btrfs_root_used(&root->root_item) + size); 819f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 820f0486c68SYan, Zheng } 821f0486c68SYan, Zheng 822f0486c68SYan, Zheng static void root_sub_used(struct btrfs_root *root, u32 size) 823f0486c68SYan, Zheng { 824f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 825f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 826f0486c68SYan, Zheng btrfs_root_used(&root->root_item) - size); 827f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 828f0486c68SYan, Zheng } 829f0486c68SYan, Zheng 830d352ac68SChris Mason /* given a node and slot number, this reads the blocks it points to. The 831d352ac68SChris Mason * extent buffer is returned with a reference taken (but unlocked). 832d352ac68SChris Mason */ 8334b231ae4SDavid Sterba struct extent_buffer *btrfs_read_node_slot(struct extent_buffer *parent, 8344b231ae4SDavid Sterba int slot) 835bb803951SChris Mason { 836ca7a79adSChris Mason int level = btrfs_header_level(parent); 837416bc658SJosef Bacik struct extent_buffer *eb; 838581c1760SQu Wenruo struct btrfs_key first_key; 839416bc658SJosef Bacik 840fb770ae4SLiu Bo if (slot < 0 || slot >= btrfs_header_nritems(parent)) 841fb770ae4SLiu Bo return ERR_PTR(-ENOENT); 842ca7a79adSChris Mason 843ca7a79adSChris Mason BUG_ON(level == 0); 844ca7a79adSChris Mason 845581c1760SQu Wenruo btrfs_node_key_to_cpu(parent, &first_key, slot); 846d0d20b0fSDavid Sterba eb = read_tree_block(parent->fs_info, btrfs_node_blockptr(parent, slot), 8471b7ec85eSJosef Bacik btrfs_header_owner(parent), 848581c1760SQu Wenruo btrfs_node_ptr_generation(parent, slot), 849581c1760SQu Wenruo level - 1, &first_key); 8504eb150d6SQu Wenruo if (IS_ERR(eb)) 8514eb150d6SQu Wenruo return eb; 8524eb150d6SQu Wenruo if (!extent_buffer_uptodate(eb)) { 853416bc658SJosef Bacik free_extent_buffer(eb); 8544eb150d6SQu Wenruo return ERR_PTR(-EIO); 855416bc658SJosef Bacik } 856416bc658SJosef Bacik 857416bc658SJosef Bacik return eb; 858bb803951SChris Mason } 859bb803951SChris Mason 860d352ac68SChris Mason /* 861d352ac68SChris Mason * node level balancing, used to make sure nodes are in proper order for 862d352ac68SChris Mason * item deletion. We balance from the top down, so we have to make sure 863d352ac68SChris Mason * that a deletion won't leave an node completely empty later on. 864d352ac68SChris Mason */ 865e02119d5SChris Mason static noinline int balance_level(struct btrfs_trans_handle *trans, 86698ed5174SChris Mason struct btrfs_root *root, 86798ed5174SChris Mason struct btrfs_path *path, int level) 868bb803951SChris Mason { 8690b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 8705f39d397SChris Mason struct extent_buffer *right = NULL; 8715f39d397SChris Mason struct extent_buffer *mid; 8725f39d397SChris Mason struct extent_buffer *left = NULL; 8735f39d397SChris Mason struct extent_buffer *parent = NULL; 874bb803951SChris Mason int ret = 0; 875bb803951SChris Mason int wret; 876bb803951SChris Mason int pslot; 877bb803951SChris Mason int orig_slot = path->slots[level]; 87879f95c82SChris Mason u64 orig_ptr; 879bb803951SChris Mason 88098e6b1ebSLiu Bo ASSERT(level > 0); 881bb803951SChris Mason 8825f39d397SChris Mason mid = path->nodes[level]; 883b4ce94deSChris Mason 884ac5887c8SJosef Bacik WARN_ON(path->locks[level] != BTRFS_WRITE_LOCK); 8857bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 8867bb86316SChris Mason 8871d4f8a0cSChris Mason orig_ptr = btrfs_node_blockptr(mid, orig_slot); 88879f95c82SChris Mason 889a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 8905f39d397SChris Mason parent = path->nodes[level + 1]; 891bb803951SChris Mason pslot = path->slots[level + 1]; 892a05a9bb1SLi Zefan } 893bb803951SChris Mason 89440689478SChris Mason /* 89540689478SChris Mason * deal with the case where there is only one pointer in the root 89640689478SChris Mason * by promoting the node below to a root 89740689478SChris Mason */ 8985f39d397SChris Mason if (!parent) { 8995f39d397SChris Mason struct extent_buffer *child; 900bb803951SChris Mason 9015f39d397SChris Mason if (btrfs_header_nritems(mid) != 1) 902bb803951SChris Mason return 0; 903bb803951SChris Mason 904bb803951SChris Mason /* promote the child to a root */ 9054b231ae4SDavid Sterba child = btrfs_read_node_slot(mid, 0); 906fb770ae4SLiu Bo if (IS_ERR(child)) { 907fb770ae4SLiu Bo ret = PTR_ERR(child); 9080b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 909305a26afSMark Fasheh goto enospc; 910305a26afSMark Fasheh } 911305a26afSMark Fasheh 912925baeddSChris Mason btrfs_tree_lock(child); 9139631e4ccSJosef Bacik ret = btrfs_cow_block(trans, root, child, mid, 0, &child, 9149631e4ccSJosef Bacik BTRFS_NESTING_COW); 915f0486c68SYan, Zheng if (ret) { 916f0486c68SYan, Zheng btrfs_tree_unlock(child); 917f0486c68SYan, Zheng free_extent_buffer(child); 918f0486c68SYan, Zheng goto enospc; 919f0486c68SYan, Zheng } 9202f375ab9SYan 921406808abSFilipe Manana ret = btrfs_tree_mod_log_insert_root(root->node, child, true); 922d9d19a01SDavid Sterba BUG_ON(ret < 0); 923240f62c8SChris Mason rcu_assign_pointer(root->node, child); 924925baeddSChris Mason 9250b86a832SChris Mason add_root_to_dirty_list(root); 926925baeddSChris Mason btrfs_tree_unlock(child); 927b4ce94deSChris Mason 928925baeddSChris Mason path->locks[level] = 0; 929bb803951SChris Mason path->nodes[level] = NULL; 9306a884d7dSDavid Sterba btrfs_clean_tree_block(mid); 931925baeddSChris Mason btrfs_tree_unlock(mid); 932bb803951SChris Mason /* once for the path */ 9335f39d397SChris Mason free_extent_buffer(mid); 934f0486c68SYan, Zheng 935f0486c68SYan, Zheng root_sub_used(root, mid->len); 9367a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), mid, 0, 1); 937bb803951SChris Mason /* once for the root ptr */ 9383083ee2eSJosef Bacik free_extent_buffer_stale(mid); 939f0486c68SYan, Zheng return 0; 940bb803951SChris Mason } 9415f39d397SChris Mason if (btrfs_header_nritems(mid) > 9420b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 4) 943bb803951SChris Mason return 0; 944bb803951SChris Mason 9454b231ae4SDavid Sterba left = btrfs_read_node_slot(parent, pslot - 1); 946fb770ae4SLiu Bo if (IS_ERR(left)) 947fb770ae4SLiu Bo left = NULL; 948fb770ae4SLiu Bo 9495f39d397SChris Mason if (left) { 950bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT); 9515f39d397SChris Mason wret = btrfs_cow_block(trans, root, left, 9529631e4ccSJosef Bacik parent, pslot - 1, &left, 953bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW); 95454aa1f4dSChris Mason if (wret) { 95554aa1f4dSChris Mason ret = wret; 95654aa1f4dSChris Mason goto enospc; 95754aa1f4dSChris Mason } 9582cc58cf2SChris Mason } 959fb770ae4SLiu Bo 9604b231ae4SDavid Sterba right = btrfs_read_node_slot(parent, pslot + 1); 961fb770ae4SLiu Bo if (IS_ERR(right)) 962fb770ae4SLiu Bo right = NULL; 963fb770ae4SLiu Bo 9645f39d397SChris Mason if (right) { 965bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT); 9665f39d397SChris Mason wret = btrfs_cow_block(trans, root, right, 9679631e4ccSJosef Bacik parent, pslot + 1, &right, 968bf59a5a2SJosef Bacik BTRFS_NESTING_RIGHT_COW); 9692cc58cf2SChris Mason if (wret) { 9702cc58cf2SChris Mason ret = wret; 9712cc58cf2SChris Mason goto enospc; 9722cc58cf2SChris Mason } 9732cc58cf2SChris Mason } 9742cc58cf2SChris Mason 9752cc58cf2SChris Mason /* first, try to make some room in the middle buffer */ 9765f39d397SChris Mason if (left) { 9775f39d397SChris Mason orig_slot += btrfs_header_nritems(left); 978d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 1); 97979f95c82SChris Mason if (wret < 0) 98079f95c82SChris Mason ret = wret; 981bb803951SChris Mason } 98279f95c82SChris Mason 98379f95c82SChris Mason /* 98479f95c82SChris Mason * then try to empty the right most buffer into the middle 98579f95c82SChris Mason */ 9865f39d397SChris Mason if (right) { 987d30a668fSDavid Sterba wret = push_node_left(trans, mid, right, 1); 98854aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 98979f95c82SChris Mason ret = wret; 9905f39d397SChris Mason if (btrfs_header_nritems(right) == 0) { 9916a884d7dSDavid Sterba btrfs_clean_tree_block(right); 992925baeddSChris Mason btrfs_tree_unlock(right); 993afe5fea7STsutomu Itoh del_ptr(root, path, level + 1, pslot + 1); 994f0486c68SYan, Zheng root_sub_used(root, right->len); 9957a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), right, 9967a163608SFilipe Manana 0, 1); 9973083ee2eSJosef Bacik free_extent_buffer_stale(right); 998f0486c68SYan, Zheng right = NULL; 999bb803951SChris Mason } else { 10005f39d397SChris Mason struct btrfs_disk_key right_key; 10015f39d397SChris Mason btrfs_node_key(right, &right_key, 0); 1002f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot + 1, 1003f3a84ccdSFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE, GFP_NOFS); 10040e82bcfeSDavid Sterba BUG_ON(ret < 0); 10055f39d397SChris Mason btrfs_set_node_key(parent, &right_key, pslot + 1); 10065f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 1007bb803951SChris Mason } 1008bb803951SChris Mason } 10095f39d397SChris Mason if (btrfs_header_nritems(mid) == 1) { 101079f95c82SChris Mason /* 101179f95c82SChris Mason * we're not allowed to leave a node with one item in the 101279f95c82SChris Mason * tree during a delete. A deletion from lower in the tree 101379f95c82SChris Mason * could try to delete the only pointer in this node. 101479f95c82SChris Mason * So, pull some keys from the left. 101579f95c82SChris Mason * There has to be a left pointer at this point because 101679f95c82SChris Mason * otherwise we would have pulled some pointers from the 101779f95c82SChris Mason * right 101879f95c82SChris Mason */ 1019305a26afSMark Fasheh if (!left) { 1020305a26afSMark Fasheh ret = -EROFS; 10210b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 1022305a26afSMark Fasheh goto enospc; 1023305a26afSMark Fasheh } 102455d32ed8SDavid Sterba wret = balance_node_right(trans, mid, left); 102554aa1f4dSChris Mason if (wret < 0) { 102679f95c82SChris Mason ret = wret; 102754aa1f4dSChris Mason goto enospc; 102854aa1f4dSChris Mason } 1029bce4eae9SChris Mason if (wret == 1) { 1030d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 1); 1031bce4eae9SChris Mason if (wret < 0) 1032bce4eae9SChris Mason ret = wret; 1033bce4eae9SChris Mason } 103479f95c82SChris Mason BUG_ON(wret == 1); 103579f95c82SChris Mason } 10365f39d397SChris Mason if (btrfs_header_nritems(mid) == 0) { 10376a884d7dSDavid Sterba btrfs_clean_tree_block(mid); 1038925baeddSChris Mason btrfs_tree_unlock(mid); 1039afe5fea7STsutomu Itoh del_ptr(root, path, level + 1, pslot); 1040f0486c68SYan, Zheng root_sub_used(root, mid->len); 10417a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), mid, 0, 1); 10423083ee2eSJosef Bacik free_extent_buffer_stale(mid); 1043f0486c68SYan, Zheng mid = NULL; 104479f95c82SChris Mason } else { 104579f95c82SChris Mason /* update the parent key to reflect our changes */ 10465f39d397SChris Mason struct btrfs_disk_key mid_key; 10475f39d397SChris Mason btrfs_node_key(mid, &mid_key, 0); 1048f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot, 1049f3a84ccdSFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE, GFP_NOFS); 10500e82bcfeSDavid Sterba BUG_ON(ret < 0); 10515f39d397SChris Mason btrfs_set_node_key(parent, &mid_key, pslot); 10525f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 105379f95c82SChris Mason } 1054bb803951SChris Mason 105579f95c82SChris Mason /* update the path */ 10565f39d397SChris Mason if (left) { 10575f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 105867439dadSDavid Sterba atomic_inc(&left->refs); 1059925baeddSChris Mason /* left was locked after cow */ 10605f39d397SChris Mason path->nodes[level] = left; 1061bb803951SChris Mason path->slots[level + 1] -= 1; 1062bb803951SChris Mason path->slots[level] = orig_slot; 1063925baeddSChris Mason if (mid) { 1064925baeddSChris Mason btrfs_tree_unlock(mid); 10655f39d397SChris Mason free_extent_buffer(mid); 1066925baeddSChris Mason } 1067bb803951SChris Mason } else { 10685f39d397SChris Mason orig_slot -= btrfs_header_nritems(left); 1069bb803951SChris Mason path->slots[level] = orig_slot; 1070bb803951SChris Mason } 1071bb803951SChris Mason } 107279f95c82SChris Mason /* double check we haven't messed things up */ 1073e20d96d6SChris Mason if (orig_ptr != 10745f39d397SChris Mason btrfs_node_blockptr(path->nodes[level], path->slots[level])) 107579f95c82SChris Mason BUG(); 107654aa1f4dSChris Mason enospc: 1077925baeddSChris Mason if (right) { 1078925baeddSChris Mason btrfs_tree_unlock(right); 10795f39d397SChris Mason free_extent_buffer(right); 1080925baeddSChris Mason } 1081925baeddSChris Mason if (left) { 1082925baeddSChris Mason if (path->nodes[level] != left) 1083925baeddSChris Mason btrfs_tree_unlock(left); 10845f39d397SChris Mason free_extent_buffer(left); 1085925baeddSChris Mason } 1086bb803951SChris Mason return ret; 1087bb803951SChris Mason } 1088bb803951SChris Mason 1089d352ac68SChris Mason /* Node balancing for insertion. Here we only split or push nodes around 1090d352ac68SChris Mason * when they are completely full. This is also done top down, so we 1091d352ac68SChris Mason * have to be pessimistic. 1092d352ac68SChris Mason */ 1093d397712bSChris Mason static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans, 1094e66f709bSChris Mason struct btrfs_root *root, 1095e66f709bSChris Mason struct btrfs_path *path, int level) 1096e66f709bSChris Mason { 10970b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 10985f39d397SChris Mason struct extent_buffer *right = NULL; 10995f39d397SChris Mason struct extent_buffer *mid; 11005f39d397SChris Mason struct extent_buffer *left = NULL; 11015f39d397SChris Mason struct extent_buffer *parent = NULL; 1102e66f709bSChris Mason int ret = 0; 1103e66f709bSChris Mason int wret; 1104e66f709bSChris Mason int pslot; 1105e66f709bSChris Mason int orig_slot = path->slots[level]; 1106e66f709bSChris Mason 1107e66f709bSChris Mason if (level == 0) 1108e66f709bSChris Mason return 1; 1109e66f709bSChris Mason 11105f39d397SChris Mason mid = path->nodes[level]; 11117bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 1112e66f709bSChris Mason 1113a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 11145f39d397SChris Mason parent = path->nodes[level + 1]; 1115e66f709bSChris Mason pslot = path->slots[level + 1]; 1116a05a9bb1SLi Zefan } 1117e66f709bSChris Mason 11185f39d397SChris Mason if (!parent) 1119e66f709bSChris Mason return 1; 1120e66f709bSChris Mason 11214b231ae4SDavid Sterba left = btrfs_read_node_slot(parent, pslot - 1); 1122fb770ae4SLiu Bo if (IS_ERR(left)) 1123fb770ae4SLiu Bo left = NULL; 1124e66f709bSChris Mason 1125e66f709bSChris Mason /* first, try to make some room in the middle buffer */ 11265f39d397SChris Mason if (left) { 1127e66f709bSChris Mason u32 left_nr; 1128925baeddSChris Mason 1129bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT); 1130b4ce94deSChris Mason 11315f39d397SChris Mason left_nr = btrfs_header_nritems(left); 11320b246afaSJeff Mahoney if (left_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) { 113333ade1f8SChris Mason wret = 1; 113433ade1f8SChris Mason } else { 11355f39d397SChris Mason ret = btrfs_cow_block(trans, root, left, parent, 11369631e4ccSJosef Bacik pslot - 1, &left, 1137bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW); 113854aa1f4dSChris Mason if (ret) 113954aa1f4dSChris Mason wret = 1; 114054aa1f4dSChris Mason else { 1141d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 0); 114254aa1f4dSChris Mason } 114333ade1f8SChris Mason } 1144e66f709bSChris Mason if (wret < 0) 1145e66f709bSChris Mason ret = wret; 1146e66f709bSChris Mason if (wret == 0) { 11475f39d397SChris Mason struct btrfs_disk_key disk_key; 1148e66f709bSChris Mason orig_slot += left_nr; 11495f39d397SChris Mason btrfs_node_key(mid, &disk_key, 0); 1150f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot, 1151f3a84ccdSFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE, GFP_NOFS); 11520e82bcfeSDavid Sterba BUG_ON(ret < 0); 11535f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot); 11545f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 11555f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 11565f39d397SChris Mason path->nodes[level] = left; 1157e66f709bSChris Mason path->slots[level + 1] -= 1; 1158e66f709bSChris Mason path->slots[level] = orig_slot; 1159925baeddSChris Mason btrfs_tree_unlock(mid); 11605f39d397SChris Mason free_extent_buffer(mid); 1161e66f709bSChris Mason } else { 1162e66f709bSChris Mason orig_slot -= 11635f39d397SChris Mason btrfs_header_nritems(left); 1164e66f709bSChris Mason path->slots[level] = orig_slot; 1165925baeddSChris Mason btrfs_tree_unlock(left); 11665f39d397SChris Mason free_extent_buffer(left); 1167e66f709bSChris Mason } 1168e66f709bSChris Mason return 0; 1169e66f709bSChris Mason } 1170925baeddSChris Mason btrfs_tree_unlock(left); 11715f39d397SChris Mason free_extent_buffer(left); 1172e66f709bSChris Mason } 11734b231ae4SDavid Sterba right = btrfs_read_node_slot(parent, pslot + 1); 1174fb770ae4SLiu Bo if (IS_ERR(right)) 1175fb770ae4SLiu Bo right = NULL; 1176e66f709bSChris Mason 1177e66f709bSChris Mason /* 1178e66f709bSChris Mason * then try to empty the right most buffer into the middle 1179e66f709bSChris Mason */ 11805f39d397SChris Mason if (right) { 118133ade1f8SChris Mason u32 right_nr; 1182b4ce94deSChris Mason 1183bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT); 1184b4ce94deSChris Mason 11855f39d397SChris Mason right_nr = btrfs_header_nritems(right); 11860b246afaSJeff Mahoney if (right_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) { 118733ade1f8SChris Mason wret = 1; 118833ade1f8SChris Mason } else { 11895f39d397SChris Mason ret = btrfs_cow_block(trans, root, right, 11905f39d397SChris Mason parent, pslot + 1, 1191bf59a5a2SJosef Bacik &right, BTRFS_NESTING_RIGHT_COW); 119254aa1f4dSChris Mason if (ret) 119354aa1f4dSChris Mason wret = 1; 119454aa1f4dSChris Mason else { 119555d32ed8SDavid Sterba wret = balance_node_right(trans, right, mid); 119633ade1f8SChris Mason } 119754aa1f4dSChris Mason } 1198e66f709bSChris Mason if (wret < 0) 1199e66f709bSChris Mason ret = wret; 1200e66f709bSChris Mason if (wret == 0) { 12015f39d397SChris Mason struct btrfs_disk_key disk_key; 12025f39d397SChris Mason 12035f39d397SChris Mason btrfs_node_key(right, &disk_key, 0); 1204f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot + 1, 1205f3a84ccdSFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE, GFP_NOFS); 12060e82bcfeSDavid Sterba BUG_ON(ret < 0); 12075f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot + 1); 12085f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 12095f39d397SChris Mason 12105f39d397SChris Mason if (btrfs_header_nritems(mid) <= orig_slot) { 12115f39d397SChris Mason path->nodes[level] = right; 1212e66f709bSChris Mason path->slots[level + 1] += 1; 1213e66f709bSChris Mason path->slots[level] = orig_slot - 12145f39d397SChris Mason btrfs_header_nritems(mid); 1215925baeddSChris Mason btrfs_tree_unlock(mid); 12165f39d397SChris Mason free_extent_buffer(mid); 1217e66f709bSChris Mason } else { 1218925baeddSChris Mason btrfs_tree_unlock(right); 12195f39d397SChris Mason free_extent_buffer(right); 1220e66f709bSChris Mason } 1221e66f709bSChris Mason return 0; 1222e66f709bSChris Mason } 1223925baeddSChris Mason btrfs_tree_unlock(right); 12245f39d397SChris Mason free_extent_buffer(right); 1225e66f709bSChris Mason } 1226e66f709bSChris Mason return 1; 1227e66f709bSChris Mason } 1228e66f709bSChris Mason 122974123bd7SChris Mason /* 1230d352ac68SChris Mason * readahead one full node of leaves, finding things that are close 1231d352ac68SChris Mason * to the block in 'slot', and triggering ra on them. 12323c69faecSChris Mason */ 12332ff7e61eSJeff Mahoney static void reada_for_search(struct btrfs_fs_info *fs_info, 1234e02119d5SChris Mason struct btrfs_path *path, 123501f46658SChris Mason int level, int slot, u64 objectid) 12363c69faecSChris Mason { 12375f39d397SChris Mason struct extent_buffer *node; 123801f46658SChris Mason struct btrfs_disk_key disk_key; 12393c69faecSChris Mason u32 nritems; 12403c69faecSChris Mason u64 search; 1241a7175319SChris Mason u64 target; 12426b80053dSChris Mason u64 nread = 0; 1243ace75066SFilipe Manana u64 nread_max; 12446b80053dSChris Mason u32 nr; 12456b80053dSChris Mason u32 blocksize; 12466b80053dSChris Mason u32 nscan = 0; 1247db94535dSChris Mason 1248ace75066SFilipe Manana if (level != 1 && path->reada != READA_FORWARD_ALWAYS) 12493c69faecSChris Mason return; 12503c69faecSChris Mason 12516702ed49SChris Mason if (!path->nodes[level]) 12526702ed49SChris Mason return; 12536702ed49SChris Mason 12545f39d397SChris Mason node = path->nodes[level]; 1255925baeddSChris Mason 1256ace75066SFilipe Manana /* 1257ace75066SFilipe Manana * Since the time between visiting leaves is much shorter than the time 1258ace75066SFilipe Manana * between visiting nodes, limit read ahead of nodes to 1, to avoid too 1259ace75066SFilipe Manana * much IO at once (possibly random). 1260ace75066SFilipe Manana */ 1261ace75066SFilipe Manana if (path->reada == READA_FORWARD_ALWAYS) { 1262ace75066SFilipe Manana if (level > 1) 1263ace75066SFilipe Manana nread_max = node->fs_info->nodesize; 1264ace75066SFilipe Manana else 1265ace75066SFilipe Manana nread_max = SZ_128K; 1266ace75066SFilipe Manana } else { 1267ace75066SFilipe Manana nread_max = SZ_64K; 1268ace75066SFilipe Manana } 1269ace75066SFilipe Manana 12703c69faecSChris Mason search = btrfs_node_blockptr(node, slot); 12710b246afaSJeff Mahoney blocksize = fs_info->nodesize; 1272069a2e37SFilipe Manana if (path->reada != READA_FORWARD_ALWAYS) { 1273069a2e37SFilipe Manana struct extent_buffer *eb; 1274069a2e37SFilipe Manana 12750b246afaSJeff Mahoney eb = find_extent_buffer(fs_info, search); 12765f39d397SChris Mason if (eb) { 12775f39d397SChris Mason free_extent_buffer(eb); 12783c69faecSChris Mason return; 12793c69faecSChris Mason } 1280069a2e37SFilipe Manana } 12813c69faecSChris Mason 1282a7175319SChris Mason target = search; 12836b80053dSChris Mason 12845f39d397SChris Mason nritems = btrfs_header_nritems(node); 12856b80053dSChris Mason nr = slot; 128625b8b936SJosef Bacik 12873c69faecSChris Mason while (1) { 1288e4058b54SDavid Sterba if (path->reada == READA_BACK) { 12896b80053dSChris Mason if (nr == 0) 12903c69faecSChris Mason break; 12916b80053dSChris Mason nr--; 1292ace75066SFilipe Manana } else if (path->reada == READA_FORWARD || 1293ace75066SFilipe Manana path->reada == READA_FORWARD_ALWAYS) { 12946b80053dSChris Mason nr++; 12956b80053dSChris Mason if (nr >= nritems) 12966b80053dSChris Mason break; 12973c69faecSChris Mason } 1298e4058b54SDavid Sterba if (path->reada == READA_BACK && objectid) { 129901f46658SChris Mason btrfs_node_key(node, &disk_key, nr); 130001f46658SChris Mason if (btrfs_disk_key_objectid(&disk_key) != objectid) 130101f46658SChris Mason break; 130201f46658SChris Mason } 13036b80053dSChris Mason search = btrfs_node_blockptr(node, nr); 1304ace75066SFilipe Manana if (path->reada == READA_FORWARD_ALWAYS || 1305ace75066SFilipe Manana (search <= target && target - search <= 65536) || 1306a7175319SChris Mason (search > target && search - target <= 65536)) { 1307bfb484d9SJosef Bacik btrfs_readahead_node_child(node, nr); 13086b80053dSChris Mason nread += blocksize; 13093c69faecSChris Mason } 13106b80053dSChris Mason nscan++; 1311ace75066SFilipe Manana if (nread > nread_max || nscan > 32) 13126b80053dSChris Mason break; 13133c69faecSChris Mason } 13143c69faecSChris Mason } 1315925baeddSChris Mason 1316bfb484d9SJosef Bacik static noinline void reada_for_balance(struct btrfs_path *path, int level) 1317b4ce94deSChris Mason { 1318bfb484d9SJosef Bacik struct extent_buffer *parent; 1319b4ce94deSChris Mason int slot; 1320b4ce94deSChris Mason int nritems; 1321b4ce94deSChris Mason 13228c594ea8SChris Mason parent = path->nodes[level + 1]; 1323b4ce94deSChris Mason if (!parent) 13240b08851fSJosef Bacik return; 1325b4ce94deSChris Mason 1326b4ce94deSChris Mason nritems = btrfs_header_nritems(parent); 13278c594ea8SChris Mason slot = path->slots[level + 1]; 1328b4ce94deSChris Mason 1329bfb484d9SJosef Bacik if (slot > 0) 1330bfb484d9SJosef Bacik btrfs_readahead_node_child(parent, slot - 1); 1331bfb484d9SJosef Bacik if (slot + 1 < nritems) 1332bfb484d9SJosef Bacik btrfs_readahead_node_child(parent, slot + 1); 1333b4ce94deSChris Mason } 1334b4ce94deSChris Mason 1335b4ce94deSChris Mason 1336b4ce94deSChris Mason /* 1337d397712bSChris Mason * when we walk down the tree, it is usually safe to unlock the higher layers 1338d397712bSChris Mason * in the tree. The exceptions are when our path goes through slot 0, because 1339d397712bSChris Mason * operations on the tree might require changing key pointers higher up in the 1340d397712bSChris Mason * tree. 1341d352ac68SChris Mason * 1342d397712bSChris Mason * callers might also have set path->keep_locks, which tells this code to keep 1343d397712bSChris Mason * the lock if the path points to the last slot in the block. This is part of 1344d397712bSChris Mason * walking through the tree, and selecting the next slot in the higher block. 1345d352ac68SChris Mason * 1346d397712bSChris Mason * lowest_unlock sets the lowest level in the tree we're allowed to unlock. so 1347d397712bSChris Mason * if lowest_unlock is 1, level 0 won't be unlocked 1348d352ac68SChris Mason */ 1349e02119d5SChris Mason static noinline void unlock_up(struct btrfs_path *path, int level, 1350f7c79f30SChris Mason int lowest_unlock, int min_write_lock_level, 1351f7c79f30SChris Mason int *write_lock_level) 1352925baeddSChris Mason { 1353925baeddSChris Mason int i; 1354925baeddSChris Mason int skip_level = level; 1355c1227996SNikolay Borisov bool check_skip = true; 1356925baeddSChris Mason 1357925baeddSChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 1358925baeddSChris Mason if (!path->nodes[i]) 1359925baeddSChris Mason break; 1360925baeddSChris Mason if (!path->locks[i]) 1361925baeddSChris Mason break; 1362c1227996SNikolay Borisov 1363c1227996SNikolay Borisov if (check_skip) { 1364c1227996SNikolay Borisov if (path->slots[i] == 0) { 1365925baeddSChris Mason skip_level = i + 1; 1366925baeddSChris Mason continue; 1367925baeddSChris Mason } 1368c1227996SNikolay Borisov 1369c1227996SNikolay Borisov if (path->keep_locks) { 1370925baeddSChris Mason u32 nritems; 1371c1227996SNikolay Borisov 1372c1227996SNikolay Borisov nritems = btrfs_header_nritems(path->nodes[i]); 1373051e1b9fSChris Mason if (nritems < 1 || path->slots[i] >= nritems - 1) { 1374925baeddSChris Mason skip_level = i + 1; 1375925baeddSChris Mason continue; 1376925baeddSChris Mason } 1377925baeddSChris Mason } 1378c1227996SNikolay Borisov } 1379051e1b9fSChris Mason 1380d80bb3f9SLiu Bo if (i >= lowest_unlock && i > skip_level) { 1381c1227996SNikolay Borisov check_skip = false; 1382c1227996SNikolay Borisov btrfs_tree_unlock_rw(path->nodes[i], path->locks[i]); 1383925baeddSChris Mason path->locks[i] = 0; 1384f7c79f30SChris Mason if (write_lock_level && 1385f7c79f30SChris Mason i > min_write_lock_level && 1386f7c79f30SChris Mason i <= *write_lock_level) { 1387f7c79f30SChris Mason *write_lock_level = i - 1; 1388f7c79f30SChris Mason } 1389925baeddSChris Mason } 1390925baeddSChris Mason } 1391925baeddSChris Mason } 1392925baeddSChris Mason 13933c69faecSChris Mason /* 1394376a21d7SFilipe Manana * Helper function for btrfs_search_slot() and other functions that do a search 1395376a21d7SFilipe Manana * on a btree. The goal is to find a tree block in the cache (the radix tree at 1396376a21d7SFilipe Manana * fs_info->buffer_radix), but if we can't find it, or it's not up to date, read 1397376a21d7SFilipe Manana * its pages from disk. 1398c8c42864SChris Mason * 1399376a21d7SFilipe Manana * Returns -EAGAIN, with the path unlocked, if the caller needs to repeat the 1400376a21d7SFilipe Manana * whole btree search, starting again from the current root node. 1401c8c42864SChris Mason */ 1402c8c42864SChris Mason static int 1403d07b8528SLiu Bo read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, 1404c8c42864SChris Mason struct extent_buffer **eb_ret, int level, int slot, 1405cda79c54SDavid Sterba const struct btrfs_key *key) 1406c8c42864SChris Mason { 14070b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 1408c8c42864SChris Mason u64 blocknr; 1409c8c42864SChris Mason u64 gen; 1410c8c42864SChris Mason struct extent_buffer *tmp; 1411581c1760SQu Wenruo struct btrfs_key first_key; 141276a05b35SChris Mason int ret; 1413581c1760SQu Wenruo int parent_level; 1414b246666eSFilipe Manana bool unlock_up; 1415c8c42864SChris Mason 1416b246666eSFilipe Manana unlock_up = ((level + 1 < BTRFS_MAX_LEVEL) && p->locks[level + 1]); 1417213ff4b7SNikolay Borisov blocknr = btrfs_node_blockptr(*eb_ret, slot); 1418213ff4b7SNikolay Borisov gen = btrfs_node_ptr_generation(*eb_ret, slot); 1419213ff4b7SNikolay Borisov parent_level = btrfs_header_level(*eb_ret); 1420213ff4b7SNikolay Borisov btrfs_node_key_to_cpu(*eb_ret, &first_key, slot); 1421c8c42864SChris Mason 1422b246666eSFilipe Manana /* 1423b246666eSFilipe Manana * If we need to read an extent buffer from disk and we are holding locks 1424b246666eSFilipe Manana * on upper level nodes, we unlock all the upper nodes before reading the 1425b246666eSFilipe Manana * extent buffer, and then return -EAGAIN to the caller as it needs to 1426b246666eSFilipe Manana * restart the search. We don't release the lock on the current level 1427b246666eSFilipe Manana * because we need to walk this node to figure out which blocks to read. 1428b246666eSFilipe Manana */ 14290b246afaSJeff Mahoney tmp = find_extent_buffer(fs_info, blocknr); 1430cb44921aSChris Mason if (tmp) { 1431ace75066SFilipe Manana if (p->reada == READA_FORWARD_ALWAYS) 1432ace75066SFilipe Manana reada_for_search(fs_info, p, level, slot, key->objectid); 1433ace75066SFilipe Manana 1434b9fab919SChris Mason /* first we do an atomic uptodate check */ 1435b9fab919SChris Mason if (btrfs_buffer_uptodate(tmp, gen, 1) > 0) { 1436448de471SQu Wenruo /* 1437448de471SQu Wenruo * Do extra check for first_key, eb can be stale due to 1438448de471SQu Wenruo * being cached, read from scrub, or have multiple 1439448de471SQu Wenruo * parents (shared tree blocks). 1440448de471SQu Wenruo */ 1441e064d5e9SDavid Sterba if (btrfs_verify_level_key(tmp, 1442448de471SQu Wenruo parent_level - 1, &first_key, gen)) { 1443448de471SQu Wenruo free_extent_buffer(tmp); 1444448de471SQu Wenruo return -EUCLEAN; 1445448de471SQu Wenruo } 1446c8c42864SChris Mason *eb_ret = tmp; 1447c8c42864SChris Mason return 0; 1448c8c42864SChris Mason } 1449bdf7c00eSJosef Bacik 1450b246666eSFilipe Manana if (unlock_up) 1451b246666eSFilipe Manana btrfs_unlock_up_safe(p, level + 1); 1452b246666eSFilipe Manana 1453b9fab919SChris Mason /* now we're allowed to do a blocking uptodate check */ 14546a2e9dc4SFilipe Manana ret = btrfs_read_extent_buffer(tmp, gen, parent_level - 1, &first_key); 14559a4ffa1bSQu Wenruo if (ret) { 1456cb44921aSChris Mason free_extent_buffer(tmp); 1457b3b4aa74SDavid Sterba btrfs_release_path(p); 1458cb44921aSChris Mason return -EIO; 1459cb44921aSChris Mason } 146088c602abSQu Wenruo if (btrfs_check_eb_owner(tmp, root->root_key.objectid)) { 146188c602abSQu Wenruo free_extent_buffer(tmp); 146288c602abSQu Wenruo btrfs_release_path(p); 146388c602abSQu Wenruo return -EUCLEAN; 146488c602abSQu Wenruo } 1465b246666eSFilipe Manana 1466b246666eSFilipe Manana if (unlock_up) 1467b246666eSFilipe Manana ret = -EAGAIN; 1468b246666eSFilipe Manana 1469b246666eSFilipe Manana goto out; 14709a4ffa1bSQu Wenruo } 1471c8c42864SChris Mason 1472b246666eSFilipe Manana if (unlock_up) { 14738c594ea8SChris Mason btrfs_unlock_up_safe(p, level + 1); 14744bb59055SFilipe Manana ret = -EAGAIN; 14754bb59055SFilipe Manana } else { 14764bb59055SFilipe Manana ret = 0; 14774bb59055SFilipe Manana } 14788c594ea8SChris Mason 1479e4058b54SDavid Sterba if (p->reada != READA_NONE) 14802ff7e61eSJeff Mahoney reada_for_search(fs_info, p, level, slot, key->objectid); 1481c8c42864SChris Mason 14821b7ec85eSJosef Bacik tmp = read_tree_block(fs_info, blocknr, root->root_key.objectid, 14831b7ec85eSJosef Bacik gen, parent_level - 1, &first_key); 14844eb150d6SQu Wenruo if (IS_ERR(tmp)) { 14854eb150d6SQu Wenruo btrfs_release_path(p); 14864eb150d6SQu Wenruo return PTR_ERR(tmp); 14874eb150d6SQu Wenruo } 148876a05b35SChris Mason /* 148976a05b35SChris Mason * If the read above didn't mark this buffer up to date, 149076a05b35SChris Mason * it will never end up being up to date. Set ret to EIO now 149176a05b35SChris Mason * and give up so that our caller doesn't loop forever 149276a05b35SChris Mason * on our EAGAINs. 149376a05b35SChris Mason */ 1494e6a1d6fdSLiu Bo if (!extent_buffer_uptodate(tmp)) 149576a05b35SChris Mason ret = -EIO; 149602a3307aSLiu Bo 1497b246666eSFilipe Manana out: 14984bb59055SFilipe Manana if (ret == 0) { 14994bb59055SFilipe Manana *eb_ret = tmp; 15004bb59055SFilipe Manana } else { 15014bb59055SFilipe Manana free_extent_buffer(tmp); 150202a3307aSLiu Bo btrfs_release_path(p); 15034bb59055SFilipe Manana } 15044bb59055SFilipe Manana 150576a05b35SChris Mason return ret; 1506c8c42864SChris Mason } 1507c8c42864SChris Mason 1508c8c42864SChris Mason /* 1509c8c42864SChris Mason * helper function for btrfs_search_slot. This does all of the checks 1510c8c42864SChris Mason * for node-level blocks and does any balancing required based on 1511c8c42864SChris Mason * the ins_len. 1512c8c42864SChris Mason * 1513c8c42864SChris Mason * If no extra work was required, zero is returned. If we had to 1514c8c42864SChris Mason * drop the path, -EAGAIN is returned and btrfs_search_slot must 1515c8c42864SChris Mason * start over 1516c8c42864SChris Mason */ 1517c8c42864SChris Mason static int 1518c8c42864SChris Mason setup_nodes_for_search(struct btrfs_trans_handle *trans, 1519c8c42864SChris Mason struct btrfs_root *root, struct btrfs_path *p, 1520bd681513SChris Mason struct extent_buffer *b, int level, int ins_len, 1521bd681513SChris Mason int *write_lock_level) 1522c8c42864SChris Mason { 15230b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 152495b982deSNikolay Borisov int ret = 0; 15250b246afaSJeff Mahoney 1526c8c42864SChris Mason if ((p->search_for_split || ins_len > 0) && btrfs_header_nritems(b) >= 15270b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3) { 1528c8c42864SChris Mason 1529bd681513SChris Mason if (*write_lock_level < level + 1) { 1530bd681513SChris Mason *write_lock_level = level + 1; 1531bd681513SChris Mason btrfs_release_path(p); 153295b982deSNikolay Borisov return -EAGAIN; 1533bd681513SChris Mason } 1534bd681513SChris Mason 1535bfb484d9SJosef Bacik reada_for_balance(p, level); 153695b982deSNikolay Borisov ret = split_node(trans, root, p, level); 1537c8c42864SChris Mason 1538c8c42864SChris Mason b = p->nodes[level]; 1539c8c42864SChris Mason } else if (ins_len < 0 && btrfs_header_nritems(b) < 15400b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 2) { 1541c8c42864SChris Mason 1542bd681513SChris Mason if (*write_lock_level < level + 1) { 1543bd681513SChris Mason *write_lock_level = level + 1; 1544bd681513SChris Mason btrfs_release_path(p); 154595b982deSNikolay Borisov return -EAGAIN; 1546bd681513SChris Mason } 1547bd681513SChris Mason 1548bfb484d9SJosef Bacik reada_for_balance(p, level); 154995b982deSNikolay Borisov ret = balance_level(trans, root, p, level); 155095b982deSNikolay Borisov if (ret) 155195b982deSNikolay Borisov return ret; 1552c8c42864SChris Mason 1553c8c42864SChris Mason b = p->nodes[level]; 1554c8c42864SChris Mason if (!b) { 1555b3b4aa74SDavid Sterba btrfs_release_path(p); 155695b982deSNikolay Borisov return -EAGAIN; 1557c8c42864SChris Mason } 1558c8c42864SChris Mason BUG_ON(btrfs_header_nritems(b) == 1); 1559c8c42864SChris Mason } 1560c8c42864SChris Mason return ret; 1561c8c42864SChris Mason } 1562c8c42864SChris Mason 1563381cf658SDavid Sterba int btrfs_find_item(struct btrfs_root *fs_root, struct btrfs_path *path, 1564e33d5c3dSKelley Nielsen u64 iobjectid, u64 ioff, u8 key_type, 1565e33d5c3dSKelley Nielsen struct btrfs_key *found_key) 1566e33d5c3dSKelley Nielsen { 1567e33d5c3dSKelley Nielsen int ret; 1568e33d5c3dSKelley Nielsen struct btrfs_key key; 1569e33d5c3dSKelley Nielsen struct extent_buffer *eb; 1570381cf658SDavid Sterba 1571381cf658SDavid Sterba ASSERT(path); 15721d4c08e0SDavid Sterba ASSERT(found_key); 1573e33d5c3dSKelley Nielsen 1574e33d5c3dSKelley Nielsen key.type = key_type; 1575e33d5c3dSKelley Nielsen key.objectid = iobjectid; 1576e33d5c3dSKelley Nielsen key.offset = ioff; 1577e33d5c3dSKelley Nielsen 1578e33d5c3dSKelley Nielsen ret = btrfs_search_slot(NULL, fs_root, &key, path, 0, 0); 15791d4c08e0SDavid Sterba if (ret < 0) 1580e33d5c3dSKelley Nielsen return ret; 1581e33d5c3dSKelley Nielsen 1582e33d5c3dSKelley Nielsen eb = path->nodes[0]; 1583e33d5c3dSKelley Nielsen if (ret && path->slots[0] >= btrfs_header_nritems(eb)) { 1584e33d5c3dSKelley Nielsen ret = btrfs_next_leaf(fs_root, path); 1585e33d5c3dSKelley Nielsen if (ret) 1586e33d5c3dSKelley Nielsen return ret; 1587e33d5c3dSKelley Nielsen eb = path->nodes[0]; 1588e33d5c3dSKelley Nielsen } 1589e33d5c3dSKelley Nielsen 1590e33d5c3dSKelley Nielsen btrfs_item_key_to_cpu(eb, found_key, path->slots[0]); 1591e33d5c3dSKelley Nielsen if (found_key->type != key.type || 1592e33d5c3dSKelley Nielsen found_key->objectid != key.objectid) 1593e33d5c3dSKelley Nielsen return 1; 1594e33d5c3dSKelley Nielsen 1595e33d5c3dSKelley Nielsen return 0; 1596e33d5c3dSKelley Nielsen } 1597e33d5c3dSKelley Nielsen 15981fc28d8eSLiu Bo static struct extent_buffer *btrfs_search_slot_get_root(struct btrfs_root *root, 15991fc28d8eSLiu Bo struct btrfs_path *p, 16001fc28d8eSLiu Bo int write_lock_level) 16011fc28d8eSLiu Bo { 16021fc28d8eSLiu Bo struct extent_buffer *b; 1603120de408SJosef Bacik int root_lock = 0; 16041fc28d8eSLiu Bo int level = 0; 16051fc28d8eSLiu Bo 16061fc28d8eSLiu Bo if (p->search_commit_root) { 16071fc28d8eSLiu Bo b = root->commit_root; 160867439dadSDavid Sterba atomic_inc(&b->refs); 16091fc28d8eSLiu Bo level = btrfs_header_level(b); 1610f9ddfd05SLiu Bo /* 1611f9ddfd05SLiu Bo * Ensure that all callers have set skip_locking when 1612f9ddfd05SLiu Bo * p->search_commit_root = 1. 1613f9ddfd05SLiu Bo */ 1614f9ddfd05SLiu Bo ASSERT(p->skip_locking == 1); 16151fc28d8eSLiu Bo 16161fc28d8eSLiu Bo goto out; 16171fc28d8eSLiu Bo } 16181fc28d8eSLiu Bo 16191fc28d8eSLiu Bo if (p->skip_locking) { 16201fc28d8eSLiu Bo b = btrfs_root_node(root); 16211fc28d8eSLiu Bo level = btrfs_header_level(b); 16221fc28d8eSLiu Bo goto out; 16231fc28d8eSLiu Bo } 16241fc28d8eSLiu Bo 1625120de408SJosef Bacik /* We try very hard to do read locks on the root */ 1626120de408SJosef Bacik root_lock = BTRFS_READ_LOCK; 1627120de408SJosef Bacik 16281fc28d8eSLiu Bo /* 1629662c653bSLiu Bo * If the level is set to maximum, we can skip trying to get the read 1630662c653bSLiu Bo * lock. 1631662c653bSLiu Bo */ 1632662c653bSLiu Bo if (write_lock_level < BTRFS_MAX_LEVEL) { 1633662c653bSLiu Bo /* 1634662c653bSLiu Bo * We don't know the level of the root node until we actually 1635662c653bSLiu Bo * have it read locked 16361fc28d8eSLiu Bo */ 16371bb96598SJosef Bacik b = btrfs_read_lock_root_node(root); 16381fc28d8eSLiu Bo level = btrfs_header_level(b); 16391fc28d8eSLiu Bo if (level > write_lock_level) 16401fc28d8eSLiu Bo goto out; 16411fc28d8eSLiu Bo 1642662c653bSLiu Bo /* Whoops, must trade for write lock */ 16431fc28d8eSLiu Bo btrfs_tree_read_unlock(b); 16441fc28d8eSLiu Bo free_extent_buffer(b); 1645662c653bSLiu Bo } 1646662c653bSLiu Bo 16471fc28d8eSLiu Bo b = btrfs_lock_root_node(root); 16481fc28d8eSLiu Bo root_lock = BTRFS_WRITE_LOCK; 16491fc28d8eSLiu Bo 16501fc28d8eSLiu Bo /* The level might have changed, check again */ 16511fc28d8eSLiu Bo level = btrfs_header_level(b); 16521fc28d8eSLiu Bo 16531fc28d8eSLiu Bo out: 1654120de408SJosef Bacik /* 1655120de408SJosef Bacik * The root may have failed to write out at some point, and thus is no 1656120de408SJosef Bacik * longer valid, return an error in this case. 1657120de408SJosef Bacik */ 1658120de408SJosef Bacik if (!extent_buffer_uptodate(b)) { 1659120de408SJosef Bacik if (root_lock) 1660120de408SJosef Bacik btrfs_tree_unlock_rw(b, root_lock); 1661120de408SJosef Bacik free_extent_buffer(b); 1662120de408SJosef Bacik return ERR_PTR(-EIO); 1663120de408SJosef Bacik } 1664120de408SJosef Bacik 16651fc28d8eSLiu Bo p->nodes[level] = b; 16661fc28d8eSLiu Bo if (!p->skip_locking) 16671fc28d8eSLiu Bo p->locks[level] = root_lock; 16681fc28d8eSLiu Bo /* 16691fc28d8eSLiu Bo * Callers are responsible for dropping b's references. 16701fc28d8eSLiu Bo */ 16711fc28d8eSLiu Bo return b; 16721fc28d8eSLiu Bo } 16731fc28d8eSLiu Bo 1674d96b3424SFilipe Manana /* 1675d96b3424SFilipe Manana * Replace the extent buffer at the lowest level of the path with a cloned 1676d96b3424SFilipe Manana * version. The purpose is to be able to use it safely, after releasing the 1677d96b3424SFilipe Manana * commit root semaphore, even if relocation is happening in parallel, the 1678d96b3424SFilipe Manana * transaction used for relocation is committed and the extent buffer is 1679d96b3424SFilipe Manana * reallocated in the next transaction. 1680d96b3424SFilipe Manana * 1681d96b3424SFilipe Manana * This is used in a context where the caller does not prevent transaction 1682d96b3424SFilipe Manana * commits from happening, either by holding a transaction handle or holding 1683d96b3424SFilipe Manana * some lock, while it's doing searches through a commit root. 1684d96b3424SFilipe Manana * At the moment it's only used for send operations. 1685d96b3424SFilipe Manana */ 1686d96b3424SFilipe Manana static int finish_need_commit_sem_search(struct btrfs_path *path) 1687d96b3424SFilipe Manana { 1688d96b3424SFilipe Manana const int i = path->lowest_level; 1689d96b3424SFilipe Manana const int slot = path->slots[i]; 1690d96b3424SFilipe Manana struct extent_buffer *lowest = path->nodes[i]; 1691d96b3424SFilipe Manana struct extent_buffer *clone; 1692d96b3424SFilipe Manana 1693d96b3424SFilipe Manana ASSERT(path->need_commit_sem); 1694d96b3424SFilipe Manana 1695d96b3424SFilipe Manana if (!lowest) 1696d96b3424SFilipe Manana return 0; 1697d96b3424SFilipe Manana 1698d96b3424SFilipe Manana lockdep_assert_held_read(&lowest->fs_info->commit_root_sem); 1699d96b3424SFilipe Manana 1700d96b3424SFilipe Manana clone = btrfs_clone_extent_buffer(lowest); 1701d96b3424SFilipe Manana if (!clone) 1702d96b3424SFilipe Manana return -ENOMEM; 1703d96b3424SFilipe Manana 1704d96b3424SFilipe Manana btrfs_release_path(path); 1705d96b3424SFilipe Manana path->nodes[i] = clone; 1706d96b3424SFilipe Manana path->slots[i] = slot; 1707d96b3424SFilipe Manana 1708d96b3424SFilipe Manana return 0; 1709d96b3424SFilipe Manana } 17101fc28d8eSLiu Bo 1711e2e58d0fSFilipe Manana static inline int search_for_key_slot(struct extent_buffer *eb, 1712e2e58d0fSFilipe Manana int search_low_slot, 1713e2e58d0fSFilipe Manana const struct btrfs_key *key, 1714e2e58d0fSFilipe Manana int prev_cmp, 1715e2e58d0fSFilipe Manana int *slot) 1716e2e58d0fSFilipe Manana { 1717e2e58d0fSFilipe Manana /* 1718e2e58d0fSFilipe Manana * If a previous call to btrfs_bin_search() on a parent node returned an 1719e2e58d0fSFilipe Manana * exact match (prev_cmp == 0), we can safely assume the target key will 1720e2e58d0fSFilipe Manana * always be at slot 0 on lower levels, since each key pointer 1721e2e58d0fSFilipe Manana * (struct btrfs_key_ptr) refers to the lowest key accessible from the 1722e2e58d0fSFilipe Manana * subtree it points to. Thus we can skip searching lower levels. 1723e2e58d0fSFilipe Manana */ 1724e2e58d0fSFilipe Manana if (prev_cmp == 0) { 1725e2e58d0fSFilipe Manana *slot = 0; 1726e2e58d0fSFilipe Manana return 0; 1727e2e58d0fSFilipe Manana } 1728e2e58d0fSFilipe Manana 1729e2e58d0fSFilipe Manana return generic_bin_search(eb, search_low_slot, key, slot); 1730e2e58d0fSFilipe Manana } 1731e2e58d0fSFilipe Manana 1732109324cfSFilipe Manana static int search_leaf(struct btrfs_trans_handle *trans, 1733109324cfSFilipe Manana struct btrfs_root *root, 1734109324cfSFilipe Manana const struct btrfs_key *key, 1735109324cfSFilipe Manana struct btrfs_path *path, 1736109324cfSFilipe Manana int ins_len, 1737109324cfSFilipe Manana int prev_cmp) 1738109324cfSFilipe Manana { 1739109324cfSFilipe Manana struct extent_buffer *leaf = path->nodes[0]; 1740109324cfSFilipe Manana int leaf_free_space = -1; 1741109324cfSFilipe Manana int search_low_slot = 0; 1742109324cfSFilipe Manana int ret; 1743109324cfSFilipe Manana bool do_bin_search = true; 1744109324cfSFilipe Manana 1745109324cfSFilipe Manana /* 1746109324cfSFilipe Manana * If we are doing an insertion, the leaf has enough free space and the 1747109324cfSFilipe Manana * destination slot for the key is not slot 0, then we can unlock our 1748109324cfSFilipe Manana * write lock on the parent, and any other upper nodes, before doing the 1749109324cfSFilipe Manana * binary search on the leaf (with search_for_key_slot()), allowing other 1750109324cfSFilipe Manana * tasks to lock the parent and any other upper nodes. 1751109324cfSFilipe Manana */ 1752109324cfSFilipe Manana if (ins_len > 0) { 1753109324cfSFilipe Manana /* 1754109324cfSFilipe Manana * Cache the leaf free space, since we will need it later and it 1755109324cfSFilipe Manana * will not change until then. 1756109324cfSFilipe Manana */ 1757109324cfSFilipe Manana leaf_free_space = btrfs_leaf_free_space(leaf); 1758109324cfSFilipe Manana 1759109324cfSFilipe Manana /* 1760109324cfSFilipe Manana * !path->locks[1] means we have a single node tree, the leaf is 1761109324cfSFilipe Manana * the root of the tree. 1762109324cfSFilipe Manana */ 1763109324cfSFilipe Manana if (path->locks[1] && leaf_free_space >= ins_len) { 1764109324cfSFilipe Manana struct btrfs_disk_key first_key; 1765109324cfSFilipe Manana 1766109324cfSFilipe Manana ASSERT(btrfs_header_nritems(leaf) > 0); 1767109324cfSFilipe Manana btrfs_item_key(leaf, &first_key, 0); 1768109324cfSFilipe Manana 1769109324cfSFilipe Manana /* 1770109324cfSFilipe Manana * Doing the extra comparison with the first key is cheap, 1771109324cfSFilipe Manana * taking into account that the first key is very likely 1772109324cfSFilipe Manana * already in a cache line because it immediately follows 1773109324cfSFilipe Manana * the extent buffer's header and we have recently accessed 1774109324cfSFilipe Manana * the header's level field. 1775109324cfSFilipe Manana */ 1776109324cfSFilipe Manana ret = comp_keys(&first_key, key); 1777109324cfSFilipe Manana if (ret < 0) { 1778109324cfSFilipe Manana /* 1779109324cfSFilipe Manana * The first key is smaller than the key we want 1780109324cfSFilipe Manana * to insert, so we are safe to unlock all upper 1781109324cfSFilipe Manana * nodes and we have to do the binary search. 1782109324cfSFilipe Manana * 1783109324cfSFilipe Manana * We do use btrfs_unlock_up_safe() and not 1784109324cfSFilipe Manana * unlock_up() because the later does not unlock 1785109324cfSFilipe Manana * nodes with a slot of 0 - we can safely unlock 1786109324cfSFilipe Manana * any node even if its slot is 0 since in this 1787109324cfSFilipe Manana * case the key does not end up at slot 0 of the 1788109324cfSFilipe Manana * leaf and there's no need to split the leaf. 1789109324cfSFilipe Manana */ 1790109324cfSFilipe Manana btrfs_unlock_up_safe(path, 1); 1791109324cfSFilipe Manana search_low_slot = 1; 1792109324cfSFilipe Manana } else { 1793109324cfSFilipe Manana /* 1794109324cfSFilipe Manana * The first key is >= then the key we want to 1795109324cfSFilipe Manana * insert, so we can skip the binary search as 1796109324cfSFilipe Manana * the target key will be at slot 0. 1797109324cfSFilipe Manana * 1798109324cfSFilipe Manana * We can not unlock upper nodes when the key is 1799109324cfSFilipe Manana * less than the first key, because we will need 1800109324cfSFilipe Manana * to update the key at slot 0 of the parent node 1801109324cfSFilipe Manana * and possibly of other upper nodes too. 1802109324cfSFilipe Manana * If the key matches the first key, then we can 1803109324cfSFilipe Manana * unlock all the upper nodes, using 1804109324cfSFilipe Manana * btrfs_unlock_up_safe() instead of unlock_up() 1805109324cfSFilipe Manana * as stated above. 1806109324cfSFilipe Manana */ 1807109324cfSFilipe Manana if (ret == 0) 1808109324cfSFilipe Manana btrfs_unlock_up_safe(path, 1); 1809109324cfSFilipe Manana /* 1810109324cfSFilipe Manana * ret is already 0 or 1, matching the result of 1811109324cfSFilipe Manana * a btrfs_bin_search() call, so there is no need 1812109324cfSFilipe Manana * to adjust it. 1813109324cfSFilipe Manana */ 1814109324cfSFilipe Manana do_bin_search = false; 1815109324cfSFilipe Manana path->slots[0] = 0; 1816109324cfSFilipe Manana } 1817109324cfSFilipe Manana } 1818109324cfSFilipe Manana } 1819109324cfSFilipe Manana 1820109324cfSFilipe Manana if (do_bin_search) { 1821109324cfSFilipe Manana ret = search_for_key_slot(leaf, search_low_slot, key, 1822109324cfSFilipe Manana prev_cmp, &path->slots[0]); 1823109324cfSFilipe Manana if (ret < 0) 1824109324cfSFilipe Manana return ret; 1825109324cfSFilipe Manana } 1826109324cfSFilipe Manana 1827109324cfSFilipe Manana if (ins_len > 0) { 1828109324cfSFilipe Manana /* 1829109324cfSFilipe Manana * Item key already exists. In this case, if we are allowed to 1830109324cfSFilipe Manana * insert the item (for example, in dir_item case, item key 1831109324cfSFilipe Manana * collision is allowed), it will be merged with the original 1832109324cfSFilipe Manana * item. Only the item size grows, no new btrfs item will be 1833109324cfSFilipe Manana * added. If search_for_extension is not set, ins_len already 1834109324cfSFilipe Manana * accounts the size btrfs_item, deduct it here so leaf space 1835109324cfSFilipe Manana * check will be correct. 1836109324cfSFilipe Manana */ 1837109324cfSFilipe Manana if (ret == 0 && !path->search_for_extension) { 1838109324cfSFilipe Manana ASSERT(ins_len >= sizeof(struct btrfs_item)); 1839109324cfSFilipe Manana ins_len -= sizeof(struct btrfs_item); 1840109324cfSFilipe Manana } 1841109324cfSFilipe Manana 1842109324cfSFilipe Manana ASSERT(leaf_free_space >= 0); 1843109324cfSFilipe Manana 1844109324cfSFilipe Manana if (leaf_free_space < ins_len) { 1845109324cfSFilipe Manana int err; 1846109324cfSFilipe Manana 1847109324cfSFilipe Manana err = split_leaf(trans, root, key, path, ins_len, 1848109324cfSFilipe Manana (ret == 0)); 1849bb8e9a60SFilipe Manana ASSERT(err <= 0); 1850bb8e9a60SFilipe Manana if (WARN_ON(err > 0)) 1851bb8e9a60SFilipe Manana err = -EUCLEAN; 1852109324cfSFilipe Manana if (err) 1853109324cfSFilipe Manana ret = err; 1854109324cfSFilipe Manana } 1855109324cfSFilipe Manana } 1856109324cfSFilipe Manana 1857109324cfSFilipe Manana return ret; 1858109324cfSFilipe Manana } 1859109324cfSFilipe Manana 1860c8c42864SChris Mason /* 18614271eceaSNikolay Borisov * btrfs_search_slot - look for a key in a tree and perform necessary 18624271eceaSNikolay Borisov * modifications to preserve tree invariants. 186374123bd7SChris Mason * 18644271eceaSNikolay Borisov * @trans: Handle of transaction, used when modifying the tree 18654271eceaSNikolay Borisov * @p: Holds all btree nodes along the search path 18664271eceaSNikolay Borisov * @root: The root node of the tree 18674271eceaSNikolay Borisov * @key: The key we are looking for 18689a664971Sethanwu * @ins_len: Indicates purpose of search: 18699a664971Sethanwu * >0 for inserts it's size of item inserted (*) 18709a664971Sethanwu * <0 for deletions 18719a664971Sethanwu * 0 for plain searches, not modifying the tree 18729a664971Sethanwu * 18739a664971Sethanwu * (*) If size of item inserted doesn't include 18749a664971Sethanwu * sizeof(struct btrfs_item), then p->search_for_extension must 18759a664971Sethanwu * be set. 18764271eceaSNikolay Borisov * @cow: boolean should CoW operations be performed. Must always be 1 18774271eceaSNikolay Borisov * when modifying the tree. 187897571fd0SChris Mason * 18794271eceaSNikolay Borisov * If @ins_len > 0, nodes and leaves will be split as we walk down the tree. 18804271eceaSNikolay Borisov * If @ins_len < 0, nodes will be merged as we walk down the tree (if possible) 18814271eceaSNikolay Borisov * 18824271eceaSNikolay Borisov * If @key is found, 0 is returned and you can find the item in the leaf level 18834271eceaSNikolay Borisov * of the path (level 0) 18844271eceaSNikolay Borisov * 18854271eceaSNikolay Borisov * If @key isn't found, 1 is returned and the leaf level of the path (level 0) 18864271eceaSNikolay Borisov * points to the slot where it should be inserted 18874271eceaSNikolay Borisov * 18884271eceaSNikolay Borisov * If an error is encountered while searching the tree a negative error number 18894271eceaSNikolay Borisov * is returned 189074123bd7SChris Mason */ 1891310712b2SOmar Sandoval int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root, 1892310712b2SOmar Sandoval const struct btrfs_key *key, struct btrfs_path *p, 1893310712b2SOmar Sandoval int ins_len, int cow) 1894be0e5c09SChris Mason { 1895d96b3424SFilipe Manana struct btrfs_fs_info *fs_info = root->fs_info; 18965f39d397SChris Mason struct extent_buffer *b; 1897be0e5c09SChris Mason int slot; 1898be0e5c09SChris Mason int ret; 189933c66f43SYan Zheng int err; 1900be0e5c09SChris Mason int level; 1901925baeddSChris Mason int lowest_unlock = 1; 1902bd681513SChris Mason /* everything at write_lock_level or lower must be write locked */ 1903bd681513SChris Mason int write_lock_level = 0; 19049f3a7427SChris Mason u8 lowest_level = 0; 1905f7c79f30SChris Mason int min_write_lock_level; 1906d7396f07SFilipe David Borba Manana int prev_cmp; 19079f3a7427SChris Mason 19086702ed49SChris Mason lowest_level = p->lowest_level; 1909323ac95bSChris Mason WARN_ON(lowest_level && ins_len > 0); 191022b0ebdaSChris Mason WARN_ON(p->nodes[0] != NULL); 1911eb653de1SFilipe David Borba Manana BUG_ON(!cow && ins_len); 191225179201SJosef Bacik 1913bd681513SChris Mason if (ins_len < 0) { 1914925baeddSChris Mason lowest_unlock = 2; 191565b51a00SChris Mason 1916bd681513SChris Mason /* when we are removing items, we might have to go up to level 1917bd681513SChris Mason * two as we update tree pointers Make sure we keep write 1918bd681513SChris Mason * for those levels as well 1919bd681513SChris Mason */ 1920bd681513SChris Mason write_lock_level = 2; 1921bd681513SChris Mason } else if (ins_len > 0) { 1922bd681513SChris Mason /* 1923bd681513SChris Mason * for inserting items, make sure we have a write lock on 1924bd681513SChris Mason * level 1 so we can update keys 1925bd681513SChris Mason */ 1926bd681513SChris Mason write_lock_level = 1; 1927bd681513SChris Mason } 1928bd681513SChris Mason 1929bd681513SChris Mason if (!cow) 1930bd681513SChris Mason write_lock_level = -1; 1931bd681513SChris Mason 193209a2a8f9SJosef Bacik if (cow && (p->keep_locks || p->lowest_level)) 1933bd681513SChris Mason write_lock_level = BTRFS_MAX_LEVEL; 1934bd681513SChris Mason 1935f7c79f30SChris Mason min_write_lock_level = write_lock_level; 1936f7c79f30SChris Mason 1937d96b3424SFilipe Manana if (p->need_commit_sem) { 1938d96b3424SFilipe Manana ASSERT(p->search_commit_root); 1939d96b3424SFilipe Manana down_read(&fs_info->commit_root_sem); 1940d96b3424SFilipe Manana } 1941d96b3424SFilipe Manana 1942bb803951SChris Mason again: 1943d7396f07SFilipe David Borba Manana prev_cmp = -1; 19441fc28d8eSLiu Bo b = btrfs_search_slot_get_root(root, p, write_lock_level); 1945be6821f8SFilipe Manana if (IS_ERR(b)) { 1946be6821f8SFilipe Manana ret = PTR_ERR(b); 1947be6821f8SFilipe Manana goto done; 1948be6821f8SFilipe Manana } 1949925baeddSChris Mason 1950eb60ceacSChris Mason while (b) { 1951f624d976SQu Wenruo int dec = 0; 1952f624d976SQu Wenruo 19535f39d397SChris Mason level = btrfs_header_level(b); 195465b51a00SChris Mason 195502217ed2SChris Mason if (cow) { 19569ea2c7c9SNikolay Borisov bool last_level = (level == (BTRFS_MAX_LEVEL - 1)); 19579ea2c7c9SNikolay Borisov 1958c8c42864SChris Mason /* 1959c8c42864SChris Mason * if we don't really need to cow this block 1960c8c42864SChris Mason * then we don't want to set the path blocking, 1961c8c42864SChris Mason * so we test it here 1962c8c42864SChris Mason */ 19635963ffcaSJosef Bacik if (!should_cow_block(trans, root, b)) 196465b51a00SChris Mason goto cow_done; 19655d4f98a2SYan Zheng 1966bd681513SChris Mason /* 1967bd681513SChris Mason * must have write locks on this node and the 1968bd681513SChris Mason * parent 1969bd681513SChris Mason */ 19705124e00eSJosef Bacik if (level > write_lock_level || 19715124e00eSJosef Bacik (level + 1 > write_lock_level && 19725124e00eSJosef Bacik level + 1 < BTRFS_MAX_LEVEL && 19735124e00eSJosef Bacik p->nodes[level + 1])) { 1974bd681513SChris Mason write_lock_level = level + 1; 1975bd681513SChris Mason btrfs_release_path(p); 1976bd681513SChris Mason goto again; 1977bd681513SChris Mason } 1978bd681513SChris Mason 19799ea2c7c9SNikolay Borisov if (last_level) 19809ea2c7c9SNikolay Borisov err = btrfs_cow_block(trans, root, b, NULL, 0, 19819631e4ccSJosef Bacik &b, 19829631e4ccSJosef Bacik BTRFS_NESTING_COW); 19839ea2c7c9SNikolay Borisov else 198433c66f43SYan Zheng err = btrfs_cow_block(trans, root, b, 1985e20d96d6SChris Mason p->nodes[level + 1], 19869631e4ccSJosef Bacik p->slots[level + 1], &b, 19879631e4ccSJosef Bacik BTRFS_NESTING_COW); 198833c66f43SYan Zheng if (err) { 198933c66f43SYan Zheng ret = err; 199065b51a00SChris Mason goto done; 199154aa1f4dSChris Mason } 199202217ed2SChris Mason } 199365b51a00SChris Mason cow_done: 1994eb60ceacSChris Mason p->nodes[level] = b; 1995b4ce94deSChris Mason 1996b4ce94deSChris Mason /* 1997b4ce94deSChris Mason * we have a lock on b and as long as we aren't changing 1998b4ce94deSChris Mason * the tree, there is no way to for the items in b to change. 1999b4ce94deSChris Mason * It is safe to drop the lock on our parent before we 2000b4ce94deSChris Mason * go through the expensive btree search on b. 2001b4ce94deSChris Mason * 2002eb653de1SFilipe David Borba Manana * If we're inserting or deleting (ins_len != 0), then we might 2003eb653de1SFilipe David Borba Manana * be changing slot zero, which may require changing the parent. 2004eb653de1SFilipe David Borba Manana * So, we can't drop the lock until after we know which slot 2005eb653de1SFilipe David Borba Manana * we're operating on. 2006b4ce94deSChris Mason */ 2007eb653de1SFilipe David Borba Manana if (!ins_len && !p->keep_locks) { 2008eb653de1SFilipe David Borba Manana int u = level + 1; 2009eb653de1SFilipe David Borba Manana 2010eb653de1SFilipe David Borba Manana if (u < BTRFS_MAX_LEVEL && p->locks[u]) { 2011eb653de1SFilipe David Borba Manana btrfs_tree_unlock_rw(p->nodes[u], p->locks[u]); 2012eb653de1SFilipe David Borba Manana p->locks[u] = 0; 2013eb653de1SFilipe David Borba Manana } 2014eb653de1SFilipe David Borba Manana } 2015b4ce94deSChris Mason 2016e2e58d0fSFilipe Manana if (level == 0) { 2017109324cfSFilipe Manana if (ins_len > 0) 2018e5e1c174SFilipe Manana ASSERT(write_lock_level >= 1); 2019bd681513SChris Mason 2020109324cfSFilipe Manana ret = search_leaf(trans, root, key, p, ins_len, prev_cmp); 2021459931ecSChris Mason if (!p->search_for_split) 2022f7c79f30SChris Mason unlock_up(p, level, lowest_unlock, 20234b6f8e96SLiu Bo min_write_lock_level, NULL); 202465b51a00SChris Mason goto done; 202565b51a00SChris Mason } 2026e2e58d0fSFilipe Manana 2027e2e58d0fSFilipe Manana ret = search_for_key_slot(b, 0, key, prev_cmp, &slot); 2028e2e58d0fSFilipe Manana if (ret < 0) 2029e2e58d0fSFilipe Manana goto done; 2030e2e58d0fSFilipe Manana prev_cmp = ret; 2031e2e58d0fSFilipe Manana 2032f624d976SQu Wenruo if (ret && slot > 0) { 2033f624d976SQu Wenruo dec = 1; 2034f624d976SQu Wenruo slot--; 2035f624d976SQu Wenruo } 2036f624d976SQu Wenruo p->slots[level] = slot; 2037f624d976SQu Wenruo err = setup_nodes_for_search(trans, root, p, b, level, ins_len, 2038f624d976SQu Wenruo &write_lock_level); 2039f624d976SQu Wenruo if (err == -EAGAIN) 2040f624d976SQu Wenruo goto again; 2041f624d976SQu Wenruo if (err) { 2042f624d976SQu Wenruo ret = err; 2043f624d976SQu Wenruo goto done; 2044f624d976SQu Wenruo } 2045f624d976SQu Wenruo b = p->nodes[level]; 2046f624d976SQu Wenruo slot = p->slots[level]; 2047f624d976SQu Wenruo 2048f624d976SQu Wenruo /* 2049f624d976SQu Wenruo * Slot 0 is special, if we change the key we have to update 2050f624d976SQu Wenruo * the parent pointer which means we must have a write lock on 2051f624d976SQu Wenruo * the parent 2052f624d976SQu Wenruo */ 2053f624d976SQu Wenruo if (slot == 0 && ins_len && write_lock_level < level + 1) { 2054f624d976SQu Wenruo write_lock_level = level + 1; 2055f624d976SQu Wenruo btrfs_release_path(p); 2056f624d976SQu Wenruo goto again; 2057f624d976SQu Wenruo } 2058f624d976SQu Wenruo 2059f624d976SQu Wenruo unlock_up(p, level, lowest_unlock, min_write_lock_level, 2060f624d976SQu Wenruo &write_lock_level); 2061f624d976SQu Wenruo 2062f624d976SQu Wenruo if (level == lowest_level) { 2063f624d976SQu Wenruo if (dec) 2064f624d976SQu Wenruo p->slots[level]++; 2065f624d976SQu Wenruo goto done; 2066f624d976SQu Wenruo } 2067f624d976SQu Wenruo 2068f624d976SQu Wenruo err = read_block_for_search(root, p, &b, level, slot, key); 2069f624d976SQu Wenruo if (err == -EAGAIN) 2070f624d976SQu Wenruo goto again; 2071f624d976SQu Wenruo if (err) { 2072f624d976SQu Wenruo ret = err; 2073f624d976SQu Wenruo goto done; 2074f624d976SQu Wenruo } 2075f624d976SQu Wenruo 2076f624d976SQu Wenruo if (!p->skip_locking) { 2077f624d976SQu Wenruo level = btrfs_header_level(b); 2078*b40130b2SJosef Bacik 2079*b40130b2SJosef Bacik btrfs_maybe_reset_lockdep_class(root, b); 2080*b40130b2SJosef Bacik 2081f624d976SQu Wenruo if (level <= write_lock_level) { 2082f624d976SQu Wenruo btrfs_tree_lock(b); 2083f624d976SQu Wenruo p->locks[level] = BTRFS_WRITE_LOCK; 2084f624d976SQu Wenruo } else { 2085fe596ca3SJosef Bacik btrfs_tree_read_lock(b); 2086f624d976SQu Wenruo p->locks[level] = BTRFS_READ_LOCK; 2087f624d976SQu Wenruo } 2088f624d976SQu Wenruo p->nodes[level] = b; 2089f624d976SQu Wenruo } 209065b51a00SChris Mason } 209165b51a00SChris Mason ret = 1; 209265b51a00SChris Mason done: 20935f5bc6b1SFilipe Manana if (ret < 0 && !p->skip_release_on_error) 2094b3b4aa74SDavid Sterba btrfs_release_path(p); 2095d96b3424SFilipe Manana 2096d96b3424SFilipe Manana if (p->need_commit_sem) { 2097d96b3424SFilipe Manana int ret2; 2098d96b3424SFilipe Manana 2099d96b3424SFilipe Manana ret2 = finish_need_commit_sem_search(p); 2100d96b3424SFilipe Manana up_read(&fs_info->commit_root_sem); 2101d96b3424SFilipe Manana if (ret2) 2102d96b3424SFilipe Manana ret = ret2; 2103d96b3424SFilipe Manana } 2104d96b3424SFilipe Manana 2105be0e5c09SChris Mason return ret; 2106be0e5c09SChris Mason } 2107f75e2b79SJosef Bacik ALLOW_ERROR_INJECTION(btrfs_search_slot, ERRNO); 2108be0e5c09SChris Mason 210974123bd7SChris Mason /* 21105d9e75c4SJan Schmidt * Like btrfs_search_slot, this looks for a key in the given tree. It uses the 21115d9e75c4SJan Schmidt * current state of the tree together with the operations recorded in the tree 21125d9e75c4SJan Schmidt * modification log to search for the key in a previous version of this tree, as 21135d9e75c4SJan Schmidt * denoted by the time_seq parameter. 21145d9e75c4SJan Schmidt * 21155d9e75c4SJan Schmidt * Naturally, there is no support for insert, delete or cow operations. 21165d9e75c4SJan Schmidt * 21175d9e75c4SJan Schmidt * The resulting path and return value will be set up as if we called 21185d9e75c4SJan Schmidt * btrfs_search_slot at that point in time with ins_len and cow both set to 0. 21195d9e75c4SJan Schmidt */ 2120310712b2SOmar Sandoval int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key, 21215d9e75c4SJan Schmidt struct btrfs_path *p, u64 time_seq) 21225d9e75c4SJan Schmidt { 21230b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 21245d9e75c4SJan Schmidt struct extent_buffer *b; 21255d9e75c4SJan Schmidt int slot; 21265d9e75c4SJan Schmidt int ret; 21275d9e75c4SJan Schmidt int err; 21285d9e75c4SJan Schmidt int level; 21295d9e75c4SJan Schmidt int lowest_unlock = 1; 21305d9e75c4SJan Schmidt u8 lowest_level = 0; 21315d9e75c4SJan Schmidt 21325d9e75c4SJan Schmidt lowest_level = p->lowest_level; 21335d9e75c4SJan Schmidt WARN_ON(p->nodes[0] != NULL); 21345d9e75c4SJan Schmidt 21355d9e75c4SJan Schmidt if (p->search_commit_root) { 21365d9e75c4SJan Schmidt BUG_ON(time_seq); 21375d9e75c4SJan Schmidt return btrfs_search_slot(NULL, root, key, p, 0, 0); 21385d9e75c4SJan Schmidt } 21395d9e75c4SJan Schmidt 21405d9e75c4SJan Schmidt again: 2141f3a84ccdSFilipe Manana b = btrfs_get_old_root(root, time_seq); 2142315bed43SNikolay Borisov if (!b) { 2143315bed43SNikolay Borisov ret = -EIO; 2144315bed43SNikolay Borisov goto done; 2145315bed43SNikolay Borisov } 21465d9e75c4SJan Schmidt level = btrfs_header_level(b); 21475d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 21485d9e75c4SJan Schmidt 21495d9e75c4SJan Schmidt while (b) { 2150abe9339dSQu Wenruo int dec = 0; 2151abe9339dSQu Wenruo 21525d9e75c4SJan Schmidt level = btrfs_header_level(b); 21535d9e75c4SJan Schmidt p->nodes[level] = b; 21545d9e75c4SJan Schmidt 21555d9e75c4SJan Schmidt /* 21565d9e75c4SJan Schmidt * we have a lock on b and as long as we aren't changing 21575d9e75c4SJan Schmidt * the tree, there is no way to for the items in b to change. 21585d9e75c4SJan Schmidt * It is safe to drop the lock on our parent before we 21595d9e75c4SJan Schmidt * go through the expensive btree search on b. 21605d9e75c4SJan Schmidt */ 21615d9e75c4SJan Schmidt btrfs_unlock_up_safe(p, level + 1); 21625d9e75c4SJan Schmidt 2163995e9a16SNikolay Borisov ret = btrfs_bin_search(b, key, &slot); 2164cbca7d59SFilipe Manana if (ret < 0) 2165cbca7d59SFilipe Manana goto done; 21665d9e75c4SJan Schmidt 2167abe9339dSQu Wenruo if (level == 0) { 2168abe9339dSQu Wenruo p->slots[level] = slot; 2169abe9339dSQu Wenruo unlock_up(p, level, lowest_unlock, 0, NULL); 2170abe9339dSQu Wenruo goto done; 2171abe9339dSQu Wenruo } 2172abe9339dSQu Wenruo 21735d9e75c4SJan Schmidt if (ret && slot > 0) { 21745d9e75c4SJan Schmidt dec = 1; 2175abe9339dSQu Wenruo slot--; 21765d9e75c4SJan Schmidt } 21775d9e75c4SJan Schmidt p->slots[level] = slot; 21785d9e75c4SJan Schmidt unlock_up(p, level, lowest_unlock, 0, NULL); 21795d9e75c4SJan Schmidt 21805d9e75c4SJan Schmidt if (level == lowest_level) { 21815d9e75c4SJan Schmidt if (dec) 21825d9e75c4SJan Schmidt p->slots[level]++; 21835d9e75c4SJan Schmidt goto done; 21845d9e75c4SJan Schmidt } 21855d9e75c4SJan Schmidt 2186abe9339dSQu Wenruo err = read_block_for_search(root, p, &b, level, slot, key); 21875d9e75c4SJan Schmidt if (err == -EAGAIN) 21885d9e75c4SJan Schmidt goto again; 21895d9e75c4SJan Schmidt if (err) { 21905d9e75c4SJan Schmidt ret = err; 21915d9e75c4SJan Schmidt goto done; 21925d9e75c4SJan Schmidt } 21935d9e75c4SJan Schmidt 21945d9e75c4SJan Schmidt level = btrfs_header_level(b); 21955d9e75c4SJan Schmidt btrfs_tree_read_lock(b); 2196f3a84ccdSFilipe Manana b = btrfs_tree_mod_log_rewind(fs_info, p, b, time_seq); 2197db7f3436SJosef Bacik if (!b) { 2198db7f3436SJosef Bacik ret = -ENOMEM; 2199db7f3436SJosef Bacik goto done; 2200db7f3436SJosef Bacik } 22015d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 22025d9e75c4SJan Schmidt p->nodes[level] = b; 22035d9e75c4SJan Schmidt } 22045d9e75c4SJan Schmidt ret = 1; 22055d9e75c4SJan Schmidt done: 22065d9e75c4SJan Schmidt if (ret < 0) 22075d9e75c4SJan Schmidt btrfs_release_path(p); 22085d9e75c4SJan Schmidt 22095d9e75c4SJan Schmidt return ret; 22105d9e75c4SJan Schmidt } 22115d9e75c4SJan Schmidt 22125d9e75c4SJan Schmidt /* 22132f38b3e1SArne Jansen * helper to use instead of search slot if no exact match is needed but 22142f38b3e1SArne Jansen * instead the next or previous item should be returned. 22152f38b3e1SArne Jansen * When find_higher is true, the next higher item is returned, the next lower 22162f38b3e1SArne Jansen * otherwise. 22172f38b3e1SArne Jansen * When return_any and find_higher are both true, and no higher item is found, 22182f38b3e1SArne Jansen * return the next lower instead. 22192f38b3e1SArne Jansen * When return_any is true and find_higher is false, and no lower item is found, 22202f38b3e1SArne Jansen * return the next higher instead. 22212f38b3e1SArne Jansen * It returns 0 if any item is found, 1 if none is found (tree empty), and 22222f38b3e1SArne Jansen * < 0 on error 22232f38b3e1SArne Jansen */ 22242f38b3e1SArne Jansen int btrfs_search_slot_for_read(struct btrfs_root *root, 2225310712b2SOmar Sandoval const struct btrfs_key *key, 2226310712b2SOmar Sandoval struct btrfs_path *p, int find_higher, 2227310712b2SOmar Sandoval int return_any) 22282f38b3e1SArne Jansen { 22292f38b3e1SArne Jansen int ret; 22302f38b3e1SArne Jansen struct extent_buffer *leaf; 22312f38b3e1SArne Jansen 22322f38b3e1SArne Jansen again: 22332f38b3e1SArne Jansen ret = btrfs_search_slot(NULL, root, key, p, 0, 0); 22342f38b3e1SArne Jansen if (ret <= 0) 22352f38b3e1SArne Jansen return ret; 22362f38b3e1SArne Jansen /* 22372f38b3e1SArne Jansen * a return value of 1 means the path is at the position where the 22382f38b3e1SArne Jansen * item should be inserted. Normally this is the next bigger item, 22392f38b3e1SArne Jansen * but in case the previous item is the last in a leaf, path points 22402f38b3e1SArne Jansen * to the first free slot in the previous leaf, i.e. at an invalid 22412f38b3e1SArne Jansen * item. 22422f38b3e1SArne Jansen */ 22432f38b3e1SArne Jansen leaf = p->nodes[0]; 22442f38b3e1SArne Jansen 22452f38b3e1SArne Jansen if (find_higher) { 22462f38b3e1SArne Jansen if (p->slots[0] >= btrfs_header_nritems(leaf)) { 22472f38b3e1SArne Jansen ret = btrfs_next_leaf(root, p); 22482f38b3e1SArne Jansen if (ret <= 0) 22492f38b3e1SArne Jansen return ret; 22502f38b3e1SArne Jansen if (!return_any) 22512f38b3e1SArne Jansen return 1; 22522f38b3e1SArne Jansen /* 22532f38b3e1SArne Jansen * no higher item found, return the next 22542f38b3e1SArne Jansen * lower instead 22552f38b3e1SArne Jansen */ 22562f38b3e1SArne Jansen return_any = 0; 22572f38b3e1SArne Jansen find_higher = 0; 22582f38b3e1SArne Jansen btrfs_release_path(p); 22592f38b3e1SArne Jansen goto again; 22602f38b3e1SArne Jansen } 22612f38b3e1SArne Jansen } else { 22622f38b3e1SArne Jansen if (p->slots[0] == 0) { 22632f38b3e1SArne Jansen ret = btrfs_prev_leaf(root, p); 2264e6793769SArne Jansen if (ret < 0) 22652f38b3e1SArne Jansen return ret; 2266e6793769SArne Jansen if (!ret) { 226723c6bf6aSFilipe David Borba Manana leaf = p->nodes[0]; 226823c6bf6aSFilipe David Borba Manana if (p->slots[0] == btrfs_header_nritems(leaf)) 226923c6bf6aSFilipe David Borba Manana p->slots[0]--; 2270e6793769SArne Jansen return 0; 2271e6793769SArne Jansen } 22722f38b3e1SArne Jansen if (!return_any) 22732f38b3e1SArne Jansen return 1; 22742f38b3e1SArne Jansen /* 22752f38b3e1SArne Jansen * no lower item found, return the next 22762f38b3e1SArne Jansen * higher instead 22772f38b3e1SArne Jansen */ 22782f38b3e1SArne Jansen return_any = 0; 22792f38b3e1SArne Jansen find_higher = 1; 22802f38b3e1SArne Jansen btrfs_release_path(p); 22812f38b3e1SArne Jansen goto again; 2282e6793769SArne Jansen } else { 22832f38b3e1SArne Jansen --p->slots[0]; 22842f38b3e1SArne Jansen } 22852f38b3e1SArne Jansen } 22862f38b3e1SArne Jansen return 0; 22872f38b3e1SArne Jansen } 22882f38b3e1SArne Jansen 22892f38b3e1SArne Jansen /* 22900ff40a91SMarcos Paulo de Souza * Execute search and call btrfs_previous_item to traverse backwards if the item 22910ff40a91SMarcos Paulo de Souza * was not found. 22920ff40a91SMarcos Paulo de Souza * 22930ff40a91SMarcos Paulo de Souza * Return 0 if found, 1 if not found and < 0 if error. 22940ff40a91SMarcos Paulo de Souza */ 22950ff40a91SMarcos Paulo de Souza int btrfs_search_backwards(struct btrfs_root *root, struct btrfs_key *key, 22960ff40a91SMarcos Paulo de Souza struct btrfs_path *path) 22970ff40a91SMarcos Paulo de Souza { 22980ff40a91SMarcos Paulo de Souza int ret; 22990ff40a91SMarcos Paulo de Souza 23000ff40a91SMarcos Paulo de Souza ret = btrfs_search_slot(NULL, root, key, path, 0, 0); 23010ff40a91SMarcos Paulo de Souza if (ret > 0) 23020ff40a91SMarcos Paulo de Souza ret = btrfs_previous_item(root, path, key->objectid, key->type); 23030ff40a91SMarcos Paulo de Souza 23040ff40a91SMarcos Paulo de Souza if (ret == 0) 23050ff40a91SMarcos Paulo de Souza btrfs_item_key_to_cpu(path->nodes[0], key, path->slots[0]); 23060ff40a91SMarcos Paulo de Souza 23070ff40a91SMarcos Paulo de Souza return ret; 23080ff40a91SMarcos Paulo de Souza } 23090ff40a91SMarcos Paulo de Souza 231062142be3SGabriel Niebler /** 231162142be3SGabriel Niebler * Search for a valid slot for the given path. 231262142be3SGabriel Niebler * 231362142be3SGabriel Niebler * @root: The root node of the tree. 231462142be3SGabriel Niebler * @key: Will contain a valid item if found. 231562142be3SGabriel Niebler * @path: The starting point to validate the slot. 231662142be3SGabriel Niebler * 231762142be3SGabriel Niebler * Return: 0 if the item is valid 231862142be3SGabriel Niebler * 1 if not found 231962142be3SGabriel Niebler * <0 if error. 232062142be3SGabriel Niebler */ 232162142be3SGabriel Niebler int btrfs_get_next_valid_item(struct btrfs_root *root, struct btrfs_key *key, 232262142be3SGabriel Niebler struct btrfs_path *path) 232362142be3SGabriel Niebler { 232462142be3SGabriel Niebler while (1) { 232562142be3SGabriel Niebler int ret; 232662142be3SGabriel Niebler const int slot = path->slots[0]; 232762142be3SGabriel Niebler const struct extent_buffer *leaf = path->nodes[0]; 232862142be3SGabriel Niebler 232962142be3SGabriel Niebler /* This is where we start walking the path. */ 233062142be3SGabriel Niebler if (slot >= btrfs_header_nritems(leaf)) { 233162142be3SGabriel Niebler /* 233262142be3SGabriel Niebler * If we've reached the last slot in this leaf we need 233362142be3SGabriel Niebler * to go to the next leaf and reset the path. 233462142be3SGabriel Niebler */ 233562142be3SGabriel Niebler ret = btrfs_next_leaf(root, path); 233662142be3SGabriel Niebler if (ret) 233762142be3SGabriel Niebler return ret; 233862142be3SGabriel Niebler continue; 233962142be3SGabriel Niebler } 234062142be3SGabriel Niebler /* Store the found, valid item in @key. */ 234162142be3SGabriel Niebler btrfs_item_key_to_cpu(leaf, key, slot); 234262142be3SGabriel Niebler break; 234362142be3SGabriel Niebler } 234462142be3SGabriel Niebler return 0; 234562142be3SGabriel Niebler } 234662142be3SGabriel Niebler 23470ff40a91SMarcos Paulo de Souza /* 234874123bd7SChris Mason * adjust the pointers going up the tree, starting at level 234974123bd7SChris Mason * making sure the right key of each node is points to 'key'. 235074123bd7SChris Mason * This is used after shifting pointers to the left, so it stops 235174123bd7SChris Mason * fixing up pointers when a given leaf/node is not in slot 0 of the 235274123bd7SChris Mason * higher levels 2353aa5d6bedSChris Mason * 235474123bd7SChris Mason */ 2355b167fa91SNikolay Borisov static void fixup_low_keys(struct btrfs_path *path, 23565f39d397SChris Mason struct btrfs_disk_key *key, int level) 2357be0e5c09SChris Mason { 2358be0e5c09SChris Mason int i; 23595f39d397SChris Mason struct extent_buffer *t; 23600e82bcfeSDavid Sterba int ret; 23615f39d397SChris Mason 2362234b63a0SChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 2363be0e5c09SChris Mason int tslot = path->slots[i]; 23640e82bcfeSDavid Sterba 2365eb60ceacSChris Mason if (!path->nodes[i]) 2366be0e5c09SChris Mason break; 23675f39d397SChris Mason t = path->nodes[i]; 2368f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(t, tslot, 2369f3a84ccdSFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE, GFP_ATOMIC); 23700e82bcfeSDavid Sterba BUG_ON(ret < 0); 23715f39d397SChris Mason btrfs_set_node_key(t, key, tslot); 2372d6025579SChris Mason btrfs_mark_buffer_dirty(path->nodes[i]); 2373be0e5c09SChris Mason if (tslot != 0) 2374be0e5c09SChris Mason break; 2375be0e5c09SChris Mason } 2376be0e5c09SChris Mason } 2377be0e5c09SChris Mason 237874123bd7SChris Mason /* 237931840ae1SZheng Yan * update item key. 238031840ae1SZheng Yan * 238131840ae1SZheng Yan * This function isn't completely safe. It's the caller's responsibility 238231840ae1SZheng Yan * that the new key won't break the order 238331840ae1SZheng Yan */ 2384b7a0365eSDaniel Dressler void btrfs_set_item_key_safe(struct btrfs_fs_info *fs_info, 2385b7a0365eSDaniel Dressler struct btrfs_path *path, 2386310712b2SOmar Sandoval const struct btrfs_key *new_key) 238731840ae1SZheng Yan { 238831840ae1SZheng Yan struct btrfs_disk_key disk_key; 238931840ae1SZheng Yan struct extent_buffer *eb; 239031840ae1SZheng Yan int slot; 239131840ae1SZheng Yan 239231840ae1SZheng Yan eb = path->nodes[0]; 239331840ae1SZheng Yan slot = path->slots[0]; 239431840ae1SZheng Yan if (slot > 0) { 239531840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot - 1); 23967c15d410SQu Wenruo if (unlikely(comp_keys(&disk_key, new_key) >= 0)) { 23977c15d410SQu Wenruo btrfs_crit(fs_info, 23987c15d410SQu Wenruo "slot %u key (%llu %u %llu) new key (%llu %u %llu)", 23997c15d410SQu Wenruo slot, btrfs_disk_key_objectid(&disk_key), 24007c15d410SQu Wenruo btrfs_disk_key_type(&disk_key), 24017c15d410SQu Wenruo btrfs_disk_key_offset(&disk_key), 24027c15d410SQu Wenruo new_key->objectid, new_key->type, 24037c15d410SQu Wenruo new_key->offset); 24047c15d410SQu Wenruo btrfs_print_leaf(eb); 24057c15d410SQu Wenruo BUG(); 24067c15d410SQu Wenruo } 240731840ae1SZheng Yan } 240831840ae1SZheng Yan if (slot < btrfs_header_nritems(eb) - 1) { 240931840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot + 1); 24107c15d410SQu Wenruo if (unlikely(comp_keys(&disk_key, new_key) <= 0)) { 24117c15d410SQu Wenruo btrfs_crit(fs_info, 24127c15d410SQu Wenruo "slot %u key (%llu %u %llu) new key (%llu %u %llu)", 24137c15d410SQu Wenruo slot, btrfs_disk_key_objectid(&disk_key), 24147c15d410SQu Wenruo btrfs_disk_key_type(&disk_key), 24157c15d410SQu Wenruo btrfs_disk_key_offset(&disk_key), 24167c15d410SQu Wenruo new_key->objectid, new_key->type, 24177c15d410SQu Wenruo new_key->offset); 24187c15d410SQu Wenruo btrfs_print_leaf(eb); 24197c15d410SQu Wenruo BUG(); 24207c15d410SQu Wenruo } 242131840ae1SZheng Yan } 242231840ae1SZheng Yan 242331840ae1SZheng Yan btrfs_cpu_key_to_disk(&disk_key, new_key); 242431840ae1SZheng Yan btrfs_set_item_key(eb, &disk_key, slot); 242531840ae1SZheng Yan btrfs_mark_buffer_dirty(eb); 242631840ae1SZheng Yan if (slot == 0) 2427b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 242831840ae1SZheng Yan } 242931840ae1SZheng Yan 243031840ae1SZheng Yan /* 2431d16c702fSQu Wenruo * Check key order of two sibling extent buffers. 2432d16c702fSQu Wenruo * 2433d16c702fSQu Wenruo * Return true if something is wrong. 2434d16c702fSQu Wenruo * Return false if everything is fine. 2435d16c702fSQu Wenruo * 2436d16c702fSQu Wenruo * Tree-checker only works inside one tree block, thus the following 2437d16c702fSQu Wenruo * corruption can not be detected by tree-checker: 2438d16c702fSQu Wenruo * 2439d16c702fSQu Wenruo * Leaf @left | Leaf @right 2440d16c702fSQu Wenruo * -------------------------------------------------------------- 2441d16c702fSQu Wenruo * | 1 | 2 | 3 | 4 | 5 | f6 | | 7 | 8 | 2442d16c702fSQu Wenruo * 2443d16c702fSQu Wenruo * Key f6 in leaf @left itself is valid, but not valid when the next 2444d16c702fSQu Wenruo * key in leaf @right is 7. 2445d16c702fSQu Wenruo * This can only be checked at tree block merge time. 2446d16c702fSQu Wenruo * And since tree checker has ensured all key order in each tree block 2447d16c702fSQu Wenruo * is correct, we only need to bother the last key of @left and the first 2448d16c702fSQu Wenruo * key of @right. 2449d16c702fSQu Wenruo */ 2450d16c702fSQu Wenruo static bool check_sibling_keys(struct extent_buffer *left, 2451d16c702fSQu Wenruo struct extent_buffer *right) 2452d16c702fSQu Wenruo { 2453d16c702fSQu Wenruo struct btrfs_key left_last; 2454d16c702fSQu Wenruo struct btrfs_key right_first; 2455d16c702fSQu Wenruo int level = btrfs_header_level(left); 2456d16c702fSQu Wenruo int nr_left = btrfs_header_nritems(left); 2457d16c702fSQu Wenruo int nr_right = btrfs_header_nritems(right); 2458d16c702fSQu Wenruo 2459d16c702fSQu Wenruo /* No key to check in one of the tree blocks */ 2460d16c702fSQu Wenruo if (!nr_left || !nr_right) 2461d16c702fSQu Wenruo return false; 2462d16c702fSQu Wenruo 2463d16c702fSQu Wenruo if (level) { 2464d16c702fSQu Wenruo btrfs_node_key_to_cpu(left, &left_last, nr_left - 1); 2465d16c702fSQu Wenruo btrfs_node_key_to_cpu(right, &right_first, 0); 2466d16c702fSQu Wenruo } else { 2467d16c702fSQu Wenruo btrfs_item_key_to_cpu(left, &left_last, nr_left - 1); 2468d16c702fSQu Wenruo btrfs_item_key_to_cpu(right, &right_first, 0); 2469d16c702fSQu Wenruo } 2470d16c702fSQu Wenruo 2471d16c702fSQu Wenruo if (btrfs_comp_cpu_keys(&left_last, &right_first) >= 0) { 2472d16c702fSQu Wenruo btrfs_crit(left->fs_info, 2473d16c702fSQu Wenruo "bad key order, sibling blocks, left last (%llu %u %llu) right first (%llu %u %llu)", 2474d16c702fSQu Wenruo left_last.objectid, left_last.type, 2475d16c702fSQu Wenruo left_last.offset, right_first.objectid, 2476d16c702fSQu Wenruo right_first.type, right_first.offset); 2477d16c702fSQu Wenruo return true; 2478d16c702fSQu Wenruo } 2479d16c702fSQu Wenruo return false; 2480d16c702fSQu Wenruo } 2481d16c702fSQu Wenruo 2482d16c702fSQu Wenruo /* 248374123bd7SChris Mason * try to push data from one node into the next node left in the 248479f95c82SChris Mason * tree. 2485aa5d6bedSChris Mason * 2486aa5d6bedSChris Mason * returns 0 if some ptrs were pushed left, < 0 if there was some horrible 2487aa5d6bedSChris Mason * error, and > 0 if there was no room in the left hand block. 248874123bd7SChris Mason */ 248998ed5174SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 24902ff7e61eSJeff Mahoney struct extent_buffer *dst, 2491971a1f66SChris Mason struct extent_buffer *src, int empty) 2492be0e5c09SChris Mason { 2493d30a668fSDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info; 2494be0e5c09SChris Mason int push_items = 0; 2495bb803951SChris Mason int src_nritems; 2496bb803951SChris Mason int dst_nritems; 2497aa5d6bedSChris Mason int ret = 0; 2498be0e5c09SChris Mason 24995f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 25005f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 25010b246afaSJeff Mahoney push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems; 25027bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 25037bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 250454aa1f4dSChris Mason 2505bce4eae9SChris Mason if (!empty && src_nritems <= 8) 2506971a1f66SChris Mason return 1; 2507971a1f66SChris Mason 2508d397712bSChris Mason if (push_items <= 0) 2509be0e5c09SChris Mason return 1; 2510be0e5c09SChris Mason 2511bce4eae9SChris Mason if (empty) { 2512971a1f66SChris Mason push_items = min(src_nritems, push_items); 2513bce4eae9SChris Mason if (push_items < src_nritems) { 2514bce4eae9SChris Mason /* leave at least 8 pointers in the node if 2515bce4eae9SChris Mason * we aren't going to empty it 2516bce4eae9SChris Mason */ 2517bce4eae9SChris Mason if (src_nritems - push_items < 8) { 2518bce4eae9SChris Mason if (push_items <= 8) 2519bce4eae9SChris Mason return 1; 2520bce4eae9SChris Mason push_items -= 8; 2521bce4eae9SChris Mason } 2522bce4eae9SChris Mason } 2523bce4eae9SChris Mason } else 2524bce4eae9SChris Mason push_items = min(src_nritems - 8, push_items); 252579f95c82SChris Mason 2526d16c702fSQu Wenruo /* dst is the left eb, src is the middle eb */ 2527d16c702fSQu Wenruo if (check_sibling_keys(dst, src)) { 2528d16c702fSQu Wenruo ret = -EUCLEAN; 2529d16c702fSQu Wenruo btrfs_abort_transaction(trans, ret); 2530d16c702fSQu Wenruo return ret; 2531d16c702fSQu Wenruo } 2532f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_eb_copy(dst, src, dst_nritems, 0, push_items); 25335de865eeSFilipe David Borba Manana if (ret) { 253466642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 25355de865eeSFilipe David Borba Manana return ret; 25365de865eeSFilipe David Borba Manana } 25375f39d397SChris Mason copy_extent_buffer(dst, src, 25385f39d397SChris Mason btrfs_node_key_ptr_offset(dst_nritems), 25395f39d397SChris Mason btrfs_node_key_ptr_offset(0), 2540123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 25415f39d397SChris Mason 2542bb803951SChris Mason if (push_items < src_nritems) { 254357911b8bSJan Schmidt /* 2544f3a84ccdSFilipe Manana * Don't call btrfs_tree_mod_log_insert_move() here, key removal 2545f3a84ccdSFilipe Manana * was already fully logged by btrfs_tree_mod_log_eb_copy() above. 254657911b8bSJan Schmidt */ 25475f39d397SChris Mason memmove_extent_buffer(src, btrfs_node_key_ptr_offset(0), 25485f39d397SChris Mason btrfs_node_key_ptr_offset(push_items), 2549e2fa7227SChris Mason (src_nritems - push_items) * 2550123abc88SChris Mason sizeof(struct btrfs_key_ptr)); 2551bb803951SChris Mason } 25525f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 25535f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 25545f39d397SChris Mason btrfs_mark_buffer_dirty(src); 25555f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 255631840ae1SZheng Yan 2557bb803951SChris Mason return ret; 2558be0e5c09SChris Mason } 2559be0e5c09SChris Mason 256097571fd0SChris Mason /* 256179f95c82SChris Mason * try to push data from one node into the next node right in the 256279f95c82SChris Mason * tree. 256379f95c82SChris Mason * 256479f95c82SChris Mason * returns 0 if some ptrs were pushed, < 0 if there was some horrible 256579f95c82SChris Mason * error, and > 0 if there was no room in the right hand block. 256679f95c82SChris Mason * 256779f95c82SChris Mason * this will only push up to 1/2 the contents of the left node over 256879f95c82SChris Mason */ 25695f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 25705f39d397SChris Mason struct extent_buffer *dst, 25715f39d397SChris Mason struct extent_buffer *src) 257279f95c82SChris Mason { 257355d32ed8SDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info; 257479f95c82SChris Mason int push_items = 0; 257579f95c82SChris Mason int max_push; 257679f95c82SChris Mason int src_nritems; 257779f95c82SChris Mason int dst_nritems; 257879f95c82SChris Mason int ret = 0; 257979f95c82SChris Mason 25807bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 25817bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 25827bb86316SChris Mason 25835f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 25845f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 25850b246afaSJeff Mahoney push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems; 2586d397712bSChris Mason if (push_items <= 0) 258779f95c82SChris Mason return 1; 2588bce4eae9SChris Mason 2589d397712bSChris Mason if (src_nritems < 4) 2590bce4eae9SChris Mason return 1; 259179f95c82SChris Mason 259279f95c82SChris Mason max_push = src_nritems / 2 + 1; 259379f95c82SChris Mason /* don't try to empty the node */ 2594d397712bSChris Mason if (max_push >= src_nritems) 259579f95c82SChris Mason return 1; 2596252c38f0SYan 259779f95c82SChris Mason if (max_push < push_items) 259879f95c82SChris Mason push_items = max_push; 259979f95c82SChris Mason 2600d16c702fSQu Wenruo /* dst is the right eb, src is the middle eb */ 2601d16c702fSQu Wenruo if (check_sibling_keys(src, dst)) { 2602d16c702fSQu Wenruo ret = -EUCLEAN; 2603d16c702fSQu Wenruo btrfs_abort_transaction(trans, ret); 2604d16c702fSQu Wenruo return ret; 2605d16c702fSQu Wenruo } 2606f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_move(dst, push_items, 0, dst_nritems); 2607bf1d3425SDavid Sterba BUG_ON(ret < 0); 26085f39d397SChris Mason memmove_extent_buffer(dst, btrfs_node_key_ptr_offset(push_items), 26095f39d397SChris Mason btrfs_node_key_ptr_offset(0), 26105f39d397SChris Mason (dst_nritems) * 26115f39d397SChris Mason sizeof(struct btrfs_key_ptr)); 2612d6025579SChris Mason 2613f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_eb_copy(dst, src, 0, src_nritems - push_items, 2614ed874f0dSDavid Sterba push_items); 26155de865eeSFilipe David Borba Manana if (ret) { 261666642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 26175de865eeSFilipe David Borba Manana return ret; 26185de865eeSFilipe David Borba Manana } 26195f39d397SChris Mason copy_extent_buffer(dst, src, 26205f39d397SChris Mason btrfs_node_key_ptr_offset(0), 26215f39d397SChris Mason btrfs_node_key_ptr_offset(src_nritems - push_items), 2622123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 262379f95c82SChris Mason 26245f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 26255f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 262679f95c82SChris Mason 26275f39d397SChris Mason btrfs_mark_buffer_dirty(src); 26285f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 262931840ae1SZheng Yan 263079f95c82SChris Mason return ret; 263179f95c82SChris Mason } 263279f95c82SChris Mason 263379f95c82SChris Mason /* 263497571fd0SChris Mason * helper function to insert a new root level in the tree. 263597571fd0SChris Mason * A new node is allocated, and a single item is inserted to 263697571fd0SChris Mason * point to the existing root 2637aa5d6bedSChris Mason * 2638aa5d6bedSChris Mason * returns zero on success or < 0 on failure. 263997571fd0SChris Mason */ 2640d397712bSChris Mason static noinline int insert_new_root(struct btrfs_trans_handle *trans, 26415f39d397SChris Mason struct btrfs_root *root, 2642fdd99c72SLiu Bo struct btrfs_path *path, int level) 264374123bd7SChris Mason { 26440b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 26457bb86316SChris Mason u64 lower_gen; 26465f39d397SChris Mason struct extent_buffer *lower; 26475f39d397SChris Mason struct extent_buffer *c; 2648925baeddSChris Mason struct extent_buffer *old; 26495f39d397SChris Mason struct btrfs_disk_key lower_key; 2650d9d19a01SDavid Sterba int ret; 26515c680ed6SChris Mason 26525c680ed6SChris Mason BUG_ON(path->nodes[level]); 26535c680ed6SChris Mason BUG_ON(path->nodes[level-1] != root->node); 26545c680ed6SChris Mason 26557bb86316SChris Mason lower = path->nodes[level-1]; 26567bb86316SChris Mason if (level == 1) 26577bb86316SChris Mason btrfs_item_key(lower, &lower_key, 0); 26587bb86316SChris Mason else 26597bb86316SChris Mason btrfs_node_key(lower, &lower_key, 0); 26607bb86316SChris Mason 266179bd3712SFilipe Manana c = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 266279bd3712SFilipe Manana &lower_key, level, root->node->start, 0, 2663cf6f34aaSJosef Bacik BTRFS_NESTING_NEW_ROOT); 26645f39d397SChris Mason if (IS_ERR(c)) 26655f39d397SChris Mason return PTR_ERR(c); 2666925baeddSChris Mason 26670b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 2668f0486c68SYan, Zheng 26695f39d397SChris Mason btrfs_set_header_nritems(c, 1); 26705f39d397SChris Mason btrfs_set_node_key(c, &lower_key, 0); 2671db94535dSChris Mason btrfs_set_node_blockptr(c, 0, lower->start); 26727bb86316SChris Mason lower_gen = btrfs_header_generation(lower); 267331840ae1SZheng Yan WARN_ON(lower_gen != trans->transid); 26747bb86316SChris Mason 26757bb86316SChris Mason btrfs_set_node_ptr_generation(c, 0, lower_gen); 26765f39d397SChris Mason 26775f39d397SChris Mason btrfs_mark_buffer_dirty(c); 2678d5719762SChris Mason 2679925baeddSChris Mason old = root->node; 2680406808abSFilipe Manana ret = btrfs_tree_mod_log_insert_root(root->node, c, false); 2681d9d19a01SDavid Sterba BUG_ON(ret < 0); 2682240f62c8SChris Mason rcu_assign_pointer(root->node, c); 2683925baeddSChris Mason 2684925baeddSChris Mason /* the super has an extra ref to root->node */ 2685925baeddSChris Mason free_extent_buffer(old); 2686925baeddSChris Mason 26870b86a832SChris Mason add_root_to_dirty_list(root); 268867439dadSDavid Sterba atomic_inc(&c->refs); 26895f39d397SChris Mason path->nodes[level] = c; 2690ac5887c8SJosef Bacik path->locks[level] = BTRFS_WRITE_LOCK; 269174123bd7SChris Mason path->slots[level] = 0; 269274123bd7SChris Mason return 0; 269374123bd7SChris Mason } 26945c680ed6SChris Mason 26955c680ed6SChris Mason /* 26965c680ed6SChris Mason * worker function to insert a single pointer in a node. 26975c680ed6SChris Mason * the node should have enough room for the pointer already 269897571fd0SChris Mason * 26995c680ed6SChris Mason * slot and level indicate where you want the key to go, and 27005c680ed6SChris Mason * blocknr is the block the key points to. 27015c680ed6SChris Mason */ 2702143bede5SJeff Mahoney static void insert_ptr(struct btrfs_trans_handle *trans, 27036ad3cf6dSDavid Sterba struct btrfs_path *path, 2704143bede5SJeff Mahoney struct btrfs_disk_key *key, u64 bytenr, 2705c3e06965SJan Schmidt int slot, int level) 27065c680ed6SChris Mason { 27075f39d397SChris Mason struct extent_buffer *lower; 27085c680ed6SChris Mason int nritems; 2709f3ea38daSJan Schmidt int ret; 27105c680ed6SChris Mason 27115c680ed6SChris Mason BUG_ON(!path->nodes[level]); 271249d0c642SFilipe Manana btrfs_assert_tree_write_locked(path->nodes[level]); 27135f39d397SChris Mason lower = path->nodes[level]; 27145f39d397SChris Mason nritems = btrfs_header_nritems(lower); 2715c293498bSStoyan Gaydarov BUG_ON(slot > nritems); 27166ad3cf6dSDavid Sterba BUG_ON(nritems == BTRFS_NODEPTRS_PER_BLOCK(trans->fs_info)); 271774123bd7SChris Mason if (slot != nritems) { 2718bf1d3425SDavid Sterba if (level) { 2719f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_move(lower, slot + 1, 2720f3a84ccdSFilipe Manana slot, nritems - slot); 2721bf1d3425SDavid Sterba BUG_ON(ret < 0); 2722bf1d3425SDavid Sterba } 27235f39d397SChris Mason memmove_extent_buffer(lower, 27245f39d397SChris Mason btrfs_node_key_ptr_offset(slot + 1), 27255f39d397SChris Mason btrfs_node_key_ptr_offset(slot), 2726123abc88SChris Mason (nritems - slot) * sizeof(struct btrfs_key_ptr)); 272774123bd7SChris Mason } 2728c3e06965SJan Schmidt if (level) { 2729f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(lower, slot, 2730f3a84ccdSFilipe Manana BTRFS_MOD_LOG_KEY_ADD, GFP_NOFS); 2731f3ea38daSJan Schmidt BUG_ON(ret < 0); 2732f3ea38daSJan Schmidt } 27335f39d397SChris Mason btrfs_set_node_key(lower, key, slot); 2734db94535dSChris Mason btrfs_set_node_blockptr(lower, slot, bytenr); 273574493f7aSChris Mason WARN_ON(trans->transid == 0); 273674493f7aSChris Mason btrfs_set_node_ptr_generation(lower, slot, trans->transid); 27375f39d397SChris Mason btrfs_set_header_nritems(lower, nritems + 1); 27385f39d397SChris Mason btrfs_mark_buffer_dirty(lower); 273974123bd7SChris Mason } 274074123bd7SChris Mason 274197571fd0SChris Mason /* 274297571fd0SChris Mason * split the node at the specified level in path in two. 274397571fd0SChris Mason * The path is corrected to point to the appropriate node after the split 274497571fd0SChris Mason * 274597571fd0SChris Mason * Before splitting this tries to make some room in the node by pushing 274697571fd0SChris Mason * left and right, if either one works, it returns right away. 2747aa5d6bedSChris Mason * 2748aa5d6bedSChris Mason * returns 0 on success and < 0 on failure 274997571fd0SChris Mason */ 2750e02119d5SChris Mason static noinline int split_node(struct btrfs_trans_handle *trans, 2751e02119d5SChris Mason struct btrfs_root *root, 2752e02119d5SChris Mason struct btrfs_path *path, int level) 2753be0e5c09SChris Mason { 27540b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 27555f39d397SChris Mason struct extent_buffer *c; 27565f39d397SChris Mason struct extent_buffer *split; 27575f39d397SChris Mason struct btrfs_disk_key disk_key; 2758be0e5c09SChris Mason int mid; 27595c680ed6SChris Mason int ret; 27607518a238SChris Mason u32 c_nritems; 2761be0e5c09SChris Mason 27625f39d397SChris Mason c = path->nodes[level]; 27637bb86316SChris Mason WARN_ON(btrfs_header_generation(c) != trans->transid); 27645f39d397SChris Mason if (c == root->node) { 2765d9abbf1cSJan Schmidt /* 276690f8d62eSJan Schmidt * trying to split the root, lets make a new one 276790f8d62eSJan Schmidt * 2768fdd99c72SLiu Bo * tree mod log: We don't log_removal old root in 276990f8d62eSJan Schmidt * insert_new_root, because that root buffer will be kept as a 277090f8d62eSJan Schmidt * normal node. We are going to log removal of half of the 2771f3a84ccdSFilipe Manana * elements below with btrfs_tree_mod_log_eb_copy(). We're 2772f3a84ccdSFilipe Manana * holding a tree lock on the buffer, which is why we cannot 2773f3a84ccdSFilipe Manana * race with other tree_mod_log users. 2774d9abbf1cSJan Schmidt */ 2775fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, level + 1); 27765c680ed6SChris Mason if (ret) 27775c680ed6SChris Mason return ret; 2778b3612421SChris Mason } else { 2779e66f709bSChris Mason ret = push_nodes_for_insert(trans, root, path, level); 27805f39d397SChris Mason c = path->nodes[level]; 27815f39d397SChris Mason if (!ret && btrfs_header_nritems(c) < 27820b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3) 2783e66f709bSChris Mason return 0; 278454aa1f4dSChris Mason if (ret < 0) 278554aa1f4dSChris Mason return ret; 27865c680ed6SChris Mason } 2787e66f709bSChris Mason 27885f39d397SChris Mason c_nritems = btrfs_header_nritems(c); 27895d4f98a2SYan Zheng mid = (c_nritems + 1) / 2; 27905d4f98a2SYan Zheng btrfs_node_key(c, &disk_key, mid); 27917bb86316SChris Mason 279279bd3712SFilipe Manana split = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 279379bd3712SFilipe Manana &disk_key, level, c->start, 0, 279479bd3712SFilipe Manana BTRFS_NESTING_SPLIT); 27955f39d397SChris Mason if (IS_ERR(split)) 27965f39d397SChris Mason return PTR_ERR(split); 279754aa1f4dSChris Mason 27980b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 2799bc877d28SNikolay Borisov ASSERT(btrfs_header_level(c) == level); 28005f39d397SChris Mason 2801f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_eb_copy(split, c, 0, mid, c_nritems - mid); 28025de865eeSFilipe David Borba Manana if (ret) { 280366642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 28045de865eeSFilipe David Borba Manana return ret; 28055de865eeSFilipe David Borba Manana } 28065f39d397SChris Mason copy_extent_buffer(split, c, 28075f39d397SChris Mason btrfs_node_key_ptr_offset(0), 28085f39d397SChris Mason btrfs_node_key_ptr_offset(mid), 2809123abc88SChris Mason (c_nritems - mid) * sizeof(struct btrfs_key_ptr)); 28105f39d397SChris Mason btrfs_set_header_nritems(split, c_nritems - mid); 28115f39d397SChris Mason btrfs_set_header_nritems(c, mid); 2812aa5d6bedSChris Mason 28135f39d397SChris Mason btrfs_mark_buffer_dirty(c); 28145f39d397SChris Mason btrfs_mark_buffer_dirty(split); 28155f39d397SChris Mason 28166ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, split->start, 2817c3e06965SJan Schmidt path->slots[level + 1] + 1, level + 1); 2818aa5d6bedSChris Mason 28195de08d7dSChris Mason if (path->slots[level] >= mid) { 28205c680ed6SChris Mason path->slots[level] -= mid; 2821925baeddSChris Mason btrfs_tree_unlock(c); 28225f39d397SChris Mason free_extent_buffer(c); 28235f39d397SChris Mason path->nodes[level] = split; 28245c680ed6SChris Mason path->slots[level + 1] += 1; 2825eb60ceacSChris Mason } else { 2826925baeddSChris Mason btrfs_tree_unlock(split); 28275f39d397SChris Mason free_extent_buffer(split); 2828be0e5c09SChris Mason } 2829d5286a92SNikolay Borisov return 0; 2830be0e5c09SChris Mason } 2831be0e5c09SChris Mason 283274123bd7SChris Mason /* 283374123bd7SChris Mason * how many bytes are required to store the items in a leaf. start 283474123bd7SChris Mason * and nr indicate which items in the leaf to check. This totals up the 283574123bd7SChris Mason * space used both by the item structs and the item data 283674123bd7SChris Mason */ 28375f39d397SChris Mason static int leaf_space_used(struct extent_buffer *l, int start, int nr) 2838be0e5c09SChris Mason { 2839be0e5c09SChris Mason int data_len; 28405f39d397SChris Mason int nritems = btrfs_header_nritems(l); 2841d4dbff95SChris Mason int end = min(nritems, start + nr) - 1; 2842be0e5c09SChris Mason 2843be0e5c09SChris Mason if (!nr) 2844be0e5c09SChris Mason return 0; 28453212fa14SJosef Bacik data_len = btrfs_item_offset(l, start) + btrfs_item_size(l, start); 28463212fa14SJosef Bacik data_len = data_len - btrfs_item_offset(l, end); 28470783fcfcSChris Mason data_len += sizeof(struct btrfs_item) * nr; 2848d4dbff95SChris Mason WARN_ON(data_len < 0); 2849be0e5c09SChris Mason return data_len; 2850be0e5c09SChris Mason } 2851be0e5c09SChris Mason 285274123bd7SChris Mason /* 2853d4dbff95SChris Mason * The space between the end of the leaf items and 2854d4dbff95SChris Mason * the start of the leaf data. IOW, how much room 2855d4dbff95SChris Mason * the leaf has left for both items and data 2856d4dbff95SChris Mason */ 2857e902baacSDavid Sterba noinline int btrfs_leaf_free_space(struct extent_buffer *leaf) 2858d4dbff95SChris Mason { 2859e902baacSDavid Sterba struct btrfs_fs_info *fs_info = leaf->fs_info; 28605f39d397SChris Mason int nritems = btrfs_header_nritems(leaf); 28615f39d397SChris Mason int ret; 28620b246afaSJeff Mahoney 28630b246afaSJeff Mahoney ret = BTRFS_LEAF_DATA_SIZE(fs_info) - leaf_space_used(leaf, 0, nritems); 28645f39d397SChris Mason if (ret < 0) { 28650b246afaSJeff Mahoney btrfs_crit(fs_info, 2866efe120a0SFrank Holton "leaf free space ret %d, leaf data size %lu, used %d nritems %d", 2867da17066cSJeff Mahoney ret, 28680b246afaSJeff Mahoney (unsigned long) BTRFS_LEAF_DATA_SIZE(fs_info), 28695f39d397SChris Mason leaf_space_used(leaf, 0, nritems), nritems); 28705f39d397SChris Mason } 28715f39d397SChris Mason return ret; 2872d4dbff95SChris Mason } 2873d4dbff95SChris Mason 287499d8f83cSChris Mason /* 287599d8f83cSChris Mason * min slot controls the lowest index we're willing to push to the 287699d8f83cSChris Mason * right. We'll push up to and including min_slot, but no lower 287799d8f83cSChris Mason */ 2878f72f0010SDavid Sterba static noinline int __push_leaf_right(struct btrfs_path *path, 287944871b1bSChris Mason int data_size, int empty, 288044871b1bSChris Mason struct extent_buffer *right, 288199d8f83cSChris Mason int free_space, u32 left_nritems, 288299d8f83cSChris Mason u32 min_slot) 288300ec4c51SChris Mason { 2884f72f0010SDavid Sterba struct btrfs_fs_info *fs_info = right->fs_info; 28855f39d397SChris Mason struct extent_buffer *left = path->nodes[0]; 288644871b1bSChris Mason struct extent_buffer *upper = path->nodes[1]; 2887cfed81a0SChris Mason struct btrfs_map_token token; 28885f39d397SChris Mason struct btrfs_disk_key disk_key; 288900ec4c51SChris Mason int slot; 289034a38218SChris Mason u32 i; 289100ec4c51SChris Mason int push_space = 0; 289200ec4c51SChris Mason int push_items = 0; 289334a38218SChris Mason u32 nr; 28947518a238SChris Mason u32 right_nritems; 28955f39d397SChris Mason u32 data_end; 2896db94535dSChris Mason u32 this_item_size; 289700ec4c51SChris Mason 289834a38218SChris Mason if (empty) 289934a38218SChris Mason nr = 0; 290034a38218SChris Mason else 290199d8f83cSChris Mason nr = max_t(u32, 1, min_slot); 290234a38218SChris Mason 290331840ae1SZheng Yan if (path->slots[0] >= left_nritems) 290487b29b20SYan Zheng push_space += data_size; 290531840ae1SZheng Yan 290644871b1bSChris Mason slot = path->slots[1]; 290734a38218SChris Mason i = left_nritems - 1; 290834a38218SChris Mason while (i >= nr) { 290931840ae1SZheng Yan if (!empty && push_items > 0) { 291031840ae1SZheng Yan if (path->slots[0] > i) 291131840ae1SZheng Yan break; 291231840ae1SZheng Yan if (path->slots[0] == i) { 2913e902baacSDavid Sterba int space = btrfs_leaf_free_space(left); 2914e902baacSDavid Sterba 291531840ae1SZheng Yan if (space + push_space * 2 > free_space) 291631840ae1SZheng Yan break; 291731840ae1SZheng Yan } 291831840ae1SZheng Yan } 291931840ae1SZheng Yan 292000ec4c51SChris Mason if (path->slots[0] == i) 292187b29b20SYan Zheng push_space += data_size; 2922db94535dSChris Mason 29233212fa14SJosef Bacik this_item_size = btrfs_item_size(left, i); 292474794207SJosef Bacik if (this_item_size + sizeof(struct btrfs_item) + 292574794207SJosef Bacik push_space > free_space) 292600ec4c51SChris Mason break; 292731840ae1SZheng Yan 292800ec4c51SChris Mason push_items++; 292974794207SJosef Bacik push_space += this_item_size + sizeof(struct btrfs_item); 293034a38218SChris Mason if (i == 0) 293134a38218SChris Mason break; 293234a38218SChris Mason i--; 2933db94535dSChris Mason } 29345f39d397SChris Mason 2935925baeddSChris Mason if (push_items == 0) 2936925baeddSChris Mason goto out_unlock; 29375f39d397SChris Mason 29386c1500f2SJulia Lawall WARN_ON(!empty && push_items == left_nritems); 29395f39d397SChris Mason 294000ec4c51SChris Mason /* push left to right */ 29415f39d397SChris Mason right_nritems = btrfs_header_nritems(right); 294234a38218SChris Mason 2943dc2e724eSJosef Bacik push_space = btrfs_item_data_end(left, left_nritems - push_items); 29448f881e8cSDavid Sterba push_space -= leaf_data_end(left); 29455f39d397SChris Mason 294600ec4c51SChris Mason /* make room in the right data area */ 29478f881e8cSDavid Sterba data_end = leaf_data_end(right); 29485f39d397SChris Mason memmove_extent_buffer(right, 29493d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + data_end - push_space, 29503d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + data_end, 29510b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - data_end); 29525f39d397SChris Mason 295300ec4c51SChris Mason /* copy from the left data area */ 29543d9ec8c4SNikolay Borisov copy_extent_buffer(right, left, BTRFS_LEAF_DATA_OFFSET + 29550b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - push_space, 29568f881e8cSDavid Sterba BTRFS_LEAF_DATA_OFFSET + leaf_data_end(left), 2957d6025579SChris Mason push_space); 29585f39d397SChris Mason 29595f39d397SChris Mason memmove_extent_buffer(right, btrfs_item_nr_offset(push_items), 29605f39d397SChris Mason btrfs_item_nr_offset(0), 29610783fcfcSChris Mason right_nritems * sizeof(struct btrfs_item)); 29625f39d397SChris Mason 296300ec4c51SChris Mason /* copy the items from left to right */ 29645f39d397SChris Mason copy_extent_buffer(right, left, btrfs_item_nr_offset(0), 29655f39d397SChris Mason btrfs_item_nr_offset(left_nritems - push_items), 29660783fcfcSChris Mason push_items * sizeof(struct btrfs_item)); 296700ec4c51SChris Mason 296800ec4c51SChris Mason /* update the item pointers */ 2969c82f823cSDavid Sterba btrfs_init_map_token(&token, right); 29707518a238SChris Mason right_nritems += push_items; 29715f39d397SChris Mason btrfs_set_header_nritems(right, right_nritems); 29720b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info); 29737518a238SChris Mason for (i = 0; i < right_nritems; i++) { 29743212fa14SJosef Bacik push_space -= btrfs_token_item_size(&token, i); 29753212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, push_space); 2976db94535dSChris Mason } 2977db94535dSChris Mason 29787518a238SChris Mason left_nritems -= push_items; 29795f39d397SChris Mason btrfs_set_header_nritems(left, left_nritems); 298000ec4c51SChris Mason 298134a38218SChris Mason if (left_nritems) 29825f39d397SChris Mason btrfs_mark_buffer_dirty(left); 2983f0486c68SYan, Zheng else 29846a884d7dSDavid Sterba btrfs_clean_tree_block(left); 2985f0486c68SYan, Zheng 29865f39d397SChris Mason btrfs_mark_buffer_dirty(right); 2987a429e513SChris Mason 29885f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 29895f39d397SChris Mason btrfs_set_node_key(upper, &disk_key, slot + 1); 2990d6025579SChris Mason btrfs_mark_buffer_dirty(upper); 299102217ed2SChris Mason 299200ec4c51SChris Mason /* then fixup the leaf pointer in the path */ 29937518a238SChris Mason if (path->slots[0] >= left_nritems) { 29947518a238SChris Mason path->slots[0] -= left_nritems; 2995925baeddSChris Mason if (btrfs_header_nritems(path->nodes[0]) == 0) 29966a884d7dSDavid Sterba btrfs_clean_tree_block(path->nodes[0]); 2997925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 29985f39d397SChris Mason free_extent_buffer(path->nodes[0]); 29995f39d397SChris Mason path->nodes[0] = right; 300000ec4c51SChris Mason path->slots[1] += 1; 300100ec4c51SChris Mason } else { 3002925baeddSChris Mason btrfs_tree_unlock(right); 30035f39d397SChris Mason free_extent_buffer(right); 300400ec4c51SChris Mason } 300500ec4c51SChris Mason return 0; 3006925baeddSChris Mason 3007925baeddSChris Mason out_unlock: 3008925baeddSChris Mason btrfs_tree_unlock(right); 3009925baeddSChris Mason free_extent_buffer(right); 3010925baeddSChris Mason return 1; 301100ec4c51SChris Mason } 3012925baeddSChris Mason 301300ec4c51SChris Mason /* 301444871b1bSChris Mason * push some data in the path leaf to the right, trying to free up at 301574123bd7SChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 301644871b1bSChris Mason * 301744871b1bSChris Mason * returns 1 if the push failed because the other node didn't have enough 301844871b1bSChris Mason * room, 0 if everything worked out and < 0 if there were major errors. 301999d8f83cSChris Mason * 302099d8f83cSChris Mason * this will push starting from min_slot to the end of the leaf. It won't 302199d8f83cSChris Mason * push any slot lower than min_slot 302274123bd7SChris Mason */ 302344871b1bSChris Mason static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root 302499d8f83cSChris Mason *root, struct btrfs_path *path, 302599d8f83cSChris Mason int min_data_size, int data_size, 302699d8f83cSChris Mason int empty, u32 min_slot) 3027be0e5c09SChris Mason { 302844871b1bSChris Mason struct extent_buffer *left = path->nodes[0]; 302944871b1bSChris Mason struct extent_buffer *right; 303044871b1bSChris Mason struct extent_buffer *upper; 303144871b1bSChris Mason int slot; 303244871b1bSChris Mason int free_space; 303344871b1bSChris Mason u32 left_nritems; 303444871b1bSChris Mason int ret; 303544871b1bSChris Mason 303644871b1bSChris Mason if (!path->nodes[1]) 303744871b1bSChris Mason return 1; 303844871b1bSChris Mason 303944871b1bSChris Mason slot = path->slots[1]; 304044871b1bSChris Mason upper = path->nodes[1]; 304144871b1bSChris Mason if (slot >= btrfs_header_nritems(upper) - 1) 304244871b1bSChris Mason return 1; 304344871b1bSChris Mason 304449d0c642SFilipe Manana btrfs_assert_tree_write_locked(path->nodes[1]); 304544871b1bSChris Mason 30464b231ae4SDavid Sterba right = btrfs_read_node_slot(upper, slot + 1); 3047fb770ae4SLiu Bo /* 3048fb770ae4SLiu Bo * slot + 1 is not valid or we fail to read the right node, 3049fb770ae4SLiu Bo * no big deal, just return. 3050fb770ae4SLiu Bo */ 3051fb770ae4SLiu Bo if (IS_ERR(right)) 305291ca338dSTsutomu Itoh return 1; 305391ca338dSTsutomu Itoh 3054bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT); 305544871b1bSChris Mason 3056e902baacSDavid Sterba free_space = btrfs_leaf_free_space(right); 305744871b1bSChris Mason if (free_space < data_size) 305844871b1bSChris Mason goto out_unlock; 305944871b1bSChris Mason 306044871b1bSChris Mason ret = btrfs_cow_block(trans, root, right, upper, 3061bf59a5a2SJosef Bacik slot + 1, &right, BTRFS_NESTING_RIGHT_COW); 306244871b1bSChris Mason if (ret) 306344871b1bSChris Mason goto out_unlock; 306444871b1bSChris Mason 306544871b1bSChris Mason left_nritems = btrfs_header_nritems(left); 306644871b1bSChris Mason if (left_nritems == 0) 306744871b1bSChris Mason goto out_unlock; 306844871b1bSChris Mason 3069d16c702fSQu Wenruo if (check_sibling_keys(left, right)) { 3070d16c702fSQu Wenruo ret = -EUCLEAN; 3071d16c702fSQu Wenruo btrfs_tree_unlock(right); 3072d16c702fSQu Wenruo free_extent_buffer(right); 3073d16c702fSQu Wenruo return ret; 3074d16c702fSQu Wenruo } 30752ef1fed2SFilipe David Borba Manana if (path->slots[0] == left_nritems && !empty) { 30762ef1fed2SFilipe David Borba Manana /* Key greater than all keys in the leaf, right neighbor has 30772ef1fed2SFilipe David Borba Manana * enough room for it and we're not emptying our leaf to delete 30782ef1fed2SFilipe David Borba Manana * it, therefore use right neighbor to insert the new item and 307952042d8eSAndrea Gelmini * no need to touch/dirty our left leaf. */ 30802ef1fed2SFilipe David Borba Manana btrfs_tree_unlock(left); 30812ef1fed2SFilipe David Borba Manana free_extent_buffer(left); 30822ef1fed2SFilipe David Borba Manana path->nodes[0] = right; 30832ef1fed2SFilipe David Borba Manana path->slots[0] = 0; 30842ef1fed2SFilipe David Borba Manana path->slots[1]++; 30852ef1fed2SFilipe David Borba Manana return 0; 30862ef1fed2SFilipe David Borba Manana } 30872ef1fed2SFilipe David Borba Manana 3088f72f0010SDavid Sterba return __push_leaf_right(path, min_data_size, empty, 308999d8f83cSChris Mason right, free_space, left_nritems, min_slot); 309044871b1bSChris Mason out_unlock: 309144871b1bSChris Mason btrfs_tree_unlock(right); 309244871b1bSChris Mason free_extent_buffer(right); 309344871b1bSChris Mason return 1; 309444871b1bSChris Mason } 309544871b1bSChris Mason 309644871b1bSChris Mason /* 309744871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 309844871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 309999d8f83cSChris Mason * 310099d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 310199d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us do all the 310299d8f83cSChris Mason * items 310344871b1bSChris Mason */ 31048087c193SDavid Sterba static noinline int __push_leaf_left(struct btrfs_path *path, int data_size, 310544871b1bSChris Mason int empty, struct extent_buffer *left, 310699d8f83cSChris Mason int free_space, u32 right_nritems, 310799d8f83cSChris Mason u32 max_slot) 310844871b1bSChris Mason { 31098087c193SDavid Sterba struct btrfs_fs_info *fs_info = left->fs_info; 31105f39d397SChris Mason struct btrfs_disk_key disk_key; 31115f39d397SChris Mason struct extent_buffer *right = path->nodes[0]; 3112be0e5c09SChris Mason int i; 3113be0e5c09SChris Mason int push_space = 0; 3114be0e5c09SChris Mason int push_items = 0; 31157518a238SChris Mason u32 old_left_nritems; 311634a38218SChris Mason u32 nr; 3117aa5d6bedSChris Mason int ret = 0; 3118db94535dSChris Mason u32 this_item_size; 3119db94535dSChris Mason u32 old_left_item_size; 3120cfed81a0SChris Mason struct btrfs_map_token token; 3121cfed81a0SChris Mason 312234a38218SChris Mason if (empty) 312399d8f83cSChris Mason nr = min(right_nritems, max_slot); 312434a38218SChris Mason else 312599d8f83cSChris Mason nr = min(right_nritems - 1, max_slot); 312634a38218SChris Mason 312734a38218SChris Mason for (i = 0; i < nr; i++) { 312831840ae1SZheng Yan if (!empty && push_items > 0) { 312931840ae1SZheng Yan if (path->slots[0] < i) 313031840ae1SZheng Yan break; 313131840ae1SZheng Yan if (path->slots[0] == i) { 3132e902baacSDavid Sterba int space = btrfs_leaf_free_space(right); 3133e902baacSDavid Sterba 313431840ae1SZheng Yan if (space + push_space * 2 > free_space) 313531840ae1SZheng Yan break; 313631840ae1SZheng Yan } 313731840ae1SZheng Yan } 313831840ae1SZheng Yan 3139be0e5c09SChris Mason if (path->slots[0] == i) 314087b29b20SYan Zheng push_space += data_size; 3141db94535dSChris Mason 31423212fa14SJosef Bacik this_item_size = btrfs_item_size(right, i); 314374794207SJosef Bacik if (this_item_size + sizeof(struct btrfs_item) + push_space > 314474794207SJosef Bacik free_space) 3145be0e5c09SChris Mason break; 3146db94535dSChris Mason 3147be0e5c09SChris Mason push_items++; 314874794207SJosef Bacik push_space += this_item_size + sizeof(struct btrfs_item); 3149be0e5c09SChris Mason } 3150db94535dSChris Mason 3151be0e5c09SChris Mason if (push_items == 0) { 3152925baeddSChris Mason ret = 1; 3153925baeddSChris Mason goto out; 3154be0e5c09SChris Mason } 3155fae7f21cSDulshani Gunawardhana WARN_ON(!empty && push_items == btrfs_header_nritems(right)); 31565f39d397SChris Mason 3157be0e5c09SChris Mason /* push data from right to left */ 31585f39d397SChris Mason copy_extent_buffer(left, right, 31595f39d397SChris Mason btrfs_item_nr_offset(btrfs_header_nritems(left)), 31605f39d397SChris Mason btrfs_item_nr_offset(0), 31615f39d397SChris Mason push_items * sizeof(struct btrfs_item)); 31625f39d397SChris Mason 31630b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info) - 31643212fa14SJosef Bacik btrfs_item_offset(right, push_items - 1); 31655f39d397SChris Mason 31663d9ec8c4SNikolay Borisov copy_extent_buffer(left, right, BTRFS_LEAF_DATA_OFFSET + 31678f881e8cSDavid Sterba leaf_data_end(left) - push_space, 31683d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + 31693212fa14SJosef Bacik btrfs_item_offset(right, push_items - 1), 3170be0e5c09SChris Mason push_space); 31715f39d397SChris Mason old_left_nritems = btrfs_header_nritems(left); 317287b29b20SYan Zheng BUG_ON(old_left_nritems <= 0); 3173eb60ceacSChris Mason 3174c82f823cSDavid Sterba btrfs_init_map_token(&token, left); 31753212fa14SJosef Bacik old_left_item_size = btrfs_item_offset(left, old_left_nritems - 1); 3176be0e5c09SChris Mason for (i = old_left_nritems; i < old_left_nritems + push_items; i++) { 31775f39d397SChris Mason u32 ioff; 3178db94535dSChris Mason 31793212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 31803212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, 3181cc4c13d5SDavid Sterba ioff - (BTRFS_LEAF_DATA_SIZE(fs_info) - old_left_item_size)); 3182be0e5c09SChris Mason } 31835f39d397SChris Mason btrfs_set_header_nritems(left, old_left_nritems + push_items); 3184be0e5c09SChris Mason 3185be0e5c09SChris Mason /* fixup right node */ 318631b1a2bdSJulia Lawall if (push_items > right_nritems) 318731b1a2bdSJulia Lawall WARN(1, KERN_CRIT "push items %d nr %u\n", push_items, 3188d397712bSChris Mason right_nritems); 318934a38218SChris Mason 319034a38218SChris Mason if (push_items < right_nritems) { 31913212fa14SJosef Bacik push_space = btrfs_item_offset(right, push_items - 1) - 31928f881e8cSDavid Sterba leaf_data_end(right); 31933d9ec8c4SNikolay Borisov memmove_extent_buffer(right, BTRFS_LEAF_DATA_OFFSET + 31940b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - push_space, 31953d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + 31968f881e8cSDavid Sterba leaf_data_end(right), push_space); 31975f39d397SChris Mason 31985f39d397SChris Mason memmove_extent_buffer(right, btrfs_item_nr_offset(0), 31995f39d397SChris Mason btrfs_item_nr_offset(push_items), 32005f39d397SChris Mason (btrfs_header_nritems(right) - push_items) * 32010783fcfcSChris Mason sizeof(struct btrfs_item)); 320234a38218SChris Mason } 3203c82f823cSDavid Sterba 3204c82f823cSDavid Sterba btrfs_init_map_token(&token, right); 3205eef1c494SYan right_nritems -= push_items; 3206eef1c494SYan btrfs_set_header_nritems(right, right_nritems); 32070b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info); 32085f39d397SChris Mason for (i = 0; i < right_nritems; i++) { 32093212fa14SJosef Bacik push_space = push_space - btrfs_token_item_size(&token, i); 32103212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, push_space); 3211db94535dSChris Mason } 3212eb60ceacSChris Mason 32135f39d397SChris Mason btrfs_mark_buffer_dirty(left); 321434a38218SChris Mason if (right_nritems) 32155f39d397SChris Mason btrfs_mark_buffer_dirty(right); 3216f0486c68SYan, Zheng else 32176a884d7dSDavid Sterba btrfs_clean_tree_block(right); 3218098f59c2SChris Mason 32195f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 3220b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 3221be0e5c09SChris Mason 3222be0e5c09SChris Mason /* then fixup the leaf pointer in the path */ 3223be0e5c09SChris Mason if (path->slots[0] < push_items) { 3224be0e5c09SChris Mason path->slots[0] += old_left_nritems; 3225925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 32265f39d397SChris Mason free_extent_buffer(path->nodes[0]); 32275f39d397SChris Mason path->nodes[0] = left; 3228be0e5c09SChris Mason path->slots[1] -= 1; 3229be0e5c09SChris Mason } else { 3230925baeddSChris Mason btrfs_tree_unlock(left); 32315f39d397SChris Mason free_extent_buffer(left); 3232be0e5c09SChris Mason path->slots[0] -= push_items; 3233be0e5c09SChris Mason } 3234eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 3235aa5d6bedSChris Mason return ret; 3236925baeddSChris Mason out: 3237925baeddSChris Mason btrfs_tree_unlock(left); 3238925baeddSChris Mason free_extent_buffer(left); 3239925baeddSChris Mason return ret; 3240be0e5c09SChris Mason } 3241be0e5c09SChris Mason 324274123bd7SChris Mason /* 324344871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 324444871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 324599d8f83cSChris Mason * 324699d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 324799d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us push all the 324899d8f83cSChris Mason * items 324944871b1bSChris Mason */ 325044871b1bSChris Mason static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root 325199d8f83cSChris Mason *root, struct btrfs_path *path, int min_data_size, 325299d8f83cSChris Mason int data_size, int empty, u32 max_slot) 325344871b1bSChris Mason { 325444871b1bSChris Mason struct extent_buffer *right = path->nodes[0]; 325544871b1bSChris Mason struct extent_buffer *left; 325644871b1bSChris Mason int slot; 325744871b1bSChris Mason int free_space; 325844871b1bSChris Mason u32 right_nritems; 325944871b1bSChris Mason int ret = 0; 326044871b1bSChris Mason 326144871b1bSChris Mason slot = path->slots[1]; 326244871b1bSChris Mason if (slot == 0) 326344871b1bSChris Mason return 1; 326444871b1bSChris Mason if (!path->nodes[1]) 326544871b1bSChris Mason return 1; 326644871b1bSChris Mason 326744871b1bSChris Mason right_nritems = btrfs_header_nritems(right); 326844871b1bSChris Mason if (right_nritems == 0) 326944871b1bSChris Mason return 1; 327044871b1bSChris Mason 327149d0c642SFilipe Manana btrfs_assert_tree_write_locked(path->nodes[1]); 327244871b1bSChris Mason 32734b231ae4SDavid Sterba left = btrfs_read_node_slot(path->nodes[1], slot - 1); 3274fb770ae4SLiu Bo /* 3275fb770ae4SLiu Bo * slot - 1 is not valid or we fail to read the left node, 3276fb770ae4SLiu Bo * no big deal, just return. 3277fb770ae4SLiu Bo */ 3278fb770ae4SLiu Bo if (IS_ERR(left)) 327991ca338dSTsutomu Itoh return 1; 328091ca338dSTsutomu Itoh 3281bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT); 328244871b1bSChris Mason 3283e902baacSDavid Sterba free_space = btrfs_leaf_free_space(left); 328444871b1bSChris Mason if (free_space < data_size) { 328544871b1bSChris Mason ret = 1; 328644871b1bSChris Mason goto out; 328744871b1bSChris Mason } 328844871b1bSChris Mason 328944871b1bSChris Mason ret = btrfs_cow_block(trans, root, left, 32909631e4ccSJosef Bacik path->nodes[1], slot - 1, &left, 3291bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW); 329244871b1bSChris Mason if (ret) { 329344871b1bSChris Mason /* we hit -ENOSPC, but it isn't fatal here */ 329479787eaaSJeff Mahoney if (ret == -ENOSPC) 329544871b1bSChris Mason ret = 1; 329644871b1bSChris Mason goto out; 329744871b1bSChris Mason } 329844871b1bSChris Mason 3299d16c702fSQu Wenruo if (check_sibling_keys(left, right)) { 3300d16c702fSQu Wenruo ret = -EUCLEAN; 3301d16c702fSQu Wenruo goto out; 3302d16c702fSQu Wenruo } 33038087c193SDavid Sterba return __push_leaf_left(path, min_data_size, 330499d8f83cSChris Mason empty, left, free_space, right_nritems, 330599d8f83cSChris Mason max_slot); 330644871b1bSChris Mason out: 330744871b1bSChris Mason btrfs_tree_unlock(left); 330844871b1bSChris Mason free_extent_buffer(left); 330944871b1bSChris Mason return ret; 331044871b1bSChris Mason } 331144871b1bSChris Mason 331244871b1bSChris Mason /* 331374123bd7SChris Mason * split the path's leaf in two, making sure there is at least data_size 331474123bd7SChris Mason * available for the resulting leaf level of the path. 331574123bd7SChris Mason */ 3316143bede5SJeff Mahoney static noinline void copy_for_split(struct btrfs_trans_handle *trans, 331744871b1bSChris Mason struct btrfs_path *path, 331844871b1bSChris Mason struct extent_buffer *l, 331944871b1bSChris Mason struct extent_buffer *right, 332044871b1bSChris Mason int slot, int mid, int nritems) 3321be0e5c09SChris Mason { 332294f94ad9SDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info; 3323be0e5c09SChris Mason int data_copy_size; 3324be0e5c09SChris Mason int rt_data_off; 3325be0e5c09SChris Mason int i; 3326d4dbff95SChris Mason struct btrfs_disk_key disk_key; 3327cfed81a0SChris Mason struct btrfs_map_token token; 3328cfed81a0SChris Mason 33295f39d397SChris Mason nritems = nritems - mid; 33305f39d397SChris Mason btrfs_set_header_nritems(right, nritems); 3331dc2e724eSJosef Bacik data_copy_size = btrfs_item_data_end(l, mid) - leaf_data_end(l); 33325f39d397SChris Mason 33335f39d397SChris Mason copy_extent_buffer(right, l, btrfs_item_nr_offset(0), 33345f39d397SChris Mason btrfs_item_nr_offset(mid), 33355f39d397SChris Mason nritems * sizeof(struct btrfs_item)); 33365f39d397SChris Mason 33375f39d397SChris Mason copy_extent_buffer(right, l, 33383d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + BTRFS_LEAF_DATA_SIZE(fs_info) - 33393d9ec8c4SNikolay Borisov data_copy_size, BTRFS_LEAF_DATA_OFFSET + 33408f881e8cSDavid Sterba leaf_data_end(l), data_copy_size); 334174123bd7SChris Mason 3342dc2e724eSJosef Bacik rt_data_off = BTRFS_LEAF_DATA_SIZE(fs_info) - btrfs_item_data_end(l, mid); 33435f39d397SChris Mason 3344c82f823cSDavid Sterba btrfs_init_map_token(&token, right); 33455f39d397SChris Mason for (i = 0; i < nritems; i++) { 3346db94535dSChris Mason u32 ioff; 3347db94535dSChris Mason 33483212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 33493212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff + rt_data_off); 33500783fcfcSChris Mason } 335174123bd7SChris Mason 33525f39d397SChris Mason btrfs_set_header_nritems(l, mid); 33535f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 33546ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, right->start, path->slots[1] + 1, 1); 33555f39d397SChris Mason 33565f39d397SChris Mason btrfs_mark_buffer_dirty(right); 33575f39d397SChris Mason btrfs_mark_buffer_dirty(l); 3358eb60ceacSChris Mason BUG_ON(path->slots[0] != slot); 33595f39d397SChris Mason 3360be0e5c09SChris Mason if (mid <= slot) { 3361925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 33625f39d397SChris Mason free_extent_buffer(path->nodes[0]); 33635f39d397SChris Mason path->nodes[0] = right; 3364be0e5c09SChris Mason path->slots[0] -= mid; 3365be0e5c09SChris Mason path->slots[1] += 1; 3366925baeddSChris Mason } else { 3367925baeddSChris Mason btrfs_tree_unlock(right); 33685f39d397SChris Mason free_extent_buffer(right); 3369925baeddSChris Mason } 33705f39d397SChris Mason 3371eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 337244871b1bSChris Mason } 337344871b1bSChris Mason 337444871b1bSChris Mason /* 337599d8f83cSChris Mason * double splits happen when we need to insert a big item in the middle 337699d8f83cSChris Mason * of a leaf. A double split can leave us with 3 mostly empty leaves: 337799d8f83cSChris Mason * leaf: [ slots 0 - N] [ our target ] [ N + 1 - total in leaf ] 337899d8f83cSChris Mason * A B C 337999d8f83cSChris Mason * 338099d8f83cSChris Mason * We avoid this by trying to push the items on either side of our target 338199d8f83cSChris Mason * into the adjacent leaves. If all goes well we can avoid the double split 338299d8f83cSChris Mason * completely. 338399d8f83cSChris Mason */ 338499d8f83cSChris Mason static noinline int push_for_double_split(struct btrfs_trans_handle *trans, 338599d8f83cSChris Mason struct btrfs_root *root, 338699d8f83cSChris Mason struct btrfs_path *path, 338799d8f83cSChris Mason int data_size) 338899d8f83cSChris Mason { 338999d8f83cSChris Mason int ret; 339099d8f83cSChris Mason int progress = 0; 339199d8f83cSChris Mason int slot; 339299d8f83cSChris Mason u32 nritems; 33935a4267caSFilipe David Borba Manana int space_needed = data_size; 339499d8f83cSChris Mason 339599d8f83cSChris Mason slot = path->slots[0]; 33965a4267caSFilipe David Borba Manana if (slot < btrfs_header_nritems(path->nodes[0])) 3397e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(path->nodes[0]); 339899d8f83cSChris Mason 339999d8f83cSChris Mason /* 340099d8f83cSChris Mason * try to push all the items after our slot into the 340199d8f83cSChris Mason * right leaf 340299d8f83cSChris Mason */ 34035a4267caSFilipe David Borba Manana ret = push_leaf_right(trans, root, path, 1, space_needed, 0, slot); 340499d8f83cSChris Mason if (ret < 0) 340599d8f83cSChris Mason return ret; 340699d8f83cSChris Mason 340799d8f83cSChris Mason if (ret == 0) 340899d8f83cSChris Mason progress++; 340999d8f83cSChris Mason 341099d8f83cSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 341199d8f83cSChris Mason /* 341299d8f83cSChris Mason * our goal is to get our slot at the start or end of a leaf. If 341399d8f83cSChris Mason * we've done so we're done 341499d8f83cSChris Mason */ 341599d8f83cSChris Mason if (path->slots[0] == 0 || path->slots[0] == nritems) 341699d8f83cSChris Mason return 0; 341799d8f83cSChris Mason 3418e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= data_size) 341999d8f83cSChris Mason return 0; 342099d8f83cSChris Mason 342199d8f83cSChris Mason /* try to push all the items before our slot into the next leaf */ 342299d8f83cSChris Mason slot = path->slots[0]; 3423263d3995SFilipe Manana space_needed = data_size; 3424263d3995SFilipe Manana if (slot > 0) 3425e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(path->nodes[0]); 34265a4267caSFilipe David Borba Manana ret = push_leaf_left(trans, root, path, 1, space_needed, 0, slot); 342799d8f83cSChris Mason if (ret < 0) 342899d8f83cSChris Mason return ret; 342999d8f83cSChris Mason 343099d8f83cSChris Mason if (ret == 0) 343199d8f83cSChris Mason progress++; 343299d8f83cSChris Mason 343399d8f83cSChris Mason if (progress) 343499d8f83cSChris Mason return 0; 343599d8f83cSChris Mason return 1; 343699d8f83cSChris Mason } 343799d8f83cSChris Mason 343899d8f83cSChris Mason /* 343944871b1bSChris Mason * split the path's leaf in two, making sure there is at least data_size 344044871b1bSChris Mason * available for the resulting leaf level of the path. 344144871b1bSChris Mason * 344244871b1bSChris Mason * returns 0 if all went well and < 0 on failure. 344344871b1bSChris Mason */ 344444871b1bSChris Mason static noinline int split_leaf(struct btrfs_trans_handle *trans, 344544871b1bSChris Mason struct btrfs_root *root, 3446310712b2SOmar Sandoval const struct btrfs_key *ins_key, 344744871b1bSChris Mason struct btrfs_path *path, int data_size, 344844871b1bSChris Mason int extend) 344944871b1bSChris Mason { 34505d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 345144871b1bSChris Mason struct extent_buffer *l; 345244871b1bSChris Mason u32 nritems; 345344871b1bSChris Mason int mid; 345444871b1bSChris Mason int slot; 345544871b1bSChris Mason struct extent_buffer *right; 3456b7a0365eSDaniel Dressler struct btrfs_fs_info *fs_info = root->fs_info; 345744871b1bSChris Mason int ret = 0; 345844871b1bSChris Mason int wret; 34595d4f98a2SYan Zheng int split; 346044871b1bSChris Mason int num_doubles = 0; 346199d8f83cSChris Mason int tried_avoid_double = 0; 346244871b1bSChris Mason 3463a5719521SYan, Zheng l = path->nodes[0]; 3464a5719521SYan, Zheng slot = path->slots[0]; 34653212fa14SJosef Bacik if (extend && data_size + btrfs_item_size(l, slot) + 34660b246afaSJeff Mahoney sizeof(struct btrfs_item) > BTRFS_LEAF_DATA_SIZE(fs_info)) 3467a5719521SYan, Zheng return -EOVERFLOW; 3468a5719521SYan, Zheng 346944871b1bSChris Mason /* first try to make some room by pushing left and right */ 347033157e05SLiu Bo if (data_size && path->nodes[1]) { 34715a4267caSFilipe David Borba Manana int space_needed = data_size; 34725a4267caSFilipe David Borba Manana 34735a4267caSFilipe David Borba Manana if (slot < btrfs_header_nritems(l)) 3474e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(l); 34755a4267caSFilipe David Borba Manana 34765a4267caSFilipe David Borba Manana wret = push_leaf_right(trans, root, path, space_needed, 34775a4267caSFilipe David Borba Manana space_needed, 0, 0); 347844871b1bSChris Mason if (wret < 0) 347944871b1bSChris Mason return wret; 348044871b1bSChris Mason if (wret) { 3481263d3995SFilipe Manana space_needed = data_size; 3482263d3995SFilipe Manana if (slot > 0) 3483e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(l); 34845a4267caSFilipe David Borba Manana wret = push_leaf_left(trans, root, path, space_needed, 34855a4267caSFilipe David Borba Manana space_needed, 0, (u32)-1); 348644871b1bSChris Mason if (wret < 0) 348744871b1bSChris Mason return wret; 348844871b1bSChris Mason } 348944871b1bSChris Mason l = path->nodes[0]; 349044871b1bSChris Mason 349144871b1bSChris Mason /* did the pushes work? */ 3492e902baacSDavid Sterba if (btrfs_leaf_free_space(l) >= data_size) 349344871b1bSChris Mason return 0; 349444871b1bSChris Mason } 349544871b1bSChris Mason 349644871b1bSChris Mason if (!path->nodes[1]) { 3497fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, 1); 349844871b1bSChris Mason if (ret) 349944871b1bSChris Mason return ret; 350044871b1bSChris Mason } 350144871b1bSChris Mason again: 35025d4f98a2SYan Zheng split = 1; 350344871b1bSChris Mason l = path->nodes[0]; 350444871b1bSChris Mason slot = path->slots[0]; 350544871b1bSChris Mason nritems = btrfs_header_nritems(l); 350644871b1bSChris Mason mid = (nritems + 1) / 2; 350744871b1bSChris Mason 35085d4f98a2SYan Zheng if (mid <= slot) { 35095d4f98a2SYan Zheng if (nritems == 1 || 35105d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + data_size > 35110b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info)) { 35125d4f98a2SYan Zheng if (slot >= nritems) { 35135d4f98a2SYan Zheng split = 0; 35145d4f98a2SYan Zheng } else { 35155d4f98a2SYan Zheng mid = slot; 35165d4f98a2SYan Zheng if (mid != nritems && 35175d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 35180b246afaSJeff Mahoney data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) { 351999d8f83cSChris Mason if (data_size && !tried_avoid_double) 352099d8f83cSChris Mason goto push_for_double; 35215d4f98a2SYan Zheng split = 2; 35225d4f98a2SYan Zheng } 35235d4f98a2SYan Zheng } 35245d4f98a2SYan Zheng } 35255d4f98a2SYan Zheng } else { 35265d4f98a2SYan Zheng if (leaf_space_used(l, 0, mid) + data_size > 35270b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info)) { 35285d4f98a2SYan Zheng if (!extend && data_size && slot == 0) { 35295d4f98a2SYan Zheng split = 0; 35305d4f98a2SYan Zheng } else if ((extend || !data_size) && slot == 0) { 35315d4f98a2SYan Zheng mid = 1; 35325d4f98a2SYan Zheng } else { 35335d4f98a2SYan Zheng mid = slot; 35345d4f98a2SYan Zheng if (mid != nritems && 35355d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 35360b246afaSJeff Mahoney data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) { 353799d8f83cSChris Mason if (data_size && !tried_avoid_double) 353899d8f83cSChris Mason goto push_for_double; 35395d4f98a2SYan Zheng split = 2; 35405d4f98a2SYan Zheng } 35415d4f98a2SYan Zheng } 35425d4f98a2SYan Zheng } 35435d4f98a2SYan Zheng } 35445d4f98a2SYan Zheng 35455d4f98a2SYan Zheng if (split == 0) 35465d4f98a2SYan Zheng btrfs_cpu_key_to_disk(&disk_key, ins_key); 35475d4f98a2SYan Zheng else 35485d4f98a2SYan Zheng btrfs_item_key(l, &disk_key, mid); 35495d4f98a2SYan Zheng 3550ca9d473aSJosef Bacik /* 3551ca9d473aSJosef Bacik * We have to about BTRFS_NESTING_NEW_ROOT here if we've done a double 3552ca9d473aSJosef Bacik * split, because we're only allowed to have MAX_LOCKDEP_SUBCLASSES 3553ca9d473aSJosef Bacik * subclasses, which is 8 at the time of this patch, and we've maxed it 3554ca9d473aSJosef Bacik * out. In the future we could add a 3555ca9d473aSJosef Bacik * BTRFS_NESTING_SPLIT_THE_SPLITTENING if we need to, but for now just 3556ca9d473aSJosef Bacik * use BTRFS_NESTING_NEW_ROOT. 3557ca9d473aSJosef Bacik */ 355879bd3712SFilipe Manana right = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 355979bd3712SFilipe Manana &disk_key, 0, l->start, 0, 356079bd3712SFilipe Manana num_doubles ? BTRFS_NESTING_NEW_ROOT : 3561ca9d473aSJosef Bacik BTRFS_NESTING_SPLIT); 3562f0486c68SYan, Zheng if (IS_ERR(right)) 356344871b1bSChris Mason return PTR_ERR(right); 3564f0486c68SYan, Zheng 35650b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 356644871b1bSChris Mason 35675d4f98a2SYan Zheng if (split == 0) { 356844871b1bSChris Mason if (mid <= slot) { 356944871b1bSChris Mason btrfs_set_header_nritems(right, 0); 35706ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, 35712ff7e61eSJeff Mahoney right->start, path->slots[1] + 1, 1); 357244871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 357344871b1bSChris Mason free_extent_buffer(path->nodes[0]); 357444871b1bSChris Mason path->nodes[0] = right; 357544871b1bSChris Mason path->slots[0] = 0; 357644871b1bSChris Mason path->slots[1] += 1; 357744871b1bSChris Mason } else { 357844871b1bSChris Mason btrfs_set_header_nritems(right, 0); 35796ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, 35802ff7e61eSJeff Mahoney right->start, path->slots[1], 1); 358144871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 358244871b1bSChris Mason free_extent_buffer(path->nodes[0]); 358344871b1bSChris Mason path->nodes[0] = right; 358444871b1bSChris Mason path->slots[0] = 0; 3585143bede5SJeff Mahoney if (path->slots[1] == 0) 3586b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 35875d4f98a2SYan Zheng } 3588196e0249SLiu Bo /* 3589196e0249SLiu Bo * We create a new leaf 'right' for the required ins_len and 3590196e0249SLiu Bo * we'll do btrfs_mark_buffer_dirty() on this leaf after copying 3591196e0249SLiu Bo * the content of ins_len to 'right'. 3592196e0249SLiu Bo */ 359344871b1bSChris Mason return ret; 359444871b1bSChris Mason } 359544871b1bSChris Mason 359694f94ad9SDavid Sterba copy_for_split(trans, path, l, right, slot, mid, nritems); 359744871b1bSChris Mason 35985d4f98a2SYan Zheng if (split == 2) { 3599cc0c5538SChris Mason BUG_ON(num_doubles != 0); 3600cc0c5538SChris Mason num_doubles++; 3601cc0c5538SChris Mason goto again; 36023326d1b0SChris Mason } 360344871b1bSChris Mason 3604143bede5SJeff Mahoney return 0; 360599d8f83cSChris Mason 360699d8f83cSChris Mason push_for_double: 360799d8f83cSChris Mason push_for_double_split(trans, root, path, data_size); 360899d8f83cSChris Mason tried_avoid_double = 1; 3609e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= data_size) 361099d8f83cSChris Mason return 0; 361199d8f83cSChris Mason goto again; 3612be0e5c09SChris Mason } 3613be0e5c09SChris Mason 3614ad48fd75SYan, Zheng static noinline int setup_leaf_for_split(struct btrfs_trans_handle *trans, 3615ad48fd75SYan, Zheng struct btrfs_root *root, 3616ad48fd75SYan, Zheng struct btrfs_path *path, int ins_len) 3617ad48fd75SYan, Zheng { 3618ad48fd75SYan, Zheng struct btrfs_key key; 3619ad48fd75SYan, Zheng struct extent_buffer *leaf; 3620ad48fd75SYan, Zheng struct btrfs_file_extent_item *fi; 3621ad48fd75SYan, Zheng u64 extent_len = 0; 3622ad48fd75SYan, Zheng u32 item_size; 3623ad48fd75SYan, Zheng int ret; 3624ad48fd75SYan, Zheng 3625ad48fd75SYan, Zheng leaf = path->nodes[0]; 3626ad48fd75SYan, Zheng btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 3627ad48fd75SYan, Zheng 3628ad48fd75SYan, Zheng BUG_ON(key.type != BTRFS_EXTENT_DATA_KEY && 3629ad48fd75SYan, Zheng key.type != BTRFS_EXTENT_CSUM_KEY); 3630ad48fd75SYan, Zheng 3631e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) >= ins_len) 3632ad48fd75SYan, Zheng return 0; 3633ad48fd75SYan, Zheng 36343212fa14SJosef Bacik item_size = btrfs_item_size(leaf, path->slots[0]); 3635ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 3636ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 3637ad48fd75SYan, Zheng struct btrfs_file_extent_item); 3638ad48fd75SYan, Zheng extent_len = btrfs_file_extent_num_bytes(leaf, fi); 3639ad48fd75SYan, Zheng } 3640b3b4aa74SDavid Sterba btrfs_release_path(path); 3641ad48fd75SYan, Zheng 3642ad48fd75SYan, Zheng path->keep_locks = 1; 3643ad48fd75SYan, Zheng path->search_for_split = 1; 3644ad48fd75SYan, Zheng ret = btrfs_search_slot(trans, root, &key, path, 0, 1); 3645ad48fd75SYan, Zheng path->search_for_split = 0; 3646a8df6fe6SFilipe Manana if (ret > 0) 3647a8df6fe6SFilipe Manana ret = -EAGAIN; 3648ad48fd75SYan, Zheng if (ret < 0) 3649ad48fd75SYan, Zheng goto err; 3650ad48fd75SYan, Zheng 3651ad48fd75SYan, Zheng ret = -EAGAIN; 3652ad48fd75SYan, Zheng leaf = path->nodes[0]; 3653a8df6fe6SFilipe Manana /* if our item isn't there, return now */ 36543212fa14SJosef Bacik if (item_size != btrfs_item_size(leaf, path->slots[0])) 3655ad48fd75SYan, Zheng goto err; 3656ad48fd75SYan, Zheng 3657109f6aefSChris Mason /* the leaf has changed, it now has room. return now */ 3658e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= ins_len) 3659109f6aefSChris Mason goto err; 3660109f6aefSChris Mason 3661ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 3662ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 3663ad48fd75SYan, Zheng struct btrfs_file_extent_item); 3664ad48fd75SYan, Zheng if (extent_len != btrfs_file_extent_num_bytes(leaf, fi)) 3665ad48fd75SYan, Zheng goto err; 3666ad48fd75SYan, Zheng } 3667ad48fd75SYan, Zheng 3668ad48fd75SYan, Zheng ret = split_leaf(trans, root, &key, path, ins_len, 1); 3669f0486c68SYan, Zheng if (ret) 3670f0486c68SYan, Zheng goto err; 3671ad48fd75SYan, Zheng 3672ad48fd75SYan, Zheng path->keep_locks = 0; 3673ad48fd75SYan, Zheng btrfs_unlock_up_safe(path, 1); 3674ad48fd75SYan, Zheng return 0; 3675ad48fd75SYan, Zheng err: 3676ad48fd75SYan, Zheng path->keep_locks = 0; 3677ad48fd75SYan, Zheng return ret; 3678ad48fd75SYan, Zheng } 3679ad48fd75SYan, Zheng 368025263cd7SDavid Sterba static noinline int split_item(struct btrfs_path *path, 3681310712b2SOmar Sandoval const struct btrfs_key *new_key, 3682459931ecSChris Mason unsigned long split_offset) 3683459931ecSChris Mason { 3684459931ecSChris Mason struct extent_buffer *leaf; 3685c91666b1SJosef Bacik int orig_slot, slot; 3686ad48fd75SYan, Zheng char *buf; 3687459931ecSChris Mason u32 nritems; 3688ad48fd75SYan, Zheng u32 item_size; 3689459931ecSChris Mason u32 orig_offset; 3690459931ecSChris Mason struct btrfs_disk_key disk_key; 3691459931ecSChris Mason 3692459931ecSChris Mason leaf = path->nodes[0]; 3693e902baacSDavid Sterba BUG_ON(btrfs_leaf_free_space(leaf) < sizeof(struct btrfs_item)); 3694b9473439SChris Mason 3695c91666b1SJosef Bacik orig_slot = path->slots[0]; 36963212fa14SJosef Bacik orig_offset = btrfs_item_offset(leaf, path->slots[0]); 36973212fa14SJosef Bacik item_size = btrfs_item_size(leaf, path->slots[0]); 3698459931ecSChris Mason 3699459931ecSChris Mason buf = kmalloc(item_size, GFP_NOFS); 3700ad48fd75SYan, Zheng if (!buf) 3701ad48fd75SYan, Zheng return -ENOMEM; 3702ad48fd75SYan, Zheng 3703459931ecSChris Mason read_extent_buffer(leaf, buf, btrfs_item_ptr_offset(leaf, 3704459931ecSChris Mason path->slots[0]), item_size); 3705ad48fd75SYan, Zheng 3706459931ecSChris Mason slot = path->slots[0] + 1; 3707459931ecSChris Mason nritems = btrfs_header_nritems(leaf); 3708459931ecSChris Mason if (slot != nritems) { 3709459931ecSChris Mason /* shift the items */ 3710459931ecSChris Mason memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + 1), 3711459931ecSChris Mason btrfs_item_nr_offset(slot), 3712459931ecSChris Mason (nritems - slot) * sizeof(struct btrfs_item)); 3713459931ecSChris Mason } 3714459931ecSChris Mason 3715459931ecSChris Mason btrfs_cpu_key_to_disk(&disk_key, new_key); 3716459931ecSChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 3717459931ecSChris Mason 37183212fa14SJosef Bacik btrfs_set_item_offset(leaf, slot, orig_offset); 37193212fa14SJosef Bacik btrfs_set_item_size(leaf, slot, item_size - split_offset); 3720459931ecSChris Mason 37213212fa14SJosef Bacik btrfs_set_item_offset(leaf, orig_slot, 3722459931ecSChris Mason orig_offset + item_size - split_offset); 37233212fa14SJosef Bacik btrfs_set_item_size(leaf, orig_slot, split_offset); 3724459931ecSChris Mason 3725459931ecSChris Mason btrfs_set_header_nritems(leaf, nritems + 1); 3726459931ecSChris Mason 3727459931ecSChris Mason /* write the data for the start of the original item */ 3728459931ecSChris Mason write_extent_buffer(leaf, buf, 3729459931ecSChris Mason btrfs_item_ptr_offset(leaf, path->slots[0]), 3730459931ecSChris Mason split_offset); 3731459931ecSChris Mason 3732459931ecSChris Mason /* write the data for the new item */ 3733459931ecSChris Mason write_extent_buffer(leaf, buf + split_offset, 3734459931ecSChris Mason btrfs_item_ptr_offset(leaf, slot), 3735459931ecSChris Mason item_size - split_offset); 3736459931ecSChris Mason btrfs_mark_buffer_dirty(leaf); 3737459931ecSChris Mason 3738e902baacSDavid Sterba BUG_ON(btrfs_leaf_free_space(leaf) < 0); 3739459931ecSChris Mason kfree(buf); 3740ad48fd75SYan, Zheng return 0; 3741ad48fd75SYan, Zheng } 3742ad48fd75SYan, Zheng 3743ad48fd75SYan, Zheng /* 3744ad48fd75SYan, Zheng * This function splits a single item into two items, 3745ad48fd75SYan, Zheng * giving 'new_key' to the new item and splitting the 3746ad48fd75SYan, Zheng * old one at split_offset (from the start of the item). 3747ad48fd75SYan, Zheng * 3748ad48fd75SYan, Zheng * The path may be released by this operation. After 3749ad48fd75SYan, Zheng * the split, the path is pointing to the old item. The 3750ad48fd75SYan, Zheng * new item is going to be in the same node as the old one. 3751ad48fd75SYan, Zheng * 3752ad48fd75SYan, Zheng * Note, the item being split must be smaller enough to live alone on 3753ad48fd75SYan, Zheng * a tree block with room for one extra struct btrfs_item 3754ad48fd75SYan, Zheng * 3755ad48fd75SYan, Zheng * This allows us to split the item in place, keeping a lock on the 3756ad48fd75SYan, Zheng * leaf the entire time. 3757ad48fd75SYan, Zheng */ 3758ad48fd75SYan, Zheng int btrfs_split_item(struct btrfs_trans_handle *trans, 3759ad48fd75SYan, Zheng struct btrfs_root *root, 3760ad48fd75SYan, Zheng struct btrfs_path *path, 3761310712b2SOmar Sandoval const struct btrfs_key *new_key, 3762ad48fd75SYan, Zheng unsigned long split_offset) 3763ad48fd75SYan, Zheng { 3764ad48fd75SYan, Zheng int ret; 3765ad48fd75SYan, Zheng ret = setup_leaf_for_split(trans, root, path, 3766ad48fd75SYan, Zheng sizeof(struct btrfs_item)); 3767ad48fd75SYan, Zheng if (ret) 3768459931ecSChris Mason return ret; 3769ad48fd75SYan, Zheng 377025263cd7SDavid Sterba ret = split_item(path, new_key, split_offset); 3771ad48fd75SYan, Zheng return ret; 3772ad48fd75SYan, Zheng } 3773ad48fd75SYan, Zheng 3774ad48fd75SYan, Zheng /* 3775d352ac68SChris Mason * make the item pointed to by the path smaller. new_size indicates 3776d352ac68SChris Mason * how small to make it, and from_end tells us if we just chop bytes 3777d352ac68SChris Mason * off the end of the item or if we shift the item to chop bytes off 3778d352ac68SChris Mason * the front. 3779d352ac68SChris Mason */ 378078ac4f9eSDavid Sterba void btrfs_truncate_item(struct btrfs_path *path, u32 new_size, int from_end) 3781b18c6685SChris Mason { 3782b18c6685SChris Mason int slot; 37835f39d397SChris Mason struct extent_buffer *leaf; 3784b18c6685SChris Mason u32 nritems; 3785b18c6685SChris Mason unsigned int data_end; 3786b18c6685SChris Mason unsigned int old_data_start; 3787b18c6685SChris Mason unsigned int old_size; 3788b18c6685SChris Mason unsigned int size_diff; 3789b18c6685SChris Mason int i; 3790cfed81a0SChris Mason struct btrfs_map_token token; 3791cfed81a0SChris Mason 37925f39d397SChris Mason leaf = path->nodes[0]; 3793179e29e4SChris Mason slot = path->slots[0]; 3794179e29e4SChris Mason 37953212fa14SJosef Bacik old_size = btrfs_item_size(leaf, slot); 3796179e29e4SChris Mason if (old_size == new_size) 3797143bede5SJeff Mahoney return; 3798b18c6685SChris Mason 37995f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 38008f881e8cSDavid Sterba data_end = leaf_data_end(leaf); 3801b18c6685SChris Mason 38023212fa14SJosef Bacik old_data_start = btrfs_item_offset(leaf, slot); 3803179e29e4SChris Mason 3804b18c6685SChris Mason size_diff = old_size - new_size; 3805b18c6685SChris Mason 3806b18c6685SChris Mason BUG_ON(slot < 0); 3807b18c6685SChris Mason BUG_ON(slot >= nritems); 3808b18c6685SChris Mason 3809b18c6685SChris Mason /* 3810b18c6685SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 3811b18c6685SChris Mason */ 3812b18c6685SChris Mason /* first correct the data pointers */ 3813c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 3814b18c6685SChris Mason for (i = slot; i < nritems; i++) { 38155f39d397SChris Mason u32 ioff; 3816db94535dSChris Mason 38173212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 38183212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff + size_diff); 3819b18c6685SChris Mason } 3820db94535dSChris Mason 3821b18c6685SChris Mason /* shift the data */ 3822179e29e4SChris Mason if (from_end) { 38233d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 38243d9ec8c4SNikolay Borisov data_end + size_diff, BTRFS_LEAF_DATA_OFFSET + 3825b18c6685SChris Mason data_end, old_data_start + new_size - data_end); 3826179e29e4SChris Mason } else { 3827179e29e4SChris Mason struct btrfs_disk_key disk_key; 3828179e29e4SChris Mason u64 offset; 3829179e29e4SChris Mason 3830179e29e4SChris Mason btrfs_item_key(leaf, &disk_key, slot); 3831179e29e4SChris Mason 3832179e29e4SChris Mason if (btrfs_disk_key_type(&disk_key) == BTRFS_EXTENT_DATA_KEY) { 3833179e29e4SChris Mason unsigned long ptr; 3834179e29e4SChris Mason struct btrfs_file_extent_item *fi; 3835179e29e4SChris Mason 3836179e29e4SChris Mason fi = btrfs_item_ptr(leaf, slot, 3837179e29e4SChris Mason struct btrfs_file_extent_item); 3838179e29e4SChris Mason fi = (struct btrfs_file_extent_item *)( 3839179e29e4SChris Mason (unsigned long)fi - size_diff); 3840179e29e4SChris Mason 3841179e29e4SChris Mason if (btrfs_file_extent_type(leaf, fi) == 3842179e29e4SChris Mason BTRFS_FILE_EXTENT_INLINE) { 3843179e29e4SChris Mason ptr = btrfs_item_ptr_offset(leaf, slot); 3844179e29e4SChris Mason memmove_extent_buffer(leaf, ptr, 3845179e29e4SChris Mason (unsigned long)fi, 38467ec20afbSDavid Sterba BTRFS_FILE_EXTENT_INLINE_DATA_START); 3847179e29e4SChris Mason } 3848179e29e4SChris Mason } 3849179e29e4SChris Mason 38503d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 38513d9ec8c4SNikolay Borisov data_end + size_diff, BTRFS_LEAF_DATA_OFFSET + 3852179e29e4SChris Mason data_end, old_data_start - data_end); 3853179e29e4SChris Mason 3854179e29e4SChris Mason offset = btrfs_disk_key_offset(&disk_key); 3855179e29e4SChris Mason btrfs_set_disk_key_offset(&disk_key, offset + size_diff); 3856179e29e4SChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 3857179e29e4SChris Mason if (slot == 0) 3858b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 3859179e29e4SChris Mason } 38605f39d397SChris Mason 38613212fa14SJosef Bacik btrfs_set_item_size(leaf, slot, new_size); 38625f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 3863b18c6685SChris Mason 3864e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) { 3865a4f78750SDavid Sterba btrfs_print_leaf(leaf); 3866b18c6685SChris Mason BUG(); 38675f39d397SChris Mason } 3868b18c6685SChris Mason } 3869b18c6685SChris Mason 3870d352ac68SChris Mason /* 38718f69dbd2SStefan Behrens * make the item pointed to by the path bigger, data_size is the added size. 3872d352ac68SChris Mason */ 3873c71dd880SDavid Sterba void btrfs_extend_item(struct btrfs_path *path, u32 data_size) 38746567e837SChris Mason { 38756567e837SChris Mason int slot; 38765f39d397SChris Mason struct extent_buffer *leaf; 38776567e837SChris Mason u32 nritems; 38786567e837SChris Mason unsigned int data_end; 38796567e837SChris Mason unsigned int old_data; 38806567e837SChris Mason unsigned int old_size; 38816567e837SChris Mason int i; 3882cfed81a0SChris Mason struct btrfs_map_token token; 3883cfed81a0SChris Mason 38845f39d397SChris Mason leaf = path->nodes[0]; 38856567e837SChris Mason 38865f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 38878f881e8cSDavid Sterba data_end = leaf_data_end(leaf); 38886567e837SChris Mason 3889e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < data_size) { 3890a4f78750SDavid Sterba btrfs_print_leaf(leaf); 38916567e837SChris Mason BUG(); 38925f39d397SChris Mason } 38936567e837SChris Mason slot = path->slots[0]; 3894dc2e724eSJosef Bacik old_data = btrfs_item_data_end(leaf, slot); 38956567e837SChris Mason 38966567e837SChris Mason BUG_ON(slot < 0); 38973326d1b0SChris Mason if (slot >= nritems) { 3898a4f78750SDavid Sterba btrfs_print_leaf(leaf); 3899c71dd880SDavid Sterba btrfs_crit(leaf->fs_info, "slot %d too large, nritems %d", 3900d397712bSChris Mason slot, nritems); 3901290342f6SArnd Bergmann BUG(); 39023326d1b0SChris Mason } 39036567e837SChris Mason 39046567e837SChris Mason /* 39056567e837SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 39066567e837SChris Mason */ 39076567e837SChris Mason /* first correct the data pointers */ 3908c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 39096567e837SChris Mason for (i = slot; i < nritems; i++) { 39105f39d397SChris Mason u32 ioff; 3911db94535dSChris Mason 39123212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 39133212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff - data_size); 39146567e837SChris Mason } 39155f39d397SChris Mason 39166567e837SChris Mason /* shift the data */ 39173d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 39183d9ec8c4SNikolay Borisov data_end - data_size, BTRFS_LEAF_DATA_OFFSET + 39196567e837SChris Mason data_end, old_data - data_end); 39205f39d397SChris Mason 39216567e837SChris Mason data_end = old_data; 39223212fa14SJosef Bacik old_size = btrfs_item_size(leaf, slot); 39233212fa14SJosef Bacik btrfs_set_item_size(leaf, slot, old_size + data_size); 39245f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 39256567e837SChris Mason 3926e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) { 3927a4f78750SDavid Sterba btrfs_print_leaf(leaf); 39286567e837SChris Mason BUG(); 39295f39d397SChris Mason } 39306567e837SChris Mason } 39316567e837SChris Mason 3932da9ffb24SNikolay Borisov /** 3933da9ffb24SNikolay Borisov * setup_items_for_insert - Helper called before inserting one or more items 3934da9ffb24SNikolay Borisov * to a leaf. Main purpose is to save stack depth by doing the bulk of the work 3935da9ffb24SNikolay Borisov * in a function that doesn't call btrfs_search_slot 3936da9ffb24SNikolay Borisov * 3937da9ffb24SNikolay Borisov * @root: root we are inserting items to 3938da9ffb24SNikolay Borisov * @path: points to the leaf/slot where we are going to insert new items 3939b7ef5f3aSFilipe Manana * @batch: information about the batch of items to insert 394074123bd7SChris Mason */ 3941f0641656SFilipe Manana static void setup_items_for_insert(struct btrfs_root *root, struct btrfs_path *path, 3942b7ef5f3aSFilipe Manana const struct btrfs_item_batch *batch) 3943be0e5c09SChris Mason { 39440b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 39459c58309dSChris Mason int i; 39467518a238SChris Mason u32 nritems; 3947be0e5c09SChris Mason unsigned int data_end; 3948e2fa7227SChris Mason struct btrfs_disk_key disk_key; 394944871b1bSChris Mason struct extent_buffer *leaf; 395044871b1bSChris Mason int slot; 3951cfed81a0SChris Mason struct btrfs_map_token token; 3952fc0d82e1SNikolay Borisov u32 total_size; 3953fc0d82e1SNikolay Borisov 3954b7ef5f3aSFilipe Manana /* 3955b7ef5f3aSFilipe Manana * Before anything else, update keys in the parent and other ancestors 3956b7ef5f3aSFilipe Manana * if needed, then release the write locks on them, so that other tasks 3957b7ef5f3aSFilipe Manana * can use them while we modify the leaf. 3958b7ef5f3aSFilipe Manana */ 395924cdc847SFilipe Manana if (path->slots[0] == 0) { 3960b7ef5f3aSFilipe Manana btrfs_cpu_key_to_disk(&disk_key, &batch->keys[0]); 3961b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 396224cdc847SFilipe Manana } 396324cdc847SFilipe Manana btrfs_unlock_up_safe(path, 1); 396424cdc847SFilipe Manana 39655f39d397SChris Mason leaf = path->nodes[0]; 396644871b1bSChris Mason slot = path->slots[0]; 396774123bd7SChris Mason 39685f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 39698f881e8cSDavid Sterba data_end = leaf_data_end(leaf); 3970b7ef5f3aSFilipe Manana total_size = batch->total_data_size + (batch->nr * sizeof(struct btrfs_item)); 3971eb60ceacSChris Mason 3972e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < total_size) { 3973a4f78750SDavid Sterba btrfs_print_leaf(leaf); 39740b246afaSJeff Mahoney btrfs_crit(fs_info, "not enough freespace need %u have %d", 3975e902baacSDavid Sterba total_size, btrfs_leaf_free_space(leaf)); 3976be0e5c09SChris Mason BUG(); 3977d4dbff95SChris Mason } 39785f39d397SChris Mason 3979c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 3980be0e5c09SChris Mason if (slot != nritems) { 3981dc2e724eSJosef Bacik unsigned int old_data = btrfs_item_data_end(leaf, slot); 3982be0e5c09SChris Mason 39835f39d397SChris Mason if (old_data < data_end) { 3984a4f78750SDavid Sterba btrfs_print_leaf(leaf); 39857269ddd2SNikolay Borisov btrfs_crit(fs_info, 39867269ddd2SNikolay Borisov "item at slot %d with data offset %u beyond data end of leaf %u", 39875f39d397SChris Mason slot, old_data, data_end); 3988290342f6SArnd Bergmann BUG(); 39895f39d397SChris Mason } 3990be0e5c09SChris Mason /* 3991be0e5c09SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 3992be0e5c09SChris Mason */ 3993be0e5c09SChris Mason /* first correct the data pointers */ 39940783fcfcSChris Mason for (i = slot; i < nritems; i++) { 39955f39d397SChris Mason u32 ioff; 3996db94535dSChris Mason 39973212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 39983212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, 3999b7ef5f3aSFilipe Manana ioff - batch->total_data_size); 40000783fcfcSChris Mason } 4001be0e5c09SChris Mason /* shift the items */ 4002b7ef5f3aSFilipe Manana memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + batch->nr), 40035f39d397SChris Mason btrfs_item_nr_offset(slot), 40040783fcfcSChris Mason (nritems - slot) * sizeof(struct btrfs_item)); 4005be0e5c09SChris Mason 4006be0e5c09SChris Mason /* shift the data */ 40073d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 4008b7ef5f3aSFilipe Manana data_end - batch->total_data_size, 4009b7ef5f3aSFilipe Manana BTRFS_LEAF_DATA_OFFSET + data_end, 4010b7ef5f3aSFilipe Manana old_data - data_end); 4011be0e5c09SChris Mason data_end = old_data; 4012be0e5c09SChris Mason } 40135f39d397SChris Mason 401462e2749eSChris Mason /* setup the item for the new data */ 4015b7ef5f3aSFilipe Manana for (i = 0; i < batch->nr; i++) { 4016b7ef5f3aSFilipe Manana btrfs_cpu_key_to_disk(&disk_key, &batch->keys[i]); 40179c58309dSChris Mason btrfs_set_item_key(leaf, &disk_key, slot + i); 4018b7ef5f3aSFilipe Manana data_end -= batch->data_sizes[i]; 40193212fa14SJosef Bacik btrfs_set_token_item_offset(&token, slot + i, data_end); 40203212fa14SJosef Bacik btrfs_set_token_item_size(&token, slot + i, batch->data_sizes[i]); 40219c58309dSChris Mason } 402244871b1bSChris Mason 4023b7ef5f3aSFilipe Manana btrfs_set_header_nritems(leaf, nritems + batch->nr); 4024b9473439SChris Mason btrfs_mark_buffer_dirty(leaf); 4025aa5d6bedSChris Mason 4026e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) { 4027a4f78750SDavid Sterba btrfs_print_leaf(leaf); 4028be0e5c09SChris Mason BUG(); 40295f39d397SChris Mason } 403044871b1bSChris Mason } 403144871b1bSChris Mason 403244871b1bSChris Mason /* 4033f0641656SFilipe Manana * Insert a new item into a leaf. 4034f0641656SFilipe Manana * 4035f0641656SFilipe Manana * @root: The root of the btree. 4036f0641656SFilipe Manana * @path: A path pointing to the target leaf and slot. 4037f0641656SFilipe Manana * @key: The key of the new item. 4038f0641656SFilipe Manana * @data_size: The size of the data associated with the new key. 4039f0641656SFilipe Manana */ 4040f0641656SFilipe Manana void btrfs_setup_item_for_insert(struct btrfs_root *root, 4041f0641656SFilipe Manana struct btrfs_path *path, 4042f0641656SFilipe Manana const struct btrfs_key *key, 4043f0641656SFilipe Manana u32 data_size) 4044f0641656SFilipe Manana { 4045f0641656SFilipe Manana struct btrfs_item_batch batch; 4046f0641656SFilipe Manana 4047f0641656SFilipe Manana batch.keys = key; 4048f0641656SFilipe Manana batch.data_sizes = &data_size; 4049f0641656SFilipe Manana batch.total_data_size = data_size; 4050f0641656SFilipe Manana batch.nr = 1; 4051f0641656SFilipe Manana 4052f0641656SFilipe Manana setup_items_for_insert(root, path, &batch); 4053f0641656SFilipe Manana } 4054f0641656SFilipe Manana 4055f0641656SFilipe Manana /* 405644871b1bSChris Mason * Given a key and some data, insert items into the tree. 405744871b1bSChris Mason * This does all the path init required, making room in the tree if needed. 405844871b1bSChris Mason */ 405944871b1bSChris Mason int btrfs_insert_empty_items(struct btrfs_trans_handle *trans, 406044871b1bSChris Mason struct btrfs_root *root, 406144871b1bSChris Mason struct btrfs_path *path, 4062b7ef5f3aSFilipe Manana const struct btrfs_item_batch *batch) 406344871b1bSChris Mason { 406444871b1bSChris Mason int ret = 0; 406544871b1bSChris Mason int slot; 4066b7ef5f3aSFilipe Manana u32 total_size; 406744871b1bSChris Mason 4068b7ef5f3aSFilipe Manana total_size = batch->total_data_size + (batch->nr * sizeof(struct btrfs_item)); 4069b7ef5f3aSFilipe Manana ret = btrfs_search_slot(trans, root, &batch->keys[0], path, total_size, 1); 407044871b1bSChris Mason if (ret == 0) 407144871b1bSChris Mason return -EEXIST; 407244871b1bSChris Mason if (ret < 0) 4073143bede5SJeff Mahoney return ret; 407444871b1bSChris Mason 407544871b1bSChris Mason slot = path->slots[0]; 407644871b1bSChris Mason BUG_ON(slot < 0); 407744871b1bSChris Mason 4078b7ef5f3aSFilipe Manana setup_items_for_insert(root, path, batch); 4079143bede5SJeff Mahoney return 0; 408062e2749eSChris Mason } 408162e2749eSChris Mason 408262e2749eSChris Mason /* 408362e2749eSChris Mason * Given a key and some data, insert an item into the tree. 408462e2749eSChris Mason * This does all the path init required, making room in the tree if needed. 408562e2749eSChris Mason */ 4086310712b2SOmar Sandoval int btrfs_insert_item(struct btrfs_trans_handle *trans, struct btrfs_root *root, 4087310712b2SOmar Sandoval const struct btrfs_key *cpu_key, void *data, 4088310712b2SOmar Sandoval u32 data_size) 408962e2749eSChris Mason { 409062e2749eSChris Mason int ret = 0; 40912c90e5d6SChris Mason struct btrfs_path *path; 40925f39d397SChris Mason struct extent_buffer *leaf; 40935f39d397SChris Mason unsigned long ptr; 409462e2749eSChris Mason 40952c90e5d6SChris Mason path = btrfs_alloc_path(); 4096db5b493aSTsutomu Itoh if (!path) 4097db5b493aSTsutomu Itoh return -ENOMEM; 40982c90e5d6SChris Mason ret = btrfs_insert_empty_item(trans, root, path, cpu_key, data_size); 409962e2749eSChris Mason if (!ret) { 41005f39d397SChris Mason leaf = path->nodes[0]; 41015f39d397SChris Mason ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); 41025f39d397SChris Mason write_extent_buffer(leaf, data, ptr, data_size); 41035f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 410462e2749eSChris Mason } 41052c90e5d6SChris Mason btrfs_free_path(path); 4106aa5d6bedSChris Mason return ret; 4107be0e5c09SChris Mason } 4108be0e5c09SChris Mason 410974123bd7SChris Mason /* 4110f0641656SFilipe Manana * This function duplicates an item, giving 'new_key' to the new item. 4111f0641656SFilipe Manana * It guarantees both items live in the same tree leaf and the new item is 4112f0641656SFilipe Manana * contiguous with the original item. 4113f0641656SFilipe Manana * 4114f0641656SFilipe Manana * This allows us to split a file extent in place, keeping a lock on the leaf 4115f0641656SFilipe Manana * the entire time. 4116f0641656SFilipe Manana */ 4117f0641656SFilipe Manana int btrfs_duplicate_item(struct btrfs_trans_handle *trans, 4118f0641656SFilipe Manana struct btrfs_root *root, 4119f0641656SFilipe Manana struct btrfs_path *path, 4120f0641656SFilipe Manana const struct btrfs_key *new_key) 4121f0641656SFilipe Manana { 4122f0641656SFilipe Manana struct extent_buffer *leaf; 4123f0641656SFilipe Manana int ret; 4124f0641656SFilipe Manana u32 item_size; 4125f0641656SFilipe Manana 4126f0641656SFilipe Manana leaf = path->nodes[0]; 41273212fa14SJosef Bacik item_size = btrfs_item_size(leaf, path->slots[0]); 4128f0641656SFilipe Manana ret = setup_leaf_for_split(trans, root, path, 4129f0641656SFilipe Manana item_size + sizeof(struct btrfs_item)); 4130f0641656SFilipe Manana if (ret) 4131f0641656SFilipe Manana return ret; 4132f0641656SFilipe Manana 4133f0641656SFilipe Manana path->slots[0]++; 4134f0641656SFilipe Manana btrfs_setup_item_for_insert(root, path, new_key, item_size); 4135f0641656SFilipe Manana leaf = path->nodes[0]; 4136f0641656SFilipe Manana memcpy_extent_buffer(leaf, 4137f0641656SFilipe Manana btrfs_item_ptr_offset(leaf, path->slots[0]), 4138f0641656SFilipe Manana btrfs_item_ptr_offset(leaf, path->slots[0] - 1), 4139f0641656SFilipe Manana item_size); 4140f0641656SFilipe Manana return 0; 4141f0641656SFilipe Manana } 4142f0641656SFilipe Manana 4143f0641656SFilipe Manana /* 41445de08d7dSChris Mason * delete the pointer from a given node. 414574123bd7SChris Mason * 4146d352ac68SChris Mason * the tree should have been previously balanced so the deletion does not 4147d352ac68SChris Mason * empty a node. 414874123bd7SChris Mason */ 4149afe5fea7STsutomu Itoh static void del_ptr(struct btrfs_root *root, struct btrfs_path *path, 4150afe5fea7STsutomu Itoh int level, int slot) 4151be0e5c09SChris Mason { 41525f39d397SChris Mason struct extent_buffer *parent = path->nodes[level]; 41537518a238SChris Mason u32 nritems; 4154f3ea38daSJan Schmidt int ret; 4155be0e5c09SChris Mason 41565f39d397SChris Mason nritems = btrfs_header_nritems(parent); 4157be0e5c09SChris Mason if (slot != nritems - 1) { 4158bf1d3425SDavid Sterba if (level) { 4159f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_move(parent, slot, 4160f3a84ccdSFilipe Manana slot + 1, nritems - slot - 1); 4161bf1d3425SDavid Sterba BUG_ON(ret < 0); 4162bf1d3425SDavid Sterba } 41635f39d397SChris Mason memmove_extent_buffer(parent, 41645f39d397SChris Mason btrfs_node_key_ptr_offset(slot), 41655f39d397SChris Mason btrfs_node_key_ptr_offset(slot + 1), 4166d6025579SChris Mason sizeof(struct btrfs_key_ptr) * 4167d6025579SChris Mason (nritems - slot - 1)); 416857ba86c0SChris Mason } else if (level) { 4169f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, slot, 4170f3a84ccdSFilipe Manana BTRFS_MOD_LOG_KEY_REMOVE, GFP_NOFS); 417157ba86c0SChris Mason BUG_ON(ret < 0); 4172be0e5c09SChris Mason } 4173f3ea38daSJan Schmidt 41747518a238SChris Mason nritems--; 41755f39d397SChris Mason btrfs_set_header_nritems(parent, nritems); 41767518a238SChris Mason if (nritems == 0 && parent == root->node) { 41775f39d397SChris Mason BUG_ON(btrfs_header_level(root->node) != 1); 4178eb60ceacSChris Mason /* just turn the root into a leaf and break */ 41795f39d397SChris Mason btrfs_set_header_level(root->node, 0); 4180bb803951SChris Mason } else if (slot == 0) { 41815f39d397SChris Mason struct btrfs_disk_key disk_key; 41825f39d397SChris Mason 41835f39d397SChris Mason btrfs_node_key(parent, &disk_key, 0); 4184b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, level + 1); 4185be0e5c09SChris Mason } 4186d6025579SChris Mason btrfs_mark_buffer_dirty(parent); 4187be0e5c09SChris Mason } 4188be0e5c09SChris Mason 418974123bd7SChris Mason /* 4190323ac95bSChris Mason * a helper function to delete the leaf pointed to by path->slots[1] and 41915d4f98a2SYan Zheng * path->nodes[1]. 4192323ac95bSChris Mason * 4193323ac95bSChris Mason * This deletes the pointer in path->nodes[1] and frees the leaf 4194323ac95bSChris Mason * block extent. zero is returned if it all worked out, < 0 otherwise. 4195323ac95bSChris Mason * 4196323ac95bSChris Mason * The path must have already been setup for deleting the leaf, including 4197323ac95bSChris Mason * all the proper balancing. path->nodes[1] must be locked. 4198323ac95bSChris Mason */ 4199143bede5SJeff Mahoney static noinline void btrfs_del_leaf(struct btrfs_trans_handle *trans, 4200323ac95bSChris Mason struct btrfs_root *root, 42015d4f98a2SYan Zheng struct btrfs_path *path, 42025d4f98a2SYan Zheng struct extent_buffer *leaf) 4203323ac95bSChris Mason { 42045d4f98a2SYan Zheng WARN_ON(btrfs_header_generation(leaf) != trans->transid); 4205afe5fea7STsutomu Itoh del_ptr(root, path, 1, path->slots[1]); 4206323ac95bSChris Mason 42074d081c41SChris Mason /* 42084d081c41SChris Mason * btrfs_free_extent is expensive, we want to make sure we 42094d081c41SChris Mason * aren't holding any locks when we call it 42104d081c41SChris Mason */ 42114d081c41SChris Mason btrfs_unlock_up_safe(path, 0); 42124d081c41SChris Mason 4213f0486c68SYan, Zheng root_sub_used(root, leaf->len); 4214f0486c68SYan, Zheng 421567439dadSDavid Sterba atomic_inc(&leaf->refs); 42167a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), leaf, 0, 1); 42173083ee2eSJosef Bacik free_extent_buffer_stale(leaf); 4218323ac95bSChris Mason } 4219323ac95bSChris Mason /* 422074123bd7SChris Mason * delete the item at the leaf level in path. If that empties 422174123bd7SChris Mason * the leaf, remove it from the tree 422274123bd7SChris Mason */ 422385e21bacSChris Mason int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root, 422485e21bacSChris Mason struct btrfs_path *path, int slot, int nr) 4225be0e5c09SChris Mason { 42260b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 42275f39d397SChris Mason struct extent_buffer *leaf; 4228aa5d6bedSChris Mason int ret = 0; 4229aa5d6bedSChris Mason int wret; 42307518a238SChris Mason u32 nritems; 4231be0e5c09SChris Mason 42325f39d397SChris Mason leaf = path->nodes[0]; 42335f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 4234be0e5c09SChris Mason 423585e21bacSChris Mason if (slot + nr != nritems) { 42360cae23b6SFilipe Manana const u32 last_off = btrfs_item_offset(leaf, slot + nr - 1); 42370cae23b6SFilipe Manana const int data_end = leaf_data_end(leaf); 4238c82f823cSDavid Sterba struct btrfs_map_token token; 42390cae23b6SFilipe Manana u32 dsize = 0; 42400cae23b6SFilipe Manana int i; 42410cae23b6SFilipe Manana 42420cae23b6SFilipe Manana for (i = 0; i < nr; i++) 42430cae23b6SFilipe Manana dsize += btrfs_item_size(leaf, slot + i); 42445f39d397SChris Mason 42453d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 4246d6025579SChris Mason data_end + dsize, 42473d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + data_end, 424885e21bacSChris Mason last_off - data_end); 42495f39d397SChris Mason 4250c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 425185e21bacSChris Mason for (i = slot + nr; i < nritems; i++) { 42525f39d397SChris Mason u32 ioff; 4253db94535dSChris Mason 42543212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 42553212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff + dsize); 42560783fcfcSChris Mason } 4257db94535dSChris Mason 42585f39d397SChris Mason memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot), 425985e21bacSChris Mason btrfs_item_nr_offset(slot + nr), 42600783fcfcSChris Mason sizeof(struct btrfs_item) * 426185e21bacSChris Mason (nritems - slot - nr)); 4262be0e5c09SChris Mason } 426385e21bacSChris Mason btrfs_set_header_nritems(leaf, nritems - nr); 426485e21bacSChris Mason nritems -= nr; 42655f39d397SChris Mason 426674123bd7SChris Mason /* delete the leaf if we've emptied it */ 42677518a238SChris Mason if (nritems == 0) { 42685f39d397SChris Mason if (leaf == root->node) { 42695f39d397SChris Mason btrfs_set_header_level(leaf, 0); 42709a8dd150SChris Mason } else { 42716a884d7dSDavid Sterba btrfs_clean_tree_block(leaf); 4272143bede5SJeff Mahoney btrfs_del_leaf(trans, root, path, leaf); 42739a8dd150SChris Mason } 4274be0e5c09SChris Mason } else { 42757518a238SChris Mason int used = leaf_space_used(leaf, 0, nritems); 4276aa5d6bedSChris Mason if (slot == 0) { 42775f39d397SChris Mason struct btrfs_disk_key disk_key; 42785f39d397SChris Mason 42795f39d397SChris Mason btrfs_item_key(leaf, &disk_key, 0); 4280b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 4281aa5d6bedSChris Mason } 4282aa5d6bedSChris Mason 42837c4063d1SFilipe Manana /* 42847c4063d1SFilipe Manana * Try to delete the leaf if it is mostly empty. We do this by 42857c4063d1SFilipe Manana * trying to move all its items into its left and right neighbours. 42867c4063d1SFilipe Manana * If we can't move all the items, then we don't delete it - it's 42877c4063d1SFilipe Manana * not ideal, but future insertions might fill the leaf with more 42887c4063d1SFilipe Manana * items, or items from other leaves might be moved later into our 42897c4063d1SFilipe Manana * leaf due to deletions on those leaves. 42907c4063d1SFilipe Manana */ 42910b246afaSJeff Mahoney if (used < BTRFS_LEAF_DATA_SIZE(fs_info) / 3) { 42927c4063d1SFilipe Manana u32 min_push_space; 42937c4063d1SFilipe Manana 4294be0e5c09SChris Mason /* push_leaf_left fixes the path. 4295be0e5c09SChris Mason * make sure the path still points to our leaf 4296be0e5c09SChris Mason * for possible call to del_ptr below 4297be0e5c09SChris Mason */ 42984920c9acSChris Mason slot = path->slots[1]; 429967439dadSDavid Sterba atomic_inc(&leaf->refs); 43007c4063d1SFilipe Manana /* 43017c4063d1SFilipe Manana * We want to be able to at least push one item to the 43027c4063d1SFilipe Manana * left neighbour leaf, and that's the first item. 43037c4063d1SFilipe Manana */ 43047c4063d1SFilipe Manana min_push_space = sizeof(struct btrfs_item) + 43057c4063d1SFilipe Manana btrfs_item_size(leaf, 0); 43067c4063d1SFilipe Manana wret = push_leaf_left(trans, root, path, 0, 43077c4063d1SFilipe Manana min_push_space, 1, (u32)-1); 430854aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 4309aa5d6bedSChris Mason ret = wret; 43105f39d397SChris Mason 43115f39d397SChris Mason if (path->nodes[0] == leaf && 43125f39d397SChris Mason btrfs_header_nritems(leaf)) { 43137c4063d1SFilipe Manana /* 43147c4063d1SFilipe Manana * If we were not able to push all items from our 43157c4063d1SFilipe Manana * leaf to its left neighbour, then attempt to 43167c4063d1SFilipe Manana * either push all the remaining items to the 43177c4063d1SFilipe Manana * right neighbour or none. There's no advantage 43187c4063d1SFilipe Manana * in pushing only some items, instead of all, as 43197c4063d1SFilipe Manana * it's pointless to end up with a leaf having 43207c4063d1SFilipe Manana * too few items while the neighbours can be full 43217c4063d1SFilipe Manana * or nearly full. 43227c4063d1SFilipe Manana */ 43237c4063d1SFilipe Manana nritems = btrfs_header_nritems(leaf); 43247c4063d1SFilipe Manana min_push_space = leaf_space_used(leaf, 0, nritems); 43257c4063d1SFilipe Manana wret = push_leaf_right(trans, root, path, 0, 43267c4063d1SFilipe Manana min_push_space, 1, 0); 432754aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 4328aa5d6bedSChris Mason ret = wret; 4329aa5d6bedSChris Mason } 43305f39d397SChris Mason 43315f39d397SChris Mason if (btrfs_header_nritems(leaf) == 0) { 4332323ac95bSChris Mason path->slots[1] = slot; 4333143bede5SJeff Mahoney btrfs_del_leaf(trans, root, path, leaf); 43345f39d397SChris Mason free_extent_buffer(leaf); 4335143bede5SJeff Mahoney ret = 0; 43365de08d7dSChris Mason } else { 4337925baeddSChris Mason /* if we're still in the path, make sure 4338925baeddSChris Mason * we're dirty. Otherwise, one of the 4339925baeddSChris Mason * push_leaf functions must have already 4340925baeddSChris Mason * dirtied this buffer 4341925baeddSChris Mason */ 4342925baeddSChris Mason if (path->nodes[0] == leaf) 43435f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 43445f39d397SChris Mason free_extent_buffer(leaf); 4345be0e5c09SChris Mason } 4346d5719762SChris Mason } else { 43475f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 4348be0e5c09SChris Mason } 43499a8dd150SChris Mason } 4350aa5d6bedSChris Mason return ret; 43519a8dd150SChris Mason } 43529a8dd150SChris Mason 435397571fd0SChris Mason /* 4354925baeddSChris Mason * search the tree again to find a leaf with lesser keys 43557bb86316SChris Mason * returns 0 if it found something or 1 if there are no lesser leaves. 43567bb86316SChris Mason * returns < 0 on io errors. 4357d352ac68SChris Mason * 4358d352ac68SChris Mason * This may release the path, and so you may lose any locks held at the 4359d352ac68SChris Mason * time you call it. 43607bb86316SChris Mason */ 436116e7549fSJosef Bacik int btrfs_prev_leaf(struct btrfs_root *root, struct btrfs_path *path) 43627bb86316SChris Mason { 4363925baeddSChris Mason struct btrfs_key key; 4364925baeddSChris Mason struct btrfs_disk_key found_key; 4365925baeddSChris Mason int ret; 43667bb86316SChris Mason 4367925baeddSChris Mason btrfs_item_key_to_cpu(path->nodes[0], &key, 0); 4368925baeddSChris Mason 4369e8b0d724SFilipe David Borba Manana if (key.offset > 0) { 4370925baeddSChris Mason key.offset--; 4371e8b0d724SFilipe David Borba Manana } else if (key.type > 0) { 4372925baeddSChris Mason key.type--; 4373e8b0d724SFilipe David Borba Manana key.offset = (u64)-1; 4374e8b0d724SFilipe David Borba Manana } else if (key.objectid > 0) { 4375925baeddSChris Mason key.objectid--; 4376e8b0d724SFilipe David Borba Manana key.type = (u8)-1; 4377e8b0d724SFilipe David Borba Manana key.offset = (u64)-1; 4378e8b0d724SFilipe David Borba Manana } else { 43797bb86316SChris Mason return 1; 4380e8b0d724SFilipe David Borba Manana } 43817bb86316SChris Mason 4382b3b4aa74SDavid Sterba btrfs_release_path(path); 4383925baeddSChris Mason ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 4384925baeddSChris Mason if (ret < 0) 4385925baeddSChris Mason return ret; 4386925baeddSChris Mason btrfs_item_key(path->nodes[0], &found_key, 0); 4387925baeddSChris Mason ret = comp_keys(&found_key, &key); 4388337c6f68SFilipe Manana /* 4389337c6f68SFilipe Manana * We might have had an item with the previous key in the tree right 4390337c6f68SFilipe Manana * before we released our path. And after we released our path, that 4391337c6f68SFilipe Manana * item might have been pushed to the first slot (0) of the leaf we 4392337c6f68SFilipe Manana * were holding due to a tree balance. Alternatively, an item with the 4393337c6f68SFilipe Manana * previous key can exist as the only element of a leaf (big fat item). 4394337c6f68SFilipe Manana * Therefore account for these 2 cases, so that our callers (like 4395337c6f68SFilipe Manana * btrfs_previous_item) don't miss an existing item with a key matching 4396337c6f68SFilipe Manana * the previous key we computed above. 4397337c6f68SFilipe Manana */ 4398337c6f68SFilipe Manana if (ret <= 0) 43997bb86316SChris Mason return 0; 4400925baeddSChris Mason return 1; 44017bb86316SChris Mason } 44027bb86316SChris Mason 44033f157a2fSChris Mason /* 44043f157a2fSChris Mason * A helper function to walk down the tree starting at min_key, and looking 4405de78b51aSEric Sandeen * for nodes or leaves that are have a minimum transaction id. 4406de78b51aSEric Sandeen * This is used by the btree defrag code, and tree logging 44073f157a2fSChris Mason * 44083f157a2fSChris Mason * This does not cow, but it does stuff the starting key it finds back 44093f157a2fSChris Mason * into min_key, so you can call btrfs_search_slot with cow=1 on the 44103f157a2fSChris Mason * key and get a writable path. 44113f157a2fSChris Mason * 44123f157a2fSChris Mason * This honors path->lowest_level to prevent descent past a given level 44133f157a2fSChris Mason * of the tree. 44143f157a2fSChris Mason * 4415d352ac68SChris Mason * min_trans indicates the oldest transaction that you are interested 4416d352ac68SChris Mason * in walking through. Any nodes or leaves older than min_trans are 4417d352ac68SChris Mason * skipped over (without reading them). 4418d352ac68SChris Mason * 44193f157a2fSChris Mason * returns zero if something useful was found, < 0 on error and 1 if there 44203f157a2fSChris Mason * was nothing in the tree that matched the search criteria. 44213f157a2fSChris Mason */ 44223f157a2fSChris Mason int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key, 4423de78b51aSEric Sandeen struct btrfs_path *path, 44243f157a2fSChris Mason u64 min_trans) 44253f157a2fSChris Mason { 44263f157a2fSChris Mason struct extent_buffer *cur; 44273f157a2fSChris Mason struct btrfs_key found_key; 44283f157a2fSChris Mason int slot; 44299652480bSYan int sret; 44303f157a2fSChris Mason u32 nritems; 44313f157a2fSChris Mason int level; 44323f157a2fSChris Mason int ret = 1; 4433f98de9b9SFilipe Manana int keep_locks = path->keep_locks; 44343f157a2fSChris Mason 4435f98de9b9SFilipe Manana path->keep_locks = 1; 44363f157a2fSChris Mason again: 4437bd681513SChris Mason cur = btrfs_read_lock_root_node(root); 44383f157a2fSChris Mason level = btrfs_header_level(cur); 4439e02119d5SChris Mason WARN_ON(path->nodes[level]); 44403f157a2fSChris Mason path->nodes[level] = cur; 4441bd681513SChris Mason path->locks[level] = BTRFS_READ_LOCK; 44423f157a2fSChris Mason 44433f157a2fSChris Mason if (btrfs_header_generation(cur) < min_trans) { 44443f157a2fSChris Mason ret = 1; 44453f157a2fSChris Mason goto out; 44463f157a2fSChris Mason } 44473f157a2fSChris Mason while (1) { 44483f157a2fSChris Mason nritems = btrfs_header_nritems(cur); 44493f157a2fSChris Mason level = btrfs_header_level(cur); 4450e3b83361SQu Wenruo sret = btrfs_bin_search(cur, min_key, &slot); 4451cbca7d59SFilipe Manana if (sret < 0) { 4452cbca7d59SFilipe Manana ret = sret; 4453cbca7d59SFilipe Manana goto out; 4454cbca7d59SFilipe Manana } 44553f157a2fSChris Mason 4456323ac95bSChris Mason /* at the lowest level, we're done, setup the path and exit */ 4457323ac95bSChris Mason if (level == path->lowest_level) { 4458e02119d5SChris Mason if (slot >= nritems) 4459e02119d5SChris Mason goto find_next_key; 44603f157a2fSChris Mason ret = 0; 44613f157a2fSChris Mason path->slots[level] = slot; 44623f157a2fSChris Mason btrfs_item_key_to_cpu(cur, &found_key, slot); 44633f157a2fSChris Mason goto out; 44643f157a2fSChris Mason } 44659652480bSYan if (sret && slot > 0) 44669652480bSYan slot--; 44673f157a2fSChris Mason /* 4468de78b51aSEric Sandeen * check this node pointer against the min_trans parameters. 4469260db43cSRandy Dunlap * If it is too old, skip to the next one. 44703f157a2fSChris Mason */ 44713f157a2fSChris Mason while (slot < nritems) { 44723f157a2fSChris Mason u64 gen; 4473e02119d5SChris Mason 44743f157a2fSChris Mason gen = btrfs_node_ptr_generation(cur, slot); 44753f157a2fSChris Mason if (gen < min_trans) { 44763f157a2fSChris Mason slot++; 44773f157a2fSChris Mason continue; 44783f157a2fSChris Mason } 44793f157a2fSChris Mason break; 44803f157a2fSChris Mason } 4481e02119d5SChris Mason find_next_key: 44823f157a2fSChris Mason /* 44833f157a2fSChris Mason * we didn't find a candidate key in this node, walk forward 44843f157a2fSChris Mason * and find another one 44853f157a2fSChris Mason */ 44863f157a2fSChris Mason if (slot >= nritems) { 4487e02119d5SChris Mason path->slots[level] = slot; 4488e02119d5SChris Mason sret = btrfs_find_next_key(root, path, min_key, level, 4489de78b51aSEric Sandeen min_trans); 4490e02119d5SChris Mason if (sret == 0) { 4491b3b4aa74SDavid Sterba btrfs_release_path(path); 44923f157a2fSChris Mason goto again; 44933f157a2fSChris Mason } else { 44943f157a2fSChris Mason goto out; 44953f157a2fSChris Mason } 44963f157a2fSChris Mason } 44973f157a2fSChris Mason /* save our key for returning back */ 44983f157a2fSChris Mason btrfs_node_key_to_cpu(cur, &found_key, slot); 44993f157a2fSChris Mason path->slots[level] = slot; 45003f157a2fSChris Mason if (level == path->lowest_level) { 45013f157a2fSChris Mason ret = 0; 45023f157a2fSChris Mason goto out; 45033f157a2fSChris Mason } 45044b231ae4SDavid Sterba cur = btrfs_read_node_slot(cur, slot); 4505fb770ae4SLiu Bo if (IS_ERR(cur)) { 4506fb770ae4SLiu Bo ret = PTR_ERR(cur); 4507fb770ae4SLiu Bo goto out; 4508fb770ae4SLiu Bo } 45093f157a2fSChris Mason 4510bd681513SChris Mason btrfs_tree_read_lock(cur); 4511b4ce94deSChris Mason 4512bd681513SChris Mason path->locks[level - 1] = BTRFS_READ_LOCK; 45133f157a2fSChris Mason path->nodes[level - 1] = cur; 4514f7c79f30SChris Mason unlock_up(path, level, 1, 0, NULL); 45153f157a2fSChris Mason } 45163f157a2fSChris Mason out: 4517f98de9b9SFilipe Manana path->keep_locks = keep_locks; 4518f98de9b9SFilipe Manana if (ret == 0) { 4519f98de9b9SFilipe Manana btrfs_unlock_up_safe(path, path->lowest_level + 1); 4520f98de9b9SFilipe Manana memcpy(min_key, &found_key, sizeof(found_key)); 4521f98de9b9SFilipe Manana } 45223f157a2fSChris Mason return ret; 45233f157a2fSChris Mason } 45243f157a2fSChris Mason 45253f157a2fSChris Mason /* 45263f157a2fSChris Mason * this is similar to btrfs_next_leaf, but does not try to preserve 45273f157a2fSChris Mason * and fixup the path. It looks for and returns the next key in the 4528de78b51aSEric Sandeen * tree based on the current path and the min_trans parameters. 45293f157a2fSChris Mason * 45303f157a2fSChris Mason * 0 is returned if another key is found, < 0 if there are any errors 45313f157a2fSChris Mason * and 1 is returned if there are no higher keys in the tree 45323f157a2fSChris Mason * 45333f157a2fSChris Mason * path->keep_locks should be set to 1 on the search made before 45343f157a2fSChris Mason * calling this function. 45353f157a2fSChris Mason */ 4536e7a84565SChris Mason int btrfs_find_next_key(struct btrfs_root *root, struct btrfs_path *path, 4537de78b51aSEric Sandeen struct btrfs_key *key, int level, u64 min_trans) 4538e7a84565SChris Mason { 4539e7a84565SChris Mason int slot; 4540e7a84565SChris Mason struct extent_buffer *c; 4541e7a84565SChris Mason 45426a9fb468SJosef Bacik WARN_ON(!path->keep_locks && !path->skip_locking); 4543e7a84565SChris Mason while (level < BTRFS_MAX_LEVEL) { 4544e7a84565SChris Mason if (!path->nodes[level]) 4545e7a84565SChris Mason return 1; 4546e7a84565SChris Mason 4547e7a84565SChris Mason slot = path->slots[level] + 1; 4548e7a84565SChris Mason c = path->nodes[level]; 45493f157a2fSChris Mason next: 4550e7a84565SChris Mason if (slot >= btrfs_header_nritems(c)) { 455133c66f43SYan Zheng int ret; 455233c66f43SYan Zheng int orig_lowest; 455333c66f43SYan Zheng struct btrfs_key cur_key; 455433c66f43SYan Zheng if (level + 1 >= BTRFS_MAX_LEVEL || 455533c66f43SYan Zheng !path->nodes[level + 1]) 4556e7a84565SChris Mason return 1; 455733c66f43SYan Zheng 45586a9fb468SJosef Bacik if (path->locks[level + 1] || path->skip_locking) { 455933c66f43SYan Zheng level++; 4560e7a84565SChris Mason continue; 4561e7a84565SChris Mason } 456233c66f43SYan Zheng 456333c66f43SYan Zheng slot = btrfs_header_nritems(c) - 1; 456433c66f43SYan Zheng if (level == 0) 456533c66f43SYan Zheng btrfs_item_key_to_cpu(c, &cur_key, slot); 456633c66f43SYan Zheng else 456733c66f43SYan Zheng btrfs_node_key_to_cpu(c, &cur_key, slot); 456833c66f43SYan Zheng 456933c66f43SYan Zheng orig_lowest = path->lowest_level; 4570b3b4aa74SDavid Sterba btrfs_release_path(path); 457133c66f43SYan Zheng path->lowest_level = level; 457233c66f43SYan Zheng ret = btrfs_search_slot(NULL, root, &cur_key, path, 457333c66f43SYan Zheng 0, 0); 457433c66f43SYan Zheng path->lowest_level = orig_lowest; 457533c66f43SYan Zheng if (ret < 0) 457633c66f43SYan Zheng return ret; 457733c66f43SYan Zheng 457833c66f43SYan Zheng c = path->nodes[level]; 457933c66f43SYan Zheng slot = path->slots[level]; 458033c66f43SYan Zheng if (ret == 0) 458133c66f43SYan Zheng slot++; 458233c66f43SYan Zheng goto next; 458333c66f43SYan Zheng } 458433c66f43SYan Zheng 4585e7a84565SChris Mason if (level == 0) 4586e7a84565SChris Mason btrfs_item_key_to_cpu(c, key, slot); 45873f157a2fSChris Mason else { 45883f157a2fSChris Mason u64 gen = btrfs_node_ptr_generation(c, slot); 45893f157a2fSChris Mason 45903f157a2fSChris Mason if (gen < min_trans) { 45913f157a2fSChris Mason slot++; 45923f157a2fSChris Mason goto next; 45933f157a2fSChris Mason } 4594e7a84565SChris Mason btrfs_node_key_to_cpu(c, key, slot); 45953f157a2fSChris Mason } 4596e7a84565SChris Mason return 0; 4597e7a84565SChris Mason } 4598e7a84565SChris Mason return 1; 4599e7a84565SChris Mason } 4600e7a84565SChris Mason 46013d7806ecSJan Schmidt int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path, 46023d7806ecSJan Schmidt u64 time_seq) 46033d7806ecSJan Schmidt { 4604d97e63b6SChris Mason int slot; 46058e73f275SChris Mason int level; 46065f39d397SChris Mason struct extent_buffer *c; 46078e73f275SChris Mason struct extent_buffer *next; 4608d96b3424SFilipe Manana struct btrfs_fs_info *fs_info = root->fs_info; 4609925baeddSChris Mason struct btrfs_key key; 4610d96b3424SFilipe Manana bool need_commit_sem = false; 4611925baeddSChris Mason u32 nritems; 4612925baeddSChris Mason int ret; 46130e46318dSJosef Bacik int i; 4614925baeddSChris Mason 4615925baeddSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 4616d397712bSChris Mason if (nritems == 0) 4617925baeddSChris Mason return 1; 4618925baeddSChris Mason 46198e73f275SChris Mason btrfs_item_key_to_cpu(path->nodes[0], &key, nritems - 1); 46208e73f275SChris Mason again: 46218e73f275SChris Mason level = 1; 46228e73f275SChris Mason next = NULL; 4623b3b4aa74SDavid Sterba btrfs_release_path(path); 46248e73f275SChris Mason 4625a2135011SChris Mason path->keep_locks = 1; 46268e73f275SChris Mason 4627d96b3424SFilipe Manana if (time_seq) { 46283d7806ecSJan Schmidt ret = btrfs_search_old_slot(root, &key, path, time_seq); 4629d96b3424SFilipe Manana } else { 4630d96b3424SFilipe Manana if (path->need_commit_sem) { 4631d96b3424SFilipe Manana path->need_commit_sem = 0; 4632d96b3424SFilipe Manana need_commit_sem = true; 4633d96b3424SFilipe Manana down_read(&fs_info->commit_root_sem); 4634d96b3424SFilipe Manana } 4635925baeddSChris Mason ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 4636d96b3424SFilipe Manana } 4637925baeddSChris Mason path->keep_locks = 0; 4638925baeddSChris Mason 4639925baeddSChris Mason if (ret < 0) 4640d96b3424SFilipe Manana goto done; 4641925baeddSChris Mason 4642a2135011SChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 4643168fd7d2SChris Mason /* 4644168fd7d2SChris Mason * by releasing the path above we dropped all our locks. A balance 4645168fd7d2SChris Mason * could have added more items next to the key that used to be 4646168fd7d2SChris Mason * at the very end of the block. So, check again here and 4647168fd7d2SChris Mason * advance the path if there are now more items available. 4648168fd7d2SChris Mason */ 4649a2135011SChris Mason if (nritems > 0 && path->slots[0] < nritems - 1) { 4650e457afecSYan Zheng if (ret == 0) 4651168fd7d2SChris Mason path->slots[0]++; 46528e73f275SChris Mason ret = 0; 4653925baeddSChris Mason goto done; 4654925baeddSChris Mason } 46550b43e04fSLiu Bo /* 46560b43e04fSLiu Bo * So the above check misses one case: 46570b43e04fSLiu Bo * - after releasing the path above, someone has removed the item that 46580b43e04fSLiu Bo * used to be at the very end of the block, and balance between leafs 46590b43e04fSLiu Bo * gets another one with bigger key.offset to replace it. 46600b43e04fSLiu Bo * 46610b43e04fSLiu Bo * This one should be returned as well, or we can get leaf corruption 46620b43e04fSLiu Bo * later(esp. in __btrfs_drop_extents()). 46630b43e04fSLiu Bo * 46640b43e04fSLiu Bo * And a bit more explanation about this check, 46650b43e04fSLiu Bo * with ret > 0, the key isn't found, the path points to the slot 46660b43e04fSLiu Bo * where it should be inserted, so the path->slots[0] item must be the 46670b43e04fSLiu Bo * bigger one. 46680b43e04fSLiu Bo */ 46690b43e04fSLiu Bo if (nritems > 0 && ret > 0 && path->slots[0] == nritems - 1) { 46700b43e04fSLiu Bo ret = 0; 46710b43e04fSLiu Bo goto done; 46720b43e04fSLiu Bo } 4673d97e63b6SChris Mason 4674234b63a0SChris Mason while (level < BTRFS_MAX_LEVEL) { 46758e73f275SChris Mason if (!path->nodes[level]) { 46768e73f275SChris Mason ret = 1; 46778e73f275SChris Mason goto done; 46788e73f275SChris Mason } 46795f39d397SChris Mason 4680d97e63b6SChris Mason slot = path->slots[level] + 1; 4681d97e63b6SChris Mason c = path->nodes[level]; 46825f39d397SChris Mason if (slot >= btrfs_header_nritems(c)) { 4683d97e63b6SChris Mason level++; 46848e73f275SChris Mason if (level == BTRFS_MAX_LEVEL) { 46858e73f275SChris Mason ret = 1; 46868e73f275SChris Mason goto done; 46878e73f275SChris Mason } 4688d97e63b6SChris Mason continue; 4689d97e63b6SChris Mason } 46905f39d397SChris Mason 46910e46318dSJosef Bacik 46920e46318dSJosef Bacik /* 46930e46318dSJosef Bacik * Our current level is where we're going to start from, and to 46940e46318dSJosef Bacik * make sure lockdep doesn't complain we need to drop our locks 46950e46318dSJosef Bacik * and nodes from 0 to our current level. 46960e46318dSJosef Bacik */ 46970e46318dSJosef Bacik for (i = 0; i < level; i++) { 46980e46318dSJosef Bacik if (path->locks[level]) { 46990e46318dSJosef Bacik btrfs_tree_read_unlock(path->nodes[i]); 47000e46318dSJosef Bacik path->locks[i] = 0; 47010e46318dSJosef Bacik } 47020e46318dSJosef Bacik free_extent_buffer(path->nodes[i]); 47030e46318dSJosef Bacik path->nodes[i] = NULL; 4704925baeddSChris Mason } 47055f39d397SChris Mason 47068e73f275SChris Mason next = c; 4707d07b8528SLiu Bo ret = read_block_for_search(root, path, &next, level, 4708cda79c54SDavid Sterba slot, &key); 47098e73f275SChris Mason if (ret == -EAGAIN) 47108e73f275SChris Mason goto again; 47115f39d397SChris Mason 471276a05b35SChris Mason if (ret < 0) { 4713b3b4aa74SDavid Sterba btrfs_release_path(path); 471476a05b35SChris Mason goto done; 471576a05b35SChris Mason } 471676a05b35SChris Mason 47175cd57b2cSChris Mason if (!path->skip_locking) { 4718bd681513SChris Mason ret = btrfs_try_tree_read_lock(next); 4719d42244a0SJan Schmidt if (!ret && time_seq) { 4720d42244a0SJan Schmidt /* 4721d42244a0SJan Schmidt * If we don't get the lock, we may be racing 4722d42244a0SJan Schmidt * with push_leaf_left, holding that lock while 4723d42244a0SJan Schmidt * itself waiting for the leaf we've currently 4724d42244a0SJan Schmidt * locked. To solve this situation, we give up 4725d42244a0SJan Schmidt * on our lock and cycle. 4726d42244a0SJan Schmidt */ 4727cf538830SJan Schmidt free_extent_buffer(next); 4728d42244a0SJan Schmidt btrfs_release_path(path); 4729d42244a0SJan Schmidt cond_resched(); 4730d42244a0SJan Schmidt goto again; 4731d42244a0SJan Schmidt } 47320e46318dSJosef Bacik if (!ret) 47330e46318dSJosef Bacik btrfs_tree_read_lock(next); 4734bd681513SChris Mason } 4735d97e63b6SChris Mason break; 4736d97e63b6SChris Mason } 4737d97e63b6SChris Mason path->slots[level] = slot; 4738d97e63b6SChris Mason while (1) { 4739d97e63b6SChris Mason level--; 4740d97e63b6SChris Mason path->nodes[level] = next; 4741d97e63b6SChris Mason path->slots[level] = 0; 4742a74a4b97SChris Mason if (!path->skip_locking) 4743ffeb03cfSJosef Bacik path->locks[level] = BTRFS_READ_LOCK; 4744d97e63b6SChris Mason if (!level) 4745d97e63b6SChris Mason break; 4746b4ce94deSChris Mason 4747d07b8528SLiu Bo ret = read_block_for_search(root, path, &next, level, 4748cda79c54SDavid Sterba 0, &key); 47498e73f275SChris Mason if (ret == -EAGAIN) 47508e73f275SChris Mason goto again; 47518e73f275SChris Mason 475276a05b35SChris Mason if (ret < 0) { 4753b3b4aa74SDavid Sterba btrfs_release_path(path); 475476a05b35SChris Mason goto done; 475576a05b35SChris Mason } 475676a05b35SChris Mason 4757ffeb03cfSJosef Bacik if (!path->skip_locking) 47580e46318dSJosef Bacik btrfs_tree_read_lock(next); 4759d97e63b6SChris Mason } 47608e73f275SChris Mason ret = 0; 4761925baeddSChris Mason done: 4762f7c79f30SChris Mason unlock_up(path, 0, 1, 0, NULL); 4763d96b3424SFilipe Manana if (need_commit_sem) { 4764d96b3424SFilipe Manana int ret2; 4765d96b3424SFilipe Manana 4766d96b3424SFilipe Manana path->need_commit_sem = 1; 4767d96b3424SFilipe Manana ret2 = finish_need_commit_sem_search(path); 4768d96b3424SFilipe Manana up_read(&fs_info->commit_root_sem); 4769d96b3424SFilipe Manana if (ret2) 4770d96b3424SFilipe Manana ret = ret2; 4771d96b3424SFilipe Manana } 47728e73f275SChris Mason 47738e73f275SChris Mason return ret; 4774d97e63b6SChris Mason } 47750b86a832SChris Mason 47763f157a2fSChris Mason /* 47773f157a2fSChris Mason * this uses btrfs_prev_leaf to walk backwards in the tree, and keeps 47783f157a2fSChris Mason * searching until it gets past min_objectid or finds an item of 'type' 47793f157a2fSChris Mason * 47803f157a2fSChris Mason * returns 0 if something is found, 1 if nothing was found and < 0 on error 47813f157a2fSChris Mason */ 47820b86a832SChris Mason int btrfs_previous_item(struct btrfs_root *root, 47830b86a832SChris Mason struct btrfs_path *path, u64 min_objectid, 47840b86a832SChris Mason int type) 47850b86a832SChris Mason { 47860b86a832SChris Mason struct btrfs_key found_key; 47870b86a832SChris Mason struct extent_buffer *leaf; 4788e02119d5SChris Mason u32 nritems; 47890b86a832SChris Mason int ret; 47900b86a832SChris Mason 47910b86a832SChris Mason while (1) { 47920b86a832SChris Mason if (path->slots[0] == 0) { 47930b86a832SChris Mason ret = btrfs_prev_leaf(root, path); 47940b86a832SChris Mason if (ret != 0) 47950b86a832SChris Mason return ret; 47960b86a832SChris Mason } else { 47970b86a832SChris Mason path->slots[0]--; 47980b86a832SChris Mason } 47990b86a832SChris Mason leaf = path->nodes[0]; 4800e02119d5SChris Mason nritems = btrfs_header_nritems(leaf); 4801e02119d5SChris Mason if (nritems == 0) 4802e02119d5SChris Mason return 1; 4803e02119d5SChris Mason if (path->slots[0] == nritems) 4804e02119d5SChris Mason path->slots[0]--; 4805e02119d5SChris Mason 48060b86a832SChris Mason btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 4807e02119d5SChris Mason if (found_key.objectid < min_objectid) 4808e02119d5SChris Mason break; 48090a4eefbbSYan Zheng if (found_key.type == type) 48100a4eefbbSYan Zheng return 0; 4811e02119d5SChris Mason if (found_key.objectid == min_objectid && 4812e02119d5SChris Mason found_key.type < type) 4813e02119d5SChris Mason break; 48140b86a832SChris Mason } 48150b86a832SChris Mason return 1; 48160b86a832SChris Mason } 4817ade2e0b3SWang Shilong 4818ade2e0b3SWang Shilong /* 4819ade2e0b3SWang Shilong * search in extent tree to find a previous Metadata/Data extent item with 4820ade2e0b3SWang Shilong * min objecitd. 4821ade2e0b3SWang Shilong * 4822ade2e0b3SWang Shilong * returns 0 if something is found, 1 if nothing was found and < 0 on error 4823ade2e0b3SWang Shilong */ 4824ade2e0b3SWang Shilong int btrfs_previous_extent_item(struct btrfs_root *root, 4825ade2e0b3SWang Shilong struct btrfs_path *path, u64 min_objectid) 4826ade2e0b3SWang Shilong { 4827ade2e0b3SWang Shilong struct btrfs_key found_key; 4828ade2e0b3SWang Shilong struct extent_buffer *leaf; 4829ade2e0b3SWang Shilong u32 nritems; 4830ade2e0b3SWang Shilong int ret; 4831ade2e0b3SWang Shilong 4832ade2e0b3SWang Shilong while (1) { 4833ade2e0b3SWang Shilong if (path->slots[0] == 0) { 4834ade2e0b3SWang Shilong ret = btrfs_prev_leaf(root, path); 4835ade2e0b3SWang Shilong if (ret != 0) 4836ade2e0b3SWang Shilong return ret; 4837ade2e0b3SWang Shilong } else { 4838ade2e0b3SWang Shilong path->slots[0]--; 4839ade2e0b3SWang Shilong } 4840ade2e0b3SWang Shilong leaf = path->nodes[0]; 4841ade2e0b3SWang Shilong nritems = btrfs_header_nritems(leaf); 4842ade2e0b3SWang Shilong if (nritems == 0) 4843ade2e0b3SWang Shilong return 1; 4844ade2e0b3SWang Shilong if (path->slots[0] == nritems) 4845ade2e0b3SWang Shilong path->slots[0]--; 4846ade2e0b3SWang Shilong 4847ade2e0b3SWang Shilong btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 4848ade2e0b3SWang Shilong if (found_key.objectid < min_objectid) 4849ade2e0b3SWang Shilong break; 4850ade2e0b3SWang Shilong if (found_key.type == BTRFS_EXTENT_ITEM_KEY || 4851ade2e0b3SWang Shilong found_key.type == BTRFS_METADATA_ITEM_KEY) 4852ade2e0b3SWang Shilong return 0; 4853ade2e0b3SWang Shilong if (found_key.objectid == min_objectid && 4854ade2e0b3SWang Shilong found_key.type < BTRFS_EXTENT_ITEM_KEY) 4855ade2e0b3SWang Shilong break; 4856ade2e0b3SWang Shilong } 4857ade2e0b3SWang Shilong return 1; 4858ade2e0b3SWang Shilong } 4859