1c1d7c514SDavid Sterba // SPDX-License-Identifier: GPL-2.0 26cbd5570SChris Mason /* 3d352ac68SChris Mason * Copyright (C) 2007,2008 Oracle. All rights reserved. 46cbd5570SChris Mason */ 56cbd5570SChris Mason 6a6b6e75eSChris Mason #include <linux/sched.h> 75a0e3ad6STejun Heo #include <linux/slab.h> 8bd989ba3SJan Schmidt #include <linux/rbtree.h> 9adf02123SDavid Sterba #include <linux/mm.h> 10e41d12f5SChristoph Hellwig #include <linux/error-injection.h> 119b569ea0SJosef Bacik #include "messages.h" 12eb60ceacSChris Mason #include "ctree.h" 13eb60ceacSChris Mason #include "disk-io.h" 147f5c1516SChris Mason #include "transaction.h" 155f39d397SChris Mason #include "print-tree.h" 16925baeddSChris Mason #include "locking.h" 17de37aa51SNikolay Borisov #include "volumes.h" 18f616f5cdSQu Wenruo #include "qgroup.h" 19f3a84ccdSFilipe Manana #include "tree-mod-log.h" 2088c602abSQu Wenruo #include "tree-checker.h" 21ec8eb376SJosef Bacik #include "fs.h" 22ad1ac501SJosef Bacik #include "accessors.h" 23a0231804SJosef Bacik #include "extent-tree.h" 24*67707479SJosef Bacik #include "relocation.h" 259a8dd150SChris Mason 26226463d7SJosef Bacik static struct kmem_cache *btrfs_path_cachep; 27226463d7SJosef Bacik 28e089f05cSChris Mason static int split_node(struct btrfs_trans_handle *trans, struct btrfs_root 29e089f05cSChris Mason *root, struct btrfs_path *path, int level); 30310712b2SOmar Sandoval static int split_leaf(struct btrfs_trans_handle *trans, struct btrfs_root *root, 31310712b2SOmar Sandoval const struct btrfs_key *ins_key, struct btrfs_path *path, 32310712b2SOmar Sandoval int data_size, int extend); 335f39d397SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 342ff7e61eSJeff Mahoney struct extent_buffer *dst, 35971a1f66SChris Mason struct extent_buffer *src, int empty); 365f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 375f39d397SChris Mason struct extent_buffer *dst_buf, 385f39d397SChris Mason struct extent_buffer *src_buf); 39afe5fea7STsutomu Itoh static void del_ptr(struct btrfs_root *root, struct btrfs_path *path, 40afe5fea7STsutomu Itoh int level, int slot); 41d97e63b6SChris Mason 42af024ed2SJohannes Thumshirn static const struct btrfs_csums { 43af024ed2SJohannes Thumshirn u16 size; 4459a0fcdbSDavid Sterba const char name[10]; 4559a0fcdbSDavid Sterba const char driver[12]; 46af024ed2SJohannes Thumshirn } btrfs_csums[] = { 47af024ed2SJohannes Thumshirn [BTRFS_CSUM_TYPE_CRC32] = { .size = 4, .name = "crc32c" }, 483951e7f0SJohannes Thumshirn [BTRFS_CSUM_TYPE_XXHASH] = { .size = 8, .name = "xxhash64" }, 493831bf00SJohannes Thumshirn [BTRFS_CSUM_TYPE_SHA256] = { .size = 32, .name = "sha256" }, 50352ae07bSDavid Sterba [BTRFS_CSUM_TYPE_BLAKE2] = { .size = 32, .name = "blake2b", 51352ae07bSDavid Sterba .driver = "blake2b-256" }, 52af024ed2SJohannes Thumshirn }; 53af024ed2SJohannes Thumshirn 54af024ed2SJohannes Thumshirn int btrfs_super_csum_size(const struct btrfs_super_block *s) 55af024ed2SJohannes Thumshirn { 56af024ed2SJohannes Thumshirn u16 t = btrfs_super_csum_type(s); 57af024ed2SJohannes Thumshirn /* 58af024ed2SJohannes Thumshirn * csum type is validated at mount time 59af024ed2SJohannes Thumshirn */ 60af024ed2SJohannes Thumshirn return btrfs_csums[t].size; 61af024ed2SJohannes Thumshirn } 62af024ed2SJohannes Thumshirn 63af024ed2SJohannes Thumshirn const char *btrfs_super_csum_name(u16 csum_type) 64af024ed2SJohannes Thumshirn { 65af024ed2SJohannes Thumshirn /* csum type is validated at mount time */ 66af024ed2SJohannes Thumshirn return btrfs_csums[csum_type].name; 67af024ed2SJohannes Thumshirn } 68af024ed2SJohannes Thumshirn 69b4e967beSDavid Sterba /* 70b4e967beSDavid Sterba * Return driver name if defined, otherwise the name that's also a valid driver 71b4e967beSDavid Sterba * name 72b4e967beSDavid Sterba */ 73b4e967beSDavid Sterba const char *btrfs_super_csum_driver(u16 csum_type) 74b4e967beSDavid Sterba { 75b4e967beSDavid Sterba /* csum type is validated at mount time */ 7659a0fcdbSDavid Sterba return btrfs_csums[csum_type].driver[0] ? 7759a0fcdbSDavid Sterba btrfs_csums[csum_type].driver : 78b4e967beSDavid Sterba btrfs_csums[csum_type].name; 79b4e967beSDavid Sterba } 80b4e967beSDavid Sterba 81604997b4SDavid Sterba size_t __attribute_const__ btrfs_get_num_csums(void) 82f7cea56cSDavid Sterba { 83f7cea56cSDavid Sterba return ARRAY_SIZE(btrfs_csums); 84f7cea56cSDavid Sterba } 85f7cea56cSDavid Sterba 862c90e5d6SChris Mason struct btrfs_path *btrfs_alloc_path(void) 872c90e5d6SChris Mason { 88e2c89907SMasahiro Yamada return kmem_cache_zalloc(btrfs_path_cachep, GFP_NOFS); 892c90e5d6SChris Mason } 902c90e5d6SChris Mason 91d352ac68SChris Mason /* this also releases the path */ 922c90e5d6SChris Mason void btrfs_free_path(struct btrfs_path *p) 932c90e5d6SChris Mason { 94ff175d57SJesper Juhl if (!p) 95ff175d57SJesper Juhl return; 96b3b4aa74SDavid Sterba btrfs_release_path(p); 972c90e5d6SChris Mason kmem_cache_free(btrfs_path_cachep, p); 982c90e5d6SChris Mason } 992c90e5d6SChris Mason 100d352ac68SChris Mason /* 101d352ac68SChris Mason * path release drops references on the extent buffers in the path 102d352ac68SChris Mason * and it drops any locks held by this path 103d352ac68SChris Mason * 104d352ac68SChris Mason * It is safe to call this on paths that no locks or extent buffers held. 105d352ac68SChris Mason */ 106b3b4aa74SDavid Sterba noinline void btrfs_release_path(struct btrfs_path *p) 107eb60ceacSChris Mason { 108eb60ceacSChris Mason int i; 109a2135011SChris Mason 110234b63a0SChris Mason for (i = 0; i < BTRFS_MAX_LEVEL; i++) { 1113f157a2fSChris Mason p->slots[i] = 0; 112eb60ceacSChris Mason if (!p->nodes[i]) 113925baeddSChris Mason continue; 114925baeddSChris Mason if (p->locks[i]) { 115bd681513SChris Mason btrfs_tree_unlock_rw(p->nodes[i], p->locks[i]); 116925baeddSChris Mason p->locks[i] = 0; 117925baeddSChris Mason } 1185f39d397SChris Mason free_extent_buffer(p->nodes[i]); 1193f157a2fSChris Mason p->nodes[i] = NULL; 120eb60ceacSChris Mason } 121eb60ceacSChris Mason } 122eb60ceacSChris Mason 123d352ac68SChris Mason /* 1248bb808c6SDavid Sterba * We want the transaction abort to print stack trace only for errors where the 1258bb808c6SDavid Sterba * cause could be a bug, eg. due to ENOSPC, and not for common errors that are 1268bb808c6SDavid Sterba * caused by external factors. 1278bb808c6SDavid Sterba */ 1288bb808c6SDavid Sterba bool __cold abort_should_print_stack(int errno) 1298bb808c6SDavid Sterba { 1308bb808c6SDavid Sterba switch (errno) { 1318bb808c6SDavid Sterba case -EIO: 1328bb808c6SDavid Sterba case -EROFS: 1338bb808c6SDavid Sterba case -ENOMEM: 1348bb808c6SDavid Sterba return false; 1358bb808c6SDavid Sterba } 1368bb808c6SDavid Sterba return true; 1378bb808c6SDavid Sterba } 1388bb808c6SDavid Sterba 1398bb808c6SDavid Sterba /* 140d352ac68SChris Mason * safely gets a reference on the root node of a tree. A lock 141d352ac68SChris Mason * is not taken, so a concurrent writer may put a different node 142d352ac68SChris Mason * at the root of the tree. See btrfs_lock_root_node for the 143d352ac68SChris Mason * looping required. 144d352ac68SChris Mason * 145d352ac68SChris Mason * The extent buffer returned by this has a reference taken, so 146d352ac68SChris Mason * it won't disappear. It may stop being the root of the tree 147d352ac68SChris Mason * at any time because there are no locks held. 148d352ac68SChris Mason */ 149925baeddSChris Mason struct extent_buffer *btrfs_root_node(struct btrfs_root *root) 150925baeddSChris Mason { 151925baeddSChris Mason struct extent_buffer *eb; 152240f62c8SChris Mason 1533083ee2eSJosef Bacik while (1) { 154240f62c8SChris Mason rcu_read_lock(); 155240f62c8SChris Mason eb = rcu_dereference(root->node); 1563083ee2eSJosef Bacik 1573083ee2eSJosef Bacik /* 1583083ee2eSJosef Bacik * RCU really hurts here, we could free up the root node because 15901327610SNicholas D Steeves * it was COWed but we may not get the new root node yet so do 1603083ee2eSJosef Bacik * the inc_not_zero dance and if it doesn't work then 1613083ee2eSJosef Bacik * synchronize_rcu and try again. 1623083ee2eSJosef Bacik */ 1633083ee2eSJosef Bacik if (atomic_inc_not_zero(&eb->refs)) { 164240f62c8SChris Mason rcu_read_unlock(); 1653083ee2eSJosef Bacik break; 1663083ee2eSJosef Bacik } 1673083ee2eSJosef Bacik rcu_read_unlock(); 1683083ee2eSJosef Bacik synchronize_rcu(); 1693083ee2eSJosef Bacik } 170925baeddSChris Mason return eb; 171925baeddSChris Mason } 172925baeddSChris Mason 17392a7cc42SQu Wenruo /* 17492a7cc42SQu Wenruo * Cowonly root (not-shareable trees, everything not subvolume or reloc roots), 17592a7cc42SQu Wenruo * just get put onto a simple dirty list. Transaction walks this list to make 17692a7cc42SQu Wenruo * sure they get properly updated on disk. 177d352ac68SChris Mason */ 1780b86a832SChris Mason static void add_root_to_dirty_list(struct btrfs_root *root) 1790b86a832SChris Mason { 1800b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 1810b246afaSJeff Mahoney 182e7070be1SJosef Bacik if (test_bit(BTRFS_ROOT_DIRTY, &root->state) || 183e7070be1SJosef Bacik !test_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state)) 184e7070be1SJosef Bacik return; 185e7070be1SJosef Bacik 1860b246afaSJeff Mahoney spin_lock(&fs_info->trans_lock); 187e7070be1SJosef Bacik if (!test_and_set_bit(BTRFS_ROOT_DIRTY, &root->state)) { 188e7070be1SJosef Bacik /* Want the extent tree to be the last on the list */ 1894fd786e6SMisono Tomohiro if (root->root_key.objectid == BTRFS_EXTENT_TREE_OBJECTID) 190e7070be1SJosef Bacik list_move_tail(&root->dirty_list, 1910b246afaSJeff Mahoney &fs_info->dirty_cowonly_roots); 192e7070be1SJosef Bacik else 193e7070be1SJosef Bacik list_move(&root->dirty_list, 1940b246afaSJeff Mahoney &fs_info->dirty_cowonly_roots); 1950b86a832SChris Mason } 1960b246afaSJeff Mahoney spin_unlock(&fs_info->trans_lock); 1970b86a832SChris Mason } 1980b86a832SChris Mason 199d352ac68SChris Mason /* 200d352ac68SChris Mason * used by snapshot creation to make a copy of a root for a tree with 201d352ac68SChris Mason * a given objectid. The buffer with the new root node is returned in 202d352ac68SChris Mason * cow_ret, and this func returns zero on success or a negative error code. 203d352ac68SChris Mason */ 204be20aa9dSChris Mason int btrfs_copy_root(struct btrfs_trans_handle *trans, 205be20aa9dSChris Mason struct btrfs_root *root, 206be20aa9dSChris Mason struct extent_buffer *buf, 207be20aa9dSChris Mason struct extent_buffer **cow_ret, u64 new_root_objectid) 208be20aa9dSChris Mason { 2090b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 210be20aa9dSChris Mason struct extent_buffer *cow; 211be20aa9dSChris Mason int ret = 0; 212be20aa9dSChris Mason int level; 2135d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 214be20aa9dSChris Mason 21592a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 2160b246afaSJeff Mahoney trans->transid != fs_info->running_transaction->transid); 21792a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 21827cdeb70SMiao Xie trans->transid != root->last_trans); 219be20aa9dSChris Mason 220be20aa9dSChris Mason level = btrfs_header_level(buf); 2215d4f98a2SYan Zheng if (level == 0) 2225d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 2235d4f98a2SYan Zheng else 2245d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 22531840ae1SZheng Yan 2264d75f8a9SDavid Sterba cow = btrfs_alloc_tree_block(trans, root, 0, new_root_objectid, 227cf6f34aaSJosef Bacik &disk_key, level, buf->start, 0, 228cf6f34aaSJosef Bacik BTRFS_NESTING_NEW_ROOT); 2295d4f98a2SYan Zheng if (IS_ERR(cow)) 230be20aa9dSChris Mason return PTR_ERR(cow); 231be20aa9dSChris Mason 23258e8012cSDavid Sterba copy_extent_buffer_full(cow, buf); 233be20aa9dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 234be20aa9dSChris Mason btrfs_set_header_generation(cow, trans->transid); 2355d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 2365d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 2375d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 2385d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 2395d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 2405d4f98a2SYan Zheng else 241be20aa9dSChris Mason btrfs_set_header_owner(cow, new_root_objectid); 242be20aa9dSChris Mason 243de37aa51SNikolay Borisov write_extent_buffer_fsid(cow, fs_info->fs_devices->metadata_uuid); 2442b82032cSYan Zheng 245be20aa9dSChris Mason WARN_ON(btrfs_header_generation(buf) > trans->transid); 2465d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 247e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 2485d4f98a2SYan Zheng else 249e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 250867ed321SJosef Bacik if (ret) { 25172c9925fSFilipe Manana btrfs_tree_unlock(cow); 25272c9925fSFilipe Manana free_extent_buffer(cow); 253867ed321SJosef Bacik btrfs_abort_transaction(trans, ret); 254be20aa9dSChris Mason return ret; 255867ed321SJosef Bacik } 256be20aa9dSChris Mason 257be20aa9dSChris Mason btrfs_mark_buffer_dirty(cow); 258be20aa9dSChris Mason *cow_ret = cow; 259be20aa9dSChris Mason return 0; 260be20aa9dSChris Mason } 261be20aa9dSChris Mason 262d352ac68SChris Mason /* 2635d4f98a2SYan Zheng * check if the tree block can be shared by multiple trees 2645d4f98a2SYan Zheng */ 2655d4f98a2SYan Zheng int btrfs_block_can_be_shared(struct btrfs_root *root, 2665d4f98a2SYan Zheng struct extent_buffer *buf) 2675d4f98a2SYan Zheng { 2685d4f98a2SYan Zheng /* 26992a7cc42SQu Wenruo * Tree blocks not in shareable trees and tree roots are never shared. 27092a7cc42SQu Wenruo * If a block was allocated after the last snapshot and the block was 27192a7cc42SQu Wenruo * not allocated by tree relocation, we know the block is not shared. 2725d4f98a2SYan Zheng */ 27392a7cc42SQu Wenruo if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 2745d4f98a2SYan Zheng buf != root->node && buf != root->commit_root && 2755d4f98a2SYan Zheng (btrfs_header_generation(buf) <= 2765d4f98a2SYan Zheng btrfs_root_last_snapshot(&root->root_item) || 2775d4f98a2SYan Zheng btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC))) 2785d4f98a2SYan Zheng return 1; 279a79865c6SNikolay Borisov 2805d4f98a2SYan Zheng return 0; 2815d4f98a2SYan Zheng } 2825d4f98a2SYan Zheng 2835d4f98a2SYan Zheng static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans, 2845d4f98a2SYan Zheng struct btrfs_root *root, 2855d4f98a2SYan Zheng struct extent_buffer *buf, 286f0486c68SYan, Zheng struct extent_buffer *cow, 287f0486c68SYan, Zheng int *last_ref) 2885d4f98a2SYan Zheng { 2890b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 2905d4f98a2SYan Zheng u64 refs; 2915d4f98a2SYan Zheng u64 owner; 2925d4f98a2SYan Zheng u64 flags; 2935d4f98a2SYan Zheng u64 new_flags = 0; 2945d4f98a2SYan Zheng int ret; 2955d4f98a2SYan Zheng 2965d4f98a2SYan Zheng /* 2975d4f98a2SYan Zheng * Backrefs update rules: 2985d4f98a2SYan Zheng * 2995d4f98a2SYan Zheng * Always use full backrefs for extent pointers in tree block 3005d4f98a2SYan Zheng * allocated by tree relocation. 3015d4f98a2SYan Zheng * 3025d4f98a2SYan Zheng * If a shared tree block is no longer referenced by its owner 3035d4f98a2SYan Zheng * tree (btrfs_header_owner(buf) == root->root_key.objectid), 3045d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 3055d4f98a2SYan Zheng * 3065d4f98a2SYan Zheng * If a tree block is been relocating 3075d4f98a2SYan Zheng * (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID), 3085d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 3095d4f98a2SYan Zheng * The reason for this is some operations (such as drop tree) 3105d4f98a2SYan Zheng * are only allowed for blocks use full backrefs. 3115d4f98a2SYan Zheng */ 3125d4f98a2SYan Zheng 3135d4f98a2SYan Zheng if (btrfs_block_can_be_shared(root, buf)) { 3142ff7e61eSJeff Mahoney ret = btrfs_lookup_extent_info(trans, fs_info, buf->start, 3153173a18fSJosef Bacik btrfs_header_level(buf), 1, 3163173a18fSJosef Bacik &refs, &flags); 317be1a5564SMark Fasheh if (ret) 318be1a5564SMark Fasheh return ret; 319e5df9573SMark Fasheh if (refs == 0) { 320e5df9573SMark Fasheh ret = -EROFS; 3210b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 322e5df9573SMark Fasheh return ret; 323e5df9573SMark Fasheh } 3245d4f98a2SYan Zheng } else { 3255d4f98a2SYan Zheng refs = 1; 3265d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 3275d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 3285d4f98a2SYan Zheng flags = BTRFS_BLOCK_FLAG_FULL_BACKREF; 3295d4f98a2SYan Zheng else 3305d4f98a2SYan Zheng flags = 0; 3315d4f98a2SYan Zheng } 3325d4f98a2SYan Zheng 3335d4f98a2SYan Zheng owner = btrfs_header_owner(buf); 3345d4f98a2SYan Zheng BUG_ON(owner == BTRFS_TREE_RELOC_OBJECTID && 3355d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)); 3365d4f98a2SYan Zheng 3375d4f98a2SYan Zheng if (refs > 1) { 3385d4f98a2SYan Zheng if ((owner == root->root_key.objectid || 3395d4f98a2SYan Zheng root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && 3405d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) { 341e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, buf, 1); 342692826b2SJeff Mahoney if (ret) 343692826b2SJeff Mahoney return ret; 3445d4f98a2SYan Zheng 3455d4f98a2SYan Zheng if (root->root_key.objectid == 3465d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) { 347e339a6b0SJosef Bacik ret = btrfs_dec_ref(trans, root, buf, 0); 348692826b2SJeff Mahoney if (ret) 349692826b2SJeff Mahoney return ret; 350e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 351692826b2SJeff Mahoney if (ret) 352692826b2SJeff Mahoney return ret; 3535d4f98a2SYan Zheng } 3545d4f98a2SYan Zheng new_flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF; 3555d4f98a2SYan Zheng } else { 3565d4f98a2SYan Zheng 3575d4f98a2SYan Zheng if (root->root_key.objectid == 3585d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 359e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 3605d4f98a2SYan Zheng else 361e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 362692826b2SJeff Mahoney if (ret) 363692826b2SJeff Mahoney return ret; 3645d4f98a2SYan Zheng } 3655d4f98a2SYan Zheng if (new_flags != 0) { 366b1c79e09SJosef Bacik int level = btrfs_header_level(buf); 367b1c79e09SJosef Bacik 36842c9d0b5SDavid Sterba ret = btrfs_set_disk_extent_flags(trans, buf, 3692fe6a5a1SDavid Sterba new_flags, level); 370be1a5564SMark Fasheh if (ret) 371be1a5564SMark Fasheh return ret; 3725d4f98a2SYan Zheng } 3735d4f98a2SYan Zheng } else { 3745d4f98a2SYan Zheng if (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) { 3755d4f98a2SYan Zheng if (root->root_key.objectid == 3765d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 377e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 3785d4f98a2SYan Zheng else 379e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 380692826b2SJeff Mahoney if (ret) 381692826b2SJeff Mahoney return ret; 382e339a6b0SJosef Bacik ret = btrfs_dec_ref(trans, root, buf, 1); 383692826b2SJeff Mahoney if (ret) 384692826b2SJeff Mahoney return ret; 3855d4f98a2SYan Zheng } 3866a884d7dSDavid Sterba btrfs_clean_tree_block(buf); 387f0486c68SYan, Zheng *last_ref = 1; 3885d4f98a2SYan Zheng } 3895d4f98a2SYan Zheng return 0; 3905d4f98a2SYan Zheng } 3915d4f98a2SYan Zheng 3925d4f98a2SYan Zheng /* 393d397712bSChris Mason * does the dirty work in cow of a single block. The parent block (if 394d397712bSChris Mason * supplied) is updated to point to the new cow copy. The new buffer is marked 395d397712bSChris Mason * dirty and returned locked. If you modify the block it needs to be marked 396d397712bSChris Mason * dirty again. 397d352ac68SChris Mason * 398d352ac68SChris Mason * search_start -- an allocation hint for the new block 399d352ac68SChris Mason * 400d397712bSChris Mason * empty_size -- a hint that you plan on doing more cow. This is the size in 401d397712bSChris Mason * bytes the allocator should try to find free next to the block it returns. 402d397712bSChris Mason * This is just a hint and may be ignored by the allocator. 403d352ac68SChris Mason */ 404d397712bSChris Mason static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans, 4055f39d397SChris Mason struct btrfs_root *root, 4065f39d397SChris Mason struct extent_buffer *buf, 4075f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 4085f39d397SChris Mason struct extent_buffer **cow_ret, 4099631e4ccSJosef Bacik u64 search_start, u64 empty_size, 4109631e4ccSJosef Bacik enum btrfs_lock_nesting nest) 4116702ed49SChris Mason { 4120b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 4135d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 4145f39d397SChris Mason struct extent_buffer *cow; 415be1a5564SMark Fasheh int level, ret; 416f0486c68SYan, Zheng int last_ref = 0; 417925baeddSChris Mason int unlock_orig = 0; 4180f5053ebSGoldwyn Rodrigues u64 parent_start = 0; 4196702ed49SChris Mason 420925baeddSChris Mason if (*cow_ret == buf) 421925baeddSChris Mason unlock_orig = 1; 422925baeddSChris Mason 42349d0c642SFilipe Manana btrfs_assert_tree_write_locked(buf); 424925baeddSChris Mason 42592a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 4260b246afaSJeff Mahoney trans->transid != fs_info->running_transaction->transid); 42792a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 42827cdeb70SMiao Xie trans->transid != root->last_trans); 4295f39d397SChris Mason 4307bb86316SChris Mason level = btrfs_header_level(buf); 43131840ae1SZheng Yan 4325d4f98a2SYan Zheng if (level == 0) 4335d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 4345d4f98a2SYan Zheng else 4355d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 4365d4f98a2SYan Zheng 4370f5053ebSGoldwyn Rodrigues if ((root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && parent) 4385d4f98a2SYan Zheng parent_start = parent->start; 4395d4f98a2SYan Zheng 44079bd3712SFilipe Manana cow = btrfs_alloc_tree_block(trans, root, parent_start, 44179bd3712SFilipe Manana root->root_key.objectid, &disk_key, level, 44279bd3712SFilipe Manana search_start, empty_size, nest); 4436702ed49SChris Mason if (IS_ERR(cow)) 4446702ed49SChris Mason return PTR_ERR(cow); 4456702ed49SChris Mason 446b4ce94deSChris Mason /* cow is set to blocking by btrfs_init_new_buffer */ 447b4ce94deSChris Mason 44858e8012cSDavid Sterba copy_extent_buffer_full(cow, buf); 449db94535dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 4505f39d397SChris Mason btrfs_set_header_generation(cow, trans->transid); 4515d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 4525d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 4535d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 4545d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) 4555d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 4565d4f98a2SYan Zheng else 4575f39d397SChris Mason btrfs_set_header_owner(cow, root->root_key.objectid); 4586702ed49SChris Mason 459de37aa51SNikolay Borisov write_extent_buffer_fsid(cow, fs_info->fs_devices->metadata_uuid); 4602b82032cSYan Zheng 461be1a5564SMark Fasheh ret = update_ref_for_cow(trans, root, buf, cow, &last_ref); 462b68dc2a9SMark Fasheh if (ret) { 463572c83acSJosef Bacik btrfs_tree_unlock(cow); 464572c83acSJosef Bacik free_extent_buffer(cow); 46566642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 466b68dc2a9SMark Fasheh return ret; 467b68dc2a9SMark Fasheh } 4681a40e23bSZheng Yan 46992a7cc42SQu Wenruo if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) { 47083d4cfd4SJosef Bacik ret = btrfs_reloc_cow_block(trans, root, buf, cow); 47193314e3bSZhaolei if (ret) { 472572c83acSJosef Bacik btrfs_tree_unlock(cow); 473572c83acSJosef Bacik free_extent_buffer(cow); 47466642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 47583d4cfd4SJosef Bacik return ret; 47683d4cfd4SJosef Bacik } 47793314e3bSZhaolei } 4783fd0a558SYan, Zheng 4796702ed49SChris Mason if (buf == root->node) { 480925baeddSChris Mason WARN_ON(parent && parent != buf); 4815d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 4825d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 4835d4f98a2SYan Zheng parent_start = buf->start; 484925baeddSChris Mason 48567439dadSDavid Sterba atomic_inc(&cow->refs); 486406808abSFilipe Manana ret = btrfs_tree_mod_log_insert_root(root->node, cow, true); 487d9d19a01SDavid Sterba BUG_ON(ret < 0); 488240f62c8SChris Mason rcu_assign_pointer(root->node, cow); 489925baeddSChris Mason 4907a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), buf, 4917a163608SFilipe Manana parent_start, last_ref); 4925f39d397SChris Mason free_extent_buffer(buf); 4930b86a832SChris Mason add_root_to_dirty_list(root); 4946702ed49SChris Mason } else { 4955d4f98a2SYan Zheng WARN_ON(trans->transid != btrfs_header_generation(parent)); 496f3a84ccdSFilipe Manana btrfs_tree_mod_log_insert_key(parent, parent_slot, 49733cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 4985f39d397SChris Mason btrfs_set_node_blockptr(parent, parent_slot, 499db94535dSChris Mason cow->start); 50074493f7aSChris Mason btrfs_set_node_ptr_generation(parent, parent_slot, 50174493f7aSChris Mason trans->transid); 5026702ed49SChris Mason btrfs_mark_buffer_dirty(parent); 5035de865eeSFilipe David Borba Manana if (last_ref) { 504f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_free_eb(buf); 5055de865eeSFilipe David Borba Manana if (ret) { 506572c83acSJosef Bacik btrfs_tree_unlock(cow); 507572c83acSJosef Bacik free_extent_buffer(cow); 50866642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 5095de865eeSFilipe David Borba Manana return ret; 5105de865eeSFilipe David Borba Manana } 5115de865eeSFilipe David Borba Manana } 5127a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), buf, 5137a163608SFilipe Manana parent_start, last_ref); 5146702ed49SChris Mason } 515925baeddSChris Mason if (unlock_orig) 516925baeddSChris Mason btrfs_tree_unlock(buf); 5173083ee2eSJosef Bacik free_extent_buffer_stale(buf); 5186702ed49SChris Mason btrfs_mark_buffer_dirty(cow); 5196702ed49SChris Mason *cow_ret = cow; 5206702ed49SChris Mason return 0; 5216702ed49SChris Mason } 5226702ed49SChris Mason 5235d4f98a2SYan Zheng static inline int should_cow_block(struct btrfs_trans_handle *trans, 5245d4f98a2SYan Zheng struct btrfs_root *root, 5255d4f98a2SYan Zheng struct extent_buffer *buf) 5265d4f98a2SYan Zheng { 527f5ee5c9aSJeff Mahoney if (btrfs_is_testing(root->fs_info)) 528faa2dbf0SJosef Bacik return 0; 529fccb84c9SDavid Sterba 530d1980131SDavid Sterba /* Ensure we can see the FORCE_COW bit */ 531d1980131SDavid Sterba smp_mb__before_atomic(); 532f1ebcc74SLiu Bo 533f1ebcc74SLiu Bo /* 534f1ebcc74SLiu Bo * We do not need to cow a block if 535f1ebcc74SLiu Bo * 1) this block is not created or changed in this transaction; 536f1ebcc74SLiu Bo * 2) this block does not belong to TREE_RELOC tree; 537f1ebcc74SLiu Bo * 3) the root is not forced COW. 538f1ebcc74SLiu Bo * 539f1ebcc74SLiu Bo * What is forced COW: 54001327610SNicholas D Steeves * when we create snapshot during committing the transaction, 54152042d8eSAndrea Gelmini * after we've finished copying src root, we must COW the shared 542f1ebcc74SLiu Bo * block to ensure the metadata consistency. 543f1ebcc74SLiu Bo */ 5445d4f98a2SYan Zheng if (btrfs_header_generation(buf) == trans->transid && 5455d4f98a2SYan Zheng !btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN) && 5465d4f98a2SYan Zheng !(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID && 547f1ebcc74SLiu Bo btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)) && 54827cdeb70SMiao Xie !test_bit(BTRFS_ROOT_FORCE_COW, &root->state)) 5495d4f98a2SYan Zheng return 0; 5505d4f98a2SYan Zheng return 1; 5515d4f98a2SYan Zheng } 5525d4f98a2SYan Zheng 553d352ac68SChris Mason /* 554d352ac68SChris Mason * cows a single block, see __btrfs_cow_block for the real work. 55501327610SNicholas D Steeves * This version of it has extra checks so that a block isn't COWed more than 556d352ac68SChris Mason * once per transaction, as long as it hasn't been written yet 557d352ac68SChris Mason */ 558d397712bSChris Mason noinline int btrfs_cow_block(struct btrfs_trans_handle *trans, 5595f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *buf, 5605f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 5619631e4ccSJosef Bacik struct extent_buffer **cow_ret, 5629631e4ccSJosef Bacik enum btrfs_lock_nesting nest) 56302217ed2SChris Mason { 5640b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 5656702ed49SChris Mason u64 search_start; 566f510cfecSChris Mason int ret; 567dc17ff8fSChris Mason 56883354f07SJosef Bacik if (test_bit(BTRFS_ROOT_DELETING, &root->state)) 56983354f07SJosef Bacik btrfs_err(fs_info, 57083354f07SJosef Bacik "COW'ing blocks on a fs root that's being dropped"); 57183354f07SJosef Bacik 5720b246afaSJeff Mahoney if (trans->transaction != fs_info->running_transaction) 57331b1a2bdSJulia Lawall WARN(1, KERN_CRIT "trans %llu running %llu\n", 574c1c9ff7cSGeert Uytterhoeven trans->transid, 5750b246afaSJeff Mahoney fs_info->running_transaction->transid); 57631b1a2bdSJulia Lawall 5770b246afaSJeff Mahoney if (trans->transid != fs_info->generation) 57831b1a2bdSJulia Lawall WARN(1, KERN_CRIT "trans %llu running %llu\n", 5790b246afaSJeff Mahoney trans->transid, fs_info->generation); 580dc17ff8fSChris Mason 5815d4f98a2SYan Zheng if (!should_cow_block(trans, root, buf)) { 58202217ed2SChris Mason *cow_ret = buf; 58302217ed2SChris Mason return 0; 58402217ed2SChris Mason } 585c487685dSChris Mason 586ee22184bSByongho Lee search_start = buf->start & ~((u64)SZ_1G - 1); 587b4ce94deSChris Mason 588f616f5cdSQu Wenruo /* 589f616f5cdSQu Wenruo * Before CoWing this block for later modification, check if it's 590f616f5cdSQu Wenruo * the subtree root and do the delayed subtree trace if needed. 591f616f5cdSQu Wenruo * 592f616f5cdSQu Wenruo * Also We don't care about the error, as it's handled internally. 593f616f5cdSQu Wenruo */ 594f616f5cdSQu Wenruo btrfs_qgroup_trace_subtree_after_cow(trans, root, buf); 595f510cfecSChris Mason ret = __btrfs_cow_block(trans, root, buf, parent, 5969631e4ccSJosef Bacik parent_slot, cow_ret, search_start, 0, nest); 5971abe9b8aSliubo 5981abe9b8aSliubo trace_btrfs_cow_block(root, buf, *cow_ret); 5991abe9b8aSliubo 600f510cfecSChris Mason return ret; 6012c90e5d6SChris Mason } 602f75e2b79SJosef Bacik ALLOW_ERROR_INJECTION(btrfs_cow_block, ERRNO); 6036702ed49SChris Mason 604d352ac68SChris Mason /* 605d352ac68SChris Mason * helper function for defrag to decide if two blocks pointed to by a 606d352ac68SChris Mason * node are actually close by 607d352ac68SChris Mason */ 6086b80053dSChris Mason static int close_blocks(u64 blocknr, u64 other, u32 blocksize) 6096702ed49SChris Mason { 6106b80053dSChris Mason if (blocknr < other && other - (blocknr + blocksize) < 32768) 6116702ed49SChris Mason return 1; 6126b80053dSChris Mason if (blocknr > other && blocknr - (other + blocksize) < 32768) 6136702ed49SChris Mason return 1; 61402217ed2SChris Mason return 0; 61502217ed2SChris Mason } 61602217ed2SChris Mason 617ce6ef5abSDavid Sterba #ifdef __LITTLE_ENDIAN 618ce6ef5abSDavid Sterba 619ce6ef5abSDavid Sterba /* 620ce6ef5abSDavid Sterba * Compare two keys, on little-endian the disk order is same as CPU order and 621ce6ef5abSDavid Sterba * we can avoid the conversion. 622ce6ef5abSDavid Sterba */ 623ce6ef5abSDavid Sterba static int comp_keys(const struct btrfs_disk_key *disk_key, 624ce6ef5abSDavid Sterba const struct btrfs_key *k2) 625ce6ef5abSDavid Sterba { 626ce6ef5abSDavid Sterba const struct btrfs_key *k1 = (const struct btrfs_key *)disk_key; 627ce6ef5abSDavid Sterba 628ce6ef5abSDavid Sterba return btrfs_comp_cpu_keys(k1, k2); 629ce6ef5abSDavid Sterba } 630ce6ef5abSDavid Sterba 631ce6ef5abSDavid Sterba #else 632ce6ef5abSDavid Sterba 633081e9573SChris Mason /* 634081e9573SChris Mason * compare two keys in a memcmp fashion 635081e9573SChris Mason */ 636310712b2SOmar Sandoval static int comp_keys(const struct btrfs_disk_key *disk, 637310712b2SOmar Sandoval const struct btrfs_key *k2) 638081e9573SChris Mason { 639081e9573SChris Mason struct btrfs_key k1; 640081e9573SChris Mason 641081e9573SChris Mason btrfs_disk_key_to_cpu(&k1, disk); 642081e9573SChris Mason 64320736abaSDiego Calleja return btrfs_comp_cpu_keys(&k1, k2); 644081e9573SChris Mason } 645ce6ef5abSDavid Sterba #endif 646081e9573SChris Mason 647f3465ca4SJosef Bacik /* 648f3465ca4SJosef Bacik * same as comp_keys only with two btrfs_key's 649f3465ca4SJosef Bacik */ 650e1f60a65SDavid Sterba int __pure btrfs_comp_cpu_keys(const struct btrfs_key *k1, const struct btrfs_key *k2) 651f3465ca4SJosef Bacik { 652f3465ca4SJosef Bacik if (k1->objectid > k2->objectid) 653f3465ca4SJosef Bacik return 1; 654f3465ca4SJosef Bacik if (k1->objectid < k2->objectid) 655f3465ca4SJosef Bacik return -1; 656f3465ca4SJosef Bacik if (k1->type > k2->type) 657f3465ca4SJosef Bacik return 1; 658f3465ca4SJosef Bacik if (k1->type < k2->type) 659f3465ca4SJosef Bacik return -1; 660f3465ca4SJosef Bacik if (k1->offset > k2->offset) 661f3465ca4SJosef Bacik return 1; 662f3465ca4SJosef Bacik if (k1->offset < k2->offset) 663f3465ca4SJosef Bacik return -1; 664f3465ca4SJosef Bacik return 0; 665f3465ca4SJosef Bacik } 666081e9573SChris Mason 667d352ac68SChris Mason /* 668d352ac68SChris Mason * this is used by the defrag code to go through all the 669d352ac68SChris Mason * leaves pointed to by a node and reallocate them so that 670d352ac68SChris Mason * disk order is close to key order 671d352ac68SChris Mason */ 6726702ed49SChris Mason int btrfs_realloc_node(struct btrfs_trans_handle *trans, 6735f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *parent, 674de78b51aSEric Sandeen int start_slot, u64 *last_ret, 675a6b6e75eSChris Mason struct btrfs_key *progress) 6766702ed49SChris Mason { 6770b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 6786b80053dSChris Mason struct extent_buffer *cur; 6796702ed49SChris Mason u64 blocknr; 680e9d0b13bSChris Mason u64 search_start = *last_ret; 681e9d0b13bSChris Mason u64 last_block = 0; 6826702ed49SChris Mason u64 other; 6836702ed49SChris Mason u32 parent_nritems; 6846702ed49SChris Mason int end_slot; 6856702ed49SChris Mason int i; 6866702ed49SChris Mason int err = 0; 6876b80053dSChris Mason u32 blocksize; 688081e9573SChris Mason int progress_passed = 0; 689081e9573SChris Mason struct btrfs_disk_key disk_key; 6906702ed49SChris Mason 6910b246afaSJeff Mahoney WARN_ON(trans->transaction != fs_info->running_transaction); 6920b246afaSJeff Mahoney WARN_ON(trans->transid != fs_info->generation); 69386479a04SChris Mason 6946b80053dSChris Mason parent_nritems = btrfs_header_nritems(parent); 6950b246afaSJeff Mahoney blocksize = fs_info->nodesize; 6965dfe2be7SFilipe Manana end_slot = parent_nritems - 1; 6976702ed49SChris Mason 6985dfe2be7SFilipe Manana if (parent_nritems <= 1) 6996702ed49SChris Mason return 0; 7006702ed49SChris Mason 7015dfe2be7SFilipe Manana for (i = start_slot; i <= end_slot; i++) { 7026702ed49SChris Mason int close = 1; 703a6b6e75eSChris Mason 704081e9573SChris Mason btrfs_node_key(parent, &disk_key, i); 705081e9573SChris Mason if (!progress_passed && comp_keys(&disk_key, progress) < 0) 706081e9573SChris Mason continue; 707081e9573SChris Mason 708081e9573SChris Mason progress_passed = 1; 7096b80053dSChris Mason blocknr = btrfs_node_blockptr(parent, i); 710e9d0b13bSChris Mason if (last_block == 0) 711e9d0b13bSChris Mason last_block = blocknr; 7125708b959SChris Mason 7136702ed49SChris Mason if (i > 0) { 7146b80053dSChris Mason other = btrfs_node_blockptr(parent, i - 1); 7156b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 7166702ed49SChris Mason } 7175dfe2be7SFilipe Manana if (!close && i < end_slot) { 7186b80053dSChris Mason other = btrfs_node_blockptr(parent, i + 1); 7196b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 7206702ed49SChris Mason } 721e9d0b13bSChris Mason if (close) { 722e9d0b13bSChris Mason last_block = blocknr; 7236702ed49SChris Mason continue; 724e9d0b13bSChris Mason } 7256702ed49SChris Mason 726206983b7SJosef Bacik cur = btrfs_read_node_slot(parent, i); 727206983b7SJosef Bacik if (IS_ERR(cur)) 72864c043deSLiu Bo return PTR_ERR(cur); 729e9d0b13bSChris Mason if (search_start == 0) 7306b80053dSChris Mason search_start = last_block; 731e9d0b13bSChris Mason 732e7a84565SChris Mason btrfs_tree_lock(cur); 7336b80053dSChris Mason err = __btrfs_cow_block(trans, root, cur, parent, i, 734e7a84565SChris Mason &cur, search_start, 7356b80053dSChris Mason min(16 * blocksize, 7369631e4ccSJosef Bacik (end_slot - i) * blocksize), 7379631e4ccSJosef Bacik BTRFS_NESTING_COW); 738252c38f0SYan if (err) { 739e7a84565SChris Mason btrfs_tree_unlock(cur); 7406b80053dSChris Mason free_extent_buffer(cur); 7416702ed49SChris Mason break; 742252c38f0SYan } 743e7a84565SChris Mason search_start = cur->start; 744e7a84565SChris Mason last_block = cur->start; 745f2183bdeSChris Mason *last_ret = search_start; 746e7a84565SChris Mason btrfs_tree_unlock(cur); 747e7a84565SChris Mason free_extent_buffer(cur); 7486702ed49SChris Mason } 7496702ed49SChris Mason return err; 7506702ed49SChris Mason } 7516702ed49SChris Mason 75274123bd7SChris Mason /* 753fb81212cSFilipe Manana * Search for a key in the given extent_buffer. 7545f39d397SChris Mason * 755fb81212cSFilipe Manana * The lower boundary for the search is specified by the slot number @low. Use a 756fb81212cSFilipe Manana * value of 0 to search over the whole extent buffer. 75774123bd7SChris Mason * 758fb81212cSFilipe Manana * The slot in the extent buffer is returned via @slot. If the key exists in the 759fb81212cSFilipe Manana * extent buffer, then @slot will point to the slot where the key is, otherwise 760fb81212cSFilipe Manana * it points to the slot where you would insert the key. 761fb81212cSFilipe Manana * 762fb81212cSFilipe Manana * Slot may point to the total number of items (i.e. one position beyond the last 763fb81212cSFilipe Manana * key) if the key is bigger than the last key in the extent buffer. 76474123bd7SChris Mason */ 765fb81212cSFilipe Manana static noinline int generic_bin_search(struct extent_buffer *eb, int low, 76667d5e289SMarcos Paulo de Souza const struct btrfs_key *key, int *slot) 767be0e5c09SChris Mason { 768fb81212cSFilipe Manana unsigned long p; 769fb81212cSFilipe Manana int item_size; 77067d5e289SMarcos Paulo de Souza int high = btrfs_header_nritems(eb); 771be0e5c09SChris Mason int ret; 7725cd17f34SDavid Sterba const int key_size = sizeof(struct btrfs_disk_key); 773be0e5c09SChris Mason 7745e24e9afSLiu Bo if (low > high) { 7755e24e9afSLiu Bo btrfs_err(eb->fs_info, 7765e24e9afSLiu Bo "%s: low (%d) > high (%d) eb %llu owner %llu level %d", 7775e24e9afSLiu Bo __func__, low, high, eb->start, 7785e24e9afSLiu Bo btrfs_header_owner(eb), btrfs_header_level(eb)); 7795e24e9afSLiu Bo return -EINVAL; 7805e24e9afSLiu Bo } 7815e24e9afSLiu Bo 782fb81212cSFilipe Manana if (btrfs_header_level(eb) == 0) { 783fb81212cSFilipe Manana p = offsetof(struct btrfs_leaf, items); 784fb81212cSFilipe Manana item_size = sizeof(struct btrfs_item); 785fb81212cSFilipe Manana } else { 786fb81212cSFilipe Manana p = offsetof(struct btrfs_node, ptrs); 787fb81212cSFilipe Manana item_size = sizeof(struct btrfs_key_ptr); 788fb81212cSFilipe Manana } 789fb81212cSFilipe Manana 790be0e5c09SChris Mason while (low < high) { 7915cd17f34SDavid Sterba unsigned long oip; 7925cd17f34SDavid Sterba unsigned long offset; 7935cd17f34SDavid Sterba struct btrfs_disk_key *tmp; 7945cd17f34SDavid Sterba struct btrfs_disk_key unaligned; 7955cd17f34SDavid Sterba int mid; 7965cd17f34SDavid Sterba 797be0e5c09SChris Mason mid = (low + high) / 2; 7985f39d397SChris Mason offset = p + mid * item_size; 7995cd17f34SDavid Sterba oip = offset_in_page(offset); 8005f39d397SChris Mason 8015cd17f34SDavid Sterba if (oip + key_size <= PAGE_SIZE) { 802884b07d0SQu Wenruo const unsigned long idx = get_eb_page_index(offset); 8035cd17f34SDavid Sterba char *kaddr = page_address(eb->pages[idx]); 804934d375bSChris Mason 805884b07d0SQu Wenruo oip = get_eb_offset_in_page(eb, offset); 8065cd17f34SDavid Sterba tmp = (struct btrfs_disk_key *)(kaddr + oip); 8075cd17f34SDavid Sterba } else { 8085cd17f34SDavid Sterba read_extent_buffer(eb, &unaligned, offset, key_size); 8095f39d397SChris Mason tmp = &unaligned; 810479965d6SChris Mason } 811479965d6SChris Mason 812be0e5c09SChris Mason ret = comp_keys(tmp, key); 813be0e5c09SChris Mason 814be0e5c09SChris Mason if (ret < 0) 815be0e5c09SChris Mason low = mid + 1; 816be0e5c09SChris Mason else if (ret > 0) 817be0e5c09SChris Mason high = mid; 818be0e5c09SChris Mason else { 819be0e5c09SChris Mason *slot = mid; 820be0e5c09SChris Mason return 0; 821be0e5c09SChris Mason } 822be0e5c09SChris Mason } 823be0e5c09SChris Mason *slot = low; 824be0e5c09SChris Mason return 1; 825be0e5c09SChris Mason } 826be0e5c09SChris Mason 82797571fd0SChris Mason /* 828fb81212cSFilipe Manana * Simple binary search on an extent buffer. Works for both leaves and nodes, and 829fb81212cSFilipe Manana * always searches over the whole range of keys (slot 0 to slot 'nritems - 1'). 83097571fd0SChris Mason */ 831a74b35ecSNikolay Borisov int btrfs_bin_search(struct extent_buffer *eb, const struct btrfs_key *key, 832e3b83361SQu Wenruo int *slot) 833be0e5c09SChris Mason { 834fb81212cSFilipe Manana return generic_bin_search(eb, 0, key, slot); 835be0e5c09SChris Mason } 836be0e5c09SChris Mason 837f0486c68SYan, Zheng static void root_add_used(struct btrfs_root *root, u32 size) 838f0486c68SYan, Zheng { 839f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 840f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 841f0486c68SYan, Zheng btrfs_root_used(&root->root_item) + size); 842f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 843f0486c68SYan, Zheng } 844f0486c68SYan, Zheng 845f0486c68SYan, Zheng static void root_sub_used(struct btrfs_root *root, u32 size) 846f0486c68SYan, Zheng { 847f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 848f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 849f0486c68SYan, Zheng btrfs_root_used(&root->root_item) - size); 850f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 851f0486c68SYan, Zheng } 852f0486c68SYan, Zheng 853d352ac68SChris Mason /* given a node and slot number, this reads the blocks it points to. The 854d352ac68SChris Mason * extent buffer is returned with a reference taken (but unlocked). 855d352ac68SChris Mason */ 8564b231ae4SDavid Sterba struct extent_buffer *btrfs_read_node_slot(struct extent_buffer *parent, 8574b231ae4SDavid Sterba int slot) 858bb803951SChris Mason { 859ca7a79adSChris Mason int level = btrfs_header_level(parent); 860416bc658SJosef Bacik struct extent_buffer *eb; 861581c1760SQu Wenruo struct btrfs_key first_key; 862416bc658SJosef Bacik 863fb770ae4SLiu Bo if (slot < 0 || slot >= btrfs_header_nritems(parent)) 864fb770ae4SLiu Bo return ERR_PTR(-ENOENT); 865ca7a79adSChris Mason 866ca7a79adSChris Mason BUG_ON(level == 0); 867ca7a79adSChris Mason 868581c1760SQu Wenruo btrfs_node_key_to_cpu(parent, &first_key, slot); 869d0d20b0fSDavid Sterba eb = read_tree_block(parent->fs_info, btrfs_node_blockptr(parent, slot), 8701b7ec85eSJosef Bacik btrfs_header_owner(parent), 871581c1760SQu Wenruo btrfs_node_ptr_generation(parent, slot), 872581c1760SQu Wenruo level - 1, &first_key); 8734eb150d6SQu Wenruo if (IS_ERR(eb)) 8744eb150d6SQu Wenruo return eb; 8754eb150d6SQu Wenruo if (!extent_buffer_uptodate(eb)) { 876416bc658SJosef Bacik free_extent_buffer(eb); 8774eb150d6SQu Wenruo return ERR_PTR(-EIO); 878416bc658SJosef Bacik } 879416bc658SJosef Bacik 880416bc658SJosef Bacik return eb; 881bb803951SChris Mason } 882bb803951SChris Mason 883d352ac68SChris Mason /* 884d352ac68SChris Mason * node level balancing, used to make sure nodes are in proper order for 885d352ac68SChris Mason * item deletion. We balance from the top down, so we have to make sure 886d352ac68SChris Mason * that a deletion won't leave an node completely empty later on. 887d352ac68SChris Mason */ 888e02119d5SChris Mason static noinline int balance_level(struct btrfs_trans_handle *trans, 88998ed5174SChris Mason struct btrfs_root *root, 89098ed5174SChris Mason struct btrfs_path *path, int level) 891bb803951SChris Mason { 8920b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 8935f39d397SChris Mason struct extent_buffer *right = NULL; 8945f39d397SChris Mason struct extent_buffer *mid; 8955f39d397SChris Mason struct extent_buffer *left = NULL; 8965f39d397SChris Mason struct extent_buffer *parent = NULL; 897bb803951SChris Mason int ret = 0; 898bb803951SChris Mason int wret; 899bb803951SChris Mason int pslot; 900bb803951SChris Mason int orig_slot = path->slots[level]; 90179f95c82SChris Mason u64 orig_ptr; 902bb803951SChris Mason 90398e6b1ebSLiu Bo ASSERT(level > 0); 904bb803951SChris Mason 9055f39d397SChris Mason mid = path->nodes[level]; 906b4ce94deSChris Mason 907ac5887c8SJosef Bacik WARN_ON(path->locks[level] != BTRFS_WRITE_LOCK); 9087bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 9097bb86316SChris Mason 9101d4f8a0cSChris Mason orig_ptr = btrfs_node_blockptr(mid, orig_slot); 91179f95c82SChris Mason 912a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 9135f39d397SChris Mason parent = path->nodes[level + 1]; 914bb803951SChris Mason pslot = path->slots[level + 1]; 915a05a9bb1SLi Zefan } 916bb803951SChris Mason 91740689478SChris Mason /* 91840689478SChris Mason * deal with the case where there is only one pointer in the root 91940689478SChris Mason * by promoting the node below to a root 92040689478SChris Mason */ 9215f39d397SChris Mason if (!parent) { 9225f39d397SChris Mason struct extent_buffer *child; 923bb803951SChris Mason 9245f39d397SChris Mason if (btrfs_header_nritems(mid) != 1) 925bb803951SChris Mason return 0; 926bb803951SChris Mason 927bb803951SChris Mason /* promote the child to a root */ 9284b231ae4SDavid Sterba child = btrfs_read_node_slot(mid, 0); 929fb770ae4SLiu Bo if (IS_ERR(child)) { 930fb770ae4SLiu Bo ret = PTR_ERR(child); 9310b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 932305a26afSMark Fasheh goto enospc; 933305a26afSMark Fasheh } 934305a26afSMark Fasheh 935925baeddSChris Mason btrfs_tree_lock(child); 9369631e4ccSJosef Bacik ret = btrfs_cow_block(trans, root, child, mid, 0, &child, 9379631e4ccSJosef Bacik BTRFS_NESTING_COW); 938f0486c68SYan, Zheng if (ret) { 939f0486c68SYan, Zheng btrfs_tree_unlock(child); 940f0486c68SYan, Zheng free_extent_buffer(child); 941f0486c68SYan, Zheng goto enospc; 942f0486c68SYan, Zheng } 9432f375ab9SYan 944406808abSFilipe Manana ret = btrfs_tree_mod_log_insert_root(root->node, child, true); 945d9d19a01SDavid Sterba BUG_ON(ret < 0); 946240f62c8SChris Mason rcu_assign_pointer(root->node, child); 947925baeddSChris Mason 9480b86a832SChris Mason add_root_to_dirty_list(root); 949925baeddSChris Mason btrfs_tree_unlock(child); 950b4ce94deSChris Mason 951925baeddSChris Mason path->locks[level] = 0; 952bb803951SChris Mason path->nodes[level] = NULL; 9536a884d7dSDavid Sterba btrfs_clean_tree_block(mid); 954925baeddSChris Mason btrfs_tree_unlock(mid); 955bb803951SChris Mason /* once for the path */ 9565f39d397SChris Mason free_extent_buffer(mid); 957f0486c68SYan, Zheng 958f0486c68SYan, Zheng root_sub_used(root, mid->len); 9597a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), mid, 0, 1); 960bb803951SChris Mason /* once for the root ptr */ 9613083ee2eSJosef Bacik free_extent_buffer_stale(mid); 962f0486c68SYan, Zheng return 0; 963bb803951SChris Mason } 9645f39d397SChris Mason if (btrfs_header_nritems(mid) > 9650b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 4) 966bb803951SChris Mason return 0; 967bb803951SChris Mason 9684b231ae4SDavid Sterba left = btrfs_read_node_slot(parent, pslot - 1); 969fb770ae4SLiu Bo if (IS_ERR(left)) 970fb770ae4SLiu Bo left = NULL; 971fb770ae4SLiu Bo 9725f39d397SChris Mason if (left) { 973bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT); 9745f39d397SChris Mason wret = btrfs_cow_block(trans, root, left, 9759631e4ccSJosef Bacik parent, pslot - 1, &left, 976bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW); 97754aa1f4dSChris Mason if (wret) { 97854aa1f4dSChris Mason ret = wret; 97954aa1f4dSChris Mason goto enospc; 98054aa1f4dSChris Mason } 9812cc58cf2SChris Mason } 982fb770ae4SLiu Bo 9834b231ae4SDavid Sterba right = btrfs_read_node_slot(parent, pslot + 1); 984fb770ae4SLiu Bo if (IS_ERR(right)) 985fb770ae4SLiu Bo right = NULL; 986fb770ae4SLiu Bo 9875f39d397SChris Mason if (right) { 988bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT); 9895f39d397SChris Mason wret = btrfs_cow_block(trans, root, right, 9909631e4ccSJosef Bacik parent, pslot + 1, &right, 991bf59a5a2SJosef Bacik BTRFS_NESTING_RIGHT_COW); 9922cc58cf2SChris Mason if (wret) { 9932cc58cf2SChris Mason ret = wret; 9942cc58cf2SChris Mason goto enospc; 9952cc58cf2SChris Mason } 9962cc58cf2SChris Mason } 9972cc58cf2SChris Mason 9982cc58cf2SChris Mason /* first, try to make some room in the middle buffer */ 9995f39d397SChris Mason if (left) { 10005f39d397SChris Mason orig_slot += btrfs_header_nritems(left); 1001d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 1); 100279f95c82SChris Mason if (wret < 0) 100379f95c82SChris Mason ret = wret; 1004bb803951SChris Mason } 100579f95c82SChris Mason 100679f95c82SChris Mason /* 100779f95c82SChris Mason * then try to empty the right most buffer into the middle 100879f95c82SChris Mason */ 10095f39d397SChris Mason if (right) { 1010d30a668fSDavid Sterba wret = push_node_left(trans, mid, right, 1); 101154aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 101279f95c82SChris Mason ret = wret; 10135f39d397SChris Mason if (btrfs_header_nritems(right) == 0) { 10146a884d7dSDavid Sterba btrfs_clean_tree_block(right); 1015925baeddSChris Mason btrfs_tree_unlock(right); 1016afe5fea7STsutomu Itoh del_ptr(root, path, level + 1, pslot + 1); 1017f0486c68SYan, Zheng root_sub_used(root, right->len); 10187a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), right, 10197a163608SFilipe Manana 0, 1); 10203083ee2eSJosef Bacik free_extent_buffer_stale(right); 1021f0486c68SYan, Zheng right = NULL; 1022bb803951SChris Mason } else { 10235f39d397SChris Mason struct btrfs_disk_key right_key; 10245f39d397SChris Mason btrfs_node_key(right, &right_key, 0); 1025f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot + 1, 102633cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 10270e82bcfeSDavid Sterba BUG_ON(ret < 0); 10285f39d397SChris Mason btrfs_set_node_key(parent, &right_key, pslot + 1); 10295f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 1030bb803951SChris Mason } 1031bb803951SChris Mason } 10325f39d397SChris Mason if (btrfs_header_nritems(mid) == 1) { 103379f95c82SChris Mason /* 103479f95c82SChris Mason * we're not allowed to leave a node with one item in the 103579f95c82SChris Mason * tree during a delete. A deletion from lower in the tree 103679f95c82SChris Mason * could try to delete the only pointer in this node. 103779f95c82SChris Mason * So, pull some keys from the left. 103879f95c82SChris Mason * There has to be a left pointer at this point because 103979f95c82SChris Mason * otherwise we would have pulled some pointers from the 104079f95c82SChris Mason * right 104179f95c82SChris Mason */ 1042305a26afSMark Fasheh if (!left) { 1043305a26afSMark Fasheh ret = -EROFS; 10440b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 1045305a26afSMark Fasheh goto enospc; 1046305a26afSMark Fasheh } 104755d32ed8SDavid Sterba wret = balance_node_right(trans, mid, left); 104854aa1f4dSChris Mason if (wret < 0) { 104979f95c82SChris Mason ret = wret; 105054aa1f4dSChris Mason goto enospc; 105154aa1f4dSChris Mason } 1052bce4eae9SChris Mason if (wret == 1) { 1053d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 1); 1054bce4eae9SChris Mason if (wret < 0) 1055bce4eae9SChris Mason ret = wret; 1056bce4eae9SChris Mason } 105779f95c82SChris Mason BUG_ON(wret == 1); 105879f95c82SChris Mason } 10595f39d397SChris Mason if (btrfs_header_nritems(mid) == 0) { 10606a884d7dSDavid Sterba btrfs_clean_tree_block(mid); 1061925baeddSChris Mason btrfs_tree_unlock(mid); 1062afe5fea7STsutomu Itoh del_ptr(root, path, level + 1, pslot); 1063f0486c68SYan, Zheng root_sub_used(root, mid->len); 10647a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), mid, 0, 1); 10653083ee2eSJosef Bacik free_extent_buffer_stale(mid); 1066f0486c68SYan, Zheng mid = NULL; 106779f95c82SChris Mason } else { 106879f95c82SChris Mason /* update the parent key to reflect our changes */ 10695f39d397SChris Mason struct btrfs_disk_key mid_key; 10705f39d397SChris Mason btrfs_node_key(mid, &mid_key, 0); 1071f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot, 107233cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 10730e82bcfeSDavid Sterba BUG_ON(ret < 0); 10745f39d397SChris Mason btrfs_set_node_key(parent, &mid_key, pslot); 10755f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 107679f95c82SChris Mason } 1077bb803951SChris Mason 107879f95c82SChris Mason /* update the path */ 10795f39d397SChris Mason if (left) { 10805f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 108167439dadSDavid Sterba atomic_inc(&left->refs); 1082925baeddSChris Mason /* left was locked after cow */ 10835f39d397SChris Mason path->nodes[level] = left; 1084bb803951SChris Mason path->slots[level + 1] -= 1; 1085bb803951SChris Mason path->slots[level] = orig_slot; 1086925baeddSChris Mason if (mid) { 1087925baeddSChris Mason btrfs_tree_unlock(mid); 10885f39d397SChris Mason free_extent_buffer(mid); 1089925baeddSChris Mason } 1090bb803951SChris Mason } else { 10915f39d397SChris Mason orig_slot -= btrfs_header_nritems(left); 1092bb803951SChris Mason path->slots[level] = orig_slot; 1093bb803951SChris Mason } 1094bb803951SChris Mason } 109579f95c82SChris Mason /* double check we haven't messed things up */ 1096e20d96d6SChris Mason if (orig_ptr != 10975f39d397SChris Mason btrfs_node_blockptr(path->nodes[level], path->slots[level])) 109879f95c82SChris Mason BUG(); 109954aa1f4dSChris Mason enospc: 1100925baeddSChris Mason if (right) { 1101925baeddSChris Mason btrfs_tree_unlock(right); 11025f39d397SChris Mason free_extent_buffer(right); 1103925baeddSChris Mason } 1104925baeddSChris Mason if (left) { 1105925baeddSChris Mason if (path->nodes[level] != left) 1106925baeddSChris Mason btrfs_tree_unlock(left); 11075f39d397SChris Mason free_extent_buffer(left); 1108925baeddSChris Mason } 1109bb803951SChris Mason return ret; 1110bb803951SChris Mason } 1111bb803951SChris Mason 1112d352ac68SChris Mason /* Node balancing for insertion. Here we only split or push nodes around 1113d352ac68SChris Mason * when they are completely full. This is also done top down, so we 1114d352ac68SChris Mason * have to be pessimistic. 1115d352ac68SChris Mason */ 1116d397712bSChris Mason static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans, 1117e66f709bSChris Mason struct btrfs_root *root, 1118e66f709bSChris Mason struct btrfs_path *path, int level) 1119e66f709bSChris Mason { 11200b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 11215f39d397SChris Mason struct extent_buffer *right = NULL; 11225f39d397SChris Mason struct extent_buffer *mid; 11235f39d397SChris Mason struct extent_buffer *left = NULL; 11245f39d397SChris Mason struct extent_buffer *parent = NULL; 1125e66f709bSChris Mason int ret = 0; 1126e66f709bSChris Mason int wret; 1127e66f709bSChris Mason int pslot; 1128e66f709bSChris Mason int orig_slot = path->slots[level]; 1129e66f709bSChris Mason 1130e66f709bSChris Mason if (level == 0) 1131e66f709bSChris Mason return 1; 1132e66f709bSChris Mason 11335f39d397SChris Mason mid = path->nodes[level]; 11347bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 1135e66f709bSChris Mason 1136a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 11375f39d397SChris Mason parent = path->nodes[level + 1]; 1138e66f709bSChris Mason pslot = path->slots[level + 1]; 1139a05a9bb1SLi Zefan } 1140e66f709bSChris Mason 11415f39d397SChris Mason if (!parent) 1142e66f709bSChris Mason return 1; 1143e66f709bSChris Mason 11444b231ae4SDavid Sterba left = btrfs_read_node_slot(parent, pslot - 1); 1145fb770ae4SLiu Bo if (IS_ERR(left)) 1146fb770ae4SLiu Bo left = NULL; 1147e66f709bSChris Mason 1148e66f709bSChris Mason /* first, try to make some room in the middle buffer */ 11495f39d397SChris Mason if (left) { 1150e66f709bSChris Mason u32 left_nr; 1151925baeddSChris Mason 1152bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT); 1153b4ce94deSChris Mason 11545f39d397SChris Mason left_nr = btrfs_header_nritems(left); 11550b246afaSJeff Mahoney if (left_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) { 115633ade1f8SChris Mason wret = 1; 115733ade1f8SChris Mason } else { 11585f39d397SChris Mason ret = btrfs_cow_block(trans, root, left, parent, 11599631e4ccSJosef Bacik pslot - 1, &left, 1160bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW); 116154aa1f4dSChris Mason if (ret) 116254aa1f4dSChris Mason wret = 1; 116354aa1f4dSChris Mason else { 1164d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 0); 116554aa1f4dSChris Mason } 116633ade1f8SChris Mason } 1167e66f709bSChris Mason if (wret < 0) 1168e66f709bSChris Mason ret = wret; 1169e66f709bSChris Mason if (wret == 0) { 11705f39d397SChris Mason struct btrfs_disk_key disk_key; 1171e66f709bSChris Mason orig_slot += left_nr; 11725f39d397SChris Mason btrfs_node_key(mid, &disk_key, 0); 1173f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot, 117433cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 11750e82bcfeSDavid Sterba BUG_ON(ret < 0); 11765f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot); 11775f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 11785f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 11795f39d397SChris Mason path->nodes[level] = left; 1180e66f709bSChris Mason path->slots[level + 1] -= 1; 1181e66f709bSChris Mason path->slots[level] = orig_slot; 1182925baeddSChris Mason btrfs_tree_unlock(mid); 11835f39d397SChris Mason free_extent_buffer(mid); 1184e66f709bSChris Mason } else { 1185e66f709bSChris Mason orig_slot -= 11865f39d397SChris Mason btrfs_header_nritems(left); 1187e66f709bSChris Mason path->slots[level] = orig_slot; 1188925baeddSChris Mason btrfs_tree_unlock(left); 11895f39d397SChris Mason free_extent_buffer(left); 1190e66f709bSChris Mason } 1191e66f709bSChris Mason return 0; 1192e66f709bSChris Mason } 1193925baeddSChris Mason btrfs_tree_unlock(left); 11945f39d397SChris Mason free_extent_buffer(left); 1195e66f709bSChris Mason } 11964b231ae4SDavid Sterba right = btrfs_read_node_slot(parent, pslot + 1); 1197fb770ae4SLiu Bo if (IS_ERR(right)) 1198fb770ae4SLiu Bo right = NULL; 1199e66f709bSChris Mason 1200e66f709bSChris Mason /* 1201e66f709bSChris Mason * then try to empty the right most buffer into the middle 1202e66f709bSChris Mason */ 12035f39d397SChris Mason if (right) { 120433ade1f8SChris Mason u32 right_nr; 1205b4ce94deSChris Mason 1206bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT); 1207b4ce94deSChris Mason 12085f39d397SChris Mason right_nr = btrfs_header_nritems(right); 12090b246afaSJeff Mahoney if (right_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) { 121033ade1f8SChris Mason wret = 1; 121133ade1f8SChris Mason } else { 12125f39d397SChris Mason ret = btrfs_cow_block(trans, root, right, 12135f39d397SChris Mason parent, pslot + 1, 1214bf59a5a2SJosef Bacik &right, BTRFS_NESTING_RIGHT_COW); 121554aa1f4dSChris Mason if (ret) 121654aa1f4dSChris Mason wret = 1; 121754aa1f4dSChris Mason else { 121855d32ed8SDavid Sterba wret = balance_node_right(trans, right, mid); 121933ade1f8SChris Mason } 122054aa1f4dSChris Mason } 1221e66f709bSChris Mason if (wret < 0) 1222e66f709bSChris Mason ret = wret; 1223e66f709bSChris Mason if (wret == 0) { 12245f39d397SChris Mason struct btrfs_disk_key disk_key; 12255f39d397SChris Mason 12265f39d397SChris Mason btrfs_node_key(right, &disk_key, 0); 1227f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot + 1, 122833cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 12290e82bcfeSDavid Sterba BUG_ON(ret < 0); 12305f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot + 1); 12315f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 12325f39d397SChris Mason 12335f39d397SChris Mason if (btrfs_header_nritems(mid) <= orig_slot) { 12345f39d397SChris Mason path->nodes[level] = right; 1235e66f709bSChris Mason path->slots[level + 1] += 1; 1236e66f709bSChris Mason path->slots[level] = orig_slot - 12375f39d397SChris Mason btrfs_header_nritems(mid); 1238925baeddSChris Mason btrfs_tree_unlock(mid); 12395f39d397SChris Mason free_extent_buffer(mid); 1240e66f709bSChris Mason } else { 1241925baeddSChris Mason btrfs_tree_unlock(right); 12425f39d397SChris Mason free_extent_buffer(right); 1243e66f709bSChris Mason } 1244e66f709bSChris Mason return 0; 1245e66f709bSChris Mason } 1246925baeddSChris Mason btrfs_tree_unlock(right); 12475f39d397SChris Mason free_extent_buffer(right); 1248e66f709bSChris Mason } 1249e66f709bSChris Mason return 1; 1250e66f709bSChris Mason } 1251e66f709bSChris Mason 125274123bd7SChris Mason /* 1253d352ac68SChris Mason * readahead one full node of leaves, finding things that are close 1254d352ac68SChris Mason * to the block in 'slot', and triggering ra on them. 12553c69faecSChris Mason */ 12562ff7e61eSJeff Mahoney static void reada_for_search(struct btrfs_fs_info *fs_info, 1257e02119d5SChris Mason struct btrfs_path *path, 125801f46658SChris Mason int level, int slot, u64 objectid) 12593c69faecSChris Mason { 12605f39d397SChris Mason struct extent_buffer *node; 126101f46658SChris Mason struct btrfs_disk_key disk_key; 12623c69faecSChris Mason u32 nritems; 12633c69faecSChris Mason u64 search; 1264a7175319SChris Mason u64 target; 12656b80053dSChris Mason u64 nread = 0; 1266ace75066SFilipe Manana u64 nread_max; 12676b80053dSChris Mason u32 nr; 12686b80053dSChris Mason u32 blocksize; 12696b80053dSChris Mason u32 nscan = 0; 1270db94535dSChris Mason 1271ace75066SFilipe Manana if (level != 1 && path->reada != READA_FORWARD_ALWAYS) 12723c69faecSChris Mason return; 12733c69faecSChris Mason 12746702ed49SChris Mason if (!path->nodes[level]) 12756702ed49SChris Mason return; 12766702ed49SChris Mason 12775f39d397SChris Mason node = path->nodes[level]; 1278925baeddSChris Mason 1279ace75066SFilipe Manana /* 1280ace75066SFilipe Manana * Since the time between visiting leaves is much shorter than the time 1281ace75066SFilipe Manana * between visiting nodes, limit read ahead of nodes to 1, to avoid too 1282ace75066SFilipe Manana * much IO at once (possibly random). 1283ace75066SFilipe Manana */ 1284ace75066SFilipe Manana if (path->reada == READA_FORWARD_ALWAYS) { 1285ace75066SFilipe Manana if (level > 1) 1286ace75066SFilipe Manana nread_max = node->fs_info->nodesize; 1287ace75066SFilipe Manana else 1288ace75066SFilipe Manana nread_max = SZ_128K; 1289ace75066SFilipe Manana } else { 1290ace75066SFilipe Manana nread_max = SZ_64K; 1291ace75066SFilipe Manana } 1292ace75066SFilipe Manana 12933c69faecSChris Mason search = btrfs_node_blockptr(node, slot); 12940b246afaSJeff Mahoney blocksize = fs_info->nodesize; 1295069a2e37SFilipe Manana if (path->reada != READA_FORWARD_ALWAYS) { 1296069a2e37SFilipe Manana struct extent_buffer *eb; 1297069a2e37SFilipe Manana 12980b246afaSJeff Mahoney eb = find_extent_buffer(fs_info, search); 12995f39d397SChris Mason if (eb) { 13005f39d397SChris Mason free_extent_buffer(eb); 13013c69faecSChris Mason return; 13023c69faecSChris Mason } 1303069a2e37SFilipe Manana } 13043c69faecSChris Mason 1305a7175319SChris Mason target = search; 13066b80053dSChris Mason 13075f39d397SChris Mason nritems = btrfs_header_nritems(node); 13086b80053dSChris Mason nr = slot; 130925b8b936SJosef Bacik 13103c69faecSChris Mason while (1) { 1311e4058b54SDavid Sterba if (path->reada == READA_BACK) { 13126b80053dSChris Mason if (nr == 0) 13133c69faecSChris Mason break; 13146b80053dSChris Mason nr--; 1315ace75066SFilipe Manana } else if (path->reada == READA_FORWARD || 1316ace75066SFilipe Manana path->reada == READA_FORWARD_ALWAYS) { 13176b80053dSChris Mason nr++; 13186b80053dSChris Mason if (nr >= nritems) 13196b80053dSChris Mason break; 13203c69faecSChris Mason } 1321e4058b54SDavid Sterba if (path->reada == READA_BACK && objectid) { 132201f46658SChris Mason btrfs_node_key(node, &disk_key, nr); 132301f46658SChris Mason if (btrfs_disk_key_objectid(&disk_key) != objectid) 132401f46658SChris Mason break; 132501f46658SChris Mason } 13266b80053dSChris Mason search = btrfs_node_blockptr(node, nr); 1327ace75066SFilipe Manana if (path->reada == READA_FORWARD_ALWAYS || 1328ace75066SFilipe Manana (search <= target && target - search <= 65536) || 1329a7175319SChris Mason (search > target && search - target <= 65536)) { 1330bfb484d9SJosef Bacik btrfs_readahead_node_child(node, nr); 13316b80053dSChris Mason nread += blocksize; 13323c69faecSChris Mason } 13336b80053dSChris Mason nscan++; 1334ace75066SFilipe Manana if (nread > nread_max || nscan > 32) 13356b80053dSChris Mason break; 13363c69faecSChris Mason } 13373c69faecSChris Mason } 1338925baeddSChris Mason 1339bfb484d9SJosef Bacik static noinline void reada_for_balance(struct btrfs_path *path, int level) 1340b4ce94deSChris Mason { 1341bfb484d9SJosef Bacik struct extent_buffer *parent; 1342b4ce94deSChris Mason int slot; 1343b4ce94deSChris Mason int nritems; 1344b4ce94deSChris Mason 13458c594ea8SChris Mason parent = path->nodes[level + 1]; 1346b4ce94deSChris Mason if (!parent) 13470b08851fSJosef Bacik return; 1348b4ce94deSChris Mason 1349b4ce94deSChris Mason nritems = btrfs_header_nritems(parent); 13508c594ea8SChris Mason slot = path->slots[level + 1]; 1351b4ce94deSChris Mason 1352bfb484d9SJosef Bacik if (slot > 0) 1353bfb484d9SJosef Bacik btrfs_readahead_node_child(parent, slot - 1); 1354bfb484d9SJosef Bacik if (slot + 1 < nritems) 1355bfb484d9SJosef Bacik btrfs_readahead_node_child(parent, slot + 1); 1356b4ce94deSChris Mason } 1357b4ce94deSChris Mason 1358b4ce94deSChris Mason 1359b4ce94deSChris Mason /* 1360d397712bSChris Mason * when we walk down the tree, it is usually safe to unlock the higher layers 1361d397712bSChris Mason * in the tree. The exceptions are when our path goes through slot 0, because 1362d397712bSChris Mason * operations on the tree might require changing key pointers higher up in the 1363d397712bSChris Mason * tree. 1364d352ac68SChris Mason * 1365d397712bSChris Mason * callers might also have set path->keep_locks, which tells this code to keep 1366d397712bSChris Mason * the lock if the path points to the last slot in the block. This is part of 1367d397712bSChris Mason * walking through the tree, and selecting the next slot in the higher block. 1368d352ac68SChris Mason * 1369d397712bSChris Mason * lowest_unlock sets the lowest level in the tree we're allowed to unlock. so 1370d397712bSChris Mason * if lowest_unlock is 1, level 0 won't be unlocked 1371d352ac68SChris Mason */ 1372e02119d5SChris Mason static noinline void unlock_up(struct btrfs_path *path, int level, 1373f7c79f30SChris Mason int lowest_unlock, int min_write_lock_level, 1374f7c79f30SChris Mason int *write_lock_level) 1375925baeddSChris Mason { 1376925baeddSChris Mason int i; 1377925baeddSChris Mason int skip_level = level; 1378c1227996SNikolay Borisov bool check_skip = true; 1379925baeddSChris Mason 1380925baeddSChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 1381925baeddSChris Mason if (!path->nodes[i]) 1382925baeddSChris Mason break; 1383925baeddSChris Mason if (!path->locks[i]) 1384925baeddSChris Mason break; 1385c1227996SNikolay Borisov 1386c1227996SNikolay Borisov if (check_skip) { 1387c1227996SNikolay Borisov if (path->slots[i] == 0) { 1388925baeddSChris Mason skip_level = i + 1; 1389925baeddSChris Mason continue; 1390925baeddSChris Mason } 1391c1227996SNikolay Borisov 1392c1227996SNikolay Borisov if (path->keep_locks) { 1393925baeddSChris Mason u32 nritems; 1394c1227996SNikolay Borisov 1395c1227996SNikolay Borisov nritems = btrfs_header_nritems(path->nodes[i]); 1396051e1b9fSChris Mason if (nritems < 1 || path->slots[i] >= nritems - 1) { 1397925baeddSChris Mason skip_level = i + 1; 1398925baeddSChris Mason continue; 1399925baeddSChris Mason } 1400925baeddSChris Mason } 1401c1227996SNikolay Borisov } 1402051e1b9fSChris Mason 1403d80bb3f9SLiu Bo if (i >= lowest_unlock && i > skip_level) { 1404c1227996SNikolay Borisov check_skip = false; 1405c1227996SNikolay Borisov btrfs_tree_unlock_rw(path->nodes[i], path->locks[i]); 1406925baeddSChris Mason path->locks[i] = 0; 1407f7c79f30SChris Mason if (write_lock_level && 1408f7c79f30SChris Mason i > min_write_lock_level && 1409f7c79f30SChris Mason i <= *write_lock_level) { 1410f7c79f30SChris Mason *write_lock_level = i - 1; 1411f7c79f30SChris Mason } 1412925baeddSChris Mason } 1413925baeddSChris Mason } 1414925baeddSChris Mason } 1415925baeddSChris Mason 14163c69faecSChris Mason /* 1417376a21d7SFilipe Manana * Helper function for btrfs_search_slot() and other functions that do a search 1418376a21d7SFilipe Manana * on a btree. The goal is to find a tree block in the cache (the radix tree at 1419376a21d7SFilipe Manana * fs_info->buffer_radix), but if we can't find it, or it's not up to date, read 1420376a21d7SFilipe Manana * its pages from disk. 1421c8c42864SChris Mason * 1422376a21d7SFilipe Manana * Returns -EAGAIN, with the path unlocked, if the caller needs to repeat the 1423376a21d7SFilipe Manana * whole btree search, starting again from the current root node. 1424c8c42864SChris Mason */ 1425c8c42864SChris Mason static int 1426d07b8528SLiu Bo read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, 1427c8c42864SChris Mason struct extent_buffer **eb_ret, int level, int slot, 1428cda79c54SDavid Sterba const struct btrfs_key *key) 1429c8c42864SChris Mason { 14300b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 1431c8c42864SChris Mason u64 blocknr; 1432c8c42864SChris Mason u64 gen; 1433c8c42864SChris Mason struct extent_buffer *tmp; 1434581c1760SQu Wenruo struct btrfs_key first_key; 143576a05b35SChris Mason int ret; 1436581c1760SQu Wenruo int parent_level; 1437b246666eSFilipe Manana bool unlock_up; 1438c8c42864SChris Mason 1439b246666eSFilipe Manana unlock_up = ((level + 1 < BTRFS_MAX_LEVEL) && p->locks[level + 1]); 1440213ff4b7SNikolay Borisov blocknr = btrfs_node_blockptr(*eb_ret, slot); 1441213ff4b7SNikolay Borisov gen = btrfs_node_ptr_generation(*eb_ret, slot); 1442213ff4b7SNikolay Borisov parent_level = btrfs_header_level(*eb_ret); 1443213ff4b7SNikolay Borisov btrfs_node_key_to_cpu(*eb_ret, &first_key, slot); 1444c8c42864SChris Mason 1445b246666eSFilipe Manana /* 1446b246666eSFilipe Manana * If we need to read an extent buffer from disk and we are holding locks 1447b246666eSFilipe Manana * on upper level nodes, we unlock all the upper nodes before reading the 1448b246666eSFilipe Manana * extent buffer, and then return -EAGAIN to the caller as it needs to 1449b246666eSFilipe Manana * restart the search. We don't release the lock on the current level 1450b246666eSFilipe Manana * because we need to walk this node to figure out which blocks to read. 1451b246666eSFilipe Manana */ 14520b246afaSJeff Mahoney tmp = find_extent_buffer(fs_info, blocknr); 1453cb44921aSChris Mason if (tmp) { 1454ace75066SFilipe Manana if (p->reada == READA_FORWARD_ALWAYS) 1455ace75066SFilipe Manana reada_for_search(fs_info, p, level, slot, key->objectid); 1456ace75066SFilipe Manana 1457b9fab919SChris Mason /* first we do an atomic uptodate check */ 1458b9fab919SChris Mason if (btrfs_buffer_uptodate(tmp, gen, 1) > 0) { 1459448de471SQu Wenruo /* 1460448de471SQu Wenruo * Do extra check for first_key, eb can be stale due to 1461448de471SQu Wenruo * being cached, read from scrub, or have multiple 1462448de471SQu Wenruo * parents (shared tree blocks). 1463448de471SQu Wenruo */ 1464e064d5e9SDavid Sterba if (btrfs_verify_level_key(tmp, 1465448de471SQu Wenruo parent_level - 1, &first_key, gen)) { 1466448de471SQu Wenruo free_extent_buffer(tmp); 1467448de471SQu Wenruo return -EUCLEAN; 1468448de471SQu Wenruo } 1469c8c42864SChris Mason *eb_ret = tmp; 1470c8c42864SChris Mason return 0; 1471c8c42864SChris Mason } 1472bdf7c00eSJosef Bacik 1473857bc13fSJosef Bacik if (p->nowait) { 1474857bc13fSJosef Bacik free_extent_buffer(tmp); 1475857bc13fSJosef Bacik return -EAGAIN; 1476857bc13fSJosef Bacik } 1477857bc13fSJosef Bacik 1478b246666eSFilipe Manana if (unlock_up) 1479b246666eSFilipe Manana btrfs_unlock_up_safe(p, level + 1); 1480b246666eSFilipe Manana 1481b9fab919SChris Mason /* now we're allowed to do a blocking uptodate check */ 14826a2e9dc4SFilipe Manana ret = btrfs_read_extent_buffer(tmp, gen, parent_level - 1, &first_key); 14839a4ffa1bSQu Wenruo if (ret) { 1484cb44921aSChris Mason free_extent_buffer(tmp); 1485b3b4aa74SDavid Sterba btrfs_release_path(p); 1486cb44921aSChris Mason return -EIO; 1487cb44921aSChris Mason } 148888c602abSQu Wenruo if (btrfs_check_eb_owner(tmp, root->root_key.objectid)) { 148988c602abSQu Wenruo free_extent_buffer(tmp); 149088c602abSQu Wenruo btrfs_release_path(p); 149188c602abSQu Wenruo return -EUCLEAN; 149288c602abSQu Wenruo } 1493b246666eSFilipe Manana 1494b246666eSFilipe Manana if (unlock_up) 1495b246666eSFilipe Manana ret = -EAGAIN; 1496b246666eSFilipe Manana 1497b246666eSFilipe Manana goto out; 1498857bc13fSJosef Bacik } else if (p->nowait) { 1499857bc13fSJosef Bacik return -EAGAIN; 15009a4ffa1bSQu Wenruo } 1501c8c42864SChris Mason 1502b246666eSFilipe Manana if (unlock_up) { 15038c594ea8SChris Mason btrfs_unlock_up_safe(p, level + 1); 15044bb59055SFilipe Manana ret = -EAGAIN; 15054bb59055SFilipe Manana } else { 15064bb59055SFilipe Manana ret = 0; 15074bb59055SFilipe Manana } 15088c594ea8SChris Mason 1509e4058b54SDavid Sterba if (p->reada != READA_NONE) 15102ff7e61eSJeff Mahoney reada_for_search(fs_info, p, level, slot, key->objectid); 1511c8c42864SChris Mason 15121b7ec85eSJosef Bacik tmp = read_tree_block(fs_info, blocknr, root->root_key.objectid, 15131b7ec85eSJosef Bacik gen, parent_level - 1, &first_key); 15144eb150d6SQu Wenruo if (IS_ERR(tmp)) { 15154eb150d6SQu Wenruo btrfs_release_path(p); 15164eb150d6SQu Wenruo return PTR_ERR(tmp); 15174eb150d6SQu Wenruo } 151876a05b35SChris Mason /* 151976a05b35SChris Mason * If the read above didn't mark this buffer up to date, 152076a05b35SChris Mason * it will never end up being up to date. Set ret to EIO now 152176a05b35SChris Mason * and give up so that our caller doesn't loop forever 152276a05b35SChris Mason * on our EAGAINs. 152376a05b35SChris Mason */ 1524e6a1d6fdSLiu Bo if (!extent_buffer_uptodate(tmp)) 152576a05b35SChris Mason ret = -EIO; 152602a3307aSLiu Bo 1527b246666eSFilipe Manana out: 15284bb59055SFilipe Manana if (ret == 0) { 15294bb59055SFilipe Manana *eb_ret = tmp; 15304bb59055SFilipe Manana } else { 15314bb59055SFilipe Manana free_extent_buffer(tmp); 153202a3307aSLiu Bo btrfs_release_path(p); 15334bb59055SFilipe Manana } 15344bb59055SFilipe Manana 153576a05b35SChris Mason return ret; 1536c8c42864SChris Mason } 1537c8c42864SChris Mason 1538c8c42864SChris Mason /* 1539c8c42864SChris Mason * helper function for btrfs_search_slot. This does all of the checks 1540c8c42864SChris Mason * for node-level blocks and does any balancing required based on 1541c8c42864SChris Mason * the ins_len. 1542c8c42864SChris Mason * 1543c8c42864SChris Mason * If no extra work was required, zero is returned. If we had to 1544c8c42864SChris Mason * drop the path, -EAGAIN is returned and btrfs_search_slot must 1545c8c42864SChris Mason * start over 1546c8c42864SChris Mason */ 1547c8c42864SChris Mason static int 1548c8c42864SChris Mason setup_nodes_for_search(struct btrfs_trans_handle *trans, 1549c8c42864SChris Mason struct btrfs_root *root, struct btrfs_path *p, 1550bd681513SChris Mason struct extent_buffer *b, int level, int ins_len, 1551bd681513SChris Mason int *write_lock_level) 1552c8c42864SChris Mason { 15530b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 155495b982deSNikolay Borisov int ret = 0; 15550b246afaSJeff Mahoney 1556c8c42864SChris Mason if ((p->search_for_split || ins_len > 0) && btrfs_header_nritems(b) >= 15570b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3) { 1558c8c42864SChris Mason 1559bd681513SChris Mason if (*write_lock_level < level + 1) { 1560bd681513SChris Mason *write_lock_level = level + 1; 1561bd681513SChris Mason btrfs_release_path(p); 156295b982deSNikolay Borisov return -EAGAIN; 1563bd681513SChris Mason } 1564bd681513SChris Mason 1565bfb484d9SJosef Bacik reada_for_balance(p, level); 156695b982deSNikolay Borisov ret = split_node(trans, root, p, level); 1567c8c42864SChris Mason 1568c8c42864SChris Mason b = p->nodes[level]; 1569c8c42864SChris Mason } else if (ins_len < 0 && btrfs_header_nritems(b) < 15700b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 2) { 1571c8c42864SChris Mason 1572bd681513SChris Mason if (*write_lock_level < level + 1) { 1573bd681513SChris Mason *write_lock_level = level + 1; 1574bd681513SChris Mason btrfs_release_path(p); 157595b982deSNikolay Borisov return -EAGAIN; 1576bd681513SChris Mason } 1577bd681513SChris Mason 1578bfb484d9SJosef Bacik reada_for_balance(p, level); 157995b982deSNikolay Borisov ret = balance_level(trans, root, p, level); 158095b982deSNikolay Borisov if (ret) 158195b982deSNikolay Borisov return ret; 1582c8c42864SChris Mason 1583c8c42864SChris Mason b = p->nodes[level]; 1584c8c42864SChris Mason if (!b) { 1585b3b4aa74SDavid Sterba btrfs_release_path(p); 158695b982deSNikolay Borisov return -EAGAIN; 1587c8c42864SChris Mason } 1588c8c42864SChris Mason BUG_ON(btrfs_header_nritems(b) == 1); 1589c8c42864SChris Mason } 1590c8c42864SChris Mason return ret; 1591c8c42864SChris Mason } 1592c8c42864SChris Mason 1593381cf658SDavid Sterba int btrfs_find_item(struct btrfs_root *fs_root, struct btrfs_path *path, 1594e33d5c3dSKelley Nielsen u64 iobjectid, u64 ioff, u8 key_type, 1595e33d5c3dSKelley Nielsen struct btrfs_key *found_key) 1596e33d5c3dSKelley Nielsen { 1597e33d5c3dSKelley Nielsen int ret; 1598e33d5c3dSKelley Nielsen struct btrfs_key key; 1599e33d5c3dSKelley Nielsen struct extent_buffer *eb; 1600381cf658SDavid Sterba 1601381cf658SDavid Sterba ASSERT(path); 16021d4c08e0SDavid Sterba ASSERT(found_key); 1603e33d5c3dSKelley Nielsen 1604e33d5c3dSKelley Nielsen key.type = key_type; 1605e33d5c3dSKelley Nielsen key.objectid = iobjectid; 1606e33d5c3dSKelley Nielsen key.offset = ioff; 1607e33d5c3dSKelley Nielsen 1608e33d5c3dSKelley Nielsen ret = btrfs_search_slot(NULL, fs_root, &key, path, 0, 0); 16091d4c08e0SDavid Sterba if (ret < 0) 1610e33d5c3dSKelley Nielsen return ret; 1611e33d5c3dSKelley Nielsen 1612e33d5c3dSKelley Nielsen eb = path->nodes[0]; 1613e33d5c3dSKelley Nielsen if (ret && path->slots[0] >= btrfs_header_nritems(eb)) { 1614e33d5c3dSKelley Nielsen ret = btrfs_next_leaf(fs_root, path); 1615e33d5c3dSKelley Nielsen if (ret) 1616e33d5c3dSKelley Nielsen return ret; 1617e33d5c3dSKelley Nielsen eb = path->nodes[0]; 1618e33d5c3dSKelley Nielsen } 1619e33d5c3dSKelley Nielsen 1620e33d5c3dSKelley Nielsen btrfs_item_key_to_cpu(eb, found_key, path->slots[0]); 1621e33d5c3dSKelley Nielsen if (found_key->type != key.type || 1622e33d5c3dSKelley Nielsen found_key->objectid != key.objectid) 1623e33d5c3dSKelley Nielsen return 1; 1624e33d5c3dSKelley Nielsen 1625e33d5c3dSKelley Nielsen return 0; 1626e33d5c3dSKelley Nielsen } 1627e33d5c3dSKelley Nielsen 16281fc28d8eSLiu Bo static struct extent_buffer *btrfs_search_slot_get_root(struct btrfs_root *root, 16291fc28d8eSLiu Bo struct btrfs_path *p, 16301fc28d8eSLiu Bo int write_lock_level) 16311fc28d8eSLiu Bo { 16321fc28d8eSLiu Bo struct extent_buffer *b; 1633120de408SJosef Bacik int root_lock = 0; 16341fc28d8eSLiu Bo int level = 0; 16351fc28d8eSLiu Bo 16361fc28d8eSLiu Bo if (p->search_commit_root) { 16371fc28d8eSLiu Bo b = root->commit_root; 163867439dadSDavid Sterba atomic_inc(&b->refs); 16391fc28d8eSLiu Bo level = btrfs_header_level(b); 1640f9ddfd05SLiu Bo /* 1641f9ddfd05SLiu Bo * Ensure that all callers have set skip_locking when 1642f9ddfd05SLiu Bo * p->search_commit_root = 1. 1643f9ddfd05SLiu Bo */ 1644f9ddfd05SLiu Bo ASSERT(p->skip_locking == 1); 16451fc28d8eSLiu Bo 16461fc28d8eSLiu Bo goto out; 16471fc28d8eSLiu Bo } 16481fc28d8eSLiu Bo 16491fc28d8eSLiu Bo if (p->skip_locking) { 16501fc28d8eSLiu Bo b = btrfs_root_node(root); 16511fc28d8eSLiu Bo level = btrfs_header_level(b); 16521fc28d8eSLiu Bo goto out; 16531fc28d8eSLiu Bo } 16541fc28d8eSLiu Bo 1655120de408SJosef Bacik /* We try very hard to do read locks on the root */ 1656120de408SJosef Bacik root_lock = BTRFS_READ_LOCK; 1657120de408SJosef Bacik 16581fc28d8eSLiu Bo /* 1659662c653bSLiu Bo * If the level is set to maximum, we can skip trying to get the read 1660662c653bSLiu Bo * lock. 1661662c653bSLiu Bo */ 1662662c653bSLiu Bo if (write_lock_level < BTRFS_MAX_LEVEL) { 1663662c653bSLiu Bo /* 1664662c653bSLiu Bo * We don't know the level of the root node until we actually 1665662c653bSLiu Bo * have it read locked 16661fc28d8eSLiu Bo */ 1667857bc13fSJosef Bacik if (p->nowait) { 1668857bc13fSJosef Bacik b = btrfs_try_read_lock_root_node(root); 1669857bc13fSJosef Bacik if (IS_ERR(b)) 1670857bc13fSJosef Bacik return b; 1671857bc13fSJosef Bacik } else { 16721bb96598SJosef Bacik b = btrfs_read_lock_root_node(root); 1673857bc13fSJosef Bacik } 16741fc28d8eSLiu Bo level = btrfs_header_level(b); 16751fc28d8eSLiu Bo if (level > write_lock_level) 16761fc28d8eSLiu Bo goto out; 16771fc28d8eSLiu Bo 1678662c653bSLiu Bo /* Whoops, must trade for write lock */ 16791fc28d8eSLiu Bo btrfs_tree_read_unlock(b); 16801fc28d8eSLiu Bo free_extent_buffer(b); 1681662c653bSLiu Bo } 1682662c653bSLiu Bo 16831fc28d8eSLiu Bo b = btrfs_lock_root_node(root); 16841fc28d8eSLiu Bo root_lock = BTRFS_WRITE_LOCK; 16851fc28d8eSLiu Bo 16861fc28d8eSLiu Bo /* The level might have changed, check again */ 16871fc28d8eSLiu Bo level = btrfs_header_level(b); 16881fc28d8eSLiu Bo 16891fc28d8eSLiu Bo out: 1690120de408SJosef Bacik /* 1691120de408SJosef Bacik * The root may have failed to write out at some point, and thus is no 1692120de408SJosef Bacik * longer valid, return an error in this case. 1693120de408SJosef Bacik */ 1694120de408SJosef Bacik if (!extent_buffer_uptodate(b)) { 1695120de408SJosef Bacik if (root_lock) 1696120de408SJosef Bacik btrfs_tree_unlock_rw(b, root_lock); 1697120de408SJosef Bacik free_extent_buffer(b); 1698120de408SJosef Bacik return ERR_PTR(-EIO); 1699120de408SJosef Bacik } 1700120de408SJosef Bacik 17011fc28d8eSLiu Bo p->nodes[level] = b; 17021fc28d8eSLiu Bo if (!p->skip_locking) 17031fc28d8eSLiu Bo p->locks[level] = root_lock; 17041fc28d8eSLiu Bo /* 17051fc28d8eSLiu Bo * Callers are responsible for dropping b's references. 17061fc28d8eSLiu Bo */ 17071fc28d8eSLiu Bo return b; 17081fc28d8eSLiu Bo } 17091fc28d8eSLiu Bo 1710d96b3424SFilipe Manana /* 1711d96b3424SFilipe Manana * Replace the extent buffer at the lowest level of the path with a cloned 1712d96b3424SFilipe Manana * version. The purpose is to be able to use it safely, after releasing the 1713d96b3424SFilipe Manana * commit root semaphore, even if relocation is happening in parallel, the 1714d96b3424SFilipe Manana * transaction used for relocation is committed and the extent buffer is 1715d96b3424SFilipe Manana * reallocated in the next transaction. 1716d96b3424SFilipe Manana * 1717d96b3424SFilipe Manana * This is used in a context where the caller does not prevent transaction 1718d96b3424SFilipe Manana * commits from happening, either by holding a transaction handle or holding 1719d96b3424SFilipe Manana * some lock, while it's doing searches through a commit root. 1720d96b3424SFilipe Manana * At the moment it's only used for send operations. 1721d96b3424SFilipe Manana */ 1722d96b3424SFilipe Manana static int finish_need_commit_sem_search(struct btrfs_path *path) 1723d96b3424SFilipe Manana { 1724d96b3424SFilipe Manana const int i = path->lowest_level; 1725d96b3424SFilipe Manana const int slot = path->slots[i]; 1726d96b3424SFilipe Manana struct extent_buffer *lowest = path->nodes[i]; 1727d96b3424SFilipe Manana struct extent_buffer *clone; 1728d96b3424SFilipe Manana 1729d96b3424SFilipe Manana ASSERT(path->need_commit_sem); 1730d96b3424SFilipe Manana 1731d96b3424SFilipe Manana if (!lowest) 1732d96b3424SFilipe Manana return 0; 1733d96b3424SFilipe Manana 1734d96b3424SFilipe Manana lockdep_assert_held_read(&lowest->fs_info->commit_root_sem); 1735d96b3424SFilipe Manana 1736d96b3424SFilipe Manana clone = btrfs_clone_extent_buffer(lowest); 1737d96b3424SFilipe Manana if (!clone) 1738d96b3424SFilipe Manana return -ENOMEM; 1739d96b3424SFilipe Manana 1740d96b3424SFilipe Manana btrfs_release_path(path); 1741d96b3424SFilipe Manana path->nodes[i] = clone; 1742d96b3424SFilipe Manana path->slots[i] = slot; 1743d96b3424SFilipe Manana 1744d96b3424SFilipe Manana return 0; 1745d96b3424SFilipe Manana } 17461fc28d8eSLiu Bo 1747e2e58d0fSFilipe Manana static inline int search_for_key_slot(struct extent_buffer *eb, 1748e2e58d0fSFilipe Manana int search_low_slot, 1749e2e58d0fSFilipe Manana const struct btrfs_key *key, 1750e2e58d0fSFilipe Manana int prev_cmp, 1751e2e58d0fSFilipe Manana int *slot) 1752e2e58d0fSFilipe Manana { 1753e2e58d0fSFilipe Manana /* 1754e2e58d0fSFilipe Manana * If a previous call to btrfs_bin_search() on a parent node returned an 1755e2e58d0fSFilipe Manana * exact match (prev_cmp == 0), we can safely assume the target key will 1756e2e58d0fSFilipe Manana * always be at slot 0 on lower levels, since each key pointer 1757e2e58d0fSFilipe Manana * (struct btrfs_key_ptr) refers to the lowest key accessible from the 1758e2e58d0fSFilipe Manana * subtree it points to. Thus we can skip searching lower levels. 1759e2e58d0fSFilipe Manana */ 1760e2e58d0fSFilipe Manana if (prev_cmp == 0) { 1761e2e58d0fSFilipe Manana *slot = 0; 1762e2e58d0fSFilipe Manana return 0; 1763e2e58d0fSFilipe Manana } 1764e2e58d0fSFilipe Manana 1765e2e58d0fSFilipe Manana return generic_bin_search(eb, search_low_slot, key, slot); 1766e2e58d0fSFilipe Manana } 1767e2e58d0fSFilipe Manana 1768109324cfSFilipe Manana static int search_leaf(struct btrfs_trans_handle *trans, 1769109324cfSFilipe Manana struct btrfs_root *root, 1770109324cfSFilipe Manana const struct btrfs_key *key, 1771109324cfSFilipe Manana struct btrfs_path *path, 1772109324cfSFilipe Manana int ins_len, 1773109324cfSFilipe Manana int prev_cmp) 1774109324cfSFilipe Manana { 1775109324cfSFilipe Manana struct extent_buffer *leaf = path->nodes[0]; 1776109324cfSFilipe Manana int leaf_free_space = -1; 1777109324cfSFilipe Manana int search_low_slot = 0; 1778109324cfSFilipe Manana int ret; 1779109324cfSFilipe Manana bool do_bin_search = true; 1780109324cfSFilipe Manana 1781109324cfSFilipe Manana /* 1782109324cfSFilipe Manana * If we are doing an insertion, the leaf has enough free space and the 1783109324cfSFilipe Manana * destination slot for the key is not slot 0, then we can unlock our 1784109324cfSFilipe Manana * write lock on the parent, and any other upper nodes, before doing the 1785109324cfSFilipe Manana * binary search on the leaf (with search_for_key_slot()), allowing other 1786109324cfSFilipe Manana * tasks to lock the parent and any other upper nodes. 1787109324cfSFilipe Manana */ 1788109324cfSFilipe Manana if (ins_len > 0) { 1789109324cfSFilipe Manana /* 1790109324cfSFilipe Manana * Cache the leaf free space, since we will need it later and it 1791109324cfSFilipe Manana * will not change until then. 1792109324cfSFilipe Manana */ 1793109324cfSFilipe Manana leaf_free_space = btrfs_leaf_free_space(leaf); 1794109324cfSFilipe Manana 1795109324cfSFilipe Manana /* 1796109324cfSFilipe Manana * !path->locks[1] means we have a single node tree, the leaf is 1797109324cfSFilipe Manana * the root of the tree. 1798109324cfSFilipe Manana */ 1799109324cfSFilipe Manana if (path->locks[1] && leaf_free_space >= ins_len) { 1800109324cfSFilipe Manana struct btrfs_disk_key first_key; 1801109324cfSFilipe Manana 1802109324cfSFilipe Manana ASSERT(btrfs_header_nritems(leaf) > 0); 1803109324cfSFilipe Manana btrfs_item_key(leaf, &first_key, 0); 1804109324cfSFilipe Manana 1805109324cfSFilipe Manana /* 1806109324cfSFilipe Manana * Doing the extra comparison with the first key is cheap, 1807109324cfSFilipe Manana * taking into account that the first key is very likely 1808109324cfSFilipe Manana * already in a cache line because it immediately follows 1809109324cfSFilipe Manana * the extent buffer's header and we have recently accessed 1810109324cfSFilipe Manana * the header's level field. 1811109324cfSFilipe Manana */ 1812109324cfSFilipe Manana ret = comp_keys(&first_key, key); 1813109324cfSFilipe Manana if (ret < 0) { 1814109324cfSFilipe Manana /* 1815109324cfSFilipe Manana * The first key is smaller than the key we want 1816109324cfSFilipe Manana * to insert, so we are safe to unlock all upper 1817109324cfSFilipe Manana * nodes and we have to do the binary search. 1818109324cfSFilipe Manana * 1819109324cfSFilipe Manana * We do use btrfs_unlock_up_safe() and not 1820109324cfSFilipe Manana * unlock_up() because the later does not unlock 1821109324cfSFilipe Manana * nodes with a slot of 0 - we can safely unlock 1822109324cfSFilipe Manana * any node even if its slot is 0 since in this 1823109324cfSFilipe Manana * case the key does not end up at slot 0 of the 1824109324cfSFilipe Manana * leaf and there's no need to split the leaf. 1825109324cfSFilipe Manana */ 1826109324cfSFilipe Manana btrfs_unlock_up_safe(path, 1); 1827109324cfSFilipe Manana search_low_slot = 1; 1828109324cfSFilipe Manana } else { 1829109324cfSFilipe Manana /* 1830109324cfSFilipe Manana * The first key is >= then the key we want to 1831109324cfSFilipe Manana * insert, so we can skip the binary search as 1832109324cfSFilipe Manana * the target key will be at slot 0. 1833109324cfSFilipe Manana * 1834109324cfSFilipe Manana * We can not unlock upper nodes when the key is 1835109324cfSFilipe Manana * less than the first key, because we will need 1836109324cfSFilipe Manana * to update the key at slot 0 of the parent node 1837109324cfSFilipe Manana * and possibly of other upper nodes too. 1838109324cfSFilipe Manana * If the key matches the first key, then we can 1839109324cfSFilipe Manana * unlock all the upper nodes, using 1840109324cfSFilipe Manana * btrfs_unlock_up_safe() instead of unlock_up() 1841109324cfSFilipe Manana * as stated above. 1842109324cfSFilipe Manana */ 1843109324cfSFilipe Manana if (ret == 0) 1844109324cfSFilipe Manana btrfs_unlock_up_safe(path, 1); 1845109324cfSFilipe Manana /* 1846109324cfSFilipe Manana * ret is already 0 or 1, matching the result of 1847109324cfSFilipe Manana * a btrfs_bin_search() call, so there is no need 1848109324cfSFilipe Manana * to adjust it. 1849109324cfSFilipe Manana */ 1850109324cfSFilipe Manana do_bin_search = false; 1851109324cfSFilipe Manana path->slots[0] = 0; 1852109324cfSFilipe Manana } 1853109324cfSFilipe Manana } 1854109324cfSFilipe Manana } 1855109324cfSFilipe Manana 1856109324cfSFilipe Manana if (do_bin_search) { 1857109324cfSFilipe Manana ret = search_for_key_slot(leaf, search_low_slot, key, 1858109324cfSFilipe Manana prev_cmp, &path->slots[0]); 1859109324cfSFilipe Manana if (ret < 0) 1860109324cfSFilipe Manana return ret; 1861109324cfSFilipe Manana } 1862109324cfSFilipe Manana 1863109324cfSFilipe Manana if (ins_len > 0) { 1864109324cfSFilipe Manana /* 1865109324cfSFilipe Manana * Item key already exists. In this case, if we are allowed to 1866109324cfSFilipe Manana * insert the item (for example, in dir_item case, item key 1867109324cfSFilipe Manana * collision is allowed), it will be merged with the original 1868109324cfSFilipe Manana * item. Only the item size grows, no new btrfs item will be 1869109324cfSFilipe Manana * added. If search_for_extension is not set, ins_len already 1870109324cfSFilipe Manana * accounts the size btrfs_item, deduct it here so leaf space 1871109324cfSFilipe Manana * check will be correct. 1872109324cfSFilipe Manana */ 1873109324cfSFilipe Manana if (ret == 0 && !path->search_for_extension) { 1874109324cfSFilipe Manana ASSERT(ins_len >= sizeof(struct btrfs_item)); 1875109324cfSFilipe Manana ins_len -= sizeof(struct btrfs_item); 1876109324cfSFilipe Manana } 1877109324cfSFilipe Manana 1878109324cfSFilipe Manana ASSERT(leaf_free_space >= 0); 1879109324cfSFilipe Manana 1880109324cfSFilipe Manana if (leaf_free_space < ins_len) { 1881109324cfSFilipe Manana int err; 1882109324cfSFilipe Manana 1883109324cfSFilipe Manana err = split_leaf(trans, root, key, path, ins_len, 1884109324cfSFilipe Manana (ret == 0)); 1885bb8e9a60SFilipe Manana ASSERT(err <= 0); 1886bb8e9a60SFilipe Manana if (WARN_ON(err > 0)) 1887bb8e9a60SFilipe Manana err = -EUCLEAN; 1888109324cfSFilipe Manana if (err) 1889109324cfSFilipe Manana ret = err; 1890109324cfSFilipe Manana } 1891109324cfSFilipe Manana } 1892109324cfSFilipe Manana 1893109324cfSFilipe Manana return ret; 1894109324cfSFilipe Manana } 1895109324cfSFilipe Manana 1896c8c42864SChris Mason /* 18974271eceaSNikolay Borisov * btrfs_search_slot - look for a key in a tree and perform necessary 18984271eceaSNikolay Borisov * modifications to preserve tree invariants. 189974123bd7SChris Mason * 19004271eceaSNikolay Borisov * @trans: Handle of transaction, used when modifying the tree 19014271eceaSNikolay Borisov * @p: Holds all btree nodes along the search path 19024271eceaSNikolay Borisov * @root: The root node of the tree 19034271eceaSNikolay Borisov * @key: The key we are looking for 19049a664971Sethanwu * @ins_len: Indicates purpose of search: 19059a664971Sethanwu * >0 for inserts it's size of item inserted (*) 19069a664971Sethanwu * <0 for deletions 19079a664971Sethanwu * 0 for plain searches, not modifying the tree 19089a664971Sethanwu * 19099a664971Sethanwu * (*) If size of item inserted doesn't include 19109a664971Sethanwu * sizeof(struct btrfs_item), then p->search_for_extension must 19119a664971Sethanwu * be set. 19124271eceaSNikolay Borisov * @cow: boolean should CoW operations be performed. Must always be 1 19134271eceaSNikolay Borisov * when modifying the tree. 191497571fd0SChris Mason * 19154271eceaSNikolay Borisov * If @ins_len > 0, nodes and leaves will be split as we walk down the tree. 19164271eceaSNikolay Borisov * If @ins_len < 0, nodes will be merged as we walk down the tree (if possible) 19174271eceaSNikolay Borisov * 19184271eceaSNikolay Borisov * If @key is found, 0 is returned and you can find the item in the leaf level 19194271eceaSNikolay Borisov * of the path (level 0) 19204271eceaSNikolay Borisov * 19214271eceaSNikolay Borisov * If @key isn't found, 1 is returned and the leaf level of the path (level 0) 19224271eceaSNikolay Borisov * points to the slot where it should be inserted 19234271eceaSNikolay Borisov * 19244271eceaSNikolay Borisov * If an error is encountered while searching the tree a negative error number 19254271eceaSNikolay Borisov * is returned 192674123bd7SChris Mason */ 1927310712b2SOmar Sandoval int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root, 1928310712b2SOmar Sandoval const struct btrfs_key *key, struct btrfs_path *p, 1929310712b2SOmar Sandoval int ins_len, int cow) 1930be0e5c09SChris Mason { 1931d96b3424SFilipe Manana struct btrfs_fs_info *fs_info = root->fs_info; 19325f39d397SChris Mason struct extent_buffer *b; 1933be0e5c09SChris Mason int slot; 1934be0e5c09SChris Mason int ret; 193533c66f43SYan Zheng int err; 1936be0e5c09SChris Mason int level; 1937925baeddSChris Mason int lowest_unlock = 1; 1938bd681513SChris Mason /* everything at write_lock_level or lower must be write locked */ 1939bd681513SChris Mason int write_lock_level = 0; 19409f3a7427SChris Mason u8 lowest_level = 0; 1941f7c79f30SChris Mason int min_write_lock_level; 1942d7396f07SFilipe David Borba Manana int prev_cmp; 19439f3a7427SChris Mason 19446702ed49SChris Mason lowest_level = p->lowest_level; 1945323ac95bSChris Mason WARN_ON(lowest_level && ins_len > 0); 194622b0ebdaSChris Mason WARN_ON(p->nodes[0] != NULL); 1947eb653de1SFilipe David Borba Manana BUG_ON(!cow && ins_len); 194825179201SJosef Bacik 1949857bc13fSJosef Bacik /* 1950857bc13fSJosef Bacik * For now only allow nowait for read only operations. There's no 1951857bc13fSJosef Bacik * strict reason why we can't, we just only need it for reads so it's 1952857bc13fSJosef Bacik * only implemented for reads. 1953857bc13fSJosef Bacik */ 1954857bc13fSJosef Bacik ASSERT(!p->nowait || !cow); 1955857bc13fSJosef Bacik 1956bd681513SChris Mason if (ins_len < 0) { 1957925baeddSChris Mason lowest_unlock = 2; 195865b51a00SChris Mason 1959bd681513SChris Mason /* when we are removing items, we might have to go up to level 1960bd681513SChris Mason * two as we update tree pointers Make sure we keep write 1961bd681513SChris Mason * for those levels as well 1962bd681513SChris Mason */ 1963bd681513SChris Mason write_lock_level = 2; 1964bd681513SChris Mason } else if (ins_len > 0) { 1965bd681513SChris Mason /* 1966bd681513SChris Mason * for inserting items, make sure we have a write lock on 1967bd681513SChris Mason * level 1 so we can update keys 1968bd681513SChris Mason */ 1969bd681513SChris Mason write_lock_level = 1; 1970bd681513SChris Mason } 1971bd681513SChris Mason 1972bd681513SChris Mason if (!cow) 1973bd681513SChris Mason write_lock_level = -1; 1974bd681513SChris Mason 197509a2a8f9SJosef Bacik if (cow && (p->keep_locks || p->lowest_level)) 1976bd681513SChris Mason write_lock_level = BTRFS_MAX_LEVEL; 1977bd681513SChris Mason 1978f7c79f30SChris Mason min_write_lock_level = write_lock_level; 1979f7c79f30SChris Mason 1980d96b3424SFilipe Manana if (p->need_commit_sem) { 1981d96b3424SFilipe Manana ASSERT(p->search_commit_root); 1982857bc13fSJosef Bacik if (p->nowait) { 1983857bc13fSJosef Bacik if (!down_read_trylock(&fs_info->commit_root_sem)) 1984857bc13fSJosef Bacik return -EAGAIN; 1985857bc13fSJosef Bacik } else { 1986d96b3424SFilipe Manana down_read(&fs_info->commit_root_sem); 1987d96b3424SFilipe Manana } 1988857bc13fSJosef Bacik } 1989d96b3424SFilipe Manana 1990bb803951SChris Mason again: 1991d7396f07SFilipe David Borba Manana prev_cmp = -1; 19921fc28d8eSLiu Bo b = btrfs_search_slot_get_root(root, p, write_lock_level); 1993be6821f8SFilipe Manana if (IS_ERR(b)) { 1994be6821f8SFilipe Manana ret = PTR_ERR(b); 1995be6821f8SFilipe Manana goto done; 1996be6821f8SFilipe Manana } 1997925baeddSChris Mason 1998eb60ceacSChris Mason while (b) { 1999f624d976SQu Wenruo int dec = 0; 2000f624d976SQu Wenruo 20015f39d397SChris Mason level = btrfs_header_level(b); 200265b51a00SChris Mason 200302217ed2SChris Mason if (cow) { 20049ea2c7c9SNikolay Borisov bool last_level = (level == (BTRFS_MAX_LEVEL - 1)); 20059ea2c7c9SNikolay Borisov 2006c8c42864SChris Mason /* 2007c8c42864SChris Mason * if we don't really need to cow this block 2008c8c42864SChris Mason * then we don't want to set the path blocking, 2009c8c42864SChris Mason * so we test it here 2010c8c42864SChris Mason */ 20115963ffcaSJosef Bacik if (!should_cow_block(trans, root, b)) 201265b51a00SChris Mason goto cow_done; 20135d4f98a2SYan Zheng 2014bd681513SChris Mason /* 2015bd681513SChris Mason * must have write locks on this node and the 2016bd681513SChris Mason * parent 2017bd681513SChris Mason */ 20185124e00eSJosef Bacik if (level > write_lock_level || 20195124e00eSJosef Bacik (level + 1 > write_lock_level && 20205124e00eSJosef Bacik level + 1 < BTRFS_MAX_LEVEL && 20215124e00eSJosef Bacik p->nodes[level + 1])) { 2022bd681513SChris Mason write_lock_level = level + 1; 2023bd681513SChris Mason btrfs_release_path(p); 2024bd681513SChris Mason goto again; 2025bd681513SChris Mason } 2026bd681513SChris Mason 20279ea2c7c9SNikolay Borisov if (last_level) 20289ea2c7c9SNikolay Borisov err = btrfs_cow_block(trans, root, b, NULL, 0, 20299631e4ccSJosef Bacik &b, 20309631e4ccSJosef Bacik BTRFS_NESTING_COW); 20319ea2c7c9SNikolay Borisov else 203233c66f43SYan Zheng err = btrfs_cow_block(trans, root, b, 2033e20d96d6SChris Mason p->nodes[level + 1], 20349631e4ccSJosef Bacik p->slots[level + 1], &b, 20359631e4ccSJosef Bacik BTRFS_NESTING_COW); 203633c66f43SYan Zheng if (err) { 203733c66f43SYan Zheng ret = err; 203865b51a00SChris Mason goto done; 203954aa1f4dSChris Mason } 204002217ed2SChris Mason } 204165b51a00SChris Mason cow_done: 2042eb60ceacSChris Mason p->nodes[level] = b; 2043b4ce94deSChris Mason 2044b4ce94deSChris Mason /* 2045b4ce94deSChris Mason * we have a lock on b and as long as we aren't changing 2046b4ce94deSChris Mason * the tree, there is no way to for the items in b to change. 2047b4ce94deSChris Mason * It is safe to drop the lock on our parent before we 2048b4ce94deSChris Mason * go through the expensive btree search on b. 2049b4ce94deSChris Mason * 2050eb653de1SFilipe David Borba Manana * If we're inserting or deleting (ins_len != 0), then we might 2051eb653de1SFilipe David Borba Manana * be changing slot zero, which may require changing the parent. 2052eb653de1SFilipe David Borba Manana * So, we can't drop the lock until after we know which slot 2053eb653de1SFilipe David Borba Manana * we're operating on. 2054b4ce94deSChris Mason */ 2055eb653de1SFilipe David Borba Manana if (!ins_len && !p->keep_locks) { 2056eb653de1SFilipe David Borba Manana int u = level + 1; 2057eb653de1SFilipe David Borba Manana 2058eb653de1SFilipe David Borba Manana if (u < BTRFS_MAX_LEVEL && p->locks[u]) { 2059eb653de1SFilipe David Borba Manana btrfs_tree_unlock_rw(p->nodes[u], p->locks[u]); 2060eb653de1SFilipe David Borba Manana p->locks[u] = 0; 2061eb653de1SFilipe David Borba Manana } 2062eb653de1SFilipe David Borba Manana } 2063b4ce94deSChris Mason 2064e2e58d0fSFilipe Manana if (level == 0) { 2065109324cfSFilipe Manana if (ins_len > 0) 2066e5e1c174SFilipe Manana ASSERT(write_lock_level >= 1); 2067bd681513SChris Mason 2068109324cfSFilipe Manana ret = search_leaf(trans, root, key, p, ins_len, prev_cmp); 2069459931ecSChris Mason if (!p->search_for_split) 2070f7c79f30SChris Mason unlock_up(p, level, lowest_unlock, 20714b6f8e96SLiu Bo min_write_lock_level, NULL); 207265b51a00SChris Mason goto done; 207365b51a00SChris Mason } 2074e2e58d0fSFilipe Manana 2075e2e58d0fSFilipe Manana ret = search_for_key_slot(b, 0, key, prev_cmp, &slot); 2076e2e58d0fSFilipe Manana if (ret < 0) 2077e2e58d0fSFilipe Manana goto done; 2078e2e58d0fSFilipe Manana prev_cmp = ret; 2079e2e58d0fSFilipe Manana 2080f624d976SQu Wenruo if (ret && slot > 0) { 2081f624d976SQu Wenruo dec = 1; 2082f624d976SQu Wenruo slot--; 2083f624d976SQu Wenruo } 2084f624d976SQu Wenruo p->slots[level] = slot; 2085f624d976SQu Wenruo err = setup_nodes_for_search(trans, root, p, b, level, ins_len, 2086f624d976SQu Wenruo &write_lock_level); 2087f624d976SQu Wenruo if (err == -EAGAIN) 2088f624d976SQu Wenruo goto again; 2089f624d976SQu Wenruo if (err) { 2090f624d976SQu Wenruo ret = err; 2091f624d976SQu Wenruo goto done; 2092f624d976SQu Wenruo } 2093f624d976SQu Wenruo b = p->nodes[level]; 2094f624d976SQu Wenruo slot = p->slots[level]; 2095f624d976SQu Wenruo 2096f624d976SQu Wenruo /* 2097f624d976SQu Wenruo * Slot 0 is special, if we change the key we have to update 2098f624d976SQu Wenruo * the parent pointer which means we must have a write lock on 2099f624d976SQu Wenruo * the parent 2100f624d976SQu Wenruo */ 2101f624d976SQu Wenruo if (slot == 0 && ins_len && write_lock_level < level + 1) { 2102f624d976SQu Wenruo write_lock_level = level + 1; 2103f624d976SQu Wenruo btrfs_release_path(p); 2104f624d976SQu Wenruo goto again; 2105f624d976SQu Wenruo } 2106f624d976SQu Wenruo 2107f624d976SQu Wenruo unlock_up(p, level, lowest_unlock, min_write_lock_level, 2108f624d976SQu Wenruo &write_lock_level); 2109f624d976SQu Wenruo 2110f624d976SQu Wenruo if (level == lowest_level) { 2111f624d976SQu Wenruo if (dec) 2112f624d976SQu Wenruo p->slots[level]++; 2113f624d976SQu Wenruo goto done; 2114f624d976SQu Wenruo } 2115f624d976SQu Wenruo 2116f624d976SQu Wenruo err = read_block_for_search(root, p, &b, level, slot, key); 2117f624d976SQu Wenruo if (err == -EAGAIN) 2118f624d976SQu Wenruo goto again; 2119f624d976SQu Wenruo if (err) { 2120f624d976SQu Wenruo ret = err; 2121f624d976SQu Wenruo goto done; 2122f624d976SQu Wenruo } 2123f624d976SQu Wenruo 2124f624d976SQu Wenruo if (!p->skip_locking) { 2125f624d976SQu Wenruo level = btrfs_header_level(b); 2126b40130b2SJosef Bacik 2127b40130b2SJosef Bacik btrfs_maybe_reset_lockdep_class(root, b); 2128b40130b2SJosef Bacik 2129f624d976SQu Wenruo if (level <= write_lock_level) { 2130f624d976SQu Wenruo btrfs_tree_lock(b); 2131f624d976SQu Wenruo p->locks[level] = BTRFS_WRITE_LOCK; 2132f624d976SQu Wenruo } else { 2133857bc13fSJosef Bacik if (p->nowait) { 2134857bc13fSJosef Bacik if (!btrfs_try_tree_read_lock(b)) { 2135857bc13fSJosef Bacik free_extent_buffer(b); 2136857bc13fSJosef Bacik ret = -EAGAIN; 2137857bc13fSJosef Bacik goto done; 2138857bc13fSJosef Bacik } 2139857bc13fSJosef Bacik } else { 2140fe596ca3SJosef Bacik btrfs_tree_read_lock(b); 2141857bc13fSJosef Bacik } 2142f624d976SQu Wenruo p->locks[level] = BTRFS_READ_LOCK; 2143f624d976SQu Wenruo } 2144f624d976SQu Wenruo p->nodes[level] = b; 2145f624d976SQu Wenruo } 214665b51a00SChris Mason } 214765b51a00SChris Mason ret = 1; 214865b51a00SChris Mason done: 21495f5bc6b1SFilipe Manana if (ret < 0 && !p->skip_release_on_error) 2150b3b4aa74SDavid Sterba btrfs_release_path(p); 2151d96b3424SFilipe Manana 2152d96b3424SFilipe Manana if (p->need_commit_sem) { 2153d96b3424SFilipe Manana int ret2; 2154d96b3424SFilipe Manana 2155d96b3424SFilipe Manana ret2 = finish_need_commit_sem_search(p); 2156d96b3424SFilipe Manana up_read(&fs_info->commit_root_sem); 2157d96b3424SFilipe Manana if (ret2) 2158d96b3424SFilipe Manana ret = ret2; 2159d96b3424SFilipe Manana } 2160d96b3424SFilipe Manana 2161be0e5c09SChris Mason return ret; 2162be0e5c09SChris Mason } 2163f75e2b79SJosef Bacik ALLOW_ERROR_INJECTION(btrfs_search_slot, ERRNO); 2164be0e5c09SChris Mason 216574123bd7SChris Mason /* 21665d9e75c4SJan Schmidt * Like btrfs_search_slot, this looks for a key in the given tree. It uses the 21675d9e75c4SJan Schmidt * current state of the tree together with the operations recorded in the tree 21685d9e75c4SJan Schmidt * modification log to search for the key in a previous version of this tree, as 21695d9e75c4SJan Schmidt * denoted by the time_seq parameter. 21705d9e75c4SJan Schmidt * 21715d9e75c4SJan Schmidt * Naturally, there is no support for insert, delete or cow operations. 21725d9e75c4SJan Schmidt * 21735d9e75c4SJan Schmidt * The resulting path and return value will be set up as if we called 21745d9e75c4SJan Schmidt * btrfs_search_slot at that point in time with ins_len and cow both set to 0. 21755d9e75c4SJan Schmidt */ 2176310712b2SOmar Sandoval int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key, 21775d9e75c4SJan Schmidt struct btrfs_path *p, u64 time_seq) 21785d9e75c4SJan Schmidt { 21790b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 21805d9e75c4SJan Schmidt struct extent_buffer *b; 21815d9e75c4SJan Schmidt int slot; 21825d9e75c4SJan Schmidt int ret; 21835d9e75c4SJan Schmidt int err; 21845d9e75c4SJan Schmidt int level; 21855d9e75c4SJan Schmidt int lowest_unlock = 1; 21865d9e75c4SJan Schmidt u8 lowest_level = 0; 21875d9e75c4SJan Schmidt 21885d9e75c4SJan Schmidt lowest_level = p->lowest_level; 21895d9e75c4SJan Schmidt WARN_ON(p->nodes[0] != NULL); 2190c922b016SStefan Roesch ASSERT(!p->nowait); 21915d9e75c4SJan Schmidt 21925d9e75c4SJan Schmidt if (p->search_commit_root) { 21935d9e75c4SJan Schmidt BUG_ON(time_seq); 21945d9e75c4SJan Schmidt return btrfs_search_slot(NULL, root, key, p, 0, 0); 21955d9e75c4SJan Schmidt } 21965d9e75c4SJan Schmidt 21975d9e75c4SJan Schmidt again: 2198f3a84ccdSFilipe Manana b = btrfs_get_old_root(root, time_seq); 2199315bed43SNikolay Borisov if (!b) { 2200315bed43SNikolay Borisov ret = -EIO; 2201315bed43SNikolay Borisov goto done; 2202315bed43SNikolay Borisov } 22035d9e75c4SJan Schmidt level = btrfs_header_level(b); 22045d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 22055d9e75c4SJan Schmidt 22065d9e75c4SJan Schmidt while (b) { 2207abe9339dSQu Wenruo int dec = 0; 2208abe9339dSQu Wenruo 22095d9e75c4SJan Schmidt level = btrfs_header_level(b); 22105d9e75c4SJan Schmidt p->nodes[level] = b; 22115d9e75c4SJan Schmidt 22125d9e75c4SJan Schmidt /* 22135d9e75c4SJan Schmidt * we have a lock on b and as long as we aren't changing 22145d9e75c4SJan Schmidt * the tree, there is no way to for the items in b to change. 22155d9e75c4SJan Schmidt * It is safe to drop the lock on our parent before we 22165d9e75c4SJan Schmidt * go through the expensive btree search on b. 22175d9e75c4SJan Schmidt */ 22185d9e75c4SJan Schmidt btrfs_unlock_up_safe(p, level + 1); 22195d9e75c4SJan Schmidt 2220995e9a16SNikolay Borisov ret = btrfs_bin_search(b, key, &slot); 2221cbca7d59SFilipe Manana if (ret < 0) 2222cbca7d59SFilipe Manana goto done; 22235d9e75c4SJan Schmidt 2224abe9339dSQu Wenruo if (level == 0) { 2225abe9339dSQu Wenruo p->slots[level] = slot; 2226abe9339dSQu Wenruo unlock_up(p, level, lowest_unlock, 0, NULL); 2227abe9339dSQu Wenruo goto done; 2228abe9339dSQu Wenruo } 2229abe9339dSQu Wenruo 22305d9e75c4SJan Schmidt if (ret && slot > 0) { 22315d9e75c4SJan Schmidt dec = 1; 2232abe9339dSQu Wenruo slot--; 22335d9e75c4SJan Schmidt } 22345d9e75c4SJan Schmidt p->slots[level] = slot; 22355d9e75c4SJan Schmidt unlock_up(p, level, lowest_unlock, 0, NULL); 22365d9e75c4SJan Schmidt 22375d9e75c4SJan Schmidt if (level == lowest_level) { 22385d9e75c4SJan Schmidt if (dec) 22395d9e75c4SJan Schmidt p->slots[level]++; 22405d9e75c4SJan Schmidt goto done; 22415d9e75c4SJan Schmidt } 22425d9e75c4SJan Schmidt 2243abe9339dSQu Wenruo err = read_block_for_search(root, p, &b, level, slot, key); 22445d9e75c4SJan Schmidt if (err == -EAGAIN) 22455d9e75c4SJan Schmidt goto again; 22465d9e75c4SJan Schmidt if (err) { 22475d9e75c4SJan Schmidt ret = err; 22485d9e75c4SJan Schmidt goto done; 22495d9e75c4SJan Schmidt } 22505d9e75c4SJan Schmidt 22515d9e75c4SJan Schmidt level = btrfs_header_level(b); 22525d9e75c4SJan Schmidt btrfs_tree_read_lock(b); 2253f3a84ccdSFilipe Manana b = btrfs_tree_mod_log_rewind(fs_info, p, b, time_seq); 2254db7f3436SJosef Bacik if (!b) { 2255db7f3436SJosef Bacik ret = -ENOMEM; 2256db7f3436SJosef Bacik goto done; 2257db7f3436SJosef Bacik } 22585d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 22595d9e75c4SJan Schmidt p->nodes[level] = b; 22605d9e75c4SJan Schmidt } 22615d9e75c4SJan Schmidt ret = 1; 22625d9e75c4SJan Schmidt done: 22635d9e75c4SJan Schmidt if (ret < 0) 22645d9e75c4SJan Schmidt btrfs_release_path(p); 22655d9e75c4SJan Schmidt 22665d9e75c4SJan Schmidt return ret; 22675d9e75c4SJan Schmidt } 22685d9e75c4SJan Schmidt 22695d9e75c4SJan Schmidt /* 22702f38b3e1SArne Jansen * helper to use instead of search slot if no exact match is needed but 22712f38b3e1SArne Jansen * instead the next or previous item should be returned. 22722f38b3e1SArne Jansen * When find_higher is true, the next higher item is returned, the next lower 22732f38b3e1SArne Jansen * otherwise. 22742f38b3e1SArne Jansen * When return_any and find_higher are both true, and no higher item is found, 22752f38b3e1SArne Jansen * return the next lower instead. 22762f38b3e1SArne Jansen * When return_any is true and find_higher is false, and no lower item is found, 22772f38b3e1SArne Jansen * return the next higher instead. 22782f38b3e1SArne Jansen * It returns 0 if any item is found, 1 if none is found (tree empty), and 22792f38b3e1SArne Jansen * < 0 on error 22802f38b3e1SArne Jansen */ 22812f38b3e1SArne Jansen int btrfs_search_slot_for_read(struct btrfs_root *root, 2282310712b2SOmar Sandoval const struct btrfs_key *key, 2283310712b2SOmar Sandoval struct btrfs_path *p, int find_higher, 2284310712b2SOmar Sandoval int return_any) 22852f38b3e1SArne Jansen { 22862f38b3e1SArne Jansen int ret; 22872f38b3e1SArne Jansen struct extent_buffer *leaf; 22882f38b3e1SArne Jansen 22892f38b3e1SArne Jansen again: 22902f38b3e1SArne Jansen ret = btrfs_search_slot(NULL, root, key, p, 0, 0); 22912f38b3e1SArne Jansen if (ret <= 0) 22922f38b3e1SArne Jansen return ret; 22932f38b3e1SArne Jansen /* 22942f38b3e1SArne Jansen * a return value of 1 means the path is at the position where the 22952f38b3e1SArne Jansen * item should be inserted. Normally this is the next bigger item, 22962f38b3e1SArne Jansen * but in case the previous item is the last in a leaf, path points 22972f38b3e1SArne Jansen * to the first free slot in the previous leaf, i.e. at an invalid 22982f38b3e1SArne Jansen * item. 22992f38b3e1SArne Jansen */ 23002f38b3e1SArne Jansen leaf = p->nodes[0]; 23012f38b3e1SArne Jansen 23022f38b3e1SArne Jansen if (find_higher) { 23032f38b3e1SArne Jansen if (p->slots[0] >= btrfs_header_nritems(leaf)) { 23042f38b3e1SArne Jansen ret = btrfs_next_leaf(root, p); 23052f38b3e1SArne Jansen if (ret <= 0) 23062f38b3e1SArne Jansen return ret; 23072f38b3e1SArne Jansen if (!return_any) 23082f38b3e1SArne Jansen return 1; 23092f38b3e1SArne Jansen /* 23102f38b3e1SArne Jansen * no higher item found, return the next 23112f38b3e1SArne Jansen * lower instead 23122f38b3e1SArne Jansen */ 23132f38b3e1SArne Jansen return_any = 0; 23142f38b3e1SArne Jansen find_higher = 0; 23152f38b3e1SArne Jansen btrfs_release_path(p); 23162f38b3e1SArne Jansen goto again; 23172f38b3e1SArne Jansen } 23182f38b3e1SArne Jansen } else { 23192f38b3e1SArne Jansen if (p->slots[0] == 0) { 23202f38b3e1SArne Jansen ret = btrfs_prev_leaf(root, p); 2321e6793769SArne Jansen if (ret < 0) 23222f38b3e1SArne Jansen return ret; 2323e6793769SArne Jansen if (!ret) { 232423c6bf6aSFilipe David Borba Manana leaf = p->nodes[0]; 232523c6bf6aSFilipe David Borba Manana if (p->slots[0] == btrfs_header_nritems(leaf)) 232623c6bf6aSFilipe David Borba Manana p->slots[0]--; 2327e6793769SArne Jansen return 0; 2328e6793769SArne Jansen } 23292f38b3e1SArne Jansen if (!return_any) 23302f38b3e1SArne Jansen return 1; 23312f38b3e1SArne Jansen /* 23322f38b3e1SArne Jansen * no lower item found, return the next 23332f38b3e1SArne Jansen * higher instead 23342f38b3e1SArne Jansen */ 23352f38b3e1SArne Jansen return_any = 0; 23362f38b3e1SArne Jansen find_higher = 1; 23372f38b3e1SArne Jansen btrfs_release_path(p); 23382f38b3e1SArne Jansen goto again; 2339e6793769SArne Jansen } else { 23402f38b3e1SArne Jansen --p->slots[0]; 23412f38b3e1SArne Jansen } 23422f38b3e1SArne Jansen } 23432f38b3e1SArne Jansen return 0; 23442f38b3e1SArne Jansen } 23452f38b3e1SArne Jansen 23462f38b3e1SArne Jansen /* 23470ff40a91SMarcos Paulo de Souza * Execute search and call btrfs_previous_item to traverse backwards if the item 23480ff40a91SMarcos Paulo de Souza * was not found. 23490ff40a91SMarcos Paulo de Souza * 23500ff40a91SMarcos Paulo de Souza * Return 0 if found, 1 if not found and < 0 if error. 23510ff40a91SMarcos Paulo de Souza */ 23520ff40a91SMarcos Paulo de Souza int btrfs_search_backwards(struct btrfs_root *root, struct btrfs_key *key, 23530ff40a91SMarcos Paulo de Souza struct btrfs_path *path) 23540ff40a91SMarcos Paulo de Souza { 23550ff40a91SMarcos Paulo de Souza int ret; 23560ff40a91SMarcos Paulo de Souza 23570ff40a91SMarcos Paulo de Souza ret = btrfs_search_slot(NULL, root, key, path, 0, 0); 23580ff40a91SMarcos Paulo de Souza if (ret > 0) 23590ff40a91SMarcos Paulo de Souza ret = btrfs_previous_item(root, path, key->objectid, key->type); 23600ff40a91SMarcos Paulo de Souza 23610ff40a91SMarcos Paulo de Souza if (ret == 0) 23620ff40a91SMarcos Paulo de Souza btrfs_item_key_to_cpu(path->nodes[0], key, path->slots[0]); 23630ff40a91SMarcos Paulo de Souza 23640ff40a91SMarcos Paulo de Souza return ret; 23650ff40a91SMarcos Paulo de Souza } 23660ff40a91SMarcos Paulo de Souza 236743dd529aSDavid Sterba /* 236862142be3SGabriel Niebler * Search for a valid slot for the given path. 236962142be3SGabriel Niebler * 237062142be3SGabriel Niebler * @root: The root node of the tree. 237162142be3SGabriel Niebler * @key: Will contain a valid item if found. 237262142be3SGabriel Niebler * @path: The starting point to validate the slot. 237362142be3SGabriel Niebler * 237462142be3SGabriel Niebler * Return: 0 if the item is valid 237562142be3SGabriel Niebler * 1 if not found 237662142be3SGabriel Niebler * <0 if error. 237762142be3SGabriel Niebler */ 237862142be3SGabriel Niebler int btrfs_get_next_valid_item(struct btrfs_root *root, struct btrfs_key *key, 237962142be3SGabriel Niebler struct btrfs_path *path) 238062142be3SGabriel Niebler { 238162142be3SGabriel Niebler while (1) { 238262142be3SGabriel Niebler int ret; 238362142be3SGabriel Niebler const int slot = path->slots[0]; 238462142be3SGabriel Niebler const struct extent_buffer *leaf = path->nodes[0]; 238562142be3SGabriel Niebler 238662142be3SGabriel Niebler /* This is where we start walking the path. */ 238762142be3SGabriel Niebler if (slot >= btrfs_header_nritems(leaf)) { 238862142be3SGabriel Niebler /* 238962142be3SGabriel Niebler * If we've reached the last slot in this leaf we need 239062142be3SGabriel Niebler * to go to the next leaf and reset the path. 239162142be3SGabriel Niebler */ 239262142be3SGabriel Niebler ret = btrfs_next_leaf(root, path); 239362142be3SGabriel Niebler if (ret) 239462142be3SGabriel Niebler return ret; 239562142be3SGabriel Niebler continue; 239662142be3SGabriel Niebler } 239762142be3SGabriel Niebler /* Store the found, valid item in @key. */ 239862142be3SGabriel Niebler btrfs_item_key_to_cpu(leaf, key, slot); 239962142be3SGabriel Niebler break; 240062142be3SGabriel Niebler } 240162142be3SGabriel Niebler return 0; 240262142be3SGabriel Niebler } 240362142be3SGabriel Niebler 24040ff40a91SMarcos Paulo de Souza /* 240574123bd7SChris Mason * adjust the pointers going up the tree, starting at level 240674123bd7SChris Mason * making sure the right key of each node is points to 'key'. 240774123bd7SChris Mason * This is used after shifting pointers to the left, so it stops 240874123bd7SChris Mason * fixing up pointers when a given leaf/node is not in slot 0 of the 240974123bd7SChris Mason * higher levels 2410aa5d6bedSChris Mason * 241174123bd7SChris Mason */ 2412b167fa91SNikolay Borisov static void fixup_low_keys(struct btrfs_path *path, 24135f39d397SChris Mason struct btrfs_disk_key *key, int level) 2414be0e5c09SChris Mason { 2415be0e5c09SChris Mason int i; 24165f39d397SChris Mason struct extent_buffer *t; 24170e82bcfeSDavid Sterba int ret; 24185f39d397SChris Mason 2419234b63a0SChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 2420be0e5c09SChris Mason int tslot = path->slots[i]; 24210e82bcfeSDavid Sterba 2422eb60ceacSChris Mason if (!path->nodes[i]) 2423be0e5c09SChris Mason break; 24245f39d397SChris Mason t = path->nodes[i]; 2425f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(t, tslot, 242633cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 24270e82bcfeSDavid Sterba BUG_ON(ret < 0); 24285f39d397SChris Mason btrfs_set_node_key(t, key, tslot); 2429d6025579SChris Mason btrfs_mark_buffer_dirty(path->nodes[i]); 2430be0e5c09SChris Mason if (tslot != 0) 2431be0e5c09SChris Mason break; 2432be0e5c09SChris Mason } 2433be0e5c09SChris Mason } 2434be0e5c09SChris Mason 243574123bd7SChris Mason /* 243631840ae1SZheng Yan * update item key. 243731840ae1SZheng Yan * 243831840ae1SZheng Yan * This function isn't completely safe. It's the caller's responsibility 243931840ae1SZheng Yan * that the new key won't break the order 244031840ae1SZheng Yan */ 2441b7a0365eSDaniel Dressler void btrfs_set_item_key_safe(struct btrfs_fs_info *fs_info, 2442b7a0365eSDaniel Dressler struct btrfs_path *path, 2443310712b2SOmar Sandoval const struct btrfs_key *new_key) 244431840ae1SZheng Yan { 244531840ae1SZheng Yan struct btrfs_disk_key disk_key; 244631840ae1SZheng Yan struct extent_buffer *eb; 244731840ae1SZheng Yan int slot; 244831840ae1SZheng Yan 244931840ae1SZheng Yan eb = path->nodes[0]; 245031840ae1SZheng Yan slot = path->slots[0]; 245131840ae1SZheng Yan if (slot > 0) { 245231840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot - 1); 24537c15d410SQu Wenruo if (unlikely(comp_keys(&disk_key, new_key) >= 0)) { 24547c15d410SQu Wenruo btrfs_crit(fs_info, 24557c15d410SQu Wenruo "slot %u key (%llu %u %llu) new key (%llu %u %llu)", 24567c15d410SQu Wenruo slot, btrfs_disk_key_objectid(&disk_key), 24577c15d410SQu Wenruo btrfs_disk_key_type(&disk_key), 24587c15d410SQu Wenruo btrfs_disk_key_offset(&disk_key), 24597c15d410SQu Wenruo new_key->objectid, new_key->type, 24607c15d410SQu Wenruo new_key->offset); 24617c15d410SQu Wenruo btrfs_print_leaf(eb); 24627c15d410SQu Wenruo BUG(); 24637c15d410SQu Wenruo } 246431840ae1SZheng Yan } 246531840ae1SZheng Yan if (slot < btrfs_header_nritems(eb) - 1) { 246631840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot + 1); 24677c15d410SQu Wenruo if (unlikely(comp_keys(&disk_key, new_key) <= 0)) { 24687c15d410SQu Wenruo btrfs_crit(fs_info, 24697c15d410SQu Wenruo "slot %u key (%llu %u %llu) new key (%llu %u %llu)", 24707c15d410SQu Wenruo slot, btrfs_disk_key_objectid(&disk_key), 24717c15d410SQu Wenruo btrfs_disk_key_type(&disk_key), 24727c15d410SQu Wenruo btrfs_disk_key_offset(&disk_key), 24737c15d410SQu Wenruo new_key->objectid, new_key->type, 24747c15d410SQu Wenruo new_key->offset); 24757c15d410SQu Wenruo btrfs_print_leaf(eb); 24767c15d410SQu Wenruo BUG(); 24777c15d410SQu Wenruo } 247831840ae1SZheng Yan } 247931840ae1SZheng Yan 248031840ae1SZheng Yan btrfs_cpu_key_to_disk(&disk_key, new_key); 248131840ae1SZheng Yan btrfs_set_item_key(eb, &disk_key, slot); 248231840ae1SZheng Yan btrfs_mark_buffer_dirty(eb); 248331840ae1SZheng Yan if (slot == 0) 2484b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 248531840ae1SZheng Yan } 248631840ae1SZheng Yan 248731840ae1SZheng Yan /* 2488d16c702fSQu Wenruo * Check key order of two sibling extent buffers. 2489d16c702fSQu Wenruo * 2490d16c702fSQu Wenruo * Return true if something is wrong. 2491d16c702fSQu Wenruo * Return false if everything is fine. 2492d16c702fSQu Wenruo * 2493d16c702fSQu Wenruo * Tree-checker only works inside one tree block, thus the following 2494d16c702fSQu Wenruo * corruption can not be detected by tree-checker: 2495d16c702fSQu Wenruo * 2496d16c702fSQu Wenruo * Leaf @left | Leaf @right 2497d16c702fSQu Wenruo * -------------------------------------------------------------- 2498d16c702fSQu Wenruo * | 1 | 2 | 3 | 4 | 5 | f6 | | 7 | 8 | 2499d16c702fSQu Wenruo * 2500d16c702fSQu Wenruo * Key f6 in leaf @left itself is valid, but not valid when the next 2501d16c702fSQu Wenruo * key in leaf @right is 7. 2502d16c702fSQu Wenruo * This can only be checked at tree block merge time. 2503d16c702fSQu Wenruo * And since tree checker has ensured all key order in each tree block 2504d16c702fSQu Wenruo * is correct, we only need to bother the last key of @left and the first 2505d16c702fSQu Wenruo * key of @right. 2506d16c702fSQu Wenruo */ 2507d16c702fSQu Wenruo static bool check_sibling_keys(struct extent_buffer *left, 2508d16c702fSQu Wenruo struct extent_buffer *right) 2509d16c702fSQu Wenruo { 2510d16c702fSQu Wenruo struct btrfs_key left_last; 2511d16c702fSQu Wenruo struct btrfs_key right_first; 2512d16c702fSQu Wenruo int level = btrfs_header_level(left); 2513d16c702fSQu Wenruo int nr_left = btrfs_header_nritems(left); 2514d16c702fSQu Wenruo int nr_right = btrfs_header_nritems(right); 2515d16c702fSQu Wenruo 2516d16c702fSQu Wenruo /* No key to check in one of the tree blocks */ 2517d16c702fSQu Wenruo if (!nr_left || !nr_right) 2518d16c702fSQu Wenruo return false; 2519d16c702fSQu Wenruo 2520d16c702fSQu Wenruo if (level) { 2521d16c702fSQu Wenruo btrfs_node_key_to_cpu(left, &left_last, nr_left - 1); 2522d16c702fSQu Wenruo btrfs_node_key_to_cpu(right, &right_first, 0); 2523d16c702fSQu Wenruo } else { 2524d16c702fSQu Wenruo btrfs_item_key_to_cpu(left, &left_last, nr_left - 1); 2525d16c702fSQu Wenruo btrfs_item_key_to_cpu(right, &right_first, 0); 2526d16c702fSQu Wenruo } 2527d16c702fSQu Wenruo 2528d16c702fSQu Wenruo if (btrfs_comp_cpu_keys(&left_last, &right_first) >= 0) { 2529d16c702fSQu Wenruo btrfs_crit(left->fs_info, 2530d16c702fSQu Wenruo "bad key order, sibling blocks, left last (%llu %u %llu) right first (%llu %u %llu)", 2531d16c702fSQu Wenruo left_last.objectid, left_last.type, 2532d16c702fSQu Wenruo left_last.offset, right_first.objectid, 2533d16c702fSQu Wenruo right_first.type, right_first.offset); 2534d16c702fSQu Wenruo return true; 2535d16c702fSQu Wenruo } 2536d16c702fSQu Wenruo return false; 2537d16c702fSQu Wenruo } 2538d16c702fSQu Wenruo 2539d16c702fSQu Wenruo /* 254074123bd7SChris Mason * try to push data from one node into the next node left in the 254179f95c82SChris Mason * tree. 2542aa5d6bedSChris Mason * 2543aa5d6bedSChris Mason * returns 0 if some ptrs were pushed left, < 0 if there was some horrible 2544aa5d6bedSChris Mason * error, and > 0 if there was no room in the left hand block. 254574123bd7SChris Mason */ 254698ed5174SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 25472ff7e61eSJeff Mahoney struct extent_buffer *dst, 2548971a1f66SChris Mason struct extent_buffer *src, int empty) 2549be0e5c09SChris Mason { 2550d30a668fSDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info; 2551be0e5c09SChris Mason int push_items = 0; 2552bb803951SChris Mason int src_nritems; 2553bb803951SChris Mason int dst_nritems; 2554aa5d6bedSChris Mason int ret = 0; 2555be0e5c09SChris Mason 25565f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 25575f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 25580b246afaSJeff Mahoney push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems; 25597bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 25607bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 256154aa1f4dSChris Mason 2562bce4eae9SChris Mason if (!empty && src_nritems <= 8) 2563971a1f66SChris Mason return 1; 2564971a1f66SChris Mason 2565d397712bSChris Mason if (push_items <= 0) 2566be0e5c09SChris Mason return 1; 2567be0e5c09SChris Mason 2568bce4eae9SChris Mason if (empty) { 2569971a1f66SChris Mason push_items = min(src_nritems, push_items); 2570bce4eae9SChris Mason if (push_items < src_nritems) { 2571bce4eae9SChris Mason /* leave at least 8 pointers in the node if 2572bce4eae9SChris Mason * we aren't going to empty it 2573bce4eae9SChris Mason */ 2574bce4eae9SChris Mason if (src_nritems - push_items < 8) { 2575bce4eae9SChris Mason if (push_items <= 8) 2576bce4eae9SChris Mason return 1; 2577bce4eae9SChris Mason push_items -= 8; 2578bce4eae9SChris Mason } 2579bce4eae9SChris Mason } 2580bce4eae9SChris Mason } else 2581bce4eae9SChris Mason push_items = min(src_nritems - 8, push_items); 258279f95c82SChris Mason 2583d16c702fSQu Wenruo /* dst is the left eb, src is the middle eb */ 2584d16c702fSQu Wenruo if (check_sibling_keys(dst, src)) { 2585d16c702fSQu Wenruo ret = -EUCLEAN; 2586d16c702fSQu Wenruo btrfs_abort_transaction(trans, ret); 2587d16c702fSQu Wenruo return ret; 2588d16c702fSQu Wenruo } 2589f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_eb_copy(dst, src, dst_nritems, 0, push_items); 25905de865eeSFilipe David Borba Manana if (ret) { 259166642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 25925de865eeSFilipe David Borba Manana return ret; 25935de865eeSFilipe David Borba Manana } 25945f39d397SChris Mason copy_extent_buffer(dst, src, 25955f39d397SChris Mason btrfs_node_key_ptr_offset(dst_nritems), 25965f39d397SChris Mason btrfs_node_key_ptr_offset(0), 2597123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 25985f39d397SChris Mason 2599bb803951SChris Mason if (push_items < src_nritems) { 260057911b8bSJan Schmidt /* 2601f3a84ccdSFilipe Manana * Don't call btrfs_tree_mod_log_insert_move() here, key removal 2602f3a84ccdSFilipe Manana * was already fully logged by btrfs_tree_mod_log_eb_copy() above. 260357911b8bSJan Schmidt */ 26045f39d397SChris Mason memmove_extent_buffer(src, btrfs_node_key_ptr_offset(0), 26055f39d397SChris Mason btrfs_node_key_ptr_offset(push_items), 2606e2fa7227SChris Mason (src_nritems - push_items) * 2607123abc88SChris Mason sizeof(struct btrfs_key_ptr)); 2608bb803951SChris Mason } 26095f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 26105f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 26115f39d397SChris Mason btrfs_mark_buffer_dirty(src); 26125f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 261331840ae1SZheng Yan 2614bb803951SChris Mason return ret; 2615be0e5c09SChris Mason } 2616be0e5c09SChris Mason 261797571fd0SChris Mason /* 261879f95c82SChris Mason * try to push data from one node into the next node right in the 261979f95c82SChris Mason * tree. 262079f95c82SChris Mason * 262179f95c82SChris Mason * returns 0 if some ptrs were pushed, < 0 if there was some horrible 262279f95c82SChris Mason * error, and > 0 if there was no room in the right hand block. 262379f95c82SChris Mason * 262479f95c82SChris Mason * this will only push up to 1/2 the contents of the left node over 262579f95c82SChris Mason */ 26265f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 26275f39d397SChris Mason struct extent_buffer *dst, 26285f39d397SChris Mason struct extent_buffer *src) 262979f95c82SChris Mason { 263055d32ed8SDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info; 263179f95c82SChris Mason int push_items = 0; 263279f95c82SChris Mason int max_push; 263379f95c82SChris Mason int src_nritems; 263479f95c82SChris Mason int dst_nritems; 263579f95c82SChris Mason int ret = 0; 263679f95c82SChris Mason 26377bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 26387bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 26397bb86316SChris Mason 26405f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 26415f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 26420b246afaSJeff Mahoney push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems; 2643d397712bSChris Mason if (push_items <= 0) 264479f95c82SChris Mason return 1; 2645bce4eae9SChris Mason 2646d397712bSChris Mason if (src_nritems < 4) 2647bce4eae9SChris Mason return 1; 264879f95c82SChris Mason 264979f95c82SChris Mason max_push = src_nritems / 2 + 1; 265079f95c82SChris Mason /* don't try to empty the node */ 2651d397712bSChris Mason if (max_push >= src_nritems) 265279f95c82SChris Mason return 1; 2653252c38f0SYan 265479f95c82SChris Mason if (max_push < push_items) 265579f95c82SChris Mason push_items = max_push; 265679f95c82SChris Mason 2657d16c702fSQu Wenruo /* dst is the right eb, src is the middle eb */ 2658d16c702fSQu Wenruo if (check_sibling_keys(src, dst)) { 2659d16c702fSQu Wenruo ret = -EUCLEAN; 2660d16c702fSQu Wenruo btrfs_abort_transaction(trans, ret); 2661d16c702fSQu Wenruo return ret; 2662d16c702fSQu Wenruo } 2663f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_move(dst, push_items, 0, dst_nritems); 2664bf1d3425SDavid Sterba BUG_ON(ret < 0); 26655f39d397SChris Mason memmove_extent_buffer(dst, btrfs_node_key_ptr_offset(push_items), 26665f39d397SChris Mason btrfs_node_key_ptr_offset(0), 26675f39d397SChris Mason (dst_nritems) * 26685f39d397SChris Mason sizeof(struct btrfs_key_ptr)); 2669d6025579SChris Mason 2670f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_eb_copy(dst, src, 0, src_nritems - push_items, 2671ed874f0dSDavid Sterba push_items); 26725de865eeSFilipe David Borba Manana if (ret) { 267366642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 26745de865eeSFilipe David Borba Manana return ret; 26755de865eeSFilipe David Borba Manana } 26765f39d397SChris Mason copy_extent_buffer(dst, src, 26775f39d397SChris Mason btrfs_node_key_ptr_offset(0), 26785f39d397SChris Mason btrfs_node_key_ptr_offset(src_nritems - push_items), 2679123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 268079f95c82SChris Mason 26815f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 26825f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 268379f95c82SChris Mason 26845f39d397SChris Mason btrfs_mark_buffer_dirty(src); 26855f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 268631840ae1SZheng Yan 268779f95c82SChris Mason return ret; 268879f95c82SChris Mason } 268979f95c82SChris Mason 269079f95c82SChris Mason /* 269197571fd0SChris Mason * helper function to insert a new root level in the tree. 269297571fd0SChris Mason * A new node is allocated, and a single item is inserted to 269397571fd0SChris Mason * point to the existing root 2694aa5d6bedSChris Mason * 2695aa5d6bedSChris Mason * returns zero on success or < 0 on failure. 269697571fd0SChris Mason */ 2697d397712bSChris Mason static noinline int insert_new_root(struct btrfs_trans_handle *trans, 26985f39d397SChris Mason struct btrfs_root *root, 2699fdd99c72SLiu Bo struct btrfs_path *path, int level) 270074123bd7SChris Mason { 27010b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 27027bb86316SChris Mason u64 lower_gen; 27035f39d397SChris Mason struct extent_buffer *lower; 27045f39d397SChris Mason struct extent_buffer *c; 2705925baeddSChris Mason struct extent_buffer *old; 27065f39d397SChris Mason struct btrfs_disk_key lower_key; 2707d9d19a01SDavid Sterba int ret; 27085c680ed6SChris Mason 27095c680ed6SChris Mason BUG_ON(path->nodes[level]); 27105c680ed6SChris Mason BUG_ON(path->nodes[level-1] != root->node); 27115c680ed6SChris Mason 27127bb86316SChris Mason lower = path->nodes[level-1]; 27137bb86316SChris Mason if (level == 1) 27147bb86316SChris Mason btrfs_item_key(lower, &lower_key, 0); 27157bb86316SChris Mason else 27167bb86316SChris Mason btrfs_node_key(lower, &lower_key, 0); 27177bb86316SChris Mason 271879bd3712SFilipe Manana c = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 271979bd3712SFilipe Manana &lower_key, level, root->node->start, 0, 2720cf6f34aaSJosef Bacik BTRFS_NESTING_NEW_ROOT); 27215f39d397SChris Mason if (IS_ERR(c)) 27225f39d397SChris Mason return PTR_ERR(c); 2723925baeddSChris Mason 27240b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 2725f0486c68SYan, Zheng 27265f39d397SChris Mason btrfs_set_header_nritems(c, 1); 27275f39d397SChris Mason btrfs_set_node_key(c, &lower_key, 0); 2728db94535dSChris Mason btrfs_set_node_blockptr(c, 0, lower->start); 27297bb86316SChris Mason lower_gen = btrfs_header_generation(lower); 273031840ae1SZheng Yan WARN_ON(lower_gen != trans->transid); 27317bb86316SChris Mason 27327bb86316SChris Mason btrfs_set_node_ptr_generation(c, 0, lower_gen); 27335f39d397SChris Mason 27345f39d397SChris Mason btrfs_mark_buffer_dirty(c); 2735d5719762SChris Mason 2736925baeddSChris Mason old = root->node; 2737406808abSFilipe Manana ret = btrfs_tree_mod_log_insert_root(root->node, c, false); 2738d9d19a01SDavid Sterba BUG_ON(ret < 0); 2739240f62c8SChris Mason rcu_assign_pointer(root->node, c); 2740925baeddSChris Mason 2741925baeddSChris Mason /* the super has an extra ref to root->node */ 2742925baeddSChris Mason free_extent_buffer(old); 2743925baeddSChris Mason 27440b86a832SChris Mason add_root_to_dirty_list(root); 274567439dadSDavid Sterba atomic_inc(&c->refs); 27465f39d397SChris Mason path->nodes[level] = c; 2747ac5887c8SJosef Bacik path->locks[level] = BTRFS_WRITE_LOCK; 274874123bd7SChris Mason path->slots[level] = 0; 274974123bd7SChris Mason return 0; 275074123bd7SChris Mason } 27515c680ed6SChris Mason 27525c680ed6SChris Mason /* 27535c680ed6SChris Mason * worker function to insert a single pointer in a node. 27545c680ed6SChris Mason * the node should have enough room for the pointer already 275597571fd0SChris Mason * 27565c680ed6SChris Mason * slot and level indicate where you want the key to go, and 27575c680ed6SChris Mason * blocknr is the block the key points to. 27585c680ed6SChris Mason */ 2759143bede5SJeff Mahoney static void insert_ptr(struct btrfs_trans_handle *trans, 27606ad3cf6dSDavid Sterba struct btrfs_path *path, 2761143bede5SJeff Mahoney struct btrfs_disk_key *key, u64 bytenr, 2762c3e06965SJan Schmidt int slot, int level) 27635c680ed6SChris Mason { 27645f39d397SChris Mason struct extent_buffer *lower; 27655c680ed6SChris Mason int nritems; 2766f3ea38daSJan Schmidt int ret; 27675c680ed6SChris Mason 27685c680ed6SChris Mason BUG_ON(!path->nodes[level]); 276949d0c642SFilipe Manana btrfs_assert_tree_write_locked(path->nodes[level]); 27705f39d397SChris Mason lower = path->nodes[level]; 27715f39d397SChris Mason nritems = btrfs_header_nritems(lower); 2772c293498bSStoyan Gaydarov BUG_ON(slot > nritems); 27736ad3cf6dSDavid Sterba BUG_ON(nritems == BTRFS_NODEPTRS_PER_BLOCK(trans->fs_info)); 277474123bd7SChris Mason if (slot != nritems) { 2775bf1d3425SDavid Sterba if (level) { 2776f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_move(lower, slot + 1, 2777f3a84ccdSFilipe Manana slot, nritems - slot); 2778bf1d3425SDavid Sterba BUG_ON(ret < 0); 2779bf1d3425SDavid Sterba } 27805f39d397SChris Mason memmove_extent_buffer(lower, 27815f39d397SChris Mason btrfs_node_key_ptr_offset(slot + 1), 27825f39d397SChris Mason btrfs_node_key_ptr_offset(slot), 2783123abc88SChris Mason (nritems - slot) * sizeof(struct btrfs_key_ptr)); 278474123bd7SChris Mason } 2785c3e06965SJan Schmidt if (level) { 2786f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(lower, slot, 278733cff222SFilipe Manana BTRFS_MOD_LOG_KEY_ADD); 2788f3ea38daSJan Schmidt BUG_ON(ret < 0); 2789f3ea38daSJan Schmidt } 27905f39d397SChris Mason btrfs_set_node_key(lower, key, slot); 2791db94535dSChris Mason btrfs_set_node_blockptr(lower, slot, bytenr); 279274493f7aSChris Mason WARN_ON(trans->transid == 0); 279374493f7aSChris Mason btrfs_set_node_ptr_generation(lower, slot, trans->transid); 27945f39d397SChris Mason btrfs_set_header_nritems(lower, nritems + 1); 27955f39d397SChris Mason btrfs_mark_buffer_dirty(lower); 279674123bd7SChris Mason } 279774123bd7SChris Mason 279897571fd0SChris Mason /* 279997571fd0SChris Mason * split the node at the specified level in path in two. 280097571fd0SChris Mason * The path is corrected to point to the appropriate node after the split 280197571fd0SChris Mason * 280297571fd0SChris Mason * Before splitting this tries to make some room in the node by pushing 280397571fd0SChris Mason * left and right, if either one works, it returns right away. 2804aa5d6bedSChris Mason * 2805aa5d6bedSChris Mason * returns 0 on success and < 0 on failure 280697571fd0SChris Mason */ 2807e02119d5SChris Mason static noinline int split_node(struct btrfs_trans_handle *trans, 2808e02119d5SChris Mason struct btrfs_root *root, 2809e02119d5SChris Mason struct btrfs_path *path, int level) 2810be0e5c09SChris Mason { 28110b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 28125f39d397SChris Mason struct extent_buffer *c; 28135f39d397SChris Mason struct extent_buffer *split; 28145f39d397SChris Mason struct btrfs_disk_key disk_key; 2815be0e5c09SChris Mason int mid; 28165c680ed6SChris Mason int ret; 28177518a238SChris Mason u32 c_nritems; 2818be0e5c09SChris Mason 28195f39d397SChris Mason c = path->nodes[level]; 28207bb86316SChris Mason WARN_ON(btrfs_header_generation(c) != trans->transid); 28215f39d397SChris Mason if (c == root->node) { 2822d9abbf1cSJan Schmidt /* 282390f8d62eSJan Schmidt * trying to split the root, lets make a new one 282490f8d62eSJan Schmidt * 2825fdd99c72SLiu Bo * tree mod log: We don't log_removal old root in 282690f8d62eSJan Schmidt * insert_new_root, because that root buffer will be kept as a 282790f8d62eSJan Schmidt * normal node. We are going to log removal of half of the 2828f3a84ccdSFilipe Manana * elements below with btrfs_tree_mod_log_eb_copy(). We're 2829f3a84ccdSFilipe Manana * holding a tree lock on the buffer, which is why we cannot 2830f3a84ccdSFilipe Manana * race with other tree_mod_log users. 2831d9abbf1cSJan Schmidt */ 2832fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, level + 1); 28335c680ed6SChris Mason if (ret) 28345c680ed6SChris Mason return ret; 2835b3612421SChris Mason } else { 2836e66f709bSChris Mason ret = push_nodes_for_insert(trans, root, path, level); 28375f39d397SChris Mason c = path->nodes[level]; 28385f39d397SChris Mason if (!ret && btrfs_header_nritems(c) < 28390b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3) 2840e66f709bSChris Mason return 0; 284154aa1f4dSChris Mason if (ret < 0) 284254aa1f4dSChris Mason return ret; 28435c680ed6SChris Mason } 2844e66f709bSChris Mason 28455f39d397SChris Mason c_nritems = btrfs_header_nritems(c); 28465d4f98a2SYan Zheng mid = (c_nritems + 1) / 2; 28475d4f98a2SYan Zheng btrfs_node_key(c, &disk_key, mid); 28487bb86316SChris Mason 284979bd3712SFilipe Manana split = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 285079bd3712SFilipe Manana &disk_key, level, c->start, 0, 285179bd3712SFilipe Manana BTRFS_NESTING_SPLIT); 28525f39d397SChris Mason if (IS_ERR(split)) 28535f39d397SChris Mason return PTR_ERR(split); 285454aa1f4dSChris Mason 28550b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 2856bc877d28SNikolay Borisov ASSERT(btrfs_header_level(c) == level); 28575f39d397SChris Mason 2858f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_eb_copy(split, c, 0, mid, c_nritems - mid); 28595de865eeSFilipe David Borba Manana if (ret) { 286066642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 28615de865eeSFilipe David Borba Manana return ret; 28625de865eeSFilipe David Borba Manana } 28635f39d397SChris Mason copy_extent_buffer(split, c, 28645f39d397SChris Mason btrfs_node_key_ptr_offset(0), 28655f39d397SChris Mason btrfs_node_key_ptr_offset(mid), 2866123abc88SChris Mason (c_nritems - mid) * sizeof(struct btrfs_key_ptr)); 28675f39d397SChris Mason btrfs_set_header_nritems(split, c_nritems - mid); 28685f39d397SChris Mason btrfs_set_header_nritems(c, mid); 2869aa5d6bedSChris Mason 28705f39d397SChris Mason btrfs_mark_buffer_dirty(c); 28715f39d397SChris Mason btrfs_mark_buffer_dirty(split); 28725f39d397SChris Mason 28736ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, split->start, 2874c3e06965SJan Schmidt path->slots[level + 1] + 1, level + 1); 2875aa5d6bedSChris Mason 28765de08d7dSChris Mason if (path->slots[level] >= mid) { 28775c680ed6SChris Mason path->slots[level] -= mid; 2878925baeddSChris Mason btrfs_tree_unlock(c); 28795f39d397SChris Mason free_extent_buffer(c); 28805f39d397SChris Mason path->nodes[level] = split; 28815c680ed6SChris Mason path->slots[level + 1] += 1; 2882eb60ceacSChris Mason } else { 2883925baeddSChris Mason btrfs_tree_unlock(split); 28845f39d397SChris Mason free_extent_buffer(split); 2885be0e5c09SChris Mason } 2886d5286a92SNikolay Borisov return 0; 2887be0e5c09SChris Mason } 2888be0e5c09SChris Mason 288974123bd7SChris Mason /* 289074123bd7SChris Mason * how many bytes are required to store the items in a leaf. start 289174123bd7SChris Mason * and nr indicate which items in the leaf to check. This totals up the 289274123bd7SChris Mason * space used both by the item structs and the item data 289374123bd7SChris Mason */ 28945f39d397SChris Mason static int leaf_space_used(struct extent_buffer *l, int start, int nr) 2895be0e5c09SChris Mason { 2896be0e5c09SChris Mason int data_len; 28975f39d397SChris Mason int nritems = btrfs_header_nritems(l); 2898d4dbff95SChris Mason int end = min(nritems, start + nr) - 1; 2899be0e5c09SChris Mason 2900be0e5c09SChris Mason if (!nr) 2901be0e5c09SChris Mason return 0; 29023212fa14SJosef Bacik data_len = btrfs_item_offset(l, start) + btrfs_item_size(l, start); 29033212fa14SJosef Bacik data_len = data_len - btrfs_item_offset(l, end); 29040783fcfcSChris Mason data_len += sizeof(struct btrfs_item) * nr; 2905d4dbff95SChris Mason WARN_ON(data_len < 0); 2906be0e5c09SChris Mason return data_len; 2907be0e5c09SChris Mason } 2908be0e5c09SChris Mason 290974123bd7SChris Mason /* 2910d4dbff95SChris Mason * The space between the end of the leaf items and 2911d4dbff95SChris Mason * the start of the leaf data. IOW, how much room 2912d4dbff95SChris Mason * the leaf has left for both items and data 2913d4dbff95SChris Mason */ 2914e902baacSDavid Sterba noinline int btrfs_leaf_free_space(struct extent_buffer *leaf) 2915d4dbff95SChris Mason { 2916e902baacSDavid Sterba struct btrfs_fs_info *fs_info = leaf->fs_info; 29175f39d397SChris Mason int nritems = btrfs_header_nritems(leaf); 29185f39d397SChris Mason int ret; 29190b246afaSJeff Mahoney 29200b246afaSJeff Mahoney ret = BTRFS_LEAF_DATA_SIZE(fs_info) - leaf_space_used(leaf, 0, nritems); 29215f39d397SChris Mason if (ret < 0) { 29220b246afaSJeff Mahoney btrfs_crit(fs_info, 2923efe120a0SFrank Holton "leaf free space ret %d, leaf data size %lu, used %d nritems %d", 2924da17066cSJeff Mahoney ret, 29250b246afaSJeff Mahoney (unsigned long) BTRFS_LEAF_DATA_SIZE(fs_info), 29265f39d397SChris Mason leaf_space_used(leaf, 0, nritems), nritems); 29275f39d397SChris Mason } 29285f39d397SChris Mason return ret; 2929d4dbff95SChris Mason } 2930d4dbff95SChris Mason 293199d8f83cSChris Mason /* 293299d8f83cSChris Mason * min slot controls the lowest index we're willing to push to the 293399d8f83cSChris Mason * right. We'll push up to and including min_slot, but no lower 293499d8f83cSChris Mason */ 2935f72f0010SDavid Sterba static noinline int __push_leaf_right(struct btrfs_path *path, 293644871b1bSChris Mason int data_size, int empty, 293744871b1bSChris Mason struct extent_buffer *right, 293899d8f83cSChris Mason int free_space, u32 left_nritems, 293999d8f83cSChris Mason u32 min_slot) 294000ec4c51SChris Mason { 2941f72f0010SDavid Sterba struct btrfs_fs_info *fs_info = right->fs_info; 29425f39d397SChris Mason struct extent_buffer *left = path->nodes[0]; 294344871b1bSChris Mason struct extent_buffer *upper = path->nodes[1]; 2944cfed81a0SChris Mason struct btrfs_map_token token; 29455f39d397SChris Mason struct btrfs_disk_key disk_key; 294600ec4c51SChris Mason int slot; 294734a38218SChris Mason u32 i; 294800ec4c51SChris Mason int push_space = 0; 294900ec4c51SChris Mason int push_items = 0; 295034a38218SChris Mason u32 nr; 29517518a238SChris Mason u32 right_nritems; 29525f39d397SChris Mason u32 data_end; 2953db94535dSChris Mason u32 this_item_size; 295400ec4c51SChris Mason 295534a38218SChris Mason if (empty) 295634a38218SChris Mason nr = 0; 295734a38218SChris Mason else 295899d8f83cSChris Mason nr = max_t(u32, 1, min_slot); 295934a38218SChris Mason 296031840ae1SZheng Yan if (path->slots[0] >= left_nritems) 296187b29b20SYan Zheng push_space += data_size; 296231840ae1SZheng Yan 296344871b1bSChris Mason slot = path->slots[1]; 296434a38218SChris Mason i = left_nritems - 1; 296534a38218SChris Mason while (i >= nr) { 296631840ae1SZheng Yan if (!empty && push_items > 0) { 296731840ae1SZheng Yan if (path->slots[0] > i) 296831840ae1SZheng Yan break; 296931840ae1SZheng Yan if (path->slots[0] == i) { 2970e902baacSDavid Sterba int space = btrfs_leaf_free_space(left); 2971e902baacSDavid Sterba 297231840ae1SZheng Yan if (space + push_space * 2 > free_space) 297331840ae1SZheng Yan break; 297431840ae1SZheng Yan } 297531840ae1SZheng Yan } 297631840ae1SZheng Yan 297700ec4c51SChris Mason if (path->slots[0] == i) 297887b29b20SYan Zheng push_space += data_size; 2979db94535dSChris Mason 29803212fa14SJosef Bacik this_item_size = btrfs_item_size(left, i); 298174794207SJosef Bacik if (this_item_size + sizeof(struct btrfs_item) + 298274794207SJosef Bacik push_space > free_space) 298300ec4c51SChris Mason break; 298431840ae1SZheng Yan 298500ec4c51SChris Mason push_items++; 298674794207SJosef Bacik push_space += this_item_size + sizeof(struct btrfs_item); 298734a38218SChris Mason if (i == 0) 298834a38218SChris Mason break; 298934a38218SChris Mason i--; 2990db94535dSChris Mason } 29915f39d397SChris Mason 2992925baeddSChris Mason if (push_items == 0) 2993925baeddSChris Mason goto out_unlock; 29945f39d397SChris Mason 29956c1500f2SJulia Lawall WARN_ON(!empty && push_items == left_nritems); 29965f39d397SChris Mason 299700ec4c51SChris Mason /* push left to right */ 29985f39d397SChris Mason right_nritems = btrfs_header_nritems(right); 299934a38218SChris Mason 3000dc2e724eSJosef Bacik push_space = btrfs_item_data_end(left, left_nritems - push_items); 30018f881e8cSDavid Sterba push_space -= leaf_data_end(left); 30025f39d397SChris Mason 300300ec4c51SChris Mason /* make room in the right data area */ 30048f881e8cSDavid Sterba data_end = leaf_data_end(right); 30055f39d397SChris Mason memmove_extent_buffer(right, 30063d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + data_end - push_space, 30073d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + data_end, 30080b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - data_end); 30095f39d397SChris Mason 301000ec4c51SChris Mason /* copy from the left data area */ 30113d9ec8c4SNikolay Borisov copy_extent_buffer(right, left, BTRFS_LEAF_DATA_OFFSET + 30120b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - push_space, 30138f881e8cSDavid Sterba BTRFS_LEAF_DATA_OFFSET + leaf_data_end(left), 3014d6025579SChris Mason push_space); 30155f39d397SChris Mason 30165f39d397SChris Mason memmove_extent_buffer(right, btrfs_item_nr_offset(push_items), 30175f39d397SChris Mason btrfs_item_nr_offset(0), 30180783fcfcSChris Mason right_nritems * sizeof(struct btrfs_item)); 30195f39d397SChris Mason 302000ec4c51SChris Mason /* copy the items from left to right */ 30215f39d397SChris Mason copy_extent_buffer(right, left, btrfs_item_nr_offset(0), 30225f39d397SChris Mason btrfs_item_nr_offset(left_nritems - push_items), 30230783fcfcSChris Mason push_items * sizeof(struct btrfs_item)); 302400ec4c51SChris Mason 302500ec4c51SChris Mason /* update the item pointers */ 3026c82f823cSDavid Sterba btrfs_init_map_token(&token, right); 30277518a238SChris Mason right_nritems += push_items; 30285f39d397SChris Mason btrfs_set_header_nritems(right, right_nritems); 30290b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info); 30307518a238SChris Mason for (i = 0; i < right_nritems; i++) { 30313212fa14SJosef Bacik push_space -= btrfs_token_item_size(&token, i); 30323212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, push_space); 3033db94535dSChris Mason } 3034db94535dSChris Mason 30357518a238SChris Mason left_nritems -= push_items; 30365f39d397SChris Mason btrfs_set_header_nritems(left, left_nritems); 303700ec4c51SChris Mason 303834a38218SChris Mason if (left_nritems) 30395f39d397SChris Mason btrfs_mark_buffer_dirty(left); 3040f0486c68SYan, Zheng else 30416a884d7dSDavid Sterba btrfs_clean_tree_block(left); 3042f0486c68SYan, Zheng 30435f39d397SChris Mason btrfs_mark_buffer_dirty(right); 3044a429e513SChris Mason 30455f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 30465f39d397SChris Mason btrfs_set_node_key(upper, &disk_key, slot + 1); 3047d6025579SChris Mason btrfs_mark_buffer_dirty(upper); 304802217ed2SChris Mason 304900ec4c51SChris Mason /* then fixup the leaf pointer in the path */ 30507518a238SChris Mason if (path->slots[0] >= left_nritems) { 30517518a238SChris Mason path->slots[0] -= left_nritems; 3052925baeddSChris Mason if (btrfs_header_nritems(path->nodes[0]) == 0) 30536a884d7dSDavid Sterba btrfs_clean_tree_block(path->nodes[0]); 3054925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 30555f39d397SChris Mason free_extent_buffer(path->nodes[0]); 30565f39d397SChris Mason path->nodes[0] = right; 305700ec4c51SChris Mason path->slots[1] += 1; 305800ec4c51SChris Mason } else { 3059925baeddSChris Mason btrfs_tree_unlock(right); 30605f39d397SChris Mason free_extent_buffer(right); 306100ec4c51SChris Mason } 306200ec4c51SChris Mason return 0; 3063925baeddSChris Mason 3064925baeddSChris Mason out_unlock: 3065925baeddSChris Mason btrfs_tree_unlock(right); 3066925baeddSChris Mason free_extent_buffer(right); 3067925baeddSChris Mason return 1; 306800ec4c51SChris Mason } 3069925baeddSChris Mason 307000ec4c51SChris Mason /* 307144871b1bSChris Mason * push some data in the path leaf to the right, trying to free up at 307274123bd7SChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 307344871b1bSChris Mason * 307444871b1bSChris Mason * returns 1 if the push failed because the other node didn't have enough 307544871b1bSChris Mason * room, 0 if everything worked out and < 0 if there were major errors. 307699d8f83cSChris Mason * 307799d8f83cSChris Mason * this will push starting from min_slot to the end of the leaf. It won't 307899d8f83cSChris Mason * push any slot lower than min_slot 307974123bd7SChris Mason */ 308044871b1bSChris Mason static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root 308199d8f83cSChris Mason *root, struct btrfs_path *path, 308299d8f83cSChris Mason int min_data_size, int data_size, 308399d8f83cSChris Mason int empty, u32 min_slot) 3084be0e5c09SChris Mason { 308544871b1bSChris Mason struct extent_buffer *left = path->nodes[0]; 308644871b1bSChris Mason struct extent_buffer *right; 308744871b1bSChris Mason struct extent_buffer *upper; 308844871b1bSChris Mason int slot; 308944871b1bSChris Mason int free_space; 309044871b1bSChris Mason u32 left_nritems; 309144871b1bSChris Mason int ret; 309244871b1bSChris Mason 309344871b1bSChris Mason if (!path->nodes[1]) 309444871b1bSChris Mason return 1; 309544871b1bSChris Mason 309644871b1bSChris Mason slot = path->slots[1]; 309744871b1bSChris Mason upper = path->nodes[1]; 309844871b1bSChris Mason if (slot >= btrfs_header_nritems(upper) - 1) 309944871b1bSChris Mason return 1; 310044871b1bSChris Mason 310149d0c642SFilipe Manana btrfs_assert_tree_write_locked(path->nodes[1]); 310244871b1bSChris Mason 31034b231ae4SDavid Sterba right = btrfs_read_node_slot(upper, slot + 1); 3104fb770ae4SLiu Bo /* 3105fb770ae4SLiu Bo * slot + 1 is not valid or we fail to read the right node, 3106fb770ae4SLiu Bo * no big deal, just return. 3107fb770ae4SLiu Bo */ 3108fb770ae4SLiu Bo if (IS_ERR(right)) 310991ca338dSTsutomu Itoh return 1; 311091ca338dSTsutomu Itoh 3111bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT); 311244871b1bSChris Mason 3113e902baacSDavid Sterba free_space = btrfs_leaf_free_space(right); 311444871b1bSChris Mason if (free_space < data_size) 311544871b1bSChris Mason goto out_unlock; 311644871b1bSChris Mason 311744871b1bSChris Mason ret = btrfs_cow_block(trans, root, right, upper, 3118bf59a5a2SJosef Bacik slot + 1, &right, BTRFS_NESTING_RIGHT_COW); 311944871b1bSChris Mason if (ret) 312044871b1bSChris Mason goto out_unlock; 312144871b1bSChris Mason 312244871b1bSChris Mason left_nritems = btrfs_header_nritems(left); 312344871b1bSChris Mason if (left_nritems == 0) 312444871b1bSChris Mason goto out_unlock; 312544871b1bSChris Mason 3126d16c702fSQu Wenruo if (check_sibling_keys(left, right)) { 3127d16c702fSQu Wenruo ret = -EUCLEAN; 3128d16c702fSQu Wenruo btrfs_tree_unlock(right); 3129d16c702fSQu Wenruo free_extent_buffer(right); 3130d16c702fSQu Wenruo return ret; 3131d16c702fSQu Wenruo } 31322ef1fed2SFilipe David Borba Manana if (path->slots[0] == left_nritems && !empty) { 31332ef1fed2SFilipe David Borba Manana /* Key greater than all keys in the leaf, right neighbor has 31342ef1fed2SFilipe David Borba Manana * enough room for it and we're not emptying our leaf to delete 31352ef1fed2SFilipe David Borba Manana * it, therefore use right neighbor to insert the new item and 313652042d8eSAndrea Gelmini * no need to touch/dirty our left leaf. */ 31372ef1fed2SFilipe David Borba Manana btrfs_tree_unlock(left); 31382ef1fed2SFilipe David Borba Manana free_extent_buffer(left); 31392ef1fed2SFilipe David Borba Manana path->nodes[0] = right; 31402ef1fed2SFilipe David Borba Manana path->slots[0] = 0; 31412ef1fed2SFilipe David Borba Manana path->slots[1]++; 31422ef1fed2SFilipe David Borba Manana return 0; 31432ef1fed2SFilipe David Borba Manana } 31442ef1fed2SFilipe David Borba Manana 3145f72f0010SDavid Sterba return __push_leaf_right(path, min_data_size, empty, 314699d8f83cSChris Mason right, free_space, left_nritems, min_slot); 314744871b1bSChris Mason out_unlock: 314844871b1bSChris Mason btrfs_tree_unlock(right); 314944871b1bSChris Mason free_extent_buffer(right); 315044871b1bSChris Mason return 1; 315144871b1bSChris Mason } 315244871b1bSChris Mason 315344871b1bSChris Mason /* 315444871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 315544871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 315699d8f83cSChris Mason * 315799d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 315899d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us do all the 315999d8f83cSChris Mason * items 316044871b1bSChris Mason */ 31618087c193SDavid Sterba static noinline int __push_leaf_left(struct btrfs_path *path, int data_size, 316244871b1bSChris Mason int empty, struct extent_buffer *left, 316399d8f83cSChris Mason int free_space, u32 right_nritems, 316499d8f83cSChris Mason u32 max_slot) 316544871b1bSChris Mason { 31668087c193SDavid Sterba struct btrfs_fs_info *fs_info = left->fs_info; 31675f39d397SChris Mason struct btrfs_disk_key disk_key; 31685f39d397SChris Mason struct extent_buffer *right = path->nodes[0]; 3169be0e5c09SChris Mason int i; 3170be0e5c09SChris Mason int push_space = 0; 3171be0e5c09SChris Mason int push_items = 0; 31727518a238SChris Mason u32 old_left_nritems; 317334a38218SChris Mason u32 nr; 3174aa5d6bedSChris Mason int ret = 0; 3175db94535dSChris Mason u32 this_item_size; 3176db94535dSChris Mason u32 old_left_item_size; 3177cfed81a0SChris Mason struct btrfs_map_token token; 3178cfed81a0SChris Mason 317934a38218SChris Mason if (empty) 318099d8f83cSChris Mason nr = min(right_nritems, max_slot); 318134a38218SChris Mason else 318299d8f83cSChris Mason nr = min(right_nritems - 1, max_slot); 318334a38218SChris Mason 318434a38218SChris Mason for (i = 0; i < nr; i++) { 318531840ae1SZheng Yan if (!empty && push_items > 0) { 318631840ae1SZheng Yan if (path->slots[0] < i) 318731840ae1SZheng Yan break; 318831840ae1SZheng Yan if (path->slots[0] == i) { 3189e902baacSDavid Sterba int space = btrfs_leaf_free_space(right); 3190e902baacSDavid Sterba 319131840ae1SZheng Yan if (space + push_space * 2 > free_space) 319231840ae1SZheng Yan break; 319331840ae1SZheng Yan } 319431840ae1SZheng Yan } 319531840ae1SZheng Yan 3196be0e5c09SChris Mason if (path->slots[0] == i) 319787b29b20SYan Zheng push_space += data_size; 3198db94535dSChris Mason 31993212fa14SJosef Bacik this_item_size = btrfs_item_size(right, i); 320074794207SJosef Bacik if (this_item_size + sizeof(struct btrfs_item) + push_space > 320174794207SJosef Bacik free_space) 3202be0e5c09SChris Mason break; 3203db94535dSChris Mason 3204be0e5c09SChris Mason push_items++; 320574794207SJosef Bacik push_space += this_item_size + sizeof(struct btrfs_item); 3206be0e5c09SChris Mason } 3207db94535dSChris Mason 3208be0e5c09SChris Mason if (push_items == 0) { 3209925baeddSChris Mason ret = 1; 3210925baeddSChris Mason goto out; 3211be0e5c09SChris Mason } 3212fae7f21cSDulshani Gunawardhana WARN_ON(!empty && push_items == btrfs_header_nritems(right)); 32135f39d397SChris Mason 3214be0e5c09SChris Mason /* push data from right to left */ 32155f39d397SChris Mason copy_extent_buffer(left, right, 32165f39d397SChris Mason btrfs_item_nr_offset(btrfs_header_nritems(left)), 32175f39d397SChris Mason btrfs_item_nr_offset(0), 32185f39d397SChris Mason push_items * sizeof(struct btrfs_item)); 32195f39d397SChris Mason 32200b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info) - 32213212fa14SJosef Bacik btrfs_item_offset(right, push_items - 1); 32225f39d397SChris Mason 32233d9ec8c4SNikolay Borisov copy_extent_buffer(left, right, BTRFS_LEAF_DATA_OFFSET + 32248f881e8cSDavid Sterba leaf_data_end(left) - push_space, 32253d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + 32263212fa14SJosef Bacik btrfs_item_offset(right, push_items - 1), 3227be0e5c09SChris Mason push_space); 32285f39d397SChris Mason old_left_nritems = btrfs_header_nritems(left); 322987b29b20SYan Zheng BUG_ON(old_left_nritems <= 0); 3230eb60ceacSChris Mason 3231c82f823cSDavid Sterba btrfs_init_map_token(&token, left); 32323212fa14SJosef Bacik old_left_item_size = btrfs_item_offset(left, old_left_nritems - 1); 3233be0e5c09SChris Mason for (i = old_left_nritems; i < old_left_nritems + push_items; i++) { 32345f39d397SChris Mason u32 ioff; 3235db94535dSChris Mason 32363212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 32373212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, 3238cc4c13d5SDavid Sterba ioff - (BTRFS_LEAF_DATA_SIZE(fs_info) - old_left_item_size)); 3239be0e5c09SChris Mason } 32405f39d397SChris Mason btrfs_set_header_nritems(left, old_left_nritems + push_items); 3241be0e5c09SChris Mason 3242be0e5c09SChris Mason /* fixup right node */ 324331b1a2bdSJulia Lawall if (push_items > right_nritems) 324431b1a2bdSJulia Lawall WARN(1, KERN_CRIT "push items %d nr %u\n", push_items, 3245d397712bSChris Mason right_nritems); 324634a38218SChris Mason 324734a38218SChris Mason if (push_items < right_nritems) { 32483212fa14SJosef Bacik push_space = btrfs_item_offset(right, push_items - 1) - 32498f881e8cSDavid Sterba leaf_data_end(right); 32503d9ec8c4SNikolay Borisov memmove_extent_buffer(right, BTRFS_LEAF_DATA_OFFSET + 32510b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - push_space, 32523d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + 32538f881e8cSDavid Sterba leaf_data_end(right), push_space); 32545f39d397SChris Mason 32555f39d397SChris Mason memmove_extent_buffer(right, btrfs_item_nr_offset(0), 32565f39d397SChris Mason btrfs_item_nr_offset(push_items), 32575f39d397SChris Mason (btrfs_header_nritems(right) - push_items) * 32580783fcfcSChris Mason sizeof(struct btrfs_item)); 325934a38218SChris Mason } 3260c82f823cSDavid Sterba 3261c82f823cSDavid Sterba btrfs_init_map_token(&token, right); 3262eef1c494SYan right_nritems -= push_items; 3263eef1c494SYan btrfs_set_header_nritems(right, right_nritems); 32640b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info); 32655f39d397SChris Mason for (i = 0; i < right_nritems; i++) { 32663212fa14SJosef Bacik push_space = push_space - btrfs_token_item_size(&token, i); 32673212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, push_space); 3268db94535dSChris Mason } 3269eb60ceacSChris Mason 32705f39d397SChris Mason btrfs_mark_buffer_dirty(left); 327134a38218SChris Mason if (right_nritems) 32725f39d397SChris Mason btrfs_mark_buffer_dirty(right); 3273f0486c68SYan, Zheng else 32746a884d7dSDavid Sterba btrfs_clean_tree_block(right); 3275098f59c2SChris Mason 32765f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 3277b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 3278be0e5c09SChris Mason 3279be0e5c09SChris Mason /* then fixup the leaf pointer in the path */ 3280be0e5c09SChris Mason if (path->slots[0] < push_items) { 3281be0e5c09SChris Mason path->slots[0] += old_left_nritems; 3282925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 32835f39d397SChris Mason free_extent_buffer(path->nodes[0]); 32845f39d397SChris Mason path->nodes[0] = left; 3285be0e5c09SChris Mason path->slots[1] -= 1; 3286be0e5c09SChris Mason } else { 3287925baeddSChris Mason btrfs_tree_unlock(left); 32885f39d397SChris Mason free_extent_buffer(left); 3289be0e5c09SChris Mason path->slots[0] -= push_items; 3290be0e5c09SChris Mason } 3291eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 3292aa5d6bedSChris Mason return ret; 3293925baeddSChris Mason out: 3294925baeddSChris Mason btrfs_tree_unlock(left); 3295925baeddSChris Mason free_extent_buffer(left); 3296925baeddSChris Mason return ret; 3297be0e5c09SChris Mason } 3298be0e5c09SChris Mason 329974123bd7SChris Mason /* 330044871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 330144871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 330299d8f83cSChris Mason * 330399d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 330499d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us push all the 330599d8f83cSChris Mason * items 330644871b1bSChris Mason */ 330744871b1bSChris Mason static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root 330899d8f83cSChris Mason *root, struct btrfs_path *path, int min_data_size, 330999d8f83cSChris Mason int data_size, int empty, u32 max_slot) 331044871b1bSChris Mason { 331144871b1bSChris Mason struct extent_buffer *right = path->nodes[0]; 331244871b1bSChris Mason struct extent_buffer *left; 331344871b1bSChris Mason int slot; 331444871b1bSChris Mason int free_space; 331544871b1bSChris Mason u32 right_nritems; 331644871b1bSChris Mason int ret = 0; 331744871b1bSChris Mason 331844871b1bSChris Mason slot = path->slots[1]; 331944871b1bSChris Mason if (slot == 0) 332044871b1bSChris Mason return 1; 332144871b1bSChris Mason if (!path->nodes[1]) 332244871b1bSChris Mason return 1; 332344871b1bSChris Mason 332444871b1bSChris Mason right_nritems = btrfs_header_nritems(right); 332544871b1bSChris Mason if (right_nritems == 0) 332644871b1bSChris Mason return 1; 332744871b1bSChris Mason 332849d0c642SFilipe Manana btrfs_assert_tree_write_locked(path->nodes[1]); 332944871b1bSChris Mason 33304b231ae4SDavid Sterba left = btrfs_read_node_slot(path->nodes[1], slot - 1); 3331fb770ae4SLiu Bo /* 3332fb770ae4SLiu Bo * slot - 1 is not valid or we fail to read the left node, 3333fb770ae4SLiu Bo * no big deal, just return. 3334fb770ae4SLiu Bo */ 3335fb770ae4SLiu Bo if (IS_ERR(left)) 333691ca338dSTsutomu Itoh return 1; 333791ca338dSTsutomu Itoh 3338bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT); 333944871b1bSChris Mason 3340e902baacSDavid Sterba free_space = btrfs_leaf_free_space(left); 334144871b1bSChris Mason if (free_space < data_size) { 334244871b1bSChris Mason ret = 1; 334344871b1bSChris Mason goto out; 334444871b1bSChris Mason } 334544871b1bSChris Mason 334644871b1bSChris Mason ret = btrfs_cow_block(trans, root, left, 33479631e4ccSJosef Bacik path->nodes[1], slot - 1, &left, 3348bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW); 334944871b1bSChris Mason if (ret) { 335044871b1bSChris Mason /* we hit -ENOSPC, but it isn't fatal here */ 335179787eaaSJeff Mahoney if (ret == -ENOSPC) 335244871b1bSChris Mason ret = 1; 335344871b1bSChris Mason goto out; 335444871b1bSChris Mason } 335544871b1bSChris Mason 3356d16c702fSQu Wenruo if (check_sibling_keys(left, right)) { 3357d16c702fSQu Wenruo ret = -EUCLEAN; 3358d16c702fSQu Wenruo goto out; 3359d16c702fSQu Wenruo } 33608087c193SDavid Sterba return __push_leaf_left(path, min_data_size, 336199d8f83cSChris Mason empty, left, free_space, right_nritems, 336299d8f83cSChris Mason max_slot); 336344871b1bSChris Mason out: 336444871b1bSChris Mason btrfs_tree_unlock(left); 336544871b1bSChris Mason free_extent_buffer(left); 336644871b1bSChris Mason return ret; 336744871b1bSChris Mason } 336844871b1bSChris Mason 336944871b1bSChris Mason /* 337074123bd7SChris Mason * split the path's leaf in two, making sure there is at least data_size 337174123bd7SChris Mason * available for the resulting leaf level of the path. 337274123bd7SChris Mason */ 3373143bede5SJeff Mahoney static noinline void copy_for_split(struct btrfs_trans_handle *trans, 337444871b1bSChris Mason struct btrfs_path *path, 337544871b1bSChris Mason struct extent_buffer *l, 337644871b1bSChris Mason struct extent_buffer *right, 337744871b1bSChris Mason int slot, int mid, int nritems) 3378be0e5c09SChris Mason { 337994f94ad9SDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info; 3380be0e5c09SChris Mason int data_copy_size; 3381be0e5c09SChris Mason int rt_data_off; 3382be0e5c09SChris Mason int i; 3383d4dbff95SChris Mason struct btrfs_disk_key disk_key; 3384cfed81a0SChris Mason struct btrfs_map_token token; 3385cfed81a0SChris Mason 33865f39d397SChris Mason nritems = nritems - mid; 33875f39d397SChris Mason btrfs_set_header_nritems(right, nritems); 3388dc2e724eSJosef Bacik data_copy_size = btrfs_item_data_end(l, mid) - leaf_data_end(l); 33895f39d397SChris Mason 33905f39d397SChris Mason copy_extent_buffer(right, l, btrfs_item_nr_offset(0), 33915f39d397SChris Mason btrfs_item_nr_offset(mid), 33925f39d397SChris Mason nritems * sizeof(struct btrfs_item)); 33935f39d397SChris Mason 33945f39d397SChris Mason copy_extent_buffer(right, l, 33953d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + BTRFS_LEAF_DATA_SIZE(fs_info) - 33963d9ec8c4SNikolay Borisov data_copy_size, BTRFS_LEAF_DATA_OFFSET + 33978f881e8cSDavid Sterba leaf_data_end(l), data_copy_size); 339874123bd7SChris Mason 3399dc2e724eSJosef Bacik rt_data_off = BTRFS_LEAF_DATA_SIZE(fs_info) - btrfs_item_data_end(l, mid); 34005f39d397SChris Mason 3401c82f823cSDavid Sterba btrfs_init_map_token(&token, right); 34025f39d397SChris Mason for (i = 0; i < nritems; i++) { 3403db94535dSChris Mason u32 ioff; 3404db94535dSChris Mason 34053212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 34063212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff + rt_data_off); 34070783fcfcSChris Mason } 340874123bd7SChris Mason 34095f39d397SChris Mason btrfs_set_header_nritems(l, mid); 34105f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 34116ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, right->start, path->slots[1] + 1, 1); 34125f39d397SChris Mason 34135f39d397SChris Mason btrfs_mark_buffer_dirty(right); 34145f39d397SChris Mason btrfs_mark_buffer_dirty(l); 3415eb60ceacSChris Mason BUG_ON(path->slots[0] != slot); 34165f39d397SChris Mason 3417be0e5c09SChris Mason if (mid <= slot) { 3418925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 34195f39d397SChris Mason free_extent_buffer(path->nodes[0]); 34205f39d397SChris Mason path->nodes[0] = right; 3421be0e5c09SChris Mason path->slots[0] -= mid; 3422be0e5c09SChris Mason path->slots[1] += 1; 3423925baeddSChris Mason } else { 3424925baeddSChris Mason btrfs_tree_unlock(right); 34255f39d397SChris Mason free_extent_buffer(right); 3426925baeddSChris Mason } 34275f39d397SChris Mason 3428eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 342944871b1bSChris Mason } 343044871b1bSChris Mason 343144871b1bSChris Mason /* 343299d8f83cSChris Mason * double splits happen when we need to insert a big item in the middle 343399d8f83cSChris Mason * of a leaf. A double split can leave us with 3 mostly empty leaves: 343499d8f83cSChris Mason * leaf: [ slots 0 - N] [ our target ] [ N + 1 - total in leaf ] 343599d8f83cSChris Mason * A B C 343699d8f83cSChris Mason * 343799d8f83cSChris Mason * We avoid this by trying to push the items on either side of our target 343899d8f83cSChris Mason * into the adjacent leaves. If all goes well we can avoid the double split 343999d8f83cSChris Mason * completely. 344099d8f83cSChris Mason */ 344199d8f83cSChris Mason static noinline int push_for_double_split(struct btrfs_trans_handle *trans, 344299d8f83cSChris Mason struct btrfs_root *root, 344399d8f83cSChris Mason struct btrfs_path *path, 344499d8f83cSChris Mason int data_size) 344599d8f83cSChris Mason { 344699d8f83cSChris Mason int ret; 344799d8f83cSChris Mason int progress = 0; 344899d8f83cSChris Mason int slot; 344999d8f83cSChris Mason u32 nritems; 34505a4267caSFilipe David Borba Manana int space_needed = data_size; 345199d8f83cSChris Mason 345299d8f83cSChris Mason slot = path->slots[0]; 34535a4267caSFilipe David Borba Manana if (slot < btrfs_header_nritems(path->nodes[0])) 3454e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(path->nodes[0]); 345599d8f83cSChris Mason 345699d8f83cSChris Mason /* 345799d8f83cSChris Mason * try to push all the items after our slot into the 345899d8f83cSChris Mason * right leaf 345999d8f83cSChris Mason */ 34605a4267caSFilipe David Borba Manana ret = push_leaf_right(trans, root, path, 1, space_needed, 0, slot); 346199d8f83cSChris Mason if (ret < 0) 346299d8f83cSChris Mason return ret; 346399d8f83cSChris Mason 346499d8f83cSChris Mason if (ret == 0) 346599d8f83cSChris Mason progress++; 346699d8f83cSChris Mason 346799d8f83cSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 346899d8f83cSChris Mason /* 346999d8f83cSChris Mason * our goal is to get our slot at the start or end of a leaf. If 347099d8f83cSChris Mason * we've done so we're done 347199d8f83cSChris Mason */ 347299d8f83cSChris Mason if (path->slots[0] == 0 || path->slots[0] == nritems) 347399d8f83cSChris Mason return 0; 347499d8f83cSChris Mason 3475e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= data_size) 347699d8f83cSChris Mason return 0; 347799d8f83cSChris Mason 347899d8f83cSChris Mason /* try to push all the items before our slot into the next leaf */ 347999d8f83cSChris Mason slot = path->slots[0]; 3480263d3995SFilipe Manana space_needed = data_size; 3481263d3995SFilipe Manana if (slot > 0) 3482e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(path->nodes[0]); 34835a4267caSFilipe David Borba Manana ret = push_leaf_left(trans, root, path, 1, space_needed, 0, slot); 348499d8f83cSChris Mason if (ret < 0) 348599d8f83cSChris Mason return ret; 348699d8f83cSChris Mason 348799d8f83cSChris Mason if (ret == 0) 348899d8f83cSChris Mason progress++; 348999d8f83cSChris Mason 349099d8f83cSChris Mason if (progress) 349199d8f83cSChris Mason return 0; 349299d8f83cSChris Mason return 1; 349399d8f83cSChris Mason } 349499d8f83cSChris Mason 349599d8f83cSChris Mason /* 349644871b1bSChris Mason * split the path's leaf in two, making sure there is at least data_size 349744871b1bSChris Mason * available for the resulting leaf level of the path. 349844871b1bSChris Mason * 349944871b1bSChris Mason * returns 0 if all went well and < 0 on failure. 350044871b1bSChris Mason */ 350144871b1bSChris Mason static noinline int split_leaf(struct btrfs_trans_handle *trans, 350244871b1bSChris Mason struct btrfs_root *root, 3503310712b2SOmar Sandoval const struct btrfs_key *ins_key, 350444871b1bSChris Mason struct btrfs_path *path, int data_size, 350544871b1bSChris Mason int extend) 350644871b1bSChris Mason { 35075d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 350844871b1bSChris Mason struct extent_buffer *l; 350944871b1bSChris Mason u32 nritems; 351044871b1bSChris Mason int mid; 351144871b1bSChris Mason int slot; 351244871b1bSChris Mason struct extent_buffer *right; 3513b7a0365eSDaniel Dressler struct btrfs_fs_info *fs_info = root->fs_info; 351444871b1bSChris Mason int ret = 0; 351544871b1bSChris Mason int wret; 35165d4f98a2SYan Zheng int split; 351744871b1bSChris Mason int num_doubles = 0; 351899d8f83cSChris Mason int tried_avoid_double = 0; 351944871b1bSChris Mason 3520a5719521SYan, Zheng l = path->nodes[0]; 3521a5719521SYan, Zheng slot = path->slots[0]; 35223212fa14SJosef Bacik if (extend && data_size + btrfs_item_size(l, slot) + 35230b246afaSJeff Mahoney sizeof(struct btrfs_item) > BTRFS_LEAF_DATA_SIZE(fs_info)) 3524a5719521SYan, Zheng return -EOVERFLOW; 3525a5719521SYan, Zheng 352644871b1bSChris Mason /* first try to make some room by pushing left and right */ 352733157e05SLiu Bo if (data_size && path->nodes[1]) { 35285a4267caSFilipe David Borba Manana int space_needed = data_size; 35295a4267caSFilipe David Borba Manana 35305a4267caSFilipe David Borba Manana if (slot < btrfs_header_nritems(l)) 3531e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(l); 35325a4267caSFilipe David Borba Manana 35335a4267caSFilipe David Borba Manana wret = push_leaf_right(trans, root, path, space_needed, 35345a4267caSFilipe David Borba Manana space_needed, 0, 0); 353544871b1bSChris Mason if (wret < 0) 353644871b1bSChris Mason return wret; 353744871b1bSChris Mason if (wret) { 3538263d3995SFilipe Manana space_needed = data_size; 3539263d3995SFilipe Manana if (slot > 0) 3540e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(l); 35415a4267caSFilipe David Borba Manana wret = push_leaf_left(trans, root, path, space_needed, 35425a4267caSFilipe David Borba Manana space_needed, 0, (u32)-1); 354344871b1bSChris Mason if (wret < 0) 354444871b1bSChris Mason return wret; 354544871b1bSChris Mason } 354644871b1bSChris Mason l = path->nodes[0]; 354744871b1bSChris Mason 354844871b1bSChris Mason /* did the pushes work? */ 3549e902baacSDavid Sterba if (btrfs_leaf_free_space(l) >= data_size) 355044871b1bSChris Mason return 0; 355144871b1bSChris Mason } 355244871b1bSChris Mason 355344871b1bSChris Mason if (!path->nodes[1]) { 3554fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, 1); 355544871b1bSChris Mason if (ret) 355644871b1bSChris Mason return ret; 355744871b1bSChris Mason } 355844871b1bSChris Mason again: 35595d4f98a2SYan Zheng split = 1; 356044871b1bSChris Mason l = path->nodes[0]; 356144871b1bSChris Mason slot = path->slots[0]; 356244871b1bSChris Mason nritems = btrfs_header_nritems(l); 356344871b1bSChris Mason mid = (nritems + 1) / 2; 356444871b1bSChris Mason 35655d4f98a2SYan Zheng if (mid <= slot) { 35665d4f98a2SYan Zheng if (nritems == 1 || 35675d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + data_size > 35680b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info)) { 35695d4f98a2SYan Zheng if (slot >= nritems) { 35705d4f98a2SYan Zheng split = 0; 35715d4f98a2SYan Zheng } else { 35725d4f98a2SYan Zheng mid = slot; 35735d4f98a2SYan Zheng if (mid != nritems && 35745d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 35750b246afaSJeff Mahoney data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) { 357699d8f83cSChris Mason if (data_size && !tried_avoid_double) 357799d8f83cSChris Mason goto push_for_double; 35785d4f98a2SYan Zheng split = 2; 35795d4f98a2SYan Zheng } 35805d4f98a2SYan Zheng } 35815d4f98a2SYan Zheng } 35825d4f98a2SYan Zheng } else { 35835d4f98a2SYan Zheng if (leaf_space_used(l, 0, mid) + data_size > 35840b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info)) { 35855d4f98a2SYan Zheng if (!extend && data_size && slot == 0) { 35865d4f98a2SYan Zheng split = 0; 35875d4f98a2SYan Zheng } else if ((extend || !data_size) && slot == 0) { 35885d4f98a2SYan Zheng mid = 1; 35895d4f98a2SYan Zheng } else { 35905d4f98a2SYan Zheng mid = slot; 35915d4f98a2SYan Zheng if (mid != nritems && 35925d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 35930b246afaSJeff Mahoney data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) { 359499d8f83cSChris Mason if (data_size && !tried_avoid_double) 359599d8f83cSChris Mason goto push_for_double; 35965d4f98a2SYan Zheng split = 2; 35975d4f98a2SYan Zheng } 35985d4f98a2SYan Zheng } 35995d4f98a2SYan Zheng } 36005d4f98a2SYan Zheng } 36015d4f98a2SYan Zheng 36025d4f98a2SYan Zheng if (split == 0) 36035d4f98a2SYan Zheng btrfs_cpu_key_to_disk(&disk_key, ins_key); 36045d4f98a2SYan Zheng else 36055d4f98a2SYan Zheng btrfs_item_key(l, &disk_key, mid); 36065d4f98a2SYan Zheng 3607ca9d473aSJosef Bacik /* 3608ca9d473aSJosef Bacik * We have to about BTRFS_NESTING_NEW_ROOT here if we've done a double 3609ca9d473aSJosef Bacik * split, because we're only allowed to have MAX_LOCKDEP_SUBCLASSES 3610ca9d473aSJosef Bacik * subclasses, which is 8 at the time of this patch, and we've maxed it 3611ca9d473aSJosef Bacik * out. In the future we could add a 3612ca9d473aSJosef Bacik * BTRFS_NESTING_SPLIT_THE_SPLITTENING if we need to, but for now just 3613ca9d473aSJosef Bacik * use BTRFS_NESTING_NEW_ROOT. 3614ca9d473aSJosef Bacik */ 361579bd3712SFilipe Manana right = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 361679bd3712SFilipe Manana &disk_key, 0, l->start, 0, 361779bd3712SFilipe Manana num_doubles ? BTRFS_NESTING_NEW_ROOT : 3618ca9d473aSJosef Bacik BTRFS_NESTING_SPLIT); 3619f0486c68SYan, Zheng if (IS_ERR(right)) 362044871b1bSChris Mason return PTR_ERR(right); 3621f0486c68SYan, Zheng 36220b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 362344871b1bSChris Mason 36245d4f98a2SYan Zheng if (split == 0) { 362544871b1bSChris Mason if (mid <= slot) { 362644871b1bSChris Mason btrfs_set_header_nritems(right, 0); 36276ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, 36282ff7e61eSJeff Mahoney right->start, path->slots[1] + 1, 1); 362944871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 363044871b1bSChris Mason free_extent_buffer(path->nodes[0]); 363144871b1bSChris Mason path->nodes[0] = right; 363244871b1bSChris Mason path->slots[0] = 0; 363344871b1bSChris Mason path->slots[1] += 1; 363444871b1bSChris Mason } else { 363544871b1bSChris Mason btrfs_set_header_nritems(right, 0); 36366ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, 36372ff7e61eSJeff Mahoney right->start, path->slots[1], 1); 363844871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 363944871b1bSChris Mason free_extent_buffer(path->nodes[0]); 364044871b1bSChris Mason path->nodes[0] = right; 364144871b1bSChris Mason path->slots[0] = 0; 3642143bede5SJeff Mahoney if (path->slots[1] == 0) 3643b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 36445d4f98a2SYan Zheng } 3645196e0249SLiu Bo /* 3646196e0249SLiu Bo * We create a new leaf 'right' for the required ins_len and 3647196e0249SLiu Bo * we'll do btrfs_mark_buffer_dirty() on this leaf after copying 3648196e0249SLiu Bo * the content of ins_len to 'right'. 3649196e0249SLiu Bo */ 365044871b1bSChris Mason return ret; 365144871b1bSChris Mason } 365244871b1bSChris Mason 365394f94ad9SDavid Sterba copy_for_split(trans, path, l, right, slot, mid, nritems); 365444871b1bSChris Mason 36555d4f98a2SYan Zheng if (split == 2) { 3656cc0c5538SChris Mason BUG_ON(num_doubles != 0); 3657cc0c5538SChris Mason num_doubles++; 3658cc0c5538SChris Mason goto again; 36593326d1b0SChris Mason } 366044871b1bSChris Mason 3661143bede5SJeff Mahoney return 0; 366299d8f83cSChris Mason 366399d8f83cSChris Mason push_for_double: 366499d8f83cSChris Mason push_for_double_split(trans, root, path, data_size); 366599d8f83cSChris Mason tried_avoid_double = 1; 3666e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= data_size) 366799d8f83cSChris Mason return 0; 366899d8f83cSChris Mason goto again; 3669be0e5c09SChris Mason } 3670be0e5c09SChris Mason 3671ad48fd75SYan, Zheng static noinline int setup_leaf_for_split(struct btrfs_trans_handle *trans, 3672ad48fd75SYan, Zheng struct btrfs_root *root, 3673ad48fd75SYan, Zheng struct btrfs_path *path, int ins_len) 3674ad48fd75SYan, Zheng { 3675ad48fd75SYan, Zheng struct btrfs_key key; 3676ad48fd75SYan, Zheng struct extent_buffer *leaf; 3677ad48fd75SYan, Zheng struct btrfs_file_extent_item *fi; 3678ad48fd75SYan, Zheng u64 extent_len = 0; 3679ad48fd75SYan, Zheng u32 item_size; 3680ad48fd75SYan, Zheng int ret; 3681ad48fd75SYan, Zheng 3682ad48fd75SYan, Zheng leaf = path->nodes[0]; 3683ad48fd75SYan, Zheng btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 3684ad48fd75SYan, Zheng 3685ad48fd75SYan, Zheng BUG_ON(key.type != BTRFS_EXTENT_DATA_KEY && 3686ad48fd75SYan, Zheng key.type != BTRFS_EXTENT_CSUM_KEY); 3687ad48fd75SYan, Zheng 3688e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) >= ins_len) 3689ad48fd75SYan, Zheng return 0; 3690ad48fd75SYan, Zheng 36913212fa14SJosef Bacik item_size = btrfs_item_size(leaf, path->slots[0]); 3692ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 3693ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 3694ad48fd75SYan, Zheng struct btrfs_file_extent_item); 3695ad48fd75SYan, Zheng extent_len = btrfs_file_extent_num_bytes(leaf, fi); 3696ad48fd75SYan, Zheng } 3697b3b4aa74SDavid Sterba btrfs_release_path(path); 3698ad48fd75SYan, Zheng 3699ad48fd75SYan, Zheng path->keep_locks = 1; 3700ad48fd75SYan, Zheng path->search_for_split = 1; 3701ad48fd75SYan, Zheng ret = btrfs_search_slot(trans, root, &key, path, 0, 1); 3702ad48fd75SYan, Zheng path->search_for_split = 0; 3703a8df6fe6SFilipe Manana if (ret > 0) 3704a8df6fe6SFilipe Manana ret = -EAGAIN; 3705ad48fd75SYan, Zheng if (ret < 0) 3706ad48fd75SYan, Zheng goto err; 3707ad48fd75SYan, Zheng 3708ad48fd75SYan, Zheng ret = -EAGAIN; 3709ad48fd75SYan, Zheng leaf = path->nodes[0]; 3710a8df6fe6SFilipe Manana /* if our item isn't there, return now */ 37113212fa14SJosef Bacik if (item_size != btrfs_item_size(leaf, path->slots[0])) 3712ad48fd75SYan, Zheng goto err; 3713ad48fd75SYan, Zheng 3714109f6aefSChris Mason /* the leaf has changed, it now has room. return now */ 3715e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= ins_len) 3716109f6aefSChris Mason goto err; 3717109f6aefSChris Mason 3718ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 3719ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 3720ad48fd75SYan, Zheng struct btrfs_file_extent_item); 3721ad48fd75SYan, Zheng if (extent_len != btrfs_file_extent_num_bytes(leaf, fi)) 3722ad48fd75SYan, Zheng goto err; 3723ad48fd75SYan, Zheng } 3724ad48fd75SYan, Zheng 3725ad48fd75SYan, Zheng ret = split_leaf(trans, root, &key, path, ins_len, 1); 3726f0486c68SYan, Zheng if (ret) 3727f0486c68SYan, Zheng goto err; 3728ad48fd75SYan, Zheng 3729ad48fd75SYan, Zheng path->keep_locks = 0; 3730ad48fd75SYan, Zheng btrfs_unlock_up_safe(path, 1); 3731ad48fd75SYan, Zheng return 0; 3732ad48fd75SYan, Zheng err: 3733ad48fd75SYan, Zheng path->keep_locks = 0; 3734ad48fd75SYan, Zheng return ret; 3735ad48fd75SYan, Zheng } 3736ad48fd75SYan, Zheng 373725263cd7SDavid Sterba static noinline int split_item(struct btrfs_path *path, 3738310712b2SOmar Sandoval const struct btrfs_key *new_key, 3739459931ecSChris Mason unsigned long split_offset) 3740459931ecSChris Mason { 3741459931ecSChris Mason struct extent_buffer *leaf; 3742c91666b1SJosef Bacik int orig_slot, slot; 3743ad48fd75SYan, Zheng char *buf; 3744459931ecSChris Mason u32 nritems; 3745ad48fd75SYan, Zheng u32 item_size; 3746459931ecSChris Mason u32 orig_offset; 3747459931ecSChris Mason struct btrfs_disk_key disk_key; 3748459931ecSChris Mason 3749459931ecSChris Mason leaf = path->nodes[0]; 3750e902baacSDavid Sterba BUG_ON(btrfs_leaf_free_space(leaf) < sizeof(struct btrfs_item)); 3751b9473439SChris Mason 3752c91666b1SJosef Bacik orig_slot = path->slots[0]; 37533212fa14SJosef Bacik orig_offset = btrfs_item_offset(leaf, path->slots[0]); 37543212fa14SJosef Bacik item_size = btrfs_item_size(leaf, path->slots[0]); 3755459931ecSChris Mason 3756459931ecSChris Mason buf = kmalloc(item_size, GFP_NOFS); 3757ad48fd75SYan, Zheng if (!buf) 3758ad48fd75SYan, Zheng return -ENOMEM; 3759ad48fd75SYan, Zheng 3760459931ecSChris Mason read_extent_buffer(leaf, buf, btrfs_item_ptr_offset(leaf, 3761459931ecSChris Mason path->slots[0]), item_size); 3762ad48fd75SYan, Zheng 3763459931ecSChris Mason slot = path->slots[0] + 1; 3764459931ecSChris Mason nritems = btrfs_header_nritems(leaf); 3765459931ecSChris Mason if (slot != nritems) { 3766459931ecSChris Mason /* shift the items */ 3767459931ecSChris Mason memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + 1), 3768459931ecSChris Mason btrfs_item_nr_offset(slot), 3769459931ecSChris Mason (nritems - slot) * sizeof(struct btrfs_item)); 3770459931ecSChris Mason } 3771459931ecSChris Mason 3772459931ecSChris Mason btrfs_cpu_key_to_disk(&disk_key, new_key); 3773459931ecSChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 3774459931ecSChris Mason 37753212fa14SJosef Bacik btrfs_set_item_offset(leaf, slot, orig_offset); 37763212fa14SJosef Bacik btrfs_set_item_size(leaf, slot, item_size - split_offset); 3777459931ecSChris Mason 37783212fa14SJosef Bacik btrfs_set_item_offset(leaf, orig_slot, 3779459931ecSChris Mason orig_offset + item_size - split_offset); 37803212fa14SJosef Bacik btrfs_set_item_size(leaf, orig_slot, split_offset); 3781459931ecSChris Mason 3782459931ecSChris Mason btrfs_set_header_nritems(leaf, nritems + 1); 3783459931ecSChris Mason 3784459931ecSChris Mason /* write the data for the start of the original item */ 3785459931ecSChris Mason write_extent_buffer(leaf, buf, 3786459931ecSChris Mason btrfs_item_ptr_offset(leaf, path->slots[0]), 3787459931ecSChris Mason split_offset); 3788459931ecSChris Mason 3789459931ecSChris Mason /* write the data for the new item */ 3790459931ecSChris Mason write_extent_buffer(leaf, buf + split_offset, 3791459931ecSChris Mason btrfs_item_ptr_offset(leaf, slot), 3792459931ecSChris Mason item_size - split_offset); 3793459931ecSChris Mason btrfs_mark_buffer_dirty(leaf); 3794459931ecSChris Mason 3795e902baacSDavid Sterba BUG_ON(btrfs_leaf_free_space(leaf) < 0); 3796459931ecSChris Mason kfree(buf); 3797ad48fd75SYan, Zheng return 0; 3798ad48fd75SYan, Zheng } 3799ad48fd75SYan, Zheng 3800ad48fd75SYan, Zheng /* 3801ad48fd75SYan, Zheng * This function splits a single item into two items, 3802ad48fd75SYan, Zheng * giving 'new_key' to the new item and splitting the 3803ad48fd75SYan, Zheng * old one at split_offset (from the start of the item). 3804ad48fd75SYan, Zheng * 3805ad48fd75SYan, Zheng * The path may be released by this operation. After 3806ad48fd75SYan, Zheng * the split, the path is pointing to the old item. The 3807ad48fd75SYan, Zheng * new item is going to be in the same node as the old one. 3808ad48fd75SYan, Zheng * 3809ad48fd75SYan, Zheng * Note, the item being split must be smaller enough to live alone on 3810ad48fd75SYan, Zheng * a tree block with room for one extra struct btrfs_item 3811ad48fd75SYan, Zheng * 3812ad48fd75SYan, Zheng * This allows us to split the item in place, keeping a lock on the 3813ad48fd75SYan, Zheng * leaf the entire time. 3814ad48fd75SYan, Zheng */ 3815ad48fd75SYan, Zheng int btrfs_split_item(struct btrfs_trans_handle *trans, 3816ad48fd75SYan, Zheng struct btrfs_root *root, 3817ad48fd75SYan, Zheng struct btrfs_path *path, 3818310712b2SOmar Sandoval const struct btrfs_key *new_key, 3819ad48fd75SYan, Zheng unsigned long split_offset) 3820ad48fd75SYan, Zheng { 3821ad48fd75SYan, Zheng int ret; 3822ad48fd75SYan, Zheng ret = setup_leaf_for_split(trans, root, path, 3823ad48fd75SYan, Zheng sizeof(struct btrfs_item)); 3824ad48fd75SYan, Zheng if (ret) 3825459931ecSChris Mason return ret; 3826ad48fd75SYan, Zheng 382725263cd7SDavid Sterba ret = split_item(path, new_key, split_offset); 3828ad48fd75SYan, Zheng return ret; 3829ad48fd75SYan, Zheng } 3830ad48fd75SYan, Zheng 3831ad48fd75SYan, Zheng /* 3832d352ac68SChris Mason * make the item pointed to by the path smaller. new_size indicates 3833d352ac68SChris Mason * how small to make it, and from_end tells us if we just chop bytes 3834d352ac68SChris Mason * off the end of the item or if we shift the item to chop bytes off 3835d352ac68SChris Mason * the front. 3836d352ac68SChris Mason */ 383778ac4f9eSDavid Sterba void btrfs_truncate_item(struct btrfs_path *path, u32 new_size, int from_end) 3838b18c6685SChris Mason { 3839b18c6685SChris Mason int slot; 38405f39d397SChris Mason struct extent_buffer *leaf; 3841b18c6685SChris Mason u32 nritems; 3842b18c6685SChris Mason unsigned int data_end; 3843b18c6685SChris Mason unsigned int old_data_start; 3844b18c6685SChris Mason unsigned int old_size; 3845b18c6685SChris Mason unsigned int size_diff; 3846b18c6685SChris Mason int i; 3847cfed81a0SChris Mason struct btrfs_map_token token; 3848cfed81a0SChris Mason 38495f39d397SChris Mason leaf = path->nodes[0]; 3850179e29e4SChris Mason slot = path->slots[0]; 3851179e29e4SChris Mason 38523212fa14SJosef Bacik old_size = btrfs_item_size(leaf, slot); 3853179e29e4SChris Mason if (old_size == new_size) 3854143bede5SJeff Mahoney return; 3855b18c6685SChris Mason 38565f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 38578f881e8cSDavid Sterba data_end = leaf_data_end(leaf); 3858b18c6685SChris Mason 38593212fa14SJosef Bacik old_data_start = btrfs_item_offset(leaf, slot); 3860179e29e4SChris Mason 3861b18c6685SChris Mason size_diff = old_size - new_size; 3862b18c6685SChris Mason 3863b18c6685SChris Mason BUG_ON(slot < 0); 3864b18c6685SChris Mason BUG_ON(slot >= nritems); 3865b18c6685SChris Mason 3866b18c6685SChris Mason /* 3867b18c6685SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 3868b18c6685SChris Mason */ 3869b18c6685SChris Mason /* first correct the data pointers */ 3870c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 3871b18c6685SChris Mason for (i = slot; i < nritems; i++) { 38725f39d397SChris Mason u32 ioff; 3873db94535dSChris Mason 38743212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 38753212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff + size_diff); 3876b18c6685SChris Mason } 3877db94535dSChris Mason 3878b18c6685SChris Mason /* shift the data */ 3879179e29e4SChris Mason if (from_end) { 38803d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 38813d9ec8c4SNikolay Borisov data_end + size_diff, BTRFS_LEAF_DATA_OFFSET + 3882b18c6685SChris Mason data_end, old_data_start + new_size - data_end); 3883179e29e4SChris Mason } else { 3884179e29e4SChris Mason struct btrfs_disk_key disk_key; 3885179e29e4SChris Mason u64 offset; 3886179e29e4SChris Mason 3887179e29e4SChris Mason btrfs_item_key(leaf, &disk_key, slot); 3888179e29e4SChris Mason 3889179e29e4SChris Mason if (btrfs_disk_key_type(&disk_key) == BTRFS_EXTENT_DATA_KEY) { 3890179e29e4SChris Mason unsigned long ptr; 3891179e29e4SChris Mason struct btrfs_file_extent_item *fi; 3892179e29e4SChris Mason 3893179e29e4SChris Mason fi = btrfs_item_ptr(leaf, slot, 3894179e29e4SChris Mason struct btrfs_file_extent_item); 3895179e29e4SChris Mason fi = (struct btrfs_file_extent_item *)( 3896179e29e4SChris Mason (unsigned long)fi - size_diff); 3897179e29e4SChris Mason 3898179e29e4SChris Mason if (btrfs_file_extent_type(leaf, fi) == 3899179e29e4SChris Mason BTRFS_FILE_EXTENT_INLINE) { 3900179e29e4SChris Mason ptr = btrfs_item_ptr_offset(leaf, slot); 3901179e29e4SChris Mason memmove_extent_buffer(leaf, ptr, 3902179e29e4SChris Mason (unsigned long)fi, 39037ec20afbSDavid Sterba BTRFS_FILE_EXTENT_INLINE_DATA_START); 3904179e29e4SChris Mason } 3905179e29e4SChris Mason } 3906179e29e4SChris Mason 39073d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 39083d9ec8c4SNikolay Borisov data_end + size_diff, BTRFS_LEAF_DATA_OFFSET + 3909179e29e4SChris Mason data_end, old_data_start - data_end); 3910179e29e4SChris Mason 3911179e29e4SChris Mason offset = btrfs_disk_key_offset(&disk_key); 3912179e29e4SChris Mason btrfs_set_disk_key_offset(&disk_key, offset + size_diff); 3913179e29e4SChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 3914179e29e4SChris Mason if (slot == 0) 3915b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 3916179e29e4SChris Mason } 39175f39d397SChris Mason 39183212fa14SJosef Bacik btrfs_set_item_size(leaf, slot, new_size); 39195f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 3920b18c6685SChris Mason 3921e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) { 3922a4f78750SDavid Sterba btrfs_print_leaf(leaf); 3923b18c6685SChris Mason BUG(); 39245f39d397SChris Mason } 3925b18c6685SChris Mason } 3926b18c6685SChris Mason 3927d352ac68SChris Mason /* 39288f69dbd2SStefan Behrens * make the item pointed to by the path bigger, data_size is the added size. 3929d352ac68SChris Mason */ 3930c71dd880SDavid Sterba void btrfs_extend_item(struct btrfs_path *path, u32 data_size) 39316567e837SChris Mason { 39326567e837SChris Mason int slot; 39335f39d397SChris Mason struct extent_buffer *leaf; 39346567e837SChris Mason u32 nritems; 39356567e837SChris Mason unsigned int data_end; 39366567e837SChris Mason unsigned int old_data; 39376567e837SChris Mason unsigned int old_size; 39386567e837SChris Mason int i; 3939cfed81a0SChris Mason struct btrfs_map_token token; 3940cfed81a0SChris Mason 39415f39d397SChris Mason leaf = path->nodes[0]; 39426567e837SChris Mason 39435f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 39448f881e8cSDavid Sterba data_end = leaf_data_end(leaf); 39456567e837SChris Mason 3946e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < data_size) { 3947a4f78750SDavid Sterba btrfs_print_leaf(leaf); 39486567e837SChris Mason BUG(); 39495f39d397SChris Mason } 39506567e837SChris Mason slot = path->slots[0]; 3951dc2e724eSJosef Bacik old_data = btrfs_item_data_end(leaf, slot); 39526567e837SChris Mason 39536567e837SChris Mason BUG_ON(slot < 0); 39543326d1b0SChris Mason if (slot >= nritems) { 3955a4f78750SDavid Sterba btrfs_print_leaf(leaf); 3956c71dd880SDavid Sterba btrfs_crit(leaf->fs_info, "slot %d too large, nritems %d", 3957d397712bSChris Mason slot, nritems); 3958290342f6SArnd Bergmann BUG(); 39593326d1b0SChris Mason } 39606567e837SChris Mason 39616567e837SChris Mason /* 39626567e837SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 39636567e837SChris Mason */ 39646567e837SChris Mason /* first correct the data pointers */ 3965c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 39666567e837SChris Mason for (i = slot; i < nritems; i++) { 39675f39d397SChris Mason u32 ioff; 3968db94535dSChris Mason 39693212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 39703212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff - data_size); 39716567e837SChris Mason } 39725f39d397SChris Mason 39736567e837SChris Mason /* shift the data */ 39743d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 39753d9ec8c4SNikolay Borisov data_end - data_size, BTRFS_LEAF_DATA_OFFSET + 39766567e837SChris Mason data_end, old_data - data_end); 39775f39d397SChris Mason 39786567e837SChris Mason data_end = old_data; 39793212fa14SJosef Bacik old_size = btrfs_item_size(leaf, slot); 39803212fa14SJosef Bacik btrfs_set_item_size(leaf, slot, old_size + data_size); 39815f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 39826567e837SChris Mason 3983e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) { 3984a4f78750SDavid Sterba btrfs_print_leaf(leaf); 39856567e837SChris Mason BUG(); 39865f39d397SChris Mason } 39876567e837SChris Mason } 39886567e837SChris Mason 398943dd529aSDavid Sterba /* 399043dd529aSDavid Sterba * Make space in the node before inserting one or more items. 3991da9ffb24SNikolay Borisov * 3992da9ffb24SNikolay Borisov * @root: root we are inserting items to 3993da9ffb24SNikolay Borisov * @path: points to the leaf/slot where we are going to insert new items 3994b7ef5f3aSFilipe Manana * @batch: information about the batch of items to insert 399543dd529aSDavid Sterba * 399643dd529aSDavid Sterba * Main purpose is to save stack depth by doing the bulk of the work in a 399743dd529aSDavid Sterba * function that doesn't call btrfs_search_slot 399874123bd7SChris Mason */ 3999f0641656SFilipe Manana static void setup_items_for_insert(struct btrfs_root *root, struct btrfs_path *path, 4000b7ef5f3aSFilipe Manana const struct btrfs_item_batch *batch) 4001be0e5c09SChris Mason { 40020b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 40039c58309dSChris Mason int i; 40047518a238SChris Mason u32 nritems; 4005be0e5c09SChris Mason unsigned int data_end; 4006e2fa7227SChris Mason struct btrfs_disk_key disk_key; 400744871b1bSChris Mason struct extent_buffer *leaf; 400844871b1bSChris Mason int slot; 4009cfed81a0SChris Mason struct btrfs_map_token token; 4010fc0d82e1SNikolay Borisov u32 total_size; 4011fc0d82e1SNikolay Borisov 4012b7ef5f3aSFilipe Manana /* 4013b7ef5f3aSFilipe Manana * Before anything else, update keys in the parent and other ancestors 4014b7ef5f3aSFilipe Manana * if needed, then release the write locks on them, so that other tasks 4015b7ef5f3aSFilipe Manana * can use them while we modify the leaf. 4016b7ef5f3aSFilipe Manana */ 401724cdc847SFilipe Manana if (path->slots[0] == 0) { 4018b7ef5f3aSFilipe Manana btrfs_cpu_key_to_disk(&disk_key, &batch->keys[0]); 4019b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 402024cdc847SFilipe Manana } 402124cdc847SFilipe Manana btrfs_unlock_up_safe(path, 1); 402224cdc847SFilipe Manana 40235f39d397SChris Mason leaf = path->nodes[0]; 402444871b1bSChris Mason slot = path->slots[0]; 402574123bd7SChris Mason 40265f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 40278f881e8cSDavid Sterba data_end = leaf_data_end(leaf); 4028b7ef5f3aSFilipe Manana total_size = batch->total_data_size + (batch->nr * sizeof(struct btrfs_item)); 4029eb60ceacSChris Mason 4030e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < total_size) { 4031a4f78750SDavid Sterba btrfs_print_leaf(leaf); 40320b246afaSJeff Mahoney btrfs_crit(fs_info, "not enough freespace need %u have %d", 4033e902baacSDavid Sterba total_size, btrfs_leaf_free_space(leaf)); 4034be0e5c09SChris Mason BUG(); 4035d4dbff95SChris Mason } 40365f39d397SChris Mason 4037c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 4038be0e5c09SChris Mason if (slot != nritems) { 4039dc2e724eSJosef Bacik unsigned int old_data = btrfs_item_data_end(leaf, slot); 4040be0e5c09SChris Mason 40415f39d397SChris Mason if (old_data < data_end) { 4042a4f78750SDavid Sterba btrfs_print_leaf(leaf); 40437269ddd2SNikolay Borisov btrfs_crit(fs_info, 40447269ddd2SNikolay Borisov "item at slot %d with data offset %u beyond data end of leaf %u", 40455f39d397SChris Mason slot, old_data, data_end); 4046290342f6SArnd Bergmann BUG(); 40475f39d397SChris Mason } 4048be0e5c09SChris Mason /* 4049be0e5c09SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 4050be0e5c09SChris Mason */ 4051be0e5c09SChris Mason /* first correct the data pointers */ 40520783fcfcSChris Mason for (i = slot; i < nritems; i++) { 40535f39d397SChris Mason u32 ioff; 4054db94535dSChris Mason 40553212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 40563212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, 4057b7ef5f3aSFilipe Manana ioff - batch->total_data_size); 40580783fcfcSChris Mason } 4059be0e5c09SChris Mason /* shift the items */ 4060b7ef5f3aSFilipe Manana memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + batch->nr), 40615f39d397SChris Mason btrfs_item_nr_offset(slot), 40620783fcfcSChris Mason (nritems - slot) * sizeof(struct btrfs_item)); 4063be0e5c09SChris Mason 4064be0e5c09SChris Mason /* shift the data */ 40653d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 4066b7ef5f3aSFilipe Manana data_end - batch->total_data_size, 4067b7ef5f3aSFilipe Manana BTRFS_LEAF_DATA_OFFSET + data_end, 4068b7ef5f3aSFilipe Manana old_data - data_end); 4069be0e5c09SChris Mason data_end = old_data; 4070be0e5c09SChris Mason } 40715f39d397SChris Mason 407262e2749eSChris Mason /* setup the item for the new data */ 4073b7ef5f3aSFilipe Manana for (i = 0; i < batch->nr; i++) { 4074b7ef5f3aSFilipe Manana btrfs_cpu_key_to_disk(&disk_key, &batch->keys[i]); 40759c58309dSChris Mason btrfs_set_item_key(leaf, &disk_key, slot + i); 4076b7ef5f3aSFilipe Manana data_end -= batch->data_sizes[i]; 40773212fa14SJosef Bacik btrfs_set_token_item_offset(&token, slot + i, data_end); 40783212fa14SJosef Bacik btrfs_set_token_item_size(&token, slot + i, batch->data_sizes[i]); 40799c58309dSChris Mason } 408044871b1bSChris Mason 4081b7ef5f3aSFilipe Manana btrfs_set_header_nritems(leaf, nritems + batch->nr); 4082b9473439SChris Mason btrfs_mark_buffer_dirty(leaf); 4083aa5d6bedSChris Mason 4084e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) { 4085a4f78750SDavid Sterba btrfs_print_leaf(leaf); 4086be0e5c09SChris Mason BUG(); 40875f39d397SChris Mason } 408844871b1bSChris Mason } 408944871b1bSChris Mason 409044871b1bSChris Mason /* 4091f0641656SFilipe Manana * Insert a new item into a leaf. 4092f0641656SFilipe Manana * 4093f0641656SFilipe Manana * @root: The root of the btree. 4094f0641656SFilipe Manana * @path: A path pointing to the target leaf and slot. 4095f0641656SFilipe Manana * @key: The key of the new item. 4096f0641656SFilipe Manana * @data_size: The size of the data associated with the new key. 4097f0641656SFilipe Manana */ 4098f0641656SFilipe Manana void btrfs_setup_item_for_insert(struct btrfs_root *root, 4099f0641656SFilipe Manana struct btrfs_path *path, 4100f0641656SFilipe Manana const struct btrfs_key *key, 4101f0641656SFilipe Manana u32 data_size) 4102f0641656SFilipe Manana { 4103f0641656SFilipe Manana struct btrfs_item_batch batch; 4104f0641656SFilipe Manana 4105f0641656SFilipe Manana batch.keys = key; 4106f0641656SFilipe Manana batch.data_sizes = &data_size; 4107f0641656SFilipe Manana batch.total_data_size = data_size; 4108f0641656SFilipe Manana batch.nr = 1; 4109f0641656SFilipe Manana 4110f0641656SFilipe Manana setup_items_for_insert(root, path, &batch); 4111f0641656SFilipe Manana } 4112f0641656SFilipe Manana 4113f0641656SFilipe Manana /* 411444871b1bSChris Mason * Given a key and some data, insert items into the tree. 411544871b1bSChris Mason * This does all the path init required, making room in the tree if needed. 411644871b1bSChris Mason */ 411744871b1bSChris Mason int btrfs_insert_empty_items(struct btrfs_trans_handle *trans, 411844871b1bSChris Mason struct btrfs_root *root, 411944871b1bSChris Mason struct btrfs_path *path, 4120b7ef5f3aSFilipe Manana const struct btrfs_item_batch *batch) 412144871b1bSChris Mason { 412244871b1bSChris Mason int ret = 0; 412344871b1bSChris Mason int slot; 4124b7ef5f3aSFilipe Manana u32 total_size; 412544871b1bSChris Mason 4126b7ef5f3aSFilipe Manana total_size = batch->total_data_size + (batch->nr * sizeof(struct btrfs_item)); 4127b7ef5f3aSFilipe Manana ret = btrfs_search_slot(trans, root, &batch->keys[0], path, total_size, 1); 412844871b1bSChris Mason if (ret == 0) 412944871b1bSChris Mason return -EEXIST; 413044871b1bSChris Mason if (ret < 0) 4131143bede5SJeff Mahoney return ret; 413244871b1bSChris Mason 413344871b1bSChris Mason slot = path->slots[0]; 413444871b1bSChris Mason BUG_ON(slot < 0); 413544871b1bSChris Mason 4136b7ef5f3aSFilipe Manana setup_items_for_insert(root, path, batch); 4137143bede5SJeff Mahoney return 0; 413862e2749eSChris Mason } 413962e2749eSChris Mason 414062e2749eSChris Mason /* 414162e2749eSChris Mason * Given a key and some data, insert an item into the tree. 414262e2749eSChris Mason * This does all the path init required, making room in the tree if needed. 414362e2749eSChris Mason */ 4144310712b2SOmar Sandoval int btrfs_insert_item(struct btrfs_trans_handle *trans, struct btrfs_root *root, 4145310712b2SOmar Sandoval const struct btrfs_key *cpu_key, void *data, 4146310712b2SOmar Sandoval u32 data_size) 414762e2749eSChris Mason { 414862e2749eSChris Mason int ret = 0; 41492c90e5d6SChris Mason struct btrfs_path *path; 41505f39d397SChris Mason struct extent_buffer *leaf; 41515f39d397SChris Mason unsigned long ptr; 415262e2749eSChris Mason 41532c90e5d6SChris Mason path = btrfs_alloc_path(); 4154db5b493aSTsutomu Itoh if (!path) 4155db5b493aSTsutomu Itoh return -ENOMEM; 41562c90e5d6SChris Mason ret = btrfs_insert_empty_item(trans, root, path, cpu_key, data_size); 415762e2749eSChris Mason if (!ret) { 41585f39d397SChris Mason leaf = path->nodes[0]; 41595f39d397SChris Mason ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); 41605f39d397SChris Mason write_extent_buffer(leaf, data, ptr, data_size); 41615f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 416262e2749eSChris Mason } 41632c90e5d6SChris Mason btrfs_free_path(path); 4164aa5d6bedSChris Mason return ret; 4165be0e5c09SChris Mason } 4166be0e5c09SChris Mason 416774123bd7SChris Mason /* 4168f0641656SFilipe Manana * This function duplicates an item, giving 'new_key' to the new item. 4169f0641656SFilipe Manana * It guarantees both items live in the same tree leaf and the new item is 4170f0641656SFilipe Manana * contiguous with the original item. 4171f0641656SFilipe Manana * 4172f0641656SFilipe Manana * This allows us to split a file extent in place, keeping a lock on the leaf 4173f0641656SFilipe Manana * the entire time. 4174f0641656SFilipe Manana */ 4175f0641656SFilipe Manana int btrfs_duplicate_item(struct btrfs_trans_handle *trans, 4176f0641656SFilipe Manana struct btrfs_root *root, 4177f0641656SFilipe Manana struct btrfs_path *path, 4178f0641656SFilipe Manana const struct btrfs_key *new_key) 4179f0641656SFilipe Manana { 4180f0641656SFilipe Manana struct extent_buffer *leaf; 4181f0641656SFilipe Manana int ret; 4182f0641656SFilipe Manana u32 item_size; 4183f0641656SFilipe Manana 4184f0641656SFilipe Manana leaf = path->nodes[0]; 41853212fa14SJosef Bacik item_size = btrfs_item_size(leaf, path->slots[0]); 4186f0641656SFilipe Manana ret = setup_leaf_for_split(trans, root, path, 4187f0641656SFilipe Manana item_size + sizeof(struct btrfs_item)); 4188f0641656SFilipe Manana if (ret) 4189f0641656SFilipe Manana return ret; 4190f0641656SFilipe Manana 4191f0641656SFilipe Manana path->slots[0]++; 4192f0641656SFilipe Manana btrfs_setup_item_for_insert(root, path, new_key, item_size); 4193f0641656SFilipe Manana leaf = path->nodes[0]; 4194f0641656SFilipe Manana memcpy_extent_buffer(leaf, 4195f0641656SFilipe Manana btrfs_item_ptr_offset(leaf, path->slots[0]), 4196f0641656SFilipe Manana btrfs_item_ptr_offset(leaf, path->slots[0] - 1), 4197f0641656SFilipe Manana item_size); 4198f0641656SFilipe Manana return 0; 4199f0641656SFilipe Manana } 4200f0641656SFilipe Manana 4201f0641656SFilipe Manana /* 42025de08d7dSChris Mason * delete the pointer from a given node. 420374123bd7SChris Mason * 4204d352ac68SChris Mason * the tree should have been previously balanced so the deletion does not 4205d352ac68SChris Mason * empty a node. 420674123bd7SChris Mason */ 4207afe5fea7STsutomu Itoh static void del_ptr(struct btrfs_root *root, struct btrfs_path *path, 4208afe5fea7STsutomu Itoh int level, int slot) 4209be0e5c09SChris Mason { 42105f39d397SChris Mason struct extent_buffer *parent = path->nodes[level]; 42117518a238SChris Mason u32 nritems; 4212f3ea38daSJan Schmidt int ret; 4213be0e5c09SChris Mason 42145f39d397SChris Mason nritems = btrfs_header_nritems(parent); 4215be0e5c09SChris Mason if (slot != nritems - 1) { 4216bf1d3425SDavid Sterba if (level) { 4217f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_move(parent, slot, 4218f3a84ccdSFilipe Manana slot + 1, nritems - slot - 1); 4219bf1d3425SDavid Sterba BUG_ON(ret < 0); 4220bf1d3425SDavid Sterba } 42215f39d397SChris Mason memmove_extent_buffer(parent, 42225f39d397SChris Mason btrfs_node_key_ptr_offset(slot), 42235f39d397SChris Mason btrfs_node_key_ptr_offset(slot + 1), 4224d6025579SChris Mason sizeof(struct btrfs_key_ptr) * 4225d6025579SChris Mason (nritems - slot - 1)); 422657ba86c0SChris Mason } else if (level) { 4227f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, slot, 422833cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REMOVE); 422957ba86c0SChris Mason BUG_ON(ret < 0); 4230be0e5c09SChris Mason } 4231f3ea38daSJan Schmidt 42327518a238SChris Mason nritems--; 42335f39d397SChris Mason btrfs_set_header_nritems(parent, nritems); 42347518a238SChris Mason if (nritems == 0 && parent == root->node) { 42355f39d397SChris Mason BUG_ON(btrfs_header_level(root->node) != 1); 4236eb60ceacSChris Mason /* just turn the root into a leaf and break */ 42375f39d397SChris Mason btrfs_set_header_level(root->node, 0); 4238bb803951SChris Mason } else if (slot == 0) { 42395f39d397SChris Mason struct btrfs_disk_key disk_key; 42405f39d397SChris Mason 42415f39d397SChris Mason btrfs_node_key(parent, &disk_key, 0); 4242b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, level + 1); 4243be0e5c09SChris Mason } 4244d6025579SChris Mason btrfs_mark_buffer_dirty(parent); 4245be0e5c09SChris Mason } 4246be0e5c09SChris Mason 424774123bd7SChris Mason /* 4248323ac95bSChris Mason * a helper function to delete the leaf pointed to by path->slots[1] and 42495d4f98a2SYan Zheng * path->nodes[1]. 4250323ac95bSChris Mason * 4251323ac95bSChris Mason * This deletes the pointer in path->nodes[1] and frees the leaf 4252323ac95bSChris Mason * block extent. zero is returned if it all worked out, < 0 otherwise. 4253323ac95bSChris Mason * 4254323ac95bSChris Mason * The path must have already been setup for deleting the leaf, including 4255323ac95bSChris Mason * all the proper balancing. path->nodes[1] must be locked. 4256323ac95bSChris Mason */ 4257143bede5SJeff Mahoney static noinline void btrfs_del_leaf(struct btrfs_trans_handle *trans, 4258323ac95bSChris Mason struct btrfs_root *root, 42595d4f98a2SYan Zheng struct btrfs_path *path, 42605d4f98a2SYan Zheng struct extent_buffer *leaf) 4261323ac95bSChris Mason { 42625d4f98a2SYan Zheng WARN_ON(btrfs_header_generation(leaf) != trans->transid); 4263afe5fea7STsutomu Itoh del_ptr(root, path, 1, path->slots[1]); 4264323ac95bSChris Mason 42654d081c41SChris Mason /* 42664d081c41SChris Mason * btrfs_free_extent is expensive, we want to make sure we 42674d081c41SChris Mason * aren't holding any locks when we call it 42684d081c41SChris Mason */ 42694d081c41SChris Mason btrfs_unlock_up_safe(path, 0); 42704d081c41SChris Mason 4271f0486c68SYan, Zheng root_sub_used(root, leaf->len); 4272f0486c68SYan, Zheng 427367439dadSDavid Sterba atomic_inc(&leaf->refs); 42747a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), leaf, 0, 1); 42753083ee2eSJosef Bacik free_extent_buffer_stale(leaf); 4276323ac95bSChris Mason } 4277323ac95bSChris Mason /* 427874123bd7SChris Mason * delete the item at the leaf level in path. If that empties 427974123bd7SChris Mason * the leaf, remove it from the tree 428074123bd7SChris Mason */ 428185e21bacSChris Mason int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root, 428285e21bacSChris Mason struct btrfs_path *path, int slot, int nr) 4283be0e5c09SChris Mason { 42840b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 42855f39d397SChris Mason struct extent_buffer *leaf; 4286aa5d6bedSChris Mason int ret = 0; 4287aa5d6bedSChris Mason int wret; 42887518a238SChris Mason u32 nritems; 4289be0e5c09SChris Mason 42905f39d397SChris Mason leaf = path->nodes[0]; 42915f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 4292be0e5c09SChris Mason 429385e21bacSChris Mason if (slot + nr != nritems) { 42940cae23b6SFilipe Manana const u32 last_off = btrfs_item_offset(leaf, slot + nr - 1); 42950cae23b6SFilipe Manana const int data_end = leaf_data_end(leaf); 4296c82f823cSDavid Sterba struct btrfs_map_token token; 42970cae23b6SFilipe Manana u32 dsize = 0; 42980cae23b6SFilipe Manana int i; 42990cae23b6SFilipe Manana 43000cae23b6SFilipe Manana for (i = 0; i < nr; i++) 43010cae23b6SFilipe Manana dsize += btrfs_item_size(leaf, slot + i); 43025f39d397SChris Mason 43033d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 4304d6025579SChris Mason data_end + dsize, 43053d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + data_end, 430685e21bacSChris Mason last_off - data_end); 43075f39d397SChris Mason 4308c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 430985e21bacSChris Mason for (i = slot + nr; i < nritems; i++) { 43105f39d397SChris Mason u32 ioff; 4311db94535dSChris Mason 43123212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 43133212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff + dsize); 43140783fcfcSChris Mason } 4315db94535dSChris Mason 43165f39d397SChris Mason memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot), 431785e21bacSChris Mason btrfs_item_nr_offset(slot + nr), 43180783fcfcSChris Mason sizeof(struct btrfs_item) * 431985e21bacSChris Mason (nritems - slot - nr)); 4320be0e5c09SChris Mason } 432185e21bacSChris Mason btrfs_set_header_nritems(leaf, nritems - nr); 432285e21bacSChris Mason nritems -= nr; 43235f39d397SChris Mason 432474123bd7SChris Mason /* delete the leaf if we've emptied it */ 43257518a238SChris Mason if (nritems == 0) { 43265f39d397SChris Mason if (leaf == root->node) { 43275f39d397SChris Mason btrfs_set_header_level(leaf, 0); 43289a8dd150SChris Mason } else { 43296a884d7dSDavid Sterba btrfs_clean_tree_block(leaf); 4330143bede5SJeff Mahoney btrfs_del_leaf(trans, root, path, leaf); 43319a8dd150SChris Mason } 4332be0e5c09SChris Mason } else { 43337518a238SChris Mason int used = leaf_space_used(leaf, 0, nritems); 4334aa5d6bedSChris Mason if (slot == 0) { 43355f39d397SChris Mason struct btrfs_disk_key disk_key; 43365f39d397SChris Mason 43375f39d397SChris Mason btrfs_item_key(leaf, &disk_key, 0); 4338b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 4339aa5d6bedSChris Mason } 4340aa5d6bedSChris Mason 43417c4063d1SFilipe Manana /* 43427c4063d1SFilipe Manana * Try to delete the leaf if it is mostly empty. We do this by 43437c4063d1SFilipe Manana * trying to move all its items into its left and right neighbours. 43447c4063d1SFilipe Manana * If we can't move all the items, then we don't delete it - it's 43457c4063d1SFilipe Manana * not ideal, but future insertions might fill the leaf with more 43467c4063d1SFilipe Manana * items, or items from other leaves might be moved later into our 43477c4063d1SFilipe Manana * leaf due to deletions on those leaves. 43487c4063d1SFilipe Manana */ 43490b246afaSJeff Mahoney if (used < BTRFS_LEAF_DATA_SIZE(fs_info) / 3) { 43507c4063d1SFilipe Manana u32 min_push_space; 43517c4063d1SFilipe Manana 4352be0e5c09SChris Mason /* push_leaf_left fixes the path. 4353be0e5c09SChris Mason * make sure the path still points to our leaf 4354be0e5c09SChris Mason * for possible call to del_ptr below 4355be0e5c09SChris Mason */ 43564920c9acSChris Mason slot = path->slots[1]; 435767439dadSDavid Sterba atomic_inc(&leaf->refs); 43587c4063d1SFilipe Manana /* 43597c4063d1SFilipe Manana * We want to be able to at least push one item to the 43607c4063d1SFilipe Manana * left neighbour leaf, and that's the first item. 43617c4063d1SFilipe Manana */ 43627c4063d1SFilipe Manana min_push_space = sizeof(struct btrfs_item) + 43637c4063d1SFilipe Manana btrfs_item_size(leaf, 0); 43647c4063d1SFilipe Manana wret = push_leaf_left(trans, root, path, 0, 43657c4063d1SFilipe Manana min_push_space, 1, (u32)-1); 436654aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 4367aa5d6bedSChris Mason ret = wret; 43685f39d397SChris Mason 43695f39d397SChris Mason if (path->nodes[0] == leaf && 43705f39d397SChris Mason btrfs_header_nritems(leaf)) { 43717c4063d1SFilipe Manana /* 43727c4063d1SFilipe Manana * If we were not able to push all items from our 43737c4063d1SFilipe Manana * leaf to its left neighbour, then attempt to 43747c4063d1SFilipe Manana * either push all the remaining items to the 43757c4063d1SFilipe Manana * right neighbour or none. There's no advantage 43767c4063d1SFilipe Manana * in pushing only some items, instead of all, as 43777c4063d1SFilipe Manana * it's pointless to end up with a leaf having 43787c4063d1SFilipe Manana * too few items while the neighbours can be full 43797c4063d1SFilipe Manana * or nearly full. 43807c4063d1SFilipe Manana */ 43817c4063d1SFilipe Manana nritems = btrfs_header_nritems(leaf); 43827c4063d1SFilipe Manana min_push_space = leaf_space_used(leaf, 0, nritems); 43837c4063d1SFilipe Manana wret = push_leaf_right(trans, root, path, 0, 43847c4063d1SFilipe Manana min_push_space, 1, 0); 438554aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 4386aa5d6bedSChris Mason ret = wret; 4387aa5d6bedSChris Mason } 43885f39d397SChris Mason 43895f39d397SChris Mason if (btrfs_header_nritems(leaf) == 0) { 4390323ac95bSChris Mason path->slots[1] = slot; 4391143bede5SJeff Mahoney btrfs_del_leaf(trans, root, path, leaf); 43925f39d397SChris Mason free_extent_buffer(leaf); 4393143bede5SJeff Mahoney ret = 0; 43945de08d7dSChris Mason } else { 4395925baeddSChris Mason /* if we're still in the path, make sure 4396925baeddSChris Mason * we're dirty. Otherwise, one of the 4397925baeddSChris Mason * push_leaf functions must have already 4398925baeddSChris Mason * dirtied this buffer 4399925baeddSChris Mason */ 4400925baeddSChris Mason if (path->nodes[0] == leaf) 44015f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 44025f39d397SChris Mason free_extent_buffer(leaf); 4403be0e5c09SChris Mason } 4404d5719762SChris Mason } else { 44055f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 4406be0e5c09SChris Mason } 44079a8dd150SChris Mason } 4408aa5d6bedSChris Mason return ret; 44099a8dd150SChris Mason } 44109a8dd150SChris Mason 441197571fd0SChris Mason /* 4412925baeddSChris Mason * search the tree again to find a leaf with lesser keys 44137bb86316SChris Mason * returns 0 if it found something or 1 if there are no lesser leaves. 44147bb86316SChris Mason * returns < 0 on io errors. 4415d352ac68SChris Mason * 4416d352ac68SChris Mason * This may release the path, and so you may lose any locks held at the 4417d352ac68SChris Mason * time you call it. 44187bb86316SChris Mason */ 441916e7549fSJosef Bacik int btrfs_prev_leaf(struct btrfs_root *root, struct btrfs_path *path) 44207bb86316SChris Mason { 4421925baeddSChris Mason struct btrfs_key key; 4422925baeddSChris Mason struct btrfs_disk_key found_key; 4423925baeddSChris Mason int ret; 44247bb86316SChris Mason 4425925baeddSChris Mason btrfs_item_key_to_cpu(path->nodes[0], &key, 0); 4426925baeddSChris Mason 4427e8b0d724SFilipe David Borba Manana if (key.offset > 0) { 4428925baeddSChris Mason key.offset--; 4429e8b0d724SFilipe David Borba Manana } else if (key.type > 0) { 4430925baeddSChris Mason key.type--; 4431e8b0d724SFilipe David Borba Manana key.offset = (u64)-1; 4432e8b0d724SFilipe David Borba Manana } else if (key.objectid > 0) { 4433925baeddSChris Mason key.objectid--; 4434e8b0d724SFilipe David Borba Manana key.type = (u8)-1; 4435e8b0d724SFilipe David Borba Manana key.offset = (u64)-1; 4436e8b0d724SFilipe David Borba Manana } else { 44377bb86316SChris Mason return 1; 4438e8b0d724SFilipe David Borba Manana } 44397bb86316SChris Mason 4440b3b4aa74SDavid Sterba btrfs_release_path(path); 4441925baeddSChris Mason ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 4442925baeddSChris Mason if (ret < 0) 4443925baeddSChris Mason return ret; 4444925baeddSChris Mason btrfs_item_key(path->nodes[0], &found_key, 0); 4445925baeddSChris Mason ret = comp_keys(&found_key, &key); 4446337c6f68SFilipe Manana /* 4447337c6f68SFilipe Manana * We might have had an item with the previous key in the tree right 4448337c6f68SFilipe Manana * before we released our path. And after we released our path, that 4449337c6f68SFilipe Manana * item might have been pushed to the first slot (0) of the leaf we 4450337c6f68SFilipe Manana * were holding due to a tree balance. Alternatively, an item with the 4451337c6f68SFilipe Manana * previous key can exist as the only element of a leaf (big fat item). 4452337c6f68SFilipe Manana * Therefore account for these 2 cases, so that our callers (like 4453337c6f68SFilipe Manana * btrfs_previous_item) don't miss an existing item with a key matching 4454337c6f68SFilipe Manana * the previous key we computed above. 4455337c6f68SFilipe Manana */ 4456337c6f68SFilipe Manana if (ret <= 0) 44577bb86316SChris Mason return 0; 4458925baeddSChris Mason return 1; 44597bb86316SChris Mason } 44607bb86316SChris Mason 44613f157a2fSChris Mason /* 44623f157a2fSChris Mason * A helper function to walk down the tree starting at min_key, and looking 4463de78b51aSEric Sandeen * for nodes or leaves that are have a minimum transaction id. 4464de78b51aSEric Sandeen * This is used by the btree defrag code, and tree logging 44653f157a2fSChris Mason * 44663f157a2fSChris Mason * This does not cow, but it does stuff the starting key it finds back 44673f157a2fSChris Mason * into min_key, so you can call btrfs_search_slot with cow=1 on the 44683f157a2fSChris Mason * key and get a writable path. 44693f157a2fSChris Mason * 44703f157a2fSChris Mason * This honors path->lowest_level to prevent descent past a given level 44713f157a2fSChris Mason * of the tree. 44723f157a2fSChris Mason * 4473d352ac68SChris Mason * min_trans indicates the oldest transaction that you are interested 4474d352ac68SChris Mason * in walking through. Any nodes or leaves older than min_trans are 4475d352ac68SChris Mason * skipped over (without reading them). 4476d352ac68SChris Mason * 44773f157a2fSChris Mason * returns zero if something useful was found, < 0 on error and 1 if there 44783f157a2fSChris Mason * was nothing in the tree that matched the search criteria. 44793f157a2fSChris Mason */ 44803f157a2fSChris Mason int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key, 4481de78b51aSEric Sandeen struct btrfs_path *path, 44823f157a2fSChris Mason u64 min_trans) 44833f157a2fSChris Mason { 44843f157a2fSChris Mason struct extent_buffer *cur; 44853f157a2fSChris Mason struct btrfs_key found_key; 44863f157a2fSChris Mason int slot; 44879652480bSYan int sret; 44883f157a2fSChris Mason u32 nritems; 44893f157a2fSChris Mason int level; 44903f157a2fSChris Mason int ret = 1; 4491f98de9b9SFilipe Manana int keep_locks = path->keep_locks; 44923f157a2fSChris Mason 4493c922b016SStefan Roesch ASSERT(!path->nowait); 4494f98de9b9SFilipe Manana path->keep_locks = 1; 44953f157a2fSChris Mason again: 4496bd681513SChris Mason cur = btrfs_read_lock_root_node(root); 44973f157a2fSChris Mason level = btrfs_header_level(cur); 4498e02119d5SChris Mason WARN_ON(path->nodes[level]); 44993f157a2fSChris Mason path->nodes[level] = cur; 4500bd681513SChris Mason path->locks[level] = BTRFS_READ_LOCK; 45013f157a2fSChris Mason 45023f157a2fSChris Mason if (btrfs_header_generation(cur) < min_trans) { 45033f157a2fSChris Mason ret = 1; 45043f157a2fSChris Mason goto out; 45053f157a2fSChris Mason } 45063f157a2fSChris Mason while (1) { 45073f157a2fSChris Mason nritems = btrfs_header_nritems(cur); 45083f157a2fSChris Mason level = btrfs_header_level(cur); 4509e3b83361SQu Wenruo sret = btrfs_bin_search(cur, min_key, &slot); 4510cbca7d59SFilipe Manana if (sret < 0) { 4511cbca7d59SFilipe Manana ret = sret; 4512cbca7d59SFilipe Manana goto out; 4513cbca7d59SFilipe Manana } 45143f157a2fSChris Mason 4515323ac95bSChris Mason /* at the lowest level, we're done, setup the path and exit */ 4516323ac95bSChris Mason if (level == path->lowest_level) { 4517e02119d5SChris Mason if (slot >= nritems) 4518e02119d5SChris Mason goto find_next_key; 45193f157a2fSChris Mason ret = 0; 45203f157a2fSChris Mason path->slots[level] = slot; 45213f157a2fSChris Mason btrfs_item_key_to_cpu(cur, &found_key, slot); 45223f157a2fSChris Mason goto out; 45233f157a2fSChris Mason } 45249652480bSYan if (sret && slot > 0) 45259652480bSYan slot--; 45263f157a2fSChris Mason /* 4527de78b51aSEric Sandeen * check this node pointer against the min_trans parameters. 4528260db43cSRandy Dunlap * If it is too old, skip to the next one. 45293f157a2fSChris Mason */ 45303f157a2fSChris Mason while (slot < nritems) { 45313f157a2fSChris Mason u64 gen; 4532e02119d5SChris Mason 45333f157a2fSChris Mason gen = btrfs_node_ptr_generation(cur, slot); 45343f157a2fSChris Mason if (gen < min_trans) { 45353f157a2fSChris Mason slot++; 45363f157a2fSChris Mason continue; 45373f157a2fSChris Mason } 45383f157a2fSChris Mason break; 45393f157a2fSChris Mason } 4540e02119d5SChris Mason find_next_key: 45413f157a2fSChris Mason /* 45423f157a2fSChris Mason * we didn't find a candidate key in this node, walk forward 45433f157a2fSChris Mason * and find another one 45443f157a2fSChris Mason */ 45453f157a2fSChris Mason if (slot >= nritems) { 4546e02119d5SChris Mason path->slots[level] = slot; 4547e02119d5SChris Mason sret = btrfs_find_next_key(root, path, min_key, level, 4548de78b51aSEric Sandeen min_trans); 4549e02119d5SChris Mason if (sret == 0) { 4550b3b4aa74SDavid Sterba btrfs_release_path(path); 45513f157a2fSChris Mason goto again; 45523f157a2fSChris Mason } else { 45533f157a2fSChris Mason goto out; 45543f157a2fSChris Mason } 45553f157a2fSChris Mason } 45563f157a2fSChris Mason /* save our key for returning back */ 45573f157a2fSChris Mason btrfs_node_key_to_cpu(cur, &found_key, slot); 45583f157a2fSChris Mason path->slots[level] = slot; 45593f157a2fSChris Mason if (level == path->lowest_level) { 45603f157a2fSChris Mason ret = 0; 45613f157a2fSChris Mason goto out; 45623f157a2fSChris Mason } 45634b231ae4SDavid Sterba cur = btrfs_read_node_slot(cur, slot); 4564fb770ae4SLiu Bo if (IS_ERR(cur)) { 4565fb770ae4SLiu Bo ret = PTR_ERR(cur); 4566fb770ae4SLiu Bo goto out; 4567fb770ae4SLiu Bo } 45683f157a2fSChris Mason 4569bd681513SChris Mason btrfs_tree_read_lock(cur); 4570b4ce94deSChris Mason 4571bd681513SChris Mason path->locks[level - 1] = BTRFS_READ_LOCK; 45723f157a2fSChris Mason path->nodes[level - 1] = cur; 4573f7c79f30SChris Mason unlock_up(path, level, 1, 0, NULL); 45743f157a2fSChris Mason } 45753f157a2fSChris Mason out: 4576f98de9b9SFilipe Manana path->keep_locks = keep_locks; 4577f98de9b9SFilipe Manana if (ret == 0) { 4578f98de9b9SFilipe Manana btrfs_unlock_up_safe(path, path->lowest_level + 1); 4579f98de9b9SFilipe Manana memcpy(min_key, &found_key, sizeof(found_key)); 4580f98de9b9SFilipe Manana } 45813f157a2fSChris Mason return ret; 45823f157a2fSChris Mason } 45833f157a2fSChris Mason 45843f157a2fSChris Mason /* 45853f157a2fSChris Mason * this is similar to btrfs_next_leaf, but does not try to preserve 45863f157a2fSChris Mason * and fixup the path. It looks for and returns the next key in the 4587de78b51aSEric Sandeen * tree based on the current path and the min_trans parameters. 45883f157a2fSChris Mason * 45893f157a2fSChris Mason * 0 is returned if another key is found, < 0 if there are any errors 45903f157a2fSChris Mason * and 1 is returned if there are no higher keys in the tree 45913f157a2fSChris Mason * 45923f157a2fSChris Mason * path->keep_locks should be set to 1 on the search made before 45933f157a2fSChris Mason * calling this function. 45943f157a2fSChris Mason */ 4595e7a84565SChris Mason int btrfs_find_next_key(struct btrfs_root *root, struct btrfs_path *path, 4596de78b51aSEric Sandeen struct btrfs_key *key, int level, u64 min_trans) 4597e7a84565SChris Mason { 4598e7a84565SChris Mason int slot; 4599e7a84565SChris Mason struct extent_buffer *c; 4600e7a84565SChris Mason 46016a9fb468SJosef Bacik WARN_ON(!path->keep_locks && !path->skip_locking); 4602e7a84565SChris Mason while (level < BTRFS_MAX_LEVEL) { 4603e7a84565SChris Mason if (!path->nodes[level]) 4604e7a84565SChris Mason return 1; 4605e7a84565SChris Mason 4606e7a84565SChris Mason slot = path->slots[level] + 1; 4607e7a84565SChris Mason c = path->nodes[level]; 46083f157a2fSChris Mason next: 4609e7a84565SChris Mason if (slot >= btrfs_header_nritems(c)) { 461033c66f43SYan Zheng int ret; 461133c66f43SYan Zheng int orig_lowest; 461233c66f43SYan Zheng struct btrfs_key cur_key; 461333c66f43SYan Zheng if (level + 1 >= BTRFS_MAX_LEVEL || 461433c66f43SYan Zheng !path->nodes[level + 1]) 4615e7a84565SChris Mason return 1; 461633c66f43SYan Zheng 46176a9fb468SJosef Bacik if (path->locks[level + 1] || path->skip_locking) { 461833c66f43SYan Zheng level++; 4619e7a84565SChris Mason continue; 4620e7a84565SChris Mason } 462133c66f43SYan Zheng 462233c66f43SYan Zheng slot = btrfs_header_nritems(c) - 1; 462333c66f43SYan Zheng if (level == 0) 462433c66f43SYan Zheng btrfs_item_key_to_cpu(c, &cur_key, slot); 462533c66f43SYan Zheng else 462633c66f43SYan Zheng btrfs_node_key_to_cpu(c, &cur_key, slot); 462733c66f43SYan Zheng 462833c66f43SYan Zheng orig_lowest = path->lowest_level; 4629b3b4aa74SDavid Sterba btrfs_release_path(path); 463033c66f43SYan Zheng path->lowest_level = level; 463133c66f43SYan Zheng ret = btrfs_search_slot(NULL, root, &cur_key, path, 463233c66f43SYan Zheng 0, 0); 463333c66f43SYan Zheng path->lowest_level = orig_lowest; 463433c66f43SYan Zheng if (ret < 0) 463533c66f43SYan Zheng return ret; 463633c66f43SYan Zheng 463733c66f43SYan Zheng c = path->nodes[level]; 463833c66f43SYan Zheng slot = path->slots[level]; 463933c66f43SYan Zheng if (ret == 0) 464033c66f43SYan Zheng slot++; 464133c66f43SYan Zheng goto next; 464233c66f43SYan Zheng } 464333c66f43SYan Zheng 4644e7a84565SChris Mason if (level == 0) 4645e7a84565SChris Mason btrfs_item_key_to_cpu(c, key, slot); 46463f157a2fSChris Mason else { 46473f157a2fSChris Mason u64 gen = btrfs_node_ptr_generation(c, slot); 46483f157a2fSChris Mason 46493f157a2fSChris Mason if (gen < min_trans) { 46503f157a2fSChris Mason slot++; 46513f157a2fSChris Mason goto next; 46523f157a2fSChris Mason } 4653e7a84565SChris Mason btrfs_node_key_to_cpu(c, key, slot); 46543f157a2fSChris Mason } 4655e7a84565SChris Mason return 0; 4656e7a84565SChris Mason } 4657e7a84565SChris Mason return 1; 4658e7a84565SChris Mason } 4659e7a84565SChris Mason 46603d7806ecSJan Schmidt int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path, 46613d7806ecSJan Schmidt u64 time_seq) 46623d7806ecSJan Schmidt { 4663d97e63b6SChris Mason int slot; 46648e73f275SChris Mason int level; 46655f39d397SChris Mason struct extent_buffer *c; 46668e73f275SChris Mason struct extent_buffer *next; 4667d96b3424SFilipe Manana struct btrfs_fs_info *fs_info = root->fs_info; 4668925baeddSChris Mason struct btrfs_key key; 4669d96b3424SFilipe Manana bool need_commit_sem = false; 4670925baeddSChris Mason u32 nritems; 4671925baeddSChris Mason int ret; 46720e46318dSJosef Bacik int i; 4673925baeddSChris Mason 4674bdcdd86cSFilipe Manana /* 4675bdcdd86cSFilipe Manana * The nowait semantics are used only for write paths, where we don't 4676bdcdd86cSFilipe Manana * use the tree mod log and sequence numbers. 4677bdcdd86cSFilipe Manana */ 4678bdcdd86cSFilipe Manana if (time_seq) 4679c922b016SStefan Roesch ASSERT(!path->nowait); 4680c922b016SStefan Roesch 4681925baeddSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 4682d397712bSChris Mason if (nritems == 0) 4683925baeddSChris Mason return 1; 4684925baeddSChris Mason 46858e73f275SChris Mason btrfs_item_key_to_cpu(path->nodes[0], &key, nritems - 1); 46868e73f275SChris Mason again: 46878e73f275SChris Mason level = 1; 46888e73f275SChris Mason next = NULL; 4689b3b4aa74SDavid Sterba btrfs_release_path(path); 46908e73f275SChris Mason 4691a2135011SChris Mason path->keep_locks = 1; 46928e73f275SChris Mason 4693d96b3424SFilipe Manana if (time_seq) { 46943d7806ecSJan Schmidt ret = btrfs_search_old_slot(root, &key, path, time_seq); 4695d96b3424SFilipe Manana } else { 4696d96b3424SFilipe Manana if (path->need_commit_sem) { 4697d96b3424SFilipe Manana path->need_commit_sem = 0; 4698d96b3424SFilipe Manana need_commit_sem = true; 4699bdcdd86cSFilipe Manana if (path->nowait) { 4700bdcdd86cSFilipe Manana if (!down_read_trylock(&fs_info->commit_root_sem)) { 4701bdcdd86cSFilipe Manana ret = -EAGAIN; 4702bdcdd86cSFilipe Manana goto done; 4703bdcdd86cSFilipe Manana } 4704bdcdd86cSFilipe Manana } else { 4705d96b3424SFilipe Manana down_read(&fs_info->commit_root_sem); 4706d96b3424SFilipe Manana } 4707bdcdd86cSFilipe Manana } 4708925baeddSChris Mason ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 4709d96b3424SFilipe Manana } 4710925baeddSChris Mason path->keep_locks = 0; 4711925baeddSChris Mason 4712925baeddSChris Mason if (ret < 0) 4713d96b3424SFilipe Manana goto done; 4714925baeddSChris Mason 4715a2135011SChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 4716168fd7d2SChris Mason /* 4717168fd7d2SChris Mason * by releasing the path above we dropped all our locks. A balance 4718168fd7d2SChris Mason * could have added more items next to the key that used to be 4719168fd7d2SChris Mason * at the very end of the block. So, check again here and 4720168fd7d2SChris Mason * advance the path if there are now more items available. 4721168fd7d2SChris Mason */ 4722a2135011SChris Mason if (nritems > 0 && path->slots[0] < nritems - 1) { 4723e457afecSYan Zheng if (ret == 0) 4724168fd7d2SChris Mason path->slots[0]++; 47258e73f275SChris Mason ret = 0; 4726925baeddSChris Mason goto done; 4727925baeddSChris Mason } 47280b43e04fSLiu Bo /* 47290b43e04fSLiu Bo * So the above check misses one case: 47300b43e04fSLiu Bo * - after releasing the path above, someone has removed the item that 47310b43e04fSLiu Bo * used to be at the very end of the block, and balance between leafs 47320b43e04fSLiu Bo * gets another one with bigger key.offset to replace it. 47330b43e04fSLiu Bo * 47340b43e04fSLiu Bo * This one should be returned as well, or we can get leaf corruption 47350b43e04fSLiu Bo * later(esp. in __btrfs_drop_extents()). 47360b43e04fSLiu Bo * 47370b43e04fSLiu Bo * And a bit more explanation about this check, 47380b43e04fSLiu Bo * with ret > 0, the key isn't found, the path points to the slot 47390b43e04fSLiu Bo * where it should be inserted, so the path->slots[0] item must be the 47400b43e04fSLiu Bo * bigger one. 47410b43e04fSLiu Bo */ 47420b43e04fSLiu Bo if (nritems > 0 && ret > 0 && path->slots[0] == nritems - 1) { 47430b43e04fSLiu Bo ret = 0; 47440b43e04fSLiu Bo goto done; 47450b43e04fSLiu Bo } 4746d97e63b6SChris Mason 4747234b63a0SChris Mason while (level < BTRFS_MAX_LEVEL) { 47488e73f275SChris Mason if (!path->nodes[level]) { 47498e73f275SChris Mason ret = 1; 47508e73f275SChris Mason goto done; 47518e73f275SChris Mason } 47525f39d397SChris Mason 4753d97e63b6SChris Mason slot = path->slots[level] + 1; 4754d97e63b6SChris Mason c = path->nodes[level]; 47555f39d397SChris Mason if (slot >= btrfs_header_nritems(c)) { 4756d97e63b6SChris Mason level++; 47578e73f275SChris Mason if (level == BTRFS_MAX_LEVEL) { 47588e73f275SChris Mason ret = 1; 47598e73f275SChris Mason goto done; 47608e73f275SChris Mason } 4761d97e63b6SChris Mason continue; 4762d97e63b6SChris Mason } 47635f39d397SChris Mason 47640e46318dSJosef Bacik 47650e46318dSJosef Bacik /* 47660e46318dSJosef Bacik * Our current level is where we're going to start from, and to 47670e46318dSJosef Bacik * make sure lockdep doesn't complain we need to drop our locks 47680e46318dSJosef Bacik * and nodes from 0 to our current level. 47690e46318dSJosef Bacik */ 47700e46318dSJosef Bacik for (i = 0; i < level; i++) { 47710e46318dSJosef Bacik if (path->locks[level]) { 47720e46318dSJosef Bacik btrfs_tree_read_unlock(path->nodes[i]); 47730e46318dSJosef Bacik path->locks[i] = 0; 47740e46318dSJosef Bacik } 47750e46318dSJosef Bacik free_extent_buffer(path->nodes[i]); 47760e46318dSJosef Bacik path->nodes[i] = NULL; 4777925baeddSChris Mason } 47785f39d397SChris Mason 47798e73f275SChris Mason next = c; 4780d07b8528SLiu Bo ret = read_block_for_search(root, path, &next, level, 4781cda79c54SDavid Sterba slot, &key); 4782bdcdd86cSFilipe Manana if (ret == -EAGAIN && !path->nowait) 47838e73f275SChris Mason goto again; 47845f39d397SChris Mason 478576a05b35SChris Mason if (ret < 0) { 4786b3b4aa74SDavid Sterba btrfs_release_path(path); 478776a05b35SChris Mason goto done; 478876a05b35SChris Mason } 478976a05b35SChris Mason 47905cd57b2cSChris Mason if (!path->skip_locking) { 4791bd681513SChris Mason ret = btrfs_try_tree_read_lock(next); 4792bdcdd86cSFilipe Manana if (!ret && path->nowait) { 4793bdcdd86cSFilipe Manana ret = -EAGAIN; 4794bdcdd86cSFilipe Manana goto done; 4795bdcdd86cSFilipe Manana } 4796d42244a0SJan Schmidt if (!ret && time_seq) { 4797d42244a0SJan Schmidt /* 4798d42244a0SJan Schmidt * If we don't get the lock, we may be racing 4799d42244a0SJan Schmidt * with push_leaf_left, holding that lock while 4800d42244a0SJan Schmidt * itself waiting for the leaf we've currently 4801d42244a0SJan Schmidt * locked. To solve this situation, we give up 4802d42244a0SJan Schmidt * on our lock and cycle. 4803d42244a0SJan Schmidt */ 4804cf538830SJan Schmidt free_extent_buffer(next); 4805d42244a0SJan Schmidt btrfs_release_path(path); 4806d42244a0SJan Schmidt cond_resched(); 4807d42244a0SJan Schmidt goto again; 4808d42244a0SJan Schmidt } 48090e46318dSJosef Bacik if (!ret) 48100e46318dSJosef Bacik btrfs_tree_read_lock(next); 4811bd681513SChris Mason } 4812d97e63b6SChris Mason break; 4813d97e63b6SChris Mason } 4814d97e63b6SChris Mason path->slots[level] = slot; 4815d97e63b6SChris Mason while (1) { 4816d97e63b6SChris Mason level--; 4817d97e63b6SChris Mason path->nodes[level] = next; 4818d97e63b6SChris Mason path->slots[level] = 0; 4819a74a4b97SChris Mason if (!path->skip_locking) 4820ffeb03cfSJosef Bacik path->locks[level] = BTRFS_READ_LOCK; 4821d97e63b6SChris Mason if (!level) 4822d97e63b6SChris Mason break; 4823b4ce94deSChris Mason 4824d07b8528SLiu Bo ret = read_block_for_search(root, path, &next, level, 4825cda79c54SDavid Sterba 0, &key); 4826bdcdd86cSFilipe Manana if (ret == -EAGAIN && !path->nowait) 48278e73f275SChris Mason goto again; 48288e73f275SChris Mason 482976a05b35SChris Mason if (ret < 0) { 4830b3b4aa74SDavid Sterba btrfs_release_path(path); 483176a05b35SChris Mason goto done; 483276a05b35SChris Mason } 483376a05b35SChris Mason 4834bdcdd86cSFilipe Manana if (!path->skip_locking) { 4835bdcdd86cSFilipe Manana if (path->nowait) { 4836bdcdd86cSFilipe Manana if (!btrfs_try_tree_read_lock(next)) { 4837bdcdd86cSFilipe Manana ret = -EAGAIN; 4838bdcdd86cSFilipe Manana goto done; 4839bdcdd86cSFilipe Manana } 4840bdcdd86cSFilipe Manana } else { 48410e46318dSJosef Bacik btrfs_tree_read_lock(next); 4842d97e63b6SChris Mason } 4843bdcdd86cSFilipe Manana } 4844bdcdd86cSFilipe Manana } 48458e73f275SChris Mason ret = 0; 4846925baeddSChris Mason done: 4847f7c79f30SChris Mason unlock_up(path, 0, 1, 0, NULL); 4848d96b3424SFilipe Manana if (need_commit_sem) { 4849d96b3424SFilipe Manana int ret2; 4850d96b3424SFilipe Manana 4851d96b3424SFilipe Manana path->need_commit_sem = 1; 4852d96b3424SFilipe Manana ret2 = finish_need_commit_sem_search(path); 4853d96b3424SFilipe Manana up_read(&fs_info->commit_root_sem); 4854d96b3424SFilipe Manana if (ret2) 4855d96b3424SFilipe Manana ret = ret2; 4856d96b3424SFilipe Manana } 48578e73f275SChris Mason 48588e73f275SChris Mason return ret; 4859d97e63b6SChris Mason } 48600b86a832SChris Mason 4861890d2b1aSJosef Bacik int btrfs_next_old_item(struct btrfs_root *root, struct btrfs_path *path, u64 time_seq) 4862890d2b1aSJosef Bacik { 4863890d2b1aSJosef Bacik path->slots[0]++; 4864890d2b1aSJosef Bacik if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) 4865890d2b1aSJosef Bacik return btrfs_next_old_leaf(root, path, time_seq); 4866890d2b1aSJosef Bacik return 0; 4867890d2b1aSJosef Bacik } 4868890d2b1aSJosef Bacik 48693f157a2fSChris Mason /* 48703f157a2fSChris Mason * this uses btrfs_prev_leaf to walk backwards in the tree, and keeps 48713f157a2fSChris Mason * searching until it gets past min_objectid or finds an item of 'type' 48723f157a2fSChris Mason * 48733f157a2fSChris Mason * returns 0 if something is found, 1 if nothing was found and < 0 on error 48743f157a2fSChris Mason */ 48750b86a832SChris Mason int btrfs_previous_item(struct btrfs_root *root, 48760b86a832SChris Mason struct btrfs_path *path, u64 min_objectid, 48770b86a832SChris Mason int type) 48780b86a832SChris Mason { 48790b86a832SChris Mason struct btrfs_key found_key; 48800b86a832SChris Mason struct extent_buffer *leaf; 4881e02119d5SChris Mason u32 nritems; 48820b86a832SChris Mason int ret; 48830b86a832SChris Mason 48840b86a832SChris Mason while (1) { 48850b86a832SChris Mason if (path->slots[0] == 0) { 48860b86a832SChris Mason ret = btrfs_prev_leaf(root, path); 48870b86a832SChris Mason if (ret != 0) 48880b86a832SChris Mason return ret; 48890b86a832SChris Mason } else { 48900b86a832SChris Mason path->slots[0]--; 48910b86a832SChris Mason } 48920b86a832SChris Mason leaf = path->nodes[0]; 4893e02119d5SChris Mason nritems = btrfs_header_nritems(leaf); 4894e02119d5SChris Mason if (nritems == 0) 4895e02119d5SChris Mason return 1; 4896e02119d5SChris Mason if (path->slots[0] == nritems) 4897e02119d5SChris Mason path->slots[0]--; 4898e02119d5SChris Mason 48990b86a832SChris Mason btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 4900e02119d5SChris Mason if (found_key.objectid < min_objectid) 4901e02119d5SChris Mason break; 49020a4eefbbSYan Zheng if (found_key.type == type) 49030a4eefbbSYan Zheng return 0; 4904e02119d5SChris Mason if (found_key.objectid == min_objectid && 4905e02119d5SChris Mason found_key.type < type) 4906e02119d5SChris Mason break; 49070b86a832SChris Mason } 49080b86a832SChris Mason return 1; 49090b86a832SChris Mason } 4910ade2e0b3SWang Shilong 4911ade2e0b3SWang Shilong /* 4912ade2e0b3SWang Shilong * search in extent tree to find a previous Metadata/Data extent item with 4913ade2e0b3SWang Shilong * min objecitd. 4914ade2e0b3SWang Shilong * 4915ade2e0b3SWang Shilong * returns 0 if something is found, 1 if nothing was found and < 0 on error 4916ade2e0b3SWang Shilong */ 4917ade2e0b3SWang Shilong int btrfs_previous_extent_item(struct btrfs_root *root, 4918ade2e0b3SWang Shilong struct btrfs_path *path, u64 min_objectid) 4919ade2e0b3SWang Shilong { 4920ade2e0b3SWang Shilong struct btrfs_key found_key; 4921ade2e0b3SWang Shilong struct extent_buffer *leaf; 4922ade2e0b3SWang Shilong u32 nritems; 4923ade2e0b3SWang Shilong int ret; 4924ade2e0b3SWang Shilong 4925ade2e0b3SWang Shilong while (1) { 4926ade2e0b3SWang Shilong if (path->slots[0] == 0) { 4927ade2e0b3SWang Shilong ret = btrfs_prev_leaf(root, path); 4928ade2e0b3SWang Shilong if (ret != 0) 4929ade2e0b3SWang Shilong return ret; 4930ade2e0b3SWang Shilong } else { 4931ade2e0b3SWang Shilong path->slots[0]--; 4932ade2e0b3SWang Shilong } 4933ade2e0b3SWang Shilong leaf = path->nodes[0]; 4934ade2e0b3SWang Shilong nritems = btrfs_header_nritems(leaf); 4935ade2e0b3SWang Shilong if (nritems == 0) 4936ade2e0b3SWang Shilong return 1; 4937ade2e0b3SWang Shilong if (path->slots[0] == nritems) 4938ade2e0b3SWang Shilong path->slots[0]--; 4939ade2e0b3SWang Shilong 4940ade2e0b3SWang Shilong btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 4941ade2e0b3SWang Shilong if (found_key.objectid < min_objectid) 4942ade2e0b3SWang Shilong break; 4943ade2e0b3SWang Shilong if (found_key.type == BTRFS_EXTENT_ITEM_KEY || 4944ade2e0b3SWang Shilong found_key.type == BTRFS_METADATA_ITEM_KEY) 4945ade2e0b3SWang Shilong return 0; 4946ade2e0b3SWang Shilong if (found_key.objectid == min_objectid && 4947ade2e0b3SWang Shilong found_key.type < BTRFS_EXTENT_ITEM_KEY) 4948ade2e0b3SWang Shilong break; 4949ade2e0b3SWang Shilong } 4950ade2e0b3SWang Shilong return 1; 4951ade2e0b3SWang Shilong } 4952226463d7SJosef Bacik 4953226463d7SJosef Bacik int __init btrfs_ctree_init(void) 4954226463d7SJosef Bacik { 4955226463d7SJosef Bacik btrfs_path_cachep = kmem_cache_create("btrfs_path", 4956226463d7SJosef Bacik sizeof(struct btrfs_path), 0, 4957226463d7SJosef Bacik SLAB_MEM_SPREAD, NULL); 4958226463d7SJosef Bacik if (!btrfs_path_cachep) 4959226463d7SJosef Bacik return -ENOMEM; 4960226463d7SJosef Bacik return 0; 4961226463d7SJosef Bacik } 4962226463d7SJosef Bacik 4963226463d7SJosef Bacik void __cold btrfs_ctree_exit(void) 4964226463d7SJosef Bacik { 4965226463d7SJosef Bacik kmem_cache_destroy(btrfs_path_cachep); 4966226463d7SJosef Bacik } 4967