1c1d7c514SDavid Sterba // SPDX-License-Identifier: GPL-2.0 26cbd5570SChris Mason /* 3d352ac68SChris Mason * Copyright (C) 2007,2008 Oracle. All rights reserved. 46cbd5570SChris Mason */ 56cbd5570SChris Mason 6a6b6e75eSChris Mason #include <linux/sched.h> 75a0e3ad6STejun Heo #include <linux/slab.h> 8bd989ba3SJan Schmidt #include <linux/rbtree.h> 9adf02123SDavid Sterba #include <linux/mm.h> 10e41d12f5SChristoph Hellwig #include <linux/error-injection.h> 11eb60ceacSChris Mason #include "ctree.h" 12eb60ceacSChris Mason #include "disk-io.h" 137f5c1516SChris Mason #include "transaction.h" 145f39d397SChris Mason #include "print-tree.h" 15925baeddSChris Mason #include "locking.h" 16de37aa51SNikolay Borisov #include "volumes.h" 17f616f5cdSQu Wenruo #include "qgroup.h" 18f3a84ccdSFilipe Manana #include "tree-mod-log.h" 199a8dd150SChris Mason 20e089f05cSChris Mason static int split_node(struct btrfs_trans_handle *trans, struct btrfs_root 21e089f05cSChris Mason *root, struct btrfs_path *path, int level); 22310712b2SOmar Sandoval static int split_leaf(struct btrfs_trans_handle *trans, struct btrfs_root *root, 23310712b2SOmar Sandoval const struct btrfs_key *ins_key, struct btrfs_path *path, 24310712b2SOmar Sandoval int data_size, int extend); 255f39d397SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 262ff7e61eSJeff Mahoney struct extent_buffer *dst, 27971a1f66SChris Mason struct extent_buffer *src, int empty); 285f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 295f39d397SChris Mason struct extent_buffer *dst_buf, 305f39d397SChris Mason struct extent_buffer *src_buf); 31afe5fea7STsutomu Itoh static void del_ptr(struct btrfs_root *root, struct btrfs_path *path, 32afe5fea7STsutomu Itoh int level, int slot); 33d97e63b6SChris Mason 34af024ed2SJohannes Thumshirn static const struct btrfs_csums { 35af024ed2SJohannes Thumshirn u16 size; 3659a0fcdbSDavid Sterba const char name[10]; 3759a0fcdbSDavid Sterba const char driver[12]; 38af024ed2SJohannes Thumshirn } btrfs_csums[] = { 39af024ed2SJohannes Thumshirn [BTRFS_CSUM_TYPE_CRC32] = { .size = 4, .name = "crc32c" }, 403951e7f0SJohannes Thumshirn [BTRFS_CSUM_TYPE_XXHASH] = { .size = 8, .name = "xxhash64" }, 413831bf00SJohannes Thumshirn [BTRFS_CSUM_TYPE_SHA256] = { .size = 32, .name = "sha256" }, 42352ae07bSDavid Sterba [BTRFS_CSUM_TYPE_BLAKE2] = { .size = 32, .name = "blake2b", 43352ae07bSDavid Sterba .driver = "blake2b-256" }, 44af024ed2SJohannes Thumshirn }; 45af024ed2SJohannes Thumshirn 46af024ed2SJohannes Thumshirn int btrfs_super_csum_size(const struct btrfs_super_block *s) 47af024ed2SJohannes Thumshirn { 48af024ed2SJohannes Thumshirn u16 t = btrfs_super_csum_type(s); 49af024ed2SJohannes Thumshirn /* 50af024ed2SJohannes Thumshirn * csum type is validated at mount time 51af024ed2SJohannes Thumshirn */ 52af024ed2SJohannes Thumshirn return btrfs_csums[t].size; 53af024ed2SJohannes Thumshirn } 54af024ed2SJohannes Thumshirn 55af024ed2SJohannes Thumshirn const char *btrfs_super_csum_name(u16 csum_type) 56af024ed2SJohannes Thumshirn { 57af024ed2SJohannes Thumshirn /* csum type is validated at mount time */ 58af024ed2SJohannes Thumshirn return btrfs_csums[csum_type].name; 59af024ed2SJohannes Thumshirn } 60af024ed2SJohannes Thumshirn 61b4e967beSDavid Sterba /* 62b4e967beSDavid Sterba * Return driver name if defined, otherwise the name that's also a valid driver 63b4e967beSDavid Sterba * name 64b4e967beSDavid Sterba */ 65b4e967beSDavid Sterba const char *btrfs_super_csum_driver(u16 csum_type) 66b4e967beSDavid Sterba { 67b4e967beSDavid Sterba /* csum type is validated at mount time */ 6859a0fcdbSDavid Sterba return btrfs_csums[csum_type].driver[0] ? 6959a0fcdbSDavid Sterba btrfs_csums[csum_type].driver : 70b4e967beSDavid Sterba btrfs_csums[csum_type].name; 71b4e967beSDavid Sterba } 72b4e967beSDavid Sterba 73604997b4SDavid Sterba size_t __attribute_const__ btrfs_get_num_csums(void) 74f7cea56cSDavid Sterba { 75f7cea56cSDavid Sterba return ARRAY_SIZE(btrfs_csums); 76f7cea56cSDavid Sterba } 77f7cea56cSDavid Sterba 782c90e5d6SChris Mason struct btrfs_path *btrfs_alloc_path(void) 792c90e5d6SChris Mason { 80e2c89907SMasahiro Yamada return kmem_cache_zalloc(btrfs_path_cachep, GFP_NOFS); 812c90e5d6SChris Mason } 822c90e5d6SChris Mason 83d352ac68SChris Mason /* this also releases the path */ 842c90e5d6SChris Mason void btrfs_free_path(struct btrfs_path *p) 852c90e5d6SChris Mason { 86ff175d57SJesper Juhl if (!p) 87ff175d57SJesper Juhl return; 88b3b4aa74SDavid Sterba btrfs_release_path(p); 892c90e5d6SChris Mason kmem_cache_free(btrfs_path_cachep, p); 902c90e5d6SChris Mason } 912c90e5d6SChris Mason 92d352ac68SChris Mason /* 93d352ac68SChris Mason * path release drops references on the extent buffers in the path 94d352ac68SChris Mason * and it drops any locks held by this path 95d352ac68SChris Mason * 96d352ac68SChris Mason * It is safe to call this on paths that no locks or extent buffers held. 97d352ac68SChris Mason */ 98b3b4aa74SDavid Sterba noinline void btrfs_release_path(struct btrfs_path *p) 99eb60ceacSChris Mason { 100eb60ceacSChris Mason int i; 101a2135011SChris Mason 102234b63a0SChris Mason for (i = 0; i < BTRFS_MAX_LEVEL; i++) { 1033f157a2fSChris Mason p->slots[i] = 0; 104eb60ceacSChris Mason if (!p->nodes[i]) 105925baeddSChris Mason continue; 106925baeddSChris Mason if (p->locks[i]) { 107bd681513SChris Mason btrfs_tree_unlock_rw(p->nodes[i], p->locks[i]); 108925baeddSChris Mason p->locks[i] = 0; 109925baeddSChris Mason } 1105f39d397SChris Mason free_extent_buffer(p->nodes[i]); 1113f157a2fSChris Mason p->nodes[i] = NULL; 112eb60ceacSChris Mason } 113eb60ceacSChris Mason } 114eb60ceacSChris Mason 115d352ac68SChris Mason /* 116d352ac68SChris Mason * safely gets a reference on the root node of a tree. A lock 117d352ac68SChris Mason * is not taken, so a concurrent writer may put a different node 118d352ac68SChris Mason * at the root of the tree. See btrfs_lock_root_node for the 119d352ac68SChris Mason * looping required. 120d352ac68SChris Mason * 121d352ac68SChris Mason * The extent buffer returned by this has a reference taken, so 122d352ac68SChris Mason * it won't disappear. It may stop being the root of the tree 123d352ac68SChris Mason * at any time because there are no locks held. 124d352ac68SChris Mason */ 125925baeddSChris Mason struct extent_buffer *btrfs_root_node(struct btrfs_root *root) 126925baeddSChris Mason { 127925baeddSChris Mason struct extent_buffer *eb; 128240f62c8SChris Mason 1293083ee2eSJosef Bacik while (1) { 130240f62c8SChris Mason rcu_read_lock(); 131240f62c8SChris Mason eb = rcu_dereference(root->node); 1323083ee2eSJosef Bacik 1333083ee2eSJosef Bacik /* 1343083ee2eSJosef Bacik * RCU really hurts here, we could free up the root node because 13501327610SNicholas D Steeves * it was COWed but we may not get the new root node yet so do 1363083ee2eSJosef Bacik * the inc_not_zero dance and if it doesn't work then 1373083ee2eSJosef Bacik * synchronize_rcu and try again. 1383083ee2eSJosef Bacik */ 1393083ee2eSJosef Bacik if (atomic_inc_not_zero(&eb->refs)) { 140240f62c8SChris Mason rcu_read_unlock(); 1413083ee2eSJosef Bacik break; 1423083ee2eSJosef Bacik } 1433083ee2eSJosef Bacik rcu_read_unlock(); 1443083ee2eSJosef Bacik synchronize_rcu(); 1453083ee2eSJosef Bacik } 146925baeddSChris Mason return eb; 147925baeddSChris Mason } 148925baeddSChris Mason 14992a7cc42SQu Wenruo /* 15092a7cc42SQu Wenruo * Cowonly root (not-shareable trees, everything not subvolume or reloc roots), 15192a7cc42SQu Wenruo * just get put onto a simple dirty list. Transaction walks this list to make 15292a7cc42SQu Wenruo * sure they get properly updated on disk. 153d352ac68SChris Mason */ 1540b86a832SChris Mason static void add_root_to_dirty_list(struct btrfs_root *root) 1550b86a832SChris Mason { 1560b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 1570b246afaSJeff Mahoney 158e7070be1SJosef Bacik if (test_bit(BTRFS_ROOT_DIRTY, &root->state) || 159e7070be1SJosef Bacik !test_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state)) 160e7070be1SJosef Bacik return; 161e7070be1SJosef Bacik 1620b246afaSJeff Mahoney spin_lock(&fs_info->trans_lock); 163e7070be1SJosef Bacik if (!test_and_set_bit(BTRFS_ROOT_DIRTY, &root->state)) { 164e7070be1SJosef Bacik /* Want the extent tree to be the last on the list */ 1654fd786e6SMisono Tomohiro if (root->root_key.objectid == BTRFS_EXTENT_TREE_OBJECTID) 166e7070be1SJosef Bacik list_move_tail(&root->dirty_list, 1670b246afaSJeff Mahoney &fs_info->dirty_cowonly_roots); 168e7070be1SJosef Bacik else 169e7070be1SJosef Bacik list_move(&root->dirty_list, 1700b246afaSJeff Mahoney &fs_info->dirty_cowonly_roots); 1710b86a832SChris Mason } 1720b246afaSJeff Mahoney spin_unlock(&fs_info->trans_lock); 1730b86a832SChris Mason } 1740b86a832SChris Mason 175d352ac68SChris Mason /* 176d352ac68SChris Mason * used by snapshot creation to make a copy of a root for a tree with 177d352ac68SChris Mason * a given objectid. The buffer with the new root node is returned in 178d352ac68SChris Mason * cow_ret, and this func returns zero on success or a negative error code. 179d352ac68SChris Mason */ 180be20aa9dSChris Mason int btrfs_copy_root(struct btrfs_trans_handle *trans, 181be20aa9dSChris Mason struct btrfs_root *root, 182be20aa9dSChris Mason struct extent_buffer *buf, 183be20aa9dSChris Mason struct extent_buffer **cow_ret, u64 new_root_objectid) 184be20aa9dSChris Mason { 1850b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 186be20aa9dSChris Mason struct extent_buffer *cow; 187be20aa9dSChris Mason int ret = 0; 188be20aa9dSChris Mason int level; 1895d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 190be20aa9dSChris Mason 19192a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 1920b246afaSJeff Mahoney trans->transid != fs_info->running_transaction->transid); 19392a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 19427cdeb70SMiao Xie trans->transid != root->last_trans); 195be20aa9dSChris Mason 196be20aa9dSChris Mason level = btrfs_header_level(buf); 1975d4f98a2SYan Zheng if (level == 0) 1985d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 1995d4f98a2SYan Zheng else 2005d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 20131840ae1SZheng Yan 2024d75f8a9SDavid Sterba cow = btrfs_alloc_tree_block(trans, root, 0, new_root_objectid, 203cf6f34aaSJosef Bacik &disk_key, level, buf->start, 0, 204cf6f34aaSJosef Bacik BTRFS_NESTING_NEW_ROOT); 2055d4f98a2SYan Zheng if (IS_ERR(cow)) 206be20aa9dSChris Mason return PTR_ERR(cow); 207be20aa9dSChris Mason 20858e8012cSDavid Sterba copy_extent_buffer_full(cow, buf); 209be20aa9dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 210be20aa9dSChris Mason btrfs_set_header_generation(cow, trans->transid); 2115d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 2125d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 2135d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 2145d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 2155d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 2165d4f98a2SYan Zheng else 217be20aa9dSChris Mason btrfs_set_header_owner(cow, new_root_objectid); 218be20aa9dSChris Mason 219de37aa51SNikolay Borisov write_extent_buffer_fsid(cow, fs_info->fs_devices->metadata_uuid); 2202b82032cSYan Zheng 221be20aa9dSChris Mason WARN_ON(btrfs_header_generation(buf) > trans->transid); 2225d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 223e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 2245d4f98a2SYan Zheng else 225e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 226867ed321SJosef Bacik if (ret) { 22772c9925fSFilipe Manana btrfs_tree_unlock(cow); 22872c9925fSFilipe Manana free_extent_buffer(cow); 229867ed321SJosef Bacik btrfs_abort_transaction(trans, ret); 230be20aa9dSChris Mason return ret; 231867ed321SJosef Bacik } 232be20aa9dSChris Mason 233be20aa9dSChris Mason btrfs_mark_buffer_dirty(cow); 234be20aa9dSChris Mason *cow_ret = cow; 235be20aa9dSChris Mason return 0; 236be20aa9dSChris Mason } 237be20aa9dSChris Mason 238d352ac68SChris Mason /* 2395d4f98a2SYan Zheng * check if the tree block can be shared by multiple trees 2405d4f98a2SYan Zheng */ 2415d4f98a2SYan Zheng int btrfs_block_can_be_shared(struct btrfs_root *root, 2425d4f98a2SYan Zheng struct extent_buffer *buf) 2435d4f98a2SYan Zheng { 2445d4f98a2SYan Zheng /* 24592a7cc42SQu Wenruo * Tree blocks not in shareable trees and tree roots are never shared. 24692a7cc42SQu Wenruo * If a block was allocated after the last snapshot and the block was 24792a7cc42SQu Wenruo * not allocated by tree relocation, we know the block is not shared. 2485d4f98a2SYan Zheng */ 24992a7cc42SQu Wenruo if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 2505d4f98a2SYan Zheng buf != root->node && buf != root->commit_root && 2515d4f98a2SYan Zheng (btrfs_header_generation(buf) <= 2525d4f98a2SYan Zheng btrfs_root_last_snapshot(&root->root_item) || 2535d4f98a2SYan Zheng btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC))) 2545d4f98a2SYan Zheng return 1; 255a79865c6SNikolay Borisov 2565d4f98a2SYan Zheng return 0; 2575d4f98a2SYan Zheng } 2585d4f98a2SYan Zheng 2595d4f98a2SYan Zheng static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans, 2605d4f98a2SYan Zheng struct btrfs_root *root, 2615d4f98a2SYan Zheng struct extent_buffer *buf, 262f0486c68SYan, Zheng struct extent_buffer *cow, 263f0486c68SYan, Zheng int *last_ref) 2645d4f98a2SYan Zheng { 2650b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 2665d4f98a2SYan Zheng u64 refs; 2675d4f98a2SYan Zheng u64 owner; 2685d4f98a2SYan Zheng u64 flags; 2695d4f98a2SYan Zheng u64 new_flags = 0; 2705d4f98a2SYan Zheng int ret; 2715d4f98a2SYan Zheng 2725d4f98a2SYan Zheng /* 2735d4f98a2SYan Zheng * Backrefs update rules: 2745d4f98a2SYan Zheng * 2755d4f98a2SYan Zheng * Always use full backrefs for extent pointers in tree block 2765d4f98a2SYan Zheng * allocated by tree relocation. 2775d4f98a2SYan Zheng * 2785d4f98a2SYan Zheng * If a shared tree block is no longer referenced by its owner 2795d4f98a2SYan Zheng * tree (btrfs_header_owner(buf) == root->root_key.objectid), 2805d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 2815d4f98a2SYan Zheng * 2825d4f98a2SYan Zheng * If a tree block is been relocating 2835d4f98a2SYan Zheng * (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID), 2845d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 2855d4f98a2SYan Zheng * The reason for this is some operations (such as drop tree) 2865d4f98a2SYan Zheng * are only allowed for blocks use full backrefs. 2875d4f98a2SYan Zheng */ 2885d4f98a2SYan Zheng 2895d4f98a2SYan Zheng if (btrfs_block_can_be_shared(root, buf)) { 2902ff7e61eSJeff Mahoney ret = btrfs_lookup_extent_info(trans, fs_info, buf->start, 2913173a18fSJosef Bacik btrfs_header_level(buf), 1, 2923173a18fSJosef Bacik &refs, &flags); 293be1a5564SMark Fasheh if (ret) 294be1a5564SMark Fasheh return ret; 295e5df9573SMark Fasheh if (refs == 0) { 296e5df9573SMark Fasheh ret = -EROFS; 2970b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 298e5df9573SMark Fasheh return ret; 299e5df9573SMark Fasheh } 3005d4f98a2SYan Zheng } else { 3015d4f98a2SYan Zheng refs = 1; 3025d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 3035d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 3045d4f98a2SYan Zheng flags = BTRFS_BLOCK_FLAG_FULL_BACKREF; 3055d4f98a2SYan Zheng else 3065d4f98a2SYan Zheng flags = 0; 3075d4f98a2SYan Zheng } 3085d4f98a2SYan Zheng 3095d4f98a2SYan Zheng owner = btrfs_header_owner(buf); 3105d4f98a2SYan Zheng BUG_ON(owner == BTRFS_TREE_RELOC_OBJECTID && 3115d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)); 3125d4f98a2SYan Zheng 3135d4f98a2SYan Zheng if (refs > 1) { 3145d4f98a2SYan Zheng if ((owner == root->root_key.objectid || 3155d4f98a2SYan Zheng root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && 3165d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) { 317e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, buf, 1); 318692826b2SJeff Mahoney if (ret) 319692826b2SJeff Mahoney return ret; 3205d4f98a2SYan Zheng 3215d4f98a2SYan Zheng if (root->root_key.objectid == 3225d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) { 323e339a6b0SJosef Bacik ret = btrfs_dec_ref(trans, root, buf, 0); 324692826b2SJeff Mahoney if (ret) 325692826b2SJeff Mahoney return ret; 326e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 327692826b2SJeff Mahoney if (ret) 328692826b2SJeff Mahoney return ret; 3295d4f98a2SYan Zheng } 3305d4f98a2SYan Zheng new_flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF; 3315d4f98a2SYan Zheng } else { 3325d4f98a2SYan Zheng 3335d4f98a2SYan Zheng if (root->root_key.objectid == 3345d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 335e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 3365d4f98a2SYan Zheng else 337e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 338692826b2SJeff Mahoney if (ret) 339692826b2SJeff Mahoney return ret; 3405d4f98a2SYan Zheng } 3415d4f98a2SYan Zheng if (new_flags != 0) { 342b1c79e09SJosef Bacik int level = btrfs_header_level(buf); 343b1c79e09SJosef Bacik 34442c9d0b5SDavid Sterba ret = btrfs_set_disk_extent_flags(trans, buf, 345b1c79e09SJosef Bacik new_flags, level, 0); 346be1a5564SMark Fasheh if (ret) 347be1a5564SMark Fasheh return ret; 3485d4f98a2SYan Zheng } 3495d4f98a2SYan Zheng } else { 3505d4f98a2SYan Zheng if (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) { 3515d4f98a2SYan Zheng if (root->root_key.objectid == 3525d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 353e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 3545d4f98a2SYan Zheng else 355e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 356692826b2SJeff Mahoney if (ret) 357692826b2SJeff Mahoney return ret; 358e339a6b0SJosef Bacik ret = btrfs_dec_ref(trans, root, buf, 1); 359692826b2SJeff Mahoney if (ret) 360692826b2SJeff Mahoney return ret; 3615d4f98a2SYan Zheng } 3626a884d7dSDavid Sterba btrfs_clean_tree_block(buf); 363f0486c68SYan, Zheng *last_ref = 1; 3645d4f98a2SYan Zheng } 3655d4f98a2SYan Zheng return 0; 3665d4f98a2SYan Zheng } 3675d4f98a2SYan Zheng 3685d4f98a2SYan Zheng /* 369d397712bSChris Mason * does the dirty work in cow of a single block. The parent block (if 370d397712bSChris Mason * supplied) is updated to point to the new cow copy. The new buffer is marked 371d397712bSChris Mason * dirty and returned locked. If you modify the block it needs to be marked 372d397712bSChris Mason * dirty again. 373d352ac68SChris Mason * 374d352ac68SChris Mason * search_start -- an allocation hint for the new block 375d352ac68SChris Mason * 376d397712bSChris Mason * empty_size -- a hint that you plan on doing more cow. This is the size in 377d397712bSChris Mason * bytes the allocator should try to find free next to the block it returns. 378d397712bSChris Mason * This is just a hint and may be ignored by the allocator. 379d352ac68SChris Mason */ 380d397712bSChris Mason static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans, 3815f39d397SChris Mason struct btrfs_root *root, 3825f39d397SChris Mason struct extent_buffer *buf, 3835f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 3845f39d397SChris Mason struct extent_buffer **cow_ret, 3859631e4ccSJosef Bacik u64 search_start, u64 empty_size, 3869631e4ccSJosef Bacik enum btrfs_lock_nesting nest) 3876702ed49SChris Mason { 3880b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 3895d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 3905f39d397SChris Mason struct extent_buffer *cow; 391be1a5564SMark Fasheh int level, ret; 392f0486c68SYan, Zheng int last_ref = 0; 393925baeddSChris Mason int unlock_orig = 0; 3940f5053ebSGoldwyn Rodrigues u64 parent_start = 0; 3956702ed49SChris Mason 396925baeddSChris Mason if (*cow_ret == buf) 397925baeddSChris Mason unlock_orig = 1; 398925baeddSChris Mason 39949d0c642SFilipe Manana btrfs_assert_tree_write_locked(buf); 400925baeddSChris Mason 40192a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 4020b246afaSJeff Mahoney trans->transid != fs_info->running_transaction->transid); 40392a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 40427cdeb70SMiao Xie trans->transid != root->last_trans); 4055f39d397SChris Mason 4067bb86316SChris Mason level = btrfs_header_level(buf); 40731840ae1SZheng Yan 4085d4f98a2SYan Zheng if (level == 0) 4095d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 4105d4f98a2SYan Zheng else 4115d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 4125d4f98a2SYan Zheng 4130f5053ebSGoldwyn Rodrigues if ((root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && parent) 4145d4f98a2SYan Zheng parent_start = parent->start; 4155d4f98a2SYan Zheng 41679bd3712SFilipe Manana cow = btrfs_alloc_tree_block(trans, root, parent_start, 41779bd3712SFilipe Manana root->root_key.objectid, &disk_key, level, 41879bd3712SFilipe Manana search_start, empty_size, nest); 4196702ed49SChris Mason if (IS_ERR(cow)) 4206702ed49SChris Mason return PTR_ERR(cow); 4216702ed49SChris Mason 422b4ce94deSChris Mason /* cow is set to blocking by btrfs_init_new_buffer */ 423b4ce94deSChris Mason 42458e8012cSDavid Sterba copy_extent_buffer_full(cow, buf); 425db94535dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 4265f39d397SChris Mason btrfs_set_header_generation(cow, trans->transid); 4275d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 4285d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 4295d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 4305d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) 4315d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 4325d4f98a2SYan Zheng else 4335f39d397SChris Mason btrfs_set_header_owner(cow, root->root_key.objectid); 4346702ed49SChris Mason 435de37aa51SNikolay Borisov write_extent_buffer_fsid(cow, fs_info->fs_devices->metadata_uuid); 4362b82032cSYan Zheng 437be1a5564SMark Fasheh ret = update_ref_for_cow(trans, root, buf, cow, &last_ref); 438b68dc2a9SMark Fasheh if (ret) { 439572c83acSJosef Bacik btrfs_tree_unlock(cow); 440572c83acSJosef Bacik free_extent_buffer(cow); 44166642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 442b68dc2a9SMark Fasheh return ret; 443b68dc2a9SMark Fasheh } 4441a40e23bSZheng Yan 44592a7cc42SQu Wenruo if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) { 44683d4cfd4SJosef Bacik ret = btrfs_reloc_cow_block(trans, root, buf, cow); 44793314e3bSZhaolei if (ret) { 448572c83acSJosef Bacik btrfs_tree_unlock(cow); 449572c83acSJosef Bacik free_extent_buffer(cow); 45066642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 45183d4cfd4SJosef Bacik return ret; 45283d4cfd4SJosef Bacik } 45393314e3bSZhaolei } 4543fd0a558SYan, Zheng 4556702ed49SChris Mason if (buf == root->node) { 456925baeddSChris Mason WARN_ON(parent && parent != buf); 4575d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 4585d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 4595d4f98a2SYan Zheng parent_start = buf->start; 460925baeddSChris Mason 46167439dadSDavid Sterba atomic_inc(&cow->refs); 462406808abSFilipe Manana ret = btrfs_tree_mod_log_insert_root(root->node, cow, true); 463d9d19a01SDavid Sterba BUG_ON(ret < 0); 464240f62c8SChris Mason rcu_assign_pointer(root->node, cow); 465925baeddSChris Mason 4667a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), buf, 4677a163608SFilipe Manana parent_start, last_ref); 4685f39d397SChris Mason free_extent_buffer(buf); 4690b86a832SChris Mason add_root_to_dirty_list(root); 4706702ed49SChris Mason } else { 4715d4f98a2SYan Zheng WARN_ON(trans->transid != btrfs_header_generation(parent)); 472f3a84ccdSFilipe Manana btrfs_tree_mod_log_insert_key(parent, parent_slot, 473f3a84ccdSFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE, GFP_NOFS); 4745f39d397SChris Mason btrfs_set_node_blockptr(parent, parent_slot, 475db94535dSChris Mason cow->start); 47674493f7aSChris Mason btrfs_set_node_ptr_generation(parent, parent_slot, 47774493f7aSChris Mason trans->transid); 4786702ed49SChris Mason btrfs_mark_buffer_dirty(parent); 4795de865eeSFilipe David Borba Manana if (last_ref) { 480f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_free_eb(buf); 4815de865eeSFilipe David Borba Manana if (ret) { 482572c83acSJosef Bacik btrfs_tree_unlock(cow); 483572c83acSJosef Bacik free_extent_buffer(cow); 48466642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 4855de865eeSFilipe David Borba Manana return ret; 4865de865eeSFilipe David Borba Manana } 4875de865eeSFilipe David Borba Manana } 4887a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), buf, 4897a163608SFilipe Manana parent_start, last_ref); 4906702ed49SChris Mason } 491925baeddSChris Mason if (unlock_orig) 492925baeddSChris Mason btrfs_tree_unlock(buf); 4933083ee2eSJosef Bacik free_extent_buffer_stale(buf); 4946702ed49SChris Mason btrfs_mark_buffer_dirty(cow); 4956702ed49SChris Mason *cow_ret = cow; 4966702ed49SChris Mason return 0; 4976702ed49SChris Mason } 4986702ed49SChris Mason 4995d4f98a2SYan Zheng static inline int should_cow_block(struct btrfs_trans_handle *trans, 5005d4f98a2SYan Zheng struct btrfs_root *root, 5015d4f98a2SYan Zheng struct extent_buffer *buf) 5025d4f98a2SYan Zheng { 503f5ee5c9aSJeff Mahoney if (btrfs_is_testing(root->fs_info)) 504faa2dbf0SJosef Bacik return 0; 505fccb84c9SDavid Sterba 506d1980131SDavid Sterba /* Ensure we can see the FORCE_COW bit */ 507d1980131SDavid Sterba smp_mb__before_atomic(); 508f1ebcc74SLiu Bo 509f1ebcc74SLiu Bo /* 510f1ebcc74SLiu Bo * We do not need to cow a block if 511f1ebcc74SLiu Bo * 1) this block is not created or changed in this transaction; 512f1ebcc74SLiu Bo * 2) this block does not belong to TREE_RELOC tree; 513f1ebcc74SLiu Bo * 3) the root is not forced COW. 514f1ebcc74SLiu Bo * 515f1ebcc74SLiu Bo * What is forced COW: 51601327610SNicholas D Steeves * when we create snapshot during committing the transaction, 51752042d8eSAndrea Gelmini * after we've finished copying src root, we must COW the shared 518f1ebcc74SLiu Bo * block to ensure the metadata consistency. 519f1ebcc74SLiu Bo */ 5205d4f98a2SYan Zheng if (btrfs_header_generation(buf) == trans->transid && 5215d4f98a2SYan Zheng !btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN) && 5225d4f98a2SYan Zheng !(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID && 523f1ebcc74SLiu Bo btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)) && 52427cdeb70SMiao Xie !test_bit(BTRFS_ROOT_FORCE_COW, &root->state)) 5255d4f98a2SYan Zheng return 0; 5265d4f98a2SYan Zheng return 1; 5275d4f98a2SYan Zheng } 5285d4f98a2SYan Zheng 529d352ac68SChris Mason /* 530d352ac68SChris Mason * cows a single block, see __btrfs_cow_block for the real work. 53101327610SNicholas D Steeves * This version of it has extra checks so that a block isn't COWed more than 532d352ac68SChris Mason * once per transaction, as long as it hasn't been written yet 533d352ac68SChris Mason */ 534d397712bSChris Mason noinline int btrfs_cow_block(struct btrfs_trans_handle *trans, 5355f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *buf, 5365f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 5379631e4ccSJosef Bacik struct extent_buffer **cow_ret, 5389631e4ccSJosef Bacik enum btrfs_lock_nesting nest) 53902217ed2SChris Mason { 5400b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 5416702ed49SChris Mason u64 search_start; 542f510cfecSChris Mason int ret; 543dc17ff8fSChris Mason 54483354f07SJosef Bacik if (test_bit(BTRFS_ROOT_DELETING, &root->state)) 54583354f07SJosef Bacik btrfs_err(fs_info, 54683354f07SJosef Bacik "COW'ing blocks on a fs root that's being dropped"); 54783354f07SJosef Bacik 5480b246afaSJeff Mahoney if (trans->transaction != fs_info->running_transaction) 54931b1a2bdSJulia Lawall WARN(1, KERN_CRIT "trans %llu running %llu\n", 550c1c9ff7cSGeert Uytterhoeven trans->transid, 5510b246afaSJeff Mahoney fs_info->running_transaction->transid); 55231b1a2bdSJulia Lawall 5530b246afaSJeff Mahoney if (trans->transid != fs_info->generation) 55431b1a2bdSJulia Lawall WARN(1, KERN_CRIT "trans %llu running %llu\n", 5550b246afaSJeff Mahoney trans->transid, fs_info->generation); 556dc17ff8fSChris Mason 5575d4f98a2SYan Zheng if (!should_cow_block(trans, root, buf)) { 55802217ed2SChris Mason *cow_ret = buf; 55902217ed2SChris Mason return 0; 56002217ed2SChris Mason } 561c487685dSChris Mason 562ee22184bSByongho Lee search_start = buf->start & ~((u64)SZ_1G - 1); 563b4ce94deSChris Mason 564f616f5cdSQu Wenruo /* 565f616f5cdSQu Wenruo * Before CoWing this block for later modification, check if it's 566f616f5cdSQu Wenruo * the subtree root and do the delayed subtree trace if needed. 567f616f5cdSQu Wenruo * 568f616f5cdSQu Wenruo * Also We don't care about the error, as it's handled internally. 569f616f5cdSQu Wenruo */ 570f616f5cdSQu Wenruo btrfs_qgroup_trace_subtree_after_cow(trans, root, buf); 571f510cfecSChris Mason ret = __btrfs_cow_block(trans, root, buf, parent, 5729631e4ccSJosef Bacik parent_slot, cow_ret, search_start, 0, nest); 5731abe9b8aSliubo 5741abe9b8aSliubo trace_btrfs_cow_block(root, buf, *cow_ret); 5751abe9b8aSliubo 576f510cfecSChris Mason return ret; 5772c90e5d6SChris Mason } 578f75e2b79SJosef Bacik ALLOW_ERROR_INJECTION(btrfs_cow_block, ERRNO); 5796702ed49SChris Mason 580d352ac68SChris Mason /* 581d352ac68SChris Mason * helper function for defrag to decide if two blocks pointed to by a 582d352ac68SChris Mason * node are actually close by 583d352ac68SChris Mason */ 5846b80053dSChris Mason static int close_blocks(u64 blocknr, u64 other, u32 blocksize) 5856702ed49SChris Mason { 5866b80053dSChris Mason if (blocknr < other && other - (blocknr + blocksize) < 32768) 5876702ed49SChris Mason return 1; 5886b80053dSChris Mason if (blocknr > other && blocknr - (other + blocksize) < 32768) 5896702ed49SChris Mason return 1; 59002217ed2SChris Mason return 0; 59102217ed2SChris Mason } 59202217ed2SChris Mason 593ce6ef5abSDavid Sterba #ifdef __LITTLE_ENDIAN 594ce6ef5abSDavid Sterba 595ce6ef5abSDavid Sterba /* 596ce6ef5abSDavid Sterba * Compare two keys, on little-endian the disk order is same as CPU order and 597ce6ef5abSDavid Sterba * we can avoid the conversion. 598ce6ef5abSDavid Sterba */ 599ce6ef5abSDavid Sterba static int comp_keys(const struct btrfs_disk_key *disk_key, 600ce6ef5abSDavid Sterba const struct btrfs_key *k2) 601ce6ef5abSDavid Sterba { 602ce6ef5abSDavid Sterba const struct btrfs_key *k1 = (const struct btrfs_key *)disk_key; 603ce6ef5abSDavid Sterba 604ce6ef5abSDavid Sterba return btrfs_comp_cpu_keys(k1, k2); 605ce6ef5abSDavid Sterba } 606ce6ef5abSDavid Sterba 607ce6ef5abSDavid Sterba #else 608ce6ef5abSDavid Sterba 609081e9573SChris Mason /* 610081e9573SChris Mason * compare two keys in a memcmp fashion 611081e9573SChris Mason */ 612310712b2SOmar Sandoval static int comp_keys(const struct btrfs_disk_key *disk, 613310712b2SOmar Sandoval const struct btrfs_key *k2) 614081e9573SChris Mason { 615081e9573SChris Mason struct btrfs_key k1; 616081e9573SChris Mason 617081e9573SChris Mason btrfs_disk_key_to_cpu(&k1, disk); 618081e9573SChris Mason 61920736abaSDiego Calleja return btrfs_comp_cpu_keys(&k1, k2); 620081e9573SChris Mason } 621ce6ef5abSDavid Sterba #endif 622081e9573SChris Mason 623f3465ca4SJosef Bacik /* 624f3465ca4SJosef Bacik * same as comp_keys only with two btrfs_key's 625f3465ca4SJosef Bacik */ 626e1f60a65SDavid Sterba int __pure btrfs_comp_cpu_keys(const struct btrfs_key *k1, const struct btrfs_key *k2) 627f3465ca4SJosef Bacik { 628f3465ca4SJosef Bacik if (k1->objectid > k2->objectid) 629f3465ca4SJosef Bacik return 1; 630f3465ca4SJosef Bacik if (k1->objectid < k2->objectid) 631f3465ca4SJosef Bacik return -1; 632f3465ca4SJosef Bacik if (k1->type > k2->type) 633f3465ca4SJosef Bacik return 1; 634f3465ca4SJosef Bacik if (k1->type < k2->type) 635f3465ca4SJosef Bacik return -1; 636f3465ca4SJosef Bacik if (k1->offset > k2->offset) 637f3465ca4SJosef Bacik return 1; 638f3465ca4SJosef Bacik if (k1->offset < k2->offset) 639f3465ca4SJosef Bacik return -1; 640f3465ca4SJosef Bacik return 0; 641f3465ca4SJosef Bacik } 642081e9573SChris Mason 643d352ac68SChris Mason /* 644d352ac68SChris Mason * this is used by the defrag code to go through all the 645d352ac68SChris Mason * leaves pointed to by a node and reallocate them so that 646d352ac68SChris Mason * disk order is close to key order 647d352ac68SChris Mason */ 6486702ed49SChris Mason int btrfs_realloc_node(struct btrfs_trans_handle *trans, 6495f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *parent, 650de78b51aSEric Sandeen int start_slot, u64 *last_ret, 651a6b6e75eSChris Mason struct btrfs_key *progress) 6526702ed49SChris Mason { 6530b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 6546b80053dSChris Mason struct extent_buffer *cur; 6556702ed49SChris Mason u64 blocknr; 656e9d0b13bSChris Mason u64 search_start = *last_ret; 657e9d0b13bSChris Mason u64 last_block = 0; 6586702ed49SChris Mason u64 other; 6596702ed49SChris Mason u32 parent_nritems; 6606702ed49SChris Mason int end_slot; 6616702ed49SChris Mason int i; 6626702ed49SChris Mason int err = 0; 6636b80053dSChris Mason u32 blocksize; 664081e9573SChris Mason int progress_passed = 0; 665081e9573SChris Mason struct btrfs_disk_key disk_key; 6666702ed49SChris Mason 6670b246afaSJeff Mahoney WARN_ON(trans->transaction != fs_info->running_transaction); 6680b246afaSJeff Mahoney WARN_ON(trans->transid != fs_info->generation); 66986479a04SChris Mason 6706b80053dSChris Mason parent_nritems = btrfs_header_nritems(parent); 6710b246afaSJeff Mahoney blocksize = fs_info->nodesize; 6725dfe2be7SFilipe Manana end_slot = parent_nritems - 1; 6736702ed49SChris Mason 6745dfe2be7SFilipe Manana if (parent_nritems <= 1) 6756702ed49SChris Mason return 0; 6766702ed49SChris Mason 6775dfe2be7SFilipe Manana for (i = start_slot; i <= end_slot; i++) { 6786702ed49SChris Mason int close = 1; 679a6b6e75eSChris Mason 680081e9573SChris Mason btrfs_node_key(parent, &disk_key, i); 681081e9573SChris Mason if (!progress_passed && comp_keys(&disk_key, progress) < 0) 682081e9573SChris Mason continue; 683081e9573SChris Mason 684081e9573SChris Mason progress_passed = 1; 6856b80053dSChris Mason blocknr = btrfs_node_blockptr(parent, i); 686e9d0b13bSChris Mason if (last_block == 0) 687e9d0b13bSChris Mason last_block = blocknr; 6885708b959SChris Mason 6896702ed49SChris Mason if (i > 0) { 6906b80053dSChris Mason other = btrfs_node_blockptr(parent, i - 1); 6916b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 6926702ed49SChris Mason } 6935dfe2be7SFilipe Manana if (!close && i < end_slot) { 6946b80053dSChris Mason other = btrfs_node_blockptr(parent, i + 1); 6956b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 6966702ed49SChris Mason } 697e9d0b13bSChris Mason if (close) { 698e9d0b13bSChris Mason last_block = blocknr; 6996702ed49SChris Mason continue; 700e9d0b13bSChris Mason } 7016702ed49SChris Mason 702206983b7SJosef Bacik cur = btrfs_read_node_slot(parent, i); 703206983b7SJosef Bacik if (IS_ERR(cur)) 70464c043deSLiu Bo return PTR_ERR(cur); 705e9d0b13bSChris Mason if (search_start == 0) 7066b80053dSChris Mason search_start = last_block; 707e9d0b13bSChris Mason 708e7a84565SChris Mason btrfs_tree_lock(cur); 7096b80053dSChris Mason err = __btrfs_cow_block(trans, root, cur, parent, i, 710e7a84565SChris Mason &cur, search_start, 7116b80053dSChris Mason min(16 * blocksize, 7129631e4ccSJosef Bacik (end_slot - i) * blocksize), 7139631e4ccSJosef Bacik BTRFS_NESTING_COW); 714252c38f0SYan if (err) { 715e7a84565SChris Mason btrfs_tree_unlock(cur); 7166b80053dSChris Mason free_extent_buffer(cur); 7176702ed49SChris Mason break; 718252c38f0SYan } 719e7a84565SChris Mason search_start = cur->start; 720e7a84565SChris Mason last_block = cur->start; 721f2183bdeSChris Mason *last_ret = search_start; 722e7a84565SChris Mason btrfs_tree_unlock(cur); 723e7a84565SChris Mason free_extent_buffer(cur); 7246702ed49SChris Mason } 7256702ed49SChris Mason return err; 7266702ed49SChris Mason } 7276702ed49SChris Mason 72874123bd7SChris Mason /* 729fb81212cSFilipe Manana * Search for a key in the given extent_buffer. 7305f39d397SChris Mason * 731fb81212cSFilipe Manana * The lower boundary for the search is specified by the slot number @low. Use a 732fb81212cSFilipe Manana * value of 0 to search over the whole extent buffer. 73374123bd7SChris Mason * 734fb81212cSFilipe Manana * The slot in the extent buffer is returned via @slot. If the key exists in the 735fb81212cSFilipe Manana * extent buffer, then @slot will point to the slot where the key is, otherwise 736fb81212cSFilipe Manana * it points to the slot where you would insert the key. 737fb81212cSFilipe Manana * 738fb81212cSFilipe Manana * Slot may point to the total number of items (i.e. one position beyond the last 739fb81212cSFilipe Manana * key) if the key is bigger than the last key in the extent buffer. 74074123bd7SChris Mason */ 741fb81212cSFilipe Manana static noinline int generic_bin_search(struct extent_buffer *eb, int low, 74267d5e289SMarcos Paulo de Souza const struct btrfs_key *key, int *slot) 743be0e5c09SChris Mason { 744fb81212cSFilipe Manana unsigned long p; 745fb81212cSFilipe Manana int item_size; 74667d5e289SMarcos Paulo de Souza int high = btrfs_header_nritems(eb); 747be0e5c09SChris Mason int ret; 7485cd17f34SDavid Sterba const int key_size = sizeof(struct btrfs_disk_key); 749be0e5c09SChris Mason 7505e24e9afSLiu Bo if (low > high) { 7515e24e9afSLiu Bo btrfs_err(eb->fs_info, 7525e24e9afSLiu Bo "%s: low (%d) > high (%d) eb %llu owner %llu level %d", 7535e24e9afSLiu Bo __func__, low, high, eb->start, 7545e24e9afSLiu Bo btrfs_header_owner(eb), btrfs_header_level(eb)); 7555e24e9afSLiu Bo return -EINVAL; 7565e24e9afSLiu Bo } 7575e24e9afSLiu Bo 758fb81212cSFilipe Manana if (btrfs_header_level(eb) == 0) { 759fb81212cSFilipe Manana p = offsetof(struct btrfs_leaf, items); 760fb81212cSFilipe Manana item_size = sizeof(struct btrfs_item); 761fb81212cSFilipe Manana } else { 762fb81212cSFilipe Manana p = offsetof(struct btrfs_node, ptrs); 763fb81212cSFilipe Manana item_size = sizeof(struct btrfs_key_ptr); 764fb81212cSFilipe Manana } 765fb81212cSFilipe Manana 766be0e5c09SChris Mason while (low < high) { 7675cd17f34SDavid Sterba unsigned long oip; 7685cd17f34SDavid Sterba unsigned long offset; 7695cd17f34SDavid Sterba struct btrfs_disk_key *tmp; 7705cd17f34SDavid Sterba struct btrfs_disk_key unaligned; 7715cd17f34SDavid Sterba int mid; 7725cd17f34SDavid Sterba 773be0e5c09SChris Mason mid = (low + high) / 2; 7745f39d397SChris Mason offset = p + mid * item_size; 7755cd17f34SDavid Sterba oip = offset_in_page(offset); 7765f39d397SChris Mason 7775cd17f34SDavid Sterba if (oip + key_size <= PAGE_SIZE) { 778884b07d0SQu Wenruo const unsigned long idx = get_eb_page_index(offset); 7795cd17f34SDavid Sterba char *kaddr = page_address(eb->pages[idx]); 780934d375bSChris Mason 781884b07d0SQu Wenruo oip = get_eb_offset_in_page(eb, offset); 7825cd17f34SDavid Sterba tmp = (struct btrfs_disk_key *)(kaddr + oip); 7835cd17f34SDavid Sterba } else { 7845cd17f34SDavid Sterba read_extent_buffer(eb, &unaligned, offset, key_size); 7855f39d397SChris Mason tmp = &unaligned; 786479965d6SChris Mason } 787479965d6SChris Mason 788be0e5c09SChris Mason ret = comp_keys(tmp, key); 789be0e5c09SChris Mason 790be0e5c09SChris Mason if (ret < 0) 791be0e5c09SChris Mason low = mid + 1; 792be0e5c09SChris Mason else if (ret > 0) 793be0e5c09SChris Mason high = mid; 794be0e5c09SChris Mason else { 795be0e5c09SChris Mason *slot = mid; 796be0e5c09SChris Mason return 0; 797be0e5c09SChris Mason } 798be0e5c09SChris Mason } 799be0e5c09SChris Mason *slot = low; 800be0e5c09SChris Mason return 1; 801be0e5c09SChris Mason } 802be0e5c09SChris Mason 80397571fd0SChris Mason /* 804fb81212cSFilipe Manana * Simple binary search on an extent buffer. Works for both leaves and nodes, and 805fb81212cSFilipe Manana * always searches over the whole range of keys (slot 0 to slot 'nritems - 1'). 80697571fd0SChris Mason */ 807a74b35ecSNikolay Borisov int btrfs_bin_search(struct extent_buffer *eb, const struct btrfs_key *key, 808e3b83361SQu Wenruo int *slot) 809be0e5c09SChris Mason { 810fb81212cSFilipe Manana return generic_bin_search(eb, 0, key, slot); 811be0e5c09SChris Mason } 812be0e5c09SChris Mason 813f0486c68SYan, Zheng static void root_add_used(struct btrfs_root *root, u32 size) 814f0486c68SYan, Zheng { 815f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 816f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 817f0486c68SYan, Zheng btrfs_root_used(&root->root_item) + size); 818f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 819f0486c68SYan, Zheng } 820f0486c68SYan, Zheng 821f0486c68SYan, Zheng static void root_sub_used(struct btrfs_root *root, u32 size) 822f0486c68SYan, Zheng { 823f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 824f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 825f0486c68SYan, Zheng btrfs_root_used(&root->root_item) - size); 826f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 827f0486c68SYan, Zheng } 828f0486c68SYan, Zheng 829d352ac68SChris Mason /* given a node and slot number, this reads the blocks it points to. The 830d352ac68SChris Mason * extent buffer is returned with a reference taken (but unlocked). 831d352ac68SChris Mason */ 8324b231ae4SDavid Sterba struct extent_buffer *btrfs_read_node_slot(struct extent_buffer *parent, 8334b231ae4SDavid Sterba int slot) 834bb803951SChris Mason { 835ca7a79adSChris Mason int level = btrfs_header_level(parent); 836416bc658SJosef Bacik struct extent_buffer *eb; 837581c1760SQu Wenruo struct btrfs_key first_key; 838416bc658SJosef Bacik 839fb770ae4SLiu Bo if (slot < 0 || slot >= btrfs_header_nritems(parent)) 840fb770ae4SLiu Bo return ERR_PTR(-ENOENT); 841ca7a79adSChris Mason 842ca7a79adSChris Mason BUG_ON(level == 0); 843ca7a79adSChris Mason 844581c1760SQu Wenruo btrfs_node_key_to_cpu(parent, &first_key, slot); 845d0d20b0fSDavid Sterba eb = read_tree_block(parent->fs_info, btrfs_node_blockptr(parent, slot), 8461b7ec85eSJosef Bacik btrfs_header_owner(parent), 847581c1760SQu Wenruo btrfs_node_ptr_generation(parent, slot), 848581c1760SQu Wenruo level - 1, &first_key); 849fb770ae4SLiu Bo if (!IS_ERR(eb) && !extent_buffer_uptodate(eb)) { 850416bc658SJosef Bacik free_extent_buffer(eb); 851fb770ae4SLiu Bo eb = ERR_PTR(-EIO); 852416bc658SJosef Bacik } 853416bc658SJosef Bacik 854416bc658SJosef Bacik return eb; 855bb803951SChris Mason } 856bb803951SChris Mason 857d352ac68SChris Mason /* 858d352ac68SChris Mason * node level balancing, used to make sure nodes are in proper order for 859d352ac68SChris Mason * item deletion. We balance from the top down, so we have to make sure 860d352ac68SChris Mason * that a deletion won't leave an node completely empty later on. 861d352ac68SChris Mason */ 862e02119d5SChris Mason static noinline int balance_level(struct btrfs_trans_handle *trans, 86398ed5174SChris Mason struct btrfs_root *root, 86498ed5174SChris Mason struct btrfs_path *path, int level) 865bb803951SChris Mason { 8660b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 8675f39d397SChris Mason struct extent_buffer *right = NULL; 8685f39d397SChris Mason struct extent_buffer *mid; 8695f39d397SChris Mason struct extent_buffer *left = NULL; 8705f39d397SChris Mason struct extent_buffer *parent = NULL; 871bb803951SChris Mason int ret = 0; 872bb803951SChris Mason int wret; 873bb803951SChris Mason int pslot; 874bb803951SChris Mason int orig_slot = path->slots[level]; 87579f95c82SChris Mason u64 orig_ptr; 876bb803951SChris Mason 87798e6b1ebSLiu Bo ASSERT(level > 0); 878bb803951SChris Mason 8795f39d397SChris Mason mid = path->nodes[level]; 880b4ce94deSChris Mason 881ac5887c8SJosef Bacik WARN_ON(path->locks[level] != BTRFS_WRITE_LOCK); 8827bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 8837bb86316SChris Mason 8841d4f8a0cSChris Mason orig_ptr = btrfs_node_blockptr(mid, orig_slot); 88579f95c82SChris Mason 886a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 8875f39d397SChris Mason parent = path->nodes[level + 1]; 888bb803951SChris Mason pslot = path->slots[level + 1]; 889a05a9bb1SLi Zefan } 890bb803951SChris Mason 89140689478SChris Mason /* 89240689478SChris Mason * deal with the case where there is only one pointer in the root 89340689478SChris Mason * by promoting the node below to a root 89440689478SChris Mason */ 8955f39d397SChris Mason if (!parent) { 8965f39d397SChris Mason struct extent_buffer *child; 897bb803951SChris Mason 8985f39d397SChris Mason if (btrfs_header_nritems(mid) != 1) 899bb803951SChris Mason return 0; 900bb803951SChris Mason 901bb803951SChris Mason /* promote the child to a root */ 9024b231ae4SDavid Sterba child = btrfs_read_node_slot(mid, 0); 903fb770ae4SLiu Bo if (IS_ERR(child)) { 904fb770ae4SLiu Bo ret = PTR_ERR(child); 9050b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 906305a26afSMark Fasheh goto enospc; 907305a26afSMark Fasheh } 908305a26afSMark Fasheh 909925baeddSChris Mason btrfs_tree_lock(child); 9109631e4ccSJosef Bacik ret = btrfs_cow_block(trans, root, child, mid, 0, &child, 9119631e4ccSJosef Bacik BTRFS_NESTING_COW); 912f0486c68SYan, Zheng if (ret) { 913f0486c68SYan, Zheng btrfs_tree_unlock(child); 914f0486c68SYan, Zheng free_extent_buffer(child); 915f0486c68SYan, Zheng goto enospc; 916f0486c68SYan, Zheng } 9172f375ab9SYan 918406808abSFilipe Manana ret = btrfs_tree_mod_log_insert_root(root->node, child, true); 919d9d19a01SDavid Sterba BUG_ON(ret < 0); 920240f62c8SChris Mason rcu_assign_pointer(root->node, child); 921925baeddSChris Mason 9220b86a832SChris Mason add_root_to_dirty_list(root); 923925baeddSChris Mason btrfs_tree_unlock(child); 924b4ce94deSChris Mason 925925baeddSChris Mason path->locks[level] = 0; 926bb803951SChris Mason path->nodes[level] = NULL; 9276a884d7dSDavid Sterba btrfs_clean_tree_block(mid); 928925baeddSChris Mason btrfs_tree_unlock(mid); 929bb803951SChris Mason /* once for the path */ 9305f39d397SChris Mason free_extent_buffer(mid); 931f0486c68SYan, Zheng 932f0486c68SYan, Zheng root_sub_used(root, mid->len); 9337a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), mid, 0, 1); 934bb803951SChris Mason /* once for the root ptr */ 9353083ee2eSJosef Bacik free_extent_buffer_stale(mid); 936f0486c68SYan, Zheng return 0; 937bb803951SChris Mason } 9385f39d397SChris Mason if (btrfs_header_nritems(mid) > 9390b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 4) 940bb803951SChris Mason return 0; 941bb803951SChris Mason 9424b231ae4SDavid Sterba left = btrfs_read_node_slot(parent, pslot - 1); 943fb770ae4SLiu Bo if (IS_ERR(left)) 944fb770ae4SLiu Bo left = NULL; 945fb770ae4SLiu Bo 9465f39d397SChris Mason if (left) { 947bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT); 9485f39d397SChris Mason wret = btrfs_cow_block(trans, root, left, 9499631e4ccSJosef Bacik parent, pslot - 1, &left, 950bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW); 95154aa1f4dSChris Mason if (wret) { 95254aa1f4dSChris Mason ret = wret; 95354aa1f4dSChris Mason goto enospc; 95454aa1f4dSChris Mason } 9552cc58cf2SChris Mason } 956fb770ae4SLiu Bo 9574b231ae4SDavid Sterba right = btrfs_read_node_slot(parent, pslot + 1); 958fb770ae4SLiu Bo if (IS_ERR(right)) 959fb770ae4SLiu Bo right = NULL; 960fb770ae4SLiu Bo 9615f39d397SChris Mason if (right) { 962bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT); 9635f39d397SChris Mason wret = btrfs_cow_block(trans, root, right, 9649631e4ccSJosef Bacik parent, pslot + 1, &right, 965bf59a5a2SJosef Bacik BTRFS_NESTING_RIGHT_COW); 9662cc58cf2SChris Mason if (wret) { 9672cc58cf2SChris Mason ret = wret; 9682cc58cf2SChris Mason goto enospc; 9692cc58cf2SChris Mason } 9702cc58cf2SChris Mason } 9712cc58cf2SChris Mason 9722cc58cf2SChris Mason /* first, try to make some room in the middle buffer */ 9735f39d397SChris Mason if (left) { 9745f39d397SChris Mason orig_slot += btrfs_header_nritems(left); 975d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 1); 97679f95c82SChris Mason if (wret < 0) 97779f95c82SChris Mason ret = wret; 978bb803951SChris Mason } 97979f95c82SChris Mason 98079f95c82SChris Mason /* 98179f95c82SChris Mason * then try to empty the right most buffer into the middle 98279f95c82SChris Mason */ 9835f39d397SChris Mason if (right) { 984d30a668fSDavid Sterba wret = push_node_left(trans, mid, right, 1); 98554aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 98679f95c82SChris Mason ret = wret; 9875f39d397SChris Mason if (btrfs_header_nritems(right) == 0) { 9886a884d7dSDavid Sterba btrfs_clean_tree_block(right); 989925baeddSChris Mason btrfs_tree_unlock(right); 990afe5fea7STsutomu Itoh del_ptr(root, path, level + 1, pslot + 1); 991f0486c68SYan, Zheng root_sub_used(root, right->len); 9927a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), right, 9937a163608SFilipe Manana 0, 1); 9943083ee2eSJosef Bacik free_extent_buffer_stale(right); 995f0486c68SYan, Zheng right = NULL; 996bb803951SChris Mason } else { 9975f39d397SChris Mason struct btrfs_disk_key right_key; 9985f39d397SChris Mason btrfs_node_key(right, &right_key, 0); 999f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot + 1, 1000f3a84ccdSFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE, GFP_NOFS); 10010e82bcfeSDavid Sterba BUG_ON(ret < 0); 10025f39d397SChris Mason btrfs_set_node_key(parent, &right_key, pslot + 1); 10035f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 1004bb803951SChris Mason } 1005bb803951SChris Mason } 10065f39d397SChris Mason if (btrfs_header_nritems(mid) == 1) { 100779f95c82SChris Mason /* 100879f95c82SChris Mason * we're not allowed to leave a node with one item in the 100979f95c82SChris Mason * tree during a delete. A deletion from lower in the tree 101079f95c82SChris Mason * could try to delete the only pointer in this node. 101179f95c82SChris Mason * So, pull some keys from the left. 101279f95c82SChris Mason * There has to be a left pointer at this point because 101379f95c82SChris Mason * otherwise we would have pulled some pointers from the 101479f95c82SChris Mason * right 101579f95c82SChris Mason */ 1016305a26afSMark Fasheh if (!left) { 1017305a26afSMark Fasheh ret = -EROFS; 10180b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 1019305a26afSMark Fasheh goto enospc; 1020305a26afSMark Fasheh } 102155d32ed8SDavid Sterba wret = balance_node_right(trans, mid, left); 102254aa1f4dSChris Mason if (wret < 0) { 102379f95c82SChris Mason ret = wret; 102454aa1f4dSChris Mason goto enospc; 102554aa1f4dSChris Mason } 1026bce4eae9SChris Mason if (wret == 1) { 1027d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 1); 1028bce4eae9SChris Mason if (wret < 0) 1029bce4eae9SChris Mason ret = wret; 1030bce4eae9SChris Mason } 103179f95c82SChris Mason BUG_ON(wret == 1); 103279f95c82SChris Mason } 10335f39d397SChris Mason if (btrfs_header_nritems(mid) == 0) { 10346a884d7dSDavid Sterba btrfs_clean_tree_block(mid); 1035925baeddSChris Mason btrfs_tree_unlock(mid); 1036afe5fea7STsutomu Itoh del_ptr(root, path, level + 1, pslot); 1037f0486c68SYan, Zheng root_sub_used(root, mid->len); 10387a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), mid, 0, 1); 10393083ee2eSJosef Bacik free_extent_buffer_stale(mid); 1040f0486c68SYan, Zheng mid = NULL; 104179f95c82SChris Mason } else { 104279f95c82SChris Mason /* update the parent key to reflect our changes */ 10435f39d397SChris Mason struct btrfs_disk_key mid_key; 10445f39d397SChris Mason btrfs_node_key(mid, &mid_key, 0); 1045f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot, 1046f3a84ccdSFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE, GFP_NOFS); 10470e82bcfeSDavid Sterba BUG_ON(ret < 0); 10485f39d397SChris Mason btrfs_set_node_key(parent, &mid_key, pslot); 10495f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 105079f95c82SChris Mason } 1051bb803951SChris Mason 105279f95c82SChris Mason /* update the path */ 10535f39d397SChris Mason if (left) { 10545f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 105567439dadSDavid Sterba atomic_inc(&left->refs); 1056925baeddSChris Mason /* left was locked after cow */ 10575f39d397SChris Mason path->nodes[level] = left; 1058bb803951SChris Mason path->slots[level + 1] -= 1; 1059bb803951SChris Mason path->slots[level] = orig_slot; 1060925baeddSChris Mason if (mid) { 1061925baeddSChris Mason btrfs_tree_unlock(mid); 10625f39d397SChris Mason free_extent_buffer(mid); 1063925baeddSChris Mason } 1064bb803951SChris Mason } else { 10655f39d397SChris Mason orig_slot -= btrfs_header_nritems(left); 1066bb803951SChris Mason path->slots[level] = orig_slot; 1067bb803951SChris Mason } 1068bb803951SChris Mason } 106979f95c82SChris Mason /* double check we haven't messed things up */ 1070e20d96d6SChris Mason if (orig_ptr != 10715f39d397SChris Mason btrfs_node_blockptr(path->nodes[level], path->slots[level])) 107279f95c82SChris Mason BUG(); 107354aa1f4dSChris Mason enospc: 1074925baeddSChris Mason if (right) { 1075925baeddSChris Mason btrfs_tree_unlock(right); 10765f39d397SChris Mason free_extent_buffer(right); 1077925baeddSChris Mason } 1078925baeddSChris Mason if (left) { 1079925baeddSChris Mason if (path->nodes[level] != left) 1080925baeddSChris Mason btrfs_tree_unlock(left); 10815f39d397SChris Mason free_extent_buffer(left); 1082925baeddSChris Mason } 1083bb803951SChris Mason return ret; 1084bb803951SChris Mason } 1085bb803951SChris Mason 1086d352ac68SChris Mason /* Node balancing for insertion. Here we only split or push nodes around 1087d352ac68SChris Mason * when they are completely full. This is also done top down, so we 1088d352ac68SChris Mason * have to be pessimistic. 1089d352ac68SChris Mason */ 1090d397712bSChris Mason static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans, 1091e66f709bSChris Mason struct btrfs_root *root, 1092e66f709bSChris Mason struct btrfs_path *path, int level) 1093e66f709bSChris Mason { 10940b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 10955f39d397SChris Mason struct extent_buffer *right = NULL; 10965f39d397SChris Mason struct extent_buffer *mid; 10975f39d397SChris Mason struct extent_buffer *left = NULL; 10985f39d397SChris Mason struct extent_buffer *parent = NULL; 1099e66f709bSChris Mason int ret = 0; 1100e66f709bSChris Mason int wret; 1101e66f709bSChris Mason int pslot; 1102e66f709bSChris Mason int orig_slot = path->slots[level]; 1103e66f709bSChris Mason 1104e66f709bSChris Mason if (level == 0) 1105e66f709bSChris Mason return 1; 1106e66f709bSChris Mason 11075f39d397SChris Mason mid = path->nodes[level]; 11087bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 1109e66f709bSChris Mason 1110a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 11115f39d397SChris Mason parent = path->nodes[level + 1]; 1112e66f709bSChris Mason pslot = path->slots[level + 1]; 1113a05a9bb1SLi Zefan } 1114e66f709bSChris Mason 11155f39d397SChris Mason if (!parent) 1116e66f709bSChris Mason return 1; 1117e66f709bSChris Mason 11184b231ae4SDavid Sterba left = btrfs_read_node_slot(parent, pslot - 1); 1119fb770ae4SLiu Bo if (IS_ERR(left)) 1120fb770ae4SLiu Bo left = NULL; 1121e66f709bSChris Mason 1122e66f709bSChris Mason /* first, try to make some room in the middle buffer */ 11235f39d397SChris Mason if (left) { 1124e66f709bSChris Mason u32 left_nr; 1125925baeddSChris Mason 1126bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT); 1127b4ce94deSChris Mason 11285f39d397SChris Mason left_nr = btrfs_header_nritems(left); 11290b246afaSJeff Mahoney if (left_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) { 113033ade1f8SChris Mason wret = 1; 113133ade1f8SChris Mason } else { 11325f39d397SChris Mason ret = btrfs_cow_block(trans, root, left, parent, 11339631e4ccSJosef Bacik pslot - 1, &left, 1134bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW); 113554aa1f4dSChris Mason if (ret) 113654aa1f4dSChris Mason wret = 1; 113754aa1f4dSChris Mason else { 1138d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 0); 113954aa1f4dSChris Mason } 114033ade1f8SChris Mason } 1141e66f709bSChris Mason if (wret < 0) 1142e66f709bSChris Mason ret = wret; 1143e66f709bSChris Mason if (wret == 0) { 11445f39d397SChris Mason struct btrfs_disk_key disk_key; 1145e66f709bSChris Mason orig_slot += left_nr; 11465f39d397SChris Mason btrfs_node_key(mid, &disk_key, 0); 1147f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot, 1148f3a84ccdSFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE, GFP_NOFS); 11490e82bcfeSDavid Sterba BUG_ON(ret < 0); 11505f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot); 11515f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 11525f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 11535f39d397SChris Mason path->nodes[level] = left; 1154e66f709bSChris Mason path->slots[level + 1] -= 1; 1155e66f709bSChris Mason path->slots[level] = orig_slot; 1156925baeddSChris Mason btrfs_tree_unlock(mid); 11575f39d397SChris Mason free_extent_buffer(mid); 1158e66f709bSChris Mason } else { 1159e66f709bSChris Mason orig_slot -= 11605f39d397SChris Mason btrfs_header_nritems(left); 1161e66f709bSChris Mason path->slots[level] = orig_slot; 1162925baeddSChris Mason btrfs_tree_unlock(left); 11635f39d397SChris Mason free_extent_buffer(left); 1164e66f709bSChris Mason } 1165e66f709bSChris Mason return 0; 1166e66f709bSChris Mason } 1167925baeddSChris Mason btrfs_tree_unlock(left); 11685f39d397SChris Mason free_extent_buffer(left); 1169e66f709bSChris Mason } 11704b231ae4SDavid Sterba right = btrfs_read_node_slot(parent, pslot + 1); 1171fb770ae4SLiu Bo if (IS_ERR(right)) 1172fb770ae4SLiu Bo right = NULL; 1173e66f709bSChris Mason 1174e66f709bSChris Mason /* 1175e66f709bSChris Mason * then try to empty the right most buffer into the middle 1176e66f709bSChris Mason */ 11775f39d397SChris Mason if (right) { 117833ade1f8SChris Mason u32 right_nr; 1179b4ce94deSChris Mason 1180bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT); 1181b4ce94deSChris Mason 11825f39d397SChris Mason right_nr = btrfs_header_nritems(right); 11830b246afaSJeff Mahoney if (right_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) { 118433ade1f8SChris Mason wret = 1; 118533ade1f8SChris Mason } else { 11865f39d397SChris Mason ret = btrfs_cow_block(trans, root, right, 11875f39d397SChris Mason parent, pslot + 1, 1188bf59a5a2SJosef Bacik &right, BTRFS_NESTING_RIGHT_COW); 118954aa1f4dSChris Mason if (ret) 119054aa1f4dSChris Mason wret = 1; 119154aa1f4dSChris Mason else { 119255d32ed8SDavid Sterba wret = balance_node_right(trans, right, mid); 119333ade1f8SChris Mason } 119454aa1f4dSChris Mason } 1195e66f709bSChris Mason if (wret < 0) 1196e66f709bSChris Mason ret = wret; 1197e66f709bSChris Mason if (wret == 0) { 11985f39d397SChris Mason struct btrfs_disk_key disk_key; 11995f39d397SChris Mason 12005f39d397SChris Mason btrfs_node_key(right, &disk_key, 0); 1201f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot + 1, 1202f3a84ccdSFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE, GFP_NOFS); 12030e82bcfeSDavid Sterba BUG_ON(ret < 0); 12045f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot + 1); 12055f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 12065f39d397SChris Mason 12075f39d397SChris Mason if (btrfs_header_nritems(mid) <= orig_slot) { 12085f39d397SChris Mason path->nodes[level] = right; 1209e66f709bSChris Mason path->slots[level + 1] += 1; 1210e66f709bSChris Mason path->slots[level] = orig_slot - 12115f39d397SChris Mason btrfs_header_nritems(mid); 1212925baeddSChris Mason btrfs_tree_unlock(mid); 12135f39d397SChris Mason free_extent_buffer(mid); 1214e66f709bSChris Mason } else { 1215925baeddSChris Mason btrfs_tree_unlock(right); 12165f39d397SChris Mason free_extent_buffer(right); 1217e66f709bSChris Mason } 1218e66f709bSChris Mason return 0; 1219e66f709bSChris Mason } 1220925baeddSChris Mason btrfs_tree_unlock(right); 12215f39d397SChris Mason free_extent_buffer(right); 1222e66f709bSChris Mason } 1223e66f709bSChris Mason return 1; 1224e66f709bSChris Mason } 1225e66f709bSChris Mason 122674123bd7SChris Mason /* 1227d352ac68SChris Mason * readahead one full node of leaves, finding things that are close 1228d352ac68SChris Mason * to the block in 'slot', and triggering ra on them. 12293c69faecSChris Mason */ 12302ff7e61eSJeff Mahoney static void reada_for_search(struct btrfs_fs_info *fs_info, 1231e02119d5SChris Mason struct btrfs_path *path, 123201f46658SChris Mason int level, int slot, u64 objectid) 12333c69faecSChris Mason { 12345f39d397SChris Mason struct extent_buffer *node; 123501f46658SChris Mason struct btrfs_disk_key disk_key; 12363c69faecSChris Mason u32 nritems; 12373c69faecSChris Mason u64 search; 1238a7175319SChris Mason u64 target; 12396b80053dSChris Mason u64 nread = 0; 1240ace75066SFilipe Manana u64 nread_max; 12416b80053dSChris Mason u32 nr; 12426b80053dSChris Mason u32 blocksize; 12436b80053dSChris Mason u32 nscan = 0; 1244db94535dSChris Mason 1245ace75066SFilipe Manana if (level != 1 && path->reada != READA_FORWARD_ALWAYS) 12463c69faecSChris Mason return; 12473c69faecSChris Mason 12486702ed49SChris Mason if (!path->nodes[level]) 12496702ed49SChris Mason return; 12506702ed49SChris Mason 12515f39d397SChris Mason node = path->nodes[level]; 1252925baeddSChris Mason 1253ace75066SFilipe Manana /* 1254ace75066SFilipe Manana * Since the time between visiting leaves is much shorter than the time 1255ace75066SFilipe Manana * between visiting nodes, limit read ahead of nodes to 1, to avoid too 1256ace75066SFilipe Manana * much IO at once (possibly random). 1257ace75066SFilipe Manana */ 1258ace75066SFilipe Manana if (path->reada == READA_FORWARD_ALWAYS) { 1259ace75066SFilipe Manana if (level > 1) 1260ace75066SFilipe Manana nread_max = node->fs_info->nodesize; 1261ace75066SFilipe Manana else 1262ace75066SFilipe Manana nread_max = SZ_128K; 1263ace75066SFilipe Manana } else { 1264ace75066SFilipe Manana nread_max = SZ_64K; 1265ace75066SFilipe Manana } 1266ace75066SFilipe Manana 12673c69faecSChris Mason search = btrfs_node_blockptr(node, slot); 12680b246afaSJeff Mahoney blocksize = fs_info->nodesize; 1269069a2e37SFilipe Manana if (path->reada != READA_FORWARD_ALWAYS) { 1270069a2e37SFilipe Manana struct extent_buffer *eb; 1271069a2e37SFilipe Manana 12720b246afaSJeff Mahoney eb = find_extent_buffer(fs_info, search); 12735f39d397SChris Mason if (eb) { 12745f39d397SChris Mason free_extent_buffer(eb); 12753c69faecSChris Mason return; 12763c69faecSChris Mason } 1277069a2e37SFilipe Manana } 12783c69faecSChris Mason 1279a7175319SChris Mason target = search; 12806b80053dSChris Mason 12815f39d397SChris Mason nritems = btrfs_header_nritems(node); 12826b80053dSChris Mason nr = slot; 128325b8b936SJosef Bacik 12843c69faecSChris Mason while (1) { 1285e4058b54SDavid Sterba if (path->reada == READA_BACK) { 12866b80053dSChris Mason if (nr == 0) 12873c69faecSChris Mason break; 12886b80053dSChris Mason nr--; 1289ace75066SFilipe Manana } else if (path->reada == READA_FORWARD || 1290ace75066SFilipe Manana path->reada == READA_FORWARD_ALWAYS) { 12916b80053dSChris Mason nr++; 12926b80053dSChris Mason if (nr >= nritems) 12936b80053dSChris Mason break; 12943c69faecSChris Mason } 1295e4058b54SDavid Sterba if (path->reada == READA_BACK && objectid) { 129601f46658SChris Mason btrfs_node_key(node, &disk_key, nr); 129701f46658SChris Mason if (btrfs_disk_key_objectid(&disk_key) != objectid) 129801f46658SChris Mason break; 129901f46658SChris Mason } 13006b80053dSChris Mason search = btrfs_node_blockptr(node, nr); 1301ace75066SFilipe Manana if (path->reada == READA_FORWARD_ALWAYS || 1302ace75066SFilipe Manana (search <= target && target - search <= 65536) || 1303a7175319SChris Mason (search > target && search - target <= 65536)) { 1304bfb484d9SJosef Bacik btrfs_readahead_node_child(node, nr); 13056b80053dSChris Mason nread += blocksize; 13063c69faecSChris Mason } 13076b80053dSChris Mason nscan++; 1308ace75066SFilipe Manana if (nread > nread_max || nscan > 32) 13096b80053dSChris Mason break; 13103c69faecSChris Mason } 13113c69faecSChris Mason } 1312925baeddSChris Mason 1313bfb484d9SJosef Bacik static noinline void reada_for_balance(struct btrfs_path *path, int level) 1314b4ce94deSChris Mason { 1315bfb484d9SJosef Bacik struct extent_buffer *parent; 1316b4ce94deSChris Mason int slot; 1317b4ce94deSChris Mason int nritems; 1318b4ce94deSChris Mason 13198c594ea8SChris Mason parent = path->nodes[level + 1]; 1320b4ce94deSChris Mason if (!parent) 13210b08851fSJosef Bacik return; 1322b4ce94deSChris Mason 1323b4ce94deSChris Mason nritems = btrfs_header_nritems(parent); 13248c594ea8SChris Mason slot = path->slots[level + 1]; 1325b4ce94deSChris Mason 1326bfb484d9SJosef Bacik if (slot > 0) 1327bfb484d9SJosef Bacik btrfs_readahead_node_child(parent, slot - 1); 1328bfb484d9SJosef Bacik if (slot + 1 < nritems) 1329bfb484d9SJosef Bacik btrfs_readahead_node_child(parent, slot + 1); 1330b4ce94deSChris Mason } 1331b4ce94deSChris Mason 1332b4ce94deSChris Mason 1333b4ce94deSChris Mason /* 1334d397712bSChris Mason * when we walk down the tree, it is usually safe to unlock the higher layers 1335d397712bSChris Mason * in the tree. The exceptions are when our path goes through slot 0, because 1336d397712bSChris Mason * operations on the tree might require changing key pointers higher up in the 1337d397712bSChris Mason * tree. 1338d352ac68SChris Mason * 1339d397712bSChris Mason * callers might also have set path->keep_locks, which tells this code to keep 1340d397712bSChris Mason * the lock if the path points to the last slot in the block. This is part of 1341d397712bSChris Mason * walking through the tree, and selecting the next slot in the higher block. 1342d352ac68SChris Mason * 1343d397712bSChris Mason * lowest_unlock sets the lowest level in the tree we're allowed to unlock. so 1344d397712bSChris Mason * if lowest_unlock is 1, level 0 won't be unlocked 1345d352ac68SChris Mason */ 1346e02119d5SChris Mason static noinline void unlock_up(struct btrfs_path *path, int level, 1347f7c79f30SChris Mason int lowest_unlock, int min_write_lock_level, 1348f7c79f30SChris Mason int *write_lock_level) 1349925baeddSChris Mason { 1350925baeddSChris Mason int i; 1351925baeddSChris Mason int skip_level = level; 1352051e1b9fSChris Mason int no_skips = 0; 1353925baeddSChris Mason struct extent_buffer *t; 1354925baeddSChris Mason 1355925baeddSChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 1356925baeddSChris Mason if (!path->nodes[i]) 1357925baeddSChris Mason break; 1358925baeddSChris Mason if (!path->locks[i]) 1359925baeddSChris Mason break; 1360051e1b9fSChris Mason if (!no_skips && path->slots[i] == 0) { 1361925baeddSChris Mason skip_level = i + 1; 1362925baeddSChris Mason continue; 1363925baeddSChris Mason } 1364051e1b9fSChris Mason if (!no_skips && path->keep_locks) { 1365925baeddSChris Mason u32 nritems; 1366925baeddSChris Mason t = path->nodes[i]; 1367925baeddSChris Mason nritems = btrfs_header_nritems(t); 1368051e1b9fSChris Mason if (nritems < 1 || path->slots[i] >= nritems - 1) { 1369925baeddSChris Mason skip_level = i + 1; 1370925baeddSChris Mason continue; 1371925baeddSChris Mason } 1372925baeddSChris Mason } 1373051e1b9fSChris Mason if (skip_level < i && i >= lowest_unlock) 1374051e1b9fSChris Mason no_skips = 1; 1375051e1b9fSChris Mason 1376925baeddSChris Mason t = path->nodes[i]; 1377d80bb3f9SLiu Bo if (i >= lowest_unlock && i > skip_level) { 1378bd681513SChris Mason btrfs_tree_unlock_rw(t, path->locks[i]); 1379925baeddSChris Mason path->locks[i] = 0; 1380f7c79f30SChris Mason if (write_lock_level && 1381f7c79f30SChris Mason i > min_write_lock_level && 1382f7c79f30SChris Mason i <= *write_lock_level) { 1383f7c79f30SChris Mason *write_lock_level = i - 1; 1384f7c79f30SChris Mason } 1385925baeddSChris Mason } 1386925baeddSChris Mason } 1387925baeddSChris Mason } 1388925baeddSChris Mason 13893c69faecSChris Mason /* 1390c8c42864SChris Mason * helper function for btrfs_search_slot. The goal is to find a block 1391c8c42864SChris Mason * in cache without setting the path to blocking. If we find the block 1392c8c42864SChris Mason * we return zero and the path is unchanged. 1393c8c42864SChris Mason * 1394c8c42864SChris Mason * If we can't find the block, we set the path blocking and do some 1395c8c42864SChris Mason * reada. -EAGAIN is returned and the search must be repeated. 1396c8c42864SChris Mason */ 1397c8c42864SChris Mason static int 1398d07b8528SLiu Bo read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, 1399c8c42864SChris Mason struct extent_buffer **eb_ret, int level, int slot, 1400cda79c54SDavid Sterba const struct btrfs_key *key) 1401c8c42864SChris Mason { 14020b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 1403c8c42864SChris Mason u64 blocknr; 1404c8c42864SChris Mason u64 gen; 1405c8c42864SChris Mason struct extent_buffer *tmp; 1406581c1760SQu Wenruo struct btrfs_key first_key; 140776a05b35SChris Mason int ret; 1408581c1760SQu Wenruo int parent_level; 1409c8c42864SChris Mason 1410213ff4b7SNikolay Borisov blocknr = btrfs_node_blockptr(*eb_ret, slot); 1411213ff4b7SNikolay Borisov gen = btrfs_node_ptr_generation(*eb_ret, slot); 1412213ff4b7SNikolay Borisov parent_level = btrfs_header_level(*eb_ret); 1413213ff4b7SNikolay Borisov btrfs_node_key_to_cpu(*eb_ret, &first_key, slot); 1414c8c42864SChris Mason 14150b246afaSJeff Mahoney tmp = find_extent_buffer(fs_info, blocknr); 1416cb44921aSChris Mason if (tmp) { 1417ace75066SFilipe Manana if (p->reada == READA_FORWARD_ALWAYS) 1418ace75066SFilipe Manana reada_for_search(fs_info, p, level, slot, key->objectid); 1419ace75066SFilipe Manana 1420b9fab919SChris Mason /* first we do an atomic uptodate check */ 1421b9fab919SChris Mason if (btrfs_buffer_uptodate(tmp, gen, 1) > 0) { 1422448de471SQu Wenruo /* 1423448de471SQu Wenruo * Do extra check for first_key, eb can be stale due to 1424448de471SQu Wenruo * being cached, read from scrub, or have multiple 1425448de471SQu Wenruo * parents (shared tree blocks). 1426448de471SQu Wenruo */ 1427e064d5e9SDavid Sterba if (btrfs_verify_level_key(tmp, 1428448de471SQu Wenruo parent_level - 1, &first_key, gen)) { 1429448de471SQu Wenruo free_extent_buffer(tmp); 1430448de471SQu Wenruo return -EUCLEAN; 1431448de471SQu Wenruo } 1432c8c42864SChris Mason *eb_ret = tmp; 1433c8c42864SChris Mason return 0; 1434c8c42864SChris Mason } 1435bdf7c00eSJosef Bacik 1436b9fab919SChris Mason /* now we're allowed to do a blocking uptodate check */ 1437581c1760SQu Wenruo ret = btrfs_read_buffer(tmp, gen, parent_level - 1, &first_key); 1438bdf7c00eSJosef Bacik if (!ret) { 1439cb44921aSChris Mason *eb_ret = tmp; 1440cb44921aSChris Mason return 0; 1441cb44921aSChris Mason } 1442cb44921aSChris Mason free_extent_buffer(tmp); 1443b3b4aa74SDavid Sterba btrfs_release_path(p); 1444cb44921aSChris Mason return -EIO; 1445cb44921aSChris Mason } 1446c8c42864SChris Mason 1447c8c42864SChris Mason /* 1448c8c42864SChris Mason * reduce lock contention at high levels 1449c8c42864SChris Mason * of the btree by dropping locks before 145076a05b35SChris Mason * we read. Don't release the lock on the current 145176a05b35SChris Mason * level because we need to walk this node to figure 145276a05b35SChris Mason * out which blocks to read. 1453c8c42864SChris Mason */ 14548c594ea8SChris Mason btrfs_unlock_up_safe(p, level + 1); 14558c594ea8SChris Mason 1456e4058b54SDavid Sterba if (p->reada != READA_NONE) 14572ff7e61eSJeff Mahoney reada_for_search(fs_info, p, level, slot, key->objectid); 1458c8c42864SChris Mason 145976a05b35SChris Mason ret = -EAGAIN; 14601b7ec85eSJosef Bacik tmp = read_tree_block(fs_info, blocknr, root->root_key.objectid, 14611b7ec85eSJosef Bacik gen, parent_level - 1, &first_key); 146264c043deSLiu Bo if (!IS_ERR(tmp)) { 146376a05b35SChris Mason /* 146476a05b35SChris Mason * If the read above didn't mark this buffer up to date, 146576a05b35SChris Mason * it will never end up being up to date. Set ret to EIO now 146676a05b35SChris Mason * and give up so that our caller doesn't loop forever 146776a05b35SChris Mason * on our EAGAINs. 146876a05b35SChris Mason */ 1469e6a1d6fdSLiu Bo if (!extent_buffer_uptodate(tmp)) 147076a05b35SChris Mason ret = -EIO; 1471c8c42864SChris Mason free_extent_buffer(tmp); 1472c871b0f2SLiu Bo } else { 1473c871b0f2SLiu Bo ret = PTR_ERR(tmp); 147476a05b35SChris Mason } 147502a3307aSLiu Bo 147602a3307aSLiu Bo btrfs_release_path(p); 147776a05b35SChris Mason return ret; 1478c8c42864SChris Mason } 1479c8c42864SChris Mason 1480c8c42864SChris Mason /* 1481c8c42864SChris Mason * helper function for btrfs_search_slot. This does all of the checks 1482c8c42864SChris Mason * for node-level blocks and does any balancing required based on 1483c8c42864SChris Mason * the ins_len. 1484c8c42864SChris Mason * 1485c8c42864SChris Mason * If no extra work was required, zero is returned. If we had to 1486c8c42864SChris Mason * drop the path, -EAGAIN is returned and btrfs_search_slot must 1487c8c42864SChris Mason * start over 1488c8c42864SChris Mason */ 1489c8c42864SChris Mason static int 1490c8c42864SChris Mason setup_nodes_for_search(struct btrfs_trans_handle *trans, 1491c8c42864SChris Mason struct btrfs_root *root, struct btrfs_path *p, 1492bd681513SChris Mason struct extent_buffer *b, int level, int ins_len, 1493bd681513SChris Mason int *write_lock_level) 1494c8c42864SChris Mason { 14950b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 149695b982deSNikolay Borisov int ret = 0; 14970b246afaSJeff Mahoney 1498c8c42864SChris Mason if ((p->search_for_split || ins_len > 0) && btrfs_header_nritems(b) >= 14990b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3) { 1500c8c42864SChris Mason 1501bd681513SChris Mason if (*write_lock_level < level + 1) { 1502bd681513SChris Mason *write_lock_level = level + 1; 1503bd681513SChris Mason btrfs_release_path(p); 150495b982deSNikolay Borisov return -EAGAIN; 1505bd681513SChris Mason } 1506bd681513SChris Mason 1507bfb484d9SJosef Bacik reada_for_balance(p, level); 150895b982deSNikolay Borisov ret = split_node(trans, root, p, level); 1509c8c42864SChris Mason 1510c8c42864SChris Mason b = p->nodes[level]; 1511c8c42864SChris Mason } else if (ins_len < 0 && btrfs_header_nritems(b) < 15120b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 2) { 1513c8c42864SChris Mason 1514bd681513SChris Mason if (*write_lock_level < level + 1) { 1515bd681513SChris Mason *write_lock_level = level + 1; 1516bd681513SChris Mason btrfs_release_path(p); 151795b982deSNikolay Borisov return -EAGAIN; 1518bd681513SChris Mason } 1519bd681513SChris Mason 1520bfb484d9SJosef Bacik reada_for_balance(p, level); 152195b982deSNikolay Borisov ret = balance_level(trans, root, p, level); 152295b982deSNikolay Borisov if (ret) 152395b982deSNikolay Borisov return ret; 1524c8c42864SChris Mason 1525c8c42864SChris Mason b = p->nodes[level]; 1526c8c42864SChris Mason if (!b) { 1527b3b4aa74SDavid Sterba btrfs_release_path(p); 152895b982deSNikolay Borisov return -EAGAIN; 1529c8c42864SChris Mason } 1530c8c42864SChris Mason BUG_ON(btrfs_header_nritems(b) == 1); 1531c8c42864SChris Mason } 1532c8c42864SChris Mason return ret; 1533c8c42864SChris Mason } 1534c8c42864SChris Mason 1535381cf658SDavid Sterba int btrfs_find_item(struct btrfs_root *fs_root, struct btrfs_path *path, 1536e33d5c3dSKelley Nielsen u64 iobjectid, u64 ioff, u8 key_type, 1537e33d5c3dSKelley Nielsen struct btrfs_key *found_key) 1538e33d5c3dSKelley Nielsen { 1539e33d5c3dSKelley Nielsen int ret; 1540e33d5c3dSKelley Nielsen struct btrfs_key key; 1541e33d5c3dSKelley Nielsen struct extent_buffer *eb; 1542381cf658SDavid Sterba 1543381cf658SDavid Sterba ASSERT(path); 15441d4c08e0SDavid Sterba ASSERT(found_key); 1545e33d5c3dSKelley Nielsen 1546e33d5c3dSKelley Nielsen key.type = key_type; 1547e33d5c3dSKelley Nielsen key.objectid = iobjectid; 1548e33d5c3dSKelley Nielsen key.offset = ioff; 1549e33d5c3dSKelley Nielsen 1550e33d5c3dSKelley Nielsen ret = btrfs_search_slot(NULL, fs_root, &key, path, 0, 0); 15511d4c08e0SDavid Sterba if (ret < 0) 1552e33d5c3dSKelley Nielsen return ret; 1553e33d5c3dSKelley Nielsen 1554e33d5c3dSKelley Nielsen eb = path->nodes[0]; 1555e33d5c3dSKelley Nielsen if (ret && path->slots[0] >= btrfs_header_nritems(eb)) { 1556e33d5c3dSKelley Nielsen ret = btrfs_next_leaf(fs_root, path); 1557e33d5c3dSKelley Nielsen if (ret) 1558e33d5c3dSKelley Nielsen return ret; 1559e33d5c3dSKelley Nielsen eb = path->nodes[0]; 1560e33d5c3dSKelley Nielsen } 1561e33d5c3dSKelley Nielsen 1562e33d5c3dSKelley Nielsen btrfs_item_key_to_cpu(eb, found_key, path->slots[0]); 1563e33d5c3dSKelley Nielsen if (found_key->type != key.type || 1564e33d5c3dSKelley Nielsen found_key->objectid != key.objectid) 1565e33d5c3dSKelley Nielsen return 1; 1566e33d5c3dSKelley Nielsen 1567e33d5c3dSKelley Nielsen return 0; 1568e33d5c3dSKelley Nielsen } 1569e33d5c3dSKelley Nielsen 15701fc28d8eSLiu Bo static struct extent_buffer *btrfs_search_slot_get_root(struct btrfs_root *root, 15711fc28d8eSLiu Bo struct btrfs_path *p, 15721fc28d8eSLiu Bo int write_lock_level) 15731fc28d8eSLiu Bo { 15741fc28d8eSLiu Bo struct extent_buffer *b; 1575120de408SJosef Bacik int root_lock = 0; 15761fc28d8eSLiu Bo int level = 0; 15771fc28d8eSLiu Bo 15781fc28d8eSLiu Bo if (p->search_commit_root) { 15791fc28d8eSLiu Bo b = root->commit_root; 158067439dadSDavid Sterba atomic_inc(&b->refs); 15811fc28d8eSLiu Bo level = btrfs_header_level(b); 1582f9ddfd05SLiu Bo /* 1583f9ddfd05SLiu Bo * Ensure that all callers have set skip_locking when 1584f9ddfd05SLiu Bo * p->search_commit_root = 1. 1585f9ddfd05SLiu Bo */ 1586f9ddfd05SLiu Bo ASSERT(p->skip_locking == 1); 15871fc28d8eSLiu Bo 15881fc28d8eSLiu Bo goto out; 15891fc28d8eSLiu Bo } 15901fc28d8eSLiu Bo 15911fc28d8eSLiu Bo if (p->skip_locking) { 15921fc28d8eSLiu Bo b = btrfs_root_node(root); 15931fc28d8eSLiu Bo level = btrfs_header_level(b); 15941fc28d8eSLiu Bo goto out; 15951fc28d8eSLiu Bo } 15961fc28d8eSLiu Bo 1597120de408SJosef Bacik /* We try very hard to do read locks on the root */ 1598120de408SJosef Bacik root_lock = BTRFS_READ_LOCK; 1599120de408SJosef Bacik 16001fc28d8eSLiu Bo /* 1601662c653bSLiu Bo * If the level is set to maximum, we can skip trying to get the read 1602662c653bSLiu Bo * lock. 1603662c653bSLiu Bo */ 1604662c653bSLiu Bo if (write_lock_level < BTRFS_MAX_LEVEL) { 1605662c653bSLiu Bo /* 1606662c653bSLiu Bo * We don't know the level of the root node until we actually 1607662c653bSLiu Bo * have it read locked 16081fc28d8eSLiu Bo */ 16091bb96598SJosef Bacik b = btrfs_read_lock_root_node(root); 16101fc28d8eSLiu Bo level = btrfs_header_level(b); 16111fc28d8eSLiu Bo if (level > write_lock_level) 16121fc28d8eSLiu Bo goto out; 16131fc28d8eSLiu Bo 1614662c653bSLiu Bo /* Whoops, must trade for write lock */ 16151fc28d8eSLiu Bo btrfs_tree_read_unlock(b); 16161fc28d8eSLiu Bo free_extent_buffer(b); 1617662c653bSLiu Bo } 1618662c653bSLiu Bo 16191fc28d8eSLiu Bo b = btrfs_lock_root_node(root); 16201fc28d8eSLiu Bo root_lock = BTRFS_WRITE_LOCK; 16211fc28d8eSLiu Bo 16221fc28d8eSLiu Bo /* The level might have changed, check again */ 16231fc28d8eSLiu Bo level = btrfs_header_level(b); 16241fc28d8eSLiu Bo 16251fc28d8eSLiu Bo out: 1626120de408SJosef Bacik /* 1627120de408SJosef Bacik * The root may have failed to write out at some point, and thus is no 1628120de408SJosef Bacik * longer valid, return an error in this case. 1629120de408SJosef Bacik */ 1630120de408SJosef Bacik if (!extent_buffer_uptodate(b)) { 1631120de408SJosef Bacik if (root_lock) 1632120de408SJosef Bacik btrfs_tree_unlock_rw(b, root_lock); 1633120de408SJosef Bacik free_extent_buffer(b); 1634120de408SJosef Bacik return ERR_PTR(-EIO); 1635120de408SJosef Bacik } 1636120de408SJosef Bacik 16371fc28d8eSLiu Bo p->nodes[level] = b; 16381fc28d8eSLiu Bo if (!p->skip_locking) 16391fc28d8eSLiu Bo p->locks[level] = root_lock; 16401fc28d8eSLiu Bo /* 16411fc28d8eSLiu Bo * Callers are responsible for dropping b's references. 16421fc28d8eSLiu Bo */ 16431fc28d8eSLiu Bo return b; 16441fc28d8eSLiu Bo } 16451fc28d8eSLiu Bo 1646d96b3424SFilipe Manana /* 1647d96b3424SFilipe Manana * Replace the extent buffer at the lowest level of the path with a cloned 1648d96b3424SFilipe Manana * version. The purpose is to be able to use it safely, after releasing the 1649d96b3424SFilipe Manana * commit root semaphore, even if relocation is happening in parallel, the 1650d96b3424SFilipe Manana * transaction used for relocation is committed and the extent buffer is 1651d96b3424SFilipe Manana * reallocated in the next transaction. 1652d96b3424SFilipe Manana * 1653d96b3424SFilipe Manana * This is used in a context where the caller does not prevent transaction 1654d96b3424SFilipe Manana * commits from happening, either by holding a transaction handle or holding 1655d96b3424SFilipe Manana * some lock, while it's doing searches through a commit root. 1656d96b3424SFilipe Manana * At the moment it's only used for send operations. 1657d96b3424SFilipe Manana */ 1658d96b3424SFilipe Manana static int finish_need_commit_sem_search(struct btrfs_path *path) 1659d96b3424SFilipe Manana { 1660d96b3424SFilipe Manana const int i = path->lowest_level; 1661d96b3424SFilipe Manana const int slot = path->slots[i]; 1662d96b3424SFilipe Manana struct extent_buffer *lowest = path->nodes[i]; 1663d96b3424SFilipe Manana struct extent_buffer *clone; 1664d96b3424SFilipe Manana 1665d96b3424SFilipe Manana ASSERT(path->need_commit_sem); 1666d96b3424SFilipe Manana 1667d96b3424SFilipe Manana if (!lowest) 1668d96b3424SFilipe Manana return 0; 1669d96b3424SFilipe Manana 1670d96b3424SFilipe Manana lockdep_assert_held_read(&lowest->fs_info->commit_root_sem); 1671d96b3424SFilipe Manana 1672d96b3424SFilipe Manana clone = btrfs_clone_extent_buffer(lowest); 1673d96b3424SFilipe Manana if (!clone) 1674d96b3424SFilipe Manana return -ENOMEM; 1675d96b3424SFilipe Manana 1676d96b3424SFilipe Manana btrfs_release_path(path); 1677d96b3424SFilipe Manana path->nodes[i] = clone; 1678d96b3424SFilipe Manana path->slots[i] = slot; 1679d96b3424SFilipe Manana 1680d96b3424SFilipe Manana return 0; 1681d96b3424SFilipe Manana } 16821fc28d8eSLiu Bo 1683*e2e58d0fSFilipe Manana static inline int search_for_key_slot(struct extent_buffer *eb, 1684*e2e58d0fSFilipe Manana int search_low_slot, 1685*e2e58d0fSFilipe Manana const struct btrfs_key *key, 1686*e2e58d0fSFilipe Manana int prev_cmp, 1687*e2e58d0fSFilipe Manana int *slot) 1688*e2e58d0fSFilipe Manana { 1689*e2e58d0fSFilipe Manana /* 1690*e2e58d0fSFilipe Manana * If a previous call to btrfs_bin_search() on a parent node returned an 1691*e2e58d0fSFilipe Manana * exact match (prev_cmp == 0), we can safely assume the target key will 1692*e2e58d0fSFilipe Manana * always be at slot 0 on lower levels, since each key pointer 1693*e2e58d0fSFilipe Manana * (struct btrfs_key_ptr) refers to the lowest key accessible from the 1694*e2e58d0fSFilipe Manana * subtree it points to. Thus we can skip searching lower levels. 1695*e2e58d0fSFilipe Manana */ 1696*e2e58d0fSFilipe Manana if (prev_cmp == 0) { 1697*e2e58d0fSFilipe Manana *slot = 0; 1698*e2e58d0fSFilipe Manana return 0; 1699*e2e58d0fSFilipe Manana } 1700*e2e58d0fSFilipe Manana 1701*e2e58d0fSFilipe Manana return generic_bin_search(eb, search_low_slot, key, slot); 1702*e2e58d0fSFilipe Manana } 1703*e2e58d0fSFilipe Manana 1704c8c42864SChris Mason /* 17054271eceaSNikolay Borisov * btrfs_search_slot - look for a key in a tree and perform necessary 17064271eceaSNikolay Borisov * modifications to preserve tree invariants. 170774123bd7SChris Mason * 17084271eceaSNikolay Borisov * @trans: Handle of transaction, used when modifying the tree 17094271eceaSNikolay Borisov * @p: Holds all btree nodes along the search path 17104271eceaSNikolay Borisov * @root: The root node of the tree 17114271eceaSNikolay Borisov * @key: The key we are looking for 17129a664971Sethanwu * @ins_len: Indicates purpose of search: 17139a664971Sethanwu * >0 for inserts it's size of item inserted (*) 17149a664971Sethanwu * <0 for deletions 17159a664971Sethanwu * 0 for plain searches, not modifying the tree 17169a664971Sethanwu * 17179a664971Sethanwu * (*) If size of item inserted doesn't include 17189a664971Sethanwu * sizeof(struct btrfs_item), then p->search_for_extension must 17199a664971Sethanwu * be set. 17204271eceaSNikolay Borisov * @cow: boolean should CoW operations be performed. Must always be 1 17214271eceaSNikolay Borisov * when modifying the tree. 172297571fd0SChris Mason * 17234271eceaSNikolay Borisov * If @ins_len > 0, nodes and leaves will be split as we walk down the tree. 17244271eceaSNikolay Borisov * If @ins_len < 0, nodes will be merged as we walk down the tree (if possible) 17254271eceaSNikolay Borisov * 17264271eceaSNikolay Borisov * If @key is found, 0 is returned and you can find the item in the leaf level 17274271eceaSNikolay Borisov * of the path (level 0) 17284271eceaSNikolay Borisov * 17294271eceaSNikolay Borisov * If @key isn't found, 1 is returned and the leaf level of the path (level 0) 17304271eceaSNikolay Borisov * points to the slot where it should be inserted 17314271eceaSNikolay Borisov * 17324271eceaSNikolay Borisov * If an error is encountered while searching the tree a negative error number 17334271eceaSNikolay Borisov * is returned 173474123bd7SChris Mason */ 1735310712b2SOmar Sandoval int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root, 1736310712b2SOmar Sandoval const struct btrfs_key *key, struct btrfs_path *p, 1737310712b2SOmar Sandoval int ins_len, int cow) 1738be0e5c09SChris Mason { 1739d96b3424SFilipe Manana struct btrfs_fs_info *fs_info = root->fs_info; 17405f39d397SChris Mason struct extent_buffer *b; 1741be0e5c09SChris Mason int slot; 1742be0e5c09SChris Mason int ret; 174333c66f43SYan Zheng int err; 1744be0e5c09SChris Mason int level; 1745925baeddSChris Mason int lowest_unlock = 1; 1746bd681513SChris Mason /* everything at write_lock_level or lower must be write locked */ 1747bd681513SChris Mason int write_lock_level = 0; 17489f3a7427SChris Mason u8 lowest_level = 0; 1749f7c79f30SChris Mason int min_write_lock_level; 1750d7396f07SFilipe David Borba Manana int prev_cmp; 17519f3a7427SChris Mason 17526702ed49SChris Mason lowest_level = p->lowest_level; 1753323ac95bSChris Mason WARN_ON(lowest_level && ins_len > 0); 175422b0ebdaSChris Mason WARN_ON(p->nodes[0] != NULL); 1755eb653de1SFilipe David Borba Manana BUG_ON(!cow && ins_len); 175625179201SJosef Bacik 1757bd681513SChris Mason if (ins_len < 0) { 1758925baeddSChris Mason lowest_unlock = 2; 175965b51a00SChris Mason 1760bd681513SChris Mason /* when we are removing items, we might have to go up to level 1761bd681513SChris Mason * two as we update tree pointers Make sure we keep write 1762bd681513SChris Mason * for those levels as well 1763bd681513SChris Mason */ 1764bd681513SChris Mason write_lock_level = 2; 1765bd681513SChris Mason } else if (ins_len > 0) { 1766bd681513SChris Mason /* 1767bd681513SChris Mason * for inserting items, make sure we have a write lock on 1768bd681513SChris Mason * level 1 so we can update keys 1769bd681513SChris Mason */ 1770bd681513SChris Mason write_lock_level = 1; 1771bd681513SChris Mason } 1772bd681513SChris Mason 1773bd681513SChris Mason if (!cow) 1774bd681513SChris Mason write_lock_level = -1; 1775bd681513SChris Mason 177609a2a8f9SJosef Bacik if (cow && (p->keep_locks || p->lowest_level)) 1777bd681513SChris Mason write_lock_level = BTRFS_MAX_LEVEL; 1778bd681513SChris Mason 1779f7c79f30SChris Mason min_write_lock_level = write_lock_level; 1780f7c79f30SChris Mason 1781d96b3424SFilipe Manana if (p->need_commit_sem) { 1782d96b3424SFilipe Manana ASSERT(p->search_commit_root); 1783d96b3424SFilipe Manana down_read(&fs_info->commit_root_sem); 1784d96b3424SFilipe Manana } 1785d96b3424SFilipe Manana 1786bb803951SChris Mason again: 1787d7396f07SFilipe David Borba Manana prev_cmp = -1; 17881fc28d8eSLiu Bo b = btrfs_search_slot_get_root(root, p, write_lock_level); 1789be6821f8SFilipe Manana if (IS_ERR(b)) { 1790be6821f8SFilipe Manana ret = PTR_ERR(b); 1791be6821f8SFilipe Manana goto done; 1792be6821f8SFilipe Manana } 1793925baeddSChris Mason 1794eb60ceacSChris Mason while (b) { 1795f624d976SQu Wenruo int dec = 0; 1796f624d976SQu Wenruo 17975f39d397SChris Mason level = btrfs_header_level(b); 179865b51a00SChris Mason 179902217ed2SChris Mason if (cow) { 18009ea2c7c9SNikolay Borisov bool last_level = (level == (BTRFS_MAX_LEVEL - 1)); 18019ea2c7c9SNikolay Borisov 1802c8c42864SChris Mason /* 1803c8c42864SChris Mason * if we don't really need to cow this block 1804c8c42864SChris Mason * then we don't want to set the path blocking, 1805c8c42864SChris Mason * so we test it here 1806c8c42864SChris Mason */ 18075963ffcaSJosef Bacik if (!should_cow_block(trans, root, b)) 180865b51a00SChris Mason goto cow_done; 18095d4f98a2SYan Zheng 1810bd681513SChris Mason /* 1811bd681513SChris Mason * must have write locks on this node and the 1812bd681513SChris Mason * parent 1813bd681513SChris Mason */ 18145124e00eSJosef Bacik if (level > write_lock_level || 18155124e00eSJosef Bacik (level + 1 > write_lock_level && 18165124e00eSJosef Bacik level + 1 < BTRFS_MAX_LEVEL && 18175124e00eSJosef Bacik p->nodes[level + 1])) { 1818bd681513SChris Mason write_lock_level = level + 1; 1819bd681513SChris Mason btrfs_release_path(p); 1820bd681513SChris Mason goto again; 1821bd681513SChris Mason } 1822bd681513SChris Mason 18239ea2c7c9SNikolay Borisov if (last_level) 18249ea2c7c9SNikolay Borisov err = btrfs_cow_block(trans, root, b, NULL, 0, 18259631e4ccSJosef Bacik &b, 18269631e4ccSJosef Bacik BTRFS_NESTING_COW); 18279ea2c7c9SNikolay Borisov else 182833c66f43SYan Zheng err = btrfs_cow_block(trans, root, b, 1829e20d96d6SChris Mason p->nodes[level + 1], 18309631e4ccSJosef Bacik p->slots[level + 1], &b, 18319631e4ccSJosef Bacik BTRFS_NESTING_COW); 183233c66f43SYan Zheng if (err) { 183333c66f43SYan Zheng ret = err; 183465b51a00SChris Mason goto done; 183554aa1f4dSChris Mason } 183602217ed2SChris Mason } 183765b51a00SChris Mason cow_done: 1838eb60ceacSChris Mason p->nodes[level] = b; 183952398340SLiu Bo /* 184052398340SLiu Bo * Leave path with blocking locks to avoid massive 184152398340SLiu Bo * lock context switch, this is made on purpose. 184252398340SLiu Bo */ 1843b4ce94deSChris Mason 1844b4ce94deSChris Mason /* 1845b4ce94deSChris Mason * we have a lock on b and as long as we aren't changing 1846b4ce94deSChris Mason * the tree, there is no way to for the items in b to change. 1847b4ce94deSChris Mason * It is safe to drop the lock on our parent before we 1848b4ce94deSChris Mason * go through the expensive btree search on b. 1849b4ce94deSChris Mason * 1850eb653de1SFilipe David Borba Manana * If we're inserting or deleting (ins_len != 0), then we might 1851eb653de1SFilipe David Borba Manana * be changing slot zero, which may require changing the parent. 1852eb653de1SFilipe David Borba Manana * So, we can't drop the lock until after we know which slot 1853eb653de1SFilipe David Borba Manana * we're operating on. 1854b4ce94deSChris Mason */ 1855eb653de1SFilipe David Borba Manana if (!ins_len && !p->keep_locks) { 1856eb653de1SFilipe David Borba Manana int u = level + 1; 1857eb653de1SFilipe David Borba Manana 1858eb653de1SFilipe David Borba Manana if (u < BTRFS_MAX_LEVEL && p->locks[u]) { 1859eb653de1SFilipe David Borba Manana btrfs_tree_unlock_rw(p->nodes[u], p->locks[u]); 1860eb653de1SFilipe David Borba Manana p->locks[u] = 0; 1861eb653de1SFilipe David Borba Manana } 1862eb653de1SFilipe David Borba Manana } 1863b4ce94deSChris Mason 1864*e2e58d0fSFilipe Manana if (level == 0) { 1865*e2e58d0fSFilipe Manana int leaf_free_space = 0; 1866*e2e58d0fSFilipe Manana int search_low_slot = 0; 1867*e2e58d0fSFilipe Manana 1868995e9a16SNikolay Borisov /* 1869*e2e58d0fSFilipe Manana * If we are doing an insertion, the leaf has enough free 1870*e2e58d0fSFilipe Manana * space and the destination slot for the key is not slot 1871*e2e58d0fSFilipe Manana * 0, then we can unlock our write lock on the parent, and 1872*e2e58d0fSFilipe Manana * any other upper nodes, before doing the binary search 1873*e2e58d0fSFilipe Manana * on the leaf (with search_for_key_slot()), allowing other 1874*e2e58d0fSFilipe Manana * tasks to lock the parent and any other upper nodes. 1875995e9a16SNikolay Borisov */ 1876*e2e58d0fSFilipe Manana if (ins_len > 0) { 1877*e2e58d0fSFilipe Manana struct btrfs_disk_key first_key; 1878*e2e58d0fSFilipe Manana 1879*e2e58d0fSFilipe Manana /* 1880*e2e58d0fSFilipe Manana * Cache the leaf free space, since we will need it 1881*e2e58d0fSFilipe Manana * later and it will not change until then. 1882*e2e58d0fSFilipe Manana */ 1883*e2e58d0fSFilipe Manana leaf_free_space = btrfs_leaf_free_space(b); 1884*e2e58d0fSFilipe Manana 1885*e2e58d0fSFilipe Manana /* 1886*e2e58d0fSFilipe Manana * !p->locks[1] means we have a single node tree, 1887*e2e58d0fSFilipe Manana * the leaf is the root of the tree. 1888*e2e58d0fSFilipe Manana */ 1889*e2e58d0fSFilipe Manana if (!p->locks[1] || leaf_free_space < ins_len) 1890*e2e58d0fSFilipe Manana goto leaf_search; 1891*e2e58d0fSFilipe Manana 1892*e2e58d0fSFilipe Manana ASSERT(btrfs_header_nritems(b) > 0); 1893*e2e58d0fSFilipe Manana btrfs_item_key(b, &first_key, 0); 1894*e2e58d0fSFilipe Manana 1895*e2e58d0fSFilipe Manana /* 1896*e2e58d0fSFilipe Manana * Doing the extra comparison with the first key 1897*e2e58d0fSFilipe Manana * is cheap, taking into account that the first 1898*e2e58d0fSFilipe Manana * key is very likely already in a cache line 1899*e2e58d0fSFilipe Manana * because it immediately follows the extent 1900*e2e58d0fSFilipe Manana * buffer's header and we have recently accessed 1901*e2e58d0fSFilipe Manana * the header's level field. 1902*e2e58d0fSFilipe Manana */ 1903*e2e58d0fSFilipe Manana ret = comp_keys(&first_key, key); 1904*e2e58d0fSFilipe Manana if (ret < 0) { 1905*e2e58d0fSFilipe Manana /* 1906*e2e58d0fSFilipe Manana * The first key is smaller than the key 1907*e2e58d0fSFilipe Manana * we want to insert, so we are safe to 1908*e2e58d0fSFilipe Manana * unlock all upper nodes and we have to 1909*e2e58d0fSFilipe Manana * do the binary search. 1910*e2e58d0fSFilipe Manana * 1911*e2e58d0fSFilipe Manana * We do use btrfs_unlock_up_safe() and 1912*e2e58d0fSFilipe Manana * not unlock_up() because the later does 1913*e2e58d0fSFilipe Manana * not unlock nodes with a slot of 0. 1914*e2e58d0fSFilipe Manana * We can safely unlock any node even if 1915*e2e58d0fSFilipe Manana * its slot is 0 since in this case the 1916*e2e58d0fSFilipe Manana * key does not end up at slot 0 of the 1917*e2e58d0fSFilipe Manana * leaf and there's also no need to split 1918*e2e58d0fSFilipe Manana * the leaf. 1919*e2e58d0fSFilipe Manana */ 1920*e2e58d0fSFilipe Manana btrfs_unlock_up_safe(p, 1); 1921*e2e58d0fSFilipe Manana search_low_slot = 1; 1922995e9a16SNikolay Borisov } else { 1923*e2e58d0fSFilipe Manana /* 1924*e2e58d0fSFilipe Manana * The first key is >= then the key we 1925*e2e58d0fSFilipe Manana * want to insert, so we can skip the 1926*e2e58d0fSFilipe Manana * binary search as the target key will 1927*e2e58d0fSFilipe Manana * be at slot 0. 1928*e2e58d0fSFilipe Manana * 1929*e2e58d0fSFilipe Manana * We can not unlock upper nodes when 1930*e2e58d0fSFilipe Manana * the key is less than the first key, 1931*e2e58d0fSFilipe Manana * because we will need to update the key 1932*e2e58d0fSFilipe Manana * at slot 0 of the parent node and 1933*e2e58d0fSFilipe Manana * possibly of other upper nodes too. 1934*e2e58d0fSFilipe Manana * If the key matches the first key, then 1935*e2e58d0fSFilipe Manana * we can unlock all the upper nodes, 1936*e2e58d0fSFilipe Manana * using btrfs_unlock_up_safe() instead 1937*e2e58d0fSFilipe Manana * of unlock_up() as stated above. 1938*e2e58d0fSFilipe Manana */ 1939*e2e58d0fSFilipe Manana if (ret == 0) 1940*e2e58d0fSFilipe Manana btrfs_unlock_up_safe(p, 1); 1941*e2e58d0fSFilipe Manana slot = 0; 1942*e2e58d0fSFilipe Manana /* 1943*e2e58d0fSFilipe Manana * ret is already 0 or 1, matching the 1944*e2e58d0fSFilipe Manana * result of a btrfs_bin_search() call, 1945*e2e58d0fSFilipe Manana * so there is no need to adjust it. 1946*e2e58d0fSFilipe Manana */ 1947*e2e58d0fSFilipe Manana goto skip_leaf_search; 1948*e2e58d0fSFilipe Manana } 1949*e2e58d0fSFilipe Manana } 1950*e2e58d0fSFilipe Manana leaf_search: 1951*e2e58d0fSFilipe Manana ret = search_for_key_slot(b, search_low_slot, key, 1952*e2e58d0fSFilipe Manana prev_cmp, &slot); 1953415b35a5SLiu Bo if (ret < 0) 1954415b35a5SLiu Bo goto done; 1955*e2e58d0fSFilipe Manana skip_leaf_search: 1956be0e5c09SChris Mason p->slots[level] = slot; 19579a664971Sethanwu /* 19589a664971Sethanwu * Item key already exists. In this case, if we are 19599a664971Sethanwu * allowed to insert the item (for example, in dir_item 19609a664971Sethanwu * case, item key collision is allowed), it will be 19619a664971Sethanwu * merged with the original item. Only the item size 19629a664971Sethanwu * grows, no new btrfs item will be added. If 19639a664971Sethanwu * search_for_extension is not set, ins_len already 19649a664971Sethanwu * accounts the size btrfs_item, deduct it here so leaf 19659a664971Sethanwu * space check will be correct. 19669a664971Sethanwu */ 19679a664971Sethanwu if (ret == 0 && ins_len > 0 && !p->search_for_extension) { 19689a664971Sethanwu ASSERT(ins_len >= sizeof(struct btrfs_item)); 19699a664971Sethanwu ins_len -= sizeof(struct btrfs_item); 19709a664971Sethanwu } 1971*e2e58d0fSFilipe Manana if (ins_len > 0 && leaf_free_space < ins_len) { 1972bd681513SChris Mason if (write_lock_level < 1) { 1973bd681513SChris Mason write_lock_level = 1; 1974bd681513SChris Mason btrfs_release_path(p); 1975bd681513SChris Mason goto again; 1976bd681513SChris Mason } 1977bd681513SChris Mason 197833c66f43SYan Zheng err = split_leaf(trans, root, key, 1979cc0c5538SChris Mason p, ins_len, ret == 0); 1980b4ce94deSChris Mason 198133c66f43SYan Zheng BUG_ON(err > 0); 198233c66f43SYan Zheng if (err) { 198333c66f43SYan Zheng ret = err; 198465b51a00SChris Mason goto done; 198565b51a00SChris Mason } 19865c680ed6SChris Mason } 1987459931ecSChris Mason if (!p->search_for_split) 1988f7c79f30SChris Mason unlock_up(p, level, lowest_unlock, 19894b6f8e96SLiu Bo min_write_lock_level, NULL); 199065b51a00SChris Mason goto done; 199165b51a00SChris Mason } 1992*e2e58d0fSFilipe Manana 1993*e2e58d0fSFilipe Manana ret = search_for_key_slot(b, 0, key, prev_cmp, &slot); 1994*e2e58d0fSFilipe Manana if (ret < 0) 1995*e2e58d0fSFilipe Manana goto done; 1996*e2e58d0fSFilipe Manana prev_cmp = ret; 1997*e2e58d0fSFilipe Manana 1998f624d976SQu Wenruo if (ret && slot > 0) { 1999f624d976SQu Wenruo dec = 1; 2000f624d976SQu Wenruo slot--; 2001f624d976SQu Wenruo } 2002f624d976SQu Wenruo p->slots[level] = slot; 2003f624d976SQu Wenruo err = setup_nodes_for_search(trans, root, p, b, level, ins_len, 2004f624d976SQu Wenruo &write_lock_level); 2005f624d976SQu Wenruo if (err == -EAGAIN) 2006f624d976SQu Wenruo goto again; 2007f624d976SQu Wenruo if (err) { 2008f624d976SQu Wenruo ret = err; 2009f624d976SQu Wenruo goto done; 2010f624d976SQu Wenruo } 2011f624d976SQu Wenruo b = p->nodes[level]; 2012f624d976SQu Wenruo slot = p->slots[level]; 2013f624d976SQu Wenruo 2014f624d976SQu Wenruo /* 2015f624d976SQu Wenruo * Slot 0 is special, if we change the key we have to update 2016f624d976SQu Wenruo * the parent pointer which means we must have a write lock on 2017f624d976SQu Wenruo * the parent 2018f624d976SQu Wenruo */ 2019f624d976SQu Wenruo if (slot == 0 && ins_len && write_lock_level < level + 1) { 2020f624d976SQu Wenruo write_lock_level = level + 1; 2021f624d976SQu Wenruo btrfs_release_path(p); 2022f624d976SQu Wenruo goto again; 2023f624d976SQu Wenruo } 2024f624d976SQu Wenruo 2025f624d976SQu Wenruo unlock_up(p, level, lowest_unlock, min_write_lock_level, 2026f624d976SQu Wenruo &write_lock_level); 2027f624d976SQu Wenruo 2028f624d976SQu Wenruo if (level == lowest_level) { 2029f624d976SQu Wenruo if (dec) 2030f624d976SQu Wenruo p->slots[level]++; 2031f624d976SQu Wenruo goto done; 2032f624d976SQu Wenruo } 2033f624d976SQu Wenruo 2034f624d976SQu Wenruo err = read_block_for_search(root, p, &b, level, slot, key); 2035f624d976SQu Wenruo if (err == -EAGAIN) 2036f624d976SQu Wenruo goto again; 2037f624d976SQu Wenruo if (err) { 2038f624d976SQu Wenruo ret = err; 2039f624d976SQu Wenruo goto done; 2040f624d976SQu Wenruo } 2041f624d976SQu Wenruo 2042f624d976SQu Wenruo if (!p->skip_locking) { 2043f624d976SQu Wenruo level = btrfs_header_level(b); 2044f624d976SQu Wenruo if (level <= write_lock_level) { 2045f624d976SQu Wenruo btrfs_tree_lock(b); 2046f624d976SQu Wenruo p->locks[level] = BTRFS_WRITE_LOCK; 2047f624d976SQu Wenruo } else { 2048fe596ca3SJosef Bacik btrfs_tree_read_lock(b); 2049f624d976SQu Wenruo p->locks[level] = BTRFS_READ_LOCK; 2050f624d976SQu Wenruo } 2051f624d976SQu Wenruo p->nodes[level] = b; 2052f624d976SQu Wenruo } 205365b51a00SChris Mason } 205465b51a00SChris Mason ret = 1; 205565b51a00SChris Mason done: 20565f5bc6b1SFilipe Manana if (ret < 0 && !p->skip_release_on_error) 2057b3b4aa74SDavid Sterba btrfs_release_path(p); 2058d96b3424SFilipe Manana 2059d96b3424SFilipe Manana if (p->need_commit_sem) { 2060d96b3424SFilipe Manana int ret2; 2061d96b3424SFilipe Manana 2062d96b3424SFilipe Manana ret2 = finish_need_commit_sem_search(p); 2063d96b3424SFilipe Manana up_read(&fs_info->commit_root_sem); 2064d96b3424SFilipe Manana if (ret2) 2065d96b3424SFilipe Manana ret = ret2; 2066d96b3424SFilipe Manana } 2067d96b3424SFilipe Manana 2068be0e5c09SChris Mason return ret; 2069be0e5c09SChris Mason } 2070f75e2b79SJosef Bacik ALLOW_ERROR_INJECTION(btrfs_search_slot, ERRNO); 2071be0e5c09SChris Mason 207274123bd7SChris Mason /* 20735d9e75c4SJan Schmidt * Like btrfs_search_slot, this looks for a key in the given tree. It uses the 20745d9e75c4SJan Schmidt * current state of the tree together with the operations recorded in the tree 20755d9e75c4SJan Schmidt * modification log to search for the key in a previous version of this tree, as 20765d9e75c4SJan Schmidt * denoted by the time_seq parameter. 20775d9e75c4SJan Schmidt * 20785d9e75c4SJan Schmidt * Naturally, there is no support for insert, delete or cow operations. 20795d9e75c4SJan Schmidt * 20805d9e75c4SJan Schmidt * The resulting path and return value will be set up as if we called 20815d9e75c4SJan Schmidt * btrfs_search_slot at that point in time with ins_len and cow both set to 0. 20825d9e75c4SJan Schmidt */ 2083310712b2SOmar Sandoval int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key, 20845d9e75c4SJan Schmidt struct btrfs_path *p, u64 time_seq) 20855d9e75c4SJan Schmidt { 20860b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 20875d9e75c4SJan Schmidt struct extent_buffer *b; 20885d9e75c4SJan Schmidt int slot; 20895d9e75c4SJan Schmidt int ret; 20905d9e75c4SJan Schmidt int err; 20915d9e75c4SJan Schmidt int level; 20925d9e75c4SJan Schmidt int lowest_unlock = 1; 20935d9e75c4SJan Schmidt u8 lowest_level = 0; 20945d9e75c4SJan Schmidt 20955d9e75c4SJan Schmidt lowest_level = p->lowest_level; 20965d9e75c4SJan Schmidt WARN_ON(p->nodes[0] != NULL); 20975d9e75c4SJan Schmidt 20985d9e75c4SJan Schmidt if (p->search_commit_root) { 20995d9e75c4SJan Schmidt BUG_ON(time_seq); 21005d9e75c4SJan Schmidt return btrfs_search_slot(NULL, root, key, p, 0, 0); 21015d9e75c4SJan Schmidt } 21025d9e75c4SJan Schmidt 21035d9e75c4SJan Schmidt again: 2104f3a84ccdSFilipe Manana b = btrfs_get_old_root(root, time_seq); 2105315bed43SNikolay Borisov if (!b) { 2106315bed43SNikolay Borisov ret = -EIO; 2107315bed43SNikolay Borisov goto done; 2108315bed43SNikolay Borisov } 21095d9e75c4SJan Schmidt level = btrfs_header_level(b); 21105d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 21115d9e75c4SJan Schmidt 21125d9e75c4SJan Schmidt while (b) { 2113abe9339dSQu Wenruo int dec = 0; 2114abe9339dSQu Wenruo 21155d9e75c4SJan Schmidt level = btrfs_header_level(b); 21165d9e75c4SJan Schmidt p->nodes[level] = b; 21175d9e75c4SJan Schmidt 21185d9e75c4SJan Schmidt /* 21195d9e75c4SJan Schmidt * we have a lock on b and as long as we aren't changing 21205d9e75c4SJan Schmidt * the tree, there is no way to for the items in b to change. 21215d9e75c4SJan Schmidt * It is safe to drop the lock on our parent before we 21225d9e75c4SJan Schmidt * go through the expensive btree search on b. 21235d9e75c4SJan Schmidt */ 21245d9e75c4SJan Schmidt btrfs_unlock_up_safe(p, level + 1); 21255d9e75c4SJan Schmidt 2126995e9a16SNikolay Borisov ret = btrfs_bin_search(b, key, &slot); 2127cbca7d59SFilipe Manana if (ret < 0) 2128cbca7d59SFilipe Manana goto done; 21295d9e75c4SJan Schmidt 2130abe9339dSQu Wenruo if (level == 0) { 2131abe9339dSQu Wenruo p->slots[level] = slot; 2132abe9339dSQu Wenruo unlock_up(p, level, lowest_unlock, 0, NULL); 2133abe9339dSQu Wenruo goto done; 2134abe9339dSQu Wenruo } 2135abe9339dSQu Wenruo 21365d9e75c4SJan Schmidt if (ret && slot > 0) { 21375d9e75c4SJan Schmidt dec = 1; 2138abe9339dSQu Wenruo slot--; 21395d9e75c4SJan Schmidt } 21405d9e75c4SJan Schmidt p->slots[level] = slot; 21415d9e75c4SJan Schmidt unlock_up(p, level, lowest_unlock, 0, NULL); 21425d9e75c4SJan Schmidt 21435d9e75c4SJan Schmidt if (level == lowest_level) { 21445d9e75c4SJan Schmidt if (dec) 21455d9e75c4SJan Schmidt p->slots[level]++; 21465d9e75c4SJan Schmidt goto done; 21475d9e75c4SJan Schmidt } 21485d9e75c4SJan Schmidt 2149abe9339dSQu Wenruo err = read_block_for_search(root, p, &b, level, slot, key); 21505d9e75c4SJan Schmidt if (err == -EAGAIN) 21515d9e75c4SJan Schmidt goto again; 21525d9e75c4SJan Schmidt if (err) { 21535d9e75c4SJan Schmidt ret = err; 21545d9e75c4SJan Schmidt goto done; 21555d9e75c4SJan Schmidt } 21565d9e75c4SJan Schmidt 21575d9e75c4SJan Schmidt level = btrfs_header_level(b); 21585d9e75c4SJan Schmidt btrfs_tree_read_lock(b); 2159f3a84ccdSFilipe Manana b = btrfs_tree_mod_log_rewind(fs_info, p, b, time_seq); 2160db7f3436SJosef Bacik if (!b) { 2161db7f3436SJosef Bacik ret = -ENOMEM; 2162db7f3436SJosef Bacik goto done; 2163db7f3436SJosef Bacik } 21645d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 21655d9e75c4SJan Schmidt p->nodes[level] = b; 21665d9e75c4SJan Schmidt } 21675d9e75c4SJan Schmidt ret = 1; 21685d9e75c4SJan Schmidt done: 21695d9e75c4SJan Schmidt if (ret < 0) 21705d9e75c4SJan Schmidt btrfs_release_path(p); 21715d9e75c4SJan Schmidt 21725d9e75c4SJan Schmidt return ret; 21735d9e75c4SJan Schmidt } 21745d9e75c4SJan Schmidt 21755d9e75c4SJan Schmidt /* 21762f38b3e1SArne Jansen * helper to use instead of search slot if no exact match is needed but 21772f38b3e1SArne Jansen * instead the next or previous item should be returned. 21782f38b3e1SArne Jansen * When find_higher is true, the next higher item is returned, the next lower 21792f38b3e1SArne Jansen * otherwise. 21802f38b3e1SArne Jansen * When return_any and find_higher are both true, and no higher item is found, 21812f38b3e1SArne Jansen * return the next lower instead. 21822f38b3e1SArne Jansen * When return_any is true and find_higher is false, and no lower item is found, 21832f38b3e1SArne Jansen * return the next higher instead. 21842f38b3e1SArne Jansen * It returns 0 if any item is found, 1 if none is found (tree empty), and 21852f38b3e1SArne Jansen * < 0 on error 21862f38b3e1SArne Jansen */ 21872f38b3e1SArne Jansen int btrfs_search_slot_for_read(struct btrfs_root *root, 2188310712b2SOmar Sandoval const struct btrfs_key *key, 2189310712b2SOmar Sandoval struct btrfs_path *p, int find_higher, 2190310712b2SOmar Sandoval int return_any) 21912f38b3e1SArne Jansen { 21922f38b3e1SArne Jansen int ret; 21932f38b3e1SArne Jansen struct extent_buffer *leaf; 21942f38b3e1SArne Jansen 21952f38b3e1SArne Jansen again: 21962f38b3e1SArne Jansen ret = btrfs_search_slot(NULL, root, key, p, 0, 0); 21972f38b3e1SArne Jansen if (ret <= 0) 21982f38b3e1SArne Jansen return ret; 21992f38b3e1SArne Jansen /* 22002f38b3e1SArne Jansen * a return value of 1 means the path is at the position where the 22012f38b3e1SArne Jansen * item should be inserted. Normally this is the next bigger item, 22022f38b3e1SArne Jansen * but in case the previous item is the last in a leaf, path points 22032f38b3e1SArne Jansen * to the first free slot in the previous leaf, i.e. at an invalid 22042f38b3e1SArne Jansen * item. 22052f38b3e1SArne Jansen */ 22062f38b3e1SArne Jansen leaf = p->nodes[0]; 22072f38b3e1SArne Jansen 22082f38b3e1SArne Jansen if (find_higher) { 22092f38b3e1SArne Jansen if (p->slots[0] >= btrfs_header_nritems(leaf)) { 22102f38b3e1SArne Jansen ret = btrfs_next_leaf(root, p); 22112f38b3e1SArne Jansen if (ret <= 0) 22122f38b3e1SArne Jansen return ret; 22132f38b3e1SArne Jansen if (!return_any) 22142f38b3e1SArne Jansen return 1; 22152f38b3e1SArne Jansen /* 22162f38b3e1SArne Jansen * no higher item found, return the next 22172f38b3e1SArne Jansen * lower instead 22182f38b3e1SArne Jansen */ 22192f38b3e1SArne Jansen return_any = 0; 22202f38b3e1SArne Jansen find_higher = 0; 22212f38b3e1SArne Jansen btrfs_release_path(p); 22222f38b3e1SArne Jansen goto again; 22232f38b3e1SArne Jansen } 22242f38b3e1SArne Jansen } else { 22252f38b3e1SArne Jansen if (p->slots[0] == 0) { 22262f38b3e1SArne Jansen ret = btrfs_prev_leaf(root, p); 2227e6793769SArne Jansen if (ret < 0) 22282f38b3e1SArne Jansen return ret; 2229e6793769SArne Jansen if (!ret) { 223023c6bf6aSFilipe David Borba Manana leaf = p->nodes[0]; 223123c6bf6aSFilipe David Borba Manana if (p->slots[0] == btrfs_header_nritems(leaf)) 223223c6bf6aSFilipe David Borba Manana p->slots[0]--; 2233e6793769SArne Jansen return 0; 2234e6793769SArne Jansen } 22352f38b3e1SArne Jansen if (!return_any) 22362f38b3e1SArne Jansen return 1; 22372f38b3e1SArne Jansen /* 22382f38b3e1SArne Jansen * no lower item found, return the next 22392f38b3e1SArne Jansen * higher instead 22402f38b3e1SArne Jansen */ 22412f38b3e1SArne Jansen return_any = 0; 22422f38b3e1SArne Jansen find_higher = 1; 22432f38b3e1SArne Jansen btrfs_release_path(p); 22442f38b3e1SArne Jansen goto again; 2245e6793769SArne Jansen } else { 22462f38b3e1SArne Jansen --p->slots[0]; 22472f38b3e1SArne Jansen } 22482f38b3e1SArne Jansen } 22492f38b3e1SArne Jansen return 0; 22502f38b3e1SArne Jansen } 22512f38b3e1SArne Jansen 22522f38b3e1SArne Jansen /* 22530ff40a91SMarcos Paulo de Souza * Execute search and call btrfs_previous_item to traverse backwards if the item 22540ff40a91SMarcos Paulo de Souza * was not found. 22550ff40a91SMarcos Paulo de Souza * 22560ff40a91SMarcos Paulo de Souza * Return 0 if found, 1 if not found and < 0 if error. 22570ff40a91SMarcos Paulo de Souza */ 22580ff40a91SMarcos Paulo de Souza int btrfs_search_backwards(struct btrfs_root *root, struct btrfs_key *key, 22590ff40a91SMarcos Paulo de Souza struct btrfs_path *path) 22600ff40a91SMarcos Paulo de Souza { 22610ff40a91SMarcos Paulo de Souza int ret; 22620ff40a91SMarcos Paulo de Souza 22630ff40a91SMarcos Paulo de Souza ret = btrfs_search_slot(NULL, root, key, path, 0, 0); 22640ff40a91SMarcos Paulo de Souza if (ret > 0) 22650ff40a91SMarcos Paulo de Souza ret = btrfs_previous_item(root, path, key->objectid, key->type); 22660ff40a91SMarcos Paulo de Souza 22670ff40a91SMarcos Paulo de Souza if (ret == 0) 22680ff40a91SMarcos Paulo de Souza btrfs_item_key_to_cpu(path->nodes[0], key, path->slots[0]); 22690ff40a91SMarcos Paulo de Souza 22700ff40a91SMarcos Paulo de Souza return ret; 22710ff40a91SMarcos Paulo de Souza } 22720ff40a91SMarcos Paulo de Souza 22730ff40a91SMarcos Paulo de Souza /* 227474123bd7SChris Mason * adjust the pointers going up the tree, starting at level 227574123bd7SChris Mason * making sure the right key of each node is points to 'key'. 227674123bd7SChris Mason * This is used after shifting pointers to the left, so it stops 227774123bd7SChris Mason * fixing up pointers when a given leaf/node is not in slot 0 of the 227874123bd7SChris Mason * higher levels 2279aa5d6bedSChris Mason * 228074123bd7SChris Mason */ 2281b167fa91SNikolay Borisov static void fixup_low_keys(struct btrfs_path *path, 22825f39d397SChris Mason struct btrfs_disk_key *key, int level) 2283be0e5c09SChris Mason { 2284be0e5c09SChris Mason int i; 22855f39d397SChris Mason struct extent_buffer *t; 22860e82bcfeSDavid Sterba int ret; 22875f39d397SChris Mason 2288234b63a0SChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 2289be0e5c09SChris Mason int tslot = path->slots[i]; 22900e82bcfeSDavid Sterba 2291eb60ceacSChris Mason if (!path->nodes[i]) 2292be0e5c09SChris Mason break; 22935f39d397SChris Mason t = path->nodes[i]; 2294f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(t, tslot, 2295f3a84ccdSFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE, GFP_ATOMIC); 22960e82bcfeSDavid Sterba BUG_ON(ret < 0); 22975f39d397SChris Mason btrfs_set_node_key(t, key, tslot); 2298d6025579SChris Mason btrfs_mark_buffer_dirty(path->nodes[i]); 2299be0e5c09SChris Mason if (tslot != 0) 2300be0e5c09SChris Mason break; 2301be0e5c09SChris Mason } 2302be0e5c09SChris Mason } 2303be0e5c09SChris Mason 230474123bd7SChris Mason /* 230531840ae1SZheng Yan * update item key. 230631840ae1SZheng Yan * 230731840ae1SZheng Yan * This function isn't completely safe. It's the caller's responsibility 230831840ae1SZheng Yan * that the new key won't break the order 230931840ae1SZheng Yan */ 2310b7a0365eSDaniel Dressler void btrfs_set_item_key_safe(struct btrfs_fs_info *fs_info, 2311b7a0365eSDaniel Dressler struct btrfs_path *path, 2312310712b2SOmar Sandoval const struct btrfs_key *new_key) 231331840ae1SZheng Yan { 231431840ae1SZheng Yan struct btrfs_disk_key disk_key; 231531840ae1SZheng Yan struct extent_buffer *eb; 231631840ae1SZheng Yan int slot; 231731840ae1SZheng Yan 231831840ae1SZheng Yan eb = path->nodes[0]; 231931840ae1SZheng Yan slot = path->slots[0]; 232031840ae1SZheng Yan if (slot > 0) { 232131840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot - 1); 23227c15d410SQu Wenruo if (unlikely(comp_keys(&disk_key, new_key) >= 0)) { 23237c15d410SQu Wenruo btrfs_crit(fs_info, 23247c15d410SQu Wenruo "slot %u key (%llu %u %llu) new key (%llu %u %llu)", 23257c15d410SQu Wenruo slot, btrfs_disk_key_objectid(&disk_key), 23267c15d410SQu Wenruo btrfs_disk_key_type(&disk_key), 23277c15d410SQu Wenruo btrfs_disk_key_offset(&disk_key), 23287c15d410SQu Wenruo new_key->objectid, new_key->type, 23297c15d410SQu Wenruo new_key->offset); 23307c15d410SQu Wenruo btrfs_print_leaf(eb); 23317c15d410SQu Wenruo BUG(); 23327c15d410SQu Wenruo } 233331840ae1SZheng Yan } 233431840ae1SZheng Yan if (slot < btrfs_header_nritems(eb) - 1) { 233531840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot + 1); 23367c15d410SQu Wenruo if (unlikely(comp_keys(&disk_key, new_key) <= 0)) { 23377c15d410SQu Wenruo btrfs_crit(fs_info, 23387c15d410SQu Wenruo "slot %u key (%llu %u %llu) new key (%llu %u %llu)", 23397c15d410SQu Wenruo slot, btrfs_disk_key_objectid(&disk_key), 23407c15d410SQu Wenruo btrfs_disk_key_type(&disk_key), 23417c15d410SQu Wenruo btrfs_disk_key_offset(&disk_key), 23427c15d410SQu Wenruo new_key->objectid, new_key->type, 23437c15d410SQu Wenruo new_key->offset); 23447c15d410SQu Wenruo btrfs_print_leaf(eb); 23457c15d410SQu Wenruo BUG(); 23467c15d410SQu Wenruo } 234731840ae1SZheng Yan } 234831840ae1SZheng Yan 234931840ae1SZheng Yan btrfs_cpu_key_to_disk(&disk_key, new_key); 235031840ae1SZheng Yan btrfs_set_item_key(eb, &disk_key, slot); 235131840ae1SZheng Yan btrfs_mark_buffer_dirty(eb); 235231840ae1SZheng Yan if (slot == 0) 2353b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 235431840ae1SZheng Yan } 235531840ae1SZheng Yan 235631840ae1SZheng Yan /* 2357d16c702fSQu Wenruo * Check key order of two sibling extent buffers. 2358d16c702fSQu Wenruo * 2359d16c702fSQu Wenruo * Return true if something is wrong. 2360d16c702fSQu Wenruo * Return false if everything is fine. 2361d16c702fSQu Wenruo * 2362d16c702fSQu Wenruo * Tree-checker only works inside one tree block, thus the following 2363d16c702fSQu Wenruo * corruption can not be detected by tree-checker: 2364d16c702fSQu Wenruo * 2365d16c702fSQu Wenruo * Leaf @left | Leaf @right 2366d16c702fSQu Wenruo * -------------------------------------------------------------- 2367d16c702fSQu Wenruo * | 1 | 2 | 3 | 4 | 5 | f6 | | 7 | 8 | 2368d16c702fSQu Wenruo * 2369d16c702fSQu Wenruo * Key f6 in leaf @left itself is valid, but not valid when the next 2370d16c702fSQu Wenruo * key in leaf @right is 7. 2371d16c702fSQu Wenruo * This can only be checked at tree block merge time. 2372d16c702fSQu Wenruo * And since tree checker has ensured all key order in each tree block 2373d16c702fSQu Wenruo * is correct, we only need to bother the last key of @left and the first 2374d16c702fSQu Wenruo * key of @right. 2375d16c702fSQu Wenruo */ 2376d16c702fSQu Wenruo static bool check_sibling_keys(struct extent_buffer *left, 2377d16c702fSQu Wenruo struct extent_buffer *right) 2378d16c702fSQu Wenruo { 2379d16c702fSQu Wenruo struct btrfs_key left_last; 2380d16c702fSQu Wenruo struct btrfs_key right_first; 2381d16c702fSQu Wenruo int level = btrfs_header_level(left); 2382d16c702fSQu Wenruo int nr_left = btrfs_header_nritems(left); 2383d16c702fSQu Wenruo int nr_right = btrfs_header_nritems(right); 2384d16c702fSQu Wenruo 2385d16c702fSQu Wenruo /* No key to check in one of the tree blocks */ 2386d16c702fSQu Wenruo if (!nr_left || !nr_right) 2387d16c702fSQu Wenruo return false; 2388d16c702fSQu Wenruo 2389d16c702fSQu Wenruo if (level) { 2390d16c702fSQu Wenruo btrfs_node_key_to_cpu(left, &left_last, nr_left - 1); 2391d16c702fSQu Wenruo btrfs_node_key_to_cpu(right, &right_first, 0); 2392d16c702fSQu Wenruo } else { 2393d16c702fSQu Wenruo btrfs_item_key_to_cpu(left, &left_last, nr_left - 1); 2394d16c702fSQu Wenruo btrfs_item_key_to_cpu(right, &right_first, 0); 2395d16c702fSQu Wenruo } 2396d16c702fSQu Wenruo 2397d16c702fSQu Wenruo if (btrfs_comp_cpu_keys(&left_last, &right_first) >= 0) { 2398d16c702fSQu Wenruo btrfs_crit(left->fs_info, 2399d16c702fSQu Wenruo "bad key order, sibling blocks, left last (%llu %u %llu) right first (%llu %u %llu)", 2400d16c702fSQu Wenruo left_last.objectid, left_last.type, 2401d16c702fSQu Wenruo left_last.offset, right_first.objectid, 2402d16c702fSQu Wenruo right_first.type, right_first.offset); 2403d16c702fSQu Wenruo return true; 2404d16c702fSQu Wenruo } 2405d16c702fSQu Wenruo return false; 2406d16c702fSQu Wenruo } 2407d16c702fSQu Wenruo 2408d16c702fSQu Wenruo /* 240974123bd7SChris Mason * try to push data from one node into the next node left in the 241079f95c82SChris Mason * tree. 2411aa5d6bedSChris Mason * 2412aa5d6bedSChris Mason * returns 0 if some ptrs were pushed left, < 0 if there was some horrible 2413aa5d6bedSChris Mason * error, and > 0 if there was no room in the left hand block. 241474123bd7SChris Mason */ 241598ed5174SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 24162ff7e61eSJeff Mahoney struct extent_buffer *dst, 2417971a1f66SChris Mason struct extent_buffer *src, int empty) 2418be0e5c09SChris Mason { 2419d30a668fSDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info; 2420be0e5c09SChris Mason int push_items = 0; 2421bb803951SChris Mason int src_nritems; 2422bb803951SChris Mason int dst_nritems; 2423aa5d6bedSChris Mason int ret = 0; 2424be0e5c09SChris Mason 24255f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 24265f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 24270b246afaSJeff Mahoney push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems; 24287bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 24297bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 243054aa1f4dSChris Mason 2431bce4eae9SChris Mason if (!empty && src_nritems <= 8) 2432971a1f66SChris Mason return 1; 2433971a1f66SChris Mason 2434d397712bSChris Mason if (push_items <= 0) 2435be0e5c09SChris Mason return 1; 2436be0e5c09SChris Mason 2437bce4eae9SChris Mason if (empty) { 2438971a1f66SChris Mason push_items = min(src_nritems, push_items); 2439bce4eae9SChris Mason if (push_items < src_nritems) { 2440bce4eae9SChris Mason /* leave at least 8 pointers in the node if 2441bce4eae9SChris Mason * we aren't going to empty it 2442bce4eae9SChris Mason */ 2443bce4eae9SChris Mason if (src_nritems - push_items < 8) { 2444bce4eae9SChris Mason if (push_items <= 8) 2445bce4eae9SChris Mason return 1; 2446bce4eae9SChris Mason push_items -= 8; 2447bce4eae9SChris Mason } 2448bce4eae9SChris Mason } 2449bce4eae9SChris Mason } else 2450bce4eae9SChris Mason push_items = min(src_nritems - 8, push_items); 245179f95c82SChris Mason 2452d16c702fSQu Wenruo /* dst is the left eb, src is the middle eb */ 2453d16c702fSQu Wenruo if (check_sibling_keys(dst, src)) { 2454d16c702fSQu Wenruo ret = -EUCLEAN; 2455d16c702fSQu Wenruo btrfs_abort_transaction(trans, ret); 2456d16c702fSQu Wenruo return ret; 2457d16c702fSQu Wenruo } 2458f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_eb_copy(dst, src, dst_nritems, 0, push_items); 24595de865eeSFilipe David Borba Manana if (ret) { 246066642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 24615de865eeSFilipe David Borba Manana return ret; 24625de865eeSFilipe David Borba Manana } 24635f39d397SChris Mason copy_extent_buffer(dst, src, 24645f39d397SChris Mason btrfs_node_key_ptr_offset(dst_nritems), 24655f39d397SChris Mason btrfs_node_key_ptr_offset(0), 2466123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 24675f39d397SChris Mason 2468bb803951SChris Mason if (push_items < src_nritems) { 246957911b8bSJan Schmidt /* 2470f3a84ccdSFilipe Manana * Don't call btrfs_tree_mod_log_insert_move() here, key removal 2471f3a84ccdSFilipe Manana * was already fully logged by btrfs_tree_mod_log_eb_copy() above. 247257911b8bSJan Schmidt */ 24735f39d397SChris Mason memmove_extent_buffer(src, btrfs_node_key_ptr_offset(0), 24745f39d397SChris Mason btrfs_node_key_ptr_offset(push_items), 2475e2fa7227SChris Mason (src_nritems - push_items) * 2476123abc88SChris Mason sizeof(struct btrfs_key_ptr)); 2477bb803951SChris Mason } 24785f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 24795f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 24805f39d397SChris Mason btrfs_mark_buffer_dirty(src); 24815f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 248231840ae1SZheng Yan 2483bb803951SChris Mason return ret; 2484be0e5c09SChris Mason } 2485be0e5c09SChris Mason 248697571fd0SChris Mason /* 248779f95c82SChris Mason * try to push data from one node into the next node right in the 248879f95c82SChris Mason * tree. 248979f95c82SChris Mason * 249079f95c82SChris Mason * returns 0 if some ptrs were pushed, < 0 if there was some horrible 249179f95c82SChris Mason * error, and > 0 if there was no room in the right hand block. 249279f95c82SChris Mason * 249379f95c82SChris Mason * this will only push up to 1/2 the contents of the left node over 249479f95c82SChris Mason */ 24955f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 24965f39d397SChris Mason struct extent_buffer *dst, 24975f39d397SChris Mason struct extent_buffer *src) 249879f95c82SChris Mason { 249955d32ed8SDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info; 250079f95c82SChris Mason int push_items = 0; 250179f95c82SChris Mason int max_push; 250279f95c82SChris Mason int src_nritems; 250379f95c82SChris Mason int dst_nritems; 250479f95c82SChris Mason int ret = 0; 250579f95c82SChris Mason 25067bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 25077bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 25087bb86316SChris Mason 25095f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 25105f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 25110b246afaSJeff Mahoney push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems; 2512d397712bSChris Mason if (push_items <= 0) 251379f95c82SChris Mason return 1; 2514bce4eae9SChris Mason 2515d397712bSChris Mason if (src_nritems < 4) 2516bce4eae9SChris Mason return 1; 251779f95c82SChris Mason 251879f95c82SChris Mason max_push = src_nritems / 2 + 1; 251979f95c82SChris Mason /* don't try to empty the node */ 2520d397712bSChris Mason if (max_push >= src_nritems) 252179f95c82SChris Mason return 1; 2522252c38f0SYan 252379f95c82SChris Mason if (max_push < push_items) 252479f95c82SChris Mason push_items = max_push; 252579f95c82SChris Mason 2526d16c702fSQu Wenruo /* dst is the right eb, src is the middle eb */ 2527d16c702fSQu Wenruo if (check_sibling_keys(src, dst)) { 2528d16c702fSQu Wenruo ret = -EUCLEAN; 2529d16c702fSQu Wenruo btrfs_abort_transaction(trans, ret); 2530d16c702fSQu Wenruo return ret; 2531d16c702fSQu Wenruo } 2532f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_move(dst, push_items, 0, dst_nritems); 2533bf1d3425SDavid Sterba BUG_ON(ret < 0); 25345f39d397SChris Mason memmove_extent_buffer(dst, btrfs_node_key_ptr_offset(push_items), 25355f39d397SChris Mason btrfs_node_key_ptr_offset(0), 25365f39d397SChris Mason (dst_nritems) * 25375f39d397SChris Mason sizeof(struct btrfs_key_ptr)); 2538d6025579SChris Mason 2539f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_eb_copy(dst, src, 0, src_nritems - push_items, 2540ed874f0dSDavid Sterba push_items); 25415de865eeSFilipe David Borba Manana if (ret) { 254266642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 25435de865eeSFilipe David Borba Manana return ret; 25445de865eeSFilipe David Borba Manana } 25455f39d397SChris Mason copy_extent_buffer(dst, src, 25465f39d397SChris Mason btrfs_node_key_ptr_offset(0), 25475f39d397SChris Mason btrfs_node_key_ptr_offset(src_nritems - push_items), 2548123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 254979f95c82SChris Mason 25505f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 25515f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 255279f95c82SChris Mason 25535f39d397SChris Mason btrfs_mark_buffer_dirty(src); 25545f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 255531840ae1SZheng Yan 255679f95c82SChris Mason return ret; 255779f95c82SChris Mason } 255879f95c82SChris Mason 255979f95c82SChris Mason /* 256097571fd0SChris Mason * helper function to insert a new root level in the tree. 256197571fd0SChris Mason * A new node is allocated, and a single item is inserted to 256297571fd0SChris Mason * point to the existing root 2563aa5d6bedSChris Mason * 2564aa5d6bedSChris Mason * returns zero on success or < 0 on failure. 256597571fd0SChris Mason */ 2566d397712bSChris Mason static noinline int insert_new_root(struct btrfs_trans_handle *trans, 25675f39d397SChris Mason struct btrfs_root *root, 2568fdd99c72SLiu Bo struct btrfs_path *path, int level) 256974123bd7SChris Mason { 25700b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 25717bb86316SChris Mason u64 lower_gen; 25725f39d397SChris Mason struct extent_buffer *lower; 25735f39d397SChris Mason struct extent_buffer *c; 2574925baeddSChris Mason struct extent_buffer *old; 25755f39d397SChris Mason struct btrfs_disk_key lower_key; 2576d9d19a01SDavid Sterba int ret; 25775c680ed6SChris Mason 25785c680ed6SChris Mason BUG_ON(path->nodes[level]); 25795c680ed6SChris Mason BUG_ON(path->nodes[level-1] != root->node); 25805c680ed6SChris Mason 25817bb86316SChris Mason lower = path->nodes[level-1]; 25827bb86316SChris Mason if (level == 1) 25837bb86316SChris Mason btrfs_item_key(lower, &lower_key, 0); 25847bb86316SChris Mason else 25857bb86316SChris Mason btrfs_node_key(lower, &lower_key, 0); 25867bb86316SChris Mason 258779bd3712SFilipe Manana c = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 258879bd3712SFilipe Manana &lower_key, level, root->node->start, 0, 2589cf6f34aaSJosef Bacik BTRFS_NESTING_NEW_ROOT); 25905f39d397SChris Mason if (IS_ERR(c)) 25915f39d397SChris Mason return PTR_ERR(c); 2592925baeddSChris Mason 25930b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 2594f0486c68SYan, Zheng 25955f39d397SChris Mason btrfs_set_header_nritems(c, 1); 25965f39d397SChris Mason btrfs_set_node_key(c, &lower_key, 0); 2597db94535dSChris Mason btrfs_set_node_blockptr(c, 0, lower->start); 25987bb86316SChris Mason lower_gen = btrfs_header_generation(lower); 259931840ae1SZheng Yan WARN_ON(lower_gen != trans->transid); 26007bb86316SChris Mason 26017bb86316SChris Mason btrfs_set_node_ptr_generation(c, 0, lower_gen); 26025f39d397SChris Mason 26035f39d397SChris Mason btrfs_mark_buffer_dirty(c); 2604d5719762SChris Mason 2605925baeddSChris Mason old = root->node; 2606406808abSFilipe Manana ret = btrfs_tree_mod_log_insert_root(root->node, c, false); 2607d9d19a01SDavid Sterba BUG_ON(ret < 0); 2608240f62c8SChris Mason rcu_assign_pointer(root->node, c); 2609925baeddSChris Mason 2610925baeddSChris Mason /* the super has an extra ref to root->node */ 2611925baeddSChris Mason free_extent_buffer(old); 2612925baeddSChris Mason 26130b86a832SChris Mason add_root_to_dirty_list(root); 261467439dadSDavid Sterba atomic_inc(&c->refs); 26155f39d397SChris Mason path->nodes[level] = c; 2616ac5887c8SJosef Bacik path->locks[level] = BTRFS_WRITE_LOCK; 261774123bd7SChris Mason path->slots[level] = 0; 261874123bd7SChris Mason return 0; 261974123bd7SChris Mason } 26205c680ed6SChris Mason 26215c680ed6SChris Mason /* 26225c680ed6SChris Mason * worker function to insert a single pointer in a node. 26235c680ed6SChris Mason * the node should have enough room for the pointer already 262497571fd0SChris Mason * 26255c680ed6SChris Mason * slot and level indicate where you want the key to go, and 26265c680ed6SChris Mason * blocknr is the block the key points to. 26275c680ed6SChris Mason */ 2628143bede5SJeff Mahoney static void insert_ptr(struct btrfs_trans_handle *trans, 26296ad3cf6dSDavid Sterba struct btrfs_path *path, 2630143bede5SJeff Mahoney struct btrfs_disk_key *key, u64 bytenr, 2631c3e06965SJan Schmidt int slot, int level) 26325c680ed6SChris Mason { 26335f39d397SChris Mason struct extent_buffer *lower; 26345c680ed6SChris Mason int nritems; 2635f3ea38daSJan Schmidt int ret; 26365c680ed6SChris Mason 26375c680ed6SChris Mason BUG_ON(!path->nodes[level]); 263849d0c642SFilipe Manana btrfs_assert_tree_write_locked(path->nodes[level]); 26395f39d397SChris Mason lower = path->nodes[level]; 26405f39d397SChris Mason nritems = btrfs_header_nritems(lower); 2641c293498bSStoyan Gaydarov BUG_ON(slot > nritems); 26426ad3cf6dSDavid Sterba BUG_ON(nritems == BTRFS_NODEPTRS_PER_BLOCK(trans->fs_info)); 264374123bd7SChris Mason if (slot != nritems) { 2644bf1d3425SDavid Sterba if (level) { 2645f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_move(lower, slot + 1, 2646f3a84ccdSFilipe Manana slot, nritems - slot); 2647bf1d3425SDavid Sterba BUG_ON(ret < 0); 2648bf1d3425SDavid Sterba } 26495f39d397SChris Mason memmove_extent_buffer(lower, 26505f39d397SChris Mason btrfs_node_key_ptr_offset(slot + 1), 26515f39d397SChris Mason btrfs_node_key_ptr_offset(slot), 2652123abc88SChris Mason (nritems - slot) * sizeof(struct btrfs_key_ptr)); 265374123bd7SChris Mason } 2654c3e06965SJan Schmidt if (level) { 2655f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(lower, slot, 2656f3a84ccdSFilipe Manana BTRFS_MOD_LOG_KEY_ADD, GFP_NOFS); 2657f3ea38daSJan Schmidt BUG_ON(ret < 0); 2658f3ea38daSJan Schmidt } 26595f39d397SChris Mason btrfs_set_node_key(lower, key, slot); 2660db94535dSChris Mason btrfs_set_node_blockptr(lower, slot, bytenr); 266174493f7aSChris Mason WARN_ON(trans->transid == 0); 266274493f7aSChris Mason btrfs_set_node_ptr_generation(lower, slot, trans->transid); 26635f39d397SChris Mason btrfs_set_header_nritems(lower, nritems + 1); 26645f39d397SChris Mason btrfs_mark_buffer_dirty(lower); 266574123bd7SChris Mason } 266674123bd7SChris Mason 266797571fd0SChris Mason /* 266897571fd0SChris Mason * split the node at the specified level in path in two. 266997571fd0SChris Mason * The path is corrected to point to the appropriate node after the split 267097571fd0SChris Mason * 267197571fd0SChris Mason * Before splitting this tries to make some room in the node by pushing 267297571fd0SChris Mason * left and right, if either one works, it returns right away. 2673aa5d6bedSChris Mason * 2674aa5d6bedSChris Mason * returns 0 on success and < 0 on failure 267597571fd0SChris Mason */ 2676e02119d5SChris Mason static noinline int split_node(struct btrfs_trans_handle *trans, 2677e02119d5SChris Mason struct btrfs_root *root, 2678e02119d5SChris Mason struct btrfs_path *path, int level) 2679be0e5c09SChris Mason { 26800b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 26815f39d397SChris Mason struct extent_buffer *c; 26825f39d397SChris Mason struct extent_buffer *split; 26835f39d397SChris Mason struct btrfs_disk_key disk_key; 2684be0e5c09SChris Mason int mid; 26855c680ed6SChris Mason int ret; 26867518a238SChris Mason u32 c_nritems; 2687be0e5c09SChris Mason 26885f39d397SChris Mason c = path->nodes[level]; 26897bb86316SChris Mason WARN_ON(btrfs_header_generation(c) != trans->transid); 26905f39d397SChris Mason if (c == root->node) { 2691d9abbf1cSJan Schmidt /* 269290f8d62eSJan Schmidt * trying to split the root, lets make a new one 269390f8d62eSJan Schmidt * 2694fdd99c72SLiu Bo * tree mod log: We don't log_removal old root in 269590f8d62eSJan Schmidt * insert_new_root, because that root buffer will be kept as a 269690f8d62eSJan Schmidt * normal node. We are going to log removal of half of the 2697f3a84ccdSFilipe Manana * elements below with btrfs_tree_mod_log_eb_copy(). We're 2698f3a84ccdSFilipe Manana * holding a tree lock on the buffer, which is why we cannot 2699f3a84ccdSFilipe Manana * race with other tree_mod_log users. 2700d9abbf1cSJan Schmidt */ 2701fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, level + 1); 27025c680ed6SChris Mason if (ret) 27035c680ed6SChris Mason return ret; 2704b3612421SChris Mason } else { 2705e66f709bSChris Mason ret = push_nodes_for_insert(trans, root, path, level); 27065f39d397SChris Mason c = path->nodes[level]; 27075f39d397SChris Mason if (!ret && btrfs_header_nritems(c) < 27080b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3) 2709e66f709bSChris Mason return 0; 271054aa1f4dSChris Mason if (ret < 0) 271154aa1f4dSChris Mason return ret; 27125c680ed6SChris Mason } 2713e66f709bSChris Mason 27145f39d397SChris Mason c_nritems = btrfs_header_nritems(c); 27155d4f98a2SYan Zheng mid = (c_nritems + 1) / 2; 27165d4f98a2SYan Zheng btrfs_node_key(c, &disk_key, mid); 27177bb86316SChris Mason 271879bd3712SFilipe Manana split = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 271979bd3712SFilipe Manana &disk_key, level, c->start, 0, 272079bd3712SFilipe Manana BTRFS_NESTING_SPLIT); 27215f39d397SChris Mason if (IS_ERR(split)) 27225f39d397SChris Mason return PTR_ERR(split); 272354aa1f4dSChris Mason 27240b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 2725bc877d28SNikolay Borisov ASSERT(btrfs_header_level(c) == level); 27265f39d397SChris Mason 2727f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_eb_copy(split, c, 0, mid, c_nritems - mid); 27285de865eeSFilipe David Borba Manana if (ret) { 272966642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 27305de865eeSFilipe David Borba Manana return ret; 27315de865eeSFilipe David Borba Manana } 27325f39d397SChris Mason copy_extent_buffer(split, c, 27335f39d397SChris Mason btrfs_node_key_ptr_offset(0), 27345f39d397SChris Mason btrfs_node_key_ptr_offset(mid), 2735123abc88SChris Mason (c_nritems - mid) * sizeof(struct btrfs_key_ptr)); 27365f39d397SChris Mason btrfs_set_header_nritems(split, c_nritems - mid); 27375f39d397SChris Mason btrfs_set_header_nritems(c, mid); 2738aa5d6bedSChris Mason 27395f39d397SChris Mason btrfs_mark_buffer_dirty(c); 27405f39d397SChris Mason btrfs_mark_buffer_dirty(split); 27415f39d397SChris Mason 27426ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, split->start, 2743c3e06965SJan Schmidt path->slots[level + 1] + 1, level + 1); 2744aa5d6bedSChris Mason 27455de08d7dSChris Mason if (path->slots[level] >= mid) { 27465c680ed6SChris Mason path->slots[level] -= mid; 2747925baeddSChris Mason btrfs_tree_unlock(c); 27485f39d397SChris Mason free_extent_buffer(c); 27495f39d397SChris Mason path->nodes[level] = split; 27505c680ed6SChris Mason path->slots[level + 1] += 1; 2751eb60ceacSChris Mason } else { 2752925baeddSChris Mason btrfs_tree_unlock(split); 27535f39d397SChris Mason free_extent_buffer(split); 2754be0e5c09SChris Mason } 2755d5286a92SNikolay Borisov return 0; 2756be0e5c09SChris Mason } 2757be0e5c09SChris Mason 275874123bd7SChris Mason /* 275974123bd7SChris Mason * how many bytes are required to store the items in a leaf. start 276074123bd7SChris Mason * and nr indicate which items in the leaf to check. This totals up the 276174123bd7SChris Mason * space used both by the item structs and the item data 276274123bd7SChris Mason */ 27635f39d397SChris Mason static int leaf_space_used(struct extent_buffer *l, int start, int nr) 2764be0e5c09SChris Mason { 2765be0e5c09SChris Mason int data_len; 27665f39d397SChris Mason int nritems = btrfs_header_nritems(l); 2767d4dbff95SChris Mason int end = min(nritems, start + nr) - 1; 2768be0e5c09SChris Mason 2769be0e5c09SChris Mason if (!nr) 2770be0e5c09SChris Mason return 0; 27713212fa14SJosef Bacik data_len = btrfs_item_offset(l, start) + btrfs_item_size(l, start); 27723212fa14SJosef Bacik data_len = data_len - btrfs_item_offset(l, end); 27730783fcfcSChris Mason data_len += sizeof(struct btrfs_item) * nr; 2774d4dbff95SChris Mason WARN_ON(data_len < 0); 2775be0e5c09SChris Mason return data_len; 2776be0e5c09SChris Mason } 2777be0e5c09SChris Mason 277874123bd7SChris Mason /* 2779d4dbff95SChris Mason * The space between the end of the leaf items and 2780d4dbff95SChris Mason * the start of the leaf data. IOW, how much room 2781d4dbff95SChris Mason * the leaf has left for both items and data 2782d4dbff95SChris Mason */ 2783e902baacSDavid Sterba noinline int btrfs_leaf_free_space(struct extent_buffer *leaf) 2784d4dbff95SChris Mason { 2785e902baacSDavid Sterba struct btrfs_fs_info *fs_info = leaf->fs_info; 27865f39d397SChris Mason int nritems = btrfs_header_nritems(leaf); 27875f39d397SChris Mason int ret; 27880b246afaSJeff Mahoney 27890b246afaSJeff Mahoney ret = BTRFS_LEAF_DATA_SIZE(fs_info) - leaf_space_used(leaf, 0, nritems); 27905f39d397SChris Mason if (ret < 0) { 27910b246afaSJeff Mahoney btrfs_crit(fs_info, 2792efe120a0SFrank Holton "leaf free space ret %d, leaf data size %lu, used %d nritems %d", 2793da17066cSJeff Mahoney ret, 27940b246afaSJeff Mahoney (unsigned long) BTRFS_LEAF_DATA_SIZE(fs_info), 27955f39d397SChris Mason leaf_space_used(leaf, 0, nritems), nritems); 27965f39d397SChris Mason } 27975f39d397SChris Mason return ret; 2798d4dbff95SChris Mason } 2799d4dbff95SChris Mason 280099d8f83cSChris Mason /* 280199d8f83cSChris Mason * min slot controls the lowest index we're willing to push to the 280299d8f83cSChris Mason * right. We'll push up to and including min_slot, but no lower 280399d8f83cSChris Mason */ 2804f72f0010SDavid Sterba static noinline int __push_leaf_right(struct btrfs_path *path, 280544871b1bSChris Mason int data_size, int empty, 280644871b1bSChris Mason struct extent_buffer *right, 280799d8f83cSChris Mason int free_space, u32 left_nritems, 280899d8f83cSChris Mason u32 min_slot) 280900ec4c51SChris Mason { 2810f72f0010SDavid Sterba struct btrfs_fs_info *fs_info = right->fs_info; 28115f39d397SChris Mason struct extent_buffer *left = path->nodes[0]; 281244871b1bSChris Mason struct extent_buffer *upper = path->nodes[1]; 2813cfed81a0SChris Mason struct btrfs_map_token token; 28145f39d397SChris Mason struct btrfs_disk_key disk_key; 281500ec4c51SChris Mason int slot; 281634a38218SChris Mason u32 i; 281700ec4c51SChris Mason int push_space = 0; 281800ec4c51SChris Mason int push_items = 0; 281934a38218SChris Mason u32 nr; 28207518a238SChris Mason u32 right_nritems; 28215f39d397SChris Mason u32 data_end; 2822db94535dSChris Mason u32 this_item_size; 282300ec4c51SChris Mason 282434a38218SChris Mason if (empty) 282534a38218SChris Mason nr = 0; 282634a38218SChris Mason else 282799d8f83cSChris Mason nr = max_t(u32, 1, min_slot); 282834a38218SChris Mason 282931840ae1SZheng Yan if (path->slots[0] >= left_nritems) 283087b29b20SYan Zheng push_space += data_size; 283131840ae1SZheng Yan 283244871b1bSChris Mason slot = path->slots[1]; 283334a38218SChris Mason i = left_nritems - 1; 283434a38218SChris Mason while (i >= nr) { 283531840ae1SZheng Yan if (!empty && push_items > 0) { 283631840ae1SZheng Yan if (path->slots[0] > i) 283731840ae1SZheng Yan break; 283831840ae1SZheng Yan if (path->slots[0] == i) { 2839e902baacSDavid Sterba int space = btrfs_leaf_free_space(left); 2840e902baacSDavid Sterba 284131840ae1SZheng Yan if (space + push_space * 2 > free_space) 284231840ae1SZheng Yan break; 284331840ae1SZheng Yan } 284431840ae1SZheng Yan } 284531840ae1SZheng Yan 284600ec4c51SChris Mason if (path->slots[0] == i) 284787b29b20SYan Zheng push_space += data_size; 2848db94535dSChris Mason 28493212fa14SJosef Bacik this_item_size = btrfs_item_size(left, i); 285074794207SJosef Bacik if (this_item_size + sizeof(struct btrfs_item) + 285174794207SJosef Bacik push_space > free_space) 285200ec4c51SChris Mason break; 285331840ae1SZheng Yan 285400ec4c51SChris Mason push_items++; 285574794207SJosef Bacik push_space += this_item_size + sizeof(struct btrfs_item); 285634a38218SChris Mason if (i == 0) 285734a38218SChris Mason break; 285834a38218SChris Mason i--; 2859db94535dSChris Mason } 28605f39d397SChris Mason 2861925baeddSChris Mason if (push_items == 0) 2862925baeddSChris Mason goto out_unlock; 28635f39d397SChris Mason 28646c1500f2SJulia Lawall WARN_ON(!empty && push_items == left_nritems); 28655f39d397SChris Mason 286600ec4c51SChris Mason /* push left to right */ 28675f39d397SChris Mason right_nritems = btrfs_header_nritems(right); 286834a38218SChris Mason 2869dc2e724eSJosef Bacik push_space = btrfs_item_data_end(left, left_nritems - push_items); 28708f881e8cSDavid Sterba push_space -= leaf_data_end(left); 28715f39d397SChris Mason 287200ec4c51SChris Mason /* make room in the right data area */ 28738f881e8cSDavid Sterba data_end = leaf_data_end(right); 28745f39d397SChris Mason memmove_extent_buffer(right, 28753d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + data_end - push_space, 28763d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + data_end, 28770b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - data_end); 28785f39d397SChris Mason 287900ec4c51SChris Mason /* copy from the left data area */ 28803d9ec8c4SNikolay Borisov copy_extent_buffer(right, left, BTRFS_LEAF_DATA_OFFSET + 28810b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - push_space, 28828f881e8cSDavid Sterba BTRFS_LEAF_DATA_OFFSET + leaf_data_end(left), 2883d6025579SChris Mason push_space); 28845f39d397SChris Mason 28855f39d397SChris Mason memmove_extent_buffer(right, btrfs_item_nr_offset(push_items), 28865f39d397SChris Mason btrfs_item_nr_offset(0), 28870783fcfcSChris Mason right_nritems * sizeof(struct btrfs_item)); 28885f39d397SChris Mason 288900ec4c51SChris Mason /* copy the items from left to right */ 28905f39d397SChris Mason copy_extent_buffer(right, left, btrfs_item_nr_offset(0), 28915f39d397SChris Mason btrfs_item_nr_offset(left_nritems - push_items), 28920783fcfcSChris Mason push_items * sizeof(struct btrfs_item)); 289300ec4c51SChris Mason 289400ec4c51SChris Mason /* update the item pointers */ 2895c82f823cSDavid Sterba btrfs_init_map_token(&token, right); 28967518a238SChris Mason right_nritems += push_items; 28975f39d397SChris Mason btrfs_set_header_nritems(right, right_nritems); 28980b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info); 28997518a238SChris Mason for (i = 0; i < right_nritems; i++) { 29003212fa14SJosef Bacik push_space -= btrfs_token_item_size(&token, i); 29013212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, push_space); 2902db94535dSChris Mason } 2903db94535dSChris Mason 29047518a238SChris Mason left_nritems -= push_items; 29055f39d397SChris Mason btrfs_set_header_nritems(left, left_nritems); 290600ec4c51SChris Mason 290734a38218SChris Mason if (left_nritems) 29085f39d397SChris Mason btrfs_mark_buffer_dirty(left); 2909f0486c68SYan, Zheng else 29106a884d7dSDavid Sterba btrfs_clean_tree_block(left); 2911f0486c68SYan, Zheng 29125f39d397SChris Mason btrfs_mark_buffer_dirty(right); 2913a429e513SChris Mason 29145f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 29155f39d397SChris Mason btrfs_set_node_key(upper, &disk_key, slot + 1); 2916d6025579SChris Mason btrfs_mark_buffer_dirty(upper); 291702217ed2SChris Mason 291800ec4c51SChris Mason /* then fixup the leaf pointer in the path */ 29197518a238SChris Mason if (path->slots[0] >= left_nritems) { 29207518a238SChris Mason path->slots[0] -= left_nritems; 2921925baeddSChris Mason if (btrfs_header_nritems(path->nodes[0]) == 0) 29226a884d7dSDavid Sterba btrfs_clean_tree_block(path->nodes[0]); 2923925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 29245f39d397SChris Mason free_extent_buffer(path->nodes[0]); 29255f39d397SChris Mason path->nodes[0] = right; 292600ec4c51SChris Mason path->slots[1] += 1; 292700ec4c51SChris Mason } else { 2928925baeddSChris Mason btrfs_tree_unlock(right); 29295f39d397SChris Mason free_extent_buffer(right); 293000ec4c51SChris Mason } 293100ec4c51SChris Mason return 0; 2932925baeddSChris Mason 2933925baeddSChris Mason out_unlock: 2934925baeddSChris Mason btrfs_tree_unlock(right); 2935925baeddSChris Mason free_extent_buffer(right); 2936925baeddSChris Mason return 1; 293700ec4c51SChris Mason } 2938925baeddSChris Mason 293900ec4c51SChris Mason /* 294044871b1bSChris Mason * push some data in the path leaf to the right, trying to free up at 294174123bd7SChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 294244871b1bSChris Mason * 294344871b1bSChris Mason * returns 1 if the push failed because the other node didn't have enough 294444871b1bSChris Mason * room, 0 if everything worked out and < 0 if there were major errors. 294599d8f83cSChris Mason * 294699d8f83cSChris Mason * this will push starting from min_slot to the end of the leaf. It won't 294799d8f83cSChris Mason * push any slot lower than min_slot 294874123bd7SChris Mason */ 294944871b1bSChris Mason static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root 295099d8f83cSChris Mason *root, struct btrfs_path *path, 295199d8f83cSChris Mason int min_data_size, int data_size, 295299d8f83cSChris Mason int empty, u32 min_slot) 2953be0e5c09SChris Mason { 295444871b1bSChris Mason struct extent_buffer *left = path->nodes[0]; 295544871b1bSChris Mason struct extent_buffer *right; 295644871b1bSChris Mason struct extent_buffer *upper; 295744871b1bSChris Mason int slot; 295844871b1bSChris Mason int free_space; 295944871b1bSChris Mason u32 left_nritems; 296044871b1bSChris Mason int ret; 296144871b1bSChris Mason 296244871b1bSChris Mason if (!path->nodes[1]) 296344871b1bSChris Mason return 1; 296444871b1bSChris Mason 296544871b1bSChris Mason slot = path->slots[1]; 296644871b1bSChris Mason upper = path->nodes[1]; 296744871b1bSChris Mason if (slot >= btrfs_header_nritems(upper) - 1) 296844871b1bSChris Mason return 1; 296944871b1bSChris Mason 297049d0c642SFilipe Manana btrfs_assert_tree_write_locked(path->nodes[1]); 297144871b1bSChris Mason 29724b231ae4SDavid Sterba right = btrfs_read_node_slot(upper, slot + 1); 2973fb770ae4SLiu Bo /* 2974fb770ae4SLiu Bo * slot + 1 is not valid or we fail to read the right node, 2975fb770ae4SLiu Bo * no big deal, just return. 2976fb770ae4SLiu Bo */ 2977fb770ae4SLiu Bo if (IS_ERR(right)) 297891ca338dSTsutomu Itoh return 1; 297991ca338dSTsutomu Itoh 2980bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT); 298144871b1bSChris Mason 2982e902baacSDavid Sterba free_space = btrfs_leaf_free_space(right); 298344871b1bSChris Mason if (free_space < data_size) 298444871b1bSChris Mason goto out_unlock; 298544871b1bSChris Mason 298644871b1bSChris Mason /* cow and double check */ 298744871b1bSChris Mason ret = btrfs_cow_block(trans, root, right, upper, 2988bf59a5a2SJosef Bacik slot + 1, &right, BTRFS_NESTING_RIGHT_COW); 298944871b1bSChris Mason if (ret) 299044871b1bSChris Mason goto out_unlock; 299144871b1bSChris Mason 2992e902baacSDavid Sterba free_space = btrfs_leaf_free_space(right); 299344871b1bSChris Mason if (free_space < data_size) 299444871b1bSChris Mason goto out_unlock; 299544871b1bSChris Mason 299644871b1bSChris Mason left_nritems = btrfs_header_nritems(left); 299744871b1bSChris Mason if (left_nritems == 0) 299844871b1bSChris Mason goto out_unlock; 299944871b1bSChris Mason 3000d16c702fSQu Wenruo if (check_sibling_keys(left, right)) { 3001d16c702fSQu Wenruo ret = -EUCLEAN; 3002d16c702fSQu Wenruo btrfs_tree_unlock(right); 3003d16c702fSQu Wenruo free_extent_buffer(right); 3004d16c702fSQu Wenruo return ret; 3005d16c702fSQu Wenruo } 30062ef1fed2SFilipe David Borba Manana if (path->slots[0] == left_nritems && !empty) { 30072ef1fed2SFilipe David Borba Manana /* Key greater than all keys in the leaf, right neighbor has 30082ef1fed2SFilipe David Borba Manana * enough room for it and we're not emptying our leaf to delete 30092ef1fed2SFilipe David Borba Manana * it, therefore use right neighbor to insert the new item and 301052042d8eSAndrea Gelmini * no need to touch/dirty our left leaf. */ 30112ef1fed2SFilipe David Borba Manana btrfs_tree_unlock(left); 30122ef1fed2SFilipe David Borba Manana free_extent_buffer(left); 30132ef1fed2SFilipe David Borba Manana path->nodes[0] = right; 30142ef1fed2SFilipe David Borba Manana path->slots[0] = 0; 30152ef1fed2SFilipe David Borba Manana path->slots[1]++; 30162ef1fed2SFilipe David Borba Manana return 0; 30172ef1fed2SFilipe David Borba Manana } 30182ef1fed2SFilipe David Borba Manana 3019f72f0010SDavid Sterba return __push_leaf_right(path, min_data_size, empty, 302099d8f83cSChris Mason right, free_space, left_nritems, min_slot); 302144871b1bSChris Mason out_unlock: 302244871b1bSChris Mason btrfs_tree_unlock(right); 302344871b1bSChris Mason free_extent_buffer(right); 302444871b1bSChris Mason return 1; 302544871b1bSChris Mason } 302644871b1bSChris Mason 302744871b1bSChris Mason /* 302844871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 302944871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 303099d8f83cSChris Mason * 303199d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 303299d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us do all the 303399d8f83cSChris Mason * items 303444871b1bSChris Mason */ 30358087c193SDavid Sterba static noinline int __push_leaf_left(struct btrfs_path *path, int data_size, 303644871b1bSChris Mason int empty, struct extent_buffer *left, 303799d8f83cSChris Mason int free_space, u32 right_nritems, 303899d8f83cSChris Mason u32 max_slot) 303944871b1bSChris Mason { 30408087c193SDavid Sterba struct btrfs_fs_info *fs_info = left->fs_info; 30415f39d397SChris Mason struct btrfs_disk_key disk_key; 30425f39d397SChris Mason struct extent_buffer *right = path->nodes[0]; 3043be0e5c09SChris Mason int i; 3044be0e5c09SChris Mason int push_space = 0; 3045be0e5c09SChris Mason int push_items = 0; 30467518a238SChris Mason u32 old_left_nritems; 304734a38218SChris Mason u32 nr; 3048aa5d6bedSChris Mason int ret = 0; 3049db94535dSChris Mason u32 this_item_size; 3050db94535dSChris Mason u32 old_left_item_size; 3051cfed81a0SChris Mason struct btrfs_map_token token; 3052cfed81a0SChris Mason 305334a38218SChris Mason if (empty) 305499d8f83cSChris Mason nr = min(right_nritems, max_slot); 305534a38218SChris Mason else 305699d8f83cSChris Mason nr = min(right_nritems - 1, max_slot); 305734a38218SChris Mason 305834a38218SChris Mason for (i = 0; i < nr; i++) { 305931840ae1SZheng Yan if (!empty && push_items > 0) { 306031840ae1SZheng Yan if (path->slots[0] < i) 306131840ae1SZheng Yan break; 306231840ae1SZheng Yan if (path->slots[0] == i) { 3063e902baacSDavid Sterba int space = btrfs_leaf_free_space(right); 3064e902baacSDavid Sterba 306531840ae1SZheng Yan if (space + push_space * 2 > free_space) 306631840ae1SZheng Yan break; 306731840ae1SZheng Yan } 306831840ae1SZheng Yan } 306931840ae1SZheng Yan 3070be0e5c09SChris Mason if (path->slots[0] == i) 307187b29b20SYan Zheng push_space += data_size; 3072db94535dSChris Mason 30733212fa14SJosef Bacik this_item_size = btrfs_item_size(right, i); 307474794207SJosef Bacik if (this_item_size + sizeof(struct btrfs_item) + push_space > 307574794207SJosef Bacik free_space) 3076be0e5c09SChris Mason break; 3077db94535dSChris Mason 3078be0e5c09SChris Mason push_items++; 307974794207SJosef Bacik push_space += this_item_size + sizeof(struct btrfs_item); 3080be0e5c09SChris Mason } 3081db94535dSChris Mason 3082be0e5c09SChris Mason if (push_items == 0) { 3083925baeddSChris Mason ret = 1; 3084925baeddSChris Mason goto out; 3085be0e5c09SChris Mason } 3086fae7f21cSDulshani Gunawardhana WARN_ON(!empty && push_items == btrfs_header_nritems(right)); 30875f39d397SChris Mason 3088be0e5c09SChris Mason /* push data from right to left */ 30895f39d397SChris Mason copy_extent_buffer(left, right, 30905f39d397SChris Mason btrfs_item_nr_offset(btrfs_header_nritems(left)), 30915f39d397SChris Mason btrfs_item_nr_offset(0), 30925f39d397SChris Mason push_items * sizeof(struct btrfs_item)); 30935f39d397SChris Mason 30940b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info) - 30953212fa14SJosef Bacik btrfs_item_offset(right, push_items - 1); 30965f39d397SChris Mason 30973d9ec8c4SNikolay Borisov copy_extent_buffer(left, right, BTRFS_LEAF_DATA_OFFSET + 30988f881e8cSDavid Sterba leaf_data_end(left) - push_space, 30993d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + 31003212fa14SJosef Bacik btrfs_item_offset(right, push_items - 1), 3101be0e5c09SChris Mason push_space); 31025f39d397SChris Mason old_left_nritems = btrfs_header_nritems(left); 310387b29b20SYan Zheng BUG_ON(old_left_nritems <= 0); 3104eb60ceacSChris Mason 3105c82f823cSDavid Sterba btrfs_init_map_token(&token, left); 31063212fa14SJosef Bacik old_left_item_size = btrfs_item_offset(left, old_left_nritems - 1); 3107be0e5c09SChris Mason for (i = old_left_nritems; i < old_left_nritems + push_items; i++) { 31085f39d397SChris Mason u32 ioff; 3109db94535dSChris Mason 31103212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 31113212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, 3112cc4c13d5SDavid Sterba ioff - (BTRFS_LEAF_DATA_SIZE(fs_info) - old_left_item_size)); 3113be0e5c09SChris Mason } 31145f39d397SChris Mason btrfs_set_header_nritems(left, old_left_nritems + push_items); 3115be0e5c09SChris Mason 3116be0e5c09SChris Mason /* fixup right node */ 311731b1a2bdSJulia Lawall if (push_items > right_nritems) 311831b1a2bdSJulia Lawall WARN(1, KERN_CRIT "push items %d nr %u\n", push_items, 3119d397712bSChris Mason right_nritems); 312034a38218SChris Mason 312134a38218SChris Mason if (push_items < right_nritems) { 31223212fa14SJosef Bacik push_space = btrfs_item_offset(right, push_items - 1) - 31238f881e8cSDavid Sterba leaf_data_end(right); 31243d9ec8c4SNikolay Borisov memmove_extent_buffer(right, BTRFS_LEAF_DATA_OFFSET + 31250b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - push_space, 31263d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + 31278f881e8cSDavid Sterba leaf_data_end(right), push_space); 31285f39d397SChris Mason 31295f39d397SChris Mason memmove_extent_buffer(right, btrfs_item_nr_offset(0), 31305f39d397SChris Mason btrfs_item_nr_offset(push_items), 31315f39d397SChris Mason (btrfs_header_nritems(right) - push_items) * 31320783fcfcSChris Mason sizeof(struct btrfs_item)); 313334a38218SChris Mason } 3134c82f823cSDavid Sterba 3135c82f823cSDavid Sterba btrfs_init_map_token(&token, right); 3136eef1c494SYan right_nritems -= push_items; 3137eef1c494SYan btrfs_set_header_nritems(right, right_nritems); 31380b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info); 31395f39d397SChris Mason for (i = 0; i < right_nritems; i++) { 31403212fa14SJosef Bacik push_space = push_space - btrfs_token_item_size(&token, i); 31413212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, push_space); 3142db94535dSChris Mason } 3143eb60ceacSChris Mason 31445f39d397SChris Mason btrfs_mark_buffer_dirty(left); 314534a38218SChris Mason if (right_nritems) 31465f39d397SChris Mason btrfs_mark_buffer_dirty(right); 3147f0486c68SYan, Zheng else 31486a884d7dSDavid Sterba btrfs_clean_tree_block(right); 3149098f59c2SChris Mason 31505f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 3151b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 3152be0e5c09SChris Mason 3153be0e5c09SChris Mason /* then fixup the leaf pointer in the path */ 3154be0e5c09SChris Mason if (path->slots[0] < push_items) { 3155be0e5c09SChris Mason path->slots[0] += old_left_nritems; 3156925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 31575f39d397SChris Mason free_extent_buffer(path->nodes[0]); 31585f39d397SChris Mason path->nodes[0] = left; 3159be0e5c09SChris Mason path->slots[1] -= 1; 3160be0e5c09SChris Mason } else { 3161925baeddSChris Mason btrfs_tree_unlock(left); 31625f39d397SChris Mason free_extent_buffer(left); 3163be0e5c09SChris Mason path->slots[0] -= push_items; 3164be0e5c09SChris Mason } 3165eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 3166aa5d6bedSChris Mason return ret; 3167925baeddSChris Mason out: 3168925baeddSChris Mason btrfs_tree_unlock(left); 3169925baeddSChris Mason free_extent_buffer(left); 3170925baeddSChris Mason return ret; 3171be0e5c09SChris Mason } 3172be0e5c09SChris Mason 317374123bd7SChris Mason /* 317444871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 317544871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 317699d8f83cSChris Mason * 317799d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 317899d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us push all the 317999d8f83cSChris Mason * items 318044871b1bSChris Mason */ 318144871b1bSChris Mason static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root 318299d8f83cSChris Mason *root, struct btrfs_path *path, int min_data_size, 318399d8f83cSChris Mason int data_size, int empty, u32 max_slot) 318444871b1bSChris Mason { 318544871b1bSChris Mason struct extent_buffer *right = path->nodes[0]; 318644871b1bSChris Mason struct extent_buffer *left; 318744871b1bSChris Mason int slot; 318844871b1bSChris Mason int free_space; 318944871b1bSChris Mason u32 right_nritems; 319044871b1bSChris Mason int ret = 0; 319144871b1bSChris Mason 319244871b1bSChris Mason slot = path->slots[1]; 319344871b1bSChris Mason if (slot == 0) 319444871b1bSChris Mason return 1; 319544871b1bSChris Mason if (!path->nodes[1]) 319644871b1bSChris Mason return 1; 319744871b1bSChris Mason 319844871b1bSChris Mason right_nritems = btrfs_header_nritems(right); 319944871b1bSChris Mason if (right_nritems == 0) 320044871b1bSChris Mason return 1; 320144871b1bSChris Mason 320249d0c642SFilipe Manana btrfs_assert_tree_write_locked(path->nodes[1]); 320344871b1bSChris Mason 32044b231ae4SDavid Sterba left = btrfs_read_node_slot(path->nodes[1], slot - 1); 3205fb770ae4SLiu Bo /* 3206fb770ae4SLiu Bo * slot - 1 is not valid or we fail to read the left node, 3207fb770ae4SLiu Bo * no big deal, just return. 3208fb770ae4SLiu Bo */ 3209fb770ae4SLiu Bo if (IS_ERR(left)) 321091ca338dSTsutomu Itoh return 1; 321191ca338dSTsutomu Itoh 3212bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT); 321344871b1bSChris Mason 3214e902baacSDavid Sterba free_space = btrfs_leaf_free_space(left); 321544871b1bSChris Mason if (free_space < data_size) { 321644871b1bSChris Mason ret = 1; 321744871b1bSChris Mason goto out; 321844871b1bSChris Mason } 321944871b1bSChris Mason 322044871b1bSChris Mason /* cow and double check */ 322144871b1bSChris Mason ret = btrfs_cow_block(trans, root, left, 32229631e4ccSJosef Bacik path->nodes[1], slot - 1, &left, 3223bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW); 322444871b1bSChris Mason if (ret) { 322544871b1bSChris Mason /* we hit -ENOSPC, but it isn't fatal here */ 322679787eaaSJeff Mahoney if (ret == -ENOSPC) 322744871b1bSChris Mason ret = 1; 322844871b1bSChris Mason goto out; 322944871b1bSChris Mason } 323044871b1bSChris Mason 3231e902baacSDavid Sterba free_space = btrfs_leaf_free_space(left); 323244871b1bSChris Mason if (free_space < data_size) { 323344871b1bSChris Mason ret = 1; 323444871b1bSChris Mason goto out; 323544871b1bSChris Mason } 323644871b1bSChris Mason 3237d16c702fSQu Wenruo if (check_sibling_keys(left, right)) { 3238d16c702fSQu Wenruo ret = -EUCLEAN; 3239d16c702fSQu Wenruo goto out; 3240d16c702fSQu Wenruo } 32418087c193SDavid Sterba return __push_leaf_left(path, min_data_size, 324299d8f83cSChris Mason empty, left, free_space, right_nritems, 324399d8f83cSChris Mason max_slot); 324444871b1bSChris Mason out: 324544871b1bSChris Mason btrfs_tree_unlock(left); 324644871b1bSChris Mason free_extent_buffer(left); 324744871b1bSChris Mason return ret; 324844871b1bSChris Mason } 324944871b1bSChris Mason 325044871b1bSChris Mason /* 325174123bd7SChris Mason * split the path's leaf in two, making sure there is at least data_size 325274123bd7SChris Mason * available for the resulting leaf level of the path. 325374123bd7SChris Mason */ 3254143bede5SJeff Mahoney static noinline void copy_for_split(struct btrfs_trans_handle *trans, 325544871b1bSChris Mason struct btrfs_path *path, 325644871b1bSChris Mason struct extent_buffer *l, 325744871b1bSChris Mason struct extent_buffer *right, 325844871b1bSChris Mason int slot, int mid, int nritems) 3259be0e5c09SChris Mason { 326094f94ad9SDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info; 3261be0e5c09SChris Mason int data_copy_size; 3262be0e5c09SChris Mason int rt_data_off; 3263be0e5c09SChris Mason int i; 3264d4dbff95SChris Mason struct btrfs_disk_key disk_key; 3265cfed81a0SChris Mason struct btrfs_map_token token; 3266cfed81a0SChris Mason 32675f39d397SChris Mason nritems = nritems - mid; 32685f39d397SChris Mason btrfs_set_header_nritems(right, nritems); 3269dc2e724eSJosef Bacik data_copy_size = btrfs_item_data_end(l, mid) - leaf_data_end(l); 32705f39d397SChris Mason 32715f39d397SChris Mason copy_extent_buffer(right, l, btrfs_item_nr_offset(0), 32725f39d397SChris Mason btrfs_item_nr_offset(mid), 32735f39d397SChris Mason nritems * sizeof(struct btrfs_item)); 32745f39d397SChris Mason 32755f39d397SChris Mason copy_extent_buffer(right, l, 32763d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + BTRFS_LEAF_DATA_SIZE(fs_info) - 32773d9ec8c4SNikolay Borisov data_copy_size, BTRFS_LEAF_DATA_OFFSET + 32788f881e8cSDavid Sterba leaf_data_end(l), data_copy_size); 327974123bd7SChris Mason 3280dc2e724eSJosef Bacik rt_data_off = BTRFS_LEAF_DATA_SIZE(fs_info) - btrfs_item_data_end(l, mid); 32815f39d397SChris Mason 3282c82f823cSDavid Sterba btrfs_init_map_token(&token, right); 32835f39d397SChris Mason for (i = 0; i < nritems; i++) { 3284db94535dSChris Mason u32 ioff; 3285db94535dSChris Mason 32863212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 32873212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff + rt_data_off); 32880783fcfcSChris Mason } 328974123bd7SChris Mason 32905f39d397SChris Mason btrfs_set_header_nritems(l, mid); 32915f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 32926ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, right->start, path->slots[1] + 1, 1); 32935f39d397SChris Mason 32945f39d397SChris Mason btrfs_mark_buffer_dirty(right); 32955f39d397SChris Mason btrfs_mark_buffer_dirty(l); 3296eb60ceacSChris Mason BUG_ON(path->slots[0] != slot); 32975f39d397SChris Mason 3298be0e5c09SChris Mason if (mid <= slot) { 3299925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 33005f39d397SChris Mason free_extent_buffer(path->nodes[0]); 33015f39d397SChris Mason path->nodes[0] = right; 3302be0e5c09SChris Mason path->slots[0] -= mid; 3303be0e5c09SChris Mason path->slots[1] += 1; 3304925baeddSChris Mason } else { 3305925baeddSChris Mason btrfs_tree_unlock(right); 33065f39d397SChris Mason free_extent_buffer(right); 3307925baeddSChris Mason } 33085f39d397SChris Mason 3309eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 331044871b1bSChris Mason } 331144871b1bSChris Mason 331244871b1bSChris Mason /* 331399d8f83cSChris Mason * double splits happen when we need to insert a big item in the middle 331499d8f83cSChris Mason * of a leaf. A double split can leave us with 3 mostly empty leaves: 331599d8f83cSChris Mason * leaf: [ slots 0 - N] [ our target ] [ N + 1 - total in leaf ] 331699d8f83cSChris Mason * A B C 331799d8f83cSChris Mason * 331899d8f83cSChris Mason * We avoid this by trying to push the items on either side of our target 331999d8f83cSChris Mason * into the adjacent leaves. If all goes well we can avoid the double split 332099d8f83cSChris Mason * completely. 332199d8f83cSChris Mason */ 332299d8f83cSChris Mason static noinline int push_for_double_split(struct btrfs_trans_handle *trans, 332399d8f83cSChris Mason struct btrfs_root *root, 332499d8f83cSChris Mason struct btrfs_path *path, 332599d8f83cSChris Mason int data_size) 332699d8f83cSChris Mason { 332799d8f83cSChris Mason int ret; 332899d8f83cSChris Mason int progress = 0; 332999d8f83cSChris Mason int slot; 333099d8f83cSChris Mason u32 nritems; 33315a4267caSFilipe David Borba Manana int space_needed = data_size; 333299d8f83cSChris Mason 333399d8f83cSChris Mason slot = path->slots[0]; 33345a4267caSFilipe David Borba Manana if (slot < btrfs_header_nritems(path->nodes[0])) 3335e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(path->nodes[0]); 333699d8f83cSChris Mason 333799d8f83cSChris Mason /* 333899d8f83cSChris Mason * try to push all the items after our slot into the 333999d8f83cSChris Mason * right leaf 334099d8f83cSChris Mason */ 33415a4267caSFilipe David Borba Manana ret = push_leaf_right(trans, root, path, 1, space_needed, 0, slot); 334299d8f83cSChris Mason if (ret < 0) 334399d8f83cSChris Mason return ret; 334499d8f83cSChris Mason 334599d8f83cSChris Mason if (ret == 0) 334699d8f83cSChris Mason progress++; 334799d8f83cSChris Mason 334899d8f83cSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 334999d8f83cSChris Mason /* 335099d8f83cSChris Mason * our goal is to get our slot at the start or end of a leaf. If 335199d8f83cSChris Mason * we've done so we're done 335299d8f83cSChris Mason */ 335399d8f83cSChris Mason if (path->slots[0] == 0 || path->slots[0] == nritems) 335499d8f83cSChris Mason return 0; 335599d8f83cSChris Mason 3356e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= data_size) 335799d8f83cSChris Mason return 0; 335899d8f83cSChris Mason 335999d8f83cSChris Mason /* try to push all the items before our slot into the next leaf */ 336099d8f83cSChris Mason slot = path->slots[0]; 3361263d3995SFilipe Manana space_needed = data_size; 3362263d3995SFilipe Manana if (slot > 0) 3363e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(path->nodes[0]); 33645a4267caSFilipe David Borba Manana ret = push_leaf_left(trans, root, path, 1, space_needed, 0, slot); 336599d8f83cSChris Mason if (ret < 0) 336699d8f83cSChris Mason return ret; 336799d8f83cSChris Mason 336899d8f83cSChris Mason if (ret == 0) 336999d8f83cSChris Mason progress++; 337099d8f83cSChris Mason 337199d8f83cSChris Mason if (progress) 337299d8f83cSChris Mason return 0; 337399d8f83cSChris Mason return 1; 337499d8f83cSChris Mason } 337599d8f83cSChris Mason 337699d8f83cSChris Mason /* 337744871b1bSChris Mason * split the path's leaf in two, making sure there is at least data_size 337844871b1bSChris Mason * available for the resulting leaf level of the path. 337944871b1bSChris Mason * 338044871b1bSChris Mason * returns 0 if all went well and < 0 on failure. 338144871b1bSChris Mason */ 338244871b1bSChris Mason static noinline int split_leaf(struct btrfs_trans_handle *trans, 338344871b1bSChris Mason struct btrfs_root *root, 3384310712b2SOmar Sandoval const struct btrfs_key *ins_key, 338544871b1bSChris Mason struct btrfs_path *path, int data_size, 338644871b1bSChris Mason int extend) 338744871b1bSChris Mason { 33885d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 338944871b1bSChris Mason struct extent_buffer *l; 339044871b1bSChris Mason u32 nritems; 339144871b1bSChris Mason int mid; 339244871b1bSChris Mason int slot; 339344871b1bSChris Mason struct extent_buffer *right; 3394b7a0365eSDaniel Dressler struct btrfs_fs_info *fs_info = root->fs_info; 339544871b1bSChris Mason int ret = 0; 339644871b1bSChris Mason int wret; 33975d4f98a2SYan Zheng int split; 339844871b1bSChris Mason int num_doubles = 0; 339999d8f83cSChris Mason int tried_avoid_double = 0; 340044871b1bSChris Mason 3401a5719521SYan, Zheng l = path->nodes[0]; 3402a5719521SYan, Zheng slot = path->slots[0]; 34033212fa14SJosef Bacik if (extend && data_size + btrfs_item_size(l, slot) + 34040b246afaSJeff Mahoney sizeof(struct btrfs_item) > BTRFS_LEAF_DATA_SIZE(fs_info)) 3405a5719521SYan, Zheng return -EOVERFLOW; 3406a5719521SYan, Zheng 340744871b1bSChris Mason /* first try to make some room by pushing left and right */ 340833157e05SLiu Bo if (data_size && path->nodes[1]) { 34095a4267caSFilipe David Borba Manana int space_needed = data_size; 34105a4267caSFilipe David Borba Manana 34115a4267caSFilipe David Borba Manana if (slot < btrfs_header_nritems(l)) 3412e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(l); 34135a4267caSFilipe David Borba Manana 34145a4267caSFilipe David Borba Manana wret = push_leaf_right(trans, root, path, space_needed, 34155a4267caSFilipe David Borba Manana space_needed, 0, 0); 341644871b1bSChris Mason if (wret < 0) 341744871b1bSChris Mason return wret; 341844871b1bSChris Mason if (wret) { 3419263d3995SFilipe Manana space_needed = data_size; 3420263d3995SFilipe Manana if (slot > 0) 3421e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(l); 34225a4267caSFilipe David Borba Manana wret = push_leaf_left(trans, root, path, space_needed, 34235a4267caSFilipe David Borba Manana space_needed, 0, (u32)-1); 342444871b1bSChris Mason if (wret < 0) 342544871b1bSChris Mason return wret; 342644871b1bSChris Mason } 342744871b1bSChris Mason l = path->nodes[0]; 342844871b1bSChris Mason 342944871b1bSChris Mason /* did the pushes work? */ 3430e902baacSDavid Sterba if (btrfs_leaf_free_space(l) >= data_size) 343144871b1bSChris Mason return 0; 343244871b1bSChris Mason } 343344871b1bSChris Mason 343444871b1bSChris Mason if (!path->nodes[1]) { 3435fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, 1); 343644871b1bSChris Mason if (ret) 343744871b1bSChris Mason return ret; 343844871b1bSChris Mason } 343944871b1bSChris Mason again: 34405d4f98a2SYan Zheng split = 1; 344144871b1bSChris Mason l = path->nodes[0]; 344244871b1bSChris Mason slot = path->slots[0]; 344344871b1bSChris Mason nritems = btrfs_header_nritems(l); 344444871b1bSChris Mason mid = (nritems + 1) / 2; 344544871b1bSChris Mason 34465d4f98a2SYan Zheng if (mid <= slot) { 34475d4f98a2SYan Zheng if (nritems == 1 || 34485d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + data_size > 34490b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info)) { 34505d4f98a2SYan Zheng if (slot >= nritems) { 34515d4f98a2SYan Zheng split = 0; 34525d4f98a2SYan Zheng } else { 34535d4f98a2SYan Zheng mid = slot; 34545d4f98a2SYan Zheng if (mid != nritems && 34555d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 34560b246afaSJeff Mahoney data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) { 345799d8f83cSChris Mason if (data_size && !tried_avoid_double) 345899d8f83cSChris Mason goto push_for_double; 34595d4f98a2SYan Zheng split = 2; 34605d4f98a2SYan Zheng } 34615d4f98a2SYan Zheng } 34625d4f98a2SYan Zheng } 34635d4f98a2SYan Zheng } else { 34645d4f98a2SYan Zheng if (leaf_space_used(l, 0, mid) + data_size > 34650b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info)) { 34665d4f98a2SYan Zheng if (!extend && data_size && slot == 0) { 34675d4f98a2SYan Zheng split = 0; 34685d4f98a2SYan Zheng } else if ((extend || !data_size) && slot == 0) { 34695d4f98a2SYan Zheng mid = 1; 34705d4f98a2SYan Zheng } else { 34715d4f98a2SYan Zheng mid = slot; 34725d4f98a2SYan Zheng if (mid != nritems && 34735d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 34740b246afaSJeff Mahoney data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) { 347599d8f83cSChris Mason if (data_size && !tried_avoid_double) 347699d8f83cSChris Mason goto push_for_double; 34775d4f98a2SYan Zheng split = 2; 34785d4f98a2SYan Zheng } 34795d4f98a2SYan Zheng } 34805d4f98a2SYan Zheng } 34815d4f98a2SYan Zheng } 34825d4f98a2SYan Zheng 34835d4f98a2SYan Zheng if (split == 0) 34845d4f98a2SYan Zheng btrfs_cpu_key_to_disk(&disk_key, ins_key); 34855d4f98a2SYan Zheng else 34865d4f98a2SYan Zheng btrfs_item_key(l, &disk_key, mid); 34875d4f98a2SYan Zheng 3488ca9d473aSJosef Bacik /* 3489ca9d473aSJosef Bacik * We have to about BTRFS_NESTING_NEW_ROOT here if we've done a double 3490ca9d473aSJosef Bacik * split, because we're only allowed to have MAX_LOCKDEP_SUBCLASSES 3491ca9d473aSJosef Bacik * subclasses, which is 8 at the time of this patch, and we've maxed it 3492ca9d473aSJosef Bacik * out. In the future we could add a 3493ca9d473aSJosef Bacik * BTRFS_NESTING_SPLIT_THE_SPLITTENING if we need to, but for now just 3494ca9d473aSJosef Bacik * use BTRFS_NESTING_NEW_ROOT. 3495ca9d473aSJosef Bacik */ 349679bd3712SFilipe Manana right = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 349779bd3712SFilipe Manana &disk_key, 0, l->start, 0, 349879bd3712SFilipe Manana num_doubles ? BTRFS_NESTING_NEW_ROOT : 3499ca9d473aSJosef Bacik BTRFS_NESTING_SPLIT); 3500f0486c68SYan, Zheng if (IS_ERR(right)) 350144871b1bSChris Mason return PTR_ERR(right); 3502f0486c68SYan, Zheng 35030b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 350444871b1bSChris Mason 35055d4f98a2SYan Zheng if (split == 0) { 350644871b1bSChris Mason if (mid <= slot) { 350744871b1bSChris Mason btrfs_set_header_nritems(right, 0); 35086ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, 35092ff7e61eSJeff Mahoney right->start, path->slots[1] + 1, 1); 351044871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 351144871b1bSChris Mason free_extent_buffer(path->nodes[0]); 351244871b1bSChris Mason path->nodes[0] = right; 351344871b1bSChris Mason path->slots[0] = 0; 351444871b1bSChris Mason path->slots[1] += 1; 351544871b1bSChris Mason } else { 351644871b1bSChris Mason btrfs_set_header_nritems(right, 0); 35176ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, 35182ff7e61eSJeff Mahoney right->start, path->slots[1], 1); 351944871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 352044871b1bSChris Mason free_extent_buffer(path->nodes[0]); 352144871b1bSChris Mason path->nodes[0] = right; 352244871b1bSChris Mason path->slots[0] = 0; 3523143bede5SJeff Mahoney if (path->slots[1] == 0) 3524b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 35255d4f98a2SYan Zheng } 3526196e0249SLiu Bo /* 3527196e0249SLiu Bo * We create a new leaf 'right' for the required ins_len and 3528196e0249SLiu Bo * we'll do btrfs_mark_buffer_dirty() on this leaf after copying 3529196e0249SLiu Bo * the content of ins_len to 'right'. 3530196e0249SLiu Bo */ 353144871b1bSChris Mason return ret; 353244871b1bSChris Mason } 353344871b1bSChris Mason 353494f94ad9SDavid Sterba copy_for_split(trans, path, l, right, slot, mid, nritems); 353544871b1bSChris Mason 35365d4f98a2SYan Zheng if (split == 2) { 3537cc0c5538SChris Mason BUG_ON(num_doubles != 0); 3538cc0c5538SChris Mason num_doubles++; 3539cc0c5538SChris Mason goto again; 35403326d1b0SChris Mason } 354144871b1bSChris Mason 3542143bede5SJeff Mahoney return 0; 354399d8f83cSChris Mason 354499d8f83cSChris Mason push_for_double: 354599d8f83cSChris Mason push_for_double_split(trans, root, path, data_size); 354699d8f83cSChris Mason tried_avoid_double = 1; 3547e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= data_size) 354899d8f83cSChris Mason return 0; 354999d8f83cSChris Mason goto again; 3550be0e5c09SChris Mason } 3551be0e5c09SChris Mason 3552ad48fd75SYan, Zheng static noinline int setup_leaf_for_split(struct btrfs_trans_handle *trans, 3553ad48fd75SYan, Zheng struct btrfs_root *root, 3554ad48fd75SYan, Zheng struct btrfs_path *path, int ins_len) 3555ad48fd75SYan, Zheng { 3556ad48fd75SYan, Zheng struct btrfs_key key; 3557ad48fd75SYan, Zheng struct extent_buffer *leaf; 3558ad48fd75SYan, Zheng struct btrfs_file_extent_item *fi; 3559ad48fd75SYan, Zheng u64 extent_len = 0; 3560ad48fd75SYan, Zheng u32 item_size; 3561ad48fd75SYan, Zheng int ret; 3562ad48fd75SYan, Zheng 3563ad48fd75SYan, Zheng leaf = path->nodes[0]; 3564ad48fd75SYan, Zheng btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 3565ad48fd75SYan, Zheng 3566ad48fd75SYan, Zheng BUG_ON(key.type != BTRFS_EXTENT_DATA_KEY && 3567ad48fd75SYan, Zheng key.type != BTRFS_EXTENT_CSUM_KEY); 3568ad48fd75SYan, Zheng 3569e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) >= ins_len) 3570ad48fd75SYan, Zheng return 0; 3571ad48fd75SYan, Zheng 35723212fa14SJosef Bacik item_size = btrfs_item_size(leaf, path->slots[0]); 3573ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 3574ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 3575ad48fd75SYan, Zheng struct btrfs_file_extent_item); 3576ad48fd75SYan, Zheng extent_len = btrfs_file_extent_num_bytes(leaf, fi); 3577ad48fd75SYan, Zheng } 3578b3b4aa74SDavid Sterba btrfs_release_path(path); 3579ad48fd75SYan, Zheng 3580ad48fd75SYan, Zheng path->keep_locks = 1; 3581ad48fd75SYan, Zheng path->search_for_split = 1; 3582ad48fd75SYan, Zheng ret = btrfs_search_slot(trans, root, &key, path, 0, 1); 3583ad48fd75SYan, Zheng path->search_for_split = 0; 3584a8df6fe6SFilipe Manana if (ret > 0) 3585a8df6fe6SFilipe Manana ret = -EAGAIN; 3586ad48fd75SYan, Zheng if (ret < 0) 3587ad48fd75SYan, Zheng goto err; 3588ad48fd75SYan, Zheng 3589ad48fd75SYan, Zheng ret = -EAGAIN; 3590ad48fd75SYan, Zheng leaf = path->nodes[0]; 3591a8df6fe6SFilipe Manana /* if our item isn't there, return now */ 35923212fa14SJosef Bacik if (item_size != btrfs_item_size(leaf, path->slots[0])) 3593ad48fd75SYan, Zheng goto err; 3594ad48fd75SYan, Zheng 3595109f6aefSChris Mason /* the leaf has changed, it now has room. return now */ 3596e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= ins_len) 3597109f6aefSChris Mason goto err; 3598109f6aefSChris Mason 3599ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 3600ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 3601ad48fd75SYan, Zheng struct btrfs_file_extent_item); 3602ad48fd75SYan, Zheng if (extent_len != btrfs_file_extent_num_bytes(leaf, fi)) 3603ad48fd75SYan, Zheng goto err; 3604ad48fd75SYan, Zheng } 3605ad48fd75SYan, Zheng 3606ad48fd75SYan, Zheng ret = split_leaf(trans, root, &key, path, ins_len, 1); 3607f0486c68SYan, Zheng if (ret) 3608f0486c68SYan, Zheng goto err; 3609ad48fd75SYan, Zheng 3610ad48fd75SYan, Zheng path->keep_locks = 0; 3611ad48fd75SYan, Zheng btrfs_unlock_up_safe(path, 1); 3612ad48fd75SYan, Zheng return 0; 3613ad48fd75SYan, Zheng err: 3614ad48fd75SYan, Zheng path->keep_locks = 0; 3615ad48fd75SYan, Zheng return ret; 3616ad48fd75SYan, Zheng } 3617ad48fd75SYan, Zheng 361825263cd7SDavid Sterba static noinline int split_item(struct btrfs_path *path, 3619310712b2SOmar Sandoval const struct btrfs_key *new_key, 3620459931ecSChris Mason unsigned long split_offset) 3621459931ecSChris Mason { 3622459931ecSChris Mason struct extent_buffer *leaf; 3623c91666b1SJosef Bacik int orig_slot, slot; 3624ad48fd75SYan, Zheng char *buf; 3625459931ecSChris Mason u32 nritems; 3626ad48fd75SYan, Zheng u32 item_size; 3627459931ecSChris Mason u32 orig_offset; 3628459931ecSChris Mason struct btrfs_disk_key disk_key; 3629459931ecSChris Mason 3630459931ecSChris Mason leaf = path->nodes[0]; 3631e902baacSDavid Sterba BUG_ON(btrfs_leaf_free_space(leaf) < sizeof(struct btrfs_item)); 3632b9473439SChris Mason 3633c91666b1SJosef Bacik orig_slot = path->slots[0]; 36343212fa14SJosef Bacik orig_offset = btrfs_item_offset(leaf, path->slots[0]); 36353212fa14SJosef Bacik item_size = btrfs_item_size(leaf, path->slots[0]); 3636459931ecSChris Mason 3637459931ecSChris Mason buf = kmalloc(item_size, GFP_NOFS); 3638ad48fd75SYan, Zheng if (!buf) 3639ad48fd75SYan, Zheng return -ENOMEM; 3640ad48fd75SYan, Zheng 3641459931ecSChris Mason read_extent_buffer(leaf, buf, btrfs_item_ptr_offset(leaf, 3642459931ecSChris Mason path->slots[0]), item_size); 3643ad48fd75SYan, Zheng 3644459931ecSChris Mason slot = path->slots[0] + 1; 3645459931ecSChris Mason nritems = btrfs_header_nritems(leaf); 3646459931ecSChris Mason if (slot != nritems) { 3647459931ecSChris Mason /* shift the items */ 3648459931ecSChris Mason memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + 1), 3649459931ecSChris Mason btrfs_item_nr_offset(slot), 3650459931ecSChris Mason (nritems - slot) * sizeof(struct btrfs_item)); 3651459931ecSChris Mason } 3652459931ecSChris Mason 3653459931ecSChris Mason btrfs_cpu_key_to_disk(&disk_key, new_key); 3654459931ecSChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 3655459931ecSChris Mason 36563212fa14SJosef Bacik btrfs_set_item_offset(leaf, slot, orig_offset); 36573212fa14SJosef Bacik btrfs_set_item_size(leaf, slot, item_size - split_offset); 3658459931ecSChris Mason 36593212fa14SJosef Bacik btrfs_set_item_offset(leaf, orig_slot, 3660459931ecSChris Mason orig_offset + item_size - split_offset); 36613212fa14SJosef Bacik btrfs_set_item_size(leaf, orig_slot, split_offset); 3662459931ecSChris Mason 3663459931ecSChris Mason btrfs_set_header_nritems(leaf, nritems + 1); 3664459931ecSChris Mason 3665459931ecSChris Mason /* write the data for the start of the original item */ 3666459931ecSChris Mason write_extent_buffer(leaf, buf, 3667459931ecSChris Mason btrfs_item_ptr_offset(leaf, path->slots[0]), 3668459931ecSChris Mason split_offset); 3669459931ecSChris Mason 3670459931ecSChris Mason /* write the data for the new item */ 3671459931ecSChris Mason write_extent_buffer(leaf, buf + split_offset, 3672459931ecSChris Mason btrfs_item_ptr_offset(leaf, slot), 3673459931ecSChris Mason item_size - split_offset); 3674459931ecSChris Mason btrfs_mark_buffer_dirty(leaf); 3675459931ecSChris Mason 3676e902baacSDavid Sterba BUG_ON(btrfs_leaf_free_space(leaf) < 0); 3677459931ecSChris Mason kfree(buf); 3678ad48fd75SYan, Zheng return 0; 3679ad48fd75SYan, Zheng } 3680ad48fd75SYan, Zheng 3681ad48fd75SYan, Zheng /* 3682ad48fd75SYan, Zheng * This function splits a single item into two items, 3683ad48fd75SYan, Zheng * giving 'new_key' to the new item and splitting the 3684ad48fd75SYan, Zheng * old one at split_offset (from the start of the item). 3685ad48fd75SYan, Zheng * 3686ad48fd75SYan, Zheng * The path may be released by this operation. After 3687ad48fd75SYan, Zheng * the split, the path is pointing to the old item. The 3688ad48fd75SYan, Zheng * new item is going to be in the same node as the old one. 3689ad48fd75SYan, Zheng * 3690ad48fd75SYan, Zheng * Note, the item being split must be smaller enough to live alone on 3691ad48fd75SYan, Zheng * a tree block with room for one extra struct btrfs_item 3692ad48fd75SYan, Zheng * 3693ad48fd75SYan, Zheng * This allows us to split the item in place, keeping a lock on the 3694ad48fd75SYan, Zheng * leaf the entire time. 3695ad48fd75SYan, Zheng */ 3696ad48fd75SYan, Zheng int btrfs_split_item(struct btrfs_trans_handle *trans, 3697ad48fd75SYan, Zheng struct btrfs_root *root, 3698ad48fd75SYan, Zheng struct btrfs_path *path, 3699310712b2SOmar Sandoval const struct btrfs_key *new_key, 3700ad48fd75SYan, Zheng unsigned long split_offset) 3701ad48fd75SYan, Zheng { 3702ad48fd75SYan, Zheng int ret; 3703ad48fd75SYan, Zheng ret = setup_leaf_for_split(trans, root, path, 3704ad48fd75SYan, Zheng sizeof(struct btrfs_item)); 3705ad48fd75SYan, Zheng if (ret) 3706459931ecSChris Mason return ret; 3707ad48fd75SYan, Zheng 370825263cd7SDavid Sterba ret = split_item(path, new_key, split_offset); 3709ad48fd75SYan, Zheng return ret; 3710ad48fd75SYan, Zheng } 3711ad48fd75SYan, Zheng 3712ad48fd75SYan, Zheng /* 3713d352ac68SChris Mason * make the item pointed to by the path smaller. new_size indicates 3714d352ac68SChris Mason * how small to make it, and from_end tells us if we just chop bytes 3715d352ac68SChris Mason * off the end of the item or if we shift the item to chop bytes off 3716d352ac68SChris Mason * the front. 3717d352ac68SChris Mason */ 371878ac4f9eSDavid Sterba void btrfs_truncate_item(struct btrfs_path *path, u32 new_size, int from_end) 3719b18c6685SChris Mason { 3720b18c6685SChris Mason int slot; 37215f39d397SChris Mason struct extent_buffer *leaf; 3722b18c6685SChris Mason u32 nritems; 3723b18c6685SChris Mason unsigned int data_end; 3724b18c6685SChris Mason unsigned int old_data_start; 3725b18c6685SChris Mason unsigned int old_size; 3726b18c6685SChris Mason unsigned int size_diff; 3727b18c6685SChris Mason int i; 3728cfed81a0SChris Mason struct btrfs_map_token token; 3729cfed81a0SChris Mason 37305f39d397SChris Mason leaf = path->nodes[0]; 3731179e29e4SChris Mason slot = path->slots[0]; 3732179e29e4SChris Mason 37333212fa14SJosef Bacik old_size = btrfs_item_size(leaf, slot); 3734179e29e4SChris Mason if (old_size == new_size) 3735143bede5SJeff Mahoney return; 3736b18c6685SChris Mason 37375f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 37388f881e8cSDavid Sterba data_end = leaf_data_end(leaf); 3739b18c6685SChris Mason 37403212fa14SJosef Bacik old_data_start = btrfs_item_offset(leaf, slot); 3741179e29e4SChris Mason 3742b18c6685SChris Mason size_diff = old_size - new_size; 3743b18c6685SChris Mason 3744b18c6685SChris Mason BUG_ON(slot < 0); 3745b18c6685SChris Mason BUG_ON(slot >= nritems); 3746b18c6685SChris Mason 3747b18c6685SChris Mason /* 3748b18c6685SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 3749b18c6685SChris Mason */ 3750b18c6685SChris Mason /* first correct the data pointers */ 3751c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 3752b18c6685SChris Mason for (i = slot; i < nritems; i++) { 37535f39d397SChris Mason u32 ioff; 3754db94535dSChris Mason 37553212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 37563212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff + size_diff); 3757b18c6685SChris Mason } 3758db94535dSChris Mason 3759b18c6685SChris Mason /* shift the data */ 3760179e29e4SChris Mason if (from_end) { 37613d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 37623d9ec8c4SNikolay Borisov data_end + size_diff, BTRFS_LEAF_DATA_OFFSET + 3763b18c6685SChris Mason data_end, old_data_start + new_size - data_end); 3764179e29e4SChris Mason } else { 3765179e29e4SChris Mason struct btrfs_disk_key disk_key; 3766179e29e4SChris Mason u64 offset; 3767179e29e4SChris Mason 3768179e29e4SChris Mason btrfs_item_key(leaf, &disk_key, slot); 3769179e29e4SChris Mason 3770179e29e4SChris Mason if (btrfs_disk_key_type(&disk_key) == BTRFS_EXTENT_DATA_KEY) { 3771179e29e4SChris Mason unsigned long ptr; 3772179e29e4SChris Mason struct btrfs_file_extent_item *fi; 3773179e29e4SChris Mason 3774179e29e4SChris Mason fi = btrfs_item_ptr(leaf, slot, 3775179e29e4SChris Mason struct btrfs_file_extent_item); 3776179e29e4SChris Mason fi = (struct btrfs_file_extent_item *)( 3777179e29e4SChris Mason (unsigned long)fi - size_diff); 3778179e29e4SChris Mason 3779179e29e4SChris Mason if (btrfs_file_extent_type(leaf, fi) == 3780179e29e4SChris Mason BTRFS_FILE_EXTENT_INLINE) { 3781179e29e4SChris Mason ptr = btrfs_item_ptr_offset(leaf, slot); 3782179e29e4SChris Mason memmove_extent_buffer(leaf, ptr, 3783179e29e4SChris Mason (unsigned long)fi, 37847ec20afbSDavid Sterba BTRFS_FILE_EXTENT_INLINE_DATA_START); 3785179e29e4SChris Mason } 3786179e29e4SChris Mason } 3787179e29e4SChris Mason 37883d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 37893d9ec8c4SNikolay Borisov data_end + size_diff, BTRFS_LEAF_DATA_OFFSET + 3790179e29e4SChris Mason data_end, old_data_start - data_end); 3791179e29e4SChris Mason 3792179e29e4SChris Mason offset = btrfs_disk_key_offset(&disk_key); 3793179e29e4SChris Mason btrfs_set_disk_key_offset(&disk_key, offset + size_diff); 3794179e29e4SChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 3795179e29e4SChris Mason if (slot == 0) 3796b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 3797179e29e4SChris Mason } 37985f39d397SChris Mason 37993212fa14SJosef Bacik btrfs_set_item_size(leaf, slot, new_size); 38005f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 3801b18c6685SChris Mason 3802e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) { 3803a4f78750SDavid Sterba btrfs_print_leaf(leaf); 3804b18c6685SChris Mason BUG(); 38055f39d397SChris Mason } 3806b18c6685SChris Mason } 3807b18c6685SChris Mason 3808d352ac68SChris Mason /* 38098f69dbd2SStefan Behrens * make the item pointed to by the path bigger, data_size is the added size. 3810d352ac68SChris Mason */ 3811c71dd880SDavid Sterba void btrfs_extend_item(struct btrfs_path *path, u32 data_size) 38126567e837SChris Mason { 38136567e837SChris Mason int slot; 38145f39d397SChris Mason struct extent_buffer *leaf; 38156567e837SChris Mason u32 nritems; 38166567e837SChris Mason unsigned int data_end; 38176567e837SChris Mason unsigned int old_data; 38186567e837SChris Mason unsigned int old_size; 38196567e837SChris Mason int i; 3820cfed81a0SChris Mason struct btrfs_map_token token; 3821cfed81a0SChris Mason 38225f39d397SChris Mason leaf = path->nodes[0]; 38236567e837SChris Mason 38245f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 38258f881e8cSDavid Sterba data_end = leaf_data_end(leaf); 38266567e837SChris Mason 3827e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < data_size) { 3828a4f78750SDavid Sterba btrfs_print_leaf(leaf); 38296567e837SChris Mason BUG(); 38305f39d397SChris Mason } 38316567e837SChris Mason slot = path->slots[0]; 3832dc2e724eSJosef Bacik old_data = btrfs_item_data_end(leaf, slot); 38336567e837SChris Mason 38346567e837SChris Mason BUG_ON(slot < 0); 38353326d1b0SChris Mason if (slot >= nritems) { 3836a4f78750SDavid Sterba btrfs_print_leaf(leaf); 3837c71dd880SDavid Sterba btrfs_crit(leaf->fs_info, "slot %d too large, nritems %d", 3838d397712bSChris Mason slot, nritems); 3839290342f6SArnd Bergmann BUG(); 38403326d1b0SChris Mason } 38416567e837SChris Mason 38426567e837SChris Mason /* 38436567e837SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 38446567e837SChris Mason */ 38456567e837SChris Mason /* first correct the data pointers */ 3846c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 38476567e837SChris Mason for (i = slot; i < nritems; i++) { 38485f39d397SChris Mason u32 ioff; 3849db94535dSChris Mason 38503212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 38513212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff - data_size); 38526567e837SChris Mason } 38535f39d397SChris Mason 38546567e837SChris Mason /* shift the data */ 38553d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 38563d9ec8c4SNikolay Borisov data_end - data_size, BTRFS_LEAF_DATA_OFFSET + 38576567e837SChris Mason data_end, old_data - data_end); 38585f39d397SChris Mason 38596567e837SChris Mason data_end = old_data; 38603212fa14SJosef Bacik old_size = btrfs_item_size(leaf, slot); 38613212fa14SJosef Bacik btrfs_set_item_size(leaf, slot, old_size + data_size); 38625f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 38636567e837SChris Mason 3864e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) { 3865a4f78750SDavid Sterba btrfs_print_leaf(leaf); 38666567e837SChris Mason BUG(); 38675f39d397SChris Mason } 38686567e837SChris Mason } 38696567e837SChris Mason 3870da9ffb24SNikolay Borisov /** 3871da9ffb24SNikolay Borisov * setup_items_for_insert - Helper called before inserting one or more items 3872da9ffb24SNikolay Borisov * to a leaf. Main purpose is to save stack depth by doing the bulk of the work 3873da9ffb24SNikolay Borisov * in a function that doesn't call btrfs_search_slot 3874da9ffb24SNikolay Borisov * 3875da9ffb24SNikolay Borisov * @root: root we are inserting items to 3876da9ffb24SNikolay Borisov * @path: points to the leaf/slot where we are going to insert new items 3877b7ef5f3aSFilipe Manana * @batch: information about the batch of items to insert 387874123bd7SChris Mason */ 3879f0641656SFilipe Manana static void setup_items_for_insert(struct btrfs_root *root, struct btrfs_path *path, 3880b7ef5f3aSFilipe Manana const struct btrfs_item_batch *batch) 3881be0e5c09SChris Mason { 38820b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 38839c58309dSChris Mason int i; 38847518a238SChris Mason u32 nritems; 3885be0e5c09SChris Mason unsigned int data_end; 3886e2fa7227SChris Mason struct btrfs_disk_key disk_key; 388744871b1bSChris Mason struct extent_buffer *leaf; 388844871b1bSChris Mason int slot; 3889cfed81a0SChris Mason struct btrfs_map_token token; 3890fc0d82e1SNikolay Borisov u32 total_size; 3891fc0d82e1SNikolay Borisov 3892b7ef5f3aSFilipe Manana /* 3893b7ef5f3aSFilipe Manana * Before anything else, update keys in the parent and other ancestors 3894b7ef5f3aSFilipe Manana * if needed, then release the write locks on them, so that other tasks 3895b7ef5f3aSFilipe Manana * can use them while we modify the leaf. 3896b7ef5f3aSFilipe Manana */ 389724cdc847SFilipe Manana if (path->slots[0] == 0) { 3898b7ef5f3aSFilipe Manana btrfs_cpu_key_to_disk(&disk_key, &batch->keys[0]); 3899b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 390024cdc847SFilipe Manana } 390124cdc847SFilipe Manana btrfs_unlock_up_safe(path, 1); 390224cdc847SFilipe Manana 39035f39d397SChris Mason leaf = path->nodes[0]; 390444871b1bSChris Mason slot = path->slots[0]; 390574123bd7SChris Mason 39065f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 39078f881e8cSDavid Sterba data_end = leaf_data_end(leaf); 3908b7ef5f3aSFilipe Manana total_size = batch->total_data_size + (batch->nr * sizeof(struct btrfs_item)); 3909eb60ceacSChris Mason 3910e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < total_size) { 3911a4f78750SDavid Sterba btrfs_print_leaf(leaf); 39120b246afaSJeff Mahoney btrfs_crit(fs_info, "not enough freespace need %u have %d", 3913e902baacSDavid Sterba total_size, btrfs_leaf_free_space(leaf)); 3914be0e5c09SChris Mason BUG(); 3915d4dbff95SChris Mason } 39165f39d397SChris Mason 3917c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 3918be0e5c09SChris Mason if (slot != nritems) { 3919dc2e724eSJosef Bacik unsigned int old_data = btrfs_item_data_end(leaf, slot); 3920be0e5c09SChris Mason 39215f39d397SChris Mason if (old_data < data_end) { 3922a4f78750SDavid Sterba btrfs_print_leaf(leaf); 39237269ddd2SNikolay Borisov btrfs_crit(fs_info, 39247269ddd2SNikolay Borisov "item at slot %d with data offset %u beyond data end of leaf %u", 39255f39d397SChris Mason slot, old_data, data_end); 3926290342f6SArnd Bergmann BUG(); 39275f39d397SChris Mason } 3928be0e5c09SChris Mason /* 3929be0e5c09SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 3930be0e5c09SChris Mason */ 3931be0e5c09SChris Mason /* first correct the data pointers */ 39320783fcfcSChris Mason for (i = slot; i < nritems; i++) { 39335f39d397SChris Mason u32 ioff; 3934db94535dSChris Mason 39353212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 39363212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, 3937b7ef5f3aSFilipe Manana ioff - batch->total_data_size); 39380783fcfcSChris Mason } 3939be0e5c09SChris Mason /* shift the items */ 3940b7ef5f3aSFilipe Manana memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + batch->nr), 39415f39d397SChris Mason btrfs_item_nr_offset(slot), 39420783fcfcSChris Mason (nritems - slot) * sizeof(struct btrfs_item)); 3943be0e5c09SChris Mason 3944be0e5c09SChris Mason /* shift the data */ 39453d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 3946b7ef5f3aSFilipe Manana data_end - batch->total_data_size, 3947b7ef5f3aSFilipe Manana BTRFS_LEAF_DATA_OFFSET + data_end, 3948b7ef5f3aSFilipe Manana old_data - data_end); 3949be0e5c09SChris Mason data_end = old_data; 3950be0e5c09SChris Mason } 39515f39d397SChris Mason 395262e2749eSChris Mason /* setup the item for the new data */ 3953b7ef5f3aSFilipe Manana for (i = 0; i < batch->nr; i++) { 3954b7ef5f3aSFilipe Manana btrfs_cpu_key_to_disk(&disk_key, &batch->keys[i]); 39559c58309dSChris Mason btrfs_set_item_key(leaf, &disk_key, slot + i); 3956b7ef5f3aSFilipe Manana data_end -= batch->data_sizes[i]; 39573212fa14SJosef Bacik btrfs_set_token_item_offset(&token, slot + i, data_end); 39583212fa14SJosef Bacik btrfs_set_token_item_size(&token, slot + i, batch->data_sizes[i]); 39599c58309dSChris Mason } 396044871b1bSChris Mason 3961b7ef5f3aSFilipe Manana btrfs_set_header_nritems(leaf, nritems + batch->nr); 3962b9473439SChris Mason btrfs_mark_buffer_dirty(leaf); 3963aa5d6bedSChris Mason 3964e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) { 3965a4f78750SDavid Sterba btrfs_print_leaf(leaf); 3966be0e5c09SChris Mason BUG(); 39675f39d397SChris Mason } 396844871b1bSChris Mason } 396944871b1bSChris Mason 397044871b1bSChris Mason /* 3971f0641656SFilipe Manana * Insert a new item into a leaf. 3972f0641656SFilipe Manana * 3973f0641656SFilipe Manana * @root: The root of the btree. 3974f0641656SFilipe Manana * @path: A path pointing to the target leaf and slot. 3975f0641656SFilipe Manana * @key: The key of the new item. 3976f0641656SFilipe Manana * @data_size: The size of the data associated with the new key. 3977f0641656SFilipe Manana */ 3978f0641656SFilipe Manana void btrfs_setup_item_for_insert(struct btrfs_root *root, 3979f0641656SFilipe Manana struct btrfs_path *path, 3980f0641656SFilipe Manana const struct btrfs_key *key, 3981f0641656SFilipe Manana u32 data_size) 3982f0641656SFilipe Manana { 3983f0641656SFilipe Manana struct btrfs_item_batch batch; 3984f0641656SFilipe Manana 3985f0641656SFilipe Manana batch.keys = key; 3986f0641656SFilipe Manana batch.data_sizes = &data_size; 3987f0641656SFilipe Manana batch.total_data_size = data_size; 3988f0641656SFilipe Manana batch.nr = 1; 3989f0641656SFilipe Manana 3990f0641656SFilipe Manana setup_items_for_insert(root, path, &batch); 3991f0641656SFilipe Manana } 3992f0641656SFilipe Manana 3993f0641656SFilipe Manana /* 399444871b1bSChris Mason * Given a key and some data, insert items into the tree. 399544871b1bSChris Mason * This does all the path init required, making room in the tree if needed. 399644871b1bSChris Mason */ 399744871b1bSChris Mason int btrfs_insert_empty_items(struct btrfs_trans_handle *trans, 399844871b1bSChris Mason struct btrfs_root *root, 399944871b1bSChris Mason struct btrfs_path *path, 4000b7ef5f3aSFilipe Manana const struct btrfs_item_batch *batch) 400144871b1bSChris Mason { 400244871b1bSChris Mason int ret = 0; 400344871b1bSChris Mason int slot; 4004b7ef5f3aSFilipe Manana u32 total_size; 400544871b1bSChris Mason 4006b7ef5f3aSFilipe Manana total_size = batch->total_data_size + (batch->nr * sizeof(struct btrfs_item)); 4007b7ef5f3aSFilipe Manana ret = btrfs_search_slot(trans, root, &batch->keys[0], path, total_size, 1); 400844871b1bSChris Mason if (ret == 0) 400944871b1bSChris Mason return -EEXIST; 401044871b1bSChris Mason if (ret < 0) 4011143bede5SJeff Mahoney return ret; 401244871b1bSChris Mason 401344871b1bSChris Mason slot = path->slots[0]; 401444871b1bSChris Mason BUG_ON(slot < 0); 401544871b1bSChris Mason 4016b7ef5f3aSFilipe Manana setup_items_for_insert(root, path, batch); 4017143bede5SJeff Mahoney return 0; 401862e2749eSChris Mason } 401962e2749eSChris Mason 402062e2749eSChris Mason /* 402162e2749eSChris Mason * Given a key and some data, insert an item into the tree. 402262e2749eSChris Mason * This does all the path init required, making room in the tree if needed. 402362e2749eSChris Mason */ 4024310712b2SOmar Sandoval int btrfs_insert_item(struct btrfs_trans_handle *trans, struct btrfs_root *root, 4025310712b2SOmar Sandoval const struct btrfs_key *cpu_key, void *data, 4026310712b2SOmar Sandoval u32 data_size) 402762e2749eSChris Mason { 402862e2749eSChris Mason int ret = 0; 40292c90e5d6SChris Mason struct btrfs_path *path; 40305f39d397SChris Mason struct extent_buffer *leaf; 40315f39d397SChris Mason unsigned long ptr; 403262e2749eSChris Mason 40332c90e5d6SChris Mason path = btrfs_alloc_path(); 4034db5b493aSTsutomu Itoh if (!path) 4035db5b493aSTsutomu Itoh return -ENOMEM; 40362c90e5d6SChris Mason ret = btrfs_insert_empty_item(trans, root, path, cpu_key, data_size); 403762e2749eSChris Mason if (!ret) { 40385f39d397SChris Mason leaf = path->nodes[0]; 40395f39d397SChris Mason ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); 40405f39d397SChris Mason write_extent_buffer(leaf, data, ptr, data_size); 40415f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 404262e2749eSChris Mason } 40432c90e5d6SChris Mason btrfs_free_path(path); 4044aa5d6bedSChris Mason return ret; 4045be0e5c09SChris Mason } 4046be0e5c09SChris Mason 404774123bd7SChris Mason /* 4048f0641656SFilipe Manana * This function duplicates an item, giving 'new_key' to the new item. 4049f0641656SFilipe Manana * It guarantees both items live in the same tree leaf and the new item is 4050f0641656SFilipe Manana * contiguous with the original item. 4051f0641656SFilipe Manana * 4052f0641656SFilipe Manana * This allows us to split a file extent in place, keeping a lock on the leaf 4053f0641656SFilipe Manana * the entire time. 4054f0641656SFilipe Manana */ 4055f0641656SFilipe Manana int btrfs_duplicate_item(struct btrfs_trans_handle *trans, 4056f0641656SFilipe Manana struct btrfs_root *root, 4057f0641656SFilipe Manana struct btrfs_path *path, 4058f0641656SFilipe Manana const struct btrfs_key *new_key) 4059f0641656SFilipe Manana { 4060f0641656SFilipe Manana struct extent_buffer *leaf; 4061f0641656SFilipe Manana int ret; 4062f0641656SFilipe Manana u32 item_size; 4063f0641656SFilipe Manana 4064f0641656SFilipe Manana leaf = path->nodes[0]; 40653212fa14SJosef Bacik item_size = btrfs_item_size(leaf, path->slots[0]); 4066f0641656SFilipe Manana ret = setup_leaf_for_split(trans, root, path, 4067f0641656SFilipe Manana item_size + sizeof(struct btrfs_item)); 4068f0641656SFilipe Manana if (ret) 4069f0641656SFilipe Manana return ret; 4070f0641656SFilipe Manana 4071f0641656SFilipe Manana path->slots[0]++; 4072f0641656SFilipe Manana btrfs_setup_item_for_insert(root, path, new_key, item_size); 4073f0641656SFilipe Manana leaf = path->nodes[0]; 4074f0641656SFilipe Manana memcpy_extent_buffer(leaf, 4075f0641656SFilipe Manana btrfs_item_ptr_offset(leaf, path->slots[0]), 4076f0641656SFilipe Manana btrfs_item_ptr_offset(leaf, path->slots[0] - 1), 4077f0641656SFilipe Manana item_size); 4078f0641656SFilipe Manana return 0; 4079f0641656SFilipe Manana } 4080f0641656SFilipe Manana 4081f0641656SFilipe Manana /* 40825de08d7dSChris Mason * delete the pointer from a given node. 408374123bd7SChris Mason * 4084d352ac68SChris Mason * the tree should have been previously balanced so the deletion does not 4085d352ac68SChris Mason * empty a node. 408674123bd7SChris Mason */ 4087afe5fea7STsutomu Itoh static void del_ptr(struct btrfs_root *root, struct btrfs_path *path, 4088afe5fea7STsutomu Itoh int level, int slot) 4089be0e5c09SChris Mason { 40905f39d397SChris Mason struct extent_buffer *parent = path->nodes[level]; 40917518a238SChris Mason u32 nritems; 4092f3ea38daSJan Schmidt int ret; 4093be0e5c09SChris Mason 40945f39d397SChris Mason nritems = btrfs_header_nritems(parent); 4095be0e5c09SChris Mason if (slot != nritems - 1) { 4096bf1d3425SDavid Sterba if (level) { 4097f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_move(parent, slot, 4098f3a84ccdSFilipe Manana slot + 1, nritems - slot - 1); 4099bf1d3425SDavid Sterba BUG_ON(ret < 0); 4100bf1d3425SDavid Sterba } 41015f39d397SChris Mason memmove_extent_buffer(parent, 41025f39d397SChris Mason btrfs_node_key_ptr_offset(slot), 41035f39d397SChris Mason btrfs_node_key_ptr_offset(slot + 1), 4104d6025579SChris Mason sizeof(struct btrfs_key_ptr) * 4105d6025579SChris Mason (nritems - slot - 1)); 410657ba86c0SChris Mason } else if (level) { 4107f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, slot, 4108f3a84ccdSFilipe Manana BTRFS_MOD_LOG_KEY_REMOVE, GFP_NOFS); 410957ba86c0SChris Mason BUG_ON(ret < 0); 4110be0e5c09SChris Mason } 4111f3ea38daSJan Schmidt 41127518a238SChris Mason nritems--; 41135f39d397SChris Mason btrfs_set_header_nritems(parent, nritems); 41147518a238SChris Mason if (nritems == 0 && parent == root->node) { 41155f39d397SChris Mason BUG_ON(btrfs_header_level(root->node) != 1); 4116eb60ceacSChris Mason /* just turn the root into a leaf and break */ 41175f39d397SChris Mason btrfs_set_header_level(root->node, 0); 4118bb803951SChris Mason } else if (slot == 0) { 41195f39d397SChris Mason struct btrfs_disk_key disk_key; 41205f39d397SChris Mason 41215f39d397SChris Mason btrfs_node_key(parent, &disk_key, 0); 4122b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, level + 1); 4123be0e5c09SChris Mason } 4124d6025579SChris Mason btrfs_mark_buffer_dirty(parent); 4125be0e5c09SChris Mason } 4126be0e5c09SChris Mason 412774123bd7SChris Mason /* 4128323ac95bSChris Mason * a helper function to delete the leaf pointed to by path->slots[1] and 41295d4f98a2SYan Zheng * path->nodes[1]. 4130323ac95bSChris Mason * 4131323ac95bSChris Mason * This deletes the pointer in path->nodes[1] and frees the leaf 4132323ac95bSChris Mason * block extent. zero is returned if it all worked out, < 0 otherwise. 4133323ac95bSChris Mason * 4134323ac95bSChris Mason * The path must have already been setup for deleting the leaf, including 4135323ac95bSChris Mason * all the proper balancing. path->nodes[1] must be locked. 4136323ac95bSChris Mason */ 4137143bede5SJeff Mahoney static noinline void btrfs_del_leaf(struct btrfs_trans_handle *trans, 4138323ac95bSChris Mason struct btrfs_root *root, 41395d4f98a2SYan Zheng struct btrfs_path *path, 41405d4f98a2SYan Zheng struct extent_buffer *leaf) 4141323ac95bSChris Mason { 41425d4f98a2SYan Zheng WARN_ON(btrfs_header_generation(leaf) != trans->transid); 4143afe5fea7STsutomu Itoh del_ptr(root, path, 1, path->slots[1]); 4144323ac95bSChris Mason 41454d081c41SChris Mason /* 41464d081c41SChris Mason * btrfs_free_extent is expensive, we want to make sure we 41474d081c41SChris Mason * aren't holding any locks when we call it 41484d081c41SChris Mason */ 41494d081c41SChris Mason btrfs_unlock_up_safe(path, 0); 41504d081c41SChris Mason 4151f0486c68SYan, Zheng root_sub_used(root, leaf->len); 4152f0486c68SYan, Zheng 415367439dadSDavid Sterba atomic_inc(&leaf->refs); 41547a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), leaf, 0, 1); 41553083ee2eSJosef Bacik free_extent_buffer_stale(leaf); 4156323ac95bSChris Mason } 4157323ac95bSChris Mason /* 415874123bd7SChris Mason * delete the item at the leaf level in path. If that empties 415974123bd7SChris Mason * the leaf, remove it from the tree 416074123bd7SChris Mason */ 416185e21bacSChris Mason int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root, 416285e21bacSChris Mason struct btrfs_path *path, int slot, int nr) 4163be0e5c09SChris Mason { 41640b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 41655f39d397SChris Mason struct extent_buffer *leaf; 4166ce0eac2aSAlexandru Moise u32 last_off; 4167ce0eac2aSAlexandru Moise u32 dsize = 0; 4168aa5d6bedSChris Mason int ret = 0; 4169aa5d6bedSChris Mason int wret; 417085e21bacSChris Mason int i; 41717518a238SChris Mason u32 nritems; 4172be0e5c09SChris Mason 41735f39d397SChris Mason leaf = path->nodes[0]; 41743212fa14SJosef Bacik last_off = btrfs_item_offset(leaf, slot + nr - 1); 417585e21bacSChris Mason 417685e21bacSChris Mason for (i = 0; i < nr; i++) 41773212fa14SJosef Bacik dsize += btrfs_item_size(leaf, slot + i); 417885e21bacSChris Mason 41795f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 4180be0e5c09SChris Mason 418185e21bacSChris Mason if (slot + nr != nritems) { 41828f881e8cSDavid Sterba int data_end = leaf_data_end(leaf); 4183c82f823cSDavid Sterba struct btrfs_map_token token; 41845f39d397SChris Mason 41853d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 4186d6025579SChris Mason data_end + dsize, 41873d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + data_end, 418885e21bacSChris Mason last_off - data_end); 41895f39d397SChris Mason 4190c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 419185e21bacSChris Mason for (i = slot + nr; i < nritems; i++) { 41925f39d397SChris Mason u32 ioff; 4193db94535dSChris Mason 41943212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 41953212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff + dsize); 41960783fcfcSChris Mason } 4197db94535dSChris Mason 41985f39d397SChris Mason memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot), 419985e21bacSChris Mason btrfs_item_nr_offset(slot + nr), 42000783fcfcSChris Mason sizeof(struct btrfs_item) * 420185e21bacSChris Mason (nritems - slot - nr)); 4202be0e5c09SChris Mason } 420385e21bacSChris Mason btrfs_set_header_nritems(leaf, nritems - nr); 420485e21bacSChris Mason nritems -= nr; 42055f39d397SChris Mason 420674123bd7SChris Mason /* delete the leaf if we've emptied it */ 42077518a238SChris Mason if (nritems == 0) { 42085f39d397SChris Mason if (leaf == root->node) { 42095f39d397SChris Mason btrfs_set_header_level(leaf, 0); 42109a8dd150SChris Mason } else { 42116a884d7dSDavid Sterba btrfs_clean_tree_block(leaf); 4212143bede5SJeff Mahoney btrfs_del_leaf(trans, root, path, leaf); 42139a8dd150SChris Mason } 4214be0e5c09SChris Mason } else { 42157518a238SChris Mason int used = leaf_space_used(leaf, 0, nritems); 4216aa5d6bedSChris Mason if (slot == 0) { 42175f39d397SChris Mason struct btrfs_disk_key disk_key; 42185f39d397SChris Mason 42195f39d397SChris Mason btrfs_item_key(leaf, &disk_key, 0); 4220b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 4221aa5d6bedSChris Mason } 4222aa5d6bedSChris Mason 422374123bd7SChris Mason /* delete the leaf if it is mostly empty */ 42240b246afaSJeff Mahoney if (used < BTRFS_LEAF_DATA_SIZE(fs_info) / 3) { 4225be0e5c09SChris Mason /* push_leaf_left fixes the path. 4226be0e5c09SChris Mason * make sure the path still points to our leaf 4227be0e5c09SChris Mason * for possible call to del_ptr below 4228be0e5c09SChris Mason */ 42294920c9acSChris Mason slot = path->slots[1]; 423067439dadSDavid Sterba atomic_inc(&leaf->refs); 42315f39d397SChris Mason 423299d8f83cSChris Mason wret = push_leaf_left(trans, root, path, 1, 1, 423399d8f83cSChris Mason 1, (u32)-1); 423454aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 4235aa5d6bedSChris Mason ret = wret; 42365f39d397SChris Mason 42375f39d397SChris Mason if (path->nodes[0] == leaf && 42385f39d397SChris Mason btrfs_header_nritems(leaf)) { 423999d8f83cSChris Mason wret = push_leaf_right(trans, root, path, 1, 424099d8f83cSChris Mason 1, 1, 0); 424154aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 4242aa5d6bedSChris Mason ret = wret; 4243aa5d6bedSChris Mason } 42445f39d397SChris Mason 42455f39d397SChris Mason if (btrfs_header_nritems(leaf) == 0) { 4246323ac95bSChris Mason path->slots[1] = slot; 4247143bede5SJeff Mahoney btrfs_del_leaf(trans, root, path, leaf); 42485f39d397SChris Mason free_extent_buffer(leaf); 4249143bede5SJeff Mahoney ret = 0; 42505de08d7dSChris Mason } else { 4251925baeddSChris Mason /* if we're still in the path, make sure 4252925baeddSChris Mason * we're dirty. Otherwise, one of the 4253925baeddSChris Mason * push_leaf functions must have already 4254925baeddSChris Mason * dirtied this buffer 4255925baeddSChris Mason */ 4256925baeddSChris Mason if (path->nodes[0] == leaf) 42575f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 42585f39d397SChris Mason free_extent_buffer(leaf); 4259be0e5c09SChris Mason } 4260d5719762SChris Mason } else { 42615f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 4262be0e5c09SChris Mason } 42639a8dd150SChris Mason } 4264aa5d6bedSChris Mason return ret; 42659a8dd150SChris Mason } 42669a8dd150SChris Mason 426797571fd0SChris Mason /* 4268925baeddSChris Mason * search the tree again to find a leaf with lesser keys 42697bb86316SChris Mason * returns 0 if it found something or 1 if there are no lesser leaves. 42707bb86316SChris Mason * returns < 0 on io errors. 4271d352ac68SChris Mason * 4272d352ac68SChris Mason * This may release the path, and so you may lose any locks held at the 4273d352ac68SChris Mason * time you call it. 42747bb86316SChris Mason */ 427516e7549fSJosef Bacik int btrfs_prev_leaf(struct btrfs_root *root, struct btrfs_path *path) 42767bb86316SChris Mason { 4277925baeddSChris Mason struct btrfs_key key; 4278925baeddSChris Mason struct btrfs_disk_key found_key; 4279925baeddSChris Mason int ret; 42807bb86316SChris Mason 4281925baeddSChris Mason btrfs_item_key_to_cpu(path->nodes[0], &key, 0); 4282925baeddSChris Mason 4283e8b0d724SFilipe David Borba Manana if (key.offset > 0) { 4284925baeddSChris Mason key.offset--; 4285e8b0d724SFilipe David Borba Manana } else if (key.type > 0) { 4286925baeddSChris Mason key.type--; 4287e8b0d724SFilipe David Borba Manana key.offset = (u64)-1; 4288e8b0d724SFilipe David Borba Manana } else if (key.objectid > 0) { 4289925baeddSChris Mason key.objectid--; 4290e8b0d724SFilipe David Borba Manana key.type = (u8)-1; 4291e8b0d724SFilipe David Borba Manana key.offset = (u64)-1; 4292e8b0d724SFilipe David Borba Manana } else { 42937bb86316SChris Mason return 1; 4294e8b0d724SFilipe David Borba Manana } 42957bb86316SChris Mason 4296b3b4aa74SDavid Sterba btrfs_release_path(path); 4297925baeddSChris Mason ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 4298925baeddSChris Mason if (ret < 0) 4299925baeddSChris Mason return ret; 4300925baeddSChris Mason btrfs_item_key(path->nodes[0], &found_key, 0); 4301925baeddSChris Mason ret = comp_keys(&found_key, &key); 4302337c6f68SFilipe Manana /* 4303337c6f68SFilipe Manana * We might have had an item with the previous key in the tree right 4304337c6f68SFilipe Manana * before we released our path. And after we released our path, that 4305337c6f68SFilipe Manana * item might have been pushed to the first slot (0) of the leaf we 4306337c6f68SFilipe Manana * were holding due to a tree balance. Alternatively, an item with the 4307337c6f68SFilipe Manana * previous key can exist as the only element of a leaf (big fat item). 4308337c6f68SFilipe Manana * Therefore account for these 2 cases, so that our callers (like 4309337c6f68SFilipe Manana * btrfs_previous_item) don't miss an existing item with a key matching 4310337c6f68SFilipe Manana * the previous key we computed above. 4311337c6f68SFilipe Manana */ 4312337c6f68SFilipe Manana if (ret <= 0) 43137bb86316SChris Mason return 0; 4314925baeddSChris Mason return 1; 43157bb86316SChris Mason } 43167bb86316SChris Mason 43173f157a2fSChris Mason /* 43183f157a2fSChris Mason * A helper function to walk down the tree starting at min_key, and looking 4319de78b51aSEric Sandeen * for nodes or leaves that are have a minimum transaction id. 4320de78b51aSEric Sandeen * This is used by the btree defrag code, and tree logging 43213f157a2fSChris Mason * 43223f157a2fSChris Mason * This does not cow, but it does stuff the starting key it finds back 43233f157a2fSChris Mason * into min_key, so you can call btrfs_search_slot with cow=1 on the 43243f157a2fSChris Mason * key and get a writable path. 43253f157a2fSChris Mason * 43263f157a2fSChris Mason * This honors path->lowest_level to prevent descent past a given level 43273f157a2fSChris Mason * of the tree. 43283f157a2fSChris Mason * 4329d352ac68SChris Mason * min_trans indicates the oldest transaction that you are interested 4330d352ac68SChris Mason * in walking through. Any nodes or leaves older than min_trans are 4331d352ac68SChris Mason * skipped over (without reading them). 4332d352ac68SChris Mason * 43333f157a2fSChris Mason * returns zero if something useful was found, < 0 on error and 1 if there 43343f157a2fSChris Mason * was nothing in the tree that matched the search criteria. 43353f157a2fSChris Mason */ 43363f157a2fSChris Mason int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key, 4337de78b51aSEric Sandeen struct btrfs_path *path, 43383f157a2fSChris Mason u64 min_trans) 43393f157a2fSChris Mason { 43403f157a2fSChris Mason struct extent_buffer *cur; 43413f157a2fSChris Mason struct btrfs_key found_key; 43423f157a2fSChris Mason int slot; 43439652480bSYan int sret; 43443f157a2fSChris Mason u32 nritems; 43453f157a2fSChris Mason int level; 43463f157a2fSChris Mason int ret = 1; 4347f98de9b9SFilipe Manana int keep_locks = path->keep_locks; 43483f157a2fSChris Mason 4349f98de9b9SFilipe Manana path->keep_locks = 1; 43503f157a2fSChris Mason again: 4351bd681513SChris Mason cur = btrfs_read_lock_root_node(root); 43523f157a2fSChris Mason level = btrfs_header_level(cur); 4353e02119d5SChris Mason WARN_ON(path->nodes[level]); 43543f157a2fSChris Mason path->nodes[level] = cur; 4355bd681513SChris Mason path->locks[level] = BTRFS_READ_LOCK; 43563f157a2fSChris Mason 43573f157a2fSChris Mason if (btrfs_header_generation(cur) < min_trans) { 43583f157a2fSChris Mason ret = 1; 43593f157a2fSChris Mason goto out; 43603f157a2fSChris Mason } 43613f157a2fSChris Mason while (1) { 43623f157a2fSChris Mason nritems = btrfs_header_nritems(cur); 43633f157a2fSChris Mason level = btrfs_header_level(cur); 4364e3b83361SQu Wenruo sret = btrfs_bin_search(cur, min_key, &slot); 4365cbca7d59SFilipe Manana if (sret < 0) { 4366cbca7d59SFilipe Manana ret = sret; 4367cbca7d59SFilipe Manana goto out; 4368cbca7d59SFilipe Manana } 43693f157a2fSChris Mason 4370323ac95bSChris Mason /* at the lowest level, we're done, setup the path and exit */ 4371323ac95bSChris Mason if (level == path->lowest_level) { 4372e02119d5SChris Mason if (slot >= nritems) 4373e02119d5SChris Mason goto find_next_key; 43743f157a2fSChris Mason ret = 0; 43753f157a2fSChris Mason path->slots[level] = slot; 43763f157a2fSChris Mason btrfs_item_key_to_cpu(cur, &found_key, slot); 43773f157a2fSChris Mason goto out; 43783f157a2fSChris Mason } 43799652480bSYan if (sret && slot > 0) 43809652480bSYan slot--; 43813f157a2fSChris Mason /* 4382de78b51aSEric Sandeen * check this node pointer against the min_trans parameters. 4383260db43cSRandy Dunlap * If it is too old, skip to the next one. 43843f157a2fSChris Mason */ 43853f157a2fSChris Mason while (slot < nritems) { 43863f157a2fSChris Mason u64 gen; 4387e02119d5SChris Mason 43883f157a2fSChris Mason gen = btrfs_node_ptr_generation(cur, slot); 43893f157a2fSChris Mason if (gen < min_trans) { 43903f157a2fSChris Mason slot++; 43913f157a2fSChris Mason continue; 43923f157a2fSChris Mason } 43933f157a2fSChris Mason break; 43943f157a2fSChris Mason } 4395e02119d5SChris Mason find_next_key: 43963f157a2fSChris Mason /* 43973f157a2fSChris Mason * we didn't find a candidate key in this node, walk forward 43983f157a2fSChris Mason * and find another one 43993f157a2fSChris Mason */ 44003f157a2fSChris Mason if (slot >= nritems) { 4401e02119d5SChris Mason path->slots[level] = slot; 4402e02119d5SChris Mason sret = btrfs_find_next_key(root, path, min_key, level, 4403de78b51aSEric Sandeen min_trans); 4404e02119d5SChris Mason if (sret == 0) { 4405b3b4aa74SDavid Sterba btrfs_release_path(path); 44063f157a2fSChris Mason goto again; 44073f157a2fSChris Mason } else { 44083f157a2fSChris Mason goto out; 44093f157a2fSChris Mason } 44103f157a2fSChris Mason } 44113f157a2fSChris Mason /* save our key for returning back */ 44123f157a2fSChris Mason btrfs_node_key_to_cpu(cur, &found_key, slot); 44133f157a2fSChris Mason path->slots[level] = slot; 44143f157a2fSChris Mason if (level == path->lowest_level) { 44153f157a2fSChris Mason ret = 0; 44163f157a2fSChris Mason goto out; 44173f157a2fSChris Mason } 44184b231ae4SDavid Sterba cur = btrfs_read_node_slot(cur, slot); 4419fb770ae4SLiu Bo if (IS_ERR(cur)) { 4420fb770ae4SLiu Bo ret = PTR_ERR(cur); 4421fb770ae4SLiu Bo goto out; 4422fb770ae4SLiu Bo } 44233f157a2fSChris Mason 4424bd681513SChris Mason btrfs_tree_read_lock(cur); 4425b4ce94deSChris Mason 4426bd681513SChris Mason path->locks[level - 1] = BTRFS_READ_LOCK; 44273f157a2fSChris Mason path->nodes[level - 1] = cur; 4428f7c79f30SChris Mason unlock_up(path, level, 1, 0, NULL); 44293f157a2fSChris Mason } 44303f157a2fSChris Mason out: 4431f98de9b9SFilipe Manana path->keep_locks = keep_locks; 4432f98de9b9SFilipe Manana if (ret == 0) { 4433f98de9b9SFilipe Manana btrfs_unlock_up_safe(path, path->lowest_level + 1); 4434f98de9b9SFilipe Manana memcpy(min_key, &found_key, sizeof(found_key)); 4435f98de9b9SFilipe Manana } 44363f157a2fSChris Mason return ret; 44373f157a2fSChris Mason } 44383f157a2fSChris Mason 44393f157a2fSChris Mason /* 44403f157a2fSChris Mason * this is similar to btrfs_next_leaf, but does not try to preserve 44413f157a2fSChris Mason * and fixup the path. It looks for and returns the next key in the 4442de78b51aSEric Sandeen * tree based on the current path and the min_trans parameters. 44433f157a2fSChris Mason * 44443f157a2fSChris Mason * 0 is returned if another key is found, < 0 if there are any errors 44453f157a2fSChris Mason * and 1 is returned if there are no higher keys in the tree 44463f157a2fSChris Mason * 44473f157a2fSChris Mason * path->keep_locks should be set to 1 on the search made before 44483f157a2fSChris Mason * calling this function. 44493f157a2fSChris Mason */ 4450e7a84565SChris Mason int btrfs_find_next_key(struct btrfs_root *root, struct btrfs_path *path, 4451de78b51aSEric Sandeen struct btrfs_key *key, int level, u64 min_trans) 4452e7a84565SChris Mason { 4453e7a84565SChris Mason int slot; 4454e7a84565SChris Mason struct extent_buffer *c; 4455e7a84565SChris Mason 44566a9fb468SJosef Bacik WARN_ON(!path->keep_locks && !path->skip_locking); 4457e7a84565SChris Mason while (level < BTRFS_MAX_LEVEL) { 4458e7a84565SChris Mason if (!path->nodes[level]) 4459e7a84565SChris Mason return 1; 4460e7a84565SChris Mason 4461e7a84565SChris Mason slot = path->slots[level] + 1; 4462e7a84565SChris Mason c = path->nodes[level]; 44633f157a2fSChris Mason next: 4464e7a84565SChris Mason if (slot >= btrfs_header_nritems(c)) { 446533c66f43SYan Zheng int ret; 446633c66f43SYan Zheng int orig_lowest; 446733c66f43SYan Zheng struct btrfs_key cur_key; 446833c66f43SYan Zheng if (level + 1 >= BTRFS_MAX_LEVEL || 446933c66f43SYan Zheng !path->nodes[level + 1]) 4470e7a84565SChris Mason return 1; 447133c66f43SYan Zheng 44726a9fb468SJosef Bacik if (path->locks[level + 1] || path->skip_locking) { 447333c66f43SYan Zheng level++; 4474e7a84565SChris Mason continue; 4475e7a84565SChris Mason } 447633c66f43SYan Zheng 447733c66f43SYan Zheng slot = btrfs_header_nritems(c) - 1; 447833c66f43SYan Zheng if (level == 0) 447933c66f43SYan Zheng btrfs_item_key_to_cpu(c, &cur_key, slot); 448033c66f43SYan Zheng else 448133c66f43SYan Zheng btrfs_node_key_to_cpu(c, &cur_key, slot); 448233c66f43SYan Zheng 448333c66f43SYan Zheng orig_lowest = path->lowest_level; 4484b3b4aa74SDavid Sterba btrfs_release_path(path); 448533c66f43SYan Zheng path->lowest_level = level; 448633c66f43SYan Zheng ret = btrfs_search_slot(NULL, root, &cur_key, path, 448733c66f43SYan Zheng 0, 0); 448833c66f43SYan Zheng path->lowest_level = orig_lowest; 448933c66f43SYan Zheng if (ret < 0) 449033c66f43SYan Zheng return ret; 449133c66f43SYan Zheng 449233c66f43SYan Zheng c = path->nodes[level]; 449333c66f43SYan Zheng slot = path->slots[level]; 449433c66f43SYan Zheng if (ret == 0) 449533c66f43SYan Zheng slot++; 449633c66f43SYan Zheng goto next; 449733c66f43SYan Zheng } 449833c66f43SYan Zheng 4499e7a84565SChris Mason if (level == 0) 4500e7a84565SChris Mason btrfs_item_key_to_cpu(c, key, slot); 45013f157a2fSChris Mason else { 45023f157a2fSChris Mason u64 gen = btrfs_node_ptr_generation(c, slot); 45033f157a2fSChris Mason 45043f157a2fSChris Mason if (gen < min_trans) { 45053f157a2fSChris Mason slot++; 45063f157a2fSChris Mason goto next; 45073f157a2fSChris Mason } 4508e7a84565SChris Mason btrfs_node_key_to_cpu(c, key, slot); 45093f157a2fSChris Mason } 4510e7a84565SChris Mason return 0; 4511e7a84565SChris Mason } 4512e7a84565SChris Mason return 1; 4513e7a84565SChris Mason } 4514e7a84565SChris Mason 45153d7806ecSJan Schmidt int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path, 45163d7806ecSJan Schmidt u64 time_seq) 45173d7806ecSJan Schmidt { 4518d97e63b6SChris Mason int slot; 45198e73f275SChris Mason int level; 45205f39d397SChris Mason struct extent_buffer *c; 45218e73f275SChris Mason struct extent_buffer *next; 4522d96b3424SFilipe Manana struct btrfs_fs_info *fs_info = root->fs_info; 4523925baeddSChris Mason struct btrfs_key key; 4524d96b3424SFilipe Manana bool need_commit_sem = false; 4525925baeddSChris Mason u32 nritems; 4526925baeddSChris Mason int ret; 45270e46318dSJosef Bacik int i; 4528925baeddSChris Mason 4529925baeddSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 4530d397712bSChris Mason if (nritems == 0) 4531925baeddSChris Mason return 1; 4532925baeddSChris Mason 45338e73f275SChris Mason btrfs_item_key_to_cpu(path->nodes[0], &key, nritems - 1); 45348e73f275SChris Mason again: 45358e73f275SChris Mason level = 1; 45368e73f275SChris Mason next = NULL; 4537b3b4aa74SDavid Sterba btrfs_release_path(path); 45388e73f275SChris Mason 4539a2135011SChris Mason path->keep_locks = 1; 45408e73f275SChris Mason 4541d96b3424SFilipe Manana if (time_seq) { 45423d7806ecSJan Schmidt ret = btrfs_search_old_slot(root, &key, path, time_seq); 4543d96b3424SFilipe Manana } else { 4544d96b3424SFilipe Manana if (path->need_commit_sem) { 4545d96b3424SFilipe Manana path->need_commit_sem = 0; 4546d96b3424SFilipe Manana need_commit_sem = true; 4547d96b3424SFilipe Manana down_read(&fs_info->commit_root_sem); 4548d96b3424SFilipe Manana } 4549925baeddSChris Mason ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 4550d96b3424SFilipe Manana } 4551925baeddSChris Mason path->keep_locks = 0; 4552925baeddSChris Mason 4553925baeddSChris Mason if (ret < 0) 4554d96b3424SFilipe Manana goto done; 4555925baeddSChris Mason 4556a2135011SChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 4557168fd7d2SChris Mason /* 4558168fd7d2SChris Mason * by releasing the path above we dropped all our locks. A balance 4559168fd7d2SChris Mason * could have added more items next to the key that used to be 4560168fd7d2SChris Mason * at the very end of the block. So, check again here and 4561168fd7d2SChris Mason * advance the path if there are now more items available. 4562168fd7d2SChris Mason */ 4563a2135011SChris Mason if (nritems > 0 && path->slots[0] < nritems - 1) { 4564e457afecSYan Zheng if (ret == 0) 4565168fd7d2SChris Mason path->slots[0]++; 45668e73f275SChris Mason ret = 0; 4567925baeddSChris Mason goto done; 4568925baeddSChris Mason } 45690b43e04fSLiu Bo /* 45700b43e04fSLiu Bo * So the above check misses one case: 45710b43e04fSLiu Bo * - after releasing the path above, someone has removed the item that 45720b43e04fSLiu Bo * used to be at the very end of the block, and balance between leafs 45730b43e04fSLiu Bo * gets another one with bigger key.offset to replace it. 45740b43e04fSLiu Bo * 45750b43e04fSLiu Bo * This one should be returned as well, or we can get leaf corruption 45760b43e04fSLiu Bo * later(esp. in __btrfs_drop_extents()). 45770b43e04fSLiu Bo * 45780b43e04fSLiu Bo * And a bit more explanation about this check, 45790b43e04fSLiu Bo * with ret > 0, the key isn't found, the path points to the slot 45800b43e04fSLiu Bo * where it should be inserted, so the path->slots[0] item must be the 45810b43e04fSLiu Bo * bigger one. 45820b43e04fSLiu Bo */ 45830b43e04fSLiu Bo if (nritems > 0 && ret > 0 && path->slots[0] == nritems - 1) { 45840b43e04fSLiu Bo ret = 0; 45850b43e04fSLiu Bo goto done; 45860b43e04fSLiu Bo } 4587d97e63b6SChris Mason 4588234b63a0SChris Mason while (level < BTRFS_MAX_LEVEL) { 45898e73f275SChris Mason if (!path->nodes[level]) { 45908e73f275SChris Mason ret = 1; 45918e73f275SChris Mason goto done; 45928e73f275SChris Mason } 45935f39d397SChris Mason 4594d97e63b6SChris Mason slot = path->slots[level] + 1; 4595d97e63b6SChris Mason c = path->nodes[level]; 45965f39d397SChris Mason if (slot >= btrfs_header_nritems(c)) { 4597d97e63b6SChris Mason level++; 45988e73f275SChris Mason if (level == BTRFS_MAX_LEVEL) { 45998e73f275SChris Mason ret = 1; 46008e73f275SChris Mason goto done; 46018e73f275SChris Mason } 4602d97e63b6SChris Mason continue; 4603d97e63b6SChris Mason } 46045f39d397SChris Mason 46050e46318dSJosef Bacik 46060e46318dSJosef Bacik /* 46070e46318dSJosef Bacik * Our current level is where we're going to start from, and to 46080e46318dSJosef Bacik * make sure lockdep doesn't complain we need to drop our locks 46090e46318dSJosef Bacik * and nodes from 0 to our current level. 46100e46318dSJosef Bacik */ 46110e46318dSJosef Bacik for (i = 0; i < level; i++) { 46120e46318dSJosef Bacik if (path->locks[level]) { 46130e46318dSJosef Bacik btrfs_tree_read_unlock(path->nodes[i]); 46140e46318dSJosef Bacik path->locks[i] = 0; 46150e46318dSJosef Bacik } 46160e46318dSJosef Bacik free_extent_buffer(path->nodes[i]); 46170e46318dSJosef Bacik path->nodes[i] = NULL; 4618925baeddSChris Mason } 46195f39d397SChris Mason 46208e73f275SChris Mason next = c; 4621d07b8528SLiu Bo ret = read_block_for_search(root, path, &next, level, 4622cda79c54SDavid Sterba slot, &key); 46238e73f275SChris Mason if (ret == -EAGAIN) 46248e73f275SChris Mason goto again; 46255f39d397SChris Mason 462676a05b35SChris Mason if (ret < 0) { 4627b3b4aa74SDavid Sterba btrfs_release_path(path); 462876a05b35SChris Mason goto done; 462976a05b35SChris Mason } 463076a05b35SChris Mason 46315cd57b2cSChris Mason if (!path->skip_locking) { 4632bd681513SChris Mason ret = btrfs_try_tree_read_lock(next); 4633d42244a0SJan Schmidt if (!ret && time_seq) { 4634d42244a0SJan Schmidt /* 4635d42244a0SJan Schmidt * If we don't get the lock, we may be racing 4636d42244a0SJan Schmidt * with push_leaf_left, holding that lock while 4637d42244a0SJan Schmidt * itself waiting for the leaf we've currently 4638d42244a0SJan Schmidt * locked. To solve this situation, we give up 4639d42244a0SJan Schmidt * on our lock and cycle. 4640d42244a0SJan Schmidt */ 4641cf538830SJan Schmidt free_extent_buffer(next); 4642d42244a0SJan Schmidt btrfs_release_path(path); 4643d42244a0SJan Schmidt cond_resched(); 4644d42244a0SJan Schmidt goto again; 4645d42244a0SJan Schmidt } 46460e46318dSJosef Bacik if (!ret) 46470e46318dSJosef Bacik btrfs_tree_read_lock(next); 4648bd681513SChris Mason } 4649d97e63b6SChris Mason break; 4650d97e63b6SChris Mason } 4651d97e63b6SChris Mason path->slots[level] = slot; 4652d97e63b6SChris Mason while (1) { 4653d97e63b6SChris Mason level--; 4654d97e63b6SChris Mason path->nodes[level] = next; 4655d97e63b6SChris Mason path->slots[level] = 0; 4656a74a4b97SChris Mason if (!path->skip_locking) 4657ffeb03cfSJosef Bacik path->locks[level] = BTRFS_READ_LOCK; 4658d97e63b6SChris Mason if (!level) 4659d97e63b6SChris Mason break; 4660b4ce94deSChris Mason 4661d07b8528SLiu Bo ret = read_block_for_search(root, path, &next, level, 4662cda79c54SDavid Sterba 0, &key); 46638e73f275SChris Mason if (ret == -EAGAIN) 46648e73f275SChris Mason goto again; 46658e73f275SChris Mason 466676a05b35SChris Mason if (ret < 0) { 4667b3b4aa74SDavid Sterba btrfs_release_path(path); 466876a05b35SChris Mason goto done; 466976a05b35SChris Mason } 467076a05b35SChris Mason 4671ffeb03cfSJosef Bacik if (!path->skip_locking) 46720e46318dSJosef Bacik btrfs_tree_read_lock(next); 4673d97e63b6SChris Mason } 46748e73f275SChris Mason ret = 0; 4675925baeddSChris Mason done: 4676f7c79f30SChris Mason unlock_up(path, 0, 1, 0, NULL); 4677d96b3424SFilipe Manana if (need_commit_sem) { 4678d96b3424SFilipe Manana int ret2; 4679d96b3424SFilipe Manana 4680d96b3424SFilipe Manana path->need_commit_sem = 1; 4681d96b3424SFilipe Manana ret2 = finish_need_commit_sem_search(path); 4682d96b3424SFilipe Manana up_read(&fs_info->commit_root_sem); 4683d96b3424SFilipe Manana if (ret2) 4684d96b3424SFilipe Manana ret = ret2; 4685d96b3424SFilipe Manana } 46868e73f275SChris Mason 46878e73f275SChris Mason return ret; 4688d97e63b6SChris Mason } 46890b86a832SChris Mason 46903f157a2fSChris Mason /* 46913f157a2fSChris Mason * this uses btrfs_prev_leaf to walk backwards in the tree, and keeps 46923f157a2fSChris Mason * searching until it gets past min_objectid or finds an item of 'type' 46933f157a2fSChris Mason * 46943f157a2fSChris Mason * returns 0 if something is found, 1 if nothing was found and < 0 on error 46953f157a2fSChris Mason */ 46960b86a832SChris Mason int btrfs_previous_item(struct btrfs_root *root, 46970b86a832SChris Mason struct btrfs_path *path, u64 min_objectid, 46980b86a832SChris Mason int type) 46990b86a832SChris Mason { 47000b86a832SChris Mason struct btrfs_key found_key; 47010b86a832SChris Mason struct extent_buffer *leaf; 4702e02119d5SChris Mason u32 nritems; 47030b86a832SChris Mason int ret; 47040b86a832SChris Mason 47050b86a832SChris Mason while (1) { 47060b86a832SChris Mason if (path->slots[0] == 0) { 47070b86a832SChris Mason ret = btrfs_prev_leaf(root, path); 47080b86a832SChris Mason if (ret != 0) 47090b86a832SChris Mason return ret; 47100b86a832SChris Mason } else { 47110b86a832SChris Mason path->slots[0]--; 47120b86a832SChris Mason } 47130b86a832SChris Mason leaf = path->nodes[0]; 4714e02119d5SChris Mason nritems = btrfs_header_nritems(leaf); 4715e02119d5SChris Mason if (nritems == 0) 4716e02119d5SChris Mason return 1; 4717e02119d5SChris Mason if (path->slots[0] == nritems) 4718e02119d5SChris Mason path->slots[0]--; 4719e02119d5SChris Mason 47200b86a832SChris Mason btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 4721e02119d5SChris Mason if (found_key.objectid < min_objectid) 4722e02119d5SChris Mason break; 47230a4eefbbSYan Zheng if (found_key.type == type) 47240a4eefbbSYan Zheng return 0; 4725e02119d5SChris Mason if (found_key.objectid == min_objectid && 4726e02119d5SChris Mason found_key.type < type) 4727e02119d5SChris Mason break; 47280b86a832SChris Mason } 47290b86a832SChris Mason return 1; 47300b86a832SChris Mason } 4731ade2e0b3SWang Shilong 4732ade2e0b3SWang Shilong /* 4733ade2e0b3SWang Shilong * search in extent tree to find a previous Metadata/Data extent item with 4734ade2e0b3SWang Shilong * min objecitd. 4735ade2e0b3SWang Shilong * 4736ade2e0b3SWang Shilong * returns 0 if something is found, 1 if nothing was found and < 0 on error 4737ade2e0b3SWang Shilong */ 4738ade2e0b3SWang Shilong int btrfs_previous_extent_item(struct btrfs_root *root, 4739ade2e0b3SWang Shilong struct btrfs_path *path, u64 min_objectid) 4740ade2e0b3SWang Shilong { 4741ade2e0b3SWang Shilong struct btrfs_key found_key; 4742ade2e0b3SWang Shilong struct extent_buffer *leaf; 4743ade2e0b3SWang Shilong u32 nritems; 4744ade2e0b3SWang Shilong int ret; 4745ade2e0b3SWang Shilong 4746ade2e0b3SWang Shilong while (1) { 4747ade2e0b3SWang Shilong if (path->slots[0] == 0) { 4748ade2e0b3SWang Shilong ret = btrfs_prev_leaf(root, path); 4749ade2e0b3SWang Shilong if (ret != 0) 4750ade2e0b3SWang Shilong return ret; 4751ade2e0b3SWang Shilong } else { 4752ade2e0b3SWang Shilong path->slots[0]--; 4753ade2e0b3SWang Shilong } 4754ade2e0b3SWang Shilong leaf = path->nodes[0]; 4755ade2e0b3SWang Shilong nritems = btrfs_header_nritems(leaf); 4756ade2e0b3SWang Shilong if (nritems == 0) 4757ade2e0b3SWang Shilong return 1; 4758ade2e0b3SWang Shilong if (path->slots[0] == nritems) 4759ade2e0b3SWang Shilong path->slots[0]--; 4760ade2e0b3SWang Shilong 4761ade2e0b3SWang Shilong btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 4762ade2e0b3SWang Shilong if (found_key.objectid < min_objectid) 4763ade2e0b3SWang Shilong break; 4764ade2e0b3SWang Shilong if (found_key.type == BTRFS_EXTENT_ITEM_KEY || 4765ade2e0b3SWang Shilong found_key.type == BTRFS_METADATA_ITEM_KEY) 4766ade2e0b3SWang Shilong return 0; 4767ade2e0b3SWang Shilong if (found_key.objectid == min_objectid && 4768ade2e0b3SWang Shilong found_key.type < BTRFS_EXTENT_ITEM_KEY) 4769ade2e0b3SWang Shilong break; 4770ade2e0b3SWang Shilong } 4771ade2e0b3SWang Shilong return 1; 4772ade2e0b3SWang Shilong } 4773