1c1d7c514SDavid Sterba // SPDX-License-Identifier: GPL-2.0 26cbd5570SChris Mason /* 3d352ac68SChris Mason * Copyright (C) 2007,2008 Oracle. All rights reserved. 46cbd5570SChris Mason */ 56cbd5570SChris Mason 6a6b6e75eSChris Mason #include <linux/sched.h> 75a0e3ad6STejun Heo #include <linux/slab.h> 8bd989ba3SJan Schmidt #include <linux/rbtree.h> 9adf02123SDavid Sterba #include <linux/mm.h> 10e41d12f5SChristoph Hellwig #include <linux/error-injection.h> 119b569ea0SJosef Bacik #include "messages.h" 12eb60ceacSChris Mason #include "ctree.h" 13eb60ceacSChris Mason #include "disk-io.h" 147f5c1516SChris Mason #include "transaction.h" 155f39d397SChris Mason #include "print-tree.h" 16925baeddSChris Mason #include "locking.h" 17de37aa51SNikolay Borisov #include "volumes.h" 18f616f5cdSQu Wenruo #include "qgroup.h" 19f3a84ccdSFilipe Manana #include "tree-mod-log.h" 2088c602abSQu Wenruo #include "tree-checker.h" 21ec8eb376SJosef Bacik #include "fs.h" 22ad1ac501SJosef Bacik #include "accessors.h" 23a0231804SJosef Bacik #include "extent-tree.h" 2467707479SJosef Bacik #include "relocation.h" 256bfd0ffaSJosef Bacik #include "file-item.h" 269a8dd150SChris Mason 27226463d7SJosef Bacik static struct kmem_cache *btrfs_path_cachep; 28226463d7SJosef Bacik 29e089f05cSChris Mason static int split_node(struct btrfs_trans_handle *trans, struct btrfs_root 30e089f05cSChris Mason *root, struct btrfs_path *path, int level); 31310712b2SOmar Sandoval static int split_leaf(struct btrfs_trans_handle *trans, struct btrfs_root *root, 32310712b2SOmar Sandoval const struct btrfs_key *ins_key, struct btrfs_path *path, 33310712b2SOmar Sandoval int data_size, int extend); 345f39d397SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 352ff7e61eSJeff Mahoney struct extent_buffer *dst, 36971a1f66SChris Mason struct extent_buffer *src, int empty); 375f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 385f39d397SChris Mason struct extent_buffer *dst_buf, 395f39d397SChris Mason struct extent_buffer *src_buf); 40afe5fea7STsutomu Itoh static void del_ptr(struct btrfs_root *root, struct btrfs_path *path, 41afe5fea7STsutomu Itoh int level, int slot); 42d97e63b6SChris Mason 43af024ed2SJohannes Thumshirn static const struct btrfs_csums { 44af024ed2SJohannes Thumshirn u16 size; 4559a0fcdbSDavid Sterba const char name[10]; 4659a0fcdbSDavid Sterba const char driver[12]; 47af024ed2SJohannes Thumshirn } btrfs_csums[] = { 48af024ed2SJohannes Thumshirn [BTRFS_CSUM_TYPE_CRC32] = { .size = 4, .name = "crc32c" }, 493951e7f0SJohannes Thumshirn [BTRFS_CSUM_TYPE_XXHASH] = { .size = 8, .name = "xxhash64" }, 503831bf00SJohannes Thumshirn [BTRFS_CSUM_TYPE_SHA256] = { .size = 32, .name = "sha256" }, 51352ae07bSDavid Sterba [BTRFS_CSUM_TYPE_BLAKE2] = { .size = 32, .name = "blake2b", 52352ae07bSDavid Sterba .driver = "blake2b-256" }, 53af024ed2SJohannes Thumshirn }; 54af024ed2SJohannes Thumshirn 553a3178c7SJosef Bacik /* 563a3178c7SJosef Bacik * The leaf data grows from end-to-front in the node. this returns the address 573a3178c7SJosef Bacik * of the start of the last item, which is the stop of the leaf data stack. 583a3178c7SJosef Bacik */ 593a3178c7SJosef Bacik static unsigned int leaf_data_end(const struct extent_buffer *leaf) 603a3178c7SJosef Bacik { 613a3178c7SJosef Bacik u32 nr = btrfs_header_nritems(leaf); 623a3178c7SJosef Bacik 633a3178c7SJosef Bacik if (nr == 0) 643a3178c7SJosef Bacik return BTRFS_LEAF_DATA_SIZE(leaf->fs_info); 653a3178c7SJosef Bacik return btrfs_item_offset(leaf, nr - 1); 663a3178c7SJosef Bacik } 673a3178c7SJosef Bacik 68637e3b48SJosef Bacik /* 69637e3b48SJosef Bacik * Move data in a @leaf (using memmove, safe for overlapping ranges). 70637e3b48SJosef Bacik * 71637e3b48SJosef Bacik * @leaf: leaf that we're doing a memmove on 72637e3b48SJosef Bacik * @dst_offset: item data offset we're moving to 73637e3b48SJosef Bacik * @src_offset: item data offset were' moving from 74637e3b48SJosef Bacik * @len: length of the data we're moving 75637e3b48SJosef Bacik * 76637e3b48SJosef Bacik * Wrapper around memmove_extent_buffer() that takes into account the header on 77637e3b48SJosef Bacik * the leaf. The btrfs_item offset's start directly after the header, so we 78637e3b48SJosef Bacik * have to adjust any offsets to account for the header in the leaf. This 79637e3b48SJosef Bacik * handles that math to simplify the callers. 80637e3b48SJosef Bacik */ 81637e3b48SJosef Bacik static inline void memmove_leaf_data(const struct extent_buffer *leaf, 82637e3b48SJosef Bacik unsigned long dst_offset, 83637e3b48SJosef Bacik unsigned long src_offset, 84637e3b48SJosef Bacik unsigned long len) 85637e3b48SJosef Bacik { 868009adf3SJosef Bacik memmove_extent_buffer(leaf, btrfs_item_nr_offset(leaf, 0) + dst_offset, 878009adf3SJosef Bacik btrfs_item_nr_offset(leaf, 0) + src_offset, len); 88637e3b48SJosef Bacik } 89637e3b48SJosef Bacik 90637e3b48SJosef Bacik /* 91637e3b48SJosef Bacik * Copy item data from @src into @dst at the given @offset. 92637e3b48SJosef Bacik * 93637e3b48SJosef Bacik * @dst: destination leaf that we're copying into 94637e3b48SJosef Bacik * @src: source leaf that we're copying from 95637e3b48SJosef Bacik * @dst_offset: item data offset we're copying to 96637e3b48SJosef Bacik * @src_offset: item data offset were' copying from 97637e3b48SJosef Bacik * @len: length of the data we're copying 98637e3b48SJosef Bacik * 99637e3b48SJosef Bacik * Wrapper around copy_extent_buffer() that takes into account the header on 100637e3b48SJosef Bacik * the leaf. The btrfs_item offset's start directly after the header, so we 101637e3b48SJosef Bacik * have to adjust any offsets to account for the header in the leaf. This 102637e3b48SJosef Bacik * handles that math to simplify the callers. 103637e3b48SJosef Bacik */ 104637e3b48SJosef Bacik static inline void copy_leaf_data(const struct extent_buffer *dst, 105637e3b48SJosef Bacik const struct extent_buffer *src, 106637e3b48SJosef Bacik unsigned long dst_offset, 107637e3b48SJosef Bacik unsigned long src_offset, unsigned long len) 108637e3b48SJosef Bacik { 1098009adf3SJosef Bacik copy_extent_buffer(dst, src, btrfs_item_nr_offset(dst, 0) + dst_offset, 1108009adf3SJosef Bacik btrfs_item_nr_offset(src, 0) + src_offset, len); 111637e3b48SJosef Bacik } 112637e3b48SJosef Bacik 113637e3b48SJosef Bacik /* 114637e3b48SJosef Bacik * Move items in a @leaf (using memmove). 115637e3b48SJosef Bacik * 116637e3b48SJosef Bacik * @dst: destination leaf for the items 117637e3b48SJosef Bacik * @dst_item: the item nr we're copying into 118637e3b48SJosef Bacik * @src_item: the item nr we're copying from 119637e3b48SJosef Bacik * @nr_items: the number of items to copy 120637e3b48SJosef Bacik * 121637e3b48SJosef Bacik * Wrapper around memmove_extent_buffer() that does the math to get the 122637e3b48SJosef Bacik * appropriate offsets into the leaf from the item numbers. 123637e3b48SJosef Bacik */ 124637e3b48SJosef Bacik static inline void memmove_leaf_items(const struct extent_buffer *leaf, 125637e3b48SJosef Bacik int dst_item, int src_item, int nr_items) 126637e3b48SJosef Bacik { 127637e3b48SJosef Bacik memmove_extent_buffer(leaf, btrfs_item_nr_offset(leaf, dst_item), 128637e3b48SJosef Bacik btrfs_item_nr_offset(leaf, src_item), 129637e3b48SJosef Bacik nr_items * sizeof(struct btrfs_item)); 130637e3b48SJosef Bacik } 131637e3b48SJosef Bacik 132637e3b48SJosef Bacik /* 133637e3b48SJosef Bacik * Copy items from @src into @dst at the given @offset. 134637e3b48SJosef Bacik * 135637e3b48SJosef Bacik * @dst: destination leaf for the items 136637e3b48SJosef Bacik * @src: source leaf for the items 137637e3b48SJosef Bacik * @dst_item: the item nr we're copying into 138637e3b48SJosef Bacik * @src_item: the item nr we're copying from 139637e3b48SJosef Bacik * @nr_items: the number of items to copy 140637e3b48SJosef Bacik * 141637e3b48SJosef Bacik * Wrapper around copy_extent_buffer() that does the math to get the 142637e3b48SJosef Bacik * appropriate offsets into the leaf from the item numbers. 143637e3b48SJosef Bacik */ 144637e3b48SJosef Bacik static inline void copy_leaf_items(const struct extent_buffer *dst, 145637e3b48SJosef Bacik const struct extent_buffer *src, 146637e3b48SJosef Bacik int dst_item, int src_item, int nr_items) 147637e3b48SJosef Bacik { 148637e3b48SJosef Bacik copy_extent_buffer(dst, src, btrfs_item_nr_offset(dst, dst_item), 149637e3b48SJosef Bacik btrfs_item_nr_offset(src, src_item), 150637e3b48SJosef Bacik nr_items * sizeof(struct btrfs_item)); 151637e3b48SJosef Bacik } 152637e3b48SJosef Bacik 153*b3cbfb0dSJosef Bacik /* This exists for btrfs-progs usages. */ 154*b3cbfb0dSJosef Bacik u16 btrfs_csum_type_size(u16 type) 155*b3cbfb0dSJosef Bacik { 156*b3cbfb0dSJosef Bacik return btrfs_csums[type].size; 157*b3cbfb0dSJosef Bacik } 158*b3cbfb0dSJosef Bacik 159af024ed2SJohannes Thumshirn int btrfs_super_csum_size(const struct btrfs_super_block *s) 160af024ed2SJohannes Thumshirn { 161af024ed2SJohannes Thumshirn u16 t = btrfs_super_csum_type(s); 162af024ed2SJohannes Thumshirn /* 163af024ed2SJohannes Thumshirn * csum type is validated at mount time 164af024ed2SJohannes Thumshirn */ 165*b3cbfb0dSJosef Bacik return btrfs_csum_type_size(t); 166af024ed2SJohannes Thumshirn } 167af024ed2SJohannes Thumshirn 168af024ed2SJohannes Thumshirn const char *btrfs_super_csum_name(u16 csum_type) 169af024ed2SJohannes Thumshirn { 170af024ed2SJohannes Thumshirn /* csum type is validated at mount time */ 171af024ed2SJohannes Thumshirn return btrfs_csums[csum_type].name; 172af024ed2SJohannes Thumshirn } 173af024ed2SJohannes Thumshirn 174b4e967beSDavid Sterba /* 175b4e967beSDavid Sterba * Return driver name if defined, otherwise the name that's also a valid driver 176b4e967beSDavid Sterba * name 177b4e967beSDavid Sterba */ 178b4e967beSDavid Sterba const char *btrfs_super_csum_driver(u16 csum_type) 179b4e967beSDavid Sterba { 180b4e967beSDavid Sterba /* csum type is validated at mount time */ 18159a0fcdbSDavid Sterba return btrfs_csums[csum_type].driver[0] ? 18259a0fcdbSDavid Sterba btrfs_csums[csum_type].driver : 183b4e967beSDavid Sterba btrfs_csums[csum_type].name; 184b4e967beSDavid Sterba } 185b4e967beSDavid Sterba 186604997b4SDavid Sterba size_t __attribute_const__ btrfs_get_num_csums(void) 187f7cea56cSDavid Sterba { 188f7cea56cSDavid Sterba return ARRAY_SIZE(btrfs_csums); 189f7cea56cSDavid Sterba } 190f7cea56cSDavid Sterba 1912c90e5d6SChris Mason struct btrfs_path *btrfs_alloc_path(void) 1922c90e5d6SChris Mason { 193a4c853afSChenXiaoSong might_sleep(); 194a4c853afSChenXiaoSong 195e2c89907SMasahiro Yamada return kmem_cache_zalloc(btrfs_path_cachep, GFP_NOFS); 1962c90e5d6SChris Mason } 1972c90e5d6SChris Mason 198d352ac68SChris Mason /* this also releases the path */ 1992c90e5d6SChris Mason void btrfs_free_path(struct btrfs_path *p) 2002c90e5d6SChris Mason { 201ff175d57SJesper Juhl if (!p) 202ff175d57SJesper Juhl return; 203b3b4aa74SDavid Sterba btrfs_release_path(p); 2042c90e5d6SChris Mason kmem_cache_free(btrfs_path_cachep, p); 2052c90e5d6SChris Mason } 2062c90e5d6SChris Mason 207d352ac68SChris Mason /* 208d352ac68SChris Mason * path release drops references on the extent buffers in the path 209d352ac68SChris Mason * and it drops any locks held by this path 210d352ac68SChris Mason * 211d352ac68SChris Mason * It is safe to call this on paths that no locks or extent buffers held. 212d352ac68SChris Mason */ 213b3b4aa74SDavid Sterba noinline void btrfs_release_path(struct btrfs_path *p) 214eb60ceacSChris Mason { 215eb60ceacSChris Mason int i; 216a2135011SChris Mason 217234b63a0SChris Mason for (i = 0; i < BTRFS_MAX_LEVEL; i++) { 2183f157a2fSChris Mason p->slots[i] = 0; 219eb60ceacSChris Mason if (!p->nodes[i]) 220925baeddSChris Mason continue; 221925baeddSChris Mason if (p->locks[i]) { 222bd681513SChris Mason btrfs_tree_unlock_rw(p->nodes[i], p->locks[i]); 223925baeddSChris Mason p->locks[i] = 0; 224925baeddSChris Mason } 2255f39d397SChris Mason free_extent_buffer(p->nodes[i]); 2263f157a2fSChris Mason p->nodes[i] = NULL; 227eb60ceacSChris Mason } 228eb60ceacSChris Mason } 229eb60ceacSChris Mason 230d352ac68SChris Mason /* 2318bb808c6SDavid Sterba * We want the transaction abort to print stack trace only for errors where the 2328bb808c6SDavid Sterba * cause could be a bug, eg. due to ENOSPC, and not for common errors that are 2338bb808c6SDavid Sterba * caused by external factors. 2348bb808c6SDavid Sterba */ 2358bb808c6SDavid Sterba bool __cold abort_should_print_stack(int errno) 2368bb808c6SDavid Sterba { 2378bb808c6SDavid Sterba switch (errno) { 2388bb808c6SDavid Sterba case -EIO: 2398bb808c6SDavid Sterba case -EROFS: 2408bb808c6SDavid Sterba case -ENOMEM: 2418bb808c6SDavid Sterba return false; 2428bb808c6SDavid Sterba } 2438bb808c6SDavid Sterba return true; 2448bb808c6SDavid Sterba } 2458bb808c6SDavid Sterba 2468bb808c6SDavid Sterba /* 247d352ac68SChris Mason * safely gets a reference on the root node of a tree. A lock 248d352ac68SChris Mason * is not taken, so a concurrent writer may put a different node 249d352ac68SChris Mason * at the root of the tree. See btrfs_lock_root_node for the 250d352ac68SChris Mason * looping required. 251d352ac68SChris Mason * 252d352ac68SChris Mason * The extent buffer returned by this has a reference taken, so 253d352ac68SChris Mason * it won't disappear. It may stop being the root of the tree 254d352ac68SChris Mason * at any time because there are no locks held. 255d352ac68SChris Mason */ 256925baeddSChris Mason struct extent_buffer *btrfs_root_node(struct btrfs_root *root) 257925baeddSChris Mason { 258925baeddSChris Mason struct extent_buffer *eb; 259240f62c8SChris Mason 2603083ee2eSJosef Bacik while (1) { 261240f62c8SChris Mason rcu_read_lock(); 262240f62c8SChris Mason eb = rcu_dereference(root->node); 2633083ee2eSJosef Bacik 2643083ee2eSJosef Bacik /* 2653083ee2eSJosef Bacik * RCU really hurts here, we could free up the root node because 26601327610SNicholas D Steeves * it was COWed but we may not get the new root node yet so do 2673083ee2eSJosef Bacik * the inc_not_zero dance and if it doesn't work then 2683083ee2eSJosef Bacik * synchronize_rcu and try again. 2693083ee2eSJosef Bacik */ 2703083ee2eSJosef Bacik if (atomic_inc_not_zero(&eb->refs)) { 271240f62c8SChris Mason rcu_read_unlock(); 2723083ee2eSJosef Bacik break; 2733083ee2eSJosef Bacik } 2743083ee2eSJosef Bacik rcu_read_unlock(); 2753083ee2eSJosef Bacik synchronize_rcu(); 2763083ee2eSJosef Bacik } 277925baeddSChris Mason return eb; 278925baeddSChris Mason } 279925baeddSChris Mason 28092a7cc42SQu Wenruo /* 28192a7cc42SQu Wenruo * Cowonly root (not-shareable trees, everything not subvolume or reloc roots), 28292a7cc42SQu Wenruo * just get put onto a simple dirty list. Transaction walks this list to make 28392a7cc42SQu Wenruo * sure they get properly updated on disk. 284d352ac68SChris Mason */ 2850b86a832SChris Mason static void add_root_to_dirty_list(struct btrfs_root *root) 2860b86a832SChris Mason { 2870b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 2880b246afaSJeff Mahoney 289e7070be1SJosef Bacik if (test_bit(BTRFS_ROOT_DIRTY, &root->state) || 290e7070be1SJosef Bacik !test_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state)) 291e7070be1SJosef Bacik return; 292e7070be1SJosef Bacik 2930b246afaSJeff Mahoney spin_lock(&fs_info->trans_lock); 294e7070be1SJosef Bacik if (!test_and_set_bit(BTRFS_ROOT_DIRTY, &root->state)) { 295e7070be1SJosef Bacik /* Want the extent tree to be the last on the list */ 2964fd786e6SMisono Tomohiro if (root->root_key.objectid == BTRFS_EXTENT_TREE_OBJECTID) 297e7070be1SJosef Bacik list_move_tail(&root->dirty_list, 2980b246afaSJeff Mahoney &fs_info->dirty_cowonly_roots); 299e7070be1SJosef Bacik else 300e7070be1SJosef Bacik list_move(&root->dirty_list, 3010b246afaSJeff Mahoney &fs_info->dirty_cowonly_roots); 3020b86a832SChris Mason } 3030b246afaSJeff Mahoney spin_unlock(&fs_info->trans_lock); 3040b86a832SChris Mason } 3050b86a832SChris Mason 306d352ac68SChris Mason /* 307d352ac68SChris Mason * used by snapshot creation to make a copy of a root for a tree with 308d352ac68SChris Mason * a given objectid. The buffer with the new root node is returned in 309d352ac68SChris Mason * cow_ret, and this func returns zero on success or a negative error code. 310d352ac68SChris Mason */ 311be20aa9dSChris Mason int btrfs_copy_root(struct btrfs_trans_handle *trans, 312be20aa9dSChris Mason struct btrfs_root *root, 313be20aa9dSChris Mason struct extent_buffer *buf, 314be20aa9dSChris Mason struct extent_buffer **cow_ret, u64 new_root_objectid) 315be20aa9dSChris Mason { 3160b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 317be20aa9dSChris Mason struct extent_buffer *cow; 318be20aa9dSChris Mason int ret = 0; 319be20aa9dSChris Mason int level; 3205d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 321be20aa9dSChris Mason 32292a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 3230b246afaSJeff Mahoney trans->transid != fs_info->running_transaction->transid); 32492a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 32527cdeb70SMiao Xie trans->transid != root->last_trans); 326be20aa9dSChris Mason 327be20aa9dSChris Mason level = btrfs_header_level(buf); 3285d4f98a2SYan Zheng if (level == 0) 3295d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 3305d4f98a2SYan Zheng else 3315d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 33231840ae1SZheng Yan 3334d75f8a9SDavid Sterba cow = btrfs_alloc_tree_block(trans, root, 0, new_root_objectid, 334cf6f34aaSJosef Bacik &disk_key, level, buf->start, 0, 335cf6f34aaSJosef Bacik BTRFS_NESTING_NEW_ROOT); 3365d4f98a2SYan Zheng if (IS_ERR(cow)) 337be20aa9dSChris Mason return PTR_ERR(cow); 338be20aa9dSChris Mason 33958e8012cSDavid Sterba copy_extent_buffer_full(cow, buf); 340be20aa9dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 341be20aa9dSChris Mason btrfs_set_header_generation(cow, trans->transid); 3425d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 3435d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 3445d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 3455d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 3465d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 3475d4f98a2SYan Zheng else 348be20aa9dSChris Mason btrfs_set_header_owner(cow, new_root_objectid); 349be20aa9dSChris Mason 350de37aa51SNikolay Borisov write_extent_buffer_fsid(cow, fs_info->fs_devices->metadata_uuid); 3512b82032cSYan Zheng 352be20aa9dSChris Mason WARN_ON(btrfs_header_generation(buf) > trans->transid); 3535d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 354e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 3555d4f98a2SYan Zheng else 356e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 357867ed321SJosef Bacik if (ret) { 35872c9925fSFilipe Manana btrfs_tree_unlock(cow); 35972c9925fSFilipe Manana free_extent_buffer(cow); 360867ed321SJosef Bacik btrfs_abort_transaction(trans, ret); 361be20aa9dSChris Mason return ret; 362867ed321SJosef Bacik } 363be20aa9dSChris Mason 364be20aa9dSChris Mason btrfs_mark_buffer_dirty(cow); 365be20aa9dSChris Mason *cow_ret = cow; 366be20aa9dSChris Mason return 0; 367be20aa9dSChris Mason } 368be20aa9dSChris Mason 369d352ac68SChris Mason /* 3705d4f98a2SYan Zheng * check if the tree block can be shared by multiple trees 3715d4f98a2SYan Zheng */ 3725d4f98a2SYan Zheng int btrfs_block_can_be_shared(struct btrfs_root *root, 3735d4f98a2SYan Zheng struct extent_buffer *buf) 3745d4f98a2SYan Zheng { 3755d4f98a2SYan Zheng /* 37692a7cc42SQu Wenruo * Tree blocks not in shareable trees and tree roots are never shared. 37792a7cc42SQu Wenruo * If a block was allocated after the last snapshot and the block was 37892a7cc42SQu Wenruo * not allocated by tree relocation, we know the block is not shared. 3795d4f98a2SYan Zheng */ 38092a7cc42SQu Wenruo if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 3815d4f98a2SYan Zheng buf != root->node && buf != root->commit_root && 3825d4f98a2SYan Zheng (btrfs_header_generation(buf) <= 3835d4f98a2SYan Zheng btrfs_root_last_snapshot(&root->root_item) || 3845d4f98a2SYan Zheng btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC))) 3855d4f98a2SYan Zheng return 1; 386a79865c6SNikolay Borisov 3875d4f98a2SYan Zheng return 0; 3885d4f98a2SYan Zheng } 3895d4f98a2SYan Zheng 3905d4f98a2SYan Zheng static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans, 3915d4f98a2SYan Zheng struct btrfs_root *root, 3925d4f98a2SYan Zheng struct extent_buffer *buf, 393f0486c68SYan, Zheng struct extent_buffer *cow, 394f0486c68SYan, Zheng int *last_ref) 3955d4f98a2SYan Zheng { 3960b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 3975d4f98a2SYan Zheng u64 refs; 3985d4f98a2SYan Zheng u64 owner; 3995d4f98a2SYan Zheng u64 flags; 4005d4f98a2SYan Zheng u64 new_flags = 0; 4015d4f98a2SYan Zheng int ret; 4025d4f98a2SYan Zheng 4035d4f98a2SYan Zheng /* 4045d4f98a2SYan Zheng * Backrefs update rules: 4055d4f98a2SYan Zheng * 4065d4f98a2SYan Zheng * Always use full backrefs for extent pointers in tree block 4075d4f98a2SYan Zheng * allocated by tree relocation. 4085d4f98a2SYan Zheng * 4095d4f98a2SYan Zheng * If a shared tree block is no longer referenced by its owner 4105d4f98a2SYan Zheng * tree (btrfs_header_owner(buf) == root->root_key.objectid), 4115d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 4125d4f98a2SYan Zheng * 4135d4f98a2SYan Zheng * If a tree block is been relocating 4145d4f98a2SYan Zheng * (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID), 4155d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 4165d4f98a2SYan Zheng * The reason for this is some operations (such as drop tree) 4175d4f98a2SYan Zheng * are only allowed for blocks use full backrefs. 4185d4f98a2SYan Zheng */ 4195d4f98a2SYan Zheng 4205d4f98a2SYan Zheng if (btrfs_block_can_be_shared(root, buf)) { 4212ff7e61eSJeff Mahoney ret = btrfs_lookup_extent_info(trans, fs_info, buf->start, 4223173a18fSJosef Bacik btrfs_header_level(buf), 1, 4233173a18fSJosef Bacik &refs, &flags); 424be1a5564SMark Fasheh if (ret) 425be1a5564SMark Fasheh return ret; 426e5df9573SMark Fasheh if (refs == 0) { 427e5df9573SMark Fasheh ret = -EROFS; 4280b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 429e5df9573SMark Fasheh return ret; 430e5df9573SMark Fasheh } 4315d4f98a2SYan Zheng } else { 4325d4f98a2SYan Zheng refs = 1; 4335d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 4345d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 4355d4f98a2SYan Zheng flags = BTRFS_BLOCK_FLAG_FULL_BACKREF; 4365d4f98a2SYan Zheng else 4375d4f98a2SYan Zheng flags = 0; 4385d4f98a2SYan Zheng } 4395d4f98a2SYan Zheng 4405d4f98a2SYan Zheng owner = btrfs_header_owner(buf); 4415d4f98a2SYan Zheng BUG_ON(owner == BTRFS_TREE_RELOC_OBJECTID && 4425d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)); 4435d4f98a2SYan Zheng 4445d4f98a2SYan Zheng if (refs > 1) { 4455d4f98a2SYan Zheng if ((owner == root->root_key.objectid || 4465d4f98a2SYan Zheng root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && 4475d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) { 448e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, buf, 1); 449692826b2SJeff Mahoney if (ret) 450692826b2SJeff Mahoney return ret; 4515d4f98a2SYan Zheng 4525d4f98a2SYan Zheng if (root->root_key.objectid == 4535d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) { 454e339a6b0SJosef Bacik ret = btrfs_dec_ref(trans, root, buf, 0); 455692826b2SJeff Mahoney if (ret) 456692826b2SJeff Mahoney return ret; 457e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 458692826b2SJeff Mahoney if (ret) 459692826b2SJeff Mahoney return ret; 4605d4f98a2SYan Zheng } 4615d4f98a2SYan Zheng new_flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF; 4625d4f98a2SYan Zheng } else { 4635d4f98a2SYan Zheng 4645d4f98a2SYan Zheng if (root->root_key.objectid == 4655d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 466e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 4675d4f98a2SYan Zheng else 468e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 469692826b2SJeff Mahoney if (ret) 470692826b2SJeff Mahoney return ret; 4715d4f98a2SYan Zheng } 4725d4f98a2SYan Zheng if (new_flags != 0) { 4734aec05faSJosef Bacik ret = btrfs_set_disk_extent_flags(trans, buf, new_flags); 474be1a5564SMark Fasheh if (ret) 475be1a5564SMark Fasheh return ret; 4765d4f98a2SYan Zheng } 4775d4f98a2SYan Zheng } else { 4785d4f98a2SYan Zheng if (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) { 4795d4f98a2SYan Zheng if (root->root_key.objectid == 4805d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 481e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 4825d4f98a2SYan Zheng else 483e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 484692826b2SJeff Mahoney if (ret) 485692826b2SJeff Mahoney return ret; 486e339a6b0SJosef Bacik ret = btrfs_dec_ref(trans, root, buf, 1); 487692826b2SJeff Mahoney if (ret) 488692826b2SJeff Mahoney return ret; 4895d4f98a2SYan Zheng } 490190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, buf); 491f0486c68SYan, Zheng *last_ref = 1; 4925d4f98a2SYan Zheng } 4935d4f98a2SYan Zheng return 0; 4945d4f98a2SYan Zheng } 4955d4f98a2SYan Zheng 4965d4f98a2SYan Zheng /* 497d397712bSChris Mason * does the dirty work in cow of a single block. The parent block (if 498d397712bSChris Mason * supplied) is updated to point to the new cow copy. The new buffer is marked 499d397712bSChris Mason * dirty and returned locked. If you modify the block it needs to be marked 500d397712bSChris Mason * dirty again. 501d352ac68SChris Mason * 502d352ac68SChris Mason * search_start -- an allocation hint for the new block 503d352ac68SChris Mason * 504d397712bSChris Mason * empty_size -- a hint that you plan on doing more cow. This is the size in 505d397712bSChris Mason * bytes the allocator should try to find free next to the block it returns. 506d397712bSChris Mason * This is just a hint and may be ignored by the allocator. 507d352ac68SChris Mason */ 508d397712bSChris Mason static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans, 5095f39d397SChris Mason struct btrfs_root *root, 5105f39d397SChris Mason struct extent_buffer *buf, 5115f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 5125f39d397SChris Mason struct extent_buffer **cow_ret, 5139631e4ccSJosef Bacik u64 search_start, u64 empty_size, 5149631e4ccSJosef Bacik enum btrfs_lock_nesting nest) 5156702ed49SChris Mason { 5160b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 5175d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 5185f39d397SChris Mason struct extent_buffer *cow; 519be1a5564SMark Fasheh int level, ret; 520f0486c68SYan, Zheng int last_ref = 0; 521925baeddSChris Mason int unlock_orig = 0; 5220f5053ebSGoldwyn Rodrigues u64 parent_start = 0; 5236702ed49SChris Mason 524925baeddSChris Mason if (*cow_ret == buf) 525925baeddSChris Mason unlock_orig = 1; 526925baeddSChris Mason 52749d0c642SFilipe Manana btrfs_assert_tree_write_locked(buf); 528925baeddSChris Mason 52992a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 5300b246afaSJeff Mahoney trans->transid != fs_info->running_transaction->transid); 53192a7cc42SQu Wenruo WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && 53227cdeb70SMiao Xie trans->transid != root->last_trans); 5335f39d397SChris Mason 5347bb86316SChris Mason level = btrfs_header_level(buf); 53531840ae1SZheng Yan 5365d4f98a2SYan Zheng if (level == 0) 5375d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 5385d4f98a2SYan Zheng else 5395d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 5405d4f98a2SYan Zheng 5410f5053ebSGoldwyn Rodrigues if ((root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && parent) 5425d4f98a2SYan Zheng parent_start = parent->start; 5435d4f98a2SYan Zheng 54479bd3712SFilipe Manana cow = btrfs_alloc_tree_block(trans, root, parent_start, 54579bd3712SFilipe Manana root->root_key.objectid, &disk_key, level, 54679bd3712SFilipe Manana search_start, empty_size, nest); 5476702ed49SChris Mason if (IS_ERR(cow)) 5486702ed49SChris Mason return PTR_ERR(cow); 5496702ed49SChris Mason 550b4ce94deSChris Mason /* cow is set to blocking by btrfs_init_new_buffer */ 551b4ce94deSChris Mason 55258e8012cSDavid Sterba copy_extent_buffer_full(cow, buf); 553db94535dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 5545f39d397SChris Mason btrfs_set_header_generation(cow, trans->transid); 5555d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 5565d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 5575d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 5585d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) 5595d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 5605d4f98a2SYan Zheng else 5615f39d397SChris Mason btrfs_set_header_owner(cow, root->root_key.objectid); 5626702ed49SChris Mason 563de37aa51SNikolay Borisov write_extent_buffer_fsid(cow, fs_info->fs_devices->metadata_uuid); 5642b82032cSYan Zheng 565be1a5564SMark Fasheh ret = update_ref_for_cow(trans, root, buf, cow, &last_ref); 566b68dc2a9SMark Fasheh if (ret) { 567572c83acSJosef Bacik btrfs_tree_unlock(cow); 568572c83acSJosef Bacik free_extent_buffer(cow); 56966642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 570b68dc2a9SMark Fasheh return ret; 571b68dc2a9SMark Fasheh } 5721a40e23bSZheng Yan 57392a7cc42SQu Wenruo if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) { 57483d4cfd4SJosef Bacik ret = btrfs_reloc_cow_block(trans, root, buf, cow); 57593314e3bSZhaolei if (ret) { 576572c83acSJosef Bacik btrfs_tree_unlock(cow); 577572c83acSJosef Bacik free_extent_buffer(cow); 57866642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 57983d4cfd4SJosef Bacik return ret; 58083d4cfd4SJosef Bacik } 58193314e3bSZhaolei } 5823fd0a558SYan, Zheng 5836702ed49SChris Mason if (buf == root->node) { 584925baeddSChris Mason WARN_ON(parent && parent != buf); 5855d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 5865d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 5875d4f98a2SYan Zheng parent_start = buf->start; 588925baeddSChris Mason 58967439dadSDavid Sterba atomic_inc(&cow->refs); 590406808abSFilipe Manana ret = btrfs_tree_mod_log_insert_root(root->node, cow, true); 591d9d19a01SDavid Sterba BUG_ON(ret < 0); 592240f62c8SChris Mason rcu_assign_pointer(root->node, cow); 593925baeddSChris Mason 5947a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), buf, 5957a163608SFilipe Manana parent_start, last_ref); 5965f39d397SChris Mason free_extent_buffer(buf); 5970b86a832SChris Mason add_root_to_dirty_list(root); 5986702ed49SChris Mason } else { 5995d4f98a2SYan Zheng WARN_ON(trans->transid != btrfs_header_generation(parent)); 600f3a84ccdSFilipe Manana btrfs_tree_mod_log_insert_key(parent, parent_slot, 60133cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 6025f39d397SChris Mason btrfs_set_node_blockptr(parent, parent_slot, 603db94535dSChris Mason cow->start); 60474493f7aSChris Mason btrfs_set_node_ptr_generation(parent, parent_slot, 60574493f7aSChris Mason trans->transid); 6066702ed49SChris Mason btrfs_mark_buffer_dirty(parent); 6075de865eeSFilipe David Borba Manana if (last_ref) { 608f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_free_eb(buf); 6095de865eeSFilipe David Borba Manana if (ret) { 610572c83acSJosef Bacik btrfs_tree_unlock(cow); 611572c83acSJosef Bacik free_extent_buffer(cow); 61266642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 6135de865eeSFilipe David Borba Manana return ret; 6145de865eeSFilipe David Borba Manana } 6155de865eeSFilipe David Borba Manana } 6167a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), buf, 6177a163608SFilipe Manana parent_start, last_ref); 6186702ed49SChris Mason } 619925baeddSChris Mason if (unlock_orig) 620925baeddSChris Mason btrfs_tree_unlock(buf); 6213083ee2eSJosef Bacik free_extent_buffer_stale(buf); 6226702ed49SChris Mason btrfs_mark_buffer_dirty(cow); 6236702ed49SChris Mason *cow_ret = cow; 6246702ed49SChris Mason return 0; 6256702ed49SChris Mason } 6266702ed49SChris Mason 6275d4f98a2SYan Zheng static inline int should_cow_block(struct btrfs_trans_handle *trans, 6285d4f98a2SYan Zheng struct btrfs_root *root, 6295d4f98a2SYan Zheng struct extent_buffer *buf) 6305d4f98a2SYan Zheng { 631f5ee5c9aSJeff Mahoney if (btrfs_is_testing(root->fs_info)) 632faa2dbf0SJosef Bacik return 0; 633fccb84c9SDavid Sterba 634d1980131SDavid Sterba /* Ensure we can see the FORCE_COW bit */ 635d1980131SDavid Sterba smp_mb__before_atomic(); 636f1ebcc74SLiu Bo 637f1ebcc74SLiu Bo /* 638f1ebcc74SLiu Bo * We do not need to cow a block if 639f1ebcc74SLiu Bo * 1) this block is not created or changed in this transaction; 640f1ebcc74SLiu Bo * 2) this block does not belong to TREE_RELOC tree; 641f1ebcc74SLiu Bo * 3) the root is not forced COW. 642f1ebcc74SLiu Bo * 643f1ebcc74SLiu Bo * What is forced COW: 64401327610SNicholas D Steeves * when we create snapshot during committing the transaction, 64552042d8eSAndrea Gelmini * after we've finished copying src root, we must COW the shared 646f1ebcc74SLiu Bo * block to ensure the metadata consistency. 647f1ebcc74SLiu Bo */ 6485d4f98a2SYan Zheng if (btrfs_header_generation(buf) == trans->transid && 6495d4f98a2SYan Zheng !btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN) && 6505d4f98a2SYan Zheng !(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID && 651f1ebcc74SLiu Bo btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)) && 65227cdeb70SMiao Xie !test_bit(BTRFS_ROOT_FORCE_COW, &root->state)) 6535d4f98a2SYan Zheng return 0; 6545d4f98a2SYan Zheng return 1; 6555d4f98a2SYan Zheng } 6565d4f98a2SYan Zheng 657d352ac68SChris Mason /* 658d352ac68SChris Mason * cows a single block, see __btrfs_cow_block for the real work. 65901327610SNicholas D Steeves * This version of it has extra checks so that a block isn't COWed more than 660d352ac68SChris Mason * once per transaction, as long as it hasn't been written yet 661d352ac68SChris Mason */ 662d397712bSChris Mason noinline int btrfs_cow_block(struct btrfs_trans_handle *trans, 6635f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *buf, 6645f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 6659631e4ccSJosef Bacik struct extent_buffer **cow_ret, 6669631e4ccSJosef Bacik enum btrfs_lock_nesting nest) 66702217ed2SChris Mason { 6680b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 6696702ed49SChris Mason u64 search_start; 670f510cfecSChris Mason int ret; 671dc17ff8fSChris Mason 67283354f07SJosef Bacik if (test_bit(BTRFS_ROOT_DELETING, &root->state)) 67383354f07SJosef Bacik btrfs_err(fs_info, 67483354f07SJosef Bacik "COW'ing blocks on a fs root that's being dropped"); 67583354f07SJosef Bacik 6760b246afaSJeff Mahoney if (trans->transaction != fs_info->running_transaction) 67731b1a2bdSJulia Lawall WARN(1, KERN_CRIT "trans %llu running %llu\n", 678c1c9ff7cSGeert Uytterhoeven trans->transid, 6790b246afaSJeff Mahoney fs_info->running_transaction->transid); 68031b1a2bdSJulia Lawall 6810b246afaSJeff Mahoney if (trans->transid != fs_info->generation) 68231b1a2bdSJulia Lawall WARN(1, KERN_CRIT "trans %llu running %llu\n", 6830b246afaSJeff Mahoney trans->transid, fs_info->generation); 684dc17ff8fSChris Mason 6855d4f98a2SYan Zheng if (!should_cow_block(trans, root, buf)) { 68602217ed2SChris Mason *cow_ret = buf; 68702217ed2SChris Mason return 0; 68802217ed2SChris Mason } 689c487685dSChris Mason 690ee22184bSByongho Lee search_start = buf->start & ~((u64)SZ_1G - 1); 691b4ce94deSChris Mason 692f616f5cdSQu Wenruo /* 693f616f5cdSQu Wenruo * Before CoWing this block for later modification, check if it's 694f616f5cdSQu Wenruo * the subtree root and do the delayed subtree trace if needed. 695f616f5cdSQu Wenruo * 696f616f5cdSQu Wenruo * Also We don't care about the error, as it's handled internally. 697f616f5cdSQu Wenruo */ 698f616f5cdSQu Wenruo btrfs_qgroup_trace_subtree_after_cow(trans, root, buf); 699f510cfecSChris Mason ret = __btrfs_cow_block(trans, root, buf, parent, 7009631e4ccSJosef Bacik parent_slot, cow_ret, search_start, 0, nest); 7011abe9b8aSliubo 7021abe9b8aSliubo trace_btrfs_cow_block(root, buf, *cow_ret); 7031abe9b8aSliubo 704f510cfecSChris Mason return ret; 7052c90e5d6SChris Mason } 706f75e2b79SJosef Bacik ALLOW_ERROR_INJECTION(btrfs_cow_block, ERRNO); 7076702ed49SChris Mason 708d352ac68SChris Mason /* 709d352ac68SChris Mason * helper function for defrag to decide if two blocks pointed to by a 710d352ac68SChris Mason * node are actually close by 711d352ac68SChris Mason */ 7126b80053dSChris Mason static int close_blocks(u64 blocknr, u64 other, u32 blocksize) 7136702ed49SChris Mason { 7146b80053dSChris Mason if (blocknr < other && other - (blocknr + blocksize) < 32768) 7156702ed49SChris Mason return 1; 7166b80053dSChris Mason if (blocknr > other && blocknr - (other + blocksize) < 32768) 7176702ed49SChris Mason return 1; 71802217ed2SChris Mason return 0; 71902217ed2SChris Mason } 72002217ed2SChris Mason 721ce6ef5abSDavid Sterba #ifdef __LITTLE_ENDIAN 722ce6ef5abSDavid Sterba 723ce6ef5abSDavid Sterba /* 724ce6ef5abSDavid Sterba * Compare two keys, on little-endian the disk order is same as CPU order and 725ce6ef5abSDavid Sterba * we can avoid the conversion. 726ce6ef5abSDavid Sterba */ 727ce6ef5abSDavid Sterba static int comp_keys(const struct btrfs_disk_key *disk_key, 728ce6ef5abSDavid Sterba const struct btrfs_key *k2) 729ce6ef5abSDavid Sterba { 730ce6ef5abSDavid Sterba const struct btrfs_key *k1 = (const struct btrfs_key *)disk_key; 731ce6ef5abSDavid Sterba 732ce6ef5abSDavid Sterba return btrfs_comp_cpu_keys(k1, k2); 733ce6ef5abSDavid Sterba } 734ce6ef5abSDavid Sterba 735ce6ef5abSDavid Sterba #else 736ce6ef5abSDavid Sterba 737081e9573SChris Mason /* 738081e9573SChris Mason * compare two keys in a memcmp fashion 739081e9573SChris Mason */ 740310712b2SOmar Sandoval static int comp_keys(const struct btrfs_disk_key *disk, 741310712b2SOmar Sandoval const struct btrfs_key *k2) 742081e9573SChris Mason { 743081e9573SChris Mason struct btrfs_key k1; 744081e9573SChris Mason 745081e9573SChris Mason btrfs_disk_key_to_cpu(&k1, disk); 746081e9573SChris Mason 74720736abaSDiego Calleja return btrfs_comp_cpu_keys(&k1, k2); 748081e9573SChris Mason } 749ce6ef5abSDavid Sterba #endif 750081e9573SChris Mason 751f3465ca4SJosef Bacik /* 752f3465ca4SJosef Bacik * same as comp_keys only with two btrfs_key's 753f3465ca4SJosef Bacik */ 754e1f60a65SDavid Sterba int __pure btrfs_comp_cpu_keys(const struct btrfs_key *k1, const struct btrfs_key *k2) 755f3465ca4SJosef Bacik { 756f3465ca4SJosef Bacik if (k1->objectid > k2->objectid) 757f3465ca4SJosef Bacik return 1; 758f3465ca4SJosef Bacik if (k1->objectid < k2->objectid) 759f3465ca4SJosef Bacik return -1; 760f3465ca4SJosef Bacik if (k1->type > k2->type) 761f3465ca4SJosef Bacik return 1; 762f3465ca4SJosef Bacik if (k1->type < k2->type) 763f3465ca4SJosef Bacik return -1; 764f3465ca4SJosef Bacik if (k1->offset > k2->offset) 765f3465ca4SJosef Bacik return 1; 766f3465ca4SJosef Bacik if (k1->offset < k2->offset) 767f3465ca4SJosef Bacik return -1; 768f3465ca4SJosef Bacik return 0; 769f3465ca4SJosef Bacik } 770081e9573SChris Mason 771d352ac68SChris Mason /* 772d352ac68SChris Mason * this is used by the defrag code to go through all the 773d352ac68SChris Mason * leaves pointed to by a node and reallocate them so that 774d352ac68SChris Mason * disk order is close to key order 775d352ac68SChris Mason */ 7766702ed49SChris Mason int btrfs_realloc_node(struct btrfs_trans_handle *trans, 7775f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *parent, 778de78b51aSEric Sandeen int start_slot, u64 *last_ret, 779a6b6e75eSChris Mason struct btrfs_key *progress) 7806702ed49SChris Mason { 7810b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 7826b80053dSChris Mason struct extent_buffer *cur; 7836702ed49SChris Mason u64 blocknr; 784e9d0b13bSChris Mason u64 search_start = *last_ret; 785e9d0b13bSChris Mason u64 last_block = 0; 7866702ed49SChris Mason u64 other; 7876702ed49SChris Mason u32 parent_nritems; 7886702ed49SChris Mason int end_slot; 7896702ed49SChris Mason int i; 7906702ed49SChris Mason int err = 0; 7916b80053dSChris Mason u32 blocksize; 792081e9573SChris Mason int progress_passed = 0; 793081e9573SChris Mason struct btrfs_disk_key disk_key; 7946702ed49SChris Mason 7950b246afaSJeff Mahoney WARN_ON(trans->transaction != fs_info->running_transaction); 7960b246afaSJeff Mahoney WARN_ON(trans->transid != fs_info->generation); 79786479a04SChris Mason 7986b80053dSChris Mason parent_nritems = btrfs_header_nritems(parent); 7990b246afaSJeff Mahoney blocksize = fs_info->nodesize; 8005dfe2be7SFilipe Manana end_slot = parent_nritems - 1; 8016702ed49SChris Mason 8025dfe2be7SFilipe Manana if (parent_nritems <= 1) 8036702ed49SChris Mason return 0; 8046702ed49SChris Mason 8055dfe2be7SFilipe Manana for (i = start_slot; i <= end_slot; i++) { 8066702ed49SChris Mason int close = 1; 807a6b6e75eSChris Mason 808081e9573SChris Mason btrfs_node_key(parent, &disk_key, i); 809081e9573SChris Mason if (!progress_passed && comp_keys(&disk_key, progress) < 0) 810081e9573SChris Mason continue; 811081e9573SChris Mason 812081e9573SChris Mason progress_passed = 1; 8136b80053dSChris Mason blocknr = btrfs_node_blockptr(parent, i); 814e9d0b13bSChris Mason if (last_block == 0) 815e9d0b13bSChris Mason last_block = blocknr; 8165708b959SChris Mason 8176702ed49SChris Mason if (i > 0) { 8186b80053dSChris Mason other = btrfs_node_blockptr(parent, i - 1); 8196b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 8206702ed49SChris Mason } 8215dfe2be7SFilipe Manana if (!close && i < end_slot) { 8226b80053dSChris Mason other = btrfs_node_blockptr(parent, i + 1); 8236b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 8246702ed49SChris Mason } 825e9d0b13bSChris Mason if (close) { 826e9d0b13bSChris Mason last_block = blocknr; 8276702ed49SChris Mason continue; 828e9d0b13bSChris Mason } 8296702ed49SChris Mason 830206983b7SJosef Bacik cur = btrfs_read_node_slot(parent, i); 831206983b7SJosef Bacik if (IS_ERR(cur)) 83264c043deSLiu Bo return PTR_ERR(cur); 833e9d0b13bSChris Mason if (search_start == 0) 8346b80053dSChris Mason search_start = last_block; 835e9d0b13bSChris Mason 836e7a84565SChris Mason btrfs_tree_lock(cur); 8376b80053dSChris Mason err = __btrfs_cow_block(trans, root, cur, parent, i, 838e7a84565SChris Mason &cur, search_start, 8396b80053dSChris Mason min(16 * blocksize, 8409631e4ccSJosef Bacik (end_slot - i) * blocksize), 8419631e4ccSJosef Bacik BTRFS_NESTING_COW); 842252c38f0SYan if (err) { 843e7a84565SChris Mason btrfs_tree_unlock(cur); 8446b80053dSChris Mason free_extent_buffer(cur); 8456702ed49SChris Mason break; 846252c38f0SYan } 847e7a84565SChris Mason search_start = cur->start; 848e7a84565SChris Mason last_block = cur->start; 849f2183bdeSChris Mason *last_ret = search_start; 850e7a84565SChris Mason btrfs_tree_unlock(cur); 851e7a84565SChris Mason free_extent_buffer(cur); 8526702ed49SChris Mason } 8536702ed49SChris Mason return err; 8546702ed49SChris Mason } 8556702ed49SChris Mason 85674123bd7SChris Mason /* 857fb81212cSFilipe Manana * Search for a key in the given extent_buffer. 8585f39d397SChris Mason * 859a724f313SFilipe Manana * The lower boundary for the search is specified by the slot number @first_slot. 860fdf8d595SAnand Jain * Use a value of 0 to search over the whole extent buffer. Works for both 861fdf8d595SAnand Jain * leaves and nodes. 86274123bd7SChris Mason * 863fb81212cSFilipe Manana * The slot in the extent buffer is returned via @slot. If the key exists in the 864fb81212cSFilipe Manana * extent buffer, then @slot will point to the slot where the key is, otherwise 865fb81212cSFilipe Manana * it points to the slot where you would insert the key. 866fb81212cSFilipe Manana * 867fb81212cSFilipe Manana * Slot may point to the total number of items (i.e. one position beyond the last 868fb81212cSFilipe Manana * key) if the key is bigger than the last key in the extent buffer. 86974123bd7SChris Mason */ 870fdf8d595SAnand Jain int btrfs_bin_search(struct extent_buffer *eb, int first_slot, 87167d5e289SMarcos Paulo de Souza const struct btrfs_key *key, int *slot) 872be0e5c09SChris Mason { 873fb81212cSFilipe Manana unsigned long p; 874fb81212cSFilipe Manana int item_size; 875a724f313SFilipe Manana /* 876a724f313SFilipe Manana * Use unsigned types for the low and high slots, so that we get a more 877a724f313SFilipe Manana * efficient division in the search loop below. 878a724f313SFilipe Manana */ 879a724f313SFilipe Manana u32 low = first_slot; 880a724f313SFilipe Manana u32 high = btrfs_header_nritems(eb); 881be0e5c09SChris Mason int ret; 8825cd17f34SDavid Sterba const int key_size = sizeof(struct btrfs_disk_key); 883be0e5c09SChris Mason 884a724f313SFilipe Manana if (unlikely(low > high)) { 8855e24e9afSLiu Bo btrfs_err(eb->fs_info, 886a724f313SFilipe Manana "%s: low (%u) > high (%u) eb %llu owner %llu level %d", 8875e24e9afSLiu Bo __func__, low, high, eb->start, 8885e24e9afSLiu Bo btrfs_header_owner(eb), btrfs_header_level(eb)); 8895e24e9afSLiu Bo return -EINVAL; 8905e24e9afSLiu Bo } 8915e24e9afSLiu Bo 892fb81212cSFilipe Manana if (btrfs_header_level(eb) == 0) { 893fb81212cSFilipe Manana p = offsetof(struct btrfs_leaf, items); 894fb81212cSFilipe Manana item_size = sizeof(struct btrfs_item); 895fb81212cSFilipe Manana } else { 896fb81212cSFilipe Manana p = offsetof(struct btrfs_node, ptrs); 897fb81212cSFilipe Manana item_size = sizeof(struct btrfs_key_ptr); 898fb81212cSFilipe Manana } 899fb81212cSFilipe Manana 900be0e5c09SChris Mason while (low < high) { 9015cd17f34SDavid Sterba unsigned long oip; 9025cd17f34SDavid Sterba unsigned long offset; 9035cd17f34SDavid Sterba struct btrfs_disk_key *tmp; 9045cd17f34SDavid Sterba struct btrfs_disk_key unaligned; 9055cd17f34SDavid Sterba int mid; 9065cd17f34SDavid Sterba 907be0e5c09SChris Mason mid = (low + high) / 2; 9085f39d397SChris Mason offset = p + mid * item_size; 9095cd17f34SDavid Sterba oip = offset_in_page(offset); 9105f39d397SChris Mason 9115cd17f34SDavid Sterba if (oip + key_size <= PAGE_SIZE) { 912884b07d0SQu Wenruo const unsigned long idx = get_eb_page_index(offset); 9135cd17f34SDavid Sterba char *kaddr = page_address(eb->pages[idx]); 914934d375bSChris Mason 915884b07d0SQu Wenruo oip = get_eb_offset_in_page(eb, offset); 9165cd17f34SDavid Sterba tmp = (struct btrfs_disk_key *)(kaddr + oip); 9175cd17f34SDavid Sterba } else { 9185cd17f34SDavid Sterba read_extent_buffer(eb, &unaligned, offset, key_size); 9195f39d397SChris Mason tmp = &unaligned; 920479965d6SChris Mason } 921479965d6SChris Mason 922be0e5c09SChris Mason ret = comp_keys(tmp, key); 923be0e5c09SChris Mason 924be0e5c09SChris Mason if (ret < 0) 925be0e5c09SChris Mason low = mid + 1; 926be0e5c09SChris Mason else if (ret > 0) 927be0e5c09SChris Mason high = mid; 928be0e5c09SChris Mason else { 929be0e5c09SChris Mason *slot = mid; 930be0e5c09SChris Mason return 0; 931be0e5c09SChris Mason } 932be0e5c09SChris Mason } 933be0e5c09SChris Mason *slot = low; 934be0e5c09SChris Mason return 1; 935be0e5c09SChris Mason } 936be0e5c09SChris Mason 937f0486c68SYan, Zheng static void root_add_used(struct btrfs_root *root, u32 size) 938f0486c68SYan, Zheng { 939f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 940f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 941f0486c68SYan, Zheng btrfs_root_used(&root->root_item) + size); 942f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 943f0486c68SYan, Zheng } 944f0486c68SYan, Zheng 945f0486c68SYan, Zheng static void root_sub_used(struct btrfs_root *root, u32 size) 946f0486c68SYan, Zheng { 947f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 948f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 949f0486c68SYan, Zheng btrfs_root_used(&root->root_item) - size); 950f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 951f0486c68SYan, Zheng } 952f0486c68SYan, Zheng 953d352ac68SChris Mason /* given a node and slot number, this reads the blocks it points to. The 954d352ac68SChris Mason * extent buffer is returned with a reference taken (but unlocked). 955d352ac68SChris Mason */ 9564b231ae4SDavid Sterba struct extent_buffer *btrfs_read_node_slot(struct extent_buffer *parent, 9574b231ae4SDavid Sterba int slot) 958bb803951SChris Mason { 959ca7a79adSChris Mason int level = btrfs_header_level(parent); 960789d6a3aSQu Wenruo struct btrfs_tree_parent_check check = { 0 }; 961416bc658SJosef Bacik struct extent_buffer *eb; 962416bc658SJosef Bacik 963fb770ae4SLiu Bo if (slot < 0 || slot >= btrfs_header_nritems(parent)) 964fb770ae4SLiu Bo return ERR_PTR(-ENOENT); 965ca7a79adSChris Mason 966d4694728SJosef Bacik ASSERT(level); 967ca7a79adSChris Mason 968789d6a3aSQu Wenruo check.level = level - 1; 969789d6a3aSQu Wenruo check.transid = btrfs_node_ptr_generation(parent, slot); 970789d6a3aSQu Wenruo check.owner_root = btrfs_header_owner(parent); 971789d6a3aSQu Wenruo check.has_first_key = true; 972789d6a3aSQu Wenruo btrfs_node_key_to_cpu(parent, &check.first_key, slot); 973789d6a3aSQu Wenruo 974d0d20b0fSDavid Sterba eb = read_tree_block(parent->fs_info, btrfs_node_blockptr(parent, slot), 975789d6a3aSQu Wenruo &check); 9764eb150d6SQu Wenruo if (IS_ERR(eb)) 9774eb150d6SQu Wenruo return eb; 9784eb150d6SQu Wenruo if (!extent_buffer_uptodate(eb)) { 979416bc658SJosef Bacik free_extent_buffer(eb); 9804eb150d6SQu Wenruo return ERR_PTR(-EIO); 981416bc658SJosef Bacik } 982416bc658SJosef Bacik 983416bc658SJosef Bacik return eb; 984bb803951SChris Mason } 985bb803951SChris Mason 986d352ac68SChris Mason /* 987d352ac68SChris Mason * node level balancing, used to make sure nodes are in proper order for 988d352ac68SChris Mason * item deletion. We balance from the top down, so we have to make sure 989d352ac68SChris Mason * that a deletion won't leave an node completely empty later on. 990d352ac68SChris Mason */ 991e02119d5SChris Mason static noinline int balance_level(struct btrfs_trans_handle *trans, 99298ed5174SChris Mason struct btrfs_root *root, 99398ed5174SChris Mason struct btrfs_path *path, int level) 994bb803951SChris Mason { 9950b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 9965f39d397SChris Mason struct extent_buffer *right = NULL; 9975f39d397SChris Mason struct extent_buffer *mid; 9985f39d397SChris Mason struct extent_buffer *left = NULL; 9995f39d397SChris Mason struct extent_buffer *parent = NULL; 1000bb803951SChris Mason int ret = 0; 1001bb803951SChris Mason int wret; 1002bb803951SChris Mason int pslot; 1003bb803951SChris Mason int orig_slot = path->slots[level]; 100479f95c82SChris Mason u64 orig_ptr; 1005bb803951SChris Mason 100698e6b1ebSLiu Bo ASSERT(level > 0); 1007bb803951SChris Mason 10085f39d397SChris Mason mid = path->nodes[level]; 1009b4ce94deSChris Mason 1010ac5887c8SJosef Bacik WARN_ON(path->locks[level] != BTRFS_WRITE_LOCK); 10117bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 10127bb86316SChris Mason 10131d4f8a0cSChris Mason orig_ptr = btrfs_node_blockptr(mid, orig_slot); 101479f95c82SChris Mason 1015a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 10165f39d397SChris Mason parent = path->nodes[level + 1]; 1017bb803951SChris Mason pslot = path->slots[level + 1]; 1018a05a9bb1SLi Zefan } 1019bb803951SChris Mason 102040689478SChris Mason /* 102140689478SChris Mason * deal with the case where there is only one pointer in the root 102240689478SChris Mason * by promoting the node below to a root 102340689478SChris Mason */ 10245f39d397SChris Mason if (!parent) { 10255f39d397SChris Mason struct extent_buffer *child; 1026bb803951SChris Mason 10275f39d397SChris Mason if (btrfs_header_nritems(mid) != 1) 1028bb803951SChris Mason return 0; 1029bb803951SChris Mason 1030bb803951SChris Mason /* promote the child to a root */ 10314b231ae4SDavid Sterba child = btrfs_read_node_slot(mid, 0); 1032fb770ae4SLiu Bo if (IS_ERR(child)) { 1033fb770ae4SLiu Bo ret = PTR_ERR(child); 10340b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 1035305a26afSMark Fasheh goto enospc; 1036305a26afSMark Fasheh } 1037305a26afSMark Fasheh 1038925baeddSChris Mason btrfs_tree_lock(child); 10399631e4ccSJosef Bacik ret = btrfs_cow_block(trans, root, child, mid, 0, &child, 10409631e4ccSJosef Bacik BTRFS_NESTING_COW); 1041f0486c68SYan, Zheng if (ret) { 1042f0486c68SYan, Zheng btrfs_tree_unlock(child); 1043f0486c68SYan, Zheng free_extent_buffer(child); 1044f0486c68SYan, Zheng goto enospc; 1045f0486c68SYan, Zheng } 10462f375ab9SYan 1047406808abSFilipe Manana ret = btrfs_tree_mod_log_insert_root(root->node, child, true); 1048d9d19a01SDavid Sterba BUG_ON(ret < 0); 1049240f62c8SChris Mason rcu_assign_pointer(root->node, child); 1050925baeddSChris Mason 10510b86a832SChris Mason add_root_to_dirty_list(root); 1052925baeddSChris Mason btrfs_tree_unlock(child); 1053b4ce94deSChris Mason 1054925baeddSChris Mason path->locks[level] = 0; 1055bb803951SChris Mason path->nodes[level] = NULL; 1056190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, mid); 1057925baeddSChris Mason btrfs_tree_unlock(mid); 1058bb803951SChris Mason /* once for the path */ 10595f39d397SChris Mason free_extent_buffer(mid); 1060f0486c68SYan, Zheng 1061f0486c68SYan, Zheng root_sub_used(root, mid->len); 10627a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), mid, 0, 1); 1063bb803951SChris Mason /* once for the root ptr */ 10643083ee2eSJosef Bacik free_extent_buffer_stale(mid); 1065f0486c68SYan, Zheng return 0; 1066bb803951SChris Mason } 10675f39d397SChris Mason if (btrfs_header_nritems(mid) > 10680b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 4) 1069bb803951SChris Mason return 0; 1070bb803951SChris Mason 10719cf14029SJosef Bacik if (pslot) { 10724b231ae4SDavid Sterba left = btrfs_read_node_slot(parent, pslot - 1); 10739cf14029SJosef Bacik if (IS_ERR(left)) { 10749cf14029SJosef Bacik ret = PTR_ERR(left); 1075fb770ae4SLiu Bo left = NULL; 10769cf14029SJosef Bacik goto enospc; 10779cf14029SJosef Bacik } 1078fb770ae4SLiu Bo 1079bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT); 10805f39d397SChris Mason wret = btrfs_cow_block(trans, root, left, 10819631e4ccSJosef Bacik parent, pslot - 1, &left, 1082bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW); 108354aa1f4dSChris Mason if (wret) { 108454aa1f4dSChris Mason ret = wret; 108554aa1f4dSChris Mason goto enospc; 108654aa1f4dSChris Mason } 10872cc58cf2SChris Mason } 1088fb770ae4SLiu Bo 10899cf14029SJosef Bacik if (pslot + 1 < btrfs_header_nritems(parent)) { 10904b231ae4SDavid Sterba right = btrfs_read_node_slot(parent, pslot + 1); 10919cf14029SJosef Bacik if (IS_ERR(right)) { 10929cf14029SJosef Bacik ret = PTR_ERR(right); 1093fb770ae4SLiu Bo right = NULL; 10949cf14029SJosef Bacik goto enospc; 10959cf14029SJosef Bacik } 1096fb770ae4SLiu Bo 1097bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT); 10985f39d397SChris Mason wret = btrfs_cow_block(trans, root, right, 10999631e4ccSJosef Bacik parent, pslot + 1, &right, 1100bf59a5a2SJosef Bacik BTRFS_NESTING_RIGHT_COW); 11012cc58cf2SChris Mason if (wret) { 11022cc58cf2SChris Mason ret = wret; 11032cc58cf2SChris Mason goto enospc; 11042cc58cf2SChris Mason } 11052cc58cf2SChris Mason } 11062cc58cf2SChris Mason 11072cc58cf2SChris Mason /* first, try to make some room in the middle buffer */ 11085f39d397SChris Mason if (left) { 11095f39d397SChris Mason orig_slot += btrfs_header_nritems(left); 1110d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 1); 111179f95c82SChris Mason if (wret < 0) 111279f95c82SChris Mason ret = wret; 1113bb803951SChris Mason } 111479f95c82SChris Mason 111579f95c82SChris Mason /* 111679f95c82SChris Mason * then try to empty the right most buffer into the middle 111779f95c82SChris Mason */ 11185f39d397SChris Mason if (right) { 1119d30a668fSDavid Sterba wret = push_node_left(trans, mid, right, 1); 112054aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 112179f95c82SChris Mason ret = wret; 11225f39d397SChris Mason if (btrfs_header_nritems(right) == 0) { 1123190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, right); 1124925baeddSChris Mason btrfs_tree_unlock(right); 1125afe5fea7STsutomu Itoh del_ptr(root, path, level + 1, pslot + 1); 1126f0486c68SYan, Zheng root_sub_used(root, right->len); 11277a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), right, 11287a163608SFilipe Manana 0, 1); 11293083ee2eSJosef Bacik free_extent_buffer_stale(right); 1130f0486c68SYan, Zheng right = NULL; 1131bb803951SChris Mason } else { 11325f39d397SChris Mason struct btrfs_disk_key right_key; 11335f39d397SChris Mason btrfs_node_key(right, &right_key, 0); 1134f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot + 1, 113533cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 11360e82bcfeSDavid Sterba BUG_ON(ret < 0); 11375f39d397SChris Mason btrfs_set_node_key(parent, &right_key, pslot + 1); 11385f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 1139bb803951SChris Mason } 1140bb803951SChris Mason } 11415f39d397SChris Mason if (btrfs_header_nritems(mid) == 1) { 114279f95c82SChris Mason /* 114379f95c82SChris Mason * we're not allowed to leave a node with one item in the 114479f95c82SChris Mason * tree during a delete. A deletion from lower in the tree 114579f95c82SChris Mason * could try to delete the only pointer in this node. 114679f95c82SChris Mason * So, pull some keys from the left. 114779f95c82SChris Mason * There has to be a left pointer at this point because 114879f95c82SChris Mason * otherwise we would have pulled some pointers from the 114979f95c82SChris Mason * right 115079f95c82SChris Mason */ 1151305a26afSMark Fasheh if (!left) { 1152305a26afSMark Fasheh ret = -EROFS; 11530b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 1154305a26afSMark Fasheh goto enospc; 1155305a26afSMark Fasheh } 115655d32ed8SDavid Sterba wret = balance_node_right(trans, mid, left); 115754aa1f4dSChris Mason if (wret < 0) { 115879f95c82SChris Mason ret = wret; 115954aa1f4dSChris Mason goto enospc; 116054aa1f4dSChris Mason } 1161bce4eae9SChris Mason if (wret == 1) { 1162d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 1); 1163bce4eae9SChris Mason if (wret < 0) 1164bce4eae9SChris Mason ret = wret; 1165bce4eae9SChris Mason } 116679f95c82SChris Mason BUG_ON(wret == 1); 116779f95c82SChris Mason } 11685f39d397SChris Mason if (btrfs_header_nritems(mid) == 0) { 1169190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, mid); 1170925baeddSChris Mason btrfs_tree_unlock(mid); 1171afe5fea7STsutomu Itoh del_ptr(root, path, level + 1, pslot); 1172f0486c68SYan, Zheng root_sub_used(root, mid->len); 11737a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), mid, 0, 1); 11743083ee2eSJosef Bacik free_extent_buffer_stale(mid); 1175f0486c68SYan, Zheng mid = NULL; 117679f95c82SChris Mason } else { 117779f95c82SChris Mason /* update the parent key to reflect our changes */ 11785f39d397SChris Mason struct btrfs_disk_key mid_key; 11795f39d397SChris Mason btrfs_node_key(mid, &mid_key, 0); 1180f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot, 118133cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 11820e82bcfeSDavid Sterba BUG_ON(ret < 0); 11835f39d397SChris Mason btrfs_set_node_key(parent, &mid_key, pslot); 11845f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 118579f95c82SChris Mason } 1186bb803951SChris Mason 118779f95c82SChris Mason /* update the path */ 11885f39d397SChris Mason if (left) { 11895f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 119067439dadSDavid Sterba atomic_inc(&left->refs); 1191925baeddSChris Mason /* left was locked after cow */ 11925f39d397SChris Mason path->nodes[level] = left; 1193bb803951SChris Mason path->slots[level + 1] -= 1; 1194bb803951SChris Mason path->slots[level] = orig_slot; 1195925baeddSChris Mason if (mid) { 1196925baeddSChris Mason btrfs_tree_unlock(mid); 11975f39d397SChris Mason free_extent_buffer(mid); 1198925baeddSChris Mason } 1199bb803951SChris Mason } else { 12005f39d397SChris Mason orig_slot -= btrfs_header_nritems(left); 1201bb803951SChris Mason path->slots[level] = orig_slot; 1202bb803951SChris Mason } 1203bb803951SChris Mason } 120479f95c82SChris Mason /* double check we haven't messed things up */ 1205e20d96d6SChris Mason if (orig_ptr != 12065f39d397SChris Mason btrfs_node_blockptr(path->nodes[level], path->slots[level])) 120779f95c82SChris Mason BUG(); 120854aa1f4dSChris Mason enospc: 1209925baeddSChris Mason if (right) { 1210925baeddSChris Mason btrfs_tree_unlock(right); 12115f39d397SChris Mason free_extent_buffer(right); 1212925baeddSChris Mason } 1213925baeddSChris Mason if (left) { 1214925baeddSChris Mason if (path->nodes[level] != left) 1215925baeddSChris Mason btrfs_tree_unlock(left); 12165f39d397SChris Mason free_extent_buffer(left); 1217925baeddSChris Mason } 1218bb803951SChris Mason return ret; 1219bb803951SChris Mason } 1220bb803951SChris Mason 1221d352ac68SChris Mason /* Node balancing for insertion. Here we only split or push nodes around 1222d352ac68SChris Mason * when they are completely full. This is also done top down, so we 1223d352ac68SChris Mason * have to be pessimistic. 1224d352ac68SChris Mason */ 1225d397712bSChris Mason static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans, 1226e66f709bSChris Mason struct btrfs_root *root, 1227e66f709bSChris Mason struct btrfs_path *path, int level) 1228e66f709bSChris Mason { 12290b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 12305f39d397SChris Mason struct extent_buffer *right = NULL; 12315f39d397SChris Mason struct extent_buffer *mid; 12325f39d397SChris Mason struct extent_buffer *left = NULL; 12335f39d397SChris Mason struct extent_buffer *parent = NULL; 1234e66f709bSChris Mason int ret = 0; 1235e66f709bSChris Mason int wret; 1236e66f709bSChris Mason int pslot; 1237e66f709bSChris Mason int orig_slot = path->slots[level]; 1238e66f709bSChris Mason 1239e66f709bSChris Mason if (level == 0) 1240e66f709bSChris Mason return 1; 1241e66f709bSChris Mason 12425f39d397SChris Mason mid = path->nodes[level]; 12437bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 1244e66f709bSChris Mason 1245a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 12465f39d397SChris Mason parent = path->nodes[level + 1]; 1247e66f709bSChris Mason pslot = path->slots[level + 1]; 1248a05a9bb1SLi Zefan } 1249e66f709bSChris Mason 12505f39d397SChris Mason if (!parent) 1251e66f709bSChris Mason return 1; 1252e66f709bSChris Mason 12539cf14029SJosef Bacik /* first, try to make some room in the middle buffer */ 12549cf14029SJosef Bacik if (pslot) { 12559cf14029SJosef Bacik u32 left_nr; 12569cf14029SJosef Bacik 12574b231ae4SDavid Sterba left = btrfs_read_node_slot(parent, pslot - 1); 1258fb770ae4SLiu Bo if (IS_ERR(left)) 12599cf14029SJosef Bacik return PTR_ERR(left); 1260925baeddSChris Mason 1261bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT); 1262b4ce94deSChris Mason 12635f39d397SChris Mason left_nr = btrfs_header_nritems(left); 12640b246afaSJeff Mahoney if (left_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) { 126533ade1f8SChris Mason wret = 1; 126633ade1f8SChris Mason } else { 12675f39d397SChris Mason ret = btrfs_cow_block(trans, root, left, parent, 12689631e4ccSJosef Bacik pslot - 1, &left, 1269bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW); 127054aa1f4dSChris Mason if (ret) 127154aa1f4dSChris Mason wret = 1; 127254aa1f4dSChris Mason else { 1273d30a668fSDavid Sterba wret = push_node_left(trans, left, mid, 0); 127454aa1f4dSChris Mason } 127533ade1f8SChris Mason } 1276e66f709bSChris Mason if (wret < 0) 1277e66f709bSChris Mason ret = wret; 1278e66f709bSChris Mason if (wret == 0) { 12795f39d397SChris Mason struct btrfs_disk_key disk_key; 1280e66f709bSChris Mason orig_slot += left_nr; 12815f39d397SChris Mason btrfs_node_key(mid, &disk_key, 0); 1282f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot, 128333cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 12840e82bcfeSDavid Sterba BUG_ON(ret < 0); 12855f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot); 12865f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 12875f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 12885f39d397SChris Mason path->nodes[level] = left; 1289e66f709bSChris Mason path->slots[level + 1] -= 1; 1290e66f709bSChris Mason path->slots[level] = orig_slot; 1291925baeddSChris Mason btrfs_tree_unlock(mid); 12925f39d397SChris Mason free_extent_buffer(mid); 1293e66f709bSChris Mason } else { 1294e66f709bSChris Mason orig_slot -= 12955f39d397SChris Mason btrfs_header_nritems(left); 1296e66f709bSChris Mason path->slots[level] = orig_slot; 1297925baeddSChris Mason btrfs_tree_unlock(left); 12985f39d397SChris Mason free_extent_buffer(left); 1299e66f709bSChris Mason } 1300e66f709bSChris Mason return 0; 1301e66f709bSChris Mason } 1302925baeddSChris Mason btrfs_tree_unlock(left); 13035f39d397SChris Mason free_extent_buffer(left); 1304e66f709bSChris Mason } 1305e66f709bSChris Mason 1306e66f709bSChris Mason /* 1307e66f709bSChris Mason * then try to empty the right most buffer into the middle 1308e66f709bSChris Mason */ 13099cf14029SJosef Bacik if (pslot + 1 < btrfs_header_nritems(parent)) { 131033ade1f8SChris Mason u32 right_nr; 1311b4ce94deSChris Mason 13129cf14029SJosef Bacik right = btrfs_read_node_slot(parent, pslot + 1); 13139cf14029SJosef Bacik if (IS_ERR(right)) 13149cf14029SJosef Bacik return PTR_ERR(right); 13159cf14029SJosef Bacik 1316bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT); 1317b4ce94deSChris Mason 13185f39d397SChris Mason right_nr = btrfs_header_nritems(right); 13190b246afaSJeff Mahoney if (right_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) { 132033ade1f8SChris Mason wret = 1; 132133ade1f8SChris Mason } else { 13225f39d397SChris Mason ret = btrfs_cow_block(trans, root, right, 13235f39d397SChris Mason parent, pslot + 1, 1324bf59a5a2SJosef Bacik &right, BTRFS_NESTING_RIGHT_COW); 132554aa1f4dSChris Mason if (ret) 132654aa1f4dSChris Mason wret = 1; 132754aa1f4dSChris Mason else { 132855d32ed8SDavid Sterba wret = balance_node_right(trans, right, mid); 132933ade1f8SChris Mason } 133054aa1f4dSChris Mason } 1331e66f709bSChris Mason if (wret < 0) 1332e66f709bSChris Mason ret = wret; 1333e66f709bSChris Mason if (wret == 0) { 13345f39d397SChris Mason struct btrfs_disk_key disk_key; 13355f39d397SChris Mason 13365f39d397SChris Mason btrfs_node_key(right, &disk_key, 0); 1337f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, pslot + 1, 133833cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 13390e82bcfeSDavid Sterba BUG_ON(ret < 0); 13405f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot + 1); 13415f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 13425f39d397SChris Mason 13435f39d397SChris Mason if (btrfs_header_nritems(mid) <= orig_slot) { 13445f39d397SChris Mason path->nodes[level] = right; 1345e66f709bSChris Mason path->slots[level + 1] += 1; 1346e66f709bSChris Mason path->slots[level] = orig_slot - 13475f39d397SChris Mason btrfs_header_nritems(mid); 1348925baeddSChris Mason btrfs_tree_unlock(mid); 13495f39d397SChris Mason free_extent_buffer(mid); 1350e66f709bSChris Mason } else { 1351925baeddSChris Mason btrfs_tree_unlock(right); 13525f39d397SChris Mason free_extent_buffer(right); 1353e66f709bSChris Mason } 1354e66f709bSChris Mason return 0; 1355e66f709bSChris Mason } 1356925baeddSChris Mason btrfs_tree_unlock(right); 13575f39d397SChris Mason free_extent_buffer(right); 1358e66f709bSChris Mason } 1359e66f709bSChris Mason return 1; 1360e66f709bSChris Mason } 1361e66f709bSChris Mason 136274123bd7SChris Mason /* 1363d352ac68SChris Mason * readahead one full node of leaves, finding things that are close 1364d352ac68SChris Mason * to the block in 'slot', and triggering ra on them. 13653c69faecSChris Mason */ 13662ff7e61eSJeff Mahoney static void reada_for_search(struct btrfs_fs_info *fs_info, 1367e02119d5SChris Mason struct btrfs_path *path, 136801f46658SChris Mason int level, int slot, u64 objectid) 13693c69faecSChris Mason { 13705f39d397SChris Mason struct extent_buffer *node; 137101f46658SChris Mason struct btrfs_disk_key disk_key; 13723c69faecSChris Mason u32 nritems; 13733c69faecSChris Mason u64 search; 1374a7175319SChris Mason u64 target; 13756b80053dSChris Mason u64 nread = 0; 1376ace75066SFilipe Manana u64 nread_max; 13776b80053dSChris Mason u32 nr; 13786b80053dSChris Mason u32 blocksize; 13796b80053dSChris Mason u32 nscan = 0; 1380db94535dSChris Mason 1381ace75066SFilipe Manana if (level != 1 && path->reada != READA_FORWARD_ALWAYS) 13823c69faecSChris Mason return; 13833c69faecSChris Mason 13846702ed49SChris Mason if (!path->nodes[level]) 13856702ed49SChris Mason return; 13866702ed49SChris Mason 13875f39d397SChris Mason node = path->nodes[level]; 1388925baeddSChris Mason 1389ace75066SFilipe Manana /* 1390ace75066SFilipe Manana * Since the time between visiting leaves is much shorter than the time 1391ace75066SFilipe Manana * between visiting nodes, limit read ahead of nodes to 1, to avoid too 1392ace75066SFilipe Manana * much IO at once (possibly random). 1393ace75066SFilipe Manana */ 1394ace75066SFilipe Manana if (path->reada == READA_FORWARD_ALWAYS) { 1395ace75066SFilipe Manana if (level > 1) 1396ace75066SFilipe Manana nread_max = node->fs_info->nodesize; 1397ace75066SFilipe Manana else 1398ace75066SFilipe Manana nread_max = SZ_128K; 1399ace75066SFilipe Manana } else { 1400ace75066SFilipe Manana nread_max = SZ_64K; 1401ace75066SFilipe Manana } 1402ace75066SFilipe Manana 14033c69faecSChris Mason search = btrfs_node_blockptr(node, slot); 14040b246afaSJeff Mahoney blocksize = fs_info->nodesize; 1405069a2e37SFilipe Manana if (path->reada != READA_FORWARD_ALWAYS) { 1406069a2e37SFilipe Manana struct extent_buffer *eb; 1407069a2e37SFilipe Manana 14080b246afaSJeff Mahoney eb = find_extent_buffer(fs_info, search); 14095f39d397SChris Mason if (eb) { 14105f39d397SChris Mason free_extent_buffer(eb); 14113c69faecSChris Mason return; 14123c69faecSChris Mason } 1413069a2e37SFilipe Manana } 14143c69faecSChris Mason 1415a7175319SChris Mason target = search; 14166b80053dSChris Mason 14175f39d397SChris Mason nritems = btrfs_header_nritems(node); 14186b80053dSChris Mason nr = slot; 141925b8b936SJosef Bacik 14203c69faecSChris Mason while (1) { 1421e4058b54SDavid Sterba if (path->reada == READA_BACK) { 14226b80053dSChris Mason if (nr == 0) 14233c69faecSChris Mason break; 14246b80053dSChris Mason nr--; 1425ace75066SFilipe Manana } else if (path->reada == READA_FORWARD || 1426ace75066SFilipe Manana path->reada == READA_FORWARD_ALWAYS) { 14276b80053dSChris Mason nr++; 14286b80053dSChris Mason if (nr >= nritems) 14296b80053dSChris Mason break; 14303c69faecSChris Mason } 1431e4058b54SDavid Sterba if (path->reada == READA_BACK && objectid) { 143201f46658SChris Mason btrfs_node_key(node, &disk_key, nr); 143301f46658SChris Mason if (btrfs_disk_key_objectid(&disk_key) != objectid) 143401f46658SChris Mason break; 143501f46658SChris Mason } 14366b80053dSChris Mason search = btrfs_node_blockptr(node, nr); 1437ace75066SFilipe Manana if (path->reada == READA_FORWARD_ALWAYS || 1438ace75066SFilipe Manana (search <= target && target - search <= 65536) || 1439a7175319SChris Mason (search > target && search - target <= 65536)) { 1440bfb484d9SJosef Bacik btrfs_readahead_node_child(node, nr); 14416b80053dSChris Mason nread += blocksize; 14423c69faecSChris Mason } 14436b80053dSChris Mason nscan++; 1444ace75066SFilipe Manana if (nread > nread_max || nscan > 32) 14456b80053dSChris Mason break; 14463c69faecSChris Mason } 14473c69faecSChris Mason } 1448925baeddSChris Mason 1449bfb484d9SJosef Bacik static noinline void reada_for_balance(struct btrfs_path *path, int level) 1450b4ce94deSChris Mason { 1451bfb484d9SJosef Bacik struct extent_buffer *parent; 1452b4ce94deSChris Mason int slot; 1453b4ce94deSChris Mason int nritems; 1454b4ce94deSChris Mason 14558c594ea8SChris Mason parent = path->nodes[level + 1]; 1456b4ce94deSChris Mason if (!parent) 14570b08851fSJosef Bacik return; 1458b4ce94deSChris Mason 1459b4ce94deSChris Mason nritems = btrfs_header_nritems(parent); 14608c594ea8SChris Mason slot = path->slots[level + 1]; 1461b4ce94deSChris Mason 1462bfb484d9SJosef Bacik if (slot > 0) 1463bfb484d9SJosef Bacik btrfs_readahead_node_child(parent, slot - 1); 1464bfb484d9SJosef Bacik if (slot + 1 < nritems) 1465bfb484d9SJosef Bacik btrfs_readahead_node_child(parent, slot + 1); 1466b4ce94deSChris Mason } 1467b4ce94deSChris Mason 1468b4ce94deSChris Mason 1469b4ce94deSChris Mason /* 1470d397712bSChris Mason * when we walk down the tree, it is usually safe to unlock the higher layers 1471d397712bSChris Mason * in the tree. The exceptions are when our path goes through slot 0, because 1472d397712bSChris Mason * operations on the tree might require changing key pointers higher up in the 1473d397712bSChris Mason * tree. 1474d352ac68SChris Mason * 1475d397712bSChris Mason * callers might also have set path->keep_locks, which tells this code to keep 1476d397712bSChris Mason * the lock if the path points to the last slot in the block. This is part of 1477d397712bSChris Mason * walking through the tree, and selecting the next slot in the higher block. 1478d352ac68SChris Mason * 1479d397712bSChris Mason * lowest_unlock sets the lowest level in the tree we're allowed to unlock. so 1480d397712bSChris Mason * if lowest_unlock is 1, level 0 won't be unlocked 1481d352ac68SChris Mason */ 1482e02119d5SChris Mason static noinline void unlock_up(struct btrfs_path *path, int level, 1483f7c79f30SChris Mason int lowest_unlock, int min_write_lock_level, 1484f7c79f30SChris Mason int *write_lock_level) 1485925baeddSChris Mason { 1486925baeddSChris Mason int i; 1487925baeddSChris Mason int skip_level = level; 1488c1227996SNikolay Borisov bool check_skip = true; 1489925baeddSChris Mason 1490925baeddSChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 1491925baeddSChris Mason if (!path->nodes[i]) 1492925baeddSChris Mason break; 1493925baeddSChris Mason if (!path->locks[i]) 1494925baeddSChris Mason break; 1495c1227996SNikolay Borisov 1496c1227996SNikolay Borisov if (check_skip) { 1497c1227996SNikolay Borisov if (path->slots[i] == 0) { 1498925baeddSChris Mason skip_level = i + 1; 1499925baeddSChris Mason continue; 1500925baeddSChris Mason } 1501c1227996SNikolay Borisov 1502c1227996SNikolay Borisov if (path->keep_locks) { 1503925baeddSChris Mason u32 nritems; 1504c1227996SNikolay Borisov 1505c1227996SNikolay Borisov nritems = btrfs_header_nritems(path->nodes[i]); 1506051e1b9fSChris Mason if (nritems < 1 || path->slots[i] >= nritems - 1) { 1507925baeddSChris Mason skip_level = i + 1; 1508925baeddSChris Mason continue; 1509925baeddSChris Mason } 1510925baeddSChris Mason } 1511c1227996SNikolay Borisov } 1512051e1b9fSChris Mason 1513d80bb3f9SLiu Bo if (i >= lowest_unlock && i > skip_level) { 1514c1227996SNikolay Borisov check_skip = false; 1515c1227996SNikolay Borisov btrfs_tree_unlock_rw(path->nodes[i], path->locks[i]); 1516925baeddSChris Mason path->locks[i] = 0; 1517f7c79f30SChris Mason if (write_lock_level && 1518f7c79f30SChris Mason i > min_write_lock_level && 1519f7c79f30SChris Mason i <= *write_lock_level) { 1520f7c79f30SChris Mason *write_lock_level = i - 1; 1521f7c79f30SChris Mason } 1522925baeddSChris Mason } 1523925baeddSChris Mason } 1524925baeddSChris Mason } 1525925baeddSChris Mason 15263c69faecSChris Mason /* 1527376a21d7SFilipe Manana * Helper function for btrfs_search_slot() and other functions that do a search 1528376a21d7SFilipe Manana * on a btree. The goal is to find a tree block in the cache (the radix tree at 1529376a21d7SFilipe Manana * fs_info->buffer_radix), but if we can't find it, or it's not up to date, read 1530376a21d7SFilipe Manana * its pages from disk. 1531c8c42864SChris Mason * 1532376a21d7SFilipe Manana * Returns -EAGAIN, with the path unlocked, if the caller needs to repeat the 1533376a21d7SFilipe Manana * whole btree search, starting again from the current root node. 1534c8c42864SChris Mason */ 1535c8c42864SChris Mason static int 1536d07b8528SLiu Bo read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, 1537c8c42864SChris Mason struct extent_buffer **eb_ret, int level, int slot, 1538cda79c54SDavid Sterba const struct btrfs_key *key) 1539c8c42864SChris Mason { 15400b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 1541789d6a3aSQu Wenruo struct btrfs_tree_parent_check check = { 0 }; 1542c8c42864SChris Mason u64 blocknr; 1543c8c42864SChris Mason u64 gen; 1544c8c42864SChris Mason struct extent_buffer *tmp; 154576a05b35SChris Mason int ret; 1546581c1760SQu Wenruo int parent_level; 1547b246666eSFilipe Manana bool unlock_up; 1548c8c42864SChris Mason 1549b246666eSFilipe Manana unlock_up = ((level + 1 < BTRFS_MAX_LEVEL) && p->locks[level + 1]); 1550213ff4b7SNikolay Borisov blocknr = btrfs_node_blockptr(*eb_ret, slot); 1551213ff4b7SNikolay Borisov gen = btrfs_node_ptr_generation(*eb_ret, slot); 1552213ff4b7SNikolay Borisov parent_level = btrfs_header_level(*eb_ret); 1553789d6a3aSQu Wenruo btrfs_node_key_to_cpu(*eb_ret, &check.first_key, slot); 1554789d6a3aSQu Wenruo check.has_first_key = true; 1555789d6a3aSQu Wenruo check.level = parent_level - 1; 1556789d6a3aSQu Wenruo check.transid = gen; 1557789d6a3aSQu Wenruo check.owner_root = root->root_key.objectid; 1558c8c42864SChris Mason 1559b246666eSFilipe Manana /* 1560b246666eSFilipe Manana * If we need to read an extent buffer from disk and we are holding locks 1561b246666eSFilipe Manana * on upper level nodes, we unlock all the upper nodes before reading the 1562b246666eSFilipe Manana * extent buffer, and then return -EAGAIN to the caller as it needs to 1563b246666eSFilipe Manana * restart the search. We don't release the lock on the current level 1564b246666eSFilipe Manana * because we need to walk this node to figure out which blocks to read. 1565b246666eSFilipe Manana */ 15660b246afaSJeff Mahoney tmp = find_extent_buffer(fs_info, blocknr); 1567cb44921aSChris Mason if (tmp) { 1568ace75066SFilipe Manana if (p->reada == READA_FORWARD_ALWAYS) 1569ace75066SFilipe Manana reada_for_search(fs_info, p, level, slot, key->objectid); 1570ace75066SFilipe Manana 1571b9fab919SChris Mason /* first we do an atomic uptodate check */ 1572b9fab919SChris Mason if (btrfs_buffer_uptodate(tmp, gen, 1) > 0) { 1573448de471SQu Wenruo /* 1574448de471SQu Wenruo * Do extra check for first_key, eb can be stale due to 1575448de471SQu Wenruo * being cached, read from scrub, or have multiple 1576448de471SQu Wenruo * parents (shared tree blocks). 1577448de471SQu Wenruo */ 1578e064d5e9SDavid Sterba if (btrfs_verify_level_key(tmp, 1579789d6a3aSQu Wenruo parent_level - 1, &check.first_key, gen)) { 1580448de471SQu Wenruo free_extent_buffer(tmp); 1581448de471SQu Wenruo return -EUCLEAN; 1582448de471SQu Wenruo } 1583c8c42864SChris Mason *eb_ret = tmp; 1584c8c42864SChris Mason return 0; 1585c8c42864SChris Mason } 1586bdf7c00eSJosef Bacik 1587857bc13fSJosef Bacik if (p->nowait) { 1588857bc13fSJosef Bacik free_extent_buffer(tmp); 1589857bc13fSJosef Bacik return -EAGAIN; 1590857bc13fSJosef Bacik } 1591857bc13fSJosef Bacik 1592b246666eSFilipe Manana if (unlock_up) 1593b246666eSFilipe Manana btrfs_unlock_up_safe(p, level + 1); 1594b246666eSFilipe Manana 1595b9fab919SChris Mason /* now we're allowed to do a blocking uptodate check */ 1596789d6a3aSQu Wenruo ret = btrfs_read_extent_buffer(tmp, &check); 15979a4ffa1bSQu Wenruo if (ret) { 1598cb44921aSChris Mason free_extent_buffer(tmp); 1599b3b4aa74SDavid Sterba btrfs_release_path(p); 1600cb44921aSChris Mason return -EIO; 1601cb44921aSChris Mason } 160288c602abSQu Wenruo if (btrfs_check_eb_owner(tmp, root->root_key.objectid)) { 160388c602abSQu Wenruo free_extent_buffer(tmp); 160488c602abSQu Wenruo btrfs_release_path(p); 160588c602abSQu Wenruo return -EUCLEAN; 160688c602abSQu Wenruo } 1607b246666eSFilipe Manana 1608b246666eSFilipe Manana if (unlock_up) 1609b246666eSFilipe Manana ret = -EAGAIN; 1610b246666eSFilipe Manana 1611b246666eSFilipe Manana goto out; 1612857bc13fSJosef Bacik } else if (p->nowait) { 1613857bc13fSJosef Bacik return -EAGAIN; 16149a4ffa1bSQu Wenruo } 1615c8c42864SChris Mason 1616b246666eSFilipe Manana if (unlock_up) { 16178c594ea8SChris Mason btrfs_unlock_up_safe(p, level + 1); 16184bb59055SFilipe Manana ret = -EAGAIN; 16194bb59055SFilipe Manana } else { 16204bb59055SFilipe Manana ret = 0; 16214bb59055SFilipe Manana } 16228c594ea8SChris Mason 1623e4058b54SDavid Sterba if (p->reada != READA_NONE) 16242ff7e61eSJeff Mahoney reada_for_search(fs_info, p, level, slot, key->objectid); 1625c8c42864SChris Mason 1626789d6a3aSQu Wenruo tmp = read_tree_block(fs_info, blocknr, &check); 16274eb150d6SQu Wenruo if (IS_ERR(tmp)) { 16284eb150d6SQu Wenruo btrfs_release_path(p); 16294eb150d6SQu Wenruo return PTR_ERR(tmp); 16304eb150d6SQu Wenruo } 163176a05b35SChris Mason /* 163276a05b35SChris Mason * If the read above didn't mark this buffer up to date, 163376a05b35SChris Mason * it will never end up being up to date. Set ret to EIO now 163476a05b35SChris Mason * and give up so that our caller doesn't loop forever 163576a05b35SChris Mason * on our EAGAINs. 163676a05b35SChris Mason */ 1637e6a1d6fdSLiu Bo if (!extent_buffer_uptodate(tmp)) 163876a05b35SChris Mason ret = -EIO; 163902a3307aSLiu Bo 1640b246666eSFilipe Manana out: 16414bb59055SFilipe Manana if (ret == 0) { 16424bb59055SFilipe Manana *eb_ret = tmp; 16434bb59055SFilipe Manana } else { 16444bb59055SFilipe Manana free_extent_buffer(tmp); 164502a3307aSLiu Bo btrfs_release_path(p); 16464bb59055SFilipe Manana } 16474bb59055SFilipe Manana 164876a05b35SChris Mason return ret; 1649c8c42864SChris Mason } 1650c8c42864SChris Mason 1651c8c42864SChris Mason /* 1652c8c42864SChris Mason * helper function for btrfs_search_slot. This does all of the checks 1653c8c42864SChris Mason * for node-level blocks and does any balancing required based on 1654c8c42864SChris Mason * the ins_len. 1655c8c42864SChris Mason * 1656c8c42864SChris Mason * If no extra work was required, zero is returned. If we had to 1657c8c42864SChris Mason * drop the path, -EAGAIN is returned and btrfs_search_slot must 1658c8c42864SChris Mason * start over 1659c8c42864SChris Mason */ 1660c8c42864SChris Mason static int 1661c8c42864SChris Mason setup_nodes_for_search(struct btrfs_trans_handle *trans, 1662c8c42864SChris Mason struct btrfs_root *root, struct btrfs_path *p, 1663bd681513SChris Mason struct extent_buffer *b, int level, int ins_len, 1664bd681513SChris Mason int *write_lock_level) 1665c8c42864SChris Mason { 16660b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 166795b982deSNikolay Borisov int ret = 0; 16680b246afaSJeff Mahoney 1669c8c42864SChris Mason if ((p->search_for_split || ins_len > 0) && btrfs_header_nritems(b) >= 16700b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3) { 1671c8c42864SChris Mason 1672bd681513SChris Mason if (*write_lock_level < level + 1) { 1673bd681513SChris Mason *write_lock_level = level + 1; 1674bd681513SChris Mason btrfs_release_path(p); 167595b982deSNikolay Borisov return -EAGAIN; 1676bd681513SChris Mason } 1677bd681513SChris Mason 1678bfb484d9SJosef Bacik reada_for_balance(p, level); 167995b982deSNikolay Borisov ret = split_node(trans, root, p, level); 1680c8c42864SChris Mason 1681c8c42864SChris Mason b = p->nodes[level]; 1682c8c42864SChris Mason } else if (ins_len < 0 && btrfs_header_nritems(b) < 16830b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 2) { 1684c8c42864SChris Mason 1685bd681513SChris Mason if (*write_lock_level < level + 1) { 1686bd681513SChris Mason *write_lock_level = level + 1; 1687bd681513SChris Mason btrfs_release_path(p); 168895b982deSNikolay Borisov return -EAGAIN; 1689bd681513SChris Mason } 1690bd681513SChris Mason 1691bfb484d9SJosef Bacik reada_for_balance(p, level); 169295b982deSNikolay Borisov ret = balance_level(trans, root, p, level); 169395b982deSNikolay Borisov if (ret) 169495b982deSNikolay Borisov return ret; 1695c8c42864SChris Mason 1696c8c42864SChris Mason b = p->nodes[level]; 1697c8c42864SChris Mason if (!b) { 1698b3b4aa74SDavid Sterba btrfs_release_path(p); 169995b982deSNikolay Borisov return -EAGAIN; 1700c8c42864SChris Mason } 1701c8c42864SChris Mason BUG_ON(btrfs_header_nritems(b) == 1); 1702c8c42864SChris Mason } 1703c8c42864SChris Mason return ret; 1704c8c42864SChris Mason } 1705c8c42864SChris Mason 1706381cf658SDavid Sterba int btrfs_find_item(struct btrfs_root *fs_root, struct btrfs_path *path, 1707e33d5c3dSKelley Nielsen u64 iobjectid, u64 ioff, u8 key_type, 1708e33d5c3dSKelley Nielsen struct btrfs_key *found_key) 1709e33d5c3dSKelley Nielsen { 1710e33d5c3dSKelley Nielsen int ret; 1711e33d5c3dSKelley Nielsen struct btrfs_key key; 1712e33d5c3dSKelley Nielsen struct extent_buffer *eb; 1713381cf658SDavid Sterba 1714381cf658SDavid Sterba ASSERT(path); 17151d4c08e0SDavid Sterba ASSERT(found_key); 1716e33d5c3dSKelley Nielsen 1717e33d5c3dSKelley Nielsen key.type = key_type; 1718e33d5c3dSKelley Nielsen key.objectid = iobjectid; 1719e33d5c3dSKelley Nielsen key.offset = ioff; 1720e33d5c3dSKelley Nielsen 1721e33d5c3dSKelley Nielsen ret = btrfs_search_slot(NULL, fs_root, &key, path, 0, 0); 17221d4c08e0SDavid Sterba if (ret < 0) 1723e33d5c3dSKelley Nielsen return ret; 1724e33d5c3dSKelley Nielsen 1725e33d5c3dSKelley Nielsen eb = path->nodes[0]; 1726e33d5c3dSKelley Nielsen if (ret && path->slots[0] >= btrfs_header_nritems(eb)) { 1727e33d5c3dSKelley Nielsen ret = btrfs_next_leaf(fs_root, path); 1728e33d5c3dSKelley Nielsen if (ret) 1729e33d5c3dSKelley Nielsen return ret; 1730e33d5c3dSKelley Nielsen eb = path->nodes[0]; 1731e33d5c3dSKelley Nielsen } 1732e33d5c3dSKelley Nielsen 1733e33d5c3dSKelley Nielsen btrfs_item_key_to_cpu(eb, found_key, path->slots[0]); 1734e33d5c3dSKelley Nielsen if (found_key->type != key.type || 1735e33d5c3dSKelley Nielsen found_key->objectid != key.objectid) 1736e33d5c3dSKelley Nielsen return 1; 1737e33d5c3dSKelley Nielsen 1738e33d5c3dSKelley Nielsen return 0; 1739e33d5c3dSKelley Nielsen } 1740e33d5c3dSKelley Nielsen 17411fc28d8eSLiu Bo static struct extent_buffer *btrfs_search_slot_get_root(struct btrfs_root *root, 17421fc28d8eSLiu Bo struct btrfs_path *p, 17431fc28d8eSLiu Bo int write_lock_level) 17441fc28d8eSLiu Bo { 17451fc28d8eSLiu Bo struct extent_buffer *b; 1746120de408SJosef Bacik int root_lock = 0; 17471fc28d8eSLiu Bo int level = 0; 17481fc28d8eSLiu Bo 17491fc28d8eSLiu Bo if (p->search_commit_root) { 17501fc28d8eSLiu Bo b = root->commit_root; 175167439dadSDavid Sterba atomic_inc(&b->refs); 17521fc28d8eSLiu Bo level = btrfs_header_level(b); 1753f9ddfd05SLiu Bo /* 1754f9ddfd05SLiu Bo * Ensure that all callers have set skip_locking when 1755f9ddfd05SLiu Bo * p->search_commit_root = 1. 1756f9ddfd05SLiu Bo */ 1757f9ddfd05SLiu Bo ASSERT(p->skip_locking == 1); 17581fc28d8eSLiu Bo 17591fc28d8eSLiu Bo goto out; 17601fc28d8eSLiu Bo } 17611fc28d8eSLiu Bo 17621fc28d8eSLiu Bo if (p->skip_locking) { 17631fc28d8eSLiu Bo b = btrfs_root_node(root); 17641fc28d8eSLiu Bo level = btrfs_header_level(b); 17651fc28d8eSLiu Bo goto out; 17661fc28d8eSLiu Bo } 17671fc28d8eSLiu Bo 1768120de408SJosef Bacik /* We try very hard to do read locks on the root */ 1769120de408SJosef Bacik root_lock = BTRFS_READ_LOCK; 1770120de408SJosef Bacik 17711fc28d8eSLiu Bo /* 1772662c653bSLiu Bo * If the level is set to maximum, we can skip trying to get the read 1773662c653bSLiu Bo * lock. 1774662c653bSLiu Bo */ 1775662c653bSLiu Bo if (write_lock_level < BTRFS_MAX_LEVEL) { 1776662c653bSLiu Bo /* 1777662c653bSLiu Bo * We don't know the level of the root node until we actually 1778662c653bSLiu Bo * have it read locked 17791fc28d8eSLiu Bo */ 1780857bc13fSJosef Bacik if (p->nowait) { 1781857bc13fSJosef Bacik b = btrfs_try_read_lock_root_node(root); 1782857bc13fSJosef Bacik if (IS_ERR(b)) 1783857bc13fSJosef Bacik return b; 1784857bc13fSJosef Bacik } else { 17851bb96598SJosef Bacik b = btrfs_read_lock_root_node(root); 1786857bc13fSJosef Bacik } 17871fc28d8eSLiu Bo level = btrfs_header_level(b); 17881fc28d8eSLiu Bo if (level > write_lock_level) 17891fc28d8eSLiu Bo goto out; 17901fc28d8eSLiu Bo 1791662c653bSLiu Bo /* Whoops, must trade for write lock */ 17921fc28d8eSLiu Bo btrfs_tree_read_unlock(b); 17931fc28d8eSLiu Bo free_extent_buffer(b); 1794662c653bSLiu Bo } 1795662c653bSLiu Bo 17961fc28d8eSLiu Bo b = btrfs_lock_root_node(root); 17971fc28d8eSLiu Bo root_lock = BTRFS_WRITE_LOCK; 17981fc28d8eSLiu Bo 17991fc28d8eSLiu Bo /* The level might have changed, check again */ 18001fc28d8eSLiu Bo level = btrfs_header_level(b); 18011fc28d8eSLiu Bo 18021fc28d8eSLiu Bo out: 1803120de408SJosef Bacik /* 1804120de408SJosef Bacik * The root may have failed to write out at some point, and thus is no 1805120de408SJosef Bacik * longer valid, return an error in this case. 1806120de408SJosef Bacik */ 1807120de408SJosef Bacik if (!extent_buffer_uptodate(b)) { 1808120de408SJosef Bacik if (root_lock) 1809120de408SJosef Bacik btrfs_tree_unlock_rw(b, root_lock); 1810120de408SJosef Bacik free_extent_buffer(b); 1811120de408SJosef Bacik return ERR_PTR(-EIO); 1812120de408SJosef Bacik } 1813120de408SJosef Bacik 18141fc28d8eSLiu Bo p->nodes[level] = b; 18151fc28d8eSLiu Bo if (!p->skip_locking) 18161fc28d8eSLiu Bo p->locks[level] = root_lock; 18171fc28d8eSLiu Bo /* 18181fc28d8eSLiu Bo * Callers are responsible for dropping b's references. 18191fc28d8eSLiu Bo */ 18201fc28d8eSLiu Bo return b; 18211fc28d8eSLiu Bo } 18221fc28d8eSLiu Bo 1823d96b3424SFilipe Manana /* 1824d96b3424SFilipe Manana * Replace the extent buffer at the lowest level of the path with a cloned 1825d96b3424SFilipe Manana * version. The purpose is to be able to use it safely, after releasing the 1826d96b3424SFilipe Manana * commit root semaphore, even if relocation is happening in parallel, the 1827d96b3424SFilipe Manana * transaction used for relocation is committed and the extent buffer is 1828d96b3424SFilipe Manana * reallocated in the next transaction. 1829d96b3424SFilipe Manana * 1830d96b3424SFilipe Manana * This is used in a context where the caller does not prevent transaction 1831d96b3424SFilipe Manana * commits from happening, either by holding a transaction handle or holding 1832d96b3424SFilipe Manana * some lock, while it's doing searches through a commit root. 1833d96b3424SFilipe Manana * At the moment it's only used for send operations. 1834d96b3424SFilipe Manana */ 1835d96b3424SFilipe Manana static int finish_need_commit_sem_search(struct btrfs_path *path) 1836d96b3424SFilipe Manana { 1837d96b3424SFilipe Manana const int i = path->lowest_level; 1838d96b3424SFilipe Manana const int slot = path->slots[i]; 1839d96b3424SFilipe Manana struct extent_buffer *lowest = path->nodes[i]; 1840d96b3424SFilipe Manana struct extent_buffer *clone; 1841d96b3424SFilipe Manana 1842d96b3424SFilipe Manana ASSERT(path->need_commit_sem); 1843d96b3424SFilipe Manana 1844d96b3424SFilipe Manana if (!lowest) 1845d96b3424SFilipe Manana return 0; 1846d96b3424SFilipe Manana 1847d96b3424SFilipe Manana lockdep_assert_held_read(&lowest->fs_info->commit_root_sem); 1848d96b3424SFilipe Manana 1849d96b3424SFilipe Manana clone = btrfs_clone_extent_buffer(lowest); 1850d96b3424SFilipe Manana if (!clone) 1851d96b3424SFilipe Manana return -ENOMEM; 1852d96b3424SFilipe Manana 1853d96b3424SFilipe Manana btrfs_release_path(path); 1854d96b3424SFilipe Manana path->nodes[i] = clone; 1855d96b3424SFilipe Manana path->slots[i] = slot; 1856d96b3424SFilipe Manana 1857d96b3424SFilipe Manana return 0; 1858d96b3424SFilipe Manana } 18591fc28d8eSLiu Bo 1860e2e58d0fSFilipe Manana static inline int search_for_key_slot(struct extent_buffer *eb, 1861e2e58d0fSFilipe Manana int search_low_slot, 1862e2e58d0fSFilipe Manana const struct btrfs_key *key, 1863e2e58d0fSFilipe Manana int prev_cmp, 1864e2e58d0fSFilipe Manana int *slot) 1865e2e58d0fSFilipe Manana { 1866e2e58d0fSFilipe Manana /* 1867e2e58d0fSFilipe Manana * If a previous call to btrfs_bin_search() on a parent node returned an 1868e2e58d0fSFilipe Manana * exact match (prev_cmp == 0), we can safely assume the target key will 1869e2e58d0fSFilipe Manana * always be at slot 0 on lower levels, since each key pointer 1870e2e58d0fSFilipe Manana * (struct btrfs_key_ptr) refers to the lowest key accessible from the 1871e2e58d0fSFilipe Manana * subtree it points to. Thus we can skip searching lower levels. 1872e2e58d0fSFilipe Manana */ 1873e2e58d0fSFilipe Manana if (prev_cmp == 0) { 1874e2e58d0fSFilipe Manana *slot = 0; 1875e2e58d0fSFilipe Manana return 0; 1876e2e58d0fSFilipe Manana } 1877e2e58d0fSFilipe Manana 1878fdf8d595SAnand Jain return btrfs_bin_search(eb, search_low_slot, key, slot); 1879e2e58d0fSFilipe Manana } 1880e2e58d0fSFilipe Manana 1881109324cfSFilipe Manana static int search_leaf(struct btrfs_trans_handle *trans, 1882109324cfSFilipe Manana struct btrfs_root *root, 1883109324cfSFilipe Manana const struct btrfs_key *key, 1884109324cfSFilipe Manana struct btrfs_path *path, 1885109324cfSFilipe Manana int ins_len, 1886109324cfSFilipe Manana int prev_cmp) 1887109324cfSFilipe Manana { 1888109324cfSFilipe Manana struct extent_buffer *leaf = path->nodes[0]; 1889109324cfSFilipe Manana int leaf_free_space = -1; 1890109324cfSFilipe Manana int search_low_slot = 0; 1891109324cfSFilipe Manana int ret; 1892109324cfSFilipe Manana bool do_bin_search = true; 1893109324cfSFilipe Manana 1894109324cfSFilipe Manana /* 1895109324cfSFilipe Manana * If we are doing an insertion, the leaf has enough free space and the 1896109324cfSFilipe Manana * destination slot for the key is not slot 0, then we can unlock our 1897109324cfSFilipe Manana * write lock on the parent, and any other upper nodes, before doing the 1898109324cfSFilipe Manana * binary search on the leaf (with search_for_key_slot()), allowing other 1899109324cfSFilipe Manana * tasks to lock the parent and any other upper nodes. 1900109324cfSFilipe Manana */ 1901109324cfSFilipe Manana if (ins_len > 0) { 1902109324cfSFilipe Manana /* 1903109324cfSFilipe Manana * Cache the leaf free space, since we will need it later and it 1904109324cfSFilipe Manana * will not change until then. 1905109324cfSFilipe Manana */ 1906109324cfSFilipe Manana leaf_free_space = btrfs_leaf_free_space(leaf); 1907109324cfSFilipe Manana 1908109324cfSFilipe Manana /* 1909109324cfSFilipe Manana * !path->locks[1] means we have a single node tree, the leaf is 1910109324cfSFilipe Manana * the root of the tree. 1911109324cfSFilipe Manana */ 1912109324cfSFilipe Manana if (path->locks[1] && leaf_free_space >= ins_len) { 1913109324cfSFilipe Manana struct btrfs_disk_key first_key; 1914109324cfSFilipe Manana 1915109324cfSFilipe Manana ASSERT(btrfs_header_nritems(leaf) > 0); 1916109324cfSFilipe Manana btrfs_item_key(leaf, &first_key, 0); 1917109324cfSFilipe Manana 1918109324cfSFilipe Manana /* 1919109324cfSFilipe Manana * Doing the extra comparison with the first key is cheap, 1920109324cfSFilipe Manana * taking into account that the first key is very likely 1921109324cfSFilipe Manana * already in a cache line because it immediately follows 1922109324cfSFilipe Manana * the extent buffer's header and we have recently accessed 1923109324cfSFilipe Manana * the header's level field. 1924109324cfSFilipe Manana */ 1925109324cfSFilipe Manana ret = comp_keys(&first_key, key); 1926109324cfSFilipe Manana if (ret < 0) { 1927109324cfSFilipe Manana /* 1928109324cfSFilipe Manana * The first key is smaller than the key we want 1929109324cfSFilipe Manana * to insert, so we are safe to unlock all upper 1930109324cfSFilipe Manana * nodes and we have to do the binary search. 1931109324cfSFilipe Manana * 1932109324cfSFilipe Manana * We do use btrfs_unlock_up_safe() and not 1933109324cfSFilipe Manana * unlock_up() because the later does not unlock 1934109324cfSFilipe Manana * nodes with a slot of 0 - we can safely unlock 1935109324cfSFilipe Manana * any node even if its slot is 0 since in this 1936109324cfSFilipe Manana * case the key does not end up at slot 0 of the 1937109324cfSFilipe Manana * leaf and there's no need to split the leaf. 1938109324cfSFilipe Manana */ 1939109324cfSFilipe Manana btrfs_unlock_up_safe(path, 1); 1940109324cfSFilipe Manana search_low_slot = 1; 1941109324cfSFilipe Manana } else { 1942109324cfSFilipe Manana /* 1943109324cfSFilipe Manana * The first key is >= then the key we want to 1944109324cfSFilipe Manana * insert, so we can skip the binary search as 1945109324cfSFilipe Manana * the target key will be at slot 0. 1946109324cfSFilipe Manana * 1947109324cfSFilipe Manana * We can not unlock upper nodes when the key is 1948109324cfSFilipe Manana * less than the first key, because we will need 1949109324cfSFilipe Manana * to update the key at slot 0 of the parent node 1950109324cfSFilipe Manana * and possibly of other upper nodes too. 1951109324cfSFilipe Manana * If the key matches the first key, then we can 1952109324cfSFilipe Manana * unlock all the upper nodes, using 1953109324cfSFilipe Manana * btrfs_unlock_up_safe() instead of unlock_up() 1954109324cfSFilipe Manana * as stated above. 1955109324cfSFilipe Manana */ 1956109324cfSFilipe Manana if (ret == 0) 1957109324cfSFilipe Manana btrfs_unlock_up_safe(path, 1); 1958109324cfSFilipe Manana /* 1959109324cfSFilipe Manana * ret is already 0 or 1, matching the result of 1960109324cfSFilipe Manana * a btrfs_bin_search() call, so there is no need 1961109324cfSFilipe Manana * to adjust it. 1962109324cfSFilipe Manana */ 1963109324cfSFilipe Manana do_bin_search = false; 1964109324cfSFilipe Manana path->slots[0] = 0; 1965109324cfSFilipe Manana } 1966109324cfSFilipe Manana } 1967109324cfSFilipe Manana } 1968109324cfSFilipe Manana 1969109324cfSFilipe Manana if (do_bin_search) { 1970109324cfSFilipe Manana ret = search_for_key_slot(leaf, search_low_slot, key, 1971109324cfSFilipe Manana prev_cmp, &path->slots[0]); 1972109324cfSFilipe Manana if (ret < 0) 1973109324cfSFilipe Manana return ret; 1974109324cfSFilipe Manana } 1975109324cfSFilipe Manana 1976109324cfSFilipe Manana if (ins_len > 0) { 1977109324cfSFilipe Manana /* 1978109324cfSFilipe Manana * Item key already exists. In this case, if we are allowed to 1979109324cfSFilipe Manana * insert the item (for example, in dir_item case, item key 1980109324cfSFilipe Manana * collision is allowed), it will be merged with the original 1981109324cfSFilipe Manana * item. Only the item size grows, no new btrfs item will be 1982109324cfSFilipe Manana * added. If search_for_extension is not set, ins_len already 1983109324cfSFilipe Manana * accounts the size btrfs_item, deduct it here so leaf space 1984109324cfSFilipe Manana * check will be correct. 1985109324cfSFilipe Manana */ 1986109324cfSFilipe Manana if (ret == 0 && !path->search_for_extension) { 1987109324cfSFilipe Manana ASSERT(ins_len >= sizeof(struct btrfs_item)); 1988109324cfSFilipe Manana ins_len -= sizeof(struct btrfs_item); 1989109324cfSFilipe Manana } 1990109324cfSFilipe Manana 1991109324cfSFilipe Manana ASSERT(leaf_free_space >= 0); 1992109324cfSFilipe Manana 1993109324cfSFilipe Manana if (leaf_free_space < ins_len) { 1994109324cfSFilipe Manana int err; 1995109324cfSFilipe Manana 1996109324cfSFilipe Manana err = split_leaf(trans, root, key, path, ins_len, 1997109324cfSFilipe Manana (ret == 0)); 1998bb8e9a60SFilipe Manana ASSERT(err <= 0); 1999bb8e9a60SFilipe Manana if (WARN_ON(err > 0)) 2000bb8e9a60SFilipe Manana err = -EUCLEAN; 2001109324cfSFilipe Manana if (err) 2002109324cfSFilipe Manana ret = err; 2003109324cfSFilipe Manana } 2004109324cfSFilipe Manana } 2005109324cfSFilipe Manana 2006109324cfSFilipe Manana return ret; 2007109324cfSFilipe Manana } 2008109324cfSFilipe Manana 2009c8c42864SChris Mason /* 20104271eceaSNikolay Borisov * btrfs_search_slot - look for a key in a tree and perform necessary 20114271eceaSNikolay Borisov * modifications to preserve tree invariants. 201274123bd7SChris Mason * 20134271eceaSNikolay Borisov * @trans: Handle of transaction, used when modifying the tree 20144271eceaSNikolay Borisov * @p: Holds all btree nodes along the search path 20154271eceaSNikolay Borisov * @root: The root node of the tree 20164271eceaSNikolay Borisov * @key: The key we are looking for 20179a664971Sethanwu * @ins_len: Indicates purpose of search: 20189a664971Sethanwu * >0 for inserts it's size of item inserted (*) 20199a664971Sethanwu * <0 for deletions 20209a664971Sethanwu * 0 for plain searches, not modifying the tree 20219a664971Sethanwu * 20229a664971Sethanwu * (*) If size of item inserted doesn't include 20239a664971Sethanwu * sizeof(struct btrfs_item), then p->search_for_extension must 20249a664971Sethanwu * be set. 20254271eceaSNikolay Borisov * @cow: boolean should CoW operations be performed. Must always be 1 20264271eceaSNikolay Borisov * when modifying the tree. 202797571fd0SChris Mason * 20284271eceaSNikolay Borisov * If @ins_len > 0, nodes and leaves will be split as we walk down the tree. 20294271eceaSNikolay Borisov * If @ins_len < 0, nodes will be merged as we walk down the tree (if possible) 20304271eceaSNikolay Borisov * 20314271eceaSNikolay Borisov * If @key is found, 0 is returned and you can find the item in the leaf level 20324271eceaSNikolay Borisov * of the path (level 0) 20334271eceaSNikolay Borisov * 20344271eceaSNikolay Borisov * If @key isn't found, 1 is returned and the leaf level of the path (level 0) 20354271eceaSNikolay Borisov * points to the slot where it should be inserted 20364271eceaSNikolay Borisov * 20374271eceaSNikolay Borisov * If an error is encountered while searching the tree a negative error number 20384271eceaSNikolay Borisov * is returned 203974123bd7SChris Mason */ 2040310712b2SOmar Sandoval int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root, 2041310712b2SOmar Sandoval const struct btrfs_key *key, struct btrfs_path *p, 2042310712b2SOmar Sandoval int ins_len, int cow) 2043be0e5c09SChris Mason { 2044d96b3424SFilipe Manana struct btrfs_fs_info *fs_info = root->fs_info; 20455f39d397SChris Mason struct extent_buffer *b; 2046be0e5c09SChris Mason int slot; 2047be0e5c09SChris Mason int ret; 204833c66f43SYan Zheng int err; 2049be0e5c09SChris Mason int level; 2050925baeddSChris Mason int lowest_unlock = 1; 2051bd681513SChris Mason /* everything at write_lock_level or lower must be write locked */ 2052bd681513SChris Mason int write_lock_level = 0; 20539f3a7427SChris Mason u8 lowest_level = 0; 2054f7c79f30SChris Mason int min_write_lock_level; 2055d7396f07SFilipe David Borba Manana int prev_cmp; 20569f3a7427SChris Mason 2057a4c853afSChenXiaoSong might_sleep(); 2058a4c853afSChenXiaoSong 20596702ed49SChris Mason lowest_level = p->lowest_level; 2060323ac95bSChris Mason WARN_ON(lowest_level && ins_len > 0); 206122b0ebdaSChris Mason WARN_ON(p->nodes[0] != NULL); 2062eb653de1SFilipe David Borba Manana BUG_ON(!cow && ins_len); 206325179201SJosef Bacik 2064857bc13fSJosef Bacik /* 2065857bc13fSJosef Bacik * For now only allow nowait for read only operations. There's no 2066857bc13fSJosef Bacik * strict reason why we can't, we just only need it for reads so it's 2067857bc13fSJosef Bacik * only implemented for reads. 2068857bc13fSJosef Bacik */ 2069857bc13fSJosef Bacik ASSERT(!p->nowait || !cow); 2070857bc13fSJosef Bacik 2071bd681513SChris Mason if (ins_len < 0) { 2072925baeddSChris Mason lowest_unlock = 2; 207365b51a00SChris Mason 2074bd681513SChris Mason /* when we are removing items, we might have to go up to level 2075bd681513SChris Mason * two as we update tree pointers Make sure we keep write 2076bd681513SChris Mason * for those levels as well 2077bd681513SChris Mason */ 2078bd681513SChris Mason write_lock_level = 2; 2079bd681513SChris Mason } else if (ins_len > 0) { 2080bd681513SChris Mason /* 2081bd681513SChris Mason * for inserting items, make sure we have a write lock on 2082bd681513SChris Mason * level 1 so we can update keys 2083bd681513SChris Mason */ 2084bd681513SChris Mason write_lock_level = 1; 2085bd681513SChris Mason } 2086bd681513SChris Mason 2087bd681513SChris Mason if (!cow) 2088bd681513SChris Mason write_lock_level = -1; 2089bd681513SChris Mason 209009a2a8f9SJosef Bacik if (cow && (p->keep_locks || p->lowest_level)) 2091bd681513SChris Mason write_lock_level = BTRFS_MAX_LEVEL; 2092bd681513SChris Mason 2093f7c79f30SChris Mason min_write_lock_level = write_lock_level; 2094f7c79f30SChris Mason 2095d96b3424SFilipe Manana if (p->need_commit_sem) { 2096d96b3424SFilipe Manana ASSERT(p->search_commit_root); 2097857bc13fSJosef Bacik if (p->nowait) { 2098857bc13fSJosef Bacik if (!down_read_trylock(&fs_info->commit_root_sem)) 2099857bc13fSJosef Bacik return -EAGAIN; 2100857bc13fSJosef Bacik } else { 2101d96b3424SFilipe Manana down_read(&fs_info->commit_root_sem); 2102d96b3424SFilipe Manana } 2103857bc13fSJosef Bacik } 2104d96b3424SFilipe Manana 2105bb803951SChris Mason again: 2106d7396f07SFilipe David Borba Manana prev_cmp = -1; 21071fc28d8eSLiu Bo b = btrfs_search_slot_get_root(root, p, write_lock_level); 2108be6821f8SFilipe Manana if (IS_ERR(b)) { 2109be6821f8SFilipe Manana ret = PTR_ERR(b); 2110be6821f8SFilipe Manana goto done; 2111be6821f8SFilipe Manana } 2112925baeddSChris Mason 2113eb60ceacSChris Mason while (b) { 2114f624d976SQu Wenruo int dec = 0; 2115f624d976SQu Wenruo 21165f39d397SChris Mason level = btrfs_header_level(b); 211765b51a00SChris Mason 211802217ed2SChris Mason if (cow) { 21199ea2c7c9SNikolay Borisov bool last_level = (level == (BTRFS_MAX_LEVEL - 1)); 21209ea2c7c9SNikolay Borisov 2121c8c42864SChris Mason /* 2122c8c42864SChris Mason * if we don't really need to cow this block 2123c8c42864SChris Mason * then we don't want to set the path blocking, 2124c8c42864SChris Mason * so we test it here 2125c8c42864SChris Mason */ 21265963ffcaSJosef Bacik if (!should_cow_block(trans, root, b)) 212765b51a00SChris Mason goto cow_done; 21285d4f98a2SYan Zheng 2129bd681513SChris Mason /* 2130bd681513SChris Mason * must have write locks on this node and the 2131bd681513SChris Mason * parent 2132bd681513SChris Mason */ 21335124e00eSJosef Bacik if (level > write_lock_level || 21345124e00eSJosef Bacik (level + 1 > write_lock_level && 21355124e00eSJosef Bacik level + 1 < BTRFS_MAX_LEVEL && 21365124e00eSJosef Bacik p->nodes[level + 1])) { 2137bd681513SChris Mason write_lock_level = level + 1; 2138bd681513SChris Mason btrfs_release_path(p); 2139bd681513SChris Mason goto again; 2140bd681513SChris Mason } 2141bd681513SChris Mason 21429ea2c7c9SNikolay Borisov if (last_level) 21439ea2c7c9SNikolay Borisov err = btrfs_cow_block(trans, root, b, NULL, 0, 21449631e4ccSJosef Bacik &b, 21459631e4ccSJosef Bacik BTRFS_NESTING_COW); 21469ea2c7c9SNikolay Borisov else 214733c66f43SYan Zheng err = btrfs_cow_block(trans, root, b, 2148e20d96d6SChris Mason p->nodes[level + 1], 21499631e4ccSJosef Bacik p->slots[level + 1], &b, 21509631e4ccSJosef Bacik BTRFS_NESTING_COW); 215133c66f43SYan Zheng if (err) { 215233c66f43SYan Zheng ret = err; 215365b51a00SChris Mason goto done; 215454aa1f4dSChris Mason } 215502217ed2SChris Mason } 215665b51a00SChris Mason cow_done: 2157eb60ceacSChris Mason p->nodes[level] = b; 2158b4ce94deSChris Mason 2159b4ce94deSChris Mason /* 2160b4ce94deSChris Mason * we have a lock on b and as long as we aren't changing 2161b4ce94deSChris Mason * the tree, there is no way to for the items in b to change. 2162b4ce94deSChris Mason * It is safe to drop the lock on our parent before we 2163b4ce94deSChris Mason * go through the expensive btree search on b. 2164b4ce94deSChris Mason * 2165eb653de1SFilipe David Borba Manana * If we're inserting or deleting (ins_len != 0), then we might 2166eb653de1SFilipe David Borba Manana * be changing slot zero, which may require changing the parent. 2167eb653de1SFilipe David Borba Manana * So, we can't drop the lock until after we know which slot 2168eb653de1SFilipe David Borba Manana * we're operating on. 2169b4ce94deSChris Mason */ 2170eb653de1SFilipe David Borba Manana if (!ins_len && !p->keep_locks) { 2171eb653de1SFilipe David Borba Manana int u = level + 1; 2172eb653de1SFilipe David Borba Manana 2173eb653de1SFilipe David Borba Manana if (u < BTRFS_MAX_LEVEL && p->locks[u]) { 2174eb653de1SFilipe David Borba Manana btrfs_tree_unlock_rw(p->nodes[u], p->locks[u]); 2175eb653de1SFilipe David Borba Manana p->locks[u] = 0; 2176eb653de1SFilipe David Borba Manana } 2177eb653de1SFilipe David Borba Manana } 2178b4ce94deSChris Mason 2179e2e58d0fSFilipe Manana if (level == 0) { 2180109324cfSFilipe Manana if (ins_len > 0) 2181e5e1c174SFilipe Manana ASSERT(write_lock_level >= 1); 2182bd681513SChris Mason 2183109324cfSFilipe Manana ret = search_leaf(trans, root, key, p, ins_len, prev_cmp); 2184459931ecSChris Mason if (!p->search_for_split) 2185f7c79f30SChris Mason unlock_up(p, level, lowest_unlock, 21864b6f8e96SLiu Bo min_write_lock_level, NULL); 218765b51a00SChris Mason goto done; 218865b51a00SChris Mason } 2189e2e58d0fSFilipe Manana 2190e2e58d0fSFilipe Manana ret = search_for_key_slot(b, 0, key, prev_cmp, &slot); 2191e2e58d0fSFilipe Manana if (ret < 0) 2192e2e58d0fSFilipe Manana goto done; 2193e2e58d0fSFilipe Manana prev_cmp = ret; 2194e2e58d0fSFilipe Manana 2195f624d976SQu Wenruo if (ret && slot > 0) { 2196f624d976SQu Wenruo dec = 1; 2197f624d976SQu Wenruo slot--; 2198f624d976SQu Wenruo } 2199f624d976SQu Wenruo p->slots[level] = slot; 2200f624d976SQu Wenruo err = setup_nodes_for_search(trans, root, p, b, level, ins_len, 2201f624d976SQu Wenruo &write_lock_level); 2202f624d976SQu Wenruo if (err == -EAGAIN) 2203f624d976SQu Wenruo goto again; 2204f624d976SQu Wenruo if (err) { 2205f624d976SQu Wenruo ret = err; 2206f624d976SQu Wenruo goto done; 2207f624d976SQu Wenruo } 2208f624d976SQu Wenruo b = p->nodes[level]; 2209f624d976SQu Wenruo slot = p->slots[level]; 2210f624d976SQu Wenruo 2211f624d976SQu Wenruo /* 2212f624d976SQu Wenruo * Slot 0 is special, if we change the key we have to update 2213f624d976SQu Wenruo * the parent pointer which means we must have a write lock on 2214f624d976SQu Wenruo * the parent 2215f624d976SQu Wenruo */ 2216f624d976SQu Wenruo if (slot == 0 && ins_len && write_lock_level < level + 1) { 2217f624d976SQu Wenruo write_lock_level = level + 1; 2218f624d976SQu Wenruo btrfs_release_path(p); 2219f624d976SQu Wenruo goto again; 2220f624d976SQu Wenruo } 2221f624d976SQu Wenruo 2222f624d976SQu Wenruo unlock_up(p, level, lowest_unlock, min_write_lock_level, 2223f624d976SQu Wenruo &write_lock_level); 2224f624d976SQu Wenruo 2225f624d976SQu Wenruo if (level == lowest_level) { 2226f624d976SQu Wenruo if (dec) 2227f624d976SQu Wenruo p->slots[level]++; 2228f624d976SQu Wenruo goto done; 2229f624d976SQu Wenruo } 2230f624d976SQu Wenruo 2231f624d976SQu Wenruo err = read_block_for_search(root, p, &b, level, slot, key); 2232f624d976SQu Wenruo if (err == -EAGAIN) 2233f624d976SQu Wenruo goto again; 2234f624d976SQu Wenruo if (err) { 2235f624d976SQu Wenruo ret = err; 2236f624d976SQu Wenruo goto done; 2237f624d976SQu Wenruo } 2238f624d976SQu Wenruo 2239f624d976SQu Wenruo if (!p->skip_locking) { 2240f624d976SQu Wenruo level = btrfs_header_level(b); 2241b40130b2SJosef Bacik 2242b40130b2SJosef Bacik btrfs_maybe_reset_lockdep_class(root, b); 2243b40130b2SJosef Bacik 2244f624d976SQu Wenruo if (level <= write_lock_level) { 2245f624d976SQu Wenruo btrfs_tree_lock(b); 2246f624d976SQu Wenruo p->locks[level] = BTRFS_WRITE_LOCK; 2247f624d976SQu Wenruo } else { 2248857bc13fSJosef Bacik if (p->nowait) { 2249857bc13fSJosef Bacik if (!btrfs_try_tree_read_lock(b)) { 2250857bc13fSJosef Bacik free_extent_buffer(b); 2251857bc13fSJosef Bacik ret = -EAGAIN; 2252857bc13fSJosef Bacik goto done; 2253857bc13fSJosef Bacik } 2254857bc13fSJosef Bacik } else { 2255fe596ca3SJosef Bacik btrfs_tree_read_lock(b); 2256857bc13fSJosef Bacik } 2257f624d976SQu Wenruo p->locks[level] = BTRFS_READ_LOCK; 2258f624d976SQu Wenruo } 2259f624d976SQu Wenruo p->nodes[level] = b; 2260f624d976SQu Wenruo } 226165b51a00SChris Mason } 226265b51a00SChris Mason ret = 1; 226365b51a00SChris Mason done: 22645f5bc6b1SFilipe Manana if (ret < 0 && !p->skip_release_on_error) 2265b3b4aa74SDavid Sterba btrfs_release_path(p); 2266d96b3424SFilipe Manana 2267d96b3424SFilipe Manana if (p->need_commit_sem) { 2268d96b3424SFilipe Manana int ret2; 2269d96b3424SFilipe Manana 2270d96b3424SFilipe Manana ret2 = finish_need_commit_sem_search(p); 2271d96b3424SFilipe Manana up_read(&fs_info->commit_root_sem); 2272d96b3424SFilipe Manana if (ret2) 2273d96b3424SFilipe Manana ret = ret2; 2274d96b3424SFilipe Manana } 2275d96b3424SFilipe Manana 2276be0e5c09SChris Mason return ret; 2277be0e5c09SChris Mason } 2278f75e2b79SJosef Bacik ALLOW_ERROR_INJECTION(btrfs_search_slot, ERRNO); 2279be0e5c09SChris Mason 228074123bd7SChris Mason /* 22815d9e75c4SJan Schmidt * Like btrfs_search_slot, this looks for a key in the given tree. It uses the 22825d9e75c4SJan Schmidt * current state of the tree together with the operations recorded in the tree 22835d9e75c4SJan Schmidt * modification log to search for the key in a previous version of this tree, as 22845d9e75c4SJan Schmidt * denoted by the time_seq parameter. 22855d9e75c4SJan Schmidt * 22865d9e75c4SJan Schmidt * Naturally, there is no support for insert, delete or cow operations. 22875d9e75c4SJan Schmidt * 22885d9e75c4SJan Schmidt * The resulting path and return value will be set up as if we called 22895d9e75c4SJan Schmidt * btrfs_search_slot at that point in time with ins_len and cow both set to 0. 22905d9e75c4SJan Schmidt */ 2291310712b2SOmar Sandoval int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key, 22925d9e75c4SJan Schmidt struct btrfs_path *p, u64 time_seq) 22935d9e75c4SJan Schmidt { 22940b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 22955d9e75c4SJan Schmidt struct extent_buffer *b; 22965d9e75c4SJan Schmidt int slot; 22975d9e75c4SJan Schmidt int ret; 22985d9e75c4SJan Schmidt int err; 22995d9e75c4SJan Schmidt int level; 23005d9e75c4SJan Schmidt int lowest_unlock = 1; 23015d9e75c4SJan Schmidt u8 lowest_level = 0; 23025d9e75c4SJan Schmidt 23035d9e75c4SJan Schmidt lowest_level = p->lowest_level; 23045d9e75c4SJan Schmidt WARN_ON(p->nodes[0] != NULL); 2305c922b016SStefan Roesch ASSERT(!p->nowait); 23065d9e75c4SJan Schmidt 23075d9e75c4SJan Schmidt if (p->search_commit_root) { 23085d9e75c4SJan Schmidt BUG_ON(time_seq); 23095d9e75c4SJan Schmidt return btrfs_search_slot(NULL, root, key, p, 0, 0); 23105d9e75c4SJan Schmidt } 23115d9e75c4SJan Schmidt 23125d9e75c4SJan Schmidt again: 2313f3a84ccdSFilipe Manana b = btrfs_get_old_root(root, time_seq); 2314315bed43SNikolay Borisov if (!b) { 2315315bed43SNikolay Borisov ret = -EIO; 2316315bed43SNikolay Borisov goto done; 2317315bed43SNikolay Borisov } 23185d9e75c4SJan Schmidt level = btrfs_header_level(b); 23195d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 23205d9e75c4SJan Schmidt 23215d9e75c4SJan Schmidt while (b) { 2322abe9339dSQu Wenruo int dec = 0; 2323abe9339dSQu Wenruo 23245d9e75c4SJan Schmidt level = btrfs_header_level(b); 23255d9e75c4SJan Schmidt p->nodes[level] = b; 23265d9e75c4SJan Schmidt 23275d9e75c4SJan Schmidt /* 23285d9e75c4SJan Schmidt * we have a lock on b and as long as we aren't changing 23295d9e75c4SJan Schmidt * the tree, there is no way to for the items in b to change. 23305d9e75c4SJan Schmidt * It is safe to drop the lock on our parent before we 23315d9e75c4SJan Schmidt * go through the expensive btree search on b. 23325d9e75c4SJan Schmidt */ 23335d9e75c4SJan Schmidt btrfs_unlock_up_safe(p, level + 1); 23345d9e75c4SJan Schmidt 2335fdf8d595SAnand Jain ret = btrfs_bin_search(b, 0, key, &slot); 2336cbca7d59SFilipe Manana if (ret < 0) 2337cbca7d59SFilipe Manana goto done; 23385d9e75c4SJan Schmidt 2339abe9339dSQu Wenruo if (level == 0) { 2340abe9339dSQu Wenruo p->slots[level] = slot; 2341abe9339dSQu Wenruo unlock_up(p, level, lowest_unlock, 0, NULL); 2342abe9339dSQu Wenruo goto done; 2343abe9339dSQu Wenruo } 2344abe9339dSQu Wenruo 23455d9e75c4SJan Schmidt if (ret && slot > 0) { 23465d9e75c4SJan Schmidt dec = 1; 2347abe9339dSQu Wenruo slot--; 23485d9e75c4SJan Schmidt } 23495d9e75c4SJan Schmidt p->slots[level] = slot; 23505d9e75c4SJan Schmidt unlock_up(p, level, lowest_unlock, 0, NULL); 23515d9e75c4SJan Schmidt 23525d9e75c4SJan Schmidt if (level == lowest_level) { 23535d9e75c4SJan Schmidt if (dec) 23545d9e75c4SJan Schmidt p->slots[level]++; 23555d9e75c4SJan Schmidt goto done; 23565d9e75c4SJan Schmidt } 23575d9e75c4SJan Schmidt 2358abe9339dSQu Wenruo err = read_block_for_search(root, p, &b, level, slot, key); 23595d9e75c4SJan Schmidt if (err == -EAGAIN) 23605d9e75c4SJan Schmidt goto again; 23615d9e75c4SJan Schmidt if (err) { 23625d9e75c4SJan Schmidt ret = err; 23635d9e75c4SJan Schmidt goto done; 23645d9e75c4SJan Schmidt } 23655d9e75c4SJan Schmidt 23665d9e75c4SJan Schmidt level = btrfs_header_level(b); 23675d9e75c4SJan Schmidt btrfs_tree_read_lock(b); 2368f3a84ccdSFilipe Manana b = btrfs_tree_mod_log_rewind(fs_info, p, b, time_seq); 2369db7f3436SJosef Bacik if (!b) { 2370db7f3436SJosef Bacik ret = -ENOMEM; 2371db7f3436SJosef Bacik goto done; 2372db7f3436SJosef Bacik } 23735d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 23745d9e75c4SJan Schmidt p->nodes[level] = b; 23755d9e75c4SJan Schmidt } 23765d9e75c4SJan Schmidt ret = 1; 23775d9e75c4SJan Schmidt done: 23785d9e75c4SJan Schmidt if (ret < 0) 23795d9e75c4SJan Schmidt btrfs_release_path(p); 23805d9e75c4SJan Schmidt 23815d9e75c4SJan Schmidt return ret; 23825d9e75c4SJan Schmidt } 23835d9e75c4SJan Schmidt 23845d9e75c4SJan Schmidt /* 2385f469c8bdSFilipe Manana * Search the tree again to find a leaf with smaller keys. 2386f469c8bdSFilipe Manana * Returns 0 if it found something. 2387f469c8bdSFilipe Manana * Returns 1 if there are no smaller keys. 2388f469c8bdSFilipe Manana * Returns < 0 on error. 2389f469c8bdSFilipe Manana * 2390f469c8bdSFilipe Manana * This may release the path, and so you may lose any locks held at the 2391f469c8bdSFilipe Manana * time you call it. 2392f469c8bdSFilipe Manana */ 2393f469c8bdSFilipe Manana static int btrfs_prev_leaf(struct btrfs_root *root, struct btrfs_path *path) 2394f469c8bdSFilipe Manana { 2395f469c8bdSFilipe Manana struct btrfs_key key; 2396f469c8bdSFilipe Manana struct btrfs_key orig_key; 2397f469c8bdSFilipe Manana struct btrfs_disk_key found_key; 2398f469c8bdSFilipe Manana int ret; 2399f469c8bdSFilipe Manana 2400f469c8bdSFilipe Manana btrfs_item_key_to_cpu(path->nodes[0], &key, 0); 2401f469c8bdSFilipe Manana orig_key = key; 2402f469c8bdSFilipe Manana 2403f469c8bdSFilipe Manana if (key.offset > 0) { 2404f469c8bdSFilipe Manana key.offset--; 2405f469c8bdSFilipe Manana } else if (key.type > 0) { 2406f469c8bdSFilipe Manana key.type--; 2407f469c8bdSFilipe Manana key.offset = (u64)-1; 2408f469c8bdSFilipe Manana } else if (key.objectid > 0) { 2409f469c8bdSFilipe Manana key.objectid--; 2410f469c8bdSFilipe Manana key.type = (u8)-1; 2411f469c8bdSFilipe Manana key.offset = (u64)-1; 2412f469c8bdSFilipe Manana } else { 2413f469c8bdSFilipe Manana return 1; 2414f469c8bdSFilipe Manana } 2415f469c8bdSFilipe Manana 2416f469c8bdSFilipe Manana btrfs_release_path(path); 2417f469c8bdSFilipe Manana ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 2418f469c8bdSFilipe Manana if (ret <= 0) 2419f469c8bdSFilipe Manana return ret; 2420f469c8bdSFilipe Manana 2421f469c8bdSFilipe Manana /* 2422f469c8bdSFilipe Manana * Previous key not found. Even if we were at slot 0 of the leaf we had 2423f469c8bdSFilipe Manana * before releasing the path and calling btrfs_search_slot(), we now may 2424f469c8bdSFilipe Manana * be in a slot pointing to the same original key - this can happen if 2425f469c8bdSFilipe Manana * after we released the path, one of more items were moved from a 2426f469c8bdSFilipe Manana * sibling leaf into the front of the leaf we had due to an insertion 2427f469c8bdSFilipe Manana * (see push_leaf_right()). 2428f469c8bdSFilipe Manana * If we hit this case and our slot is > 0 and just decrement the slot 2429f469c8bdSFilipe Manana * so that the caller does not process the same key again, which may or 2430f469c8bdSFilipe Manana * may not break the caller, depending on its logic. 2431f469c8bdSFilipe Manana */ 2432f469c8bdSFilipe Manana if (path->slots[0] < btrfs_header_nritems(path->nodes[0])) { 2433f469c8bdSFilipe Manana btrfs_item_key(path->nodes[0], &found_key, path->slots[0]); 2434f469c8bdSFilipe Manana ret = comp_keys(&found_key, &orig_key); 2435f469c8bdSFilipe Manana if (ret == 0) { 2436f469c8bdSFilipe Manana if (path->slots[0] > 0) { 2437f469c8bdSFilipe Manana path->slots[0]--; 2438f469c8bdSFilipe Manana return 0; 2439f469c8bdSFilipe Manana } 2440f469c8bdSFilipe Manana /* 2441f469c8bdSFilipe Manana * At slot 0, same key as before, it means orig_key is 2442f469c8bdSFilipe Manana * the lowest, leftmost, key in the tree. We're done. 2443f469c8bdSFilipe Manana */ 2444f469c8bdSFilipe Manana return 1; 2445f469c8bdSFilipe Manana } 2446f469c8bdSFilipe Manana } 2447f469c8bdSFilipe Manana 2448f469c8bdSFilipe Manana btrfs_item_key(path->nodes[0], &found_key, 0); 2449f469c8bdSFilipe Manana ret = comp_keys(&found_key, &key); 2450f469c8bdSFilipe Manana /* 2451f469c8bdSFilipe Manana * We might have had an item with the previous key in the tree right 2452f469c8bdSFilipe Manana * before we released our path. And after we released our path, that 2453f469c8bdSFilipe Manana * item might have been pushed to the first slot (0) of the leaf we 2454f469c8bdSFilipe Manana * were holding due to a tree balance. Alternatively, an item with the 2455f469c8bdSFilipe Manana * previous key can exist as the only element of a leaf (big fat item). 2456f469c8bdSFilipe Manana * Therefore account for these 2 cases, so that our callers (like 2457f469c8bdSFilipe Manana * btrfs_previous_item) don't miss an existing item with a key matching 2458f469c8bdSFilipe Manana * the previous key we computed above. 2459f469c8bdSFilipe Manana */ 2460f469c8bdSFilipe Manana if (ret <= 0) 2461f469c8bdSFilipe Manana return 0; 2462f469c8bdSFilipe Manana return 1; 2463f469c8bdSFilipe Manana } 2464f469c8bdSFilipe Manana 2465f469c8bdSFilipe Manana /* 24662f38b3e1SArne Jansen * helper to use instead of search slot if no exact match is needed but 24672f38b3e1SArne Jansen * instead the next or previous item should be returned. 24682f38b3e1SArne Jansen * When find_higher is true, the next higher item is returned, the next lower 24692f38b3e1SArne Jansen * otherwise. 24702f38b3e1SArne Jansen * When return_any and find_higher are both true, and no higher item is found, 24712f38b3e1SArne Jansen * return the next lower instead. 24722f38b3e1SArne Jansen * When return_any is true and find_higher is false, and no lower item is found, 24732f38b3e1SArne Jansen * return the next higher instead. 24742f38b3e1SArne Jansen * It returns 0 if any item is found, 1 if none is found (tree empty), and 24752f38b3e1SArne Jansen * < 0 on error 24762f38b3e1SArne Jansen */ 24772f38b3e1SArne Jansen int btrfs_search_slot_for_read(struct btrfs_root *root, 2478310712b2SOmar Sandoval const struct btrfs_key *key, 2479310712b2SOmar Sandoval struct btrfs_path *p, int find_higher, 2480310712b2SOmar Sandoval int return_any) 24812f38b3e1SArne Jansen { 24822f38b3e1SArne Jansen int ret; 24832f38b3e1SArne Jansen struct extent_buffer *leaf; 24842f38b3e1SArne Jansen 24852f38b3e1SArne Jansen again: 24862f38b3e1SArne Jansen ret = btrfs_search_slot(NULL, root, key, p, 0, 0); 24872f38b3e1SArne Jansen if (ret <= 0) 24882f38b3e1SArne Jansen return ret; 24892f38b3e1SArne Jansen /* 24902f38b3e1SArne Jansen * a return value of 1 means the path is at the position where the 24912f38b3e1SArne Jansen * item should be inserted. Normally this is the next bigger item, 24922f38b3e1SArne Jansen * but in case the previous item is the last in a leaf, path points 24932f38b3e1SArne Jansen * to the first free slot in the previous leaf, i.e. at an invalid 24942f38b3e1SArne Jansen * item. 24952f38b3e1SArne Jansen */ 24962f38b3e1SArne Jansen leaf = p->nodes[0]; 24972f38b3e1SArne Jansen 24982f38b3e1SArne Jansen if (find_higher) { 24992f38b3e1SArne Jansen if (p->slots[0] >= btrfs_header_nritems(leaf)) { 25002f38b3e1SArne Jansen ret = btrfs_next_leaf(root, p); 25012f38b3e1SArne Jansen if (ret <= 0) 25022f38b3e1SArne Jansen return ret; 25032f38b3e1SArne Jansen if (!return_any) 25042f38b3e1SArne Jansen return 1; 25052f38b3e1SArne Jansen /* 25062f38b3e1SArne Jansen * no higher item found, return the next 25072f38b3e1SArne Jansen * lower instead 25082f38b3e1SArne Jansen */ 25092f38b3e1SArne Jansen return_any = 0; 25102f38b3e1SArne Jansen find_higher = 0; 25112f38b3e1SArne Jansen btrfs_release_path(p); 25122f38b3e1SArne Jansen goto again; 25132f38b3e1SArne Jansen } 25142f38b3e1SArne Jansen } else { 25152f38b3e1SArne Jansen if (p->slots[0] == 0) { 25162f38b3e1SArne Jansen ret = btrfs_prev_leaf(root, p); 2517e6793769SArne Jansen if (ret < 0) 25182f38b3e1SArne Jansen return ret; 2519e6793769SArne Jansen if (!ret) { 252023c6bf6aSFilipe David Borba Manana leaf = p->nodes[0]; 252123c6bf6aSFilipe David Borba Manana if (p->slots[0] == btrfs_header_nritems(leaf)) 252223c6bf6aSFilipe David Borba Manana p->slots[0]--; 2523e6793769SArne Jansen return 0; 2524e6793769SArne Jansen } 25252f38b3e1SArne Jansen if (!return_any) 25262f38b3e1SArne Jansen return 1; 25272f38b3e1SArne Jansen /* 25282f38b3e1SArne Jansen * no lower item found, return the next 25292f38b3e1SArne Jansen * higher instead 25302f38b3e1SArne Jansen */ 25312f38b3e1SArne Jansen return_any = 0; 25322f38b3e1SArne Jansen find_higher = 1; 25332f38b3e1SArne Jansen btrfs_release_path(p); 25342f38b3e1SArne Jansen goto again; 2535e6793769SArne Jansen } else { 25362f38b3e1SArne Jansen --p->slots[0]; 25372f38b3e1SArne Jansen } 25382f38b3e1SArne Jansen } 25392f38b3e1SArne Jansen return 0; 25402f38b3e1SArne Jansen } 25412f38b3e1SArne Jansen 25422f38b3e1SArne Jansen /* 25430ff40a91SMarcos Paulo de Souza * Execute search and call btrfs_previous_item to traverse backwards if the item 25440ff40a91SMarcos Paulo de Souza * was not found. 25450ff40a91SMarcos Paulo de Souza * 25460ff40a91SMarcos Paulo de Souza * Return 0 if found, 1 if not found and < 0 if error. 25470ff40a91SMarcos Paulo de Souza */ 25480ff40a91SMarcos Paulo de Souza int btrfs_search_backwards(struct btrfs_root *root, struct btrfs_key *key, 25490ff40a91SMarcos Paulo de Souza struct btrfs_path *path) 25500ff40a91SMarcos Paulo de Souza { 25510ff40a91SMarcos Paulo de Souza int ret; 25520ff40a91SMarcos Paulo de Souza 25530ff40a91SMarcos Paulo de Souza ret = btrfs_search_slot(NULL, root, key, path, 0, 0); 25540ff40a91SMarcos Paulo de Souza if (ret > 0) 25550ff40a91SMarcos Paulo de Souza ret = btrfs_previous_item(root, path, key->objectid, key->type); 25560ff40a91SMarcos Paulo de Souza 25570ff40a91SMarcos Paulo de Souza if (ret == 0) 25580ff40a91SMarcos Paulo de Souza btrfs_item_key_to_cpu(path->nodes[0], key, path->slots[0]); 25590ff40a91SMarcos Paulo de Souza 25600ff40a91SMarcos Paulo de Souza return ret; 25610ff40a91SMarcos Paulo de Souza } 25620ff40a91SMarcos Paulo de Souza 256343dd529aSDavid Sterba /* 256462142be3SGabriel Niebler * Search for a valid slot for the given path. 256562142be3SGabriel Niebler * 256662142be3SGabriel Niebler * @root: The root node of the tree. 256762142be3SGabriel Niebler * @key: Will contain a valid item if found. 256862142be3SGabriel Niebler * @path: The starting point to validate the slot. 256962142be3SGabriel Niebler * 257062142be3SGabriel Niebler * Return: 0 if the item is valid 257162142be3SGabriel Niebler * 1 if not found 257262142be3SGabriel Niebler * <0 if error. 257362142be3SGabriel Niebler */ 257462142be3SGabriel Niebler int btrfs_get_next_valid_item(struct btrfs_root *root, struct btrfs_key *key, 257562142be3SGabriel Niebler struct btrfs_path *path) 257662142be3SGabriel Niebler { 2577524f14bbSFilipe Manana if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) { 257862142be3SGabriel Niebler int ret; 257962142be3SGabriel Niebler 258062142be3SGabriel Niebler ret = btrfs_next_leaf(root, path); 258162142be3SGabriel Niebler if (ret) 258262142be3SGabriel Niebler return ret; 258362142be3SGabriel Niebler } 2584524f14bbSFilipe Manana 2585524f14bbSFilipe Manana btrfs_item_key_to_cpu(path->nodes[0], key, path->slots[0]); 258662142be3SGabriel Niebler return 0; 258762142be3SGabriel Niebler } 258862142be3SGabriel Niebler 25890ff40a91SMarcos Paulo de Souza /* 259074123bd7SChris Mason * adjust the pointers going up the tree, starting at level 259174123bd7SChris Mason * making sure the right key of each node is points to 'key'. 259274123bd7SChris Mason * This is used after shifting pointers to the left, so it stops 259374123bd7SChris Mason * fixing up pointers when a given leaf/node is not in slot 0 of the 259474123bd7SChris Mason * higher levels 2595aa5d6bedSChris Mason * 259674123bd7SChris Mason */ 2597b167fa91SNikolay Borisov static void fixup_low_keys(struct btrfs_path *path, 25985f39d397SChris Mason struct btrfs_disk_key *key, int level) 2599be0e5c09SChris Mason { 2600be0e5c09SChris Mason int i; 26015f39d397SChris Mason struct extent_buffer *t; 26020e82bcfeSDavid Sterba int ret; 26035f39d397SChris Mason 2604234b63a0SChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 2605be0e5c09SChris Mason int tslot = path->slots[i]; 26060e82bcfeSDavid Sterba 2607eb60ceacSChris Mason if (!path->nodes[i]) 2608be0e5c09SChris Mason break; 26095f39d397SChris Mason t = path->nodes[i]; 2610f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(t, tslot, 261133cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REPLACE); 26120e82bcfeSDavid Sterba BUG_ON(ret < 0); 26135f39d397SChris Mason btrfs_set_node_key(t, key, tslot); 2614d6025579SChris Mason btrfs_mark_buffer_dirty(path->nodes[i]); 2615be0e5c09SChris Mason if (tslot != 0) 2616be0e5c09SChris Mason break; 2617be0e5c09SChris Mason } 2618be0e5c09SChris Mason } 2619be0e5c09SChris Mason 262074123bd7SChris Mason /* 262131840ae1SZheng Yan * update item key. 262231840ae1SZheng Yan * 262331840ae1SZheng Yan * This function isn't completely safe. It's the caller's responsibility 262431840ae1SZheng Yan * that the new key won't break the order 262531840ae1SZheng Yan */ 2626b7a0365eSDaniel Dressler void btrfs_set_item_key_safe(struct btrfs_fs_info *fs_info, 2627b7a0365eSDaniel Dressler struct btrfs_path *path, 2628310712b2SOmar Sandoval const struct btrfs_key *new_key) 262931840ae1SZheng Yan { 263031840ae1SZheng Yan struct btrfs_disk_key disk_key; 263131840ae1SZheng Yan struct extent_buffer *eb; 263231840ae1SZheng Yan int slot; 263331840ae1SZheng Yan 263431840ae1SZheng Yan eb = path->nodes[0]; 263531840ae1SZheng Yan slot = path->slots[0]; 263631840ae1SZheng Yan if (slot > 0) { 263731840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot - 1); 26387c15d410SQu Wenruo if (unlikely(comp_keys(&disk_key, new_key) >= 0)) { 2639eee3b811SQu Wenruo btrfs_print_leaf(eb); 26407c15d410SQu Wenruo btrfs_crit(fs_info, 26417c15d410SQu Wenruo "slot %u key (%llu %u %llu) new key (%llu %u %llu)", 26427c15d410SQu Wenruo slot, btrfs_disk_key_objectid(&disk_key), 26437c15d410SQu Wenruo btrfs_disk_key_type(&disk_key), 26447c15d410SQu Wenruo btrfs_disk_key_offset(&disk_key), 26457c15d410SQu Wenruo new_key->objectid, new_key->type, 26467c15d410SQu Wenruo new_key->offset); 26477c15d410SQu Wenruo BUG(); 26487c15d410SQu Wenruo } 264931840ae1SZheng Yan } 265031840ae1SZheng Yan if (slot < btrfs_header_nritems(eb) - 1) { 265131840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot + 1); 26527c15d410SQu Wenruo if (unlikely(comp_keys(&disk_key, new_key) <= 0)) { 2653eee3b811SQu Wenruo btrfs_print_leaf(eb); 26547c15d410SQu Wenruo btrfs_crit(fs_info, 26557c15d410SQu Wenruo "slot %u key (%llu %u %llu) new key (%llu %u %llu)", 26567c15d410SQu Wenruo slot, btrfs_disk_key_objectid(&disk_key), 26577c15d410SQu Wenruo btrfs_disk_key_type(&disk_key), 26587c15d410SQu Wenruo btrfs_disk_key_offset(&disk_key), 26597c15d410SQu Wenruo new_key->objectid, new_key->type, 26607c15d410SQu Wenruo new_key->offset); 26617c15d410SQu Wenruo BUG(); 26627c15d410SQu Wenruo } 266331840ae1SZheng Yan } 266431840ae1SZheng Yan 266531840ae1SZheng Yan btrfs_cpu_key_to_disk(&disk_key, new_key); 266631840ae1SZheng Yan btrfs_set_item_key(eb, &disk_key, slot); 266731840ae1SZheng Yan btrfs_mark_buffer_dirty(eb); 266831840ae1SZheng Yan if (slot == 0) 2669b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 267031840ae1SZheng Yan } 267131840ae1SZheng Yan 267231840ae1SZheng Yan /* 2673d16c702fSQu Wenruo * Check key order of two sibling extent buffers. 2674d16c702fSQu Wenruo * 2675d16c702fSQu Wenruo * Return true if something is wrong. 2676d16c702fSQu Wenruo * Return false if everything is fine. 2677d16c702fSQu Wenruo * 2678d16c702fSQu Wenruo * Tree-checker only works inside one tree block, thus the following 2679d16c702fSQu Wenruo * corruption can not be detected by tree-checker: 2680d16c702fSQu Wenruo * 2681d16c702fSQu Wenruo * Leaf @left | Leaf @right 2682d16c702fSQu Wenruo * -------------------------------------------------------------- 2683d16c702fSQu Wenruo * | 1 | 2 | 3 | 4 | 5 | f6 | | 7 | 8 | 2684d16c702fSQu Wenruo * 2685d16c702fSQu Wenruo * Key f6 in leaf @left itself is valid, but not valid when the next 2686d16c702fSQu Wenruo * key in leaf @right is 7. 2687d16c702fSQu Wenruo * This can only be checked at tree block merge time. 2688d16c702fSQu Wenruo * And since tree checker has ensured all key order in each tree block 2689d16c702fSQu Wenruo * is correct, we only need to bother the last key of @left and the first 2690d16c702fSQu Wenruo * key of @right. 2691d16c702fSQu Wenruo */ 2692d16c702fSQu Wenruo static bool check_sibling_keys(struct extent_buffer *left, 2693d16c702fSQu Wenruo struct extent_buffer *right) 2694d16c702fSQu Wenruo { 2695d16c702fSQu Wenruo struct btrfs_key left_last; 2696d16c702fSQu Wenruo struct btrfs_key right_first; 2697d16c702fSQu Wenruo int level = btrfs_header_level(left); 2698d16c702fSQu Wenruo int nr_left = btrfs_header_nritems(left); 2699d16c702fSQu Wenruo int nr_right = btrfs_header_nritems(right); 2700d16c702fSQu Wenruo 2701d16c702fSQu Wenruo /* No key to check in one of the tree blocks */ 2702d16c702fSQu Wenruo if (!nr_left || !nr_right) 2703d16c702fSQu Wenruo return false; 2704d16c702fSQu Wenruo 2705d16c702fSQu Wenruo if (level) { 2706d16c702fSQu Wenruo btrfs_node_key_to_cpu(left, &left_last, nr_left - 1); 2707d16c702fSQu Wenruo btrfs_node_key_to_cpu(right, &right_first, 0); 2708d16c702fSQu Wenruo } else { 2709d16c702fSQu Wenruo btrfs_item_key_to_cpu(left, &left_last, nr_left - 1); 2710d16c702fSQu Wenruo btrfs_item_key_to_cpu(right, &right_first, 0); 2711d16c702fSQu Wenruo } 2712d16c702fSQu Wenruo 271388ad95b0SFilipe Manana if (unlikely(btrfs_comp_cpu_keys(&left_last, &right_first) >= 0)) { 2714a2cea677SFilipe Manana btrfs_crit(left->fs_info, "left extent buffer:"); 2715a2cea677SFilipe Manana btrfs_print_tree(left, false); 2716a2cea677SFilipe Manana btrfs_crit(left->fs_info, "right extent buffer:"); 2717a2cea677SFilipe Manana btrfs_print_tree(right, false); 2718d16c702fSQu Wenruo btrfs_crit(left->fs_info, 2719d16c702fSQu Wenruo "bad key order, sibling blocks, left last (%llu %u %llu) right first (%llu %u %llu)", 2720d16c702fSQu Wenruo left_last.objectid, left_last.type, 2721d16c702fSQu Wenruo left_last.offset, right_first.objectid, 2722d16c702fSQu Wenruo right_first.type, right_first.offset); 2723d16c702fSQu Wenruo return true; 2724d16c702fSQu Wenruo } 2725d16c702fSQu Wenruo return false; 2726d16c702fSQu Wenruo } 2727d16c702fSQu Wenruo 2728d16c702fSQu Wenruo /* 272974123bd7SChris Mason * try to push data from one node into the next node left in the 273079f95c82SChris Mason * tree. 2731aa5d6bedSChris Mason * 2732aa5d6bedSChris Mason * returns 0 if some ptrs were pushed left, < 0 if there was some horrible 2733aa5d6bedSChris Mason * error, and > 0 if there was no room in the left hand block. 273474123bd7SChris Mason */ 273598ed5174SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 27362ff7e61eSJeff Mahoney struct extent_buffer *dst, 2737971a1f66SChris Mason struct extent_buffer *src, int empty) 2738be0e5c09SChris Mason { 2739d30a668fSDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info; 2740be0e5c09SChris Mason int push_items = 0; 2741bb803951SChris Mason int src_nritems; 2742bb803951SChris Mason int dst_nritems; 2743aa5d6bedSChris Mason int ret = 0; 2744be0e5c09SChris Mason 27455f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 27465f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 27470b246afaSJeff Mahoney push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems; 27487bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 27497bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 275054aa1f4dSChris Mason 2751bce4eae9SChris Mason if (!empty && src_nritems <= 8) 2752971a1f66SChris Mason return 1; 2753971a1f66SChris Mason 2754d397712bSChris Mason if (push_items <= 0) 2755be0e5c09SChris Mason return 1; 2756be0e5c09SChris Mason 2757bce4eae9SChris Mason if (empty) { 2758971a1f66SChris Mason push_items = min(src_nritems, push_items); 2759bce4eae9SChris Mason if (push_items < src_nritems) { 2760bce4eae9SChris Mason /* leave at least 8 pointers in the node if 2761bce4eae9SChris Mason * we aren't going to empty it 2762bce4eae9SChris Mason */ 2763bce4eae9SChris Mason if (src_nritems - push_items < 8) { 2764bce4eae9SChris Mason if (push_items <= 8) 2765bce4eae9SChris Mason return 1; 2766bce4eae9SChris Mason push_items -= 8; 2767bce4eae9SChris Mason } 2768bce4eae9SChris Mason } 2769bce4eae9SChris Mason } else 2770bce4eae9SChris Mason push_items = min(src_nritems - 8, push_items); 277179f95c82SChris Mason 2772d16c702fSQu Wenruo /* dst is the left eb, src is the middle eb */ 2773d16c702fSQu Wenruo if (check_sibling_keys(dst, src)) { 2774d16c702fSQu Wenruo ret = -EUCLEAN; 2775d16c702fSQu Wenruo btrfs_abort_transaction(trans, ret); 2776d16c702fSQu Wenruo return ret; 2777d16c702fSQu Wenruo } 2778f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_eb_copy(dst, src, dst_nritems, 0, push_items); 27795de865eeSFilipe David Borba Manana if (ret) { 278066642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 27815de865eeSFilipe David Borba Manana return ret; 27825de865eeSFilipe David Borba Manana } 27835f39d397SChris Mason copy_extent_buffer(dst, src, 2784e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(dst, dst_nritems), 2785e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(src, 0), 2786123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 27875f39d397SChris Mason 2788bb803951SChris Mason if (push_items < src_nritems) { 278957911b8bSJan Schmidt /* 2790f3a84ccdSFilipe Manana * Don't call btrfs_tree_mod_log_insert_move() here, key removal 2791f3a84ccdSFilipe Manana * was already fully logged by btrfs_tree_mod_log_eb_copy() above. 279257911b8bSJan Schmidt */ 2793e23efd8eSJosef Bacik memmove_extent_buffer(src, btrfs_node_key_ptr_offset(src, 0), 2794e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(src, push_items), 2795e2fa7227SChris Mason (src_nritems - push_items) * 2796123abc88SChris Mason sizeof(struct btrfs_key_ptr)); 2797bb803951SChris Mason } 27985f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 27995f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 28005f39d397SChris Mason btrfs_mark_buffer_dirty(src); 28015f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 280231840ae1SZheng Yan 2803bb803951SChris Mason return ret; 2804be0e5c09SChris Mason } 2805be0e5c09SChris Mason 280697571fd0SChris Mason /* 280779f95c82SChris Mason * try to push data from one node into the next node right in the 280879f95c82SChris Mason * tree. 280979f95c82SChris Mason * 281079f95c82SChris Mason * returns 0 if some ptrs were pushed, < 0 if there was some horrible 281179f95c82SChris Mason * error, and > 0 if there was no room in the right hand block. 281279f95c82SChris Mason * 281379f95c82SChris Mason * this will only push up to 1/2 the contents of the left node over 281479f95c82SChris Mason */ 28155f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 28165f39d397SChris Mason struct extent_buffer *dst, 28175f39d397SChris Mason struct extent_buffer *src) 281879f95c82SChris Mason { 281955d32ed8SDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info; 282079f95c82SChris Mason int push_items = 0; 282179f95c82SChris Mason int max_push; 282279f95c82SChris Mason int src_nritems; 282379f95c82SChris Mason int dst_nritems; 282479f95c82SChris Mason int ret = 0; 282579f95c82SChris Mason 28267bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 28277bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 28287bb86316SChris Mason 28295f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 28305f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 28310b246afaSJeff Mahoney push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems; 2832d397712bSChris Mason if (push_items <= 0) 283379f95c82SChris Mason return 1; 2834bce4eae9SChris Mason 2835d397712bSChris Mason if (src_nritems < 4) 2836bce4eae9SChris Mason return 1; 283779f95c82SChris Mason 283879f95c82SChris Mason max_push = src_nritems / 2 + 1; 283979f95c82SChris Mason /* don't try to empty the node */ 2840d397712bSChris Mason if (max_push >= src_nritems) 284179f95c82SChris Mason return 1; 2842252c38f0SYan 284379f95c82SChris Mason if (max_push < push_items) 284479f95c82SChris Mason push_items = max_push; 284579f95c82SChris Mason 2846d16c702fSQu Wenruo /* dst is the right eb, src is the middle eb */ 2847d16c702fSQu Wenruo if (check_sibling_keys(src, dst)) { 2848d16c702fSQu Wenruo ret = -EUCLEAN; 2849d16c702fSQu Wenruo btrfs_abort_transaction(trans, ret); 2850d16c702fSQu Wenruo return ret; 2851d16c702fSQu Wenruo } 2852f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_move(dst, push_items, 0, dst_nritems); 2853bf1d3425SDavid Sterba BUG_ON(ret < 0); 2854e23efd8eSJosef Bacik memmove_extent_buffer(dst, btrfs_node_key_ptr_offset(dst, push_items), 2855e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(dst, 0), 28565f39d397SChris Mason (dst_nritems) * 28575f39d397SChris Mason sizeof(struct btrfs_key_ptr)); 2858d6025579SChris Mason 2859f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_eb_copy(dst, src, 0, src_nritems - push_items, 2860ed874f0dSDavid Sterba push_items); 28615de865eeSFilipe David Borba Manana if (ret) { 286266642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 28635de865eeSFilipe David Borba Manana return ret; 28645de865eeSFilipe David Borba Manana } 28655f39d397SChris Mason copy_extent_buffer(dst, src, 2866e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(dst, 0), 2867e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(src, src_nritems - push_items), 2868123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 286979f95c82SChris Mason 28705f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 28715f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 287279f95c82SChris Mason 28735f39d397SChris Mason btrfs_mark_buffer_dirty(src); 28745f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 287531840ae1SZheng Yan 287679f95c82SChris Mason return ret; 287779f95c82SChris Mason } 287879f95c82SChris Mason 287979f95c82SChris Mason /* 288097571fd0SChris Mason * helper function to insert a new root level in the tree. 288197571fd0SChris Mason * A new node is allocated, and a single item is inserted to 288297571fd0SChris Mason * point to the existing root 2883aa5d6bedSChris Mason * 2884aa5d6bedSChris Mason * returns zero on success or < 0 on failure. 288597571fd0SChris Mason */ 2886d397712bSChris Mason static noinline int insert_new_root(struct btrfs_trans_handle *trans, 28875f39d397SChris Mason struct btrfs_root *root, 2888fdd99c72SLiu Bo struct btrfs_path *path, int level) 288974123bd7SChris Mason { 28900b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 28917bb86316SChris Mason u64 lower_gen; 28925f39d397SChris Mason struct extent_buffer *lower; 28935f39d397SChris Mason struct extent_buffer *c; 2894925baeddSChris Mason struct extent_buffer *old; 28955f39d397SChris Mason struct btrfs_disk_key lower_key; 2896d9d19a01SDavid Sterba int ret; 28975c680ed6SChris Mason 28985c680ed6SChris Mason BUG_ON(path->nodes[level]); 28995c680ed6SChris Mason BUG_ON(path->nodes[level-1] != root->node); 29005c680ed6SChris Mason 29017bb86316SChris Mason lower = path->nodes[level-1]; 29027bb86316SChris Mason if (level == 1) 29037bb86316SChris Mason btrfs_item_key(lower, &lower_key, 0); 29047bb86316SChris Mason else 29057bb86316SChris Mason btrfs_node_key(lower, &lower_key, 0); 29067bb86316SChris Mason 290779bd3712SFilipe Manana c = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 290879bd3712SFilipe Manana &lower_key, level, root->node->start, 0, 2909cf6f34aaSJosef Bacik BTRFS_NESTING_NEW_ROOT); 29105f39d397SChris Mason if (IS_ERR(c)) 29115f39d397SChris Mason return PTR_ERR(c); 2912925baeddSChris Mason 29130b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 2914f0486c68SYan, Zheng 29155f39d397SChris Mason btrfs_set_header_nritems(c, 1); 29165f39d397SChris Mason btrfs_set_node_key(c, &lower_key, 0); 2917db94535dSChris Mason btrfs_set_node_blockptr(c, 0, lower->start); 29187bb86316SChris Mason lower_gen = btrfs_header_generation(lower); 291931840ae1SZheng Yan WARN_ON(lower_gen != trans->transid); 29207bb86316SChris Mason 29217bb86316SChris Mason btrfs_set_node_ptr_generation(c, 0, lower_gen); 29225f39d397SChris Mason 29235f39d397SChris Mason btrfs_mark_buffer_dirty(c); 2924d5719762SChris Mason 2925925baeddSChris Mason old = root->node; 2926406808abSFilipe Manana ret = btrfs_tree_mod_log_insert_root(root->node, c, false); 2927d9d19a01SDavid Sterba BUG_ON(ret < 0); 2928240f62c8SChris Mason rcu_assign_pointer(root->node, c); 2929925baeddSChris Mason 2930925baeddSChris Mason /* the super has an extra ref to root->node */ 2931925baeddSChris Mason free_extent_buffer(old); 2932925baeddSChris Mason 29330b86a832SChris Mason add_root_to_dirty_list(root); 293467439dadSDavid Sterba atomic_inc(&c->refs); 29355f39d397SChris Mason path->nodes[level] = c; 2936ac5887c8SJosef Bacik path->locks[level] = BTRFS_WRITE_LOCK; 293774123bd7SChris Mason path->slots[level] = 0; 293874123bd7SChris Mason return 0; 293974123bd7SChris Mason } 29405c680ed6SChris Mason 29415c680ed6SChris Mason /* 29425c680ed6SChris Mason * worker function to insert a single pointer in a node. 29435c680ed6SChris Mason * the node should have enough room for the pointer already 294497571fd0SChris Mason * 29455c680ed6SChris Mason * slot and level indicate where you want the key to go, and 29465c680ed6SChris Mason * blocknr is the block the key points to. 29475c680ed6SChris Mason */ 2948143bede5SJeff Mahoney static void insert_ptr(struct btrfs_trans_handle *trans, 29496ad3cf6dSDavid Sterba struct btrfs_path *path, 2950143bede5SJeff Mahoney struct btrfs_disk_key *key, u64 bytenr, 2951c3e06965SJan Schmidt int slot, int level) 29525c680ed6SChris Mason { 29535f39d397SChris Mason struct extent_buffer *lower; 29545c680ed6SChris Mason int nritems; 2955f3ea38daSJan Schmidt int ret; 29565c680ed6SChris Mason 29575c680ed6SChris Mason BUG_ON(!path->nodes[level]); 295849d0c642SFilipe Manana btrfs_assert_tree_write_locked(path->nodes[level]); 29595f39d397SChris Mason lower = path->nodes[level]; 29605f39d397SChris Mason nritems = btrfs_header_nritems(lower); 2961c293498bSStoyan Gaydarov BUG_ON(slot > nritems); 29626ad3cf6dSDavid Sterba BUG_ON(nritems == BTRFS_NODEPTRS_PER_BLOCK(trans->fs_info)); 296374123bd7SChris Mason if (slot != nritems) { 2964bf1d3425SDavid Sterba if (level) { 2965f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_move(lower, slot + 1, 2966f3a84ccdSFilipe Manana slot, nritems - slot); 2967bf1d3425SDavid Sterba BUG_ON(ret < 0); 2968bf1d3425SDavid Sterba } 29695f39d397SChris Mason memmove_extent_buffer(lower, 2970e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(lower, slot + 1), 2971e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(lower, slot), 2972123abc88SChris Mason (nritems - slot) * sizeof(struct btrfs_key_ptr)); 297374123bd7SChris Mason } 2974c3e06965SJan Schmidt if (level) { 2975f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(lower, slot, 297633cff222SFilipe Manana BTRFS_MOD_LOG_KEY_ADD); 2977f3ea38daSJan Schmidt BUG_ON(ret < 0); 2978f3ea38daSJan Schmidt } 29795f39d397SChris Mason btrfs_set_node_key(lower, key, slot); 2980db94535dSChris Mason btrfs_set_node_blockptr(lower, slot, bytenr); 298174493f7aSChris Mason WARN_ON(trans->transid == 0); 298274493f7aSChris Mason btrfs_set_node_ptr_generation(lower, slot, trans->transid); 29835f39d397SChris Mason btrfs_set_header_nritems(lower, nritems + 1); 29845f39d397SChris Mason btrfs_mark_buffer_dirty(lower); 298574123bd7SChris Mason } 298674123bd7SChris Mason 298797571fd0SChris Mason /* 298897571fd0SChris Mason * split the node at the specified level in path in two. 298997571fd0SChris Mason * The path is corrected to point to the appropriate node after the split 299097571fd0SChris Mason * 299197571fd0SChris Mason * Before splitting this tries to make some room in the node by pushing 299297571fd0SChris Mason * left and right, if either one works, it returns right away. 2993aa5d6bedSChris Mason * 2994aa5d6bedSChris Mason * returns 0 on success and < 0 on failure 299597571fd0SChris Mason */ 2996e02119d5SChris Mason static noinline int split_node(struct btrfs_trans_handle *trans, 2997e02119d5SChris Mason struct btrfs_root *root, 2998e02119d5SChris Mason struct btrfs_path *path, int level) 2999be0e5c09SChris Mason { 30000b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 30015f39d397SChris Mason struct extent_buffer *c; 30025f39d397SChris Mason struct extent_buffer *split; 30035f39d397SChris Mason struct btrfs_disk_key disk_key; 3004be0e5c09SChris Mason int mid; 30055c680ed6SChris Mason int ret; 30067518a238SChris Mason u32 c_nritems; 3007be0e5c09SChris Mason 30085f39d397SChris Mason c = path->nodes[level]; 30097bb86316SChris Mason WARN_ON(btrfs_header_generation(c) != trans->transid); 30105f39d397SChris Mason if (c == root->node) { 3011d9abbf1cSJan Schmidt /* 301290f8d62eSJan Schmidt * trying to split the root, lets make a new one 301390f8d62eSJan Schmidt * 3014fdd99c72SLiu Bo * tree mod log: We don't log_removal old root in 301590f8d62eSJan Schmidt * insert_new_root, because that root buffer will be kept as a 301690f8d62eSJan Schmidt * normal node. We are going to log removal of half of the 3017f3a84ccdSFilipe Manana * elements below with btrfs_tree_mod_log_eb_copy(). We're 3018f3a84ccdSFilipe Manana * holding a tree lock on the buffer, which is why we cannot 3019f3a84ccdSFilipe Manana * race with other tree_mod_log users. 3020d9abbf1cSJan Schmidt */ 3021fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, level + 1); 30225c680ed6SChris Mason if (ret) 30235c680ed6SChris Mason return ret; 3024b3612421SChris Mason } else { 3025e66f709bSChris Mason ret = push_nodes_for_insert(trans, root, path, level); 30265f39d397SChris Mason c = path->nodes[level]; 30275f39d397SChris Mason if (!ret && btrfs_header_nritems(c) < 30280b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3) 3029e66f709bSChris Mason return 0; 303054aa1f4dSChris Mason if (ret < 0) 303154aa1f4dSChris Mason return ret; 30325c680ed6SChris Mason } 3033e66f709bSChris Mason 30345f39d397SChris Mason c_nritems = btrfs_header_nritems(c); 30355d4f98a2SYan Zheng mid = (c_nritems + 1) / 2; 30365d4f98a2SYan Zheng btrfs_node_key(c, &disk_key, mid); 30377bb86316SChris Mason 303879bd3712SFilipe Manana split = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 303979bd3712SFilipe Manana &disk_key, level, c->start, 0, 304079bd3712SFilipe Manana BTRFS_NESTING_SPLIT); 30415f39d397SChris Mason if (IS_ERR(split)) 30425f39d397SChris Mason return PTR_ERR(split); 304354aa1f4dSChris Mason 30440b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 3045bc877d28SNikolay Borisov ASSERT(btrfs_header_level(c) == level); 30465f39d397SChris Mason 3047f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_eb_copy(split, c, 0, mid, c_nritems - mid); 30485de865eeSFilipe David Borba Manana if (ret) { 304966642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 30505de865eeSFilipe David Borba Manana return ret; 30515de865eeSFilipe David Borba Manana } 30525f39d397SChris Mason copy_extent_buffer(split, c, 3053e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(split, 0), 3054e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(c, mid), 3055123abc88SChris Mason (c_nritems - mid) * sizeof(struct btrfs_key_ptr)); 30565f39d397SChris Mason btrfs_set_header_nritems(split, c_nritems - mid); 30575f39d397SChris Mason btrfs_set_header_nritems(c, mid); 3058aa5d6bedSChris Mason 30595f39d397SChris Mason btrfs_mark_buffer_dirty(c); 30605f39d397SChris Mason btrfs_mark_buffer_dirty(split); 30615f39d397SChris Mason 30626ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, split->start, 3063c3e06965SJan Schmidt path->slots[level + 1] + 1, level + 1); 3064aa5d6bedSChris Mason 30655de08d7dSChris Mason if (path->slots[level] >= mid) { 30665c680ed6SChris Mason path->slots[level] -= mid; 3067925baeddSChris Mason btrfs_tree_unlock(c); 30685f39d397SChris Mason free_extent_buffer(c); 30695f39d397SChris Mason path->nodes[level] = split; 30705c680ed6SChris Mason path->slots[level + 1] += 1; 3071eb60ceacSChris Mason } else { 3072925baeddSChris Mason btrfs_tree_unlock(split); 30735f39d397SChris Mason free_extent_buffer(split); 3074be0e5c09SChris Mason } 3075d5286a92SNikolay Borisov return 0; 3076be0e5c09SChris Mason } 3077be0e5c09SChris Mason 307874123bd7SChris Mason /* 307974123bd7SChris Mason * how many bytes are required to store the items in a leaf. start 308074123bd7SChris Mason * and nr indicate which items in the leaf to check. This totals up the 308174123bd7SChris Mason * space used both by the item structs and the item data 308274123bd7SChris Mason */ 30836c75a589SQu Wenruo static int leaf_space_used(const struct extent_buffer *l, int start, int nr) 3084be0e5c09SChris Mason { 3085be0e5c09SChris Mason int data_len; 30865f39d397SChris Mason int nritems = btrfs_header_nritems(l); 3087d4dbff95SChris Mason int end = min(nritems, start + nr) - 1; 3088be0e5c09SChris Mason 3089be0e5c09SChris Mason if (!nr) 3090be0e5c09SChris Mason return 0; 30913212fa14SJosef Bacik data_len = btrfs_item_offset(l, start) + btrfs_item_size(l, start); 30923212fa14SJosef Bacik data_len = data_len - btrfs_item_offset(l, end); 30930783fcfcSChris Mason data_len += sizeof(struct btrfs_item) * nr; 3094d4dbff95SChris Mason WARN_ON(data_len < 0); 3095be0e5c09SChris Mason return data_len; 3096be0e5c09SChris Mason } 3097be0e5c09SChris Mason 309874123bd7SChris Mason /* 3099d4dbff95SChris Mason * The space between the end of the leaf items and 3100d4dbff95SChris Mason * the start of the leaf data. IOW, how much room 3101d4dbff95SChris Mason * the leaf has left for both items and data 3102d4dbff95SChris Mason */ 31036c75a589SQu Wenruo int btrfs_leaf_free_space(const struct extent_buffer *leaf) 3104d4dbff95SChris Mason { 3105e902baacSDavid Sterba struct btrfs_fs_info *fs_info = leaf->fs_info; 31065f39d397SChris Mason int nritems = btrfs_header_nritems(leaf); 31075f39d397SChris Mason int ret; 31080b246afaSJeff Mahoney 31090b246afaSJeff Mahoney ret = BTRFS_LEAF_DATA_SIZE(fs_info) - leaf_space_used(leaf, 0, nritems); 31105f39d397SChris Mason if (ret < 0) { 31110b246afaSJeff Mahoney btrfs_crit(fs_info, 3112efe120a0SFrank Holton "leaf free space ret %d, leaf data size %lu, used %d nritems %d", 3113da17066cSJeff Mahoney ret, 31140b246afaSJeff Mahoney (unsigned long) BTRFS_LEAF_DATA_SIZE(fs_info), 31155f39d397SChris Mason leaf_space_used(leaf, 0, nritems), nritems); 31165f39d397SChris Mason } 31175f39d397SChris Mason return ret; 3118d4dbff95SChris Mason } 3119d4dbff95SChris Mason 312099d8f83cSChris Mason /* 312199d8f83cSChris Mason * min slot controls the lowest index we're willing to push to the 312299d8f83cSChris Mason * right. We'll push up to and including min_slot, but no lower 312399d8f83cSChris Mason */ 3124ed25dab3SJosef Bacik static noinline int __push_leaf_right(struct btrfs_trans_handle *trans, 3125ed25dab3SJosef Bacik struct btrfs_path *path, 312644871b1bSChris Mason int data_size, int empty, 312744871b1bSChris Mason struct extent_buffer *right, 312899d8f83cSChris Mason int free_space, u32 left_nritems, 312999d8f83cSChris Mason u32 min_slot) 313000ec4c51SChris Mason { 3131f72f0010SDavid Sterba struct btrfs_fs_info *fs_info = right->fs_info; 31325f39d397SChris Mason struct extent_buffer *left = path->nodes[0]; 313344871b1bSChris Mason struct extent_buffer *upper = path->nodes[1]; 3134cfed81a0SChris Mason struct btrfs_map_token token; 31355f39d397SChris Mason struct btrfs_disk_key disk_key; 313600ec4c51SChris Mason int slot; 313734a38218SChris Mason u32 i; 313800ec4c51SChris Mason int push_space = 0; 313900ec4c51SChris Mason int push_items = 0; 314034a38218SChris Mason u32 nr; 31417518a238SChris Mason u32 right_nritems; 31425f39d397SChris Mason u32 data_end; 3143db94535dSChris Mason u32 this_item_size; 314400ec4c51SChris Mason 314534a38218SChris Mason if (empty) 314634a38218SChris Mason nr = 0; 314734a38218SChris Mason else 314899d8f83cSChris Mason nr = max_t(u32, 1, min_slot); 314934a38218SChris Mason 315031840ae1SZheng Yan if (path->slots[0] >= left_nritems) 315187b29b20SYan Zheng push_space += data_size; 315231840ae1SZheng Yan 315344871b1bSChris Mason slot = path->slots[1]; 315434a38218SChris Mason i = left_nritems - 1; 315534a38218SChris Mason while (i >= nr) { 315631840ae1SZheng Yan if (!empty && push_items > 0) { 315731840ae1SZheng Yan if (path->slots[0] > i) 315831840ae1SZheng Yan break; 315931840ae1SZheng Yan if (path->slots[0] == i) { 3160e902baacSDavid Sterba int space = btrfs_leaf_free_space(left); 3161e902baacSDavid Sterba 316231840ae1SZheng Yan if (space + push_space * 2 > free_space) 316331840ae1SZheng Yan break; 316431840ae1SZheng Yan } 316531840ae1SZheng Yan } 316631840ae1SZheng Yan 316700ec4c51SChris Mason if (path->slots[0] == i) 316887b29b20SYan Zheng push_space += data_size; 3169db94535dSChris Mason 31703212fa14SJosef Bacik this_item_size = btrfs_item_size(left, i); 317174794207SJosef Bacik if (this_item_size + sizeof(struct btrfs_item) + 317274794207SJosef Bacik push_space > free_space) 317300ec4c51SChris Mason break; 317431840ae1SZheng Yan 317500ec4c51SChris Mason push_items++; 317674794207SJosef Bacik push_space += this_item_size + sizeof(struct btrfs_item); 317734a38218SChris Mason if (i == 0) 317834a38218SChris Mason break; 317934a38218SChris Mason i--; 3180db94535dSChris Mason } 31815f39d397SChris Mason 3182925baeddSChris Mason if (push_items == 0) 3183925baeddSChris Mason goto out_unlock; 31845f39d397SChris Mason 31856c1500f2SJulia Lawall WARN_ON(!empty && push_items == left_nritems); 31865f39d397SChris Mason 318700ec4c51SChris Mason /* push left to right */ 31885f39d397SChris Mason right_nritems = btrfs_header_nritems(right); 318934a38218SChris Mason 3190dc2e724eSJosef Bacik push_space = btrfs_item_data_end(left, left_nritems - push_items); 31918f881e8cSDavid Sterba push_space -= leaf_data_end(left); 31925f39d397SChris Mason 319300ec4c51SChris Mason /* make room in the right data area */ 31948f881e8cSDavid Sterba data_end = leaf_data_end(right); 3195637e3b48SJosef Bacik memmove_leaf_data(right, data_end - push_space, data_end, 31960b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - data_end); 31975f39d397SChris Mason 319800ec4c51SChris Mason /* copy from the left data area */ 3199637e3b48SJosef Bacik copy_leaf_data(right, left, BTRFS_LEAF_DATA_SIZE(fs_info) - push_space, 3200637e3b48SJosef Bacik leaf_data_end(left), push_space); 32015f39d397SChris Mason 3202637e3b48SJosef Bacik memmove_leaf_items(right, push_items, 0, right_nritems); 32035f39d397SChris Mason 320400ec4c51SChris Mason /* copy the items from left to right */ 3205637e3b48SJosef Bacik copy_leaf_items(right, left, 0, left_nritems - push_items, push_items); 320600ec4c51SChris Mason 320700ec4c51SChris Mason /* update the item pointers */ 3208c82f823cSDavid Sterba btrfs_init_map_token(&token, right); 32097518a238SChris Mason right_nritems += push_items; 32105f39d397SChris Mason btrfs_set_header_nritems(right, right_nritems); 32110b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info); 32127518a238SChris Mason for (i = 0; i < right_nritems; i++) { 32133212fa14SJosef Bacik push_space -= btrfs_token_item_size(&token, i); 32143212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, push_space); 3215db94535dSChris Mason } 3216db94535dSChris Mason 32177518a238SChris Mason left_nritems -= push_items; 32185f39d397SChris Mason btrfs_set_header_nritems(left, left_nritems); 321900ec4c51SChris Mason 322034a38218SChris Mason if (left_nritems) 32215f39d397SChris Mason btrfs_mark_buffer_dirty(left); 3222f0486c68SYan, Zheng else 3223190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, left); 3224f0486c68SYan, Zheng 32255f39d397SChris Mason btrfs_mark_buffer_dirty(right); 3226a429e513SChris Mason 32275f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 32285f39d397SChris Mason btrfs_set_node_key(upper, &disk_key, slot + 1); 3229d6025579SChris Mason btrfs_mark_buffer_dirty(upper); 323002217ed2SChris Mason 323100ec4c51SChris Mason /* then fixup the leaf pointer in the path */ 32327518a238SChris Mason if (path->slots[0] >= left_nritems) { 32337518a238SChris Mason path->slots[0] -= left_nritems; 3234925baeddSChris Mason if (btrfs_header_nritems(path->nodes[0]) == 0) 3235190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, path->nodes[0]); 3236925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 32375f39d397SChris Mason free_extent_buffer(path->nodes[0]); 32385f39d397SChris Mason path->nodes[0] = right; 323900ec4c51SChris Mason path->slots[1] += 1; 324000ec4c51SChris Mason } else { 3241925baeddSChris Mason btrfs_tree_unlock(right); 32425f39d397SChris Mason free_extent_buffer(right); 324300ec4c51SChris Mason } 324400ec4c51SChris Mason return 0; 3245925baeddSChris Mason 3246925baeddSChris Mason out_unlock: 3247925baeddSChris Mason btrfs_tree_unlock(right); 3248925baeddSChris Mason free_extent_buffer(right); 3249925baeddSChris Mason return 1; 325000ec4c51SChris Mason } 3251925baeddSChris Mason 325200ec4c51SChris Mason /* 325344871b1bSChris Mason * push some data in the path leaf to the right, trying to free up at 325474123bd7SChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 325544871b1bSChris Mason * 325644871b1bSChris Mason * returns 1 if the push failed because the other node didn't have enough 325744871b1bSChris Mason * room, 0 if everything worked out and < 0 if there were major errors. 325899d8f83cSChris Mason * 325999d8f83cSChris Mason * this will push starting from min_slot to the end of the leaf. It won't 326099d8f83cSChris Mason * push any slot lower than min_slot 326174123bd7SChris Mason */ 326244871b1bSChris Mason static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root 326399d8f83cSChris Mason *root, struct btrfs_path *path, 326499d8f83cSChris Mason int min_data_size, int data_size, 326599d8f83cSChris Mason int empty, u32 min_slot) 3266be0e5c09SChris Mason { 326744871b1bSChris Mason struct extent_buffer *left = path->nodes[0]; 326844871b1bSChris Mason struct extent_buffer *right; 326944871b1bSChris Mason struct extent_buffer *upper; 327044871b1bSChris Mason int slot; 327144871b1bSChris Mason int free_space; 327244871b1bSChris Mason u32 left_nritems; 327344871b1bSChris Mason int ret; 327444871b1bSChris Mason 327544871b1bSChris Mason if (!path->nodes[1]) 327644871b1bSChris Mason return 1; 327744871b1bSChris Mason 327844871b1bSChris Mason slot = path->slots[1]; 327944871b1bSChris Mason upper = path->nodes[1]; 328044871b1bSChris Mason if (slot >= btrfs_header_nritems(upper) - 1) 328144871b1bSChris Mason return 1; 328244871b1bSChris Mason 328349d0c642SFilipe Manana btrfs_assert_tree_write_locked(path->nodes[1]); 328444871b1bSChris Mason 32854b231ae4SDavid Sterba right = btrfs_read_node_slot(upper, slot + 1); 3286fb770ae4SLiu Bo if (IS_ERR(right)) 32879cf14029SJosef Bacik return PTR_ERR(right); 328891ca338dSTsutomu Itoh 3289bf77467aSJosef Bacik __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT); 329044871b1bSChris Mason 3291e902baacSDavid Sterba free_space = btrfs_leaf_free_space(right); 329244871b1bSChris Mason if (free_space < data_size) 329344871b1bSChris Mason goto out_unlock; 329444871b1bSChris Mason 329544871b1bSChris Mason ret = btrfs_cow_block(trans, root, right, upper, 3296bf59a5a2SJosef Bacik slot + 1, &right, BTRFS_NESTING_RIGHT_COW); 329744871b1bSChris Mason if (ret) 329844871b1bSChris Mason goto out_unlock; 329944871b1bSChris Mason 330044871b1bSChris Mason left_nritems = btrfs_header_nritems(left); 330144871b1bSChris Mason if (left_nritems == 0) 330244871b1bSChris Mason goto out_unlock; 330344871b1bSChris Mason 3304d16c702fSQu Wenruo if (check_sibling_keys(left, right)) { 3305d16c702fSQu Wenruo ret = -EUCLEAN; 33069ae5afd0SFilipe Manana btrfs_abort_transaction(trans, ret); 3307d16c702fSQu Wenruo btrfs_tree_unlock(right); 3308d16c702fSQu Wenruo free_extent_buffer(right); 3309d16c702fSQu Wenruo return ret; 3310d16c702fSQu Wenruo } 33112ef1fed2SFilipe David Borba Manana if (path->slots[0] == left_nritems && !empty) { 33122ef1fed2SFilipe David Borba Manana /* Key greater than all keys in the leaf, right neighbor has 33132ef1fed2SFilipe David Borba Manana * enough room for it and we're not emptying our leaf to delete 33142ef1fed2SFilipe David Borba Manana * it, therefore use right neighbor to insert the new item and 331552042d8eSAndrea Gelmini * no need to touch/dirty our left leaf. */ 33162ef1fed2SFilipe David Borba Manana btrfs_tree_unlock(left); 33172ef1fed2SFilipe David Borba Manana free_extent_buffer(left); 33182ef1fed2SFilipe David Borba Manana path->nodes[0] = right; 33192ef1fed2SFilipe David Borba Manana path->slots[0] = 0; 33202ef1fed2SFilipe David Borba Manana path->slots[1]++; 33212ef1fed2SFilipe David Borba Manana return 0; 33222ef1fed2SFilipe David Borba Manana } 33232ef1fed2SFilipe David Borba Manana 3324ed25dab3SJosef Bacik return __push_leaf_right(trans, path, min_data_size, empty, right, 3325ed25dab3SJosef Bacik free_space, left_nritems, min_slot); 332644871b1bSChris Mason out_unlock: 332744871b1bSChris Mason btrfs_tree_unlock(right); 332844871b1bSChris Mason free_extent_buffer(right); 332944871b1bSChris Mason return 1; 333044871b1bSChris Mason } 333144871b1bSChris Mason 333244871b1bSChris Mason /* 333344871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 333444871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 333599d8f83cSChris Mason * 333699d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 333799d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us do all the 333899d8f83cSChris Mason * items 333944871b1bSChris Mason */ 3340ed25dab3SJosef Bacik static noinline int __push_leaf_left(struct btrfs_trans_handle *trans, 3341ed25dab3SJosef Bacik struct btrfs_path *path, int data_size, 334244871b1bSChris Mason int empty, struct extent_buffer *left, 334399d8f83cSChris Mason int free_space, u32 right_nritems, 334499d8f83cSChris Mason u32 max_slot) 334544871b1bSChris Mason { 33468087c193SDavid Sterba struct btrfs_fs_info *fs_info = left->fs_info; 33475f39d397SChris Mason struct btrfs_disk_key disk_key; 33485f39d397SChris Mason struct extent_buffer *right = path->nodes[0]; 3349be0e5c09SChris Mason int i; 3350be0e5c09SChris Mason int push_space = 0; 3351be0e5c09SChris Mason int push_items = 0; 33527518a238SChris Mason u32 old_left_nritems; 335334a38218SChris Mason u32 nr; 3354aa5d6bedSChris Mason int ret = 0; 3355db94535dSChris Mason u32 this_item_size; 3356db94535dSChris Mason u32 old_left_item_size; 3357cfed81a0SChris Mason struct btrfs_map_token token; 3358cfed81a0SChris Mason 335934a38218SChris Mason if (empty) 336099d8f83cSChris Mason nr = min(right_nritems, max_slot); 336134a38218SChris Mason else 336299d8f83cSChris Mason nr = min(right_nritems - 1, max_slot); 336334a38218SChris Mason 336434a38218SChris Mason for (i = 0; i < nr; i++) { 336531840ae1SZheng Yan if (!empty && push_items > 0) { 336631840ae1SZheng Yan if (path->slots[0] < i) 336731840ae1SZheng Yan break; 336831840ae1SZheng Yan if (path->slots[0] == i) { 3369e902baacSDavid Sterba int space = btrfs_leaf_free_space(right); 3370e902baacSDavid Sterba 337131840ae1SZheng Yan if (space + push_space * 2 > free_space) 337231840ae1SZheng Yan break; 337331840ae1SZheng Yan } 337431840ae1SZheng Yan } 337531840ae1SZheng Yan 3376be0e5c09SChris Mason if (path->slots[0] == i) 337787b29b20SYan Zheng push_space += data_size; 3378db94535dSChris Mason 33793212fa14SJosef Bacik this_item_size = btrfs_item_size(right, i); 338074794207SJosef Bacik if (this_item_size + sizeof(struct btrfs_item) + push_space > 338174794207SJosef Bacik free_space) 3382be0e5c09SChris Mason break; 3383db94535dSChris Mason 3384be0e5c09SChris Mason push_items++; 338574794207SJosef Bacik push_space += this_item_size + sizeof(struct btrfs_item); 3386be0e5c09SChris Mason } 3387db94535dSChris Mason 3388be0e5c09SChris Mason if (push_items == 0) { 3389925baeddSChris Mason ret = 1; 3390925baeddSChris Mason goto out; 3391be0e5c09SChris Mason } 3392fae7f21cSDulshani Gunawardhana WARN_ON(!empty && push_items == btrfs_header_nritems(right)); 33935f39d397SChris Mason 3394be0e5c09SChris Mason /* push data from right to left */ 3395637e3b48SJosef Bacik copy_leaf_items(left, right, btrfs_header_nritems(left), 0, push_items); 33965f39d397SChris Mason 33970b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info) - 33983212fa14SJosef Bacik btrfs_item_offset(right, push_items - 1); 33995f39d397SChris Mason 3400637e3b48SJosef Bacik copy_leaf_data(left, right, leaf_data_end(left) - push_space, 3401637e3b48SJosef Bacik btrfs_item_offset(right, push_items - 1), push_space); 34025f39d397SChris Mason old_left_nritems = btrfs_header_nritems(left); 340387b29b20SYan Zheng BUG_ON(old_left_nritems <= 0); 3404eb60ceacSChris Mason 3405c82f823cSDavid Sterba btrfs_init_map_token(&token, left); 34063212fa14SJosef Bacik old_left_item_size = btrfs_item_offset(left, old_left_nritems - 1); 3407be0e5c09SChris Mason for (i = old_left_nritems; i < old_left_nritems + push_items; i++) { 34085f39d397SChris Mason u32 ioff; 3409db94535dSChris Mason 34103212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 34113212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, 3412cc4c13d5SDavid Sterba ioff - (BTRFS_LEAF_DATA_SIZE(fs_info) - old_left_item_size)); 3413be0e5c09SChris Mason } 34145f39d397SChris Mason btrfs_set_header_nritems(left, old_left_nritems + push_items); 3415be0e5c09SChris Mason 3416be0e5c09SChris Mason /* fixup right node */ 341731b1a2bdSJulia Lawall if (push_items > right_nritems) 341831b1a2bdSJulia Lawall WARN(1, KERN_CRIT "push items %d nr %u\n", push_items, 3419d397712bSChris Mason right_nritems); 342034a38218SChris Mason 342134a38218SChris Mason if (push_items < right_nritems) { 34223212fa14SJosef Bacik push_space = btrfs_item_offset(right, push_items - 1) - 34238f881e8cSDavid Sterba leaf_data_end(right); 3424637e3b48SJosef Bacik memmove_leaf_data(right, 34250b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - push_space, 34268f881e8cSDavid Sterba leaf_data_end(right), push_space); 34275f39d397SChris Mason 3428637e3b48SJosef Bacik memmove_leaf_items(right, 0, push_items, 3429637e3b48SJosef Bacik btrfs_header_nritems(right) - push_items); 343034a38218SChris Mason } 3431c82f823cSDavid Sterba 3432c82f823cSDavid Sterba btrfs_init_map_token(&token, right); 3433eef1c494SYan right_nritems -= push_items; 3434eef1c494SYan btrfs_set_header_nritems(right, right_nritems); 34350b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info); 34365f39d397SChris Mason for (i = 0; i < right_nritems; i++) { 34373212fa14SJosef Bacik push_space = push_space - btrfs_token_item_size(&token, i); 34383212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, push_space); 3439db94535dSChris Mason } 3440eb60ceacSChris Mason 34415f39d397SChris Mason btrfs_mark_buffer_dirty(left); 344234a38218SChris Mason if (right_nritems) 34435f39d397SChris Mason btrfs_mark_buffer_dirty(right); 3444f0486c68SYan, Zheng else 3445190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, right); 3446098f59c2SChris Mason 34475f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 3448b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 3449be0e5c09SChris Mason 3450be0e5c09SChris Mason /* then fixup the leaf pointer in the path */ 3451be0e5c09SChris Mason if (path->slots[0] < push_items) { 3452be0e5c09SChris Mason path->slots[0] += old_left_nritems; 3453925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 34545f39d397SChris Mason free_extent_buffer(path->nodes[0]); 34555f39d397SChris Mason path->nodes[0] = left; 3456be0e5c09SChris Mason path->slots[1] -= 1; 3457be0e5c09SChris Mason } else { 3458925baeddSChris Mason btrfs_tree_unlock(left); 34595f39d397SChris Mason free_extent_buffer(left); 3460be0e5c09SChris Mason path->slots[0] -= push_items; 3461be0e5c09SChris Mason } 3462eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 3463aa5d6bedSChris Mason return ret; 3464925baeddSChris Mason out: 3465925baeddSChris Mason btrfs_tree_unlock(left); 3466925baeddSChris Mason free_extent_buffer(left); 3467925baeddSChris Mason return ret; 3468be0e5c09SChris Mason } 3469be0e5c09SChris Mason 347074123bd7SChris Mason /* 347144871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 347244871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 347399d8f83cSChris Mason * 347499d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 347599d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us push all the 347699d8f83cSChris Mason * items 347744871b1bSChris Mason */ 347844871b1bSChris Mason static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root 347999d8f83cSChris Mason *root, struct btrfs_path *path, int min_data_size, 348099d8f83cSChris Mason int data_size, int empty, u32 max_slot) 348144871b1bSChris Mason { 348244871b1bSChris Mason struct extent_buffer *right = path->nodes[0]; 348344871b1bSChris Mason struct extent_buffer *left; 348444871b1bSChris Mason int slot; 348544871b1bSChris Mason int free_space; 348644871b1bSChris Mason u32 right_nritems; 348744871b1bSChris Mason int ret = 0; 348844871b1bSChris Mason 348944871b1bSChris Mason slot = path->slots[1]; 349044871b1bSChris Mason if (slot == 0) 349144871b1bSChris Mason return 1; 349244871b1bSChris Mason if (!path->nodes[1]) 349344871b1bSChris Mason return 1; 349444871b1bSChris Mason 349544871b1bSChris Mason right_nritems = btrfs_header_nritems(right); 349644871b1bSChris Mason if (right_nritems == 0) 349744871b1bSChris Mason return 1; 349844871b1bSChris Mason 349949d0c642SFilipe Manana btrfs_assert_tree_write_locked(path->nodes[1]); 350044871b1bSChris Mason 35014b231ae4SDavid Sterba left = btrfs_read_node_slot(path->nodes[1], slot - 1); 3502fb770ae4SLiu Bo if (IS_ERR(left)) 35039cf14029SJosef Bacik return PTR_ERR(left); 350491ca338dSTsutomu Itoh 3505bf77467aSJosef Bacik __btrfs_tree_lock(left, BTRFS_NESTING_LEFT); 350644871b1bSChris Mason 3507e902baacSDavid Sterba free_space = btrfs_leaf_free_space(left); 350844871b1bSChris Mason if (free_space < data_size) { 350944871b1bSChris Mason ret = 1; 351044871b1bSChris Mason goto out; 351144871b1bSChris Mason } 351244871b1bSChris Mason 351344871b1bSChris Mason ret = btrfs_cow_block(trans, root, left, 35149631e4ccSJosef Bacik path->nodes[1], slot - 1, &left, 3515bf59a5a2SJosef Bacik BTRFS_NESTING_LEFT_COW); 351644871b1bSChris Mason if (ret) { 351744871b1bSChris Mason /* we hit -ENOSPC, but it isn't fatal here */ 351879787eaaSJeff Mahoney if (ret == -ENOSPC) 351944871b1bSChris Mason ret = 1; 352044871b1bSChris Mason goto out; 352144871b1bSChris Mason } 352244871b1bSChris Mason 3523d16c702fSQu Wenruo if (check_sibling_keys(left, right)) { 3524d16c702fSQu Wenruo ret = -EUCLEAN; 35259ae5afd0SFilipe Manana btrfs_abort_transaction(trans, ret); 3526d16c702fSQu Wenruo goto out; 3527d16c702fSQu Wenruo } 3528ed25dab3SJosef Bacik return __push_leaf_left(trans, path, min_data_size, empty, left, 3529ed25dab3SJosef Bacik free_space, right_nritems, max_slot); 353044871b1bSChris Mason out: 353144871b1bSChris Mason btrfs_tree_unlock(left); 353244871b1bSChris Mason free_extent_buffer(left); 353344871b1bSChris Mason return ret; 353444871b1bSChris Mason } 353544871b1bSChris Mason 353644871b1bSChris Mason /* 353774123bd7SChris Mason * split the path's leaf in two, making sure there is at least data_size 353874123bd7SChris Mason * available for the resulting leaf level of the path. 353974123bd7SChris Mason */ 3540143bede5SJeff Mahoney static noinline void copy_for_split(struct btrfs_trans_handle *trans, 354144871b1bSChris Mason struct btrfs_path *path, 354244871b1bSChris Mason struct extent_buffer *l, 354344871b1bSChris Mason struct extent_buffer *right, 354444871b1bSChris Mason int slot, int mid, int nritems) 3545be0e5c09SChris Mason { 354694f94ad9SDavid Sterba struct btrfs_fs_info *fs_info = trans->fs_info; 3547be0e5c09SChris Mason int data_copy_size; 3548be0e5c09SChris Mason int rt_data_off; 3549be0e5c09SChris Mason int i; 3550d4dbff95SChris Mason struct btrfs_disk_key disk_key; 3551cfed81a0SChris Mason struct btrfs_map_token token; 3552cfed81a0SChris Mason 35535f39d397SChris Mason nritems = nritems - mid; 35545f39d397SChris Mason btrfs_set_header_nritems(right, nritems); 3555dc2e724eSJosef Bacik data_copy_size = btrfs_item_data_end(l, mid) - leaf_data_end(l); 35565f39d397SChris Mason 3557637e3b48SJosef Bacik copy_leaf_items(right, l, 0, mid, nritems); 35585f39d397SChris Mason 3559637e3b48SJosef Bacik copy_leaf_data(right, l, BTRFS_LEAF_DATA_SIZE(fs_info) - data_copy_size, 35608f881e8cSDavid Sterba leaf_data_end(l), data_copy_size); 356174123bd7SChris Mason 3562dc2e724eSJosef Bacik rt_data_off = BTRFS_LEAF_DATA_SIZE(fs_info) - btrfs_item_data_end(l, mid); 35635f39d397SChris Mason 3564c82f823cSDavid Sterba btrfs_init_map_token(&token, right); 35655f39d397SChris Mason for (i = 0; i < nritems; i++) { 3566db94535dSChris Mason u32 ioff; 3567db94535dSChris Mason 35683212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 35693212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff + rt_data_off); 35700783fcfcSChris Mason } 357174123bd7SChris Mason 35725f39d397SChris Mason btrfs_set_header_nritems(l, mid); 35735f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 35746ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, right->start, path->slots[1] + 1, 1); 35755f39d397SChris Mason 35765f39d397SChris Mason btrfs_mark_buffer_dirty(right); 35775f39d397SChris Mason btrfs_mark_buffer_dirty(l); 3578eb60ceacSChris Mason BUG_ON(path->slots[0] != slot); 35795f39d397SChris Mason 3580be0e5c09SChris Mason if (mid <= slot) { 3581925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 35825f39d397SChris Mason free_extent_buffer(path->nodes[0]); 35835f39d397SChris Mason path->nodes[0] = right; 3584be0e5c09SChris Mason path->slots[0] -= mid; 3585be0e5c09SChris Mason path->slots[1] += 1; 3586925baeddSChris Mason } else { 3587925baeddSChris Mason btrfs_tree_unlock(right); 35885f39d397SChris Mason free_extent_buffer(right); 3589925baeddSChris Mason } 35905f39d397SChris Mason 3591eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 359244871b1bSChris Mason } 359344871b1bSChris Mason 359444871b1bSChris Mason /* 359599d8f83cSChris Mason * double splits happen when we need to insert a big item in the middle 359699d8f83cSChris Mason * of a leaf. A double split can leave us with 3 mostly empty leaves: 359799d8f83cSChris Mason * leaf: [ slots 0 - N] [ our target ] [ N + 1 - total in leaf ] 359899d8f83cSChris Mason * A B C 359999d8f83cSChris Mason * 360099d8f83cSChris Mason * We avoid this by trying to push the items on either side of our target 360199d8f83cSChris Mason * into the adjacent leaves. If all goes well we can avoid the double split 360299d8f83cSChris Mason * completely. 360399d8f83cSChris Mason */ 360499d8f83cSChris Mason static noinline int push_for_double_split(struct btrfs_trans_handle *trans, 360599d8f83cSChris Mason struct btrfs_root *root, 360699d8f83cSChris Mason struct btrfs_path *path, 360799d8f83cSChris Mason int data_size) 360899d8f83cSChris Mason { 360999d8f83cSChris Mason int ret; 361099d8f83cSChris Mason int progress = 0; 361199d8f83cSChris Mason int slot; 361299d8f83cSChris Mason u32 nritems; 36135a4267caSFilipe David Borba Manana int space_needed = data_size; 361499d8f83cSChris Mason 361599d8f83cSChris Mason slot = path->slots[0]; 36165a4267caSFilipe David Borba Manana if (slot < btrfs_header_nritems(path->nodes[0])) 3617e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(path->nodes[0]); 361899d8f83cSChris Mason 361999d8f83cSChris Mason /* 362099d8f83cSChris Mason * try to push all the items after our slot into the 362199d8f83cSChris Mason * right leaf 362299d8f83cSChris Mason */ 36235a4267caSFilipe David Borba Manana ret = push_leaf_right(trans, root, path, 1, space_needed, 0, slot); 362499d8f83cSChris Mason if (ret < 0) 362599d8f83cSChris Mason return ret; 362699d8f83cSChris Mason 362799d8f83cSChris Mason if (ret == 0) 362899d8f83cSChris Mason progress++; 362999d8f83cSChris Mason 363099d8f83cSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 363199d8f83cSChris Mason /* 363299d8f83cSChris Mason * our goal is to get our slot at the start or end of a leaf. If 363399d8f83cSChris Mason * we've done so we're done 363499d8f83cSChris Mason */ 363599d8f83cSChris Mason if (path->slots[0] == 0 || path->slots[0] == nritems) 363699d8f83cSChris Mason return 0; 363799d8f83cSChris Mason 3638e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= data_size) 363999d8f83cSChris Mason return 0; 364099d8f83cSChris Mason 364199d8f83cSChris Mason /* try to push all the items before our slot into the next leaf */ 364299d8f83cSChris Mason slot = path->slots[0]; 3643263d3995SFilipe Manana space_needed = data_size; 3644263d3995SFilipe Manana if (slot > 0) 3645e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(path->nodes[0]); 36465a4267caSFilipe David Borba Manana ret = push_leaf_left(trans, root, path, 1, space_needed, 0, slot); 364799d8f83cSChris Mason if (ret < 0) 364899d8f83cSChris Mason return ret; 364999d8f83cSChris Mason 365099d8f83cSChris Mason if (ret == 0) 365199d8f83cSChris Mason progress++; 365299d8f83cSChris Mason 365399d8f83cSChris Mason if (progress) 365499d8f83cSChris Mason return 0; 365599d8f83cSChris Mason return 1; 365699d8f83cSChris Mason } 365799d8f83cSChris Mason 365899d8f83cSChris Mason /* 365944871b1bSChris Mason * split the path's leaf in two, making sure there is at least data_size 366044871b1bSChris Mason * available for the resulting leaf level of the path. 366144871b1bSChris Mason * 366244871b1bSChris Mason * returns 0 if all went well and < 0 on failure. 366344871b1bSChris Mason */ 366444871b1bSChris Mason static noinline int split_leaf(struct btrfs_trans_handle *trans, 366544871b1bSChris Mason struct btrfs_root *root, 3666310712b2SOmar Sandoval const struct btrfs_key *ins_key, 366744871b1bSChris Mason struct btrfs_path *path, int data_size, 366844871b1bSChris Mason int extend) 366944871b1bSChris Mason { 36705d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 367144871b1bSChris Mason struct extent_buffer *l; 367244871b1bSChris Mason u32 nritems; 367344871b1bSChris Mason int mid; 367444871b1bSChris Mason int slot; 367544871b1bSChris Mason struct extent_buffer *right; 3676b7a0365eSDaniel Dressler struct btrfs_fs_info *fs_info = root->fs_info; 367744871b1bSChris Mason int ret = 0; 367844871b1bSChris Mason int wret; 36795d4f98a2SYan Zheng int split; 368044871b1bSChris Mason int num_doubles = 0; 368199d8f83cSChris Mason int tried_avoid_double = 0; 368244871b1bSChris Mason 3683a5719521SYan, Zheng l = path->nodes[0]; 3684a5719521SYan, Zheng slot = path->slots[0]; 36853212fa14SJosef Bacik if (extend && data_size + btrfs_item_size(l, slot) + 36860b246afaSJeff Mahoney sizeof(struct btrfs_item) > BTRFS_LEAF_DATA_SIZE(fs_info)) 3687a5719521SYan, Zheng return -EOVERFLOW; 3688a5719521SYan, Zheng 368944871b1bSChris Mason /* first try to make some room by pushing left and right */ 369033157e05SLiu Bo if (data_size && path->nodes[1]) { 36915a4267caSFilipe David Borba Manana int space_needed = data_size; 36925a4267caSFilipe David Borba Manana 36935a4267caSFilipe David Borba Manana if (slot < btrfs_header_nritems(l)) 3694e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(l); 36955a4267caSFilipe David Borba Manana 36965a4267caSFilipe David Borba Manana wret = push_leaf_right(trans, root, path, space_needed, 36975a4267caSFilipe David Borba Manana space_needed, 0, 0); 369844871b1bSChris Mason if (wret < 0) 369944871b1bSChris Mason return wret; 370044871b1bSChris Mason if (wret) { 3701263d3995SFilipe Manana space_needed = data_size; 3702263d3995SFilipe Manana if (slot > 0) 3703e902baacSDavid Sterba space_needed -= btrfs_leaf_free_space(l); 37045a4267caSFilipe David Borba Manana wret = push_leaf_left(trans, root, path, space_needed, 37055a4267caSFilipe David Borba Manana space_needed, 0, (u32)-1); 370644871b1bSChris Mason if (wret < 0) 370744871b1bSChris Mason return wret; 370844871b1bSChris Mason } 370944871b1bSChris Mason l = path->nodes[0]; 371044871b1bSChris Mason 371144871b1bSChris Mason /* did the pushes work? */ 3712e902baacSDavid Sterba if (btrfs_leaf_free_space(l) >= data_size) 371344871b1bSChris Mason return 0; 371444871b1bSChris Mason } 371544871b1bSChris Mason 371644871b1bSChris Mason if (!path->nodes[1]) { 3717fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, 1); 371844871b1bSChris Mason if (ret) 371944871b1bSChris Mason return ret; 372044871b1bSChris Mason } 372144871b1bSChris Mason again: 37225d4f98a2SYan Zheng split = 1; 372344871b1bSChris Mason l = path->nodes[0]; 372444871b1bSChris Mason slot = path->slots[0]; 372544871b1bSChris Mason nritems = btrfs_header_nritems(l); 372644871b1bSChris Mason mid = (nritems + 1) / 2; 372744871b1bSChris Mason 37285d4f98a2SYan Zheng if (mid <= slot) { 37295d4f98a2SYan Zheng if (nritems == 1 || 37305d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + data_size > 37310b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info)) { 37325d4f98a2SYan Zheng if (slot >= nritems) { 37335d4f98a2SYan Zheng split = 0; 37345d4f98a2SYan Zheng } else { 37355d4f98a2SYan Zheng mid = slot; 37365d4f98a2SYan Zheng if (mid != nritems && 37375d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 37380b246afaSJeff Mahoney data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) { 373999d8f83cSChris Mason if (data_size && !tried_avoid_double) 374099d8f83cSChris Mason goto push_for_double; 37415d4f98a2SYan Zheng split = 2; 37425d4f98a2SYan Zheng } 37435d4f98a2SYan Zheng } 37445d4f98a2SYan Zheng } 37455d4f98a2SYan Zheng } else { 37465d4f98a2SYan Zheng if (leaf_space_used(l, 0, mid) + data_size > 37470b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info)) { 37485d4f98a2SYan Zheng if (!extend && data_size && slot == 0) { 37495d4f98a2SYan Zheng split = 0; 37505d4f98a2SYan Zheng } else if ((extend || !data_size) && slot == 0) { 37515d4f98a2SYan Zheng mid = 1; 37525d4f98a2SYan Zheng } else { 37535d4f98a2SYan Zheng mid = slot; 37545d4f98a2SYan Zheng if (mid != nritems && 37555d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 37560b246afaSJeff Mahoney data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) { 375799d8f83cSChris Mason if (data_size && !tried_avoid_double) 375899d8f83cSChris Mason goto push_for_double; 37595d4f98a2SYan Zheng split = 2; 37605d4f98a2SYan Zheng } 37615d4f98a2SYan Zheng } 37625d4f98a2SYan Zheng } 37635d4f98a2SYan Zheng } 37645d4f98a2SYan Zheng 37655d4f98a2SYan Zheng if (split == 0) 37665d4f98a2SYan Zheng btrfs_cpu_key_to_disk(&disk_key, ins_key); 37675d4f98a2SYan Zheng else 37685d4f98a2SYan Zheng btrfs_item_key(l, &disk_key, mid); 37695d4f98a2SYan Zheng 3770ca9d473aSJosef Bacik /* 3771ca9d473aSJosef Bacik * We have to about BTRFS_NESTING_NEW_ROOT here if we've done a double 3772ca9d473aSJosef Bacik * split, because we're only allowed to have MAX_LOCKDEP_SUBCLASSES 3773ca9d473aSJosef Bacik * subclasses, which is 8 at the time of this patch, and we've maxed it 3774ca9d473aSJosef Bacik * out. In the future we could add a 3775ca9d473aSJosef Bacik * BTRFS_NESTING_SPLIT_THE_SPLITTENING if we need to, but for now just 3776ca9d473aSJosef Bacik * use BTRFS_NESTING_NEW_ROOT. 3777ca9d473aSJosef Bacik */ 377879bd3712SFilipe Manana right = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 377979bd3712SFilipe Manana &disk_key, 0, l->start, 0, 378079bd3712SFilipe Manana num_doubles ? BTRFS_NESTING_NEW_ROOT : 3781ca9d473aSJosef Bacik BTRFS_NESTING_SPLIT); 3782f0486c68SYan, Zheng if (IS_ERR(right)) 378344871b1bSChris Mason return PTR_ERR(right); 3784f0486c68SYan, Zheng 37850b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 378644871b1bSChris Mason 37875d4f98a2SYan Zheng if (split == 0) { 378844871b1bSChris Mason if (mid <= slot) { 378944871b1bSChris Mason btrfs_set_header_nritems(right, 0); 37906ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, 37912ff7e61eSJeff Mahoney right->start, path->slots[1] + 1, 1); 379244871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 379344871b1bSChris Mason free_extent_buffer(path->nodes[0]); 379444871b1bSChris Mason path->nodes[0] = right; 379544871b1bSChris Mason path->slots[0] = 0; 379644871b1bSChris Mason path->slots[1] += 1; 379744871b1bSChris Mason } else { 379844871b1bSChris Mason btrfs_set_header_nritems(right, 0); 37996ad3cf6dSDavid Sterba insert_ptr(trans, path, &disk_key, 38002ff7e61eSJeff Mahoney right->start, path->slots[1], 1); 380144871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 380244871b1bSChris Mason free_extent_buffer(path->nodes[0]); 380344871b1bSChris Mason path->nodes[0] = right; 380444871b1bSChris Mason path->slots[0] = 0; 3805143bede5SJeff Mahoney if (path->slots[1] == 0) 3806b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 38075d4f98a2SYan Zheng } 3808196e0249SLiu Bo /* 3809196e0249SLiu Bo * We create a new leaf 'right' for the required ins_len and 3810196e0249SLiu Bo * we'll do btrfs_mark_buffer_dirty() on this leaf after copying 3811196e0249SLiu Bo * the content of ins_len to 'right'. 3812196e0249SLiu Bo */ 381344871b1bSChris Mason return ret; 381444871b1bSChris Mason } 381544871b1bSChris Mason 381694f94ad9SDavid Sterba copy_for_split(trans, path, l, right, slot, mid, nritems); 381744871b1bSChris Mason 38185d4f98a2SYan Zheng if (split == 2) { 3819cc0c5538SChris Mason BUG_ON(num_doubles != 0); 3820cc0c5538SChris Mason num_doubles++; 3821cc0c5538SChris Mason goto again; 38223326d1b0SChris Mason } 382344871b1bSChris Mason 3824143bede5SJeff Mahoney return 0; 382599d8f83cSChris Mason 382699d8f83cSChris Mason push_for_double: 382799d8f83cSChris Mason push_for_double_split(trans, root, path, data_size); 382899d8f83cSChris Mason tried_avoid_double = 1; 3829e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= data_size) 383099d8f83cSChris Mason return 0; 383199d8f83cSChris Mason goto again; 3832be0e5c09SChris Mason } 3833be0e5c09SChris Mason 3834ad48fd75SYan, Zheng static noinline int setup_leaf_for_split(struct btrfs_trans_handle *trans, 3835ad48fd75SYan, Zheng struct btrfs_root *root, 3836ad48fd75SYan, Zheng struct btrfs_path *path, int ins_len) 3837ad48fd75SYan, Zheng { 3838ad48fd75SYan, Zheng struct btrfs_key key; 3839ad48fd75SYan, Zheng struct extent_buffer *leaf; 3840ad48fd75SYan, Zheng struct btrfs_file_extent_item *fi; 3841ad48fd75SYan, Zheng u64 extent_len = 0; 3842ad48fd75SYan, Zheng u32 item_size; 3843ad48fd75SYan, Zheng int ret; 3844ad48fd75SYan, Zheng 3845ad48fd75SYan, Zheng leaf = path->nodes[0]; 3846ad48fd75SYan, Zheng btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 3847ad48fd75SYan, Zheng 3848ad48fd75SYan, Zheng BUG_ON(key.type != BTRFS_EXTENT_DATA_KEY && 3849ad48fd75SYan, Zheng key.type != BTRFS_EXTENT_CSUM_KEY); 3850ad48fd75SYan, Zheng 3851e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) >= ins_len) 3852ad48fd75SYan, Zheng return 0; 3853ad48fd75SYan, Zheng 38543212fa14SJosef Bacik item_size = btrfs_item_size(leaf, path->slots[0]); 3855ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 3856ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 3857ad48fd75SYan, Zheng struct btrfs_file_extent_item); 3858ad48fd75SYan, Zheng extent_len = btrfs_file_extent_num_bytes(leaf, fi); 3859ad48fd75SYan, Zheng } 3860b3b4aa74SDavid Sterba btrfs_release_path(path); 3861ad48fd75SYan, Zheng 3862ad48fd75SYan, Zheng path->keep_locks = 1; 3863ad48fd75SYan, Zheng path->search_for_split = 1; 3864ad48fd75SYan, Zheng ret = btrfs_search_slot(trans, root, &key, path, 0, 1); 3865ad48fd75SYan, Zheng path->search_for_split = 0; 3866a8df6fe6SFilipe Manana if (ret > 0) 3867a8df6fe6SFilipe Manana ret = -EAGAIN; 3868ad48fd75SYan, Zheng if (ret < 0) 3869ad48fd75SYan, Zheng goto err; 3870ad48fd75SYan, Zheng 3871ad48fd75SYan, Zheng ret = -EAGAIN; 3872ad48fd75SYan, Zheng leaf = path->nodes[0]; 3873a8df6fe6SFilipe Manana /* if our item isn't there, return now */ 38743212fa14SJosef Bacik if (item_size != btrfs_item_size(leaf, path->slots[0])) 3875ad48fd75SYan, Zheng goto err; 3876ad48fd75SYan, Zheng 3877109f6aefSChris Mason /* the leaf has changed, it now has room. return now */ 3878e902baacSDavid Sterba if (btrfs_leaf_free_space(path->nodes[0]) >= ins_len) 3879109f6aefSChris Mason goto err; 3880109f6aefSChris Mason 3881ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 3882ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 3883ad48fd75SYan, Zheng struct btrfs_file_extent_item); 3884ad48fd75SYan, Zheng if (extent_len != btrfs_file_extent_num_bytes(leaf, fi)) 3885ad48fd75SYan, Zheng goto err; 3886ad48fd75SYan, Zheng } 3887ad48fd75SYan, Zheng 3888ad48fd75SYan, Zheng ret = split_leaf(trans, root, &key, path, ins_len, 1); 3889f0486c68SYan, Zheng if (ret) 3890f0486c68SYan, Zheng goto err; 3891ad48fd75SYan, Zheng 3892ad48fd75SYan, Zheng path->keep_locks = 0; 3893ad48fd75SYan, Zheng btrfs_unlock_up_safe(path, 1); 3894ad48fd75SYan, Zheng return 0; 3895ad48fd75SYan, Zheng err: 3896ad48fd75SYan, Zheng path->keep_locks = 0; 3897ad48fd75SYan, Zheng return ret; 3898ad48fd75SYan, Zheng } 3899ad48fd75SYan, Zheng 390025263cd7SDavid Sterba static noinline int split_item(struct btrfs_path *path, 3901310712b2SOmar Sandoval const struct btrfs_key *new_key, 3902459931ecSChris Mason unsigned long split_offset) 3903459931ecSChris Mason { 3904459931ecSChris Mason struct extent_buffer *leaf; 3905c91666b1SJosef Bacik int orig_slot, slot; 3906ad48fd75SYan, Zheng char *buf; 3907459931ecSChris Mason u32 nritems; 3908ad48fd75SYan, Zheng u32 item_size; 3909459931ecSChris Mason u32 orig_offset; 3910459931ecSChris Mason struct btrfs_disk_key disk_key; 3911459931ecSChris Mason 3912459931ecSChris Mason leaf = path->nodes[0]; 3913e902baacSDavid Sterba BUG_ON(btrfs_leaf_free_space(leaf) < sizeof(struct btrfs_item)); 3914b9473439SChris Mason 3915c91666b1SJosef Bacik orig_slot = path->slots[0]; 39163212fa14SJosef Bacik orig_offset = btrfs_item_offset(leaf, path->slots[0]); 39173212fa14SJosef Bacik item_size = btrfs_item_size(leaf, path->slots[0]); 3918459931ecSChris Mason 3919459931ecSChris Mason buf = kmalloc(item_size, GFP_NOFS); 3920ad48fd75SYan, Zheng if (!buf) 3921ad48fd75SYan, Zheng return -ENOMEM; 3922ad48fd75SYan, Zheng 3923459931ecSChris Mason read_extent_buffer(leaf, buf, btrfs_item_ptr_offset(leaf, 3924459931ecSChris Mason path->slots[0]), item_size); 3925ad48fd75SYan, Zheng 3926459931ecSChris Mason slot = path->slots[0] + 1; 3927459931ecSChris Mason nritems = btrfs_header_nritems(leaf); 3928459931ecSChris Mason if (slot != nritems) { 3929459931ecSChris Mason /* shift the items */ 3930637e3b48SJosef Bacik memmove_leaf_items(leaf, slot + 1, slot, nritems - slot); 3931459931ecSChris Mason } 3932459931ecSChris Mason 3933459931ecSChris Mason btrfs_cpu_key_to_disk(&disk_key, new_key); 3934459931ecSChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 3935459931ecSChris Mason 39363212fa14SJosef Bacik btrfs_set_item_offset(leaf, slot, orig_offset); 39373212fa14SJosef Bacik btrfs_set_item_size(leaf, slot, item_size - split_offset); 3938459931ecSChris Mason 39393212fa14SJosef Bacik btrfs_set_item_offset(leaf, orig_slot, 3940459931ecSChris Mason orig_offset + item_size - split_offset); 39413212fa14SJosef Bacik btrfs_set_item_size(leaf, orig_slot, split_offset); 3942459931ecSChris Mason 3943459931ecSChris Mason btrfs_set_header_nritems(leaf, nritems + 1); 3944459931ecSChris Mason 3945459931ecSChris Mason /* write the data for the start of the original item */ 3946459931ecSChris Mason write_extent_buffer(leaf, buf, 3947459931ecSChris Mason btrfs_item_ptr_offset(leaf, path->slots[0]), 3948459931ecSChris Mason split_offset); 3949459931ecSChris Mason 3950459931ecSChris Mason /* write the data for the new item */ 3951459931ecSChris Mason write_extent_buffer(leaf, buf + split_offset, 3952459931ecSChris Mason btrfs_item_ptr_offset(leaf, slot), 3953459931ecSChris Mason item_size - split_offset); 3954459931ecSChris Mason btrfs_mark_buffer_dirty(leaf); 3955459931ecSChris Mason 3956e902baacSDavid Sterba BUG_ON(btrfs_leaf_free_space(leaf) < 0); 3957459931ecSChris Mason kfree(buf); 3958ad48fd75SYan, Zheng return 0; 3959ad48fd75SYan, Zheng } 3960ad48fd75SYan, Zheng 3961ad48fd75SYan, Zheng /* 3962ad48fd75SYan, Zheng * This function splits a single item into two items, 3963ad48fd75SYan, Zheng * giving 'new_key' to the new item and splitting the 3964ad48fd75SYan, Zheng * old one at split_offset (from the start of the item). 3965ad48fd75SYan, Zheng * 3966ad48fd75SYan, Zheng * The path may be released by this operation. After 3967ad48fd75SYan, Zheng * the split, the path is pointing to the old item. The 3968ad48fd75SYan, Zheng * new item is going to be in the same node as the old one. 3969ad48fd75SYan, Zheng * 3970ad48fd75SYan, Zheng * Note, the item being split must be smaller enough to live alone on 3971ad48fd75SYan, Zheng * a tree block with room for one extra struct btrfs_item 3972ad48fd75SYan, Zheng * 3973ad48fd75SYan, Zheng * This allows us to split the item in place, keeping a lock on the 3974ad48fd75SYan, Zheng * leaf the entire time. 3975ad48fd75SYan, Zheng */ 3976ad48fd75SYan, Zheng int btrfs_split_item(struct btrfs_trans_handle *trans, 3977ad48fd75SYan, Zheng struct btrfs_root *root, 3978ad48fd75SYan, Zheng struct btrfs_path *path, 3979310712b2SOmar Sandoval const struct btrfs_key *new_key, 3980ad48fd75SYan, Zheng unsigned long split_offset) 3981ad48fd75SYan, Zheng { 3982ad48fd75SYan, Zheng int ret; 3983ad48fd75SYan, Zheng ret = setup_leaf_for_split(trans, root, path, 3984ad48fd75SYan, Zheng sizeof(struct btrfs_item)); 3985ad48fd75SYan, Zheng if (ret) 3986459931ecSChris Mason return ret; 3987ad48fd75SYan, Zheng 398825263cd7SDavid Sterba ret = split_item(path, new_key, split_offset); 3989ad48fd75SYan, Zheng return ret; 3990ad48fd75SYan, Zheng } 3991ad48fd75SYan, Zheng 3992ad48fd75SYan, Zheng /* 3993d352ac68SChris Mason * make the item pointed to by the path smaller. new_size indicates 3994d352ac68SChris Mason * how small to make it, and from_end tells us if we just chop bytes 3995d352ac68SChris Mason * off the end of the item or if we shift the item to chop bytes off 3996d352ac68SChris Mason * the front. 3997d352ac68SChris Mason */ 399878ac4f9eSDavid Sterba void btrfs_truncate_item(struct btrfs_path *path, u32 new_size, int from_end) 3999b18c6685SChris Mason { 4000b18c6685SChris Mason int slot; 40015f39d397SChris Mason struct extent_buffer *leaf; 4002b18c6685SChris Mason u32 nritems; 4003b18c6685SChris Mason unsigned int data_end; 4004b18c6685SChris Mason unsigned int old_data_start; 4005b18c6685SChris Mason unsigned int old_size; 4006b18c6685SChris Mason unsigned int size_diff; 4007b18c6685SChris Mason int i; 4008cfed81a0SChris Mason struct btrfs_map_token token; 4009cfed81a0SChris Mason 40105f39d397SChris Mason leaf = path->nodes[0]; 4011179e29e4SChris Mason slot = path->slots[0]; 4012179e29e4SChris Mason 40133212fa14SJosef Bacik old_size = btrfs_item_size(leaf, slot); 4014179e29e4SChris Mason if (old_size == new_size) 4015143bede5SJeff Mahoney return; 4016b18c6685SChris Mason 40175f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 40188f881e8cSDavid Sterba data_end = leaf_data_end(leaf); 4019b18c6685SChris Mason 40203212fa14SJosef Bacik old_data_start = btrfs_item_offset(leaf, slot); 4021179e29e4SChris Mason 4022b18c6685SChris Mason size_diff = old_size - new_size; 4023b18c6685SChris Mason 4024b18c6685SChris Mason BUG_ON(slot < 0); 4025b18c6685SChris Mason BUG_ON(slot >= nritems); 4026b18c6685SChris Mason 4027b18c6685SChris Mason /* 4028b18c6685SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 4029b18c6685SChris Mason */ 4030b18c6685SChris Mason /* first correct the data pointers */ 4031c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 4032b18c6685SChris Mason for (i = slot; i < nritems; i++) { 40335f39d397SChris Mason u32 ioff; 4034db94535dSChris Mason 40353212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 40363212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff + size_diff); 4037b18c6685SChris Mason } 4038db94535dSChris Mason 4039b18c6685SChris Mason /* shift the data */ 4040179e29e4SChris Mason if (from_end) { 4041637e3b48SJosef Bacik memmove_leaf_data(leaf, data_end + size_diff, data_end, 4042637e3b48SJosef Bacik old_data_start + new_size - data_end); 4043179e29e4SChris Mason } else { 4044179e29e4SChris Mason struct btrfs_disk_key disk_key; 4045179e29e4SChris Mason u64 offset; 4046179e29e4SChris Mason 4047179e29e4SChris Mason btrfs_item_key(leaf, &disk_key, slot); 4048179e29e4SChris Mason 4049179e29e4SChris Mason if (btrfs_disk_key_type(&disk_key) == BTRFS_EXTENT_DATA_KEY) { 4050179e29e4SChris Mason unsigned long ptr; 4051179e29e4SChris Mason struct btrfs_file_extent_item *fi; 4052179e29e4SChris Mason 4053179e29e4SChris Mason fi = btrfs_item_ptr(leaf, slot, 4054179e29e4SChris Mason struct btrfs_file_extent_item); 4055179e29e4SChris Mason fi = (struct btrfs_file_extent_item *)( 4056179e29e4SChris Mason (unsigned long)fi - size_diff); 4057179e29e4SChris Mason 4058179e29e4SChris Mason if (btrfs_file_extent_type(leaf, fi) == 4059179e29e4SChris Mason BTRFS_FILE_EXTENT_INLINE) { 4060179e29e4SChris Mason ptr = btrfs_item_ptr_offset(leaf, slot); 4061179e29e4SChris Mason memmove_extent_buffer(leaf, ptr, 4062179e29e4SChris Mason (unsigned long)fi, 40637ec20afbSDavid Sterba BTRFS_FILE_EXTENT_INLINE_DATA_START); 4064179e29e4SChris Mason } 4065179e29e4SChris Mason } 4066179e29e4SChris Mason 4067637e3b48SJosef Bacik memmove_leaf_data(leaf, data_end + size_diff, data_end, 4068637e3b48SJosef Bacik old_data_start - data_end); 4069179e29e4SChris Mason 4070179e29e4SChris Mason offset = btrfs_disk_key_offset(&disk_key); 4071179e29e4SChris Mason btrfs_set_disk_key_offset(&disk_key, offset + size_diff); 4072179e29e4SChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 4073179e29e4SChris Mason if (slot == 0) 4074b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 4075179e29e4SChris Mason } 40765f39d397SChris Mason 40773212fa14SJosef Bacik btrfs_set_item_size(leaf, slot, new_size); 40785f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 4079b18c6685SChris Mason 4080e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) { 4081a4f78750SDavid Sterba btrfs_print_leaf(leaf); 4082b18c6685SChris Mason BUG(); 40835f39d397SChris Mason } 4084b18c6685SChris Mason } 4085b18c6685SChris Mason 4086d352ac68SChris Mason /* 40878f69dbd2SStefan Behrens * make the item pointed to by the path bigger, data_size is the added size. 4088d352ac68SChris Mason */ 4089c71dd880SDavid Sterba void btrfs_extend_item(struct btrfs_path *path, u32 data_size) 40906567e837SChris Mason { 40916567e837SChris Mason int slot; 40925f39d397SChris Mason struct extent_buffer *leaf; 40936567e837SChris Mason u32 nritems; 40946567e837SChris Mason unsigned int data_end; 40956567e837SChris Mason unsigned int old_data; 40966567e837SChris Mason unsigned int old_size; 40976567e837SChris Mason int i; 4098cfed81a0SChris Mason struct btrfs_map_token token; 4099cfed81a0SChris Mason 41005f39d397SChris Mason leaf = path->nodes[0]; 41016567e837SChris Mason 41025f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 41038f881e8cSDavid Sterba data_end = leaf_data_end(leaf); 41046567e837SChris Mason 4105e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < data_size) { 4106a4f78750SDavid Sterba btrfs_print_leaf(leaf); 41076567e837SChris Mason BUG(); 41085f39d397SChris Mason } 41096567e837SChris Mason slot = path->slots[0]; 4110dc2e724eSJosef Bacik old_data = btrfs_item_data_end(leaf, slot); 41116567e837SChris Mason 41126567e837SChris Mason BUG_ON(slot < 0); 41133326d1b0SChris Mason if (slot >= nritems) { 4114a4f78750SDavid Sterba btrfs_print_leaf(leaf); 4115c71dd880SDavid Sterba btrfs_crit(leaf->fs_info, "slot %d too large, nritems %d", 4116d397712bSChris Mason slot, nritems); 4117290342f6SArnd Bergmann BUG(); 41183326d1b0SChris Mason } 41196567e837SChris Mason 41206567e837SChris Mason /* 41216567e837SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 41226567e837SChris Mason */ 41236567e837SChris Mason /* first correct the data pointers */ 4124c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 41256567e837SChris Mason for (i = slot; i < nritems; i++) { 41265f39d397SChris Mason u32 ioff; 4127db94535dSChris Mason 41283212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 41293212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff - data_size); 41306567e837SChris Mason } 41315f39d397SChris Mason 41326567e837SChris Mason /* shift the data */ 4133637e3b48SJosef Bacik memmove_leaf_data(leaf, data_end - data_size, data_end, 4134637e3b48SJosef Bacik old_data - data_end); 41355f39d397SChris Mason 41366567e837SChris Mason data_end = old_data; 41373212fa14SJosef Bacik old_size = btrfs_item_size(leaf, slot); 41383212fa14SJosef Bacik btrfs_set_item_size(leaf, slot, old_size + data_size); 41395f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 41406567e837SChris Mason 4141e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) { 4142a4f78750SDavid Sterba btrfs_print_leaf(leaf); 41436567e837SChris Mason BUG(); 41445f39d397SChris Mason } 41456567e837SChris Mason } 41466567e837SChris Mason 414743dd529aSDavid Sterba /* 414843dd529aSDavid Sterba * Make space in the node before inserting one or more items. 4149da9ffb24SNikolay Borisov * 4150da9ffb24SNikolay Borisov * @root: root we are inserting items to 4151da9ffb24SNikolay Borisov * @path: points to the leaf/slot where we are going to insert new items 4152b7ef5f3aSFilipe Manana * @batch: information about the batch of items to insert 415343dd529aSDavid Sterba * 415443dd529aSDavid Sterba * Main purpose is to save stack depth by doing the bulk of the work in a 415543dd529aSDavid Sterba * function that doesn't call btrfs_search_slot 415674123bd7SChris Mason */ 4157f0641656SFilipe Manana static void setup_items_for_insert(struct btrfs_root *root, struct btrfs_path *path, 4158b7ef5f3aSFilipe Manana const struct btrfs_item_batch *batch) 4159be0e5c09SChris Mason { 41600b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 41619c58309dSChris Mason int i; 41627518a238SChris Mason u32 nritems; 4163be0e5c09SChris Mason unsigned int data_end; 4164e2fa7227SChris Mason struct btrfs_disk_key disk_key; 416544871b1bSChris Mason struct extent_buffer *leaf; 416644871b1bSChris Mason int slot; 4167cfed81a0SChris Mason struct btrfs_map_token token; 4168fc0d82e1SNikolay Borisov u32 total_size; 4169fc0d82e1SNikolay Borisov 4170b7ef5f3aSFilipe Manana /* 4171b7ef5f3aSFilipe Manana * Before anything else, update keys in the parent and other ancestors 4172b7ef5f3aSFilipe Manana * if needed, then release the write locks on them, so that other tasks 4173b7ef5f3aSFilipe Manana * can use them while we modify the leaf. 4174b7ef5f3aSFilipe Manana */ 417524cdc847SFilipe Manana if (path->slots[0] == 0) { 4176b7ef5f3aSFilipe Manana btrfs_cpu_key_to_disk(&disk_key, &batch->keys[0]); 4177b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 417824cdc847SFilipe Manana } 417924cdc847SFilipe Manana btrfs_unlock_up_safe(path, 1); 418024cdc847SFilipe Manana 41815f39d397SChris Mason leaf = path->nodes[0]; 418244871b1bSChris Mason slot = path->slots[0]; 418374123bd7SChris Mason 41845f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 41858f881e8cSDavid Sterba data_end = leaf_data_end(leaf); 4186b7ef5f3aSFilipe Manana total_size = batch->total_data_size + (batch->nr * sizeof(struct btrfs_item)); 4187eb60ceacSChris Mason 4188e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < total_size) { 4189a4f78750SDavid Sterba btrfs_print_leaf(leaf); 41900b246afaSJeff Mahoney btrfs_crit(fs_info, "not enough freespace need %u have %d", 4191e902baacSDavid Sterba total_size, btrfs_leaf_free_space(leaf)); 4192be0e5c09SChris Mason BUG(); 4193d4dbff95SChris Mason } 41945f39d397SChris Mason 4195c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 4196be0e5c09SChris Mason if (slot != nritems) { 4197dc2e724eSJosef Bacik unsigned int old_data = btrfs_item_data_end(leaf, slot); 4198be0e5c09SChris Mason 41995f39d397SChris Mason if (old_data < data_end) { 4200a4f78750SDavid Sterba btrfs_print_leaf(leaf); 42017269ddd2SNikolay Borisov btrfs_crit(fs_info, 42027269ddd2SNikolay Borisov "item at slot %d with data offset %u beyond data end of leaf %u", 42035f39d397SChris Mason slot, old_data, data_end); 4204290342f6SArnd Bergmann BUG(); 42055f39d397SChris Mason } 4206be0e5c09SChris Mason /* 4207be0e5c09SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 4208be0e5c09SChris Mason */ 4209be0e5c09SChris Mason /* first correct the data pointers */ 42100783fcfcSChris Mason for (i = slot; i < nritems; i++) { 42115f39d397SChris Mason u32 ioff; 4212db94535dSChris Mason 42133212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 42143212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, 4215b7ef5f3aSFilipe Manana ioff - batch->total_data_size); 42160783fcfcSChris Mason } 4217be0e5c09SChris Mason /* shift the items */ 4218637e3b48SJosef Bacik memmove_leaf_items(leaf, slot + batch->nr, slot, nritems - slot); 4219be0e5c09SChris Mason 4220be0e5c09SChris Mason /* shift the data */ 4221637e3b48SJosef Bacik memmove_leaf_data(leaf, data_end - batch->total_data_size, 4222637e3b48SJosef Bacik data_end, old_data - data_end); 4223be0e5c09SChris Mason data_end = old_data; 4224be0e5c09SChris Mason } 42255f39d397SChris Mason 422662e2749eSChris Mason /* setup the item for the new data */ 4227b7ef5f3aSFilipe Manana for (i = 0; i < batch->nr; i++) { 4228b7ef5f3aSFilipe Manana btrfs_cpu_key_to_disk(&disk_key, &batch->keys[i]); 42299c58309dSChris Mason btrfs_set_item_key(leaf, &disk_key, slot + i); 4230b7ef5f3aSFilipe Manana data_end -= batch->data_sizes[i]; 42313212fa14SJosef Bacik btrfs_set_token_item_offset(&token, slot + i, data_end); 42323212fa14SJosef Bacik btrfs_set_token_item_size(&token, slot + i, batch->data_sizes[i]); 42339c58309dSChris Mason } 423444871b1bSChris Mason 4235b7ef5f3aSFilipe Manana btrfs_set_header_nritems(leaf, nritems + batch->nr); 4236b9473439SChris Mason btrfs_mark_buffer_dirty(leaf); 4237aa5d6bedSChris Mason 4238e902baacSDavid Sterba if (btrfs_leaf_free_space(leaf) < 0) { 4239a4f78750SDavid Sterba btrfs_print_leaf(leaf); 4240be0e5c09SChris Mason BUG(); 42415f39d397SChris Mason } 424244871b1bSChris Mason } 424344871b1bSChris Mason 424444871b1bSChris Mason /* 4245f0641656SFilipe Manana * Insert a new item into a leaf. 4246f0641656SFilipe Manana * 4247f0641656SFilipe Manana * @root: The root of the btree. 4248f0641656SFilipe Manana * @path: A path pointing to the target leaf and slot. 4249f0641656SFilipe Manana * @key: The key of the new item. 4250f0641656SFilipe Manana * @data_size: The size of the data associated with the new key. 4251f0641656SFilipe Manana */ 4252f0641656SFilipe Manana void btrfs_setup_item_for_insert(struct btrfs_root *root, 4253f0641656SFilipe Manana struct btrfs_path *path, 4254f0641656SFilipe Manana const struct btrfs_key *key, 4255f0641656SFilipe Manana u32 data_size) 4256f0641656SFilipe Manana { 4257f0641656SFilipe Manana struct btrfs_item_batch batch; 4258f0641656SFilipe Manana 4259f0641656SFilipe Manana batch.keys = key; 4260f0641656SFilipe Manana batch.data_sizes = &data_size; 4261f0641656SFilipe Manana batch.total_data_size = data_size; 4262f0641656SFilipe Manana batch.nr = 1; 4263f0641656SFilipe Manana 4264f0641656SFilipe Manana setup_items_for_insert(root, path, &batch); 4265f0641656SFilipe Manana } 4266f0641656SFilipe Manana 4267f0641656SFilipe Manana /* 426844871b1bSChris Mason * Given a key and some data, insert items into the tree. 426944871b1bSChris Mason * This does all the path init required, making room in the tree if needed. 427044871b1bSChris Mason */ 427144871b1bSChris Mason int btrfs_insert_empty_items(struct btrfs_trans_handle *trans, 427244871b1bSChris Mason struct btrfs_root *root, 427344871b1bSChris Mason struct btrfs_path *path, 4274b7ef5f3aSFilipe Manana const struct btrfs_item_batch *batch) 427544871b1bSChris Mason { 427644871b1bSChris Mason int ret = 0; 427744871b1bSChris Mason int slot; 4278b7ef5f3aSFilipe Manana u32 total_size; 427944871b1bSChris Mason 4280b7ef5f3aSFilipe Manana total_size = batch->total_data_size + (batch->nr * sizeof(struct btrfs_item)); 4281b7ef5f3aSFilipe Manana ret = btrfs_search_slot(trans, root, &batch->keys[0], path, total_size, 1); 428244871b1bSChris Mason if (ret == 0) 428344871b1bSChris Mason return -EEXIST; 428444871b1bSChris Mason if (ret < 0) 4285143bede5SJeff Mahoney return ret; 428644871b1bSChris Mason 428744871b1bSChris Mason slot = path->slots[0]; 428844871b1bSChris Mason BUG_ON(slot < 0); 428944871b1bSChris Mason 4290b7ef5f3aSFilipe Manana setup_items_for_insert(root, path, batch); 4291143bede5SJeff Mahoney return 0; 429262e2749eSChris Mason } 429362e2749eSChris Mason 429462e2749eSChris Mason /* 429562e2749eSChris Mason * Given a key and some data, insert an item into the tree. 429662e2749eSChris Mason * This does all the path init required, making room in the tree if needed. 429762e2749eSChris Mason */ 4298310712b2SOmar Sandoval int btrfs_insert_item(struct btrfs_trans_handle *trans, struct btrfs_root *root, 4299310712b2SOmar Sandoval const struct btrfs_key *cpu_key, void *data, 4300310712b2SOmar Sandoval u32 data_size) 430162e2749eSChris Mason { 430262e2749eSChris Mason int ret = 0; 43032c90e5d6SChris Mason struct btrfs_path *path; 43045f39d397SChris Mason struct extent_buffer *leaf; 43055f39d397SChris Mason unsigned long ptr; 430662e2749eSChris Mason 43072c90e5d6SChris Mason path = btrfs_alloc_path(); 4308db5b493aSTsutomu Itoh if (!path) 4309db5b493aSTsutomu Itoh return -ENOMEM; 43102c90e5d6SChris Mason ret = btrfs_insert_empty_item(trans, root, path, cpu_key, data_size); 431162e2749eSChris Mason if (!ret) { 43125f39d397SChris Mason leaf = path->nodes[0]; 43135f39d397SChris Mason ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); 43145f39d397SChris Mason write_extent_buffer(leaf, data, ptr, data_size); 43155f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 431662e2749eSChris Mason } 43172c90e5d6SChris Mason btrfs_free_path(path); 4318aa5d6bedSChris Mason return ret; 4319be0e5c09SChris Mason } 4320be0e5c09SChris Mason 432174123bd7SChris Mason /* 4322f0641656SFilipe Manana * This function duplicates an item, giving 'new_key' to the new item. 4323f0641656SFilipe Manana * It guarantees both items live in the same tree leaf and the new item is 4324f0641656SFilipe Manana * contiguous with the original item. 4325f0641656SFilipe Manana * 4326f0641656SFilipe Manana * This allows us to split a file extent in place, keeping a lock on the leaf 4327f0641656SFilipe Manana * the entire time. 4328f0641656SFilipe Manana */ 4329f0641656SFilipe Manana int btrfs_duplicate_item(struct btrfs_trans_handle *trans, 4330f0641656SFilipe Manana struct btrfs_root *root, 4331f0641656SFilipe Manana struct btrfs_path *path, 4332f0641656SFilipe Manana const struct btrfs_key *new_key) 4333f0641656SFilipe Manana { 4334f0641656SFilipe Manana struct extent_buffer *leaf; 4335f0641656SFilipe Manana int ret; 4336f0641656SFilipe Manana u32 item_size; 4337f0641656SFilipe Manana 4338f0641656SFilipe Manana leaf = path->nodes[0]; 43393212fa14SJosef Bacik item_size = btrfs_item_size(leaf, path->slots[0]); 4340f0641656SFilipe Manana ret = setup_leaf_for_split(trans, root, path, 4341f0641656SFilipe Manana item_size + sizeof(struct btrfs_item)); 4342f0641656SFilipe Manana if (ret) 4343f0641656SFilipe Manana return ret; 4344f0641656SFilipe Manana 4345f0641656SFilipe Manana path->slots[0]++; 4346f0641656SFilipe Manana btrfs_setup_item_for_insert(root, path, new_key, item_size); 4347f0641656SFilipe Manana leaf = path->nodes[0]; 4348f0641656SFilipe Manana memcpy_extent_buffer(leaf, 4349f0641656SFilipe Manana btrfs_item_ptr_offset(leaf, path->slots[0]), 4350f0641656SFilipe Manana btrfs_item_ptr_offset(leaf, path->slots[0] - 1), 4351f0641656SFilipe Manana item_size); 4352f0641656SFilipe Manana return 0; 4353f0641656SFilipe Manana } 4354f0641656SFilipe Manana 4355f0641656SFilipe Manana /* 43565de08d7dSChris Mason * delete the pointer from a given node. 435774123bd7SChris Mason * 4358d352ac68SChris Mason * the tree should have been previously balanced so the deletion does not 4359d352ac68SChris Mason * empty a node. 436074123bd7SChris Mason */ 4361afe5fea7STsutomu Itoh static void del_ptr(struct btrfs_root *root, struct btrfs_path *path, 4362afe5fea7STsutomu Itoh int level, int slot) 4363be0e5c09SChris Mason { 43645f39d397SChris Mason struct extent_buffer *parent = path->nodes[level]; 43657518a238SChris Mason u32 nritems; 4366f3ea38daSJan Schmidt int ret; 4367be0e5c09SChris Mason 43685f39d397SChris Mason nritems = btrfs_header_nritems(parent); 4369be0e5c09SChris Mason if (slot != nritems - 1) { 4370bf1d3425SDavid Sterba if (level) { 4371f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_move(parent, slot, 4372f3a84ccdSFilipe Manana slot + 1, nritems - slot - 1); 4373bf1d3425SDavid Sterba BUG_ON(ret < 0); 4374bf1d3425SDavid Sterba } 43755f39d397SChris Mason memmove_extent_buffer(parent, 4376e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(parent, slot), 4377e23efd8eSJosef Bacik btrfs_node_key_ptr_offset(parent, slot + 1), 4378d6025579SChris Mason sizeof(struct btrfs_key_ptr) * 4379d6025579SChris Mason (nritems - slot - 1)); 438057ba86c0SChris Mason } else if (level) { 4381f3a84ccdSFilipe Manana ret = btrfs_tree_mod_log_insert_key(parent, slot, 438233cff222SFilipe Manana BTRFS_MOD_LOG_KEY_REMOVE); 438357ba86c0SChris Mason BUG_ON(ret < 0); 4384be0e5c09SChris Mason } 4385f3ea38daSJan Schmidt 43867518a238SChris Mason nritems--; 43875f39d397SChris Mason btrfs_set_header_nritems(parent, nritems); 43887518a238SChris Mason if (nritems == 0 && parent == root->node) { 43895f39d397SChris Mason BUG_ON(btrfs_header_level(root->node) != 1); 4390eb60ceacSChris Mason /* just turn the root into a leaf and break */ 43915f39d397SChris Mason btrfs_set_header_level(root->node, 0); 4392bb803951SChris Mason } else if (slot == 0) { 43935f39d397SChris Mason struct btrfs_disk_key disk_key; 43945f39d397SChris Mason 43955f39d397SChris Mason btrfs_node_key(parent, &disk_key, 0); 4396b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, level + 1); 4397be0e5c09SChris Mason } 4398d6025579SChris Mason btrfs_mark_buffer_dirty(parent); 4399be0e5c09SChris Mason } 4400be0e5c09SChris Mason 440174123bd7SChris Mason /* 4402323ac95bSChris Mason * a helper function to delete the leaf pointed to by path->slots[1] and 44035d4f98a2SYan Zheng * path->nodes[1]. 4404323ac95bSChris Mason * 4405323ac95bSChris Mason * This deletes the pointer in path->nodes[1] and frees the leaf 4406323ac95bSChris Mason * block extent. zero is returned if it all worked out, < 0 otherwise. 4407323ac95bSChris Mason * 4408323ac95bSChris Mason * The path must have already been setup for deleting the leaf, including 4409323ac95bSChris Mason * all the proper balancing. path->nodes[1] must be locked. 4410323ac95bSChris Mason */ 4411143bede5SJeff Mahoney static noinline void btrfs_del_leaf(struct btrfs_trans_handle *trans, 4412323ac95bSChris Mason struct btrfs_root *root, 44135d4f98a2SYan Zheng struct btrfs_path *path, 44145d4f98a2SYan Zheng struct extent_buffer *leaf) 4415323ac95bSChris Mason { 44165d4f98a2SYan Zheng WARN_ON(btrfs_header_generation(leaf) != trans->transid); 4417afe5fea7STsutomu Itoh del_ptr(root, path, 1, path->slots[1]); 4418323ac95bSChris Mason 44194d081c41SChris Mason /* 44204d081c41SChris Mason * btrfs_free_extent is expensive, we want to make sure we 44214d081c41SChris Mason * aren't holding any locks when we call it 44224d081c41SChris Mason */ 44234d081c41SChris Mason btrfs_unlock_up_safe(path, 0); 44244d081c41SChris Mason 4425f0486c68SYan, Zheng root_sub_used(root, leaf->len); 4426f0486c68SYan, Zheng 442767439dadSDavid Sterba atomic_inc(&leaf->refs); 44287a163608SFilipe Manana btrfs_free_tree_block(trans, btrfs_root_id(root), leaf, 0, 1); 44293083ee2eSJosef Bacik free_extent_buffer_stale(leaf); 4430323ac95bSChris Mason } 4431323ac95bSChris Mason /* 443274123bd7SChris Mason * delete the item at the leaf level in path. If that empties 443374123bd7SChris Mason * the leaf, remove it from the tree 443474123bd7SChris Mason */ 443585e21bacSChris Mason int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root, 443685e21bacSChris Mason struct btrfs_path *path, int slot, int nr) 4437be0e5c09SChris Mason { 44380b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 44395f39d397SChris Mason struct extent_buffer *leaf; 4440aa5d6bedSChris Mason int ret = 0; 4441aa5d6bedSChris Mason int wret; 44427518a238SChris Mason u32 nritems; 4443be0e5c09SChris Mason 44445f39d397SChris Mason leaf = path->nodes[0]; 44455f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 4446be0e5c09SChris Mason 444785e21bacSChris Mason if (slot + nr != nritems) { 44480cae23b6SFilipe Manana const u32 last_off = btrfs_item_offset(leaf, slot + nr - 1); 44490cae23b6SFilipe Manana const int data_end = leaf_data_end(leaf); 4450c82f823cSDavid Sterba struct btrfs_map_token token; 44510cae23b6SFilipe Manana u32 dsize = 0; 44520cae23b6SFilipe Manana int i; 44530cae23b6SFilipe Manana 44540cae23b6SFilipe Manana for (i = 0; i < nr; i++) 44550cae23b6SFilipe Manana dsize += btrfs_item_size(leaf, slot + i); 44565f39d397SChris Mason 4457637e3b48SJosef Bacik memmove_leaf_data(leaf, data_end + dsize, data_end, 445885e21bacSChris Mason last_off - data_end); 44595f39d397SChris Mason 4460c82f823cSDavid Sterba btrfs_init_map_token(&token, leaf); 446185e21bacSChris Mason for (i = slot + nr; i < nritems; i++) { 44625f39d397SChris Mason u32 ioff; 4463db94535dSChris Mason 44643212fa14SJosef Bacik ioff = btrfs_token_item_offset(&token, i); 44653212fa14SJosef Bacik btrfs_set_token_item_offset(&token, i, ioff + dsize); 44660783fcfcSChris Mason } 4467db94535dSChris Mason 4468637e3b48SJosef Bacik memmove_leaf_items(leaf, slot, slot + nr, nritems - slot - nr); 4469be0e5c09SChris Mason } 447085e21bacSChris Mason btrfs_set_header_nritems(leaf, nritems - nr); 447185e21bacSChris Mason nritems -= nr; 44725f39d397SChris Mason 447374123bd7SChris Mason /* delete the leaf if we've emptied it */ 44747518a238SChris Mason if (nritems == 0) { 44755f39d397SChris Mason if (leaf == root->node) { 44765f39d397SChris Mason btrfs_set_header_level(leaf, 0); 44779a8dd150SChris Mason } else { 4478190a8339SJosef Bacik btrfs_clear_buffer_dirty(trans, leaf); 4479143bede5SJeff Mahoney btrfs_del_leaf(trans, root, path, leaf); 44809a8dd150SChris Mason } 4481be0e5c09SChris Mason } else { 44827518a238SChris Mason int used = leaf_space_used(leaf, 0, nritems); 4483aa5d6bedSChris Mason if (slot == 0) { 44845f39d397SChris Mason struct btrfs_disk_key disk_key; 44855f39d397SChris Mason 44865f39d397SChris Mason btrfs_item_key(leaf, &disk_key, 0); 4487b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 4488aa5d6bedSChris Mason } 4489aa5d6bedSChris Mason 44907c4063d1SFilipe Manana /* 44917c4063d1SFilipe Manana * Try to delete the leaf if it is mostly empty. We do this by 44927c4063d1SFilipe Manana * trying to move all its items into its left and right neighbours. 44937c4063d1SFilipe Manana * If we can't move all the items, then we don't delete it - it's 44947c4063d1SFilipe Manana * not ideal, but future insertions might fill the leaf with more 44957c4063d1SFilipe Manana * items, or items from other leaves might be moved later into our 44967c4063d1SFilipe Manana * leaf due to deletions on those leaves. 44977c4063d1SFilipe Manana */ 44980b246afaSJeff Mahoney if (used < BTRFS_LEAF_DATA_SIZE(fs_info) / 3) { 44997c4063d1SFilipe Manana u32 min_push_space; 45007c4063d1SFilipe Manana 4501be0e5c09SChris Mason /* push_leaf_left fixes the path. 4502be0e5c09SChris Mason * make sure the path still points to our leaf 4503be0e5c09SChris Mason * for possible call to del_ptr below 4504be0e5c09SChris Mason */ 45054920c9acSChris Mason slot = path->slots[1]; 450667439dadSDavid Sterba atomic_inc(&leaf->refs); 45077c4063d1SFilipe Manana /* 45087c4063d1SFilipe Manana * We want to be able to at least push one item to the 45097c4063d1SFilipe Manana * left neighbour leaf, and that's the first item. 45107c4063d1SFilipe Manana */ 45117c4063d1SFilipe Manana min_push_space = sizeof(struct btrfs_item) + 45127c4063d1SFilipe Manana btrfs_item_size(leaf, 0); 45137c4063d1SFilipe Manana wret = push_leaf_left(trans, root, path, 0, 45147c4063d1SFilipe Manana min_push_space, 1, (u32)-1); 451554aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 4516aa5d6bedSChris Mason ret = wret; 45175f39d397SChris Mason 45185f39d397SChris Mason if (path->nodes[0] == leaf && 45195f39d397SChris Mason btrfs_header_nritems(leaf)) { 45207c4063d1SFilipe Manana /* 45217c4063d1SFilipe Manana * If we were not able to push all items from our 45227c4063d1SFilipe Manana * leaf to its left neighbour, then attempt to 45237c4063d1SFilipe Manana * either push all the remaining items to the 45247c4063d1SFilipe Manana * right neighbour or none. There's no advantage 45257c4063d1SFilipe Manana * in pushing only some items, instead of all, as 45267c4063d1SFilipe Manana * it's pointless to end up with a leaf having 45277c4063d1SFilipe Manana * too few items while the neighbours can be full 45287c4063d1SFilipe Manana * or nearly full. 45297c4063d1SFilipe Manana */ 45307c4063d1SFilipe Manana nritems = btrfs_header_nritems(leaf); 45317c4063d1SFilipe Manana min_push_space = leaf_space_used(leaf, 0, nritems); 45327c4063d1SFilipe Manana wret = push_leaf_right(trans, root, path, 0, 45337c4063d1SFilipe Manana min_push_space, 1, 0); 453454aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 4535aa5d6bedSChris Mason ret = wret; 4536aa5d6bedSChris Mason } 45375f39d397SChris Mason 45385f39d397SChris Mason if (btrfs_header_nritems(leaf) == 0) { 4539323ac95bSChris Mason path->slots[1] = slot; 4540143bede5SJeff Mahoney btrfs_del_leaf(trans, root, path, leaf); 45415f39d397SChris Mason free_extent_buffer(leaf); 4542143bede5SJeff Mahoney ret = 0; 45435de08d7dSChris Mason } else { 4544925baeddSChris Mason /* if we're still in the path, make sure 4545925baeddSChris Mason * we're dirty. Otherwise, one of the 4546925baeddSChris Mason * push_leaf functions must have already 4547925baeddSChris Mason * dirtied this buffer 4548925baeddSChris Mason */ 4549925baeddSChris Mason if (path->nodes[0] == leaf) 45505f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 45515f39d397SChris Mason free_extent_buffer(leaf); 4552be0e5c09SChris Mason } 4553d5719762SChris Mason } else { 45545f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 4555be0e5c09SChris Mason } 45569a8dd150SChris Mason } 4557aa5d6bedSChris Mason return ret; 45589a8dd150SChris Mason } 45599a8dd150SChris Mason 456097571fd0SChris Mason /* 45613f157a2fSChris Mason * A helper function to walk down the tree starting at min_key, and looking 4562de78b51aSEric Sandeen * for nodes or leaves that are have a minimum transaction id. 4563de78b51aSEric Sandeen * This is used by the btree defrag code, and tree logging 45643f157a2fSChris Mason * 45653f157a2fSChris Mason * This does not cow, but it does stuff the starting key it finds back 45663f157a2fSChris Mason * into min_key, so you can call btrfs_search_slot with cow=1 on the 45673f157a2fSChris Mason * key and get a writable path. 45683f157a2fSChris Mason * 45693f157a2fSChris Mason * This honors path->lowest_level to prevent descent past a given level 45703f157a2fSChris Mason * of the tree. 45713f157a2fSChris Mason * 4572d352ac68SChris Mason * min_trans indicates the oldest transaction that you are interested 4573d352ac68SChris Mason * in walking through. Any nodes or leaves older than min_trans are 4574d352ac68SChris Mason * skipped over (without reading them). 4575d352ac68SChris Mason * 45763f157a2fSChris Mason * returns zero if something useful was found, < 0 on error and 1 if there 45773f157a2fSChris Mason * was nothing in the tree that matched the search criteria. 45783f157a2fSChris Mason */ 45793f157a2fSChris Mason int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key, 4580de78b51aSEric Sandeen struct btrfs_path *path, 45813f157a2fSChris Mason u64 min_trans) 45823f157a2fSChris Mason { 45833f157a2fSChris Mason struct extent_buffer *cur; 45843f157a2fSChris Mason struct btrfs_key found_key; 45853f157a2fSChris Mason int slot; 45869652480bSYan int sret; 45873f157a2fSChris Mason u32 nritems; 45883f157a2fSChris Mason int level; 45893f157a2fSChris Mason int ret = 1; 4590f98de9b9SFilipe Manana int keep_locks = path->keep_locks; 45913f157a2fSChris Mason 4592c922b016SStefan Roesch ASSERT(!path->nowait); 4593f98de9b9SFilipe Manana path->keep_locks = 1; 45943f157a2fSChris Mason again: 4595bd681513SChris Mason cur = btrfs_read_lock_root_node(root); 45963f157a2fSChris Mason level = btrfs_header_level(cur); 4597e02119d5SChris Mason WARN_ON(path->nodes[level]); 45983f157a2fSChris Mason path->nodes[level] = cur; 4599bd681513SChris Mason path->locks[level] = BTRFS_READ_LOCK; 46003f157a2fSChris Mason 46013f157a2fSChris Mason if (btrfs_header_generation(cur) < min_trans) { 46023f157a2fSChris Mason ret = 1; 46033f157a2fSChris Mason goto out; 46043f157a2fSChris Mason } 46053f157a2fSChris Mason while (1) { 46063f157a2fSChris Mason nritems = btrfs_header_nritems(cur); 46073f157a2fSChris Mason level = btrfs_header_level(cur); 4608fdf8d595SAnand Jain sret = btrfs_bin_search(cur, 0, min_key, &slot); 4609cbca7d59SFilipe Manana if (sret < 0) { 4610cbca7d59SFilipe Manana ret = sret; 4611cbca7d59SFilipe Manana goto out; 4612cbca7d59SFilipe Manana } 46133f157a2fSChris Mason 4614323ac95bSChris Mason /* at the lowest level, we're done, setup the path and exit */ 4615323ac95bSChris Mason if (level == path->lowest_level) { 4616e02119d5SChris Mason if (slot >= nritems) 4617e02119d5SChris Mason goto find_next_key; 46183f157a2fSChris Mason ret = 0; 46193f157a2fSChris Mason path->slots[level] = slot; 46203f157a2fSChris Mason btrfs_item_key_to_cpu(cur, &found_key, slot); 46213f157a2fSChris Mason goto out; 46223f157a2fSChris Mason } 46239652480bSYan if (sret && slot > 0) 46249652480bSYan slot--; 46253f157a2fSChris Mason /* 4626de78b51aSEric Sandeen * check this node pointer against the min_trans parameters. 4627260db43cSRandy Dunlap * If it is too old, skip to the next one. 46283f157a2fSChris Mason */ 46293f157a2fSChris Mason while (slot < nritems) { 46303f157a2fSChris Mason u64 gen; 4631e02119d5SChris Mason 46323f157a2fSChris Mason gen = btrfs_node_ptr_generation(cur, slot); 46333f157a2fSChris Mason if (gen < min_trans) { 46343f157a2fSChris Mason slot++; 46353f157a2fSChris Mason continue; 46363f157a2fSChris Mason } 46373f157a2fSChris Mason break; 46383f157a2fSChris Mason } 4639e02119d5SChris Mason find_next_key: 46403f157a2fSChris Mason /* 46413f157a2fSChris Mason * we didn't find a candidate key in this node, walk forward 46423f157a2fSChris Mason * and find another one 46433f157a2fSChris Mason */ 46443f157a2fSChris Mason if (slot >= nritems) { 4645e02119d5SChris Mason path->slots[level] = slot; 4646e02119d5SChris Mason sret = btrfs_find_next_key(root, path, min_key, level, 4647de78b51aSEric Sandeen min_trans); 4648e02119d5SChris Mason if (sret == 0) { 4649b3b4aa74SDavid Sterba btrfs_release_path(path); 46503f157a2fSChris Mason goto again; 46513f157a2fSChris Mason } else { 46523f157a2fSChris Mason goto out; 46533f157a2fSChris Mason } 46543f157a2fSChris Mason } 46553f157a2fSChris Mason /* save our key for returning back */ 46563f157a2fSChris Mason btrfs_node_key_to_cpu(cur, &found_key, slot); 46573f157a2fSChris Mason path->slots[level] = slot; 46583f157a2fSChris Mason if (level == path->lowest_level) { 46593f157a2fSChris Mason ret = 0; 46603f157a2fSChris Mason goto out; 46613f157a2fSChris Mason } 46624b231ae4SDavid Sterba cur = btrfs_read_node_slot(cur, slot); 4663fb770ae4SLiu Bo if (IS_ERR(cur)) { 4664fb770ae4SLiu Bo ret = PTR_ERR(cur); 4665fb770ae4SLiu Bo goto out; 4666fb770ae4SLiu Bo } 46673f157a2fSChris Mason 4668bd681513SChris Mason btrfs_tree_read_lock(cur); 4669b4ce94deSChris Mason 4670bd681513SChris Mason path->locks[level - 1] = BTRFS_READ_LOCK; 46713f157a2fSChris Mason path->nodes[level - 1] = cur; 4672f7c79f30SChris Mason unlock_up(path, level, 1, 0, NULL); 46733f157a2fSChris Mason } 46743f157a2fSChris Mason out: 4675f98de9b9SFilipe Manana path->keep_locks = keep_locks; 4676f98de9b9SFilipe Manana if (ret == 0) { 4677f98de9b9SFilipe Manana btrfs_unlock_up_safe(path, path->lowest_level + 1); 4678f98de9b9SFilipe Manana memcpy(min_key, &found_key, sizeof(found_key)); 4679f98de9b9SFilipe Manana } 46803f157a2fSChris Mason return ret; 46813f157a2fSChris Mason } 46823f157a2fSChris Mason 46833f157a2fSChris Mason /* 46843f157a2fSChris Mason * this is similar to btrfs_next_leaf, but does not try to preserve 46853f157a2fSChris Mason * and fixup the path. It looks for and returns the next key in the 4686de78b51aSEric Sandeen * tree based on the current path and the min_trans parameters. 46873f157a2fSChris Mason * 46883f157a2fSChris Mason * 0 is returned if another key is found, < 0 if there are any errors 46893f157a2fSChris Mason * and 1 is returned if there are no higher keys in the tree 46903f157a2fSChris Mason * 46913f157a2fSChris Mason * path->keep_locks should be set to 1 on the search made before 46923f157a2fSChris Mason * calling this function. 46933f157a2fSChris Mason */ 4694e7a84565SChris Mason int btrfs_find_next_key(struct btrfs_root *root, struct btrfs_path *path, 4695de78b51aSEric Sandeen struct btrfs_key *key, int level, u64 min_trans) 4696e7a84565SChris Mason { 4697e7a84565SChris Mason int slot; 4698e7a84565SChris Mason struct extent_buffer *c; 4699e7a84565SChris Mason 47006a9fb468SJosef Bacik WARN_ON(!path->keep_locks && !path->skip_locking); 4701e7a84565SChris Mason while (level < BTRFS_MAX_LEVEL) { 4702e7a84565SChris Mason if (!path->nodes[level]) 4703e7a84565SChris Mason return 1; 4704e7a84565SChris Mason 4705e7a84565SChris Mason slot = path->slots[level] + 1; 4706e7a84565SChris Mason c = path->nodes[level]; 47073f157a2fSChris Mason next: 4708e7a84565SChris Mason if (slot >= btrfs_header_nritems(c)) { 470933c66f43SYan Zheng int ret; 471033c66f43SYan Zheng int orig_lowest; 471133c66f43SYan Zheng struct btrfs_key cur_key; 471233c66f43SYan Zheng if (level + 1 >= BTRFS_MAX_LEVEL || 471333c66f43SYan Zheng !path->nodes[level + 1]) 4714e7a84565SChris Mason return 1; 471533c66f43SYan Zheng 47166a9fb468SJosef Bacik if (path->locks[level + 1] || path->skip_locking) { 471733c66f43SYan Zheng level++; 4718e7a84565SChris Mason continue; 4719e7a84565SChris Mason } 472033c66f43SYan Zheng 472133c66f43SYan Zheng slot = btrfs_header_nritems(c) - 1; 472233c66f43SYan Zheng if (level == 0) 472333c66f43SYan Zheng btrfs_item_key_to_cpu(c, &cur_key, slot); 472433c66f43SYan Zheng else 472533c66f43SYan Zheng btrfs_node_key_to_cpu(c, &cur_key, slot); 472633c66f43SYan Zheng 472733c66f43SYan Zheng orig_lowest = path->lowest_level; 4728b3b4aa74SDavid Sterba btrfs_release_path(path); 472933c66f43SYan Zheng path->lowest_level = level; 473033c66f43SYan Zheng ret = btrfs_search_slot(NULL, root, &cur_key, path, 473133c66f43SYan Zheng 0, 0); 473233c66f43SYan Zheng path->lowest_level = orig_lowest; 473333c66f43SYan Zheng if (ret < 0) 473433c66f43SYan Zheng return ret; 473533c66f43SYan Zheng 473633c66f43SYan Zheng c = path->nodes[level]; 473733c66f43SYan Zheng slot = path->slots[level]; 473833c66f43SYan Zheng if (ret == 0) 473933c66f43SYan Zheng slot++; 474033c66f43SYan Zheng goto next; 474133c66f43SYan Zheng } 474233c66f43SYan Zheng 4743e7a84565SChris Mason if (level == 0) 4744e7a84565SChris Mason btrfs_item_key_to_cpu(c, key, slot); 47453f157a2fSChris Mason else { 47463f157a2fSChris Mason u64 gen = btrfs_node_ptr_generation(c, slot); 47473f157a2fSChris Mason 47483f157a2fSChris Mason if (gen < min_trans) { 47493f157a2fSChris Mason slot++; 47503f157a2fSChris Mason goto next; 47513f157a2fSChris Mason } 4752e7a84565SChris Mason btrfs_node_key_to_cpu(c, key, slot); 47533f157a2fSChris Mason } 4754e7a84565SChris Mason return 0; 4755e7a84565SChris Mason } 4756e7a84565SChris Mason return 1; 4757e7a84565SChris Mason } 4758e7a84565SChris Mason 47593d7806ecSJan Schmidt int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path, 47603d7806ecSJan Schmidt u64 time_seq) 47613d7806ecSJan Schmidt { 4762d97e63b6SChris Mason int slot; 47638e73f275SChris Mason int level; 47645f39d397SChris Mason struct extent_buffer *c; 47658e73f275SChris Mason struct extent_buffer *next; 4766d96b3424SFilipe Manana struct btrfs_fs_info *fs_info = root->fs_info; 4767925baeddSChris Mason struct btrfs_key key; 4768d96b3424SFilipe Manana bool need_commit_sem = false; 4769925baeddSChris Mason u32 nritems; 4770925baeddSChris Mason int ret; 47710e46318dSJosef Bacik int i; 4772925baeddSChris Mason 4773bdcdd86cSFilipe Manana /* 4774bdcdd86cSFilipe Manana * The nowait semantics are used only for write paths, where we don't 4775bdcdd86cSFilipe Manana * use the tree mod log and sequence numbers. 4776bdcdd86cSFilipe Manana */ 4777bdcdd86cSFilipe Manana if (time_seq) 4778c922b016SStefan Roesch ASSERT(!path->nowait); 4779c922b016SStefan Roesch 4780925baeddSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 4781d397712bSChris Mason if (nritems == 0) 4782925baeddSChris Mason return 1; 4783925baeddSChris Mason 47848e73f275SChris Mason btrfs_item_key_to_cpu(path->nodes[0], &key, nritems - 1); 47858e73f275SChris Mason again: 47868e73f275SChris Mason level = 1; 47878e73f275SChris Mason next = NULL; 4788b3b4aa74SDavid Sterba btrfs_release_path(path); 47898e73f275SChris Mason 4790a2135011SChris Mason path->keep_locks = 1; 47918e73f275SChris Mason 4792d96b3424SFilipe Manana if (time_seq) { 47933d7806ecSJan Schmidt ret = btrfs_search_old_slot(root, &key, path, time_seq); 4794d96b3424SFilipe Manana } else { 4795d96b3424SFilipe Manana if (path->need_commit_sem) { 4796d96b3424SFilipe Manana path->need_commit_sem = 0; 4797d96b3424SFilipe Manana need_commit_sem = true; 4798bdcdd86cSFilipe Manana if (path->nowait) { 4799bdcdd86cSFilipe Manana if (!down_read_trylock(&fs_info->commit_root_sem)) { 4800bdcdd86cSFilipe Manana ret = -EAGAIN; 4801bdcdd86cSFilipe Manana goto done; 4802bdcdd86cSFilipe Manana } 4803bdcdd86cSFilipe Manana } else { 4804d96b3424SFilipe Manana down_read(&fs_info->commit_root_sem); 4805d96b3424SFilipe Manana } 4806bdcdd86cSFilipe Manana } 4807925baeddSChris Mason ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 4808d96b3424SFilipe Manana } 4809925baeddSChris Mason path->keep_locks = 0; 4810925baeddSChris Mason 4811925baeddSChris Mason if (ret < 0) 4812d96b3424SFilipe Manana goto done; 4813925baeddSChris Mason 4814a2135011SChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 4815168fd7d2SChris Mason /* 4816168fd7d2SChris Mason * by releasing the path above we dropped all our locks. A balance 4817168fd7d2SChris Mason * could have added more items next to the key that used to be 4818168fd7d2SChris Mason * at the very end of the block. So, check again here and 4819168fd7d2SChris Mason * advance the path if there are now more items available. 4820168fd7d2SChris Mason */ 4821a2135011SChris Mason if (nritems > 0 && path->slots[0] < nritems - 1) { 4822e457afecSYan Zheng if (ret == 0) 4823168fd7d2SChris Mason path->slots[0]++; 48248e73f275SChris Mason ret = 0; 4825925baeddSChris Mason goto done; 4826925baeddSChris Mason } 48270b43e04fSLiu Bo /* 48280b43e04fSLiu Bo * So the above check misses one case: 48290b43e04fSLiu Bo * - after releasing the path above, someone has removed the item that 48300b43e04fSLiu Bo * used to be at the very end of the block, and balance between leafs 48310b43e04fSLiu Bo * gets another one with bigger key.offset to replace it. 48320b43e04fSLiu Bo * 48330b43e04fSLiu Bo * This one should be returned as well, or we can get leaf corruption 48340b43e04fSLiu Bo * later(esp. in __btrfs_drop_extents()). 48350b43e04fSLiu Bo * 48360b43e04fSLiu Bo * And a bit more explanation about this check, 48370b43e04fSLiu Bo * with ret > 0, the key isn't found, the path points to the slot 48380b43e04fSLiu Bo * where it should be inserted, so the path->slots[0] item must be the 48390b43e04fSLiu Bo * bigger one. 48400b43e04fSLiu Bo */ 48410b43e04fSLiu Bo if (nritems > 0 && ret > 0 && path->slots[0] == nritems - 1) { 48420b43e04fSLiu Bo ret = 0; 48430b43e04fSLiu Bo goto done; 48440b43e04fSLiu Bo } 4845d97e63b6SChris Mason 4846234b63a0SChris Mason while (level < BTRFS_MAX_LEVEL) { 48478e73f275SChris Mason if (!path->nodes[level]) { 48488e73f275SChris Mason ret = 1; 48498e73f275SChris Mason goto done; 48508e73f275SChris Mason } 48515f39d397SChris Mason 4852d97e63b6SChris Mason slot = path->slots[level] + 1; 4853d97e63b6SChris Mason c = path->nodes[level]; 48545f39d397SChris Mason if (slot >= btrfs_header_nritems(c)) { 4855d97e63b6SChris Mason level++; 48568e73f275SChris Mason if (level == BTRFS_MAX_LEVEL) { 48578e73f275SChris Mason ret = 1; 48588e73f275SChris Mason goto done; 48598e73f275SChris Mason } 4860d97e63b6SChris Mason continue; 4861d97e63b6SChris Mason } 48625f39d397SChris Mason 48630e46318dSJosef Bacik 48640e46318dSJosef Bacik /* 48650e46318dSJosef Bacik * Our current level is where we're going to start from, and to 48660e46318dSJosef Bacik * make sure lockdep doesn't complain we need to drop our locks 48670e46318dSJosef Bacik * and nodes from 0 to our current level. 48680e46318dSJosef Bacik */ 48690e46318dSJosef Bacik for (i = 0; i < level; i++) { 48700e46318dSJosef Bacik if (path->locks[level]) { 48710e46318dSJosef Bacik btrfs_tree_read_unlock(path->nodes[i]); 48720e46318dSJosef Bacik path->locks[i] = 0; 48730e46318dSJosef Bacik } 48740e46318dSJosef Bacik free_extent_buffer(path->nodes[i]); 48750e46318dSJosef Bacik path->nodes[i] = NULL; 4876925baeddSChris Mason } 48775f39d397SChris Mason 48788e73f275SChris Mason next = c; 4879d07b8528SLiu Bo ret = read_block_for_search(root, path, &next, level, 4880cda79c54SDavid Sterba slot, &key); 4881bdcdd86cSFilipe Manana if (ret == -EAGAIN && !path->nowait) 48828e73f275SChris Mason goto again; 48835f39d397SChris Mason 488476a05b35SChris Mason if (ret < 0) { 4885b3b4aa74SDavid Sterba btrfs_release_path(path); 488676a05b35SChris Mason goto done; 488776a05b35SChris Mason } 488876a05b35SChris Mason 48895cd57b2cSChris Mason if (!path->skip_locking) { 4890bd681513SChris Mason ret = btrfs_try_tree_read_lock(next); 4891bdcdd86cSFilipe Manana if (!ret && path->nowait) { 4892bdcdd86cSFilipe Manana ret = -EAGAIN; 4893bdcdd86cSFilipe Manana goto done; 4894bdcdd86cSFilipe Manana } 4895d42244a0SJan Schmidt if (!ret && time_seq) { 4896d42244a0SJan Schmidt /* 4897d42244a0SJan Schmidt * If we don't get the lock, we may be racing 4898d42244a0SJan Schmidt * with push_leaf_left, holding that lock while 4899d42244a0SJan Schmidt * itself waiting for the leaf we've currently 4900d42244a0SJan Schmidt * locked. To solve this situation, we give up 4901d42244a0SJan Schmidt * on our lock and cycle. 4902d42244a0SJan Schmidt */ 4903cf538830SJan Schmidt free_extent_buffer(next); 4904d42244a0SJan Schmidt btrfs_release_path(path); 4905d42244a0SJan Schmidt cond_resched(); 4906d42244a0SJan Schmidt goto again; 4907d42244a0SJan Schmidt } 49080e46318dSJosef Bacik if (!ret) 49090e46318dSJosef Bacik btrfs_tree_read_lock(next); 4910bd681513SChris Mason } 4911d97e63b6SChris Mason break; 4912d97e63b6SChris Mason } 4913d97e63b6SChris Mason path->slots[level] = slot; 4914d97e63b6SChris Mason while (1) { 4915d97e63b6SChris Mason level--; 4916d97e63b6SChris Mason path->nodes[level] = next; 4917d97e63b6SChris Mason path->slots[level] = 0; 4918a74a4b97SChris Mason if (!path->skip_locking) 4919ffeb03cfSJosef Bacik path->locks[level] = BTRFS_READ_LOCK; 4920d97e63b6SChris Mason if (!level) 4921d97e63b6SChris Mason break; 4922b4ce94deSChris Mason 4923d07b8528SLiu Bo ret = read_block_for_search(root, path, &next, level, 4924cda79c54SDavid Sterba 0, &key); 4925bdcdd86cSFilipe Manana if (ret == -EAGAIN && !path->nowait) 49268e73f275SChris Mason goto again; 49278e73f275SChris Mason 492876a05b35SChris Mason if (ret < 0) { 4929b3b4aa74SDavid Sterba btrfs_release_path(path); 493076a05b35SChris Mason goto done; 493176a05b35SChris Mason } 493276a05b35SChris Mason 4933bdcdd86cSFilipe Manana if (!path->skip_locking) { 4934bdcdd86cSFilipe Manana if (path->nowait) { 4935bdcdd86cSFilipe Manana if (!btrfs_try_tree_read_lock(next)) { 4936bdcdd86cSFilipe Manana ret = -EAGAIN; 4937bdcdd86cSFilipe Manana goto done; 4938bdcdd86cSFilipe Manana } 4939bdcdd86cSFilipe Manana } else { 49400e46318dSJosef Bacik btrfs_tree_read_lock(next); 4941d97e63b6SChris Mason } 4942bdcdd86cSFilipe Manana } 4943bdcdd86cSFilipe Manana } 49448e73f275SChris Mason ret = 0; 4945925baeddSChris Mason done: 4946f7c79f30SChris Mason unlock_up(path, 0, 1, 0, NULL); 4947d96b3424SFilipe Manana if (need_commit_sem) { 4948d96b3424SFilipe Manana int ret2; 4949d96b3424SFilipe Manana 4950d96b3424SFilipe Manana path->need_commit_sem = 1; 4951d96b3424SFilipe Manana ret2 = finish_need_commit_sem_search(path); 4952d96b3424SFilipe Manana up_read(&fs_info->commit_root_sem); 4953d96b3424SFilipe Manana if (ret2) 4954d96b3424SFilipe Manana ret = ret2; 4955d96b3424SFilipe Manana } 49568e73f275SChris Mason 49578e73f275SChris Mason return ret; 4958d97e63b6SChris Mason } 49590b86a832SChris Mason 4960890d2b1aSJosef Bacik int btrfs_next_old_item(struct btrfs_root *root, struct btrfs_path *path, u64 time_seq) 4961890d2b1aSJosef Bacik { 4962890d2b1aSJosef Bacik path->slots[0]++; 4963890d2b1aSJosef Bacik if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) 4964890d2b1aSJosef Bacik return btrfs_next_old_leaf(root, path, time_seq); 4965890d2b1aSJosef Bacik return 0; 4966890d2b1aSJosef Bacik } 4967890d2b1aSJosef Bacik 49683f157a2fSChris Mason /* 49693f157a2fSChris Mason * this uses btrfs_prev_leaf to walk backwards in the tree, and keeps 49703f157a2fSChris Mason * searching until it gets past min_objectid or finds an item of 'type' 49713f157a2fSChris Mason * 49723f157a2fSChris Mason * returns 0 if something is found, 1 if nothing was found and < 0 on error 49733f157a2fSChris Mason */ 49740b86a832SChris Mason int btrfs_previous_item(struct btrfs_root *root, 49750b86a832SChris Mason struct btrfs_path *path, u64 min_objectid, 49760b86a832SChris Mason int type) 49770b86a832SChris Mason { 49780b86a832SChris Mason struct btrfs_key found_key; 49790b86a832SChris Mason struct extent_buffer *leaf; 4980e02119d5SChris Mason u32 nritems; 49810b86a832SChris Mason int ret; 49820b86a832SChris Mason 49830b86a832SChris Mason while (1) { 49840b86a832SChris Mason if (path->slots[0] == 0) { 49850b86a832SChris Mason ret = btrfs_prev_leaf(root, path); 49860b86a832SChris Mason if (ret != 0) 49870b86a832SChris Mason return ret; 49880b86a832SChris Mason } else { 49890b86a832SChris Mason path->slots[0]--; 49900b86a832SChris Mason } 49910b86a832SChris Mason leaf = path->nodes[0]; 4992e02119d5SChris Mason nritems = btrfs_header_nritems(leaf); 4993e02119d5SChris Mason if (nritems == 0) 4994e02119d5SChris Mason return 1; 4995e02119d5SChris Mason if (path->slots[0] == nritems) 4996e02119d5SChris Mason path->slots[0]--; 4997e02119d5SChris Mason 49980b86a832SChris Mason btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 4999e02119d5SChris Mason if (found_key.objectid < min_objectid) 5000e02119d5SChris Mason break; 50010a4eefbbSYan Zheng if (found_key.type == type) 50020a4eefbbSYan Zheng return 0; 5003e02119d5SChris Mason if (found_key.objectid == min_objectid && 5004e02119d5SChris Mason found_key.type < type) 5005e02119d5SChris Mason break; 50060b86a832SChris Mason } 50070b86a832SChris Mason return 1; 50080b86a832SChris Mason } 5009ade2e0b3SWang Shilong 5010ade2e0b3SWang Shilong /* 5011ade2e0b3SWang Shilong * search in extent tree to find a previous Metadata/Data extent item with 5012ade2e0b3SWang Shilong * min objecitd. 5013ade2e0b3SWang Shilong * 5014ade2e0b3SWang Shilong * returns 0 if something is found, 1 if nothing was found and < 0 on error 5015ade2e0b3SWang Shilong */ 5016ade2e0b3SWang Shilong int btrfs_previous_extent_item(struct btrfs_root *root, 5017ade2e0b3SWang Shilong struct btrfs_path *path, u64 min_objectid) 5018ade2e0b3SWang Shilong { 5019ade2e0b3SWang Shilong struct btrfs_key found_key; 5020ade2e0b3SWang Shilong struct extent_buffer *leaf; 5021ade2e0b3SWang Shilong u32 nritems; 5022ade2e0b3SWang Shilong int ret; 5023ade2e0b3SWang Shilong 5024ade2e0b3SWang Shilong while (1) { 5025ade2e0b3SWang Shilong if (path->slots[0] == 0) { 5026ade2e0b3SWang Shilong ret = btrfs_prev_leaf(root, path); 5027ade2e0b3SWang Shilong if (ret != 0) 5028ade2e0b3SWang Shilong return ret; 5029ade2e0b3SWang Shilong } else { 5030ade2e0b3SWang Shilong path->slots[0]--; 5031ade2e0b3SWang Shilong } 5032ade2e0b3SWang Shilong leaf = path->nodes[0]; 5033ade2e0b3SWang Shilong nritems = btrfs_header_nritems(leaf); 5034ade2e0b3SWang Shilong if (nritems == 0) 5035ade2e0b3SWang Shilong return 1; 5036ade2e0b3SWang Shilong if (path->slots[0] == nritems) 5037ade2e0b3SWang Shilong path->slots[0]--; 5038ade2e0b3SWang Shilong 5039ade2e0b3SWang Shilong btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 5040ade2e0b3SWang Shilong if (found_key.objectid < min_objectid) 5041ade2e0b3SWang Shilong break; 5042ade2e0b3SWang Shilong if (found_key.type == BTRFS_EXTENT_ITEM_KEY || 5043ade2e0b3SWang Shilong found_key.type == BTRFS_METADATA_ITEM_KEY) 5044ade2e0b3SWang Shilong return 0; 5045ade2e0b3SWang Shilong if (found_key.objectid == min_objectid && 5046ade2e0b3SWang Shilong found_key.type < BTRFS_EXTENT_ITEM_KEY) 5047ade2e0b3SWang Shilong break; 5048ade2e0b3SWang Shilong } 5049ade2e0b3SWang Shilong return 1; 5050ade2e0b3SWang Shilong } 5051226463d7SJosef Bacik 5052226463d7SJosef Bacik int __init btrfs_ctree_init(void) 5053226463d7SJosef Bacik { 5054226463d7SJosef Bacik btrfs_path_cachep = kmem_cache_create("btrfs_path", 5055226463d7SJosef Bacik sizeof(struct btrfs_path), 0, 5056226463d7SJosef Bacik SLAB_MEM_SPREAD, NULL); 5057226463d7SJosef Bacik if (!btrfs_path_cachep) 5058226463d7SJosef Bacik return -ENOMEM; 5059226463d7SJosef Bacik return 0; 5060226463d7SJosef Bacik } 5061226463d7SJosef Bacik 5062226463d7SJosef Bacik void __cold btrfs_ctree_exit(void) 5063226463d7SJosef Bacik { 5064226463d7SJosef Bacik kmem_cache_destroy(btrfs_path_cachep); 5065226463d7SJosef Bacik } 5066