1c1d7c514SDavid Sterba // SPDX-License-Identifier: GPL-2.0 26cbd5570SChris Mason /* 3d352ac68SChris Mason * Copyright (C) 2007,2008 Oracle. All rights reserved. 46cbd5570SChris Mason */ 56cbd5570SChris Mason 6a6b6e75eSChris Mason #include <linux/sched.h> 75a0e3ad6STejun Heo #include <linux/slab.h> 8bd989ba3SJan Schmidt #include <linux/rbtree.h> 9adf02123SDavid Sterba #include <linux/mm.h> 10eb60ceacSChris Mason #include "ctree.h" 11eb60ceacSChris Mason #include "disk-io.h" 127f5c1516SChris Mason #include "transaction.h" 135f39d397SChris Mason #include "print-tree.h" 14925baeddSChris Mason #include "locking.h" 15de37aa51SNikolay Borisov #include "volumes.h" 169a8dd150SChris Mason 17e089f05cSChris Mason static int split_node(struct btrfs_trans_handle *trans, struct btrfs_root 18e089f05cSChris Mason *root, struct btrfs_path *path, int level); 19310712b2SOmar Sandoval static int split_leaf(struct btrfs_trans_handle *trans, struct btrfs_root *root, 20310712b2SOmar Sandoval const struct btrfs_key *ins_key, struct btrfs_path *path, 21310712b2SOmar Sandoval int data_size, int extend); 225f39d397SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 232ff7e61eSJeff Mahoney struct btrfs_fs_info *fs_info, 242ff7e61eSJeff Mahoney struct extent_buffer *dst, 25971a1f66SChris Mason struct extent_buffer *src, int empty); 265f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 272ff7e61eSJeff Mahoney struct btrfs_fs_info *fs_info, 285f39d397SChris Mason struct extent_buffer *dst_buf, 295f39d397SChris Mason struct extent_buffer *src_buf); 30afe5fea7STsutomu Itoh static void del_ptr(struct btrfs_root *root, struct btrfs_path *path, 31afe5fea7STsutomu Itoh int level, int slot); 32d97e63b6SChris Mason 332c90e5d6SChris Mason struct btrfs_path *btrfs_alloc_path(void) 342c90e5d6SChris Mason { 35e2c89907SMasahiro Yamada return kmem_cache_zalloc(btrfs_path_cachep, GFP_NOFS); 362c90e5d6SChris Mason } 372c90e5d6SChris Mason 38b4ce94deSChris Mason /* 39b4ce94deSChris Mason * set all locked nodes in the path to blocking locks. This should 40b4ce94deSChris Mason * be done before scheduling 41b4ce94deSChris Mason */ 42b4ce94deSChris Mason noinline void btrfs_set_path_blocking(struct btrfs_path *p) 43b4ce94deSChris Mason { 44b4ce94deSChris Mason int i; 45b4ce94deSChris Mason for (i = 0; i < BTRFS_MAX_LEVEL; i++) { 46bd681513SChris Mason if (!p->nodes[i] || !p->locks[i]) 47bd681513SChris Mason continue; 48bd681513SChris Mason btrfs_set_lock_blocking_rw(p->nodes[i], p->locks[i]); 49bd681513SChris Mason if (p->locks[i] == BTRFS_READ_LOCK) 50bd681513SChris Mason p->locks[i] = BTRFS_READ_LOCK_BLOCKING; 51bd681513SChris Mason else if (p->locks[i] == BTRFS_WRITE_LOCK) 52bd681513SChris Mason p->locks[i] = BTRFS_WRITE_LOCK_BLOCKING; 53b4ce94deSChris Mason } 54b4ce94deSChris Mason } 55b4ce94deSChris Mason 56d352ac68SChris Mason /* this also releases the path */ 572c90e5d6SChris Mason void btrfs_free_path(struct btrfs_path *p) 582c90e5d6SChris Mason { 59ff175d57SJesper Juhl if (!p) 60ff175d57SJesper Juhl return; 61b3b4aa74SDavid Sterba btrfs_release_path(p); 622c90e5d6SChris Mason kmem_cache_free(btrfs_path_cachep, p); 632c90e5d6SChris Mason } 642c90e5d6SChris Mason 65d352ac68SChris Mason /* 66d352ac68SChris Mason * path release drops references on the extent buffers in the path 67d352ac68SChris Mason * and it drops any locks held by this path 68d352ac68SChris Mason * 69d352ac68SChris Mason * It is safe to call this on paths that no locks or extent buffers held. 70d352ac68SChris Mason */ 71b3b4aa74SDavid Sterba noinline void btrfs_release_path(struct btrfs_path *p) 72eb60ceacSChris Mason { 73eb60ceacSChris Mason int i; 74a2135011SChris Mason 75234b63a0SChris Mason for (i = 0; i < BTRFS_MAX_LEVEL; i++) { 763f157a2fSChris Mason p->slots[i] = 0; 77eb60ceacSChris Mason if (!p->nodes[i]) 78925baeddSChris Mason continue; 79925baeddSChris Mason if (p->locks[i]) { 80bd681513SChris Mason btrfs_tree_unlock_rw(p->nodes[i], p->locks[i]); 81925baeddSChris Mason p->locks[i] = 0; 82925baeddSChris Mason } 835f39d397SChris Mason free_extent_buffer(p->nodes[i]); 843f157a2fSChris Mason p->nodes[i] = NULL; 85eb60ceacSChris Mason } 86eb60ceacSChris Mason } 87eb60ceacSChris Mason 88d352ac68SChris Mason /* 89d352ac68SChris Mason * safely gets a reference on the root node of a tree. A lock 90d352ac68SChris Mason * is not taken, so a concurrent writer may put a different node 91d352ac68SChris Mason * at the root of the tree. See btrfs_lock_root_node for the 92d352ac68SChris Mason * looping required. 93d352ac68SChris Mason * 94d352ac68SChris Mason * The extent buffer returned by this has a reference taken, so 95d352ac68SChris Mason * it won't disappear. It may stop being the root of the tree 96d352ac68SChris Mason * at any time because there are no locks held. 97d352ac68SChris Mason */ 98925baeddSChris Mason struct extent_buffer *btrfs_root_node(struct btrfs_root *root) 99925baeddSChris Mason { 100925baeddSChris Mason struct extent_buffer *eb; 101240f62c8SChris Mason 1023083ee2eSJosef Bacik while (1) { 103240f62c8SChris Mason rcu_read_lock(); 104240f62c8SChris Mason eb = rcu_dereference(root->node); 1053083ee2eSJosef Bacik 1063083ee2eSJosef Bacik /* 1073083ee2eSJosef Bacik * RCU really hurts here, we could free up the root node because 10801327610SNicholas D Steeves * it was COWed but we may not get the new root node yet so do 1093083ee2eSJosef Bacik * the inc_not_zero dance and if it doesn't work then 1103083ee2eSJosef Bacik * synchronize_rcu and try again. 1113083ee2eSJosef Bacik */ 1123083ee2eSJosef Bacik if (atomic_inc_not_zero(&eb->refs)) { 113240f62c8SChris Mason rcu_read_unlock(); 1143083ee2eSJosef Bacik break; 1153083ee2eSJosef Bacik } 1163083ee2eSJosef Bacik rcu_read_unlock(); 1173083ee2eSJosef Bacik synchronize_rcu(); 1183083ee2eSJosef Bacik } 119925baeddSChris Mason return eb; 120925baeddSChris Mason } 121925baeddSChris Mason 122d352ac68SChris Mason /* loop around taking references on and locking the root node of the 123d352ac68SChris Mason * tree until you end up with a lock on the root. A locked buffer 124d352ac68SChris Mason * is returned, with a reference held. 125d352ac68SChris Mason */ 126925baeddSChris Mason struct extent_buffer *btrfs_lock_root_node(struct btrfs_root *root) 127925baeddSChris Mason { 128925baeddSChris Mason struct extent_buffer *eb; 129925baeddSChris Mason 130925baeddSChris Mason while (1) { 131925baeddSChris Mason eb = btrfs_root_node(root); 132925baeddSChris Mason btrfs_tree_lock(eb); 133240f62c8SChris Mason if (eb == root->node) 134925baeddSChris Mason break; 135925baeddSChris Mason btrfs_tree_unlock(eb); 136925baeddSChris Mason free_extent_buffer(eb); 137925baeddSChris Mason } 138925baeddSChris Mason return eb; 139925baeddSChris Mason } 140925baeddSChris Mason 141bd681513SChris Mason /* loop around taking references on and locking the root node of the 142bd681513SChris Mason * tree until you end up with a lock on the root. A locked buffer 143bd681513SChris Mason * is returned, with a reference held. 144bd681513SChris Mason */ 14584f7d8e6SJosef Bacik struct extent_buffer *btrfs_read_lock_root_node(struct btrfs_root *root) 146bd681513SChris Mason { 147bd681513SChris Mason struct extent_buffer *eb; 148bd681513SChris Mason 149bd681513SChris Mason while (1) { 150bd681513SChris Mason eb = btrfs_root_node(root); 151bd681513SChris Mason btrfs_tree_read_lock(eb); 152bd681513SChris Mason if (eb == root->node) 153bd681513SChris Mason break; 154bd681513SChris Mason btrfs_tree_read_unlock(eb); 155bd681513SChris Mason free_extent_buffer(eb); 156bd681513SChris Mason } 157bd681513SChris Mason return eb; 158bd681513SChris Mason } 159bd681513SChris Mason 160d352ac68SChris Mason /* cowonly root (everything not a reference counted cow subvolume), just get 161d352ac68SChris Mason * put onto a simple dirty list. transaction.c walks this to make sure they 162d352ac68SChris Mason * get properly updated on disk. 163d352ac68SChris Mason */ 1640b86a832SChris Mason static void add_root_to_dirty_list(struct btrfs_root *root) 1650b86a832SChris Mason { 1660b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 1670b246afaSJeff Mahoney 168e7070be1SJosef Bacik if (test_bit(BTRFS_ROOT_DIRTY, &root->state) || 169e7070be1SJosef Bacik !test_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state)) 170e7070be1SJosef Bacik return; 171e7070be1SJosef Bacik 1720b246afaSJeff Mahoney spin_lock(&fs_info->trans_lock); 173e7070be1SJosef Bacik if (!test_and_set_bit(BTRFS_ROOT_DIRTY, &root->state)) { 174e7070be1SJosef Bacik /* Want the extent tree to be the last on the list */ 1754fd786e6SMisono Tomohiro if (root->root_key.objectid == BTRFS_EXTENT_TREE_OBJECTID) 176e7070be1SJosef Bacik list_move_tail(&root->dirty_list, 1770b246afaSJeff Mahoney &fs_info->dirty_cowonly_roots); 178e7070be1SJosef Bacik else 179e7070be1SJosef Bacik list_move(&root->dirty_list, 1800b246afaSJeff Mahoney &fs_info->dirty_cowonly_roots); 1810b86a832SChris Mason } 1820b246afaSJeff Mahoney spin_unlock(&fs_info->trans_lock); 1830b86a832SChris Mason } 1840b86a832SChris Mason 185d352ac68SChris Mason /* 186d352ac68SChris Mason * used by snapshot creation to make a copy of a root for a tree with 187d352ac68SChris Mason * a given objectid. The buffer with the new root node is returned in 188d352ac68SChris Mason * cow_ret, and this func returns zero on success or a negative error code. 189d352ac68SChris Mason */ 190be20aa9dSChris Mason int btrfs_copy_root(struct btrfs_trans_handle *trans, 191be20aa9dSChris Mason struct btrfs_root *root, 192be20aa9dSChris Mason struct extent_buffer *buf, 193be20aa9dSChris Mason struct extent_buffer **cow_ret, u64 new_root_objectid) 194be20aa9dSChris Mason { 1950b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 196be20aa9dSChris Mason struct extent_buffer *cow; 197be20aa9dSChris Mason int ret = 0; 198be20aa9dSChris Mason int level; 1995d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 200be20aa9dSChris Mason 20127cdeb70SMiao Xie WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) && 2020b246afaSJeff Mahoney trans->transid != fs_info->running_transaction->transid); 20327cdeb70SMiao Xie WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) && 20427cdeb70SMiao Xie trans->transid != root->last_trans); 205be20aa9dSChris Mason 206be20aa9dSChris Mason level = btrfs_header_level(buf); 2075d4f98a2SYan Zheng if (level == 0) 2085d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 2095d4f98a2SYan Zheng else 2105d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 21131840ae1SZheng Yan 2124d75f8a9SDavid Sterba cow = btrfs_alloc_tree_block(trans, root, 0, new_root_objectid, 2134d75f8a9SDavid Sterba &disk_key, level, buf->start, 0); 2145d4f98a2SYan Zheng if (IS_ERR(cow)) 215be20aa9dSChris Mason return PTR_ERR(cow); 216be20aa9dSChris Mason 21758e8012cSDavid Sterba copy_extent_buffer_full(cow, buf); 218be20aa9dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 219be20aa9dSChris Mason btrfs_set_header_generation(cow, trans->transid); 2205d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 2215d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 2225d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 2235d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 2245d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 2255d4f98a2SYan Zheng else 226be20aa9dSChris Mason btrfs_set_header_owner(cow, new_root_objectid); 227be20aa9dSChris Mason 228de37aa51SNikolay Borisov write_extent_buffer_fsid(cow, fs_info->fs_devices->metadata_uuid); 2292b82032cSYan Zheng 230be20aa9dSChris Mason WARN_ON(btrfs_header_generation(buf) > trans->transid); 2315d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 232e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 2335d4f98a2SYan Zheng else 234e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 2354aec2b52SChris Mason 236be20aa9dSChris Mason if (ret) 237be20aa9dSChris Mason return ret; 238be20aa9dSChris Mason 239be20aa9dSChris Mason btrfs_mark_buffer_dirty(cow); 240be20aa9dSChris Mason *cow_ret = cow; 241be20aa9dSChris Mason return 0; 242be20aa9dSChris Mason } 243be20aa9dSChris Mason 244bd989ba3SJan Schmidt enum mod_log_op { 245bd989ba3SJan Schmidt MOD_LOG_KEY_REPLACE, 246bd989ba3SJan Schmidt MOD_LOG_KEY_ADD, 247bd989ba3SJan Schmidt MOD_LOG_KEY_REMOVE, 248bd989ba3SJan Schmidt MOD_LOG_KEY_REMOVE_WHILE_FREEING, 249bd989ba3SJan Schmidt MOD_LOG_KEY_REMOVE_WHILE_MOVING, 250bd989ba3SJan Schmidt MOD_LOG_MOVE_KEYS, 251bd989ba3SJan Schmidt MOD_LOG_ROOT_REPLACE, 252bd989ba3SJan Schmidt }; 253bd989ba3SJan Schmidt 254bd989ba3SJan Schmidt struct tree_mod_root { 255bd989ba3SJan Schmidt u64 logical; 256bd989ba3SJan Schmidt u8 level; 257bd989ba3SJan Schmidt }; 258bd989ba3SJan Schmidt 259bd989ba3SJan Schmidt struct tree_mod_elem { 260bd989ba3SJan Schmidt struct rb_node node; 261298cfd36SChandan Rajendra u64 logical; 262097b8a7cSJan Schmidt u64 seq; 263bd989ba3SJan Schmidt enum mod_log_op op; 264bd989ba3SJan Schmidt 265bd989ba3SJan Schmidt /* this is used for MOD_LOG_KEY_* and MOD_LOG_MOVE_KEYS operations */ 266bd989ba3SJan Schmidt int slot; 267bd989ba3SJan Schmidt 268bd989ba3SJan Schmidt /* this is used for MOD_LOG_KEY* and MOD_LOG_ROOT_REPLACE */ 269bd989ba3SJan Schmidt u64 generation; 270bd989ba3SJan Schmidt 271bd989ba3SJan Schmidt /* those are used for op == MOD_LOG_KEY_{REPLACE,REMOVE} */ 272bd989ba3SJan Schmidt struct btrfs_disk_key key; 273bd989ba3SJan Schmidt u64 blockptr; 274bd989ba3SJan Schmidt 275bd989ba3SJan Schmidt /* this is used for op == MOD_LOG_MOVE_KEYS */ 276b6dfa35bSDavid Sterba struct { 277b6dfa35bSDavid Sterba int dst_slot; 278b6dfa35bSDavid Sterba int nr_items; 279b6dfa35bSDavid Sterba } move; 280bd989ba3SJan Schmidt 281bd989ba3SJan Schmidt /* this is used for op == MOD_LOG_ROOT_REPLACE */ 282bd989ba3SJan Schmidt struct tree_mod_root old_root; 283bd989ba3SJan Schmidt }; 284bd989ba3SJan Schmidt 285097b8a7cSJan Schmidt /* 286fcebe456SJosef Bacik * Pull a new tree mod seq number for our operation. 287fc36ed7eSJan Schmidt */ 288fcebe456SJosef Bacik static inline u64 btrfs_inc_tree_mod_seq(struct btrfs_fs_info *fs_info) 289fc36ed7eSJan Schmidt { 290fc36ed7eSJan Schmidt return atomic64_inc_return(&fs_info->tree_mod_seq); 291fc36ed7eSJan Schmidt } 292fc36ed7eSJan Schmidt 293fc36ed7eSJan Schmidt /* 294097b8a7cSJan Schmidt * This adds a new blocker to the tree mod log's blocker list if the @elem 295097b8a7cSJan Schmidt * passed does not already have a sequence number set. So when a caller expects 296097b8a7cSJan Schmidt * to record tree modifications, it should ensure to set elem->seq to zero 297097b8a7cSJan Schmidt * before calling btrfs_get_tree_mod_seq. 298097b8a7cSJan Schmidt * Returns a fresh, unused tree log modification sequence number, even if no new 299097b8a7cSJan Schmidt * blocker was added. 300097b8a7cSJan Schmidt */ 301097b8a7cSJan Schmidt u64 btrfs_get_tree_mod_seq(struct btrfs_fs_info *fs_info, 302bd989ba3SJan Schmidt struct seq_list *elem) 303bd989ba3SJan Schmidt { 304b1a09f1eSDavid Sterba write_lock(&fs_info->tree_mod_log_lock); 305bd989ba3SJan Schmidt spin_lock(&fs_info->tree_mod_seq_lock); 306097b8a7cSJan Schmidt if (!elem->seq) { 307fcebe456SJosef Bacik elem->seq = btrfs_inc_tree_mod_seq(fs_info); 308097b8a7cSJan Schmidt list_add_tail(&elem->list, &fs_info->tree_mod_seq_list); 309097b8a7cSJan Schmidt } 310bd989ba3SJan Schmidt spin_unlock(&fs_info->tree_mod_seq_lock); 311b1a09f1eSDavid Sterba write_unlock(&fs_info->tree_mod_log_lock); 312097b8a7cSJan Schmidt 313fcebe456SJosef Bacik return elem->seq; 314bd989ba3SJan Schmidt } 315bd989ba3SJan Schmidt 316bd989ba3SJan Schmidt void btrfs_put_tree_mod_seq(struct btrfs_fs_info *fs_info, 317bd989ba3SJan Schmidt struct seq_list *elem) 318bd989ba3SJan Schmidt { 319bd989ba3SJan Schmidt struct rb_root *tm_root; 320bd989ba3SJan Schmidt struct rb_node *node; 321bd989ba3SJan Schmidt struct rb_node *next; 322bd989ba3SJan Schmidt struct seq_list *cur_elem; 323bd989ba3SJan Schmidt struct tree_mod_elem *tm; 324bd989ba3SJan Schmidt u64 min_seq = (u64)-1; 325bd989ba3SJan Schmidt u64 seq_putting = elem->seq; 326bd989ba3SJan Schmidt 327bd989ba3SJan Schmidt if (!seq_putting) 328bd989ba3SJan Schmidt return; 329bd989ba3SJan Schmidt 330bd989ba3SJan Schmidt spin_lock(&fs_info->tree_mod_seq_lock); 331bd989ba3SJan Schmidt list_del(&elem->list); 332097b8a7cSJan Schmidt elem->seq = 0; 333bd989ba3SJan Schmidt 334bd989ba3SJan Schmidt list_for_each_entry(cur_elem, &fs_info->tree_mod_seq_list, list) { 335097b8a7cSJan Schmidt if (cur_elem->seq < min_seq) { 336bd989ba3SJan Schmidt if (seq_putting > cur_elem->seq) { 337bd989ba3SJan Schmidt /* 338bd989ba3SJan Schmidt * blocker with lower sequence number exists, we 339bd989ba3SJan Schmidt * cannot remove anything from the log 340bd989ba3SJan Schmidt */ 341097b8a7cSJan Schmidt spin_unlock(&fs_info->tree_mod_seq_lock); 342097b8a7cSJan Schmidt return; 343bd989ba3SJan Schmidt } 344bd989ba3SJan Schmidt min_seq = cur_elem->seq; 345bd989ba3SJan Schmidt } 346bd989ba3SJan Schmidt } 347097b8a7cSJan Schmidt spin_unlock(&fs_info->tree_mod_seq_lock); 348097b8a7cSJan Schmidt 349097b8a7cSJan Schmidt /* 350bd989ba3SJan Schmidt * anything that's lower than the lowest existing (read: blocked) 351bd989ba3SJan Schmidt * sequence number can be removed from the tree. 352bd989ba3SJan Schmidt */ 353b1a09f1eSDavid Sterba write_lock(&fs_info->tree_mod_log_lock); 354bd989ba3SJan Schmidt tm_root = &fs_info->tree_mod_log; 355bd989ba3SJan Schmidt for (node = rb_first(tm_root); node; node = next) { 356bd989ba3SJan Schmidt next = rb_next(node); 3576b4df8b6SGeliang Tang tm = rb_entry(node, struct tree_mod_elem, node); 358097b8a7cSJan Schmidt if (tm->seq > min_seq) 359bd989ba3SJan Schmidt continue; 360bd989ba3SJan Schmidt rb_erase(node, tm_root); 361bd989ba3SJan Schmidt kfree(tm); 362bd989ba3SJan Schmidt } 363b1a09f1eSDavid Sterba write_unlock(&fs_info->tree_mod_log_lock); 364bd989ba3SJan Schmidt } 365bd989ba3SJan Schmidt 366bd989ba3SJan Schmidt /* 367bd989ba3SJan Schmidt * key order of the log: 368298cfd36SChandan Rajendra * node/leaf start address -> sequence 369bd989ba3SJan Schmidt * 370298cfd36SChandan Rajendra * The 'start address' is the logical address of the *new* root node 371298cfd36SChandan Rajendra * for root replace operations, or the logical address of the affected 372298cfd36SChandan Rajendra * block for all other operations. 3735de865eeSFilipe David Borba Manana * 374b1a09f1eSDavid Sterba * Note: must be called with write lock for fs_info::tree_mod_log_lock. 375bd989ba3SJan Schmidt */ 376bd989ba3SJan Schmidt static noinline int 377bd989ba3SJan Schmidt __tree_mod_log_insert(struct btrfs_fs_info *fs_info, struct tree_mod_elem *tm) 378bd989ba3SJan Schmidt { 379bd989ba3SJan Schmidt struct rb_root *tm_root; 380bd989ba3SJan Schmidt struct rb_node **new; 381bd989ba3SJan Schmidt struct rb_node *parent = NULL; 382bd989ba3SJan Schmidt struct tree_mod_elem *cur; 383bd989ba3SJan Schmidt 384fcebe456SJosef Bacik tm->seq = btrfs_inc_tree_mod_seq(fs_info); 385bd989ba3SJan Schmidt 386bd989ba3SJan Schmidt tm_root = &fs_info->tree_mod_log; 387bd989ba3SJan Schmidt new = &tm_root->rb_node; 388bd989ba3SJan Schmidt while (*new) { 3896b4df8b6SGeliang Tang cur = rb_entry(*new, struct tree_mod_elem, node); 390bd989ba3SJan Schmidt parent = *new; 391298cfd36SChandan Rajendra if (cur->logical < tm->logical) 392bd989ba3SJan Schmidt new = &((*new)->rb_left); 393298cfd36SChandan Rajendra else if (cur->logical > tm->logical) 394bd989ba3SJan Schmidt new = &((*new)->rb_right); 395097b8a7cSJan Schmidt else if (cur->seq < tm->seq) 396bd989ba3SJan Schmidt new = &((*new)->rb_left); 397097b8a7cSJan Schmidt else if (cur->seq > tm->seq) 398bd989ba3SJan Schmidt new = &((*new)->rb_right); 3995de865eeSFilipe David Borba Manana else 4005de865eeSFilipe David Borba Manana return -EEXIST; 401bd989ba3SJan Schmidt } 402bd989ba3SJan Schmidt 403bd989ba3SJan Schmidt rb_link_node(&tm->node, parent, new); 404bd989ba3SJan Schmidt rb_insert_color(&tm->node, tm_root); 4055de865eeSFilipe David Borba Manana return 0; 406bd989ba3SJan Schmidt } 407bd989ba3SJan Schmidt 408097b8a7cSJan Schmidt /* 409097b8a7cSJan Schmidt * Determines if logging can be omitted. Returns 1 if it can. Otherwise, it 410097b8a7cSJan Schmidt * returns zero with the tree_mod_log_lock acquired. The caller must hold 411097b8a7cSJan Schmidt * this until all tree mod log insertions are recorded in the rb tree and then 412b1a09f1eSDavid Sterba * write unlock fs_info::tree_mod_log_lock. 413097b8a7cSJan Schmidt */ 414e9b7fd4dSJan Schmidt static inline int tree_mod_dont_log(struct btrfs_fs_info *fs_info, 415e9b7fd4dSJan Schmidt struct extent_buffer *eb) { 416e9b7fd4dSJan Schmidt smp_mb(); 417e9b7fd4dSJan Schmidt if (list_empty(&(fs_info)->tree_mod_seq_list)) 418e9b7fd4dSJan Schmidt return 1; 419097b8a7cSJan Schmidt if (eb && btrfs_header_level(eb) == 0) 420e9b7fd4dSJan Schmidt return 1; 4215de865eeSFilipe David Borba Manana 422b1a09f1eSDavid Sterba write_lock(&fs_info->tree_mod_log_lock); 4235de865eeSFilipe David Borba Manana if (list_empty(&(fs_info)->tree_mod_seq_list)) { 424b1a09f1eSDavid Sterba write_unlock(&fs_info->tree_mod_log_lock); 4255de865eeSFilipe David Borba Manana return 1; 4265de865eeSFilipe David Borba Manana } 4275de865eeSFilipe David Borba Manana 428e9b7fd4dSJan Schmidt return 0; 429e9b7fd4dSJan Schmidt } 430e9b7fd4dSJan Schmidt 4315de865eeSFilipe David Borba Manana /* Similar to tree_mod_dont_log, but doesn't acquire any locks. */ 4325de865eeSFilipe David Borba Manana static inline int tree_mod_need_log(const struct btrfs_fs_info *fs_info, 4335de865eeSFilipe David Borba Manana struct extent_buffer *eb) 4345de865eeSFilipe David Borba Manana { 4355de865eeSFilipe David Borba Manana smp_mb(); 4365de865eeSFilipe David Borba Manana if (list_empty(&(fs_info)->tree_mod_seq_list)) 4375de865eeSFilipe David Borba Manana return 0; 4385de865eeSFilipe David Borba Manana if (eb && btrfs_header_level(eb) == 0) 4395de865eeSFilipe David Borba Manana return 0; 4405de865eeSFilipe David Borba Manana 4415de865eeSFilipe David Borba Manana return 1; 4425de865eeSFilipe David Borba Manana } 4435de865eeSFilipe David Borba Manana 4445de865eeSFilipe David Borba Manana static struct tree_mod_elem * 4455de865eeSFilipe David Borba Manana alloc_tree_mod_elem(struct extent_buffer *eb, int slot, 446bd989ba3SJan Schmidt enum mod_log_op op, gfp_t flags) 447bd989ba3SJan Schmidt { 448097b8a7cSJan Schmidt struct tree_mod_elem *tm; 449bd989ba3SJan Schmidt 450c8cc6341SJosef Bacik tm = kzalloc(sizeof(*tm), flags); 451c8cc6341SJosef Bacik if (!tm) 4525de865eeSFilipe David Borba Manana return NULL; 453bd989ba3SJan Schmidt 454298cfd36SChandan Rajendra tm->logical = eb->start; 455bd989ba3SJan Schmidt if (op != MOD_LOG_KEY_ADD) { 456bd989ba3SJan Schmidt btrfs_node_key(eb, &tm->key, slot); 457bd989ba3SJan Schmidt tm->blockptr = btrfs_node_blockptr(eb, slot); 458bd989ba3SJan Schmidt } 459bd989ba3SJan Schmidt tm->op = op; 460bd989ba3SJan Schmidt tm->slot = slot; 461bd989ba3SJan Schmidt tm->generation = btrfs_node_ptr_generation(eb, slot); 4625de865eeSFilipe David Borba Manana RB_CLEAR_NODE(&tm->node); 463bd989ba3SJan Schmidt 4645de865eeSFilipe David Borba Manana return tm; 465097b8a7cSJan Schmidt } 466097b8a7cSJan Schmidt 467e09c2efeSDavid Sterba static noinline int tree_mod_log_insert_key(struct extent_buffer *eb, int slot, 468097b8a7cSJan Schmidt enum mod_log_op op, gfp_t flags) 469097b8a7cSJan Schmidt { 4705de865eeSFilipe David Borba Manana struct tree_mod_elem *tm; 4715de865eeSFilipe David Borba Manana int ret; 4725de865eeSFilipe David Borba Manana 473e09c2efeSDavid Sterba if (!tree_mod_need_log(eb->fs_info, eb)) 474097b8a7cSJan Schmidt return 0; 475097b8a7cSJan Schmidt 4765de865eeSFilipe David Borba Manana tm = alloc_tree_mod_elem(eb, slot, op, flags); 4775de865eeSFilipe David Borba Manana if (!tm) 4785de865eeSFilipe David Borba Manana return -ENOMEM; 4795de865eeSFilipe David Borba Manana 480e09c2efeSDavid Sterba if (tree_mod_dont_log(eb->fs_info, eb)) { 4815de865eeSFilipe David Borba Manana kfree(tm); 4825de865eeSFilipe David Borba Manana return 0; 4835de865eeSFilipe David Borba Manana } 4845de865eeSFilipe David Borba Manana 485e09c2efeSDavid Sterba ret = __tree_mod_log_insert(eb->fs_info, tm); 486b1a09f1eSDavid Sterba write_unlock(&eb->fs_info->tree_mod_log_lock); 4875de865eeSFilipe David Borba Manana if (ret) 4885de865eeSFilipe David Borba Manana kfree(tm); 4895de865eeSFilipe David Borba Manana 4905de865eeSFilipe David Borba Manana return ret; 491097b8a7cSJan Schmidt } 492097b8a7cSJan Schmidt 4936074d45fSDavid Sterba static noinline int tree_mod_log_insert_move(struct extent_buffer *eb, 4946074d45fSDavid Sterba int dst_slot, int src_slot, int nr_items) 495bd989ba3SJan Schmidt { 4965de865eeSFilipe David Borba Manana struct tree_mod_elem *tm = NULL; 4975de865eeSFilipe David Borba Manana struct tree_mod_elem **tm_list = NULL; 4985de865eeSFilipe David Borba Manana int ret = 0; 499bd989ba3SJan Schmidt int i; 5005de865eeSFilipe David Borba Manana int locked = 0; 501bd989ba3SJan Schmidt 5026074d45fSDavid Sterba if (!tree_mod_need_log(eb->fs_info, eb)) 503f395694cSJan Schmidt return 0; 504bd989ba3SJan Schmidt 505176ef8f5SDavid Sterba tm_list = kcalloc(nr_items, sizeof(struct tree_mod_elem *), GFP_NOFS); 5065de865eeSFilipe David Borba Manana if (!tm_list) 5075de865eeSFilipe David Borba Manana return -ENOMEM; 508bd989ba3SJan Schmidt 509176ef8f5SDavid Sterba tm = kzalloc(sizeof(*tm), GFP_NOFS); 5105de865eeSFilipe David Borba Manana if (!tm) { 5115de865eeSFilipe David Borba Manana ret = -ENOMEM; 5125de865eeSFilipe David Borba Manana goto free_tms; 5135de865eeSFilipe David Borba Manana } 514f395694cSJan Schmidt 515298cfd36SChandan Rajendra tm->logical = eb->start; 516bd989ba3SJan Schmidt tm->slot = src_slot; 517bd989ba3SJan Schmidt tm->move.dst_slot = dst_slot; 518bd989ba3SJan Schmidt tm->move.nr_items = nr_items; 519bd989ba3SJan Schmidt tm->op = MOD_LOG_MOVE_KEYS; 520bd989ba3SJan Schmidt 5215de865eeSFilipe David Borba Manana for (i = 0; i + dst_slot < src_slot && i < nr_items; i++) { 5225de865eeSFilipe David Borba Manana tm_list[i] = alloc_tree_mod_elem(eb, i + dst_slot, 523176ef8f5SDavid Sterba MOD_LOG_KEY_REMOVE_WHILE_MOVING, GFP_NOFS); 5245de865eeSFilipe David Borba Manana if (!tm_list[i]) { 5255de865eeSFilipe David Borba Manana ret = -ENOMEM; 5265de865eeSFilipe David Borba Manana goto free_tms; 5275de865eeSFilipe David Borba Manana } 528bd989ba3SJan Schmidt } 529bd989ba3SJan Schmidt 5306074d45fSDavid Sterba if (tree_mod_dont_log(eb->fs_info, eb)) 5315de865eeSFilipe David Borba Manana goto free_tms; 5325de865eeSFilipe David Borba Manana locked = 1; 5335de865eeSFilipe David Borba Manana 5345de865eeSFilipe David Borba Manana /* 5355de865eeSFilipe David Borba Manana * When we override something during the move, we log these removals. 5365de865eeSFilipe David Borba Manana * This can only happen when we move towards the beginning of the 5375de865eeSFilipe David Borba Manana * buffer, i.e. dst_slot < src_slot. 5385de865eeSFilipe David Borba Manana */ 5395de865eeSFilipe David Borba Manana for (i = 0; i + dst_slot < src_slot && i < nr_items; i++) { 5406074d45fSDavid Sterba ret = __tree_mod_log_insert(eb->fs_info, tm_list[i]); 5415de865eeSFilipe David Borba Manana if (ret) 5425de865eeSFilipe David Borba Manana goto free_tms; 5435de865eeSFilipe David Borba Manana } 5445de865eeSFilipe David Borba Manana 5456074d45fSDavid Sterba ret = __tree_mod_log_insert(eb->fs_info, tm); 5465de865eeSFilipe David Borba Manana if (ret) 5475de865eeSFilipe David Borba Manana goto free_tms; 548b1a09f1eSDavid Sterba write_unlock(&eb->fs_info->tree_mod_log_lock); 5495de865eeSFilipe David Borba Manana kfree(tm_list); 5505de865eeSFilipe David Borba Manana 5515de865eeSFilipe David Borba Manana return 0; 5525de865eeSFilipe David Borba Manana free_tms: 5535de865eeSFilipe David Borba Manana for (i = 0; i < nr_items; i++) { 5545de865eeSFilipe David Borba Manana if (tm_list[i] && !RB_EMPTY_NODE(&tm_list[i]->node)) 5556074d45fSDavid Sterba rb_erase(&tm_list[i]->node, &eb->fs_info->tree_mod_log); 5565de865eeSFilipe David Borba Manana kfree(tm_list[i]); 5575de865eeSFilipe David Borba Manana } 5585de865eeSFilipe David Borba Manana if (locked) 559b1a09f1eSDavid Sterba write_unlock(&eb->fs_info->tree_mod_log_lock); 5605de865eeSFilipe David Borba Manana kfree(tm_list); 5615de865eeSFilipe David Borba Manana kfree(tm); 5625de865eeSFilipe David Borba Manana 5635de865eeSFilipe David Borba Manana return ret; 5645de865eeSFilipe David Borba Manana } 5655de865eeSFilipe David Borba Manana 5665de865eeSFilipe David Borba Manana static inline int 5675de865eeSFilipe David Borba Manana __tree_mod_log_free_eb(struct btrfs_fs_info *fs_info, 5685de865eeSFilipe David Borba Manana struct tree_mod_elem **tm_list, 5695de865eeSFilipe David Borba Manana int nritems) 570097b8a7cSJan Schmidt { 5715de865eeSFilipe David Borba Manana int i, j; 572097b8a7cSJan Schmidt int ret; 573097b8a7cSJan Schmidt 574097b8a7cSJan Schmidt for (i = nritems - 1; i >= 0; i--) { 5755de865eeSFilipe David Borba Manana ret = __tree_mod_log_insert(fs_info, tm_list[i]); 5765de865eeSFilipe David Borba Manana if (ret) { 5775de865eeSFilipe David Borba Manana for (j = nritems - 1; j > i; j--) 5785de865eeSFilipe David Borba Manana rb_erase(&tm_list[j]->node, 5795de865eeSFilipe David Borba Manana &fs_info->tree_mod_log); 5805de865eeSFilipe David Borba Manana return ret; 581097b8a7cSJan Schmidt } 582097b8a7cSJan Schmidt } 583097b8a7cSJan Schmidt 5845de865eeSFilipe David Borba Manana return 0; 5855de865eeSFilipe David Borba Manana } 5865de865eeSFilipe David Borba Manana 58795b757c1SDavid Sterba static noinline int tree_mod_log_insert_root(struct extent_buffer *old_root, 58895b757c1SDavid Sterba struct extent_buffer *new_root, int log_removal) 589bd989ba3SJan Schmidt { 59095b757c1SDavid Sterba struct btrfs_fs_info *fs_info = old_root->fs_info; 5915de865eeSFilipe David Borba Manana struct tree_mod_elem *tm = NULL; 5925de865eeSFilipe David Borba Manana struct tree_mod_elem **tm_list = NULL; 5935de865eeSFilipe David Borba Manana int nritems = 0; 5945de865eeSFilipe David Borba Manana int ret = 0; 5955de865eeSFilipe David Borba Manana int i; 596bd989ba3SJan Schmidt 5975de865eeSFilipe David Borba Manana if (!tree_mod_need_log(fs_info, NULL)) 598097b8a7cSJan Schmidt return 0; 599097b8a7cSJan Schmidt 6005de865eeSFilipe David Borba Manana if (log_removal && btrfs_header_level(old_root) > 0) { 6015de865eeSFilipe David Borba Manana nritems = btrfs_header_nritems(old_root); 60231e818feSDavid Sterba tm_list = kcalloc(nritems, sizeof(struct tree_mod_elem *), 603bcc8e07fSDavid Sterba GFP_NOFS); 6045de865eeSFilipe David Borba Manana if (!tm_list) { 6055de865eeSFilipe David Borba Manana ret = -ENOMEM; 6065de865eeSFilipe David Borba Manana goto free_tms; 6075de865eeSFilipe David Borba Manana } 6085de865eeSFilipe David Borba Manana for (i = 0; i < nritems; i++) { 6095de865eeSFilipe David Borba Manana tm_list[i] = alloc_tree_mod_elem(old_root, i, 610bcc8e07fSDavid Sterba MOD_LOG_KEY_REMOVE_WHILE_FREEING, GFP_NOFS); 6115de865eeSFilipe David Borba Manana if (!tm_list[i]) { 6125de865eeSFilipe David Borba Manana ret = -ENOMEM; 6135de865eeSFilipe David Borba Manana goto free_tms; 6145de865eeSFilipe David Borba Manana } 6155de865eeSFilipe David Borba Manana } 6165de865eeSFilipe David Borba Manana } 617d9abbf1cSJan Schmidt 618bcc8e07fSDavid Sterba tm = kzalloc(sizeof(*tm), GFP_NOFS); 6195de865eeSFilipe David Borba Manana if (!tm) { 6205de865eeSFilipe David Borba Manana ret = -ENOMEM; 6215de865eeSFilipe David Borba Manana goto free_tms; 6225de865eeSFilipe David Borba Manana } 623bd989ba3SJan Schmidt 624298cfd36SChandan Rajendra tm->logical = new_root->start; 625bd989ba3SJan Schmidt tm->old_root.logical = old_root->start; 626bd989ba3SJan Schmidt tm->old_root.level = btrfs_header_level(old_root); 627bd989ba3SJan Schmidt tm->generation = btrfs_header_generation(old_root); 628bd989ba3SJan Schmidt tm->op = MOD_LOG_ROOT_REPLACE; 629bd989ba3SJan Schmidt 6305de865eeSFilipe David Borba Manana if (tree_mod_dont_log(fs_info, NULL)) 6315de865eeSFilipe David Borba Manana goto free_tms; 6325de865eeSFilipe David Borba Manana 6335de865eeSFilipe David Borba Manana if (tm_list) 6345de865eeSFilipe David Borba Manana ret = __tree_mod_log_free_eb(fs_info, tm_list, nritems); 6355de865eeSFilipe David Borba Manana if (!ret) 6365de865eeSFilipe David Borba Manana ret = __tree_mod_log_insert(fs_info, tm); 6375de865eeSFilipe David Borba Manana 638b1a09f1eSDavid Sterba write_unlock(&fs_info->tree_mod_log_lock); 6395de865eeSFilipe David Borba Manana if (ret) 6405de865eeSFilipe David Borba Manana goto free_tms; 6415de865eeSFilipe David Borba Manana kfree(tm_list); 6425de865eeSFilipe David Borba Manana 6435de865eeSFilipe David Borba Manana return ret; 6445de865eeSFilipe David Borba Manana 6455de865eeSFilipe David Borba Manana free_tms: 6465de865eeSFilipe David Borba Manana if (tm_list) { 6475de865eeSFilipe David Borba Manana for (i = 0; i < nritems; i++) 6485de865eeSFilipe David Borba Manana kfree(tm_list[i]); 6495de865eeSFilipe David Borba Manana kfree(tm_list); 6505de865eeSFilipe David Borba Manana } 6515de865eeSFilipe David Borba Manana kfree(tm); 6525de865eeSFilipe David Borba Manana 6535de865eeSFilipe David Borba Manana return ret; 654bd989ba3SJan Schmidt } 655bd989ba3SJan Schmidt 656bd989ba3SJan Schmidt static struct tree_mod_elem * 657bd989ba3SJan Schmidt __tree_mod_log_search(struct btrfs_fs_info *fs_info, u64 start, u64 min_seq, 658bd989ba3SJan Schmidt int smallest) 659bd989ba3SJan Schmidt { 660bd989ba3SJan Schmidt struct rb_root *tm_root; 661bd989ba3SJan Schmidt struct rb_node *node; 662bd989ba3SJan Schmidt struct tree_mod_elem *cur = NULL; 663bd989ba3SJan Schmidt struct tree_mod_elem *found = NULL; 664bd989ba3SJan Schmidt 665b1a09f1eSDavid Sterba read_lock(&fs_info->tree_mod_log_lock); 666bd989ba3SJan Schmidt tm_root = &fs_info->tree_mod_log; 667bd989ba3SJan Schmidt node = tm_root->rb_node; 668bd989ba3SJan Schmidt while (node) { 6696b4df8b6SGeliang Tang cur = rb_entry(node, struct tree_mod_elem, node); 670298cfd36SChandan Rajendra if (cur->logical < start) { 671bd989ba3SJan Schmidt node = node->rb_left; 672298cfd36SChandan Rajendra } else if (cur->logical > start) { 673bd989ba3SJan Schmidt node = node->rb_right; 674097b8a7cSJan Schmidt } else if (cur->seq < min_seq) { 675bd989ba3SJan Schmidt node = node->rb_left; 676bd989ba3SJan Schmidt } else if (!smallest) { 677bd989ba3SJan Schmidt /* we want the node with the highest seq */ 678bd989ba3SJan Schmidt if (found) 679097b8a7cSJan Schmidt BUG_ON(found->seq > cur->seq); 680bd989ba3SJan Schmidt found = cur; 681bd989ba3SJan Schmidt node = node->rb_left; 682097b8a7cSJan Schmidt } else if (cur->seq > min_seq) { 683bd989ba3SJan Schmidt /* we want the node with the smallest seq */ 684bd989ba3SJan Schmidt if (found) 685097b8a7cSJan Schmidt BUG_ON(found->seq < cur->seq); 686bd989ba3SJan Schmidt found = cur; 687bd989ba3SJan Schmidt node = node->rb_right; 688bd989ba3SJan Schmidt } else { 689bd989ba3SJan Schmidt found = cur; 690bd989ba3SJan Schmidt break; 691bd989ba3SJan Schmidt } 692bd989ba3SJan Schmidt } 693b1a09f1eSDavid Sterba read_unlock(&fs_info->tree_mod_log_lock); 694bd989ba3SJan Schmidt 695bd989ba3SJan Schmidt return found; 696bd989ba3SJan Schmidt } 697bd989ba3SJan Schmidt 698bd989ba3SJan Schmidt /* 699bd989ba3SJan Schmidt * this returns the element from the log with the smallest time sequence 700bd989ba3SJan Schmidt * value that's in the log (the oldest log item). any element with a time 701bd989ba3SJan Schmidt * sequence lower than min_seq will be ignored. 702bd989ba3SJan Schmidt */ 703bd989ba3SJan Schmidt static struct tree_mod_elem * 704bd989ba3SJan Schmidt tree_mod_log_search_oldest(struct btrfs_fs_info *fs_info, u64 start, 705bd989ba3SJan Schmidt u64 min_seq) 706bd989ba3SJan Schmidt { 707bd989ba3SJan Schmidt return __tree_mod_log_search(fs_info, start, min_seq, 1); 708bd989ba3SJan Schmidt } 709bd989ba3SJan Schmidt 710bd989ba3SJan Schmidt /* 711bd989ba3SJan Schmidt * this returns the element from the log with the largest time sequence 712bd989ba3SJan Schmidt * value that's in the log (the most recent log item). any element with 713bd989ba3SJan Schmidt * a time sequence lower than min_seq will be ignored. 714bd989ba3SJan Schmidt */ 715bd989ba3SJan Schmidt static struct tree_mod_elem * 716bd989ba3SJan Schmidt tree_mod_log_search(struct btrfs_fs_info *fs_info, u64 start, u64 min_seq) 717bd989ba3SJan Schmidt { 718bd989ba3SJan Schmidt return __tree_mod_log_search(fs_info, start, min_seq, 0); 719bd989ba3SJan Schmidt } 720bd989ba3SJan Schmidt 7215de865eeSFilipe David Borba Manana static noinline int 722bd989ba3SJan Schmidt tree_mod_log_eb_copy(struct btrfs_fs_info *fs_info, struct extent_buffer *dst, 723bd989ba3SJan Schmidt struct extent_buffer *src, unsigned long dst_offset, 72490f8d62eSJan Schmidt unsigned long src_offset, int nr_items) 725bd989ba3SJan Schmidt { 7265de865eeSFilipe David Borba Manana int ret = 0; 7275de865eeSFilipe David Borba Manana struct tree_mod_elem **tm_list = NULL; 7285de865eeSFilipe David Borba Manana struct tree_mod_elem **tm_list_add, **tm_list_rem; 729bd989ba3SJan Schmidt int i; 7305de865eeSFilipe David Borba Manana int locked = 0; 731bd989ba3SJan Schmidt 7325de865eeSFilipe David Borba Manana if (!tree_mod_need_log(fs_info, NULL)) 7335de865eeSFilipe David Borba Manana return 0; 734bd989ba3SJan Schmidt 735c8cc6341SJosef Bacik if (btrfs_header_level(dst) == 0 && btrfs_header_level(src) == 0) 7365de865eeSFilipe David Borba Manana return 0; 7375de865eeSFilipe David Borba Manana 73831e818feSDavid Sterba tm_list = kcalloc(nr_items * 2, sizeof(struct tree_mod_elem *), 7395de865eeSFilipe David Borba Manana GFP_NOFS); 7405de865eeSFilipe David Borba Manana if (!tm_list) 7415de865eeSFilipe David Borba Manana return -ENOMEM; 7425de865eeSFilipe David Borba Manana 7435de865eeSFilipe David Borba Manana tm_list_add = tm_list; 7445de865eeSFilipe David Borba Manana tm_list_rem = tm_list + nr_items; 7455de865eeSFilipe David Borba Manana for (i = 0; i < nr_items; i++) { 7465de865eeSFilipe David Borba Manana tm_list_rem[i] = alloc_tree_mod_elem(src, i + src_offset, 7475de865eeSFilipe David Borba Manana MOD_LOG_KEY_REMOVE, GFP_NOFS); 7485de865eeSFilipe David Borba Manana if (!tm_list_rem[i]) { 7495de865eeSFilipe David Borba Manana ret = -ENOMEM; 7505de865eeSFilipe David Borba Manana goto free_tms; 7515de865eeSFilipe David Borba Manana } 7525de865eeSFilipe David Borba Manana 7535de865eeSFilipe David Borba Manana tm_list_add[i] = alloc_tree_mod_elem(dst, i + dst_offset, 7545de865eeSFilipe David Borba Manana MOD_LOG_KEY_ADD, GFP_NOFS); 7555de865eeSFilipe David Borba Manana if (!tm_list_add[i]) { 7565de865eeSFilipe David Borba Manana ret = -ENOMEM; 7575de865eeSFilipe David Borba Manana goto free_tms; 7585de865eeSFilipe David Borba Manana } 7595de865eeSFilipe David Borba Manana } 7605de865eeSFilipe David Borba Manana 7615de865eeSFilipe David Borba Manana if (tree_mod_dont_log(fs_info, NULL)) 7625de865eeSFilipe David Borba Manana goto free_tms; 7635de865eeSFilipe David Borba Manana locked = 1; 764bd989ba3SJan Schmidt 765bd989ba3SJan Schmidt for (i = 0; i < nr_items; i++) { 7665de865eeSFilipe David Borba Manana ret = __tree_mod_log_insert(fs_info, tm_list_rem[i]); 7675de865eeSFilipe David Borba Manana if (ret) 7685de865eeSFilipe David Borba Manana goto free_tms; 7695de865eeSFilipe David Borba Manana ret = __tree_mod_log_insert(fs_info, tm_list_add[i]); 7705de865eeSFilipe David Borba Manana if (ret) 7715de865eeSFilipe David Borba Manana goto free_tms; 772bd989ba3SJan Schmidt } 7735de865eeSFilipe David Borba Manana 774b1a09f1eSDavid Sterba write_unlock(&fs_info->tree_mod_log_lock); 7755de865eeSFilipe David Borba Manana kfree(tm_list); 7765de865eeSFilipe David Borba Manana 7775de865eeSFilipe David Borba Manana return 0; 7785de865eeSFilipe David Borba Manana 7795de865eeSFilipe David Borba Manana free_tms: 7805de865eeSFilipe David Borba Manana for (i = 0; i < nr_items * 2; i++) { 7815de865eeSFilipe David Borba Manana if (tm_list[i] && !RB_EMPTY_NODE(&tm_list[i]->node)) 7825de865eeSFilipe David Borba Manana rb_erase(&tm_list[i]->node, &fs_info->tree_mod_log); 7835de865eeSFilipe David Borba Manana kfree(tm_list[i]); 7845de865eeSFilipe David Borba Manana } 7855de865eeSFilipe David Borba Manana if (locked) 786b1a09f1eSDavid Sterba write_unlock(&fs_info->tree_mod_log_lock); 7875de865eeSFilipe David Borba Manana kfree(tm_list); 7885de865eeSFilipe David Borba Manana 7895de865eeSFilipe David Borba Manana return ret; 790bd989ba3SJan Schmidt } 791bd989ba3SJan Schmidt 792db7279a2SDavid Sterba static noinline int tree_mod_log_free_eb(struct extent_buffer *eb) 793bd989ba3SJan Schmidt { 7945de865eeSFilipe David Borba Manana struct tree_mod_elem **tm_list = NULL; 7955de865eeSFilipe David Borba Manana int nritems = 0; 7965de865eeSFilipe David Borba Manana int i; 7975de865eeSFilipe David Borba Manana int ret = 0; 7985de865eeSFilipe David Borba Manana 7995de865eeSFilipe David Borba Manana if (btrfs_header_level(eb) == 0) 8005de865eeSFilipe David Borba Manana return 0; 8015de865eeSFilipe David Borba Manana 802db7279a2SDavid Sterba if (!tree_mod_need_log(eb->fs_info, NULL)) 8035de865eeSFilipe David Borba Manana return 0; 8045de865eeSFilipe David Borba Manana 8055de865eeSFilipe David Borba Manana nritems = btrfs_header_nritems(eb); 80631e818feSDavid Sterba tm_list = kcalloc(nritems, sizeof(struct tree_mod_elem *), GFP_NOFS); 8075de865eeSFilipe David Borba Manana if (!tm_list) 8085de865eeSFilipe David Borba Manana return -ENOMEM; 8095de865eeSFilipe David Borba Manana 8105de865eeSFilipe David Borba Manana for (i = 0; i < nritems; i++) { 8115de865eeSFilipe David Borba Manana tm_list[i] = alloc_tree_mod_elem(eb, i, 8125de865eeSFilipe David Borba Manana MOD_LOG_KEY_REMOVE_WHILE_FREEING, GFP_NOFS); 8135de865eeSFilipe David Borba Manana if (!tm_list[i]) { 8145de865eeSFilipe David Borba Manana ret = -ENOMEM; 8155de865eeSFilipe David Borba Manana goto free_tms; 8165de865eeSFilipe David Borba Manana } 8175de865eeSFilipe David Borba Manana } 8185de865eeSFilipe David Borba Manana 819db7279a2SDavid Sterba if (tree_mod_dont_log(eb->fs_info, eb)) 8205de865eeSFilipe David Borba Manana goto free_tms; 8215de865eeSFilipe David Borba Manana 822db7279a2SDavid Sterba ret = __tree_mod_log_free_eb(eb->fs_info, tm_list, nritems); 823b1a09f1eSDavid Sterba write_unlock(&eb->fs_info->tree_mod_log_lock); 8245de865eeSFilipe David Borba Manana if (ret) 8255de865eeSFilipe David Borba Manana goto free_tms; 8265de865eeSFilipe David Borba Manana kfree(tm_list); 8275de865eeSFilipe David Borba Manana 8285de865eeSFilipe David Borba Manana return 0; 8295de865eeSFilipe David Borba Manana 8305de865eeSFilipe David Borba Manana free_tms: 8315de865eeSFilipe David Borba Manana for (i = 0; i < nritems; i++) 8325de865eeSFilipe David Borba Manana kfree(tm_list[i]); 8335de865eeSFilipe David Borba Manana kfree(tm_list); 8345de865eeSFilipe David Borba Manana 8355de865eeSFilipe David Borba Manana return ret; 836bd989ba3SJan Schmidt } 837bd989ba3SJan Schmidt 838d352ac68SChris Mason /* 8395d4f98a2SYan Zheng * check if the tree block can be shared by multiple trees 8405d4f98a2SYan Zheng */ 8415d4f98a2SYan Zheng int btrfs_block_can_be_shared(struct btrfs_root *root, 8425d4f98a2SYan Zheng struct extent_buffer *buf) 8435d4f98a2SYan Zheng { 8445d4f98a2SYan Zheng /* 84501327610SNicholas D Steeves * Tree blocks not in reference counted trees and tree roots 8465d4f98a2SYan Zheng * are never shared. If a block was allocated after the last 8475d4f98a2SYan Zheng * snapshot and the block was not allocated by tree relocation, 8485d4f98a2SYan Zheng * we know the block is not shared. 8495d4f98a2SYan Zheng */ 85027cdeb70SMiao Xie if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) && 8515d4f98a2SYan Zheng buf != root->node && buf != root->commit_root && 8525d4f98a2SYan Zheng (btrfs_header_generation(buf) <= 8535d4f98a2SYan Zheng btrfs_root_last_snapshot(&root->root_item) || 8545d4f98a2SYan Zheng btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC))) 8555d4f98a2SYan Zheng return 1; 856a79865c6SNikolay Borisov 8575d4f98a2SYan Zheng return 0; 8585d4f98a2SYan Zheng } 8595d4f98a2SYan Zheng 8605d4f98a2SYan Zheng static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans, 8615d4f98a2SYan Zheng struct btrfs_root *root, 8625d4f98a2SYan Zheng struct extent_buffer *buf, 863f0486c68SYan, Zheng struct extent_buffer *cow, 864f0486c68SYan, Zheng int *last_ref) 8655d4f98a2SYan Zheng { 8660b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 8675d4f98a2SYan Zheng u64 refs; 8685d4f98a2SYan Zheng u64 owner; 8695d4f98a2SYan Zheng u64 flags; 8705d4f98a2SYan Zheng u64 new_flags = 0; 8715d4f98a2SYan Zheng int ret; 8725d4f98a2SYan Zheng 8735d4f98a2SYan Zheng /* 8745d4f98a2SYan Zheng * Backrefs update rules: 8755d4f98a2SYan Zheng * 8765d4f98a2SYan Zheng * Always use full backrefs for extent pointers in tree block 8775d4f98a2SYan Zheng * allocated by tree relocation. 8785d4f98a2SYan Zheng * 8795d4f98a2SYan Zheng * If a shared tree block is no longer referenced by its owner 8805d4f98a2SYan Zheng * tree (btrfs_header_owner(buf) == root->root_key.objectid), 8815d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 8825d4f98a2SYan Zheng * 8835d4f98a2SYan Zheng * If a tree block is been relocating 8845d4f98a2SYan Zheng * (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID), 8855d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 8865d4f98a2SYan Zheng * The reason for this is some operations (such as drop tree) 8875d4f98a2SYan Zheng * are only allowed for blocks use full backrefs. 8885d4f98a2SYan Zheng */ 8895d4f98a2SYan Zheng 8905d4f98a2SYan Zheng if (btrfs_block_can_be_shared(root, buf)) { 8912ff7e61eSJeff Mahoney ret = btrfs_lookup_extent_info(trans, fs_info, buf->start, 8923173a18fSJosef Bacik btrfs_header_level(buf), 1, 8933173a18fSJosef Bacik &refs, &flags); 894be1a5564SMark Fasheh if (ret) 895be1a5564SMark Fasheh return ret; 896e5df9573SMark Fasheh if (refs == 0) { 897e5df9573SMark Fasheh ret = -EROFS; 8980b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 899e5df9573SMark Fasheh return ret; 900e5df9573SMark Fasheh } 9015d4f98a2SYan Zheng } else { 9025d4f98a2SYan Zheng refs = 1; 9035d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 9045d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 9055d4f98a2SYan Zheng flags = BTRFS_BLOCK_FLAG_FULL_BACKREF; 9065d4f98a2SYan Zheng else 9075d4f98a2SYan Zheng flags = 0; 9085d4f98a2SYan Zheng } 9095d4f98a2SYan Zheng 9105d4f98a2SYan Zheng owner = btrfs_header_owner(buf); 9115d4f98a2SYan Zheng BUG_ON(owner == BTRFS_TREE_RELOC_OBJECTID && 9125d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)); 9135d4f98a2SYan Zheng 9145d4f98a2SYan Zheng if (refs > 1) { 9155d4f98a2SYan Zheng if ((owner == root->root_key.objectid || 9165d4f98a2SYan Zheng root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && 9175d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) { 918e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, buf, 1); 919692826b2SJeff Mahoney if (ret) 920692826b2SJeff Mahoney return ret; 9215d4f98a2SYan Zheng 9225d4f98a2SYan Zheng if (root->root_key.objectid == 9235d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) { 924e339a6b0SJosef Bacik ret = btrfs_dec_ref(trans, root, buf, 0); 925692826b2SJeff Mahoney if (ret) 926692826b2SJeff Mahoney return ret; 927e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 928692826b2SJeff Mahoney if (ret) 929692826b2SJeff Mahoney return ret; 9305d4f98a2SYan Zheng } 9315d4f98a2SYan Zheng new_flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF; 9325d4f98a2SYan Zheng } else { 9335d4f98a2SYan Zheng 9345d4f98a2SYan Zheng if (root->root_key.objectid == 9355d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 936e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 9375d4f98a2SYan Zheng else 938e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 939692826b2SJeff Mahoney if (ret) 940692826b2SJeff Mahoney return ret; 9415d4f98a2SYan Zheng } 9425d4f98a2SYan Zheng if (new_flags != 0) { 943b1c79e09SJosef Bacik int level = btrfs_header_level(buf); 944b1c79e09SJosef Bacik 9452ff7e61eSJeff Mahoney ret = btrfs_set_disk_extent_flags(trans, fs_info, 9465d4f98a2SYan Zheng buf->start, 9475d4f98a2SYan Zheng buf->len, 948b1c79e09SJosef Bacik new_flags, level, 0); 949be1a5564SMark Fasheh if (ret) 950be1a5564SMark Fasheh return ret; 9515d4f98a2SYan Zheng } 9525d4f98a2SYan Zheng } else { 9535d4f98a2SYan Zheng if (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) { 9545d4f98a2SYan Zheng if (root->root_key.objectid == 9555d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 956e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 1); 9575d4f98a2SYan Zheng else 958e339a6b0SJosef Bacik ret = btrfs_inc_ref(trans, root, cow, 0); 959692826b2SJeff Mahoney if (ret) 960692826b2SJeff Mahoney return ret; 961e339a6b0SJosef Bacik ret = btrfs_dec_ref(trans, root, buf, 1); 962692826b2SJeff Mahoney if (ret) 963692826b2SJeff Mahoney return ret; 9645d4f98a2SYan Zheng } 9657c302b49SDavid Sterba clean_tree_block(fs_info, buf); 966f0486c68SYan, Zheng *last_ref = 1; 9675d4f98a2SYan Zheng } 9685d4f98a2SYan Zheng return 0; 9695d4f98a2SYan Zheng } 9705d4f98a2SYan Zheng 9715d4f98a2SYan Zheng /* 972d397712bSChris Mason * does the dirty work in cow of a single block. The parent block (if 973d397712bSChris Mason * supplied) is updated to point to the new cow copy. The new buffer is marked 974d397712bSChris Mason * dirty and returned locked. If you modify the block it needs to be marked 975d397712bSChris Mason * dirty again. 976d352ac68SChris Mason * 977d352ac68SChris Mason * search_start -- an allocation hint for the new block 978d352ac68SChris Mason * 979d397712bSChris Mason * empty_size -- a hint that you plan on doing more cow. This is the size in 980d397712bSChris Mason * bytes the allocator should try to find free next to the block it returns. 981d397712bSChris Mason * This is just a hint and may be ignored by the allocator. 982d352ac68SChris Mason */ 983d397712bSChris Mason static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans, 9845f39d397SChris Mason struct btrfs_root *root, 9855f39d397SChris Mason struct extent_buffer *buf, 9865f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 9875f39d397SChris Mason struct extent_buffer **cow_ret, 9889fa8cfe7SChris Mason u64 search_start, u64 empty_size) 9896702ed49SChris Mason { 9900b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 9915d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 9925f39d397SChris Mason struct extent_buffer *cow; 993be1a5564SMark Fasheh int level, ret; 994f0486c68SYan, Zheng int last_ref = 0; 995925baeddSChris Mason int unlock_orig = 0; 9960f5053ebSGoldwyn Rodrigues u64 parent_start = 0; 9976702ed49SChris Mason 998925baeddSChris Mason if (*cow_ret == buf) 999925baeddSChris Mason unlock_orig = 1; 1000925baeddSChris Mason 1001b9447ef8SChris Mason btrfs_assert_tree_locked(buf); 1002925baeddSChris Mason 100327cdeb70SMiao Xie WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) && 10040b246afaSJeff Mahoney trans->transid != fs_info->running_transaction->transid); 100527cdeb70SMiao Xie WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) && 100627cdeb70SMiao Xie trans->transid != root->last_trans); 10075f39d397SChris Mason 10087bb86316SChris Mason level = btrfs_header_level(buf); 100931840ae1SZheng Yan 10105d4f98a2SYan Zheng if (level == 0) 10115d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 10125d4f98a2SYan Zheng else 10135d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 10145d4f98a2SYan Zheng 10150f5053ebSGoldwyn Rodrigues if ((root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && parent) 10165d4f98a2SYan Zheng parent_start = parent->start; 10175d4f98a2SYan Zheng 10185ce55557SFilipe Manana /* 10195ce55557SFilipe Manana * If we are COWing a node/leaf from the extent, chunk or device trees, 10205ce55557SFilipe Manana * make sure that we do not finish block group creation of pending block 10215ce55557SFilipe Manana * groups. We do this to avoid a deadlock. 10225ce55557SFilipe Manana * COWing can result in allocation of a new chunk, and flushing pending 10235ce55557SFilipe Manana * block groups (btrfs_create_pending_block_groups()) can be triggered 10245ce55557SFilipe Manana * when finishing allocation of a new chunk. Creation of a pending block 10255ce55557SFilipe Manana * group modifies the extent, chunk and device trees, therefore we could 10265ce55557SFilipe Manana * deadlock with ourselves since we are holding a lock on an extent 10275ce55557SFilipe Manana * buffer that btrfs_create_pending_block_groups() may try to COW later. 10285ce55557SFilipe Manana */ 10295ce55557SFilipe Manana if (root == fs_info->extent_root || 10305ce55557SFilipe Manana root == fs_info->chunk_root || 10315ce55557SFilipe Manana root == fs_info->dev_root) 10325ce55557SFilipe Manana trans->can_flush_pending_bgs = false; 10335ce55557SFilipe Manana 10344d75f8a9SDavid Sterba cow = btrfs_alloc_tree_block(trans, root, parent_start, 10354d75f8a9SDavid Sterba root->root_key.objectid, &disk_key, level, 10364d75f8a9SDavid Sterba search_start, empty_size); 10375ce55557SFilipe Manana trans->can_flush_pending_bgs = true; 10386702ed49SChris Mason if (IS_ERR(cow)) 10396702ed49SChris Mason return PTR_ERR(cow); 10406702ed49SChris Mason 1041b4ce94deSChris Mason /* cow is set to blocking by btrfs_init_new_buffer */ 1042b4ce94deSChris Mason 104358e8012cSDavid Sterba copy_extent_buffer_full(cow, buf); 1044db94535dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 10455f39d397SChris Mason btrfs_set_header_generation(cow, trans->transid); 10465d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 10475d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 10485d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 10495d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) 10505d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 10515d4f98a2SYan Zheng else 10525f39d397SChris Mason btrfs_set_header_owner(cow, root->root_key.objectid); 10536702ed49SChris Mason 1054de37aa51SNikolay Borisov write_extent_buffer_fsid(cow, fs_info->fs_devices->metadata_uuid); 10552b82032cSYan Zheng 1056be1a5564SMark Fasheh ret = update_ref_for_cow(trans, root, buf, cow, &last_ref); 1057b68dc2a9SMark Fasheh if (ret) { 105866642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 1059b68dc2a9SMark Fasheh return ret; 1060b68dc2a9SMark Fasheh } 10611a40e23bSZheng Yan 106227cdeb70SMiao Xie if (test_bit(BTRFS_ROOT_REF_COWS, &root->state)) { 106383d4cfd4SJosef Bacik ret = btrfs_reloc_cow_block(trans, root, buf, cow); 106493314e3bSZhaolei if (ret) { 106566642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 106683d4cfd4SJosef Bacik return ret; 106783d4cfd4SJosef Bacik } 106893314e3bSZhaolei } 10693fd0a558SYan, Zheng 10706702ed49SChris Mason if (buf == root->node) { 1071925baeddSChris Mason WARN_ON(parent && parent != buf); 10725d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 10735d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 10745d4f98a2SYan Zheng parent_start = buf->start; 1075925baeddSChris Mason 10765f39d397SChris Mason extent_buffer_get(cow); 1077d9d19a01SDavid Sterba ret = tree_mod_log_insert_root(root->node, cow, 1); 1078d9d19a01SDavid Sterba BUG_ON(ret < 0); 1079240f62c8SChris Mason rcu_assign_pointer(root->node, cow); 1080925baeddSChris Mason 1081f0486c68SYan, Zheng btrfs_free_tree_block(trans, root, buf, parent_start, 10825581a51aSJan Schmidt last_ref); 10835f39d397SChris Mason free_extent_buffer(buf); 10840b86a832SChris Mason add_root_to_dirty_list(root); 10856702ed49SChris Mason } else { 10865d4f98a2SYan Zheng WARN_ON(trans->transid != btrfs_header_generation(parent)); 1087e09c2efeSDavid Sterba tree_mod_log_insert_key(parent, parent_slot, 1088c8cc6341SJosef Bacik MOD_LOG_KEY_REPLACE, GFP_NOFS); 10895f39d397SChris Mason btrfs_set_node_blockptr(parent, parent_slot, 1090db94535dSChris Mason cow->start); 109174493f7aSChris Mason btrfs_set_node_ptr_generation(parent, parent_slot, 109274493f7aSChris Mason trans->transid); 10936702ed49SChris Mason btrfs_mark_buffer_dirty(parent); 10945de865eeSFilipe David Borba Manana if (last_ref) { 1095db7279a2SDavid Sterba ret = tree_mod_log_free_eb(buf); 10965de865eeSFilipe David Borba Manana if (ret) { 109766642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 10985de865eeSFilipe David Borba Manana return ret; 10995de865eeSFilipe David Borba Manana } 11005de865eeSFilipe David Borba Manana } 1101f0486c68SYan, Zheng btrfs_free_tree_block(trans, root, buf, parent_start, 11025581a51aSJan Schmidt last_ref); 11036702ed49SChris Mason } 1104925baeddSChris Mason if (unlock_orig) 1105925baeddSChris Mason btrfs_tree_unlock(buf); 11063083ee2eSJosef Bacik free_extent_buffer_stale(buf); 11076702ed49SChris Mason btrfs_mark_buffer_dirty(cow); 11086702ed49SChris Mason *cow_ret = cow; 11096702ed49SChris Mason return 0; 11106702ed49SChris Mason } 11116702ed49SChris Mason 11125d9e75c4SJan Schmidt /* 11135d9e75c4SJan Schmidt * returns the logical address of the oldest predecessor of the given root. 11145d9e75c4SJan Schmidt * entries older than time_seq are ignored. 11155d9e75c4SJan Schmidt */ 1116bcd24dabSDavid Sterba static struct tree_mod_elem *__tree_mod_log_oldest_root( 111730b0463aSJan Schmidt struct extent_buffer *eb_root, u64 time_seq) 11185d9e75c4SJan Schmidt { 11195d9e75c4SJan Schmidt struct tree_mod_elem *tm; 11205d9e75c4SJan Schmidt struct tree_mod_elem *found = NULL; 112130b0463aSJan Schmidt u64 root_logical = eb_root->start; 11225d9e75c4SJan Schmidt int looped = 0; 11235d9e75c4SJan Schmidt 11245d9e75c4SJan Schmidt if (!time_seq) 112535a3621bSStefan Behrens return NULL; 11265d9e75c4SJan Schmidt 11275d9e75c4SJan Schmidt /* 1128298cfd36SChandan Rajendra * the very last operation that's logged for a root is the 1129298cfd36SChandan Rajendra * replacement operation (if it is replaced at all). this has 1130298cfd36SChandan Rajendra * the logical address of the *new* root, making it the very 1131298cfd36SChandan Rajendra * first operation that's logged for this root. 11325d9e75c4SJan Schmidt */ 11335d9e75c4SJan Schmidt while (1) { 1134bcd24dabSDavid Sterba tm = tree_mod_log_search_oldest(eb_root->fs_info, root_logical, 11355d9e75c4SJan Schmidt time_seq); 11365d9e75c4SJan Schmidt if (!looped && !tm) 113735a3621bSStefan Behrens return NULL; 11385d9e75c4SJan Schmidt /* 113928da9fb4SJan Schmidt * if there are no tree operation for the oldest root, we simply 114028da9fb4SJan Schmidt * return it. this should only happen if that (old) root is at 114128da9fb4SJan Schmidt * level 0. 11425d9e75c4SJan Schmidt */ 114328da9fb4SJan Schmidt if (!tm) 114428da9fb4SJan Schmidt break; 11455d9e75c4SJan Schmidt 114628da9fb4SJan Schmidt /* 114728da9fb4SJan Schmidt * if there's an operation that's not a root replacement, we 114828da9fb4SJan Schmidt * found the oldest version of our root. normally, we'll find a 114928da9fb4SJan Schmidt * MOD_LOG_KEY_REMOVE_WHILE_FREEING operation here. 115028da9fb4SJan Schmidt */ 11515d9e75c4SJan Schmidt if (tm->op != MOD_LOG_ROOT_REPLACE) 11525d9e75c4SJan Schmidt break; 11535d9e75c4SJan Schmidt 11545d9e75c4SJan Schmidt found = tm; 11555d9e75c4SJan Schmidt root_logical = tm->old_root.logical; 11565d9e75c4SJan Schmidt looped = 1; 11575d9e75c4SJan Schmidt } 11585d9e75c4SJan Schmidt 1159a95236d9SJan Schmidt /* if there's no old root to return, return what we found instead */ 1160a95236d9SJan Schmidt if (!found) 1161a95236d9SJan Schmidt found = tm; 1162a95236d9SJan Schmidt 11635d9e75c4SJan Schmidt return found; 11645d9e75c4SJan Schmidt } 11655d9e75c4SJan Schmidt 11665d9e75c4SJan Schmidt /* 11675d9e75c4SJan Schmidt * tm is a pointer to the first operation to rewind within eb. then, all 116801327610SNicholas D Steeves * previous operations will be rewound (until we reach something older than 11695d9e75c4SJan Schmidt * time_seq). 11705d9e75c4SJan Schmidt */ 11715d9e75c4SJan Schmidt static void 1172f1ca7e98SJosef Bacik __tree_mod_log_rewind(struct btrfs_fs_info *fs_info, struct extent_buffer *eb, 1173f1ca7e98SJosef Bacik u64 time_seq, struct tree_mod_elem *first_tm) 11745d9e75c4SJan Schmidt { 11755d9e75c4SJan Schmidt u32 n; 11765d9e75c4SJan Schmidt struct rb_node *next; 11775d9e75c4SJan Schmidt struct tree_mod_elem *tm = first_tm; 11785d9e75c4SJan Schmidt unsigned long o_dst; 11795d9e75c4SJan Schmidt unsigned long o_src; 11805d9e75c4SJan Schmidt unsigned long p_size = sizeof(struct btrfs_key_ptr); 11815d9e75c4SJan Schmidt 11825d9e75c4SJan Schmidt n = btrfs_header_nritems(eb); 1183b1a09f1eSDavid Sterba read_lock(&fs_info->tree_mod_log_lock); 1184097b8a7cSJan Schmidt while (tm && tm->seq >= time_seq) { 11855d9e75c4SJan Schmidt /* 11865d9e75c4SJan Schmidt * all the operations are recorded with the operator used for 11875d9e75c4SJan Schmidt * the modification. as we're going backwards, we do the 11885d9e75c4SJan Schmidt * opposite of each operation here. 11895d9e75c4SJan Schmidt */ 11905d9e75c4SJan Schmidt switch (tm->op) { 11915d9e75c4SJan Schmidt case MOD_LOG_KEY_REMOVE_WHILE_FREEING: 11925d9e75c4SJan Schmidt BUG_ON(tm->slot < n); 11931c697d4aSEric Sandeen /* Fallthrough */ 119495c80bb1SLiu Bo case MOD_LOG_KEY_REMOVE_WHILE_MOVING: 11954c3e6969SChris Mason case MOD_LOG_KEY_REMOVE: 11965d9e75c4SJan Schmidt btrfs_set_node_key(eb, &tm->key, tm->slot); 11975d9e75c4SJan Schmidt btrfs_set_node_blockptr(eb, tm->slot, tm->blockptr); 11985d9e75c4SJan Schmidt btrfs_set_node_ptr_generation(eb, tm->slot, 11995d9e75c4SJan Schmidt tm->generation); 12004c3e6969SChris Mason n++; 12015d9e75c4SJan Schmidt break; 12025d9e75c4SJan Schmidt case MOD_LOG_KEY_REPLACE: 12035d9e75c4SJan Schmidt BUG_ON(tm->slot >= n); 12045d9e75c4SJan Schmidt btrfs_set_node_key(eb, &tm->key, tm->slot); 12055d9e75c4SJan Schmidt btrfs_set_node_blockptr(eb, tm->slot, tm->blockptr); 12065d9e75c4SJan Schmidt btrfs_set_node_ptr_generation(eb, tm->slot, 12075d9e75c4SJan Schmidt tm->generation); 12085d9e75c4SJan Schmidt break; 12095d9e75c4SJan Schmidt case MOD_LOG_KEY_ADD: 121019956c7eSJan Schmidt /* if a move operation is needed it's in the log */ 12115d9e75c4SJan Schmidt n--; 12125d9e75c4SJan Schmidt break; 12135d9e75c4SJan Schmidt case MOD_LOG_MOVE_KEYS: 1214c3193108SJan Schmidt o_dst = btrfs_node_key_ptr_offset(tm->slot); 1215c3193108SJan Schmidt o_src = btrfs_node_key_ptr_offset(tm->move.dst_slot); 1216c3193108SJan Schmidt memmove_extent_buffer(eb, o_dst, o_src, 12175d9e75c4SJan Schmidt tm->move.nr_items * p_size); 12185d9e75c4SJan Schmidt break; 12195d9e75c4SJan Schmidt case MOD_LOG_ROOT_REPLACE: 12205d9e75c4SJan Schmidt /* 12215d9e75c4SJan Schmidt * this operation is special. for roots, this must be 12225d9e75c4SJan Schmidt * handled explicitly before rewinding. 12235d9e75c4SJan Schmidt * for non-roots, this operation may exist if the node 12245d9e75c4SJan Schmidt * was a root: root A -> child B; then A gets empty and 12255d9e75c4SJan Schmidt * B is promoted to the new root. in the mod log, we'll 12265d9e75c4SJan Schmidt * have a root-replace operation for B, a tree block 12275d9e75c4SJan Schmidt * that is no root. we simply ignore that operation. 12285d9e75c4SJan Schmidt */ 12295d9e75c4SJan Schmidt break; 12305d9e75c4SJan Schmidt } 12315d9e75c4SJan Schmidt next = rb_next(&tm->node); 12325d9e75c4SJan Schmidt if (!next) 12335d9e75c4SJan Schmidt break; 12346b4df8b6SGeliang Tang tm = rb_entry(next, struct tree_mod_elem, node); 1235298cfd36SChandan Rajendra if (tm->logical != first_tm->logical) 12365d9e75c4SJan Schmidt break; 12375d9e75c4SJan Schmidt } 1238b1a09f1eSDavid Sterba read_unlock(&fs_info->tree_mod_log_lock); 12395d9e75c4SJan Schmidt btrfs_set_header_nritems(eb, n); 12405d9e75c4SJan Schmidt } 12415d9e75c4SJan Schmidt 124247fb091fSJan Schmidt /* 124301327610SNicholas D Steeves * Called with eb read locked. If the buffer cannot be rewound, the same buffer 124447fb091fSJan Schmidt * is returned. If rewind operations happen, a fresh buffer is returned. The 124547fb091fSJan Schmidt * returned buffer is always read-locked. If the returned buffer is not the 124647fb091fSJan Schmidt * input buffer, the lock on the input buffer is released and the input buffer 124747fb091fSJan Schmidt * is freed (its refcount is decremented). 124847fb091fSJan Schmidt */ 12495d9e75c4SJan Schmidt static struct extent_buffer * 12509ec72677SJosef Bacik tree_mod_log_rewind(struct btrfs_fs_info *fs_info, struct btrfs_path *path, 12519ec72677SJosef Bacik struct extent_buffer *eb, u64 time_seq) 12525d9e75c4SJan Schmidt { 12535d9e75c4SJan Schmidt struct extent_buffer *eb_rewin; 12545d9e75c4SJan Schmidt struct tree_mod_elem *tm; 12555d9e75c4SJan Schmidt 12565d9e75c4SJan Schmidt if (!time_seq) 12575d9e75c4SJan Schmidt return eb; 12585d9e75c4SJan Schmidt 12595d9e75c4SJan Schmidt if (btrfs_header_level(eb) == 0) 12605d9e75c4SJan Schmidt return eb; 12615d9e75c4SJan Schmidt 12625d9e75c4SJan Schmidt tm = tree_mod_log_search(fs_info, eb->start, time_seq); 12635d9e75c4SJan Schmidt if (!tm) 12645d9e75c4SJan Schmidt return eb; 12655d9e75c4SJan Schmidt 12669ec72677SJosef Bacik btrfs_set_path_blocking(path); 12679ec72677SJosef Bacik btrfs_set_lock_blocking_rw(eb, BTRFS_READ_LOCK); 12689ec72677SJosef Bacik 12695d9e75c4SJan Schmidt if (tm->op == MOD_LOG_KEY_REMOVE_WHILE_FREEING) { 12705d9e75c4SJan Schmidt BUG_ON(tm->slot != 0); 1271da17066cSJeff Mahoney eb_rewin = alloc_dummy_extent_buffer(fs_info, eb->start); 1272db7f3436SJosef Bacik if (!eb_rewin) { 12739ec72677SJosef Bacik btrfs_tree_read_unlock_blocking(eb); 1274db7f3436SJosef Bacik free_extent_buffer(eb); 1275db7f3436SJosef Bacik return NULL; 1276db7f3436SJosef Bacik } 12775d9e75c4SJan Schmidt btrfs_set_header_bytenr(eb_rewin, eb->start); 12785d9e75c4SJan Schmidt btrfs_set_header_backref_rev(eb_rewin, 12795d9e75c4SJan Schmidt btrfs_header_backref_rev(eb)); 12805d9e75c4SJan Schmidt btrfs_set_header_owner(eb_rewin, btrfs_header_owner(eb)); 1281c3193108SJan Schmidt btrfs_set_header_level(eb_rewin, btrfs_header_level(eb)); 12825d9e75c4SJan Schmidt } else { 12835d9e75c4SJan Schmidt eb_rewin = btrfs_clone_extent_buffer(eb); 1284db7f3436SJosef Bacik if (!eb_rewin) { 12859ec72677SJosef Bacik btrfs_tree_read_unlock_blocking(eb); 1286db7f3436SJosef Bacik free_extent_buffer(eb); 1287db7f3436SJosef Bacik return NULL; 1288db7f3436SJosef Bacik } 12895d9e75c4SJan Schmidt } 12905d9e75c4SJan Schmidt 12919ec72677SJosef Bacik btrfs_tree_read_unlock_blocking(eb); 12925d9e75c4SJan Schmidt free_extent_buffer(eb); 12935d9e75c4SJan Schmidt 129447fb091fSJan Schmidt btrfs_tree_read_lock(eb_rewin); 1295f1ca7e98SJosef Bacik __tree_mod_log_rewind(fs_info, eb_rewin, time_seq, tm); 129657911b8bSJan Schmidt WARN_ON(btrfs_header_nritems(eb_rewin) > 1297da17066cSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info)); 12985d9e75c4SJan Schmidt 12995d9e75c4SJan Schmidt return eb_rewin; 13005d9e75c4SJan Schmidt } 13015d9e75c4SJan Schmidt 13028ba97a15SJan Schmidt /* 13038ba97a15SJan Schmidt * get_old_root() rewinds the state of @root's root node to the given @time_seq 13048ba97a15SJan Schmidt * value. If there are no changes, the current root->root_node is returned. If 13058ba97a15SJan Schmidt * anything changed in between, there's a fresh buffer allocated on which the 13068ba97a15SJan Schmidt * rewind operations are done. In any case, the returned buffer is read locked. 13078ba97a15SJan Schmidt * Returns NULL on error (with no locks held). 13088ba97a15SJan Schmidt */ 13095d9e75c4SJan Schmidt static inline struct extent_buffer * 13105d9e75c4SJan Schmidt get_old_root(struct btrfs_root *root, u64 time_seq) 13115d9e75c4SJan Schmidt { 13120b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 13135d9e75c4SJan Schmidt struct tree_mod_elem *tm; 131430b0463aSJan Schmidt struct extent_buffer *eb = NULL; 131530b0463aSJan Schmidt struct extent_buffer *eb_root; 13167bfdcf7fSLiu Bo struct extent_buffer *old; 1317a95236d9SJan Schmidt struct tree_mod_root *old_root = NULL; 13184325edd0SChris Mason u64 old_generation = 0; 1319a95236d9SJan Schmidt u64 logical; 1320581c1760SQu Wenruo int level; 13215d9e75c4SJan Schmidt 132230b0463aSJan Schmidt eb_root = btrfs_read_lock_root_node(root); 1323bcd24dabSDavid Sterba tm = __tree_mod_log_oldest_root(eb_root, time_seq); 13245d9e75c4SJan Schmidt if (!tm) 132530b0463aSJan Schmidt return eb_root; 13265d9e75c4SJan Schmidt 1327a95236d9SJan Schmidt if (tm->op == MOD_LOG_ROOT_REPLACE) { 13285d9e75c4SJan Schmidt old_root = &tm->old_root; 13295d9e75c4SJan Schmidt old_generation = tm->generation; 1330a95236d9SJan Schmidt logical = old_root->logical; 1331581c1760SQu Wenruo level = old_root->level; 1332a95236d9SJan Schmidt } else { 133330b0463aSJan Schmidt logical = eb_root->start; 1334581c1760SQu Wenruo level = btrfs_header_level(eb_root); 1335a95236d9SJan Schmidt } 13365d9e75c4SJan Schmidt 13370b246afaSJeff Mahoney tm = tree_mod_log_search(fs_info, logical, time_seq); 1338834328a8SJan Schmidt if (old_root && tm && tm->op != MOD_LOG_KEY_REMOVE_WHILE_FREEING) { 133930b0463aSJan Schmidt btrfs_tree_read_unlock(eb_root); 134030b0463aSJan Schmidt free_extent_buffer(eb_root); 1341581c1760SQu Wenruo old = read_tree_block(fs_info, logical, 0, level, NULL); 134264c043deSLiu Bo if (WARN_ON(IS_ERR(old) || !extent_buffer_uptodate(old))) { 134364c043deSLiu Bo if (!IS_ERR(old)) 1344416bc658SJosef Bacik free_extent_buffer(old); 13450b246afaSJeff Mahoney btrfs_warn(fs_info, 13460b246afaSJeff Mahoney "failed to read tree block %llu from get_old_root", 13470b246afaSJeff Mahoney logical); 1348834328a8SJan Schmidt } else { 13497bfdcf7fSLiu Bo eb = btrfs_clone_extent_buffer(old); 13507bfdcf7fSLiu Bo free_extent_buffer(old); 1351834328a8SJan Schmidt } 1352834328a8SJan Schmidt } else if (old_root) { 135330b0463aSJan Schmidt btrfs_tree_read_unlock(eb_root); 135430b0463aSJan Schmidt free_extent_buffer(eb_root); 13550b246afaSJeff Mahoney eb = alloc_dummy_extent_buffer(fs_info, logical); 1356834328a8SJan Schmidt } else { 13579ec72677SJosef Bacik btrfs_set_lock_blocking_rw(eb_root, BTRFS_READ_LOCK); 135830b0463aSJan Schmidt eb = btrfs_clone_extent_buffer(eb_root); 13599ec72677SJosef Bacik btrfs_tree_read_unlock_blocking(eb_root); 136030b0463aSJan Schmidt free_extent_buffer(eb_root); 1361834328a8SJan Schmidt } 1362834328a8SJan Schmidt 13638ba97a15SJan Schmidt if (!eb) 13648ba97a15SJan Schmidt return NULL; 13658ba97a15SJan Schmidt btrfs_tree_read_lock(eb); 1366a95236d9SJan Schmidt if (old_root) { 13675d9e75c4SJan Schmidt btrfs_set_header_bytenr(eb, eb->start); 13685d9e75c4SJan Schmidt btrfs_set_header_backref_rev(eb, BTRFS_MIXED_BACKREF_REV); 136930b0463aSJan Schmidt btrfs_set_header_owner(eb, btrfs_header_owner(eb_root)); 13705d9e75c4SJan Schmidt btrfs_set_header_level(eb, old_root->level); 13715d9e75c4SJan Schmidt btrfs_set_header_generation(eb, old_generation); 1372a95236d9SJan Schmidt } 137328da9fb4SJan Schmidt if (tm) 13740b246afaSJeff Mahoney __tree_mod_log_rewind(fs_info, eb, time_seq, tm); 137528da9fb4SJan Schmidt else 137628da9fb4SJan Schmidt WARN_ON(btrfs_header_level(eb) != 0); 13770b246afaSJeff Mahoney WARN_ON(btrfs_header_nritems(eb) > BTRFS_NODEPTRS_PER_BLOCK(fs_info)); 13785d9e75c4SJan Schmidt 13795d9e75c4SJan Schmidt return eb; 13805d9e75c4SJan Schmidt } 13815d9e75c4SJan Schmidt 13825b6602e7SJan Schmidt int btrfs_old_root_level(struct btrfs_root *root, u64 time_seq) 13835b6602e7SJan Schmidt { 13845b6602e7SJan Schmidt struct tree_mod_elem *tm; 13855b6602e7SJan Schmidt int level; 138630b0463aSJan Schmidt struct extent_buffer *eb_root = btrfs_root_node(root); 13875b6602e7SJan Schmidt 1388bcd24dabSDavid Sterba tm = __tree_mod_log_oldest_root(eb_root, time_seq); 13895b6602e7SJan Schmidt if (tm && tm->op == MOD_LOG_ROOT_REPLACE) { 13905b6602e7SJan Schmidt level = tm->old_root.level; 13915b6602e7SJan Schmidt } else { 139230b0463aSJan Schmidt level = btrfs_header_level(eb_root); 13935b6602e7SJan Schmidt } 139430b0463aSJan Schmidt free_extent_buffer(eb_root); 13955b6602e7SJan Schmidt 13965b6602e7SJan Schmidt return level; 13975b6602e7SJan Schmidt } 13985b6602e7SJan Schmidt 13995d4f98a2SYan Zheng static inline int should_cow_block(struct btrfs_trans_handle *trans, 14005d4f98a2SYan Zheng struct btrfs_root *root, 14015d4f98a2SYan Zheng struct extent_buffer *buf) 14025d4f98a2SYan Zheng { 1403f5ee5c9aSJeff Mahoney if (btrfs_is_testing(root->fs_info)) 1404faa2dbf0SJosef Bacik return 0; 1405fccb84c9SDavid Sterba 1406d1980131SDavid Sterba /* Ensure we can see the FORCE_COW bit */ 1407d1980131SDavid Sterba smp_mb__before_atomic(); 1408f1ebcc74SLiu Bo 1409f1ebcc74SLiu Bo /* 1410f1ebcc74SLiu Bo * We do not need to cow a block if 1411f1ebcc74SLiu Bo * 1) this block is not created or changed in this transaction; 1412f1ebcc74SLiu Bo * 2) this block does not belong to TREE_RELOC tree; 1413f1ebcc74SLiu Bo * 3) the root is not forced COW. 1414f1ebcc74SLiu Bo * 1415f1ebcc74SLiu Bo * What is forced COW: 141601327610SNicholas D Steeves * when we create snapshot during committing the transaction, 1417f1ebcc74SLiu Bo * after we've finished coping src root, we must COW the shared 1418f1ebcc74SLiu Bo * block to ensure the metadata consistency. 1419f1ebcc74SLiu Bo */ 14205d4f98a2SYan Zheng if (btrfs_header_generation(buf) == trans->transid && 14215d4f98a2SYan Zheng !btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN) && 14225d4f98a2SYan Zheng !(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID && 1423f1ebcc74SLiu Bo btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)) && 142427cdeb70SMiao Xie !test_bit(BTRFS_ROOT_FORCE_COW, &root->state)) 14255d4f98a2SYan Zheng return 0; 14265d4f98a2SYan Zheng return 1; 14275d4f98a2SYan Zheng } 14285d4f98a2SYan Zheng 1429d352ac68SChris Mason /* 1430d352ac68SChris Mason * cows a single block, see __btrfs_cow_block for the real work. 143101327610SNicholas D Steeves * This version of it has extra checks so that a block isn't COWed more than 1432d352ac68SChris Mason * once per transaction, as long as it hasn't been written yet 1433d352ac68SChris Mason */ 1434d397712bSChris Mason noinline int btrfs_cow_block(struct btrfs_trans_handle *trans, 14355f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *buf, 14365f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 14379fa8cfe7SChris Mason struct extent_buffer **cow_ret) 143802217ed2SChris Mason { 14390b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 14406702ed49SChris Mason u64 search_start; 1441f510cfecSChris Mason int ret; 1442dc17ff8fSChris Mason 14430b246afaSJeff Mahoney if (trans->transaction != fs_info->running_transaction) 144431b1a2bdSJulia Lawall WARN(1, KERN_CRIT "trans %llu running %llu\n", 1445c1c9ff7cSGeert Uytterhoeven trans->transid, 14460b246afaSJeff Mahoney fs_info->running_transaction->transid); 144731b1a2bdSJulia Lawall 14480b246afaSJeff Mahoney if (trans->transid != fs_info->generation) 144931b1a2bdSJulia Lawall WARN(1, KERN_CRIT "trans %llu running %llu\n", 14500b246afaSJeff Mahoney trans->transid, fs_info->generation); 1451dc17ff8fSChris Mason 14525d4f98a2SYan Zheng if (!should_cow_block(trans, root, buf)) { 145364c12921SJeff Mahoney trans->dirty = true; 145402217ed2SChris Mason *cow_ret = buf; 145502217ed2SChris Mason return 0; 145602217ed2SChris Mason } 1457c487685dSChris Mason 1458ee22184bSByongho Lee search_start = buf->start & ~((u64)SZ_1G - 1); 1459b4ce94deSChris Mason 1460b4ce94deSChris Mason if (parent) 1461b4ce94deSChris Mason btrfs_set_lock_blocking(parent); 1462b4ce94deSChris Mason btrfs_set_lock_blocking(buf); 1463b4ce94deSChris Mason 1464f510cfecSChris Mason ret = __btrfs_cow_block(trans, root, buf, parent, 14659fa8cfe7SChris Mason parent_slot, cow_ret, search_start, 0); 14661abe9b8aSliubo 14671abe9b8aSliubo trace_btrfs_cow_block(root, buf, *cow_ret); 14681abe9b8aSliubo 1469f510cfecSChris Mason return ret; 14702c90e5d6SChris Mason } 14716702ed49SChris Mason 1472d352ac68SChris Mason /* 1473d352ac68SChris Mason * helper function for defrag to decide if two blocks pointed to by a 1474d352ac68SChris Mason * node are actually close by 1475d352ac68SChris Mason */ 14766b80053dSChris Mason static int close_blocks(u64 blocknr, u64 other, u32 blocksize) 14776702ed49SChris Mason { 14786b80053dSChris Mason if (blocknr < other && other - (blocknr + blocksize) < 32768) 14796702ed49SChris Mason return 1; 14806b80053dSChris Mason if (blocknr > other && blocknr - (other + blocksize) < 32768) 14816702ed49SChris Mason return 1; 148202217ed2SChris Mason return 0; 148302217ed2SChris Mason } 148402217ed2SChris Mason 1485081e9573SChris Mason /* 1486081e9573SChris Mason * compare two keys in a memcmp fashion 1487081e9573SChris Mason */ 1488310712b2SOmar Sandoval static int comp_keys(const struct btrfs_disk_key *disk, 1489310712b2SOmar Sandoval const struct btrfs_key *k2) 1490081e9573SChris Mason { 1491081e9573SChris Mason struct btrfs_key k1; 1492081e9573SChris Mason 1493081e9573SChris Mason btrfs_disk_key_to_cpu(&k1, disk); 1494081e9573SChris Mason 149520736abaSDiego Calleja return btrfs_comp_cpu_keys(&k1, k2); 1496081e9573SChris Mason } 1497081e9573SChris Mason 1498f3465ca4SJosef Bacik /* 1499f3465ca4SJosef Bacik * same as comp_keys only with two btrfs_key's 1500f3465ca4SJosef Bacik */ 1501310712b2SOmar Sandoval int btrfs_comp_cpu_keys(const struct btrfs_key *k1, const struct btrfs_key *k2) 1502f3465ca4SJosef Bacik { 1503f3465ca4SJosef Bacik if (k1->objectid > k2->objectid) 1504f3465ca4SJosef Bacik return 1; 1505f3465ca4SJosef Bacik if (k1->objectid < k2->objectid) 1506f3465ca4SJosef Bacik return -1; 1507f3465ca4SJosef Bacik if (k1->type > k2->type) 1508f3465ca4SJosef Bacik return 1; 1509f3465ca4SJosef Bacik if (k1->type < k2->type) 1510f3465ca4SJosef Bacik return -1; 1511f3465ca4SJosef Bacik if (k1->offset > k2->offset) 1512f3465ca4SJosef Bacik return 1; 1513f3465ca4SJosef Bacik if (k1->offset < k2->offset) 1514f3465ca4SJosef Bacik return -1; 1515f3465ca4SJosef Bacik return 0; 1516f3465ca4SJosef Bacik } 1517081e9573SChris Mason 1518d352ac68SChris Mason /* 1519d352ac68SChris Mason * this is used by the defrag code to go through all the 1520d352ac68SChris Mason * leaves pointed to by a node and reallocate them so that 1521d352ac68SChris Mason * disk order is close to key order 1522d352ac68SChris Mason */ 15236702ed49SChris Mason int btrfs_realloc_node(struct btrfs_trans_handle *trans, 15245f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *parent, 1525de78b51aSEric Sandeen int start_slot, u64 *last_ret, 1526a6b6e75eSChris Mason struct btrfs_key *progress) 15276702ed49SChris Mason { 15280b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 15296b80053dSChris Mason struct extent_buffer *cur; 15306702ed49SChris Mason u64 blocknr; 1531ca7a79adSChris Mason u64 gen; 1532e9d0b13bSChris Mason u64 search_start = *last_ret; 1533e9d0b13bSChris Mason u64 last_block = 0; 15346702ed49SChris Mason u64 other; 15356702ed49SChris Mason u32 parent_nritems; 15366702ed49SChris Mason int end_slot; 15376702ed49SChris Mason int i; 15386702ed49SChris Mason int err = 0; 1539f2183bdeSChris Mason int parent_level; 15406b80053dSChris Mason int uptodate; 15416b80053dSChris Mason u32 blocksize; 1542081e9573SChris Mason int progress_passed = 0; 1543081e9573SChris Mason struct btrfs_disk_key disk_key; 15446702ed49SChris Mason 15455708b959SChris Mason parent_level = btrfs_header_level(parent); 15465708b959SChris Mason 15470b246afaSJeff Mahoney WARN_ON(trans->transaction != fs_info->running_transaction); 15480b246afaSJeff Mahoney WARN_ON(trans->transid != fs_info->generation); 154986479a04SChris Mason 15506b80053dSChris Mason parent_nritems = btrfs_header_nritems(parent); 15510b246afaSJeff Mahoney blocksize = fs_info->nodesize; 15525dfe2be7SFilipe Manana end_slot = parent_nritems - 1; 15536702ed49SChris Mason 15545dfe2be7SFilipe Manana if (parent_nritems <= 1) 15556702ed49SChris Mason return 0; 15566702ed49SChris Mason 1557b4ce94deSChris Mason btrfs_set_lock_blocking(parent); 1558b4ce94deSChris Mason 15595dfe2be7SFilipe Manana for (i = start_slot; i <= end_slot; i++) { 1560581c1760SQu Wenruo struct btrfs_key first_key; 15616702ed49SChris Mason int close = 1; 1562a6b6e75eSChris Mason 1563081e9573SChris Mason btrfs_node_key(parent, &disk_key, i); 1564081e9573SChris Mason if (!progress_passed && comp_keys(&disk_key, progress) < 0) 1565081e9573SChris Mason continue; 1566081e9573SChris Mason 1567081e9573SChris Mason progress_passed = 1; 15686b80053dSChris Mason blocknr = btrfs_node_blockptr(parent, i); 1569ca7a79adSChris Mason gen = btrfs_node_ptr_generation(parent, i); 1570581c1760SQu Wenruo btrfs_node_key_to_cpu(parent, &first_key, i); 1571e9d0b13bSChris Mason if (last_block == 0) 1572e9d0b13bSChris Mason last_block = blocknr; 15735708b959SChris Mason 15746702ed49SChris Mason if (i > 0) { 15756b80053dSChris Mason other = btrfs_node_blockptr(parent, i - 1); 15766b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 15776702ed49SChris Mason } 15785dfe2be7SFilipe Manana if (!close && i < end_slot) { 15796b80053dSChris Mason other = btrfs_node_blockptr(parent, i + 1); 15806b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 15816702ed49SChris Mason } 1582e9d0b13bSChris Mason if (close) { 1583e9d0b13bSChris Mason last_block = blocknr; 15846702ed49SChris Mason continue; 1585e9d0b13bSChris Mason } 15866702ed49SChris Mason 15870b246afaSJeff Mahoney cur = find_extent_buffer(fs_info, blocknr); 15886b80053dSChris Mason if (cur) 1589b9fab919SChris Mason uptodate = btrfs_buffer_uptodate(cur, gen, 0); 15906b80053dSChris Mason else 15916b80053dSChris Mason uptodate = 0; 15925708b959SChris Mason if (!cur || !uptodate) { 15936b80053dSChris Mason if (!cur) { 1594581c1760SQu Wenruo cur = read_tree_block(fs_info, blocknr, gen, 1595581c1760SQu Wenruo parent_level - 1, 1596581c1760SQu Wenruo &first_key); 159764c043deSLiu Bo if (IS_ERR(cur)) { 159864c043deSLiu Bo return PTR_ERR(cur); 159964c043deSLiu Bo } else if (!extent_buffer_uptodate(cur)) { 1600416bc658SJosef Bacik free_extent_buffer(cur); 160197d9a8a4STsutomu Itoh return -EIO; 1602416bc658SJosef Bacik } 16036b80053dSChris Mason } else if (!uptodate) { 1604581c1760SQu Wenruo err = btrfs_read_buffer(cur, gen, 1605581c1760SQu Wenruo parent_level - 1,&first_key); 1606018642a1STsutomu Itoh if (err) { 1607018642a1STsutomu Itoh free_extent_buffer(cur); 1608018642a1STsutomu Itoh return err; 1609018642a1STsutomu Itoh } 16106702ed49SChris Mason } 1611f2183bdeSChris Mason } 1612e9d0b13bSChris Mason if (search_start == 0) 16136b80053dSChris Mason search_start = last_block; 1614e9d0b13bSChris Mason 1615e7a84565SChris Mason btrfs_tree_lock(cur); 1616b4ce94deSChris Mason btrfs_set_lock_blocking(cur); 16176b80053dSChris Mason err = __btrfs_cow_block(trans, root, cur, parent, i, 1618e7a84565SChris Mason &cur, search_start, 16196b80053dSChris Mason min(16 * blocksize, 16209fa8cfe7SChris Mason (end_slot - i) * blocksize)); 1621252c38f0SYan if (err) { 1622e7a84565SChris Mason btrfs_tree_unlock(cur); 16236b80053dSChris Mason free_extent_buffer(cur); 16246702ed49SChris Mason break; 1625252c38f0SYan } 1626e7a84565SChris Mason search_start = cur->start; 1627e7a84565SChris Mason last_block = cur->start; 1628f2183bdeSChris Mason *last_ret = search_start; 1629e7a84565SChris Mason btrfs_tree_unlock(cur); 1630e7a84565SChris Mason free_extent_buffer(cur); 16316702ed49SChris Mason } 16326702ed49SChris Mason return err; 16336702ed49SChris Mason } 16346702ed49SChris Mason 163574123bd7SChris Mason /* 16365f39d397SChris Mason * search for key in the extent_buffer. The items start at offset p, 16375f39d397SChris Mason * and they are item_size apart. There are 'max' items in p. 16385f39d397SChris Mason * 163974123bd7SChris Mason * the slot in the array is returned via slot, and it points to 164074123bd7SChris Mason * the place where you would insert key if it is not found in 164174123bd7SChris Mason * the array. 164274123bd7SChris Mason * 164374123bd7SChris Mason * slot may point to max if the key is bigger than all of the keys 164474123bd7SChris Mason */ 1645e02119d5SChris Mason static noinline int generic_bin_search(struct extent_buffer *eb, 1646310712b2SOmar Sandoval unsigned long p, int item_size, 1647310712b2SOmar Sandoval const struct btrfs_key *key, 1648be0e5c09SChris Mason int max, int *slot) 1649be0e5c09SChris Mason { 1650be0e5c09SChris Mason int low = 0; 1651be0e5c09SChris Mason int high = max; 1652be0e5c09SChris Mason int mid; 1653be0e5c09SChris Mason int ret; 1654479965d6SChris Mason struct btrfs_disk_key *tmp = NULL; 16555f39d397SChris Mason struct btrfs_disk_key unaligned; 16565f39d397SChris Mason unsigned long offset; 16575f39d397SChris Mason char *kaddr = NULL; 16585f39d397SChris Mason unsigned long map_start = 0; 16595f39d397SChris Mason unsigned long map_len = 0; 1660479965d6SChris Mason int err; 1661be0e5c09SChris Mason 16625e24e9afSLiu Bo if (low > high) { 16635e24e9afSLiu Bo btrfs_err(eb->fs_info, 16645e24e9afSLiu Bo "%s: low (%d) > high (%d) eb %llu owner %llu level %d", 16655e24e9afSLiu Bo __func__, low, high, eb->start, 16665e24e9afSLiu Bo btrfs_header_owner(eb), btrfs_header_level(eb)); 16675e24e9afSLiu Bo return -EINVAL; 16685e24e9afSLiu Bo } 16695e24e9afSLiu Bo 1670be0e5c09SChris Mason while (low < high) { 1671be0e5c09SChris Mason mid = (low + high) / 2; 16725f39d397SChris Mason offset = p + mid * item_size; 16735f39d397SChris Mason 1674a6591715SChris Mason if (!kaddr || offset < map_start || 16755f39d397SChris Mason (offset + sizeof(struct btrfs_disk_key)) > 16765f39d397SChris Mason map_start + map_len) { 1677934d375bSChris Mason 1678934d375bSChris Mason err = map_private_extent_buffer(eb, offset, 1679479965d6SChris Mason sizeof(struct btrfs_disk_key), 1680a6591715SChris Mason &kaddr, &map_start, &map_len); 16815f39d397SChris Mason 1682479965d6SChris Mason if (!err) { 1683479965d6SChris Mason tmp = (struct btrfs_disk_key *)(kaddr + offset - 1684479965d6SChris Mason map_start); 1685415b35a5SLiu Bo } else if (err == 1) { 16865f39d397SChris Mason read_extent_buffer(eb, &unaligned, 16875f39d397SChris Mason offset, sizeof(unaligned)); 16885f39d397SChris Mason tmp = &unaligned; 1689415b35a5SLiu Bo } else { 1690415b35a5SLiu Bo return err; 1691479965d6SChris Mason } 1692479965d6SChris Mason 16935f39d397SChris Mason } else { 16945f39d397SChris Mason tmp = (struct btrfs_disk_key *)(kaddr + offset - 16955f39d397SChris Mason map_start); 16965f39d397SChris Mason } 1697be0e5c09SChris Mason ret = comp_keys(tmp, key); 1698be0e5c09SChris Mason 1699be0e5c09SChris Mason if (ret < 0) 1700be0e5c09SChris Mason low = mid + 1; 1701be0e5c09SChris Mason else if (ret > 0) 1702be0e5c09SChris Mason high = mid; 1703be0e5c09SChris Mason else { 1704be0e5c09SChris Mason *slot = mid; 1705be0e5c09SChris Mason return 0; 1706be0e5c09SChris Mason } 1707be0e5c09SChris Mason } 1708be0e5c09SChris Mason *slot = low; 1709be0e5c09SChris Mason return 1; 1710be0e5c09SChris Mason } 1711be0e5c09SChris Mason 171297571fd0SChris Mason /* 171397571fd0SChris Mason * simple bin_search frontend that does the right thing for 171497571fd0SChris Mason * leaves vs nodes 171597571fd0SChris Mason */ 1716a74b35ecSNikolay Borisov int btrfs_bin_search(struct extent_buffer *eb, const struct btrfs_key *key, 17175f39d397SChris Mason int level, int *slot) 1718be0e5c09SChris Mason { 1719f775738fSWang Sheng-Hui if (level == 0) 17205f39d397SChris Mason return generic_bin_search(eb, 17215f39d397SChris Mason offsetof(struct btrfs_leaf, items), 17220783fcfcSChris Mason sizeof(struct btrfs_item), 17235f39d397SChris Mason key, btrfs_header_nritems(eb), 17247518a238SChris Mason slot); 1725f775738fSWang Sheng-Hui else 17265f39d397SChris Mason return generic_bin_search(eb, 17275f39d397SChris Mason offsetof(struct btrfs_node, ptrs), 1728123abc88SChris Mason sizeof(struct btrfs_key_ptr), 17295f39d397SChris Mason key, btrfs_header_nritems(eb), 17307518a238SChris Mason slot); 1731be0e5c09SChris Mason } 1732be0e5c09SChris Mason 1733f0486c68SYan, Zheng static void root_add_used(struct btrfs_root *root, u32 size) 1734f0486c68SYan, Zheng { 1735f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 1736f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 1737f0486c68SYan, Zheng btrfs_root_used(&root->root_item) + size); 1738f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 1739f0486c68SYan, Zheng } 1740f0486c68SYan, Zheng 1741f0486c68SYan, Zheng static void root_sub_used(struct btrfs_root *root, u32 size) 1742f0486c68SYan, Zheng { 1743f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 1744f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 1745f0486c68SYan, Zheng btrfs_root_used(&root->root_item) - size); 1746f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 1747f0486c68SYan, Zheng } 1748f0486c68SYan, Zheng 1749d352ac68SChris Mason /* given a node and slot number, this reads the blocks it points to. The 1750d352ac68SChris Mason * extent buffer is returned with a reference taken (but unlocked). 1751d352ac68SChris Mason */ 17522ff7e61eSJeff Mahoney static noinline struct extent_buffer * 17532ff7e61eSJeff Mahoney read_node_slot(struct btrfs_fs_info *fs_info, struct extent_buffer *parent, 17542ff7e61eSJeff Mahoney int slot) 1755bb803951SChris Mason { 1756ca7a79adSChris Mason int level = btrfs_header_level(parent); 1757416bc658SJosef Bacik struct extent_buffer *eb; 1758581c1760SQu Wenruo struct btrfs_key first_key; 1759416bc658SJosef Bacik 1760fb770ae4SLiu Bo if (slot < 0 || slot >= btrfs_header_nritems(parent)) 1761fb770ae4SLiu Bo return ERR_PTR(-ENOENT); 1762ca7a79adSChris Mason 1763ca7a79adSChris Mason BUG_ON(level == 0); 1764ca7a79adSChris Mason 1765581c1760SQu Wenruo btrfs_node_key_to_cpu(parent, &first_key, slot); 17662ff7e61eSJeff Mahoney eb = read_tree_block(fs_info, btrfs_node_blockptr(parent, slot), 1767581c1760SQu Wenruo btrfs_node_ptr_generation(parent, slot), 1768581c1760SQu Wenruo level - 1, &first_key); 1769fb770ae4SLiu Bo if (!IS_ERR(eb) && !extent_buffer_uptodate(eb)) { 1770416bc658SJosef Bacik free_extent_buffer(eb); 1771fb770ae4SLiu Bo eb = ERR_PTR(-EIO); 1772416bc658SJosef Bacik } 1773416bc658SJosef Bacik 1774416bc658SJosef Bacik return eb; 1775bb803951SChris Mason } 1776bb803951SChris Mason 1777d352ac68SChris Mason /* 1778d352ac68SChris Mason * node level balancing, used to make sure nodes are in proper order for 1779d352ac68SChris Mason * item deletion. We balance from the top down, so we have to make sure 1780d352ac68SChris Mason * that a deletion won't leave an node completely empty later on. 1781d352ac68SChris Mason */ 1782e02119d5SChris Mason static noinline int balance_level(struct btrfs_trans_handle *trans, 178398ed5174SChris Mason struct btrfs_root *root, 178498ed5174SChris Mason struct btrfs_path *path, int level) 1785bb803951SChris Mason { 17860b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 17875f39d397SChris Mason struct extent_buffer *right = NULL; 17885f39d397SChris Mason struct extent_buffer *mid; 17895f39d397SChris Mason struct extent_buffer *left = NULL; 17905f39d397SChris Mason struct extent_buffer *parent = NULL; 1791bb803951SChris Mason int ret = 0; 1792bb803951SChris Mason int wret; 1793bb803951SChris Mason int pslot; 1794bb803951SChris Mason int orig_slot = path->slots[level]; 179579f95c82SChris Mason u64 orig_ptr; 1796bb803951SChris Mason 179798e6b1ebSLiu Bo ASSERT(level > 0); 1798bb803951SChris Mason 17995f39d397SChris Mason mid = path->nodes[level]; 1800b4ce94deSChris Mason 1801bd681513SChris Mason WARN_ON(path->locks[level] != BTRFS_WRITE_LOCK && 1802bd681513SChris Mason path->locks[level] != BTRFS_WRITE_LOCK_BLOCKING); 18037bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 18047bb86316SChris Mason 18051d4f8a0cSChris Mason orig_ptr = btrfs_node_blockptr(mid, orig_slot); 180679f95c82SChris Mason 1807a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 18085f39d397SChris Mason parent = path->nodes[level + 1]; 1809bb803951SChris Mason pslot = path->slots[level + 1]; 1810a05a9bb1SLi Zefan } 1811bb803951SChris Mason 181240689478SChris Mason /* 181340689478SChris Mason * deal with the case where there is only one pointer in the root 181440689478SChris Mason * by promoting the node below to a root 181540689478SChris Mason */ 18165f39d397SChris Mason if (!parent) { 18175f39d397SChris Mason struct extent_buffer *child; 1818bb803951SChris Mason 18195f39d397SChris Mason if (btrfs_header_nritems(mid) != 1) 1820bb803951SChris Mason return 0; 1821bb803951SChris Mason 1822bb803951SChris Mason /* promote the child to a root */ 18232ff7e61eSJeff Mahoney child = read_node_slot(fs_info, mid, 0); 1824fb770ae4SLiu Bo if (IS_ERR(child)) { 1825fb770ae4SLiu Bo ret = PTR_ERR(child); 18260b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 1827305a26afSMark Fasheh goto enospc; 1828305a26afSMark Fasheh } 1829305a26afSMark Fasheh 1830925baeddSChris Mason btrfs_tree_lock(child); 1831b4ce94deSChris Mason btrfs_set_lock_blocking(child); 18329fa8cfe7SChris Mason ret = btrfs_cow_block(trans, root, child, mid, 0, &child); 1833f0486c68SYan, Zheng if (ret) { 1834f0486c68SYan, Zheng btrfs_tree_unlock(child); 1835f0486c68SYan, Zheng free_extent_buffer(child); 1836f0486c68SYan, Zheng goto enospc; 1837f0486c68SYan, Zheng } 18382f375ab9SYan 1839d9d19a01SDavid Sterba ret = tree_mod_log_insert_root(root->node, child, 1); 1840d9d19a01SDavid Sterba BUG_ON(ret < 0); 1841240f62c8SChris Mason rcu_assign_pointer(root->node, child); 1842925baeddSChris Mason 18430b86a832SChris Mason add_root_to_dirty_list(root); 1844925baeddSChris Mason btrfs_tree_unlock(child); 1845b4ce94deSChris Mason 1846925baeddSChris Mason path->locks[level] = 0; 1847bb803951SChris Mason path->nodes[level] = NULL; 18487c302b49SDavid Sterba clean_tree_block(fs_info, mid); 1849925baeddSChris Mason btrfs_tree_unlock(mid); 1850bb803951SChris Mason /* once for the path */ 18515f39d397SChris Mason free_extent_buffer(mid); 1852f0486c68SYan, Zheng 1853f0486c68SYan, Zheng root_sub_used(root, mid->len); 18545581a51aSJan Schmidt btrfs_free_tree_block(trans, root, mid, 0, 1); 1855bb803951SChris Mason /* once for the root ptr */ 18563083ee2eSJosef Bacik free_extent_buffer_stale(mid); 1857f0486c68SYan, Zheng return 0; 1858bb803951SChris Mason } 18595f39d397SChris Mason if (btrfs_header_nritems(mid) > 18600b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 4) 1861bb803951SChris Mason return 0; 1862bb803951SChris Mason 18632ff7e61eSJeff Mahoney left = read_node_slot(fs_info, parent, pslot - 1); 1864fb770ae4SLiu Bo if (IS_ERR(left)) 1865fb770ae4SLiu Bo left = NULL; 1866fb770ae4SLiu Bo 18675f39d397SChris Mason if (left) { 1868925baeddSChris Mason btrfs_tree_lock(left); 1869b4ce94deSChris Mason btrfs_set_lock_blocking(left); 18705f39d397SChris Mason wret = btrfs_cow_block(trans, root, left, 18719fa8cfe7SChris Mason parent, pslot - 1, &left); 187254aa1f4dSChris Mason if (wret) { 187354aa1f4dSChris Mason ret = wret; 187454aa1f4dSChris Mason goto enospc; 187554aa1f4dSChris Mason } 18762cc58cf2SChris Mason } 1877fb770ae4SLiu Bo 18782ff7e61eSJeff Mahoney right = read_node_slot(fs_info, parent, pslot + 1); 1879fb770ae4SLiu Bo if (IS_ERR(right)) 1880fb770ae4SLiu Bo right = NULL; 1881fb770ae4SLiu Bo 18825f39d397SChris Mason if (right) { 1883925baeddSChris Mason btrfs_tree_lock(right); 1884b4ce94deSChris Mason btrfs_set_lock_blocking(right); 18855f39d397SChris Mason wret = btrfs_cow_block(trans, root, right, 18869fa8cfe7SChris Mason parent, pslot + 1, &right); 18872cc58cf2SChris Mason if (wret) { 18882cc58cf2SChris Mason ret = wret; 18892cc58cf2SChris Mason goto enospc; 18902cc58cf2SChris Mason } 18912cc58cf2SChris Mason } 18922cc58cf2SChris Mason 18932cc58cf2SChris Mason /* first, try to make some room in the middle buffer */ 18945f39d397SChris Mason if (left) { 18955f39d397SChris Mason orig_slot += btrfs_header_nritems(left); 18962ff7e61eSJeff Mahoney wret = push_node_left(trans, fs_info, left, mid, 1); 189779f95c82SChris Mason if (wret < 0) 189879f95c82SChris Mason ret = wret; 1899bb803951SChris Mason } 190079f95c82SChris Mason 190179f95c82SChris Mason /* 190279f95c82SChris Mason * then try to empty the right most buffer into the middle 190379f95c82SChris Mason */ 19045f39d397SChris Mason if (right) { 19052ff7e61eSJeff Mahoney wret = push_node_left(trans, fs_info, mid, right, 1); 190654aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 190779f95c82SChris Mason ret = wret; 19085f39d397SChris Mason if (btrfs_header_nritems(right) == 0) { 19097c302b49SDavid Sterba clean_tree_block(fs_info, right); 1910925baeddSChris Mason btrfs_tree_unlock(right); 1911afe5fea7STsutomu Itoh del_ptr(root, path, level + 1, pslot + 1); 1912f0486c68SYan, Zheng root_sub_used(root, right->len); 19135581a51aSJan Schmidt btrfs_free_tree_block(trans, root, right, 0, 1); 19143083ee2eSJosef Bacik free_extent_buffer_stale(right); 1915f0486c68SYan, Zheng right = NULL; 1916bb803951SChris Mason } else { 19175f39d397SChris Mason struct btrfs_disk_key right_key; 19185f39d397SChris Mason btrfs_node_key(right, &right_key, 0); 19190e82bcfeSDavid Sterba ret = tree_mod_log_insert_key(parent, pslot + 1, 19200e82bcfeSDavid Sterba MOD_LOG_KEY_REPLACE, GFP_NOFS); 19210e82bcfeSDavid Sterba BUG_ON(ret < 0); 19225f39d397SChris Mason btrfs_set_node_key(parent, &right_key, pslot + 1); 19235f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 1924bb803951SChris Mason } 1925bb803951SChris Mason } 19265f39d397SChris Mason if (btrfs_header_nritems(mid) == 1) { 192779f95c82SChris Mason /* 192879f95c82SChris Mason * we're not allowed to leave a node with one item in the 192979f95c82SChris Mason * tree during a delete. A deletion from lower in the tree 193079f95c82SChris Mason * could try to delete the only pointer in this node. 193179f95c82SChris Mason * So, pull some keys from the left. 193279f95c82SChris Mason * There has to be a left pointer at this point because 193379f95c82SChris Mason * otherwise we would have pulled some pointers from the 193479f95c82SChris Mason * right 193579f95c82SChris Mason */ 1936305a26afSMark Fasheh if (!left) { 1937305a26afSMark Fasheh ret = -EROFS; 19380b246afaSJeff Mahoney btrfs_handle_fs_error(fs_info, ret, NULL); 1939305a26afSMark Fasheh goto enospc; 1940305a26afSMark Fasheh } 19412ff7e61eSJeff Mahoney wret = balance_node_right(trans, fs_info, mid, left); 194254aa1f4dSChris Mason if (wret < 0) { 194379f95c82SChris Mason ret = wret; 194454aa1f4dSChris Mason goto enospc; 194554aa1f4dSChris Mason } 1946bce4eae9SChris Mason if (wret == 1) { 19472ff7e61eSJeff Mahoney wret = push_node_left(trans, fs_info, left, mid, 1); 1948bce4eae9SChris Mason if (wret < 0) 1949bce4eae9SChris Mason ret = wret; 1950bce4eae9SChris Mason } 195179f95c82SChris Mason BUG_ON(wret == 1); 195279f95c82SChris Mason } 19535f39d397SChris Mason if (btrfs_header_nritems(mid) == 0) { 19547c302b49SDavid Sterba clean_tree_block(fs_info, mid); 1955925baeddSChris Mason btrfs_tree_unlock(mid); 1956afe5fea7STsutomu Itoh del_ptr(root, path, level + 1, pslot); 1957f0486c68SYan, Zheng root_sub_used(root, mid->len); 19585581a51aSJan Schmidt btrfs_free_tree_block(trans, root, mid, 0, 1); 19593083ee2eSJosef Bacik free_extent_buffer_stale(mid); 1960f0486c68SYan, Zheng mid = NULL; 196179f95c82SChris Mason } else { 196279f95c82SChris Mason /* update the parent key to reflect our changes */ 19635f39d397SChris Mason struct btrfs_disk_key mid_key; 19645f39d397SChris Mason btrfs_node_key(mid, &mid_key, 0); 19650e82bcfeSDavid Sterba ret = tree_mod_log_insert_key(parent, pslot, 19660e82bcfeSDavid Sterba MOD_LOG_KEY_REPLACE, GFP_NOFS); 19670e82bcfeSDavid Sterba BUG_ON(ret < 0); 19685f39d397SChris Mason btrfs_set_node_key(parent, &mid_key, pslot); 19695f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 197079f95c82SChris Mason } 1971bb803951SChris Mason 197279f95c82SChris Mason /* update the path */ 19735f39d397SChris Mason if (left) { 19745f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 19755f39d397SChris Mason extent_buffer_get(left); 1976925baeddSChris Mason /* left was locked after cow */ 19775f39d397SChris Mason path->nodes[level] = left; 1978bb803951SChris Mason path->slots[level + 1] -= 1; 1979bb803951SChris Mason path->slots[level] = orig_slot; 1980925baeddSChris Mason if (mid) { 1981925baeddSChris Mason btrfs_tree_unlock(mid); 19825f39d397SChris Mason free_extent_buffer(mid); 1983925baeddSChris Mason } 1984bb803951SChris Mason } else { 19855f39d397SChris Mason orig_slot -= btrfs_header_nritems(left); 1986bb803951SChris Mason path->slots[level] = orig_slot; 1987bb803951SChris Mason } 1988bb803951SChris Mason } 198979f95c82SChris Mason /* double check we haven't messed things up */ 1990e20d96d6SChris Mason if (orig_ptr != 19915f39d397SChris Mason btrfs_node_blockptr(path->nodes[level], path->slots[level])) 199279f95c82SChris Mason BUG(); 199354aa1f4dSChris Mason enospc: 1994925baeddSChris Mason if (right) { 1995925baeddSChris Mason btrfs_tree_unlock(right); 19965f39d397SChris Mason free_extent_buffer(right); 1997925baeddSChris Mason } 1998925baeddSChris Mason if (left) { 1999925baeddSChris Mason if (path->nodes[level] != left) 2000925baeddSChris Mason btrfs_tree_unlock(left); 20015f39d397SChris Mason free_extent_buffer(left); 2002925baeddSChris Mason } 2003bb803951SChris Mason return ret; 2004bb803951SChris Mason } 2005bb803951SChris Mason 2006d352ac68SChris Mason /* Node balancing for insertion. Here we only split or push nodes around 2007d352ac68SChris Mason * when they are completely full. This is also done top down, so we 2008d352ac68SChris Mason * have to be pessimistic. 2009d352ac68SChris Mason */ 2010d397712bSChris Mason static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans, 2011e66f709bSChris Mason struct btrfs_root *root, 2012e66f709bSChris Mason struct btrfs_path *path, int level) 2013e66f709bSChris Mason { 20140b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 20155f39d397SChris Mason struct extent_buffer *right = NULL; 20165f39d397SChris Mason struct extent_buffer *mid; 20175f39d397SChris Mason struct extent_buffer *left = NULL; 20185f39d397SChris Mason struct extent_buffer *parent = NULL; 2019e66f709bSChris Mason int ret = 0; 2020e66f709bSChris Mason int wret; 2021e66f709bSChris Mason int pslot; 2022e66f709bSChris Mason int orig_slot = path->slots[level]; 2023e66f709bSChris Mason 2024e66f709bSChris Mason if (level == 0) 2025e66f709bSChris Mason return 1; 2026e66f709bSChris Mason 20275f39d397SChris Mason mid = path->nodes[level]; 20287bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 2029e66f709bSChris Mason 2030a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 20315f39d397SChris Mason parent = path->nodes[level + 1]; 2032e66f709bSChris Mason pslot = path->slots[level + 1]; 2033a05a9bb1SLi Zefan } 2034e66f709bSChris Mason 20355f39d397SChris Mason if (!parent) 2036e66f709bSChris Mason return 1; 2037e66f709bSChris Mason 20382ff7e61eSJeff Mahoney left = read_node_slot(fs_info, parent, pslot - 1); 2039fb770ae4SLiu Bo if (IS_ERR(left)) 2040fb770ae4SLiu Bo left = NULL; 2041e66f709bSChris Mason 2042e66f709bSChris Mason /* first, try to make some room in the middle buffer */ 20435f39d397SChris Mason if (left) { 2044e66f709bSChris Mason u32 left_nr; 2045925baeddSChris Mason 2046925baeddSChris Mason btrfs_tree_lock(left); 2047b4ce94deSChris Mason btrfs_set_lock_blocking(left); 2048b4ce94deSChris Mason 20495f39d397SChris Mason left_nr = btrfs_header_nritems(left); 20500b246afaSJeff Mahoney if (left_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) { 205133ade1f8SChris Mason wret = 1; 205233ade1f8SChris Mason } else { 20535f39d397SChris Mason ret = btrfs_cow_block(trans, root, left, parent, 20549fa8cfe7SChris Mason pslot - 1, &left); 205554aa1f4dSChris Mason if (ret) 205654aa1f4dSChris Mason wret = 1; 205754aa1f4dSChris Mason else { 20582ff7e61eSJeff Mahoney wret = push_node_left(trans, fs_info, 2059971a1f66SChris Mason left, mid, 0); 206054aa1f4dSChris Mason } 206133ade1f8SChris Mason } 2062e66f709bSChris Mason if (wret < 0) 2063e66f709bSChris Mason ret = wret; 2064e66f709bSChris Mason if (wret == 0) { 20655f39d397SChris Mason struct btrfs_disk_key disk_key; 2066e66f709bSChris Mason orig_slot += left_nr; 20675f39d397SChris Mason btrfs_node_key(mid, &disk_key, 0); 20680e82bcfeSDavid Sterba ret = tree_mod_log_insert_key(parent, pslot, 20690e82bcfeSDavid Sterba MOD_LOG_KEY_REPLACE, GFP_NOFS); 20700e82bcfeSDavid Sterba BUG_ON(ret < 0); 20715f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot); 20725f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 20735f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 20745f39d397SChris Mason path->nodes[level] = left; 2075e66f709bSChris Mason path->slots[level + 1] -= 1; 2076e66f709bSChris Mason path->slots[level] = orig_slot; 2077925baeddSChris Mason btrfs_tree_unlock(mid); 20785f39d397SChris Mason free_extent_buffer(mid); 2079e66f709bSChris Mason } else { 2080e66f709bSChris Mason orig_slot -= 20815f39d397SChris Mason btrfs_header_nritems(left); 2082e66f709bSChris Mason path->slots[level] = orig_slot; 2083925baeddSChris Mason btrfs_tree_unlock(left); 20845f39d397SChris Mason free_extent_buffer(left); 2085e66f709bSChris Mason } 2086e66f709bSChris Mason return 0; 2087e66f709bSChris Mason } 2088925baeddSChris Mason btrfs_tree_unlock(left); 20895f39d397SChris Mason free_extent_buffer(left); 2090e66f709bSChris Mason } 20912ff7e61eSJeff Mahoney right = read_node_slot(fs_info, parent, pslot + 1); 2092fb770ae4SLiu Bo if (IS_ERR(right)) 2093fb770ae4SLiu Bo right = NULL; 2094e66f709bSChris Mason 2095e66f709bSChris Mason /* 2096e66f709bSChris Mason * then try to empty the right most buffer into the middle 2097e66f709bSChris Mason */ 20985f39d397SChris Mason if (right) { 209933ade1f8SChris Mason u32 right_nr; 2100b4ce94deSChris Mason 2101925baeddSChris Mason btrfs_tree_lock(right); 2102b4ce94deSChris Mason btrfs_set_lock_blocking(right); 2103b4ce94deSChris Mason 21045f39d397SChris Mason right_nr = btrfs_header_nritems(right); 21050b246afaSJeff Mahoney if (right_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) { 210633ade1f8SChris Mason wret = 1; 210733ade1f8SChris Mason } else { 21085f39d397SChris Mason ret = btrfs_cow_block(trans, root, right, 21095f39d397SChris Mason parent, pslot + 1, 21109fa8cfe7SChris Mason &right); 211154aa1f4dSChris Mason if (ret) 211254aa1f4dSChris Mason wret = 1; 211354aa1f4dSChris Mason else { 21142ff7e61eSJeff Mahoney wret = balance_node_right(trans, fs_info, 21155f39d397SChris Mason right, mid); 211633ade1f8SChris Mason } 211754aa1f4dSChris Mason } 2118e66f709bSChris Mason if (wret < 0) 2119e66f709bSChris Mason ret = wret; 2120e66f709bSChris Mason if (wret == 0) { 21215f39d397SChris Mason struct btrfs_disk_key disk_key; 21225f39d397SChris Mason 21235f39d397SChris Mason btrfs_node_key(right, &disk_key, 0); 21240e82bcfeSDavid Sterba ret = tree_mod_log_insert_key(parent, pslot + 1, 21250e82bcfeSDavid Sterba MOD_LOG_KEY_REPLACE, GFP_NOFS); 21260e82bcfeSDavid Sterba BUG_ON(ret < 0); 21275f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot + 1); 21285f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 21295f39d397SChris Mason 21305f39d397SChris Mason if (btrfs_header_nritems(mid) <= orig_slot) { 21315f39d397SChris Mason path->nodes[level] = right; 2132e66f709bSChris Mason path->slots[level + 1] += 1; 2133e66f709bSChris Mason path->slots[level] = orig_slot - 21345f39d397SChris Mason btrfs_header_nritems(mid); 2135925baeddSChris Mason btrfs_tree_unlock(mid); 21365f39d397SChris Mason free_extent_buffer(mid); 2137e66f709bSChris Mason } else { 2138925baeddSChris Mason btrfs_tree_unlock(right); 21395f39d397SChris Mason free_extent_buffer(right); 2140e66f709bSChris Mason } 2141e66f709bSChris Mason return 0; 2142e66f709bSChris Mason } 2143925baeddSChris Mason btrfs_tree_unlock(right); 21445f39d397SChris Mason free_extent_buffer(right); 2145e66f709bSChris Mason } 2146e66f709bSChris Mason return 1; 2147e66f709bSChris Mason } 2148e66f709bSChris Mason 214974123bd7SChris Mason /* 2150d352ac68SChris Mason * readahead one full node of leaves, finding things that are close 2151d352ac68SChris Mason * to the block in 'slot', and triggering ra on them. 21523c69faecSChris Mason */ 21532ff7e61eSJeff Mahoney static void reada_for_search(struct btrfs_fs_info *fs_info, 2154e02119d5SChris Mason struct btrfs_path *path, 215501f46658SChris Mason int level, int slot, u64 objectid) 21563c69faecSChris Mason { 21575f39d397SChris Mason struct extent_buffer *node; 215801f46658SChris Mason struct btrfs_disk_key disk_key; 21593c69faecSChris Mason u32 nritems; 21603c69faecSChris Mason u64 search; 2161a7175319SChris Mason u64 target; 21626b80053dSChris Mason u64 nread = 0; 21635f39d397SChris Mason struct extent_buffer *eb; 21646b80053dSChris Mason u32 nr; 21656b80053dSChris Mason u32 blocksize; 21666b80053dSChris Mason u32 nscan = 0; 2167db94535dSChris Mason 2168a6b6e75eSChris Mason if (level != 1) 21693c69faecSChris Mason return; 21703c69faecSChris Mason 21716702ed49SChris Mason if (!path->nodes[level]) 21726702ed49SChris Mason return; 21736702ed49SChris Mason 21745f39d397SChris Mason node = path->nodes[level]; 2175925baeddSChris Mason 21763c69faecSChris Mason search = btrfs_node_blockptr(node, slot); 21770b246afaSJeff Mahoney blocksize = fs_info->nodesize; 21780b246afaSJeff Mahoney eb = find_extent_buffer(fs_info, search); 21795f39d397SChris Mason if (eb) { 21805f39d397SChris Mason free_extent_buffer(eb); 21813c69faecSChris Mason return; 21823c69faecSChris Mason } 21833c69faecSChris Mason 2184a7175319SChris Mason target = search; 21856b80053dSChris Mason 21865f39d397SChris Mason nritems = btrfs_header_nritems(node); 21876b80053dSChris Mason nr = slot; 218825b8b936SJosef Bacik 21893c69faecSChris Mason while (1) { 2190e4058b54SDavid Sterba if (path->reada == READA_BACK) { 21916b80053dSChris Mason if (nr == 0) 21923c69faecSChris Mason break; 21936b80053dSChris Mason nr--; 2194e4058b54SDavid Sterba } else if (path->reada == READA_FORWARD) { 21956b80053dSChris Mason nr++; 21966b80053dSChris Mason if (nr >= nritems) 21976b80053dSChris Mason break; 21983c69faecSChris Mason } 2199e4058b54SDavid Sterba if (path->reada == READA_BACK && objectid) { 220001f46658SChris Mason btrfs_node_key(node, &disk_key, nr); 220101f46658SChris Mason if (btrfs_disk_key_objectid(&disk_key) != objectid) 220201f46658SChris Mason break; 220301f46658SChris Mason } 22046b80053dSChris Mason search = btrfs_node_blockptr(node, nr); 2205a7175319SChris Mason if ((search <= target && target - search <= 65536) || 2206a7175319SChris Mason (search > target && search - target <= 65536)) { 22072ff7e61eSJeff Mahoney readahead_tree_block(fs_info, search); 22086b80053dSChris Mason nread += blocksize; 22093c69faecSChris Mason } 22106b80053dSChris Mason nscan++; 2211a7175319SChris Mason if ((nread > 65536 || nscan > 32)) 22126b80053dSChris Mason break; 22133c69faecSChris Mason } 22143c69faecSChris Mason } 2215925baeddSChris Mason 22162ff7e61eSJeff Mahoney static noinline void reada_for_balance(struct btrfs_fs_info *fs_info, 2217b4ce94deSChris Mason struct btrfs_path *path, int level) 2218b4ce94deSChris Mason { 2219b4ce94deSChris Mason int slot; 2220b4ce94deSChris Mason int nritems; 2221b4ce94deSChris Mason struct extent_buffer *parent; 2222b4ce94deSChris Mason struct extent_buffer *eb; 2223b4ce94deSChris Mason u64 gen; 2224b4ce94deSChris Mason u64 block1 = 0; 2225b4ce94deSChris Mason u64 block2 = 0; 2226b4ce94deSChris Mason 22278c594ea8SChris Mason parent = path->nodes[level + 1]; 2228b4ce94deSChris Mason if (!parent) 22290b08851fSJosef Bacik return; 2230b4ce94deSChris Mason 2231b4ce94deSChris Mason nritems = btrfs_header_nritems(parent); 22328c594ea8SChris Mason slot = path->slots[level + 1]; 2233b4ce94deSChris Mason 2234b4ce94deSChris Mason if (slot > 0) { 2235b4ce94deSChris Mason block1 = btrfs_node_blockptr(parent, slot - 1); 2236b4ce94deSChris Mason gen = btrfs_node_ptr_generation(parent, slot - 1); 22370b246afaSJeff Mahoney eb = find_extent_buffer(fs_info, block1); 2238b9fab919SChris Mason /* 2239b9fab919SChris Mason * if we get -eagain from btrfs_buffer_uptodate, we 2240b9fab919SChris Mason * don't want to return eagain here. That will loop 2241b9fab919SChris Mason * forever 2242b9fab919SChris Mason */ 2243b9fab919SChris Mason if (eb && btrfs_buffer_uptodate(eb, gen, 1) != 0) 2244b4ce94deSChris Mason block1 = 0; 2245b4ce94deSChris Mason free_extent_buffer(eb); 2246b4ce94deSChris Mason } 22478c594ea8SChris Mason if (slot + 1 < nritems) { 2248b4ce94deSChris Mason block2 = btrfs_node_blockptr(parent, slot + 1); 2249b4ce94deSChris Mason gen = btrfs_node_ptr_generation(parent, slot + 1); 22500b246afaSJeff Mahoney eb = find_extent_buffer(fs_info, block2); 2251b9fab919SChris Mason if (eb && btrfs_buffer_uptodate(eb, gen, 1) != 0) 2252b4ce94deSChris Mason block2 = 0; 2253b4ce94deSChris Mason free_extent_buffer(eb); 2254b4ce94deSChris Mason } 22558c594ea8SChris Mason 2256b4ce94deSChris Mason if (block1) 22572ff7e61eSJeff Mahoney readahead_tree_block(fs_info, block1); 2258b4ce94deSChris Mason if (block2) 22592ff7e61eSJeff Mahoney readahead_tree_block(fs_info, block2); 2260b4ce94deSChris Mason } 2261b4ce94deSChris Mason 2262b4ce94deSChris Mason 2263b4ce94deSChris Mason /* 2264d397712bSChris Mason * when we walk down the tree, it is usually safe to unlock the higher layers 2265d397712bSChris Mason * in the tree. The exceptions are when our path goes through slot 0, because 2266d397712bSChris Mason * operations on the tree might require changing key pointers higher up in the 2267d397712bSChris Mason * tree. 2268d352ac68SChris Mason * 2269d397712bSChris Mason * callers might also have set path->keep_locks, which tells this code to keep 2270d397712bSChris Mason * the lock if the path points to the last slot in the block. This is part of 2271d397712bSChris Mason * walking through the tree, and selecting the next slot in the higher block. 2272d352ac68SChris Mason * 2273d397712bSChris Mason * lowest_unlock sets the lowest level in the tree we're allowed to unlock. so 2274d397712bSChris Mason * if lowest_unlock is 1, level 0 won't be unlocked 2275d352ac68SChris Mason */ 2276e02119d5SChris Mason static noinline void unlock_up(struct btrfs_path *path, int level, 2277f7c79f30SChris Mason int lowest_unlock, int min_write_lock_level, 2278f7c79f30SChris Mason int *write_lock_level) 2279925baeddSChris Mason { 2280925baeddSChris Mason int i; 2281925baeddSChris Mason int skip_level = level; 2282051e1b9fSChris Mason int no_skips = 0; 2283925baeddSChris Mason struct extent_buffer *t; 2284925baeddSChris Mason 2285925baeddSChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 2286925baeddSChris Mason if (!path->nodes[i]) 2287925baeddSChris Mason break; 2288925baeddSChris Mason if (!path->locks[i]) 2289925baeddSChris Mason break; 2290051e1b9fSChris Mason if (!no_skips && path->slots[i] == 0) { 2291925baeddSChris Mason skip_level = i + 1; 2292925baeddSChris Mason continue; 2293925baeddSChris Mason } 2294051e1b9fSChris Mason if (!no_skips && path->keep_locks) { 2295925baeddSChris Mason u32 nritems; 2296925baeddSChris Mason t = path->nodes[i]; 2297925baeddSChris Mason nritems = btrfs_header_nritems(t); 2298051e1b9fSChris Mason if (nritems < 1 || path->slots[i] >= nritems - 1) { 2299925baeddSChris Mason skip_level = i + 1; 2300925baeddSChris Mason continue; 2301925baeddSChris Mason } 2302925baeddSChris Mason } 2303051e1b9fSChris Mason if (skip_level < i && i >= lowest_unlock) 2304051e1b9fSChris Mason no_skips = 1; 2305051e1b9fSChris Mason 2306925baeddSChris Mason t = path->nodes[i]; 2307d80bb3f9SLiu Bo if (i >= lowest_unlock && i > skip_level) { 2308bd681513SChris Mason btrfs_tree_unlock_rw(t, path->locks[i]); 2309925baeddSChris Mason path->locks[i] = 0; 2310f7c79f30SChris Mason if (write_lock_level && 2311f7c79f30SChris Mason i > min_write_lock_level && 2312f7c79f30SChris Mason i <= *write_lock_level) { 2313f7c79f30SChris Mason *write_lock_level = i - 1; 2314f7c79f30SChris Mason } 2315925baeddSChris Mason } 2316925baeddSChris Mason } 2317925baeddSChris Mason } 2318925baeddSChris Mason 23193c69faecSChris Mason /* 2320b4ce94deSChris Mason * This releases any locks held in the path starting at level and 2321b4ce94deSChris Mason * going all the way up to the root. 2322b4ce94deSChris Mason * 2323b4ce94deSChris Mason * btrfs_search_slot will keep the lock held on higher nodes in a few 2324b4ce94deSChris Mason * corner cases, such as COW of the block at slot zero in the node. This 2325b4ce94deSChris Mason * ignores those rules, and it should only be called when there are no 2326b4ce94deSChris Mason * more updates to be done higher up in the tree. 2327b4ce94deSChris Mason */ 2328b4ce94deSChris Mason noinline void btrfs_unlock_up_safe(struct btrfs_path *path, int level) 2329b4ce94deSChris Mason { 2330b4ce94deSChris Mason int i; 2331b4ce94deSChris Mason 233209a2a8f9SJosef Bacik if (path->keep_locks) 2333b4ce94deSChris Mason return; 2334b4ce94deSChris Mason 2335b4ce94deSChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 2336b4ce94deSChris Mason if (!path->nodes[i]) 233712f4daccSChris Mason continue; 2338b4ce94deSChris Mason if (!path->locks[i]) 233912f4daccSChris Mason continue; 2340bd681513SChris Mason btrfs_tree_unlock_rw(path->nodes[i], path->locks[i]); 2341b4ce94deSChris Mason path->locks[i] = 0; 2342b4ce94deSChris Mason } 2343b4ce94deSChris Mason } 2344b4ce94deSChris Mason 2345b4ce94deSChris Mason /* 2346c8c42864SChris Mason * helper function for btrfs_search_slot. The goal is to find a block 2347c8c42864SChris Mason * in cache without setting the path to blocking. If we find the block 2348c8c42864SChris Mason * we return zero and the path is unchanged. 2349c8c42864SChris Mason * 2350c8c42864SChris Mason * If we can't find the block, we set the path blocking and do some 2351c8c42864SChris Mason * reada. -EAGAIN is returned and the search must be repeated. 2352c8c42864SChris Mason */ 2353c8c42864SChris Mason static int 2354d07b8528SLiu Bo read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, 2355c8c42864SChris Mason struct extent_buffer **eb_ret, int level, int slot, 2356cda79c54SDavid Sterba const struct btrfs_key *key) 2357c8c42864SChris Mason { 23580b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 2359c8c42864SChris Mason u64 blocknr; 2360c8c42864SChris Mason u64 gen; 2361c8c42864SChris Mason struct extent_buffer *b = *eb_ret; 2362c8c42864SChris Mason struct extent_buffer *tmp; 2363581c1760SQu Wenruo struct btrfs_key first_key; 236476a05b35SChris Mason int ret; 2365581c1760SQu Wenruo int parent_level; 2366c8c42864SChris Mason 2367c8c42864SChris Mason blocknr = btrfs_node_blockptr(b, slot); 2368c8c42864SChris Mason gen = btrfs_node_ptr_generation(b, slot); 2369581c1760SQu Wenruo parent_level = btrfs_header_level(b); 2370581c1760SQu Wenruo btrfs_node_key_to_cpu(b, &first_key, slot); 2371c8c42864SChris Mason 23720b246afaSJeff Mahoney tmp = find_extent_buffer(fs_info, blocknr); 2373cb44921aSChris Mason if (tmp) { 2374b9fab919SChris Mason /* first we do an atomic uptodate check */ 2375b9fab919SChris Mason if (btrfs_buffer_uptodate(tmp, gen, 1) > 0) { 2376c8c42864SChris Mason *eb_ret = tmp; 2377c8c42864SChris Mason return 0; 2378c8c42864SChris Mason } 2379bdf7c00eSJosef Bacik 2380cb44921aSChris Mason /* the pages were up to date, but we failed 2381cb44921aSChris Mason * the generation number check. Do a full 2382cb44921aSChris Mason * read for the generation number that is correct. 2383cb44921aSChris Mason * We must do this without dropping locks so 2384cb44921aSChris Mason * we can trust our generation number 2385cb44921aSChris Mason */ 2386bd681513SChris Mason btrfs_set_path_blocking(p); 2387bd681513SChris Mason 2388b9fab919SChris Mason /* now we're allowed to do a blocking uptodate check */ 2389581c1760SQu Wenruo ret = btrfs_read_buffer(tmp, gen, parent_level - 1, &first_key); 2390bdf7c00eSJosef Bacik if (!ret) { 2391cb44921aSChris Mason *eb_ret = tmp; 2392cb44921aSChris Mason return 0; 2393cb44921aSChris Mason } 2394cb44921aSChris Mason free_extent_buffer(tmp); 2395b3b4aa74SDavid Sterba btrfs_release_path(p); 2396cb44921aSChris Mason return -EIO; 2397cb44921aSChris Mason } 2398c8c42864SChris Mason 2399c8c42864SChris Mason /* 2400c8c42864SChris Mason * reduce lock contention at high levels 2401c8c42864SChris Mason * of the btree by dropping locks before 240276a05b35SChris Mason * we read. Don't release the lock on the current 240376a05b35SChris Mason * level because we need to walk this node to figure 240476a05b35SChris Mason * out which blocks to read. 2405c8c42864SChris Mason */ 24068c594ea8SChris Mason btrfs_unlock_up_safe(p, level + 1); 24078c594ea8SChris Mason btrfs_set_path_blocking(p); 24088c594ea8SChris Mason 2409e4058b54SDavid Sterba if (p->reada != READA_NONE) 24102ff7e61eSJeff Mahoney reada_for_search(fs_info, p, level, slot, key->objectid); 2411c8c42864SChris Mason 241276a05b35SChris Mason ret = -EAGAIN; 241302a3307aSLiu Bo tmp = read_tree_block(fs_info, blocknr, gen, parent_level - 1, 2414581c1760SQu Wenruo &first_key); 241564c043deSLiu Bo if (!IS_ERR(tmp)) { 241676a05b35SChris Mason /* 241776a05b35SChris Mason * If the read above didn't mark this buffer up to date, 241876a05b35SChris Mason * it will never end up being up to date. Set ret to EIO now 241976a05b35SChris Mason * and give up so that our caller doesn't loop forever 242076a05b35SChris Mason * on our EAGAINs. 242176a05b35SChris Mason */ 2422e6a1d6fdSLiu Bo if (!extent_buffer_uptodate(tmp)) 242376a05b35SChris Mason ret = -EIO; 2424c8c42864SChris Mason free_extent_buffer(tmp); 2425c871b0f2SLiu Bo } else { 2426c871b0f2SLiu Bo ret = PTR_ERR(tmp); 242776a05b35SChris Mason } 242802a3307aSLiu Bo 242902a3307aSLiu Bo btrfs_release_path(p); 243076a05b35SChris Mason return ret; 2431c8c42864SChris Mason } 2432c8c42864SChris Mason 2433c8c42864SChris Mason /* 2434c8c42864SChris Mason * helper function for btrfs_search_slot. This does all of the checks 2435c8c42864SChris Mason * for node-level blocks and does any balancing required based on 2436c8c42864SChris Mason * the ins_len. 2437c8c42864SChris Mason * 2438c8c42864SChris Mason * If no extra work was required, zero is returned. If we had to 2439c8c42864SChris Mason * drop the path, -EAGAIN is returned and btrfs_search_slot must 2440c8c42864SChris Mason * start over 2441c8c42864SChris Mason */ 2442c8c42864SChris Mason static int 2443c8c42864SChris Mason setup_nodes_for_search(struct btrfs_trans_handle *trans, 2444c8c42864SChris Mason struct btrfs_root *root, struct btrfs_path *p, 2445bd681513SChris Mason struct extent_buffer *b, int level, int ins_len, 2446bd681513SChris Mason int *write_lock_level) 2447c8c42864SChris Mason { 24480b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 2449c8c42864SChris Mason int ret; 24500b246afaSJeff Mahoney 2451c8c42864SChris Mason if ((p->search_for_split || ins_len > 0) && btrfs_header_nritems(b) >= 24520b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3) { 2453c8c42864SChris Mason int sret; 2454c8c42864SChris Mason 2455bd681513SChris Mason if (*write_lock_level < level + 1) { 2456bd681513SChris Mason *write_lock_level = level + 1; 2457bd681513SChris Mason btrfs_release_path(p); 2458bd681513SChris Mason goto again; 2459bd681513SChris Mason } 2460bd681513SChris Mason 2461c8c42864SChris Mason btrfs_set_path_blocking(p); 24622ff7e61eSJeff Mahoney reada_for_balance(fs_info, p, level); 2463c8c42864SChris Mason sret = split_node(trans, root, p, level); 2464c8c42864SChris Mason 2465c8c42864SChris Mason BUG_ON(sret > 0); 2466c8c42864SChris Mason if (sret) { 2467c8c42864SChris Mason ret = sret; 2468c8c42864SChris Mason goto done; 2469c8c42864SChris Mason } 2470c8c42864SChris Mason b = p->nodes[level]; 2471c8c42864SChris Mason } else if (ins_len < 0 && btrfs_header_nritems(b) < 24720b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 2) { 2473c8c42864SChris Mason int sret; 2474c8c42864SChris Mason 2475bd681513SChris Mason if (*write_lock_level < level + 1) { 2476bd681513SChris Mason *write_lock_level = level + 1; 2477bd681513SChris Mason btrfs_release_path(p); 2478bd681513SChris Mason goto again; 2479bd681513SChris Mason } 2480bd681513SChris Mason 2481c8c42864SChris Mason btrfs_set_path_blocking(p); 24822ff7e61eSJeff Mahoney reada_for_balance(fs_info, p, level); 2483c8c42864SChris Mason sret = balance_level(trans, root, p, level); 2484c8c42864SChris Mason 2485c8c42864SChris Mason if (sret) { 2486c8c42864SChris Mason ret = sret; 2487c8c42864SChris Mason goto done; 2488c8c42864SChris Mason } 2489c8c42864SChris Mason b = p->nodes[level]; 2490c8c42864SChris Mason if (!b) { 2491b3b4aa74SDavid Sterba btrfs_release_path(p); 2492c8c42864SChris Mason goto again; 2493c8c42864SChris Mason } 2494c8c42864SChris Mason BUG_ON(btrfs_header_nritems(b) == 1); 2495c8c42864SChris Mason } 2496c8c42864SChris Mason return 0; 2497c8c42864SChris Mason 2498c8c42864SChris Mason again: 2499c8c42864SChris Mason ret = -EAGAIN; 2500c8c42864SChris Mason done: 2501c8c42864SChris Mason return ret; 2502c8c42864SChris Mason } 2503c8c42864SChris Mason 2504d7396f07SFilipe David Borba Manana static void key_search_validate(struct extent_buffer *b, 2505310712b2SOmar Sandoval const struct btrfs_key *key, 2506d7396f07SFilipe David Borba Manana int level) 2507d7396f07SFilipe David Borba Manana { 2508d7396f07SFilipe David Borba Manana #ifdef CONFIG_BTRFS_ASSERT 2509d7396f07SFilipe David Borba Manana struct btrfs_disk_key disk_key; 2510d7396f07SFilipe David Borba Manana 2511d7396f07SFilipe David Borba Manana btrfs_cpu_key_to_disk(&disk_key, key); 2512d7396f07SFilipe David Borba Manana 2513d7396f07SFilipe David Borba Manana if (level == 0) 2514d7396f07SFilipe David Borba Manana ASSERT(!memcmp_extent_buffer(b, &disk_key, 2515d7396f07SFilipe David Borba Manana offsetof(struct btrfs_leaf, items[0].key), 2516d7396f07SFilipe David Borba Manana sizeof(disk_key))); 2517d7396f07SFilipe David Borba Manana else 2518d7396f07SFilipe David Borba Manana ASSERT(!memcmp_extent_buffer(b, &disk_key, 2519d7396f07SFilipe David Borba Manana offsetof(struct btrfs_node, ptrs[0].key), 2520d7396f07SFilipe David Borba Manana sizeof(disk_key))); 2521d7396f07SFilipe David Borba Manana #endif 2522d7396f07SFilipe David Borba Manana } 2523d7396f07SFilipe David Borba Manana 2524310712b2SOmar Sandoval static int key_search(struct extent_buffer *b, const struct btrfs_key *key, 2525d7396f07SFilipe David Borba Manana int level, int *prev_cmp, int *slot) 2526d7396f07SFilipe David Borba Manana { 2527d7396f07SFilipe David Borba Manana if (*prev_cmp != 0) { 2528a74b35ecSNikolay Borisov *prev_cmp = btrfs_bin_search(b, key, level, slot); 2529d7396f07SFilipe David Borba Manana return *prev_cmp; 2530d7396f07SFilipe David Borba Manana } 2531d7396f07SFilipe David Borba Manana 2532d7396f07SFilipe David Borba Manana key_search_validate(b, key, level); 2533d7396f07SFilipe David Borba Manana *slot = 0; 2534d7396f07SFilipe David Borba Manana 2535d7396f07SFilipe David Borba Manana return 0; 2536d7396f07SFilipe David Borba Manana } 2537d7396f07SFilipe David Borba Manana 2538381cf658SDavid Sterba int btrfs_find_item(struct btrfs_root *fs_root, struct btrfs_path *path, 2539e33d5c3dSKelley Nielsen u64 iobjectid, u64 ioff, u8 key_type, 2540e33d5c3dSKelley Nielsen struct btrfs_key *found_key) 2541e33d5c3dSKelley Nielsen { 2542e33d5c3dSKelley Nielsen int ret; 2543e33d5c3dSKelley Nielsen struct btrfs_key key; 2544e33d5c3dSKelley Nielsen struct extent_buffer *eb; 2545381cf658SDavid Sterba 2546381cf658SDavid Sterba ASSERT(path); 25471d4c08e0SDavid Sterba ASSERT(found_key); 2548e33d5c3dSKelley Nielsen 2549e33d5c3dSKelley Nielsen key.type = key_type; 2550e33d5c3dSKelley Nielsen key.objectid = iobjectid; 2551e33d5c3dSKelley Nielsen key.offset = ioff; 2552e33d5c3dSKelley Nielsen 2553e33d5c3dSKelley Nielsen ret = btrfs_search_slot(NULL, fs_root, &key, path, 0, 0); 25541d4c08e0SDavid Sterba if (ret < 0) 2555e33d5c3dSKelley Nielsen return ret; 2556e33d5c3dSKelley Nielsen 2557e33d5c3dSKelley Nielsen eb = path->nodes[0]; 2558e33d5c3dSKelley Nielsen if (ret && path->slots[0] >= btrfs_header_nritems(eb)) { 2559e33d5c3dSKelley Nielsen ret = btrfs_next_leaf(fs_root, path); 2560e33d5c3dSKelley Nielsen if (ret) 2561e33d5c3dSKelley Nielsen return ret; 2562e33d5c3dSKelley Nielsen eb = path->nodes[0]; 2563e33d5c3dSKelley Nielsen } 2564e33d5c3dSKelley Nielsen 2565e33d5c3dSKelley Nielsen btrfs_item_key_to_cpu(eb, found_key, path->slots[0]); 2566e33d5c3dSKelley Nielsen if (found_key->type != key.type || 2567e33d5c3dSKelley Nielsen found_key->objectid != key.objectid) 2568e33d5c3dSKelley Nielsen return 1; 2569e33d5c3dSKelley Nielsen 2570e33d5c3dSKelley Nielsen return 0; 2571e33d5c3dSKelley Nielsen } 2572e33d5c3dSKelley Nielsen 25731fc28d8eSLiu Bo static struct extent_buffer *btrfs_search_slot_get_root(struct btrfs_root *root, 25741fc28d8eSLiu Bo struct btrfs_path *p, 25751fc28d8eSLiu Bo int write_lock_level) 25761fc28d8eSLiu Bo { 25771fc28d8eSLiu Bo struct btrfs_fs_info *fs_info = root->fs_info; 25781fc28d8eSLiu Bo struct extent_buffer *b; 25791fc28d8eSLiu Bo int root_lock; 25801fc28d8eSLiu Bo int level = 0; 25811fc28d8eSLiu Bo 25821fc28d8eSLiu Bo /* We try very hard to do read locks on the root */ 25831fc28d8eSLiu Bo root_lock = BTRFS_READ_LOCK; 25841fc28d8eSLiu Bo 25851fc28d8eSLiu Bo if (p->search_commit_root) { 25861fc28d8eSLiu Bo /* The commit roots are read only so we always do read locks */ 25871fc28d8eSLiu Bo if (p->need_commit_sem) 25881fc28d8eSLiu Bo down_read(&fs_info->commit_root_sem); 25891fc28d8eSLiu Bo b = root->commit_root; 25901fc28d8eSLiu Bo extent_buffer_get(b); 25911fc28d8eSLiu Bo level = btrfs_header_level(b); 25921fc28d8eSLiu Bo if (p->need_commit_sem) 25931fc28d8eSLiu Bo up_read(&fs_info->commit_root_sem); 2594f9ddfd05SLiu Bo /* 2595f9ddfd05SLiu Bo * Ensure that all callers have set skip_locking when 2596f9ddfd05SLiu Bo * p->search_commit_root = 1. 2597f9ddfd05SLiu Bo */ 2598f9ddfd05SLiu Bo ASSERT(p->skip_locking == 1); 25991fc28d8eSLiu Bo 26001fc28d8eSLiu Bo goto out; 26011fc28d8eSLiu Bo } 26021fc28d8eSLiu Bo 26031fc28d8eSLiu Bo if (p->skip_locking) { 26041fc28d8eSLiu Bo b = btrfs_root_node(root); 26051fc28d8eSLiu Bo level = btrfs_header_level(b); 26061fc28d8eSLiu Bo goto out; 26071fc28d8eSLiu Bo } 26081fc28d8eSLiu Bo 26091fc28d8eSLiu Bo /* 2610662c653bSLiu Bo * If the level is set to maximum, we can skip trying to get the read 2611662c653bSLiu Bo * lock. 2612662c653bSLiu Bo */ 2613662c653bSLiu Bo if (write_lock_level < BTRFS_MAX_LEVEL) { 2614662c653bSLiu Bo /* 2615662c653bSLiu Bo * We don't know the level of the root node until we actually 2616662c653bSLiu Bo * have it read locked 26171fc28d8eSLiu Bo */ 26181fc28d8eSLiu Bo b = btrfs_read_lock_root_node(root); 26191fc28d8eSLiu Bo level = btrfs_header_level(b); 26201fc28d8eSLiu Bo if (level > write_lock_level) 26211fc28d8eSLiu Bo goto out; 26221fc28d8eSLiu Bo 2623662c653bSLiu Bo /* Whoops, must trade for write lock */ 26241fc28d8eSLiu Bo btrfs_tree_read_unlock(b); 26251fc28d8eSLiu Bo free_extent_buffer(b); 2626662c653bSLiu Bo } 2627662c653bSLiu Bo 26281fc28d8eSLiu Bo b = btrfs_lock_root_node(root); 26291fc28d8eSLiu Bo root_lock = BTRFS_WRITE_LOCK; 26301fc28d8eSLiu Bo 26311fc28d8eSLiu Bo /* The level might have changed, check again */ 26321fc28d8eSLiu Bo level = btrfs_header_level(b); 26331fc28d8eSLiu Bo 26341fc28d8eSLiu Bo out: 26351fc28d8eSLiu Bo p->nodes[level] = b; 26361fc28d8eSLiu Bo if (!p->skip_locking) 26371fc28d8eSLiu Bo p->locks[level] = root_lock; 26381fc28d8eSLiu Bo /* 26391fc28d8eSLiu Bo * Callers are responsible for dropping b's references. 26401fc28d8eSLiu Bo */ 26411fc28d8eSLiu Bo return b; 26421fc28d8eSLiu Bo } 26431fc28d8eSLiu Bo 26441fc28d8eSLiu Bo 2645c8c42864SChris Mason /* 26464271eceaSNikolay Borisov * btrfs_search_slot - look for a key in a tree and perform necessary 26474271eceaSNikolay Borisov * modifications to preserve tree invariants. 264874123bd7SChris Mason * 26494271eceaSNikolay Borisov * @trans: Handle of transaction, used when modifying the tree 26504271eceaSNikolay Borisov * @p: Holds all btree nodes along the search path 26514271eceaSNikolay Borisov * @root: The root node of the tree 26524271eceaSNikolay Borisov * @key: The key we are looking for 26534271eceaSNikolay Borisov * @ins_len: Indicates purpose of search, for inserts it is 1, for 26544271eceaSNikolay Borisov * deletions it's -1. 0 for plain searches 26554271eceaSNikolay Borisov * @cow: boolean should CoW operations be performed. Must always be 1 26564271eceaSNikolay Borisov * when modifying the tree. 265797571fd0SChris Mason * 26584271eceaSNikolay Borisov * If @ins_len > 0, nodes and leaves will be split as we walk down the tree. 26594271eceaSNikolay Borisov * If @ins_len < 0, nodes will be merged as we walk down the tree (if possible) 26604271eceaSNikolay Borisov * 26614271eceaSNikolay Borisov * If @key is found, 0 is returned and you can find the item in the leaf level 26624271eceaSNikolay Borisov * of the path (level 0) 26634271eceaSNikolay Borisov * 26644271eceaSNikolay Borisov * If @key isn't found, 1 is returned and the leaf level of the path (level 0) 26654271eceaSNikolay Borisov * points to the slot where it should be inserted 26664271eceaSNikolay Borisov * 26674271eceaSNikolay Borisov * If an error is encountered while searching the tree a negative error number 26684271eceaSNikolay Borisov * is returned 266974123bd7SChris Mason */ 2670310712b2SOmar Sandoval int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root, 2671310712b2SOmar Sandoval const struct btrfs_key *key, struct btrfs_path *p, 2672310712b2SOmar Sandoval int ins_len, int cow) 2673be0e5c09SChris Mason { 26740b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 26755f39d397SChris Mason struct extent_buffer *b; 2676be0e5c09SChris Mason int slot; 2677be0e5c09SChris Mason int ret; 267833c66f43SYan Zheng int err; 2679be0e5c09SChris Mason int level; 2680925baeddSChris Mason int lowest_unlock = 1; 2681bd681513SChris Mason /* everything at write_lock_level or lower must be write locked */ 2682bd681513SChris Mason int write_lock_level = 0; 26839f3a7427SChris Mason u8 lowest_level = 0; 2684f7c79f30SChris Mason int min_write_lock_level; 2685d7396f07SFilipe David Borba Manana int prev_cmp; 26869f3a7427SChris Mason 26876702ed49SChris Mason lowest_level = p->lowest_level; 2688323ac95bSChris Mason WARN_ON(lowest_level && ins_len > 0); 268922b0ebdaSChris Mason WARN_ON(p->nodes[0] != NULL); 2690eb653de1SFilipe David Borba Manana BUG_ON(!cow && ins_len); 269125179201SJosef Bacik 2692bd681513SChris Mason if (ins_len < 0) { 2693925baeddSChris Mason lowest_unlock = 2; 269465b51a00SChris Mason 2695bd681513SChris Mason /* when we are removing items, we might have to go up to level 2696bd681513SChris Mason * two as we update tree pointers Make sure we keep write 2697bd681513SChris Mason * for those levels as well 2698bd681513SChris Mason */ 2699bd681513SChris Mason write_lock_level = 2; 2700bd681513SChris Mason } else if (ins_len > 0) { 2701bd681513SChris Mason /* 2702bd681513SChris Mason * for inserting items, make sure we have a write lock on 2703bd681513SChris Mason * level 1 so we can update keys 2704bd681513SChris Mason */ 2705bd681513SChris Mason write_lock_level = 1; 2706bd681513SChris Mason } 2707bd681513SChris Mason 2708bd681513SChris Mason if (!cow) 2709bd681513SChris Mason write_lock_level = -1; 2710bd681513SChris Mason 271109a2a8f9SJosef Bacik if (cow && (p->keep_locks || p->lowest_level)) 2712bd681513SChris Mason write_lock_level = BTRFS_MAX_LEVEL; 2713bd681513SChris Mason 2714f7c79f30SChris Mason min_write_lock_level = write_lock_level; 2715f7c79f30SChris Mason 2716bb803951SChris Mason again: 2717d7396f07SFilipe David Borba Manana prev_cmp = -1; 27181fc28d8eSLiu Bo b = btrfs_search_slot_get_root(root, p, write_lock_level); 2719925baeddSChris Mason 2720eb60ceacSChris Mason while (b) { 27215f39d397SChris Mason level = btrfs_header_level(b); 272265b51a00SChris Mason 272365b51a00SChris Mason /* 272465b51a00SChris Mason * setup the path here so we can release it under lock 272565b51a00SChris Mason * contention with the cow code 272665b51a00SChris Mason */ 272702217ed2SChris Mason if (cow) { 27289ea2c7c9SNikolay Borisov bool last_level = (level == (BTRFS_MAX_LEVEL - 1)); 27299ea2c7c9SNikolay Borisov 2730c8c42864SChris Mason /* 2731c8c42864SChris Mason * if we don't really need to cow this block 2732c8c42864SChris Mason * then we don't want to set the path blocking, 2733c8c42864SChris Mason * so we test it here 2734c8c42864SChris Mason */ 273564c12921SJeff Mahoney if (!should_cow_block(trans, root, b)) { 273664c12921SJeff Mahoney trans->dirty = true; 273765b51a00SChris Mason goto cow_done; 273864c12921SJeff Mahoney } 27395d4f98a2SYan Zheng 2740bd681513SChris Mason /* 2741bd681513SChris Mason * must have write locks on this node and the 2742bd681513SChris Mason * parent 2743bd681513SChris Mason */ 27445124e00eSJosef Bacik if (level > write_lock_level || 27455124e00eSJosef Bacik (level + 1 > write_lock_level && 27465124e00eSJosef Bacik level + 1 < BTRFS_MAX_LEVEL && 27475124e00eSJosef Bacik p->nodes[level + 1])) { 2748bd681513SChris Mason write_lock_level = level + 1; 2749bd681513SChris Mason btrfs_release_path(p); 2750bd681513SChris Mason goto again; 2751bd681513SChris Mason } 2752bd681513SChris Mason 2753160f4089SFilipe Manana btrfs_set_path_blocking(p); 27549ea2c7c9SNikolay Borisov if (last_level) 27559ea2c7c9SNikolay Borisov err = btrfs_cow_block(trans, root, b, NULL, 0, 27569ea2c7c9SNikolay Borisov &b); 27579ea2c7c9SNikolay Borisov else 275833c66f43SYan Zheng err = btrfs_cow_block(trans, root, b, 2759e20d96d6SChris Mason p->nodes[level + 1], 27609fa8cfe7SChris Mason p->slots[level + 1], &b); 276133c66f43SYan Zheng if (err) { 276233c66f43SYan Zheng ret = err; 276365b51a00SChris Mason goto done; 276454aa1f4dSChris Mason } 276502217ed2SChris Mason } 276665b51a00SChris Mason cow_done: 2767eb60ceacSChris Mason p->nodes[level] = b; 276852398340SLiu Bo /* 276952398340SLiu Bo * Leave path with blocking locks to avoid massive 277052398340SLiu Bo * lock context switch, this is made on purpose. 277152398340SLiu Bo */ 2772b4ce94deSChris Mason 2773b4ce94deSChris Mason /* 2774b4ce94deSChris Mason * we have a lock on b and as long as we aren't changing 2775b4ce94deSChris Mason * the tree, there is no way to for the items in b to change. 2776b4ce94deSChris Mason * It is safe to drop the lock on our parent before we 2777b4ce94deSChris Mason * go through the expensive btree search on b. 2778b4ce94deSChris Mason * 2779eb653de1SFilipe David Borba Manana * If we're inserting or deleting (ins_len != 0), then we might 2780eb653de1SFilipe David Borba Manana * be changing slot zero, which may require changing the parent. 2781eb653de1SFilipe David Borba Manana * So, we can't drop the lock until after we know which slot 2782eb653de1SFilipe David Borba Manana * we're operating on. 2783b4ce94deSChris Mason */ 2784eb653de1SFilipe David Borba Manana if (!ins_len && !p->keep_locks) { 2785eb653de1SFilipe David Borba Manana int u = level + 1; 2786eb653de1SFilipe David Borba Manana 2787eb653de1SFilipe David Borba Manana if (u < BTRFS_MAX_LEVEL && p->locks[u]) { 2788eb653de1SFilipe David Borba Manana btrfs_tree_unlock_rw(p->nodes[u], p->locks[u]); 2789eb653de1SFilipe David Borba Manana p->locks[u] = 0; 2790eb653de1SFilipe David Borba Manana } 2791eb653de1SFilipe David Borba Manana } 2792b4ce94deSChris Mason 2793d7396f07SFilipe David Borba Manana ret = key_search(b, key, level, &prev_cmp, &slot); 2794415b35a5SLiu Bo if (ret < 0) 2795415b35a5SLiu Bo goto done; 2796b4ce94deSChris Mason 27975f39d397SChris Mason if (level != 0) { 279833c66f43SYan Zheng int dec = 0; 279933c66f43SYan Zheng if (ret && slot > 0) { 280033c66f43SYan Zheng dec = 1; 2801be0e5c09SChris Mason slot -= 1; 280233c66f43SYan Zheng } 2803be0e5c09SChris Mason p->slots[level] = slot; 280433c66f43SYan Zheng err = setup_nodes_for_search(trans, root, p, b, level, 2805bd681513SChris Mason ins_len, &write_lock_level); 280633c66f43SYan Zheng if (err == -EAGAIN) 2807b4ce94deSChris Mason goto again; 280833c66f43SYan Zheng if (err) { 280933c66f43SYan Zheng ret = err; 281065b51a00SChris Mason goto done; 281133c66f43SYan Zheng } 28125c680ed6SChris Mason b = p->nodes[level]; 28135c680ed6SChris Mason slot = p->slots[level]; 2814b4ce94deSChris Mason 2815bd681513SChris Mason /* 2816bd681513SChris Mason * slot 0 is special, if we change the key 2817bd681513SChris Mason * we have to update the parent pointer 2818bd681513SChris Mason * which means we must have a write lock 2819bd681513SChris Mason * on the parent 2820bd681513SChris Mason */ 2821eb653de1SFilipe David Borba Manana if (slot == 0 && ins_len && 2822bd681513SChris Mason write_lock_level < level + 1) { 2823bd681513SChris Mason write_lock_level = level + 1; 2824bd681513SChris Mason btrfs_release_path(p); 2825bd681513SChris Mason goto again; 2826bd681513SChris Mason } 2827bd681513SChris Mason 2828f7c79f30SChris Mason unlock_up(p, level, lowest_unlock, 2829f7c79f30SChris Mason min_write_lock_level, &write_lock_level); 2830f9efa9c7SChris Mason 2831925baeddSChris Mason if (level == lowest_level) { 283233c66f43SYan Zheng if (dec) 283333c66f43SYan Zheng p->slots[level]++; 28345b21f2edSZheng Yan goto done; 2835925baeddSChris Mason } 2836ca7a79adSChris Mason 2837d07b8528SLiu Bo err = read_block_for_search(root, p, &b, level, 2838cda79c54SDavid Sterba slot, key); 283933c66f43SYan Zheng if (err == -EAGAIN) 2840051e1b9fSChris Mason goto again; 284133c66f43SYan Zheng if (err) { 284233c66f43SYan Zheng ret = err; 284376a05b35SChris Mason goto done; 284433c66f43SYan Zheng } 284576a05b35SChris Mason 2846b4ce94deSChris Mason if (!p->skip_locking) { 2847bd681513SChris Mason level = btrfs_header_level(b); 2848bd681513SChris Mason if (level <= write_lock_level) { 2849bd681513SChris Mason err = btrfs_try_tree_write_lock(b); 285033c66f43SYan Zheng if (!err) { 2851b4ce94deSChris Mason btrfs_set_path_blocking(p); 2852925baeddSChris Mason btrfs_tree_lock(b); 2853b4ce94deSChris Mason } 2854bd681513SChris Mason p->locks[level] = BTRFS_WRITE_LOCK; 2855bd681513SChris Mason } else { 2856f82c458aSChris Mason err = btrfs_tree_read_lock_atomic(b); 2857bd681513SChris Mason if (!err) { 2858bd681513SChris Mason btrfs_set_path_blocking(p); 2859bd681513SChris Mason btrfs_tree_read_lock(b); 2860bd681513SChris Mason } 2861bd681513SChris Mason p->locks[level] = BTRFS_READ_LOCK; 2862bd681513SChris Mason } 2863bd681513SChris Mason p->nodes[level] = b; 2864b4ce94deSChris Mason } 2865be0e5c09SChris Mason } else { 2866be0e5c09SChris Mason p->slots[level] = slot; 286787b29b20SYan Zheng if (ins_len > 0 && 28682ff7e61eSJeff Mahoney btrfs_leaf_free_space(fs_info, b) < ins_len) { 2869bd681513SChris Mason if (write_lock_level < 1) { 2870bd681513SChris Mason write_lock_level = 1; 2871bd681513SChris Mason btrfs_release_path(p); 2872bd681513SChris Mason goto again; 2873bd681513SChris Mason } 2874bd681513SChris Mason 2875b4ce94deSChris Mason btrfs_set_path_blocking(p); 287633c66f43SYan Zheng err = split_leaf(trans, root, key, 2877cc0c5538SChris Mason p, ins_len, ret == 0); 2878b4ce94deSChris Mason 287933c66f43SYan Zheng BUG_ON(err > 0); 288033c66f43SYan Zheng if (err) { 288133c66f43SYan Zheng ret = err; 288265b51a00SChris Mason goto done; 288365b51a00SChris Mason } 28845c680ed6SChris Mason } 2885459931ecSChris Mason if (!p->search_for_split) 2886f7c79f30SChris Mason unlock_up(p, level, lowest_unlock, 28874b6f8e96SLiu Bo min_write_lock_level, NULL); 288865b51a00SChris Mason goto done; 288965b51a00SChris Mason } 289065b51a00SChris Mason } 289165b51a00SChris Mason ret = 1; 289265b51a00SChris Mason done: 2893b4ce94deSChris Mason /* 2894b4ce94deSChris Mason * we don't really know what they plan on doing with the path 2895b4ce94deSChris Mason * from here on, so for now just mark it as blocking 2896b4ce94deSChris Mason */ 2897b9473439SChris Mason if (!p->leave_spinning) 2898b4ce94deSChris Mason btrfs_set_path_blocking(p); 28995f5bc6b1SFilipe Manana if (ret < 0 && !p->skip_release_on_error) 2900b3b4aa74SDavid Sterba btrfs_release_path(p); 2901be0e5c09SChris Mason return ret; 2902be0e5c09SChris Mason } 2903be0e5c09SChris Mason 290474123bd7SChris Mason /* 29055d9e75c4SJan Schmidt * Like btrfs_search_slot, this looks for a key in the given tree. It uses the 29065d9e75c4SJan Schmidt * current state of the tree together with the operations recorded in the tree 29075d9e75c4SJan Schmidt * modification log to search for the key in a previous version of this tree, as 29085d9e75c4SJan Schmidt * denoted by the time_seq parameter. 29095d9e75c4SJan Schmidt * 29105d9e75c4SJan Schmidt * Naturally, there is no support for insert, delete or cow operations. 29115d9e75c4SJan Schmidt * 29125d9e75c4SJan Schmidt * The resulting path and return value will be set up as if we called 29135d9e75c4SJan Schmidt * btrfs_search_slot at that point in time with ins_len and cow both set to 0. 29145d9e75c4SJan Schmidt */ 2915310712b2SOmar Sandoval int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key, 29165d9e75c4SJan Schmidt struct btrfs_path *p, u64 time_seq) 29175d9e75c4SJan Schmidt { 29180b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 29195d9e75c4SJan Schmidt struct extent_buffer *b; 29205d9e75c4SJan Schmidt int slot; 29215d9e75c4SJan Schmidt int ret; 29225d9e75c4SJan Schmidt int err; 29235d9e75c4SJan Schmidt int level; 29245d9e75c4SJan Schmidt int lowest_unlock = 1; 29255d9e75c4SJan Schmidt u8 lowest_level = 0; 2926d4b4087cSJosef Bacik int prev_cmp = -1; 29275d9e75c4SJan Schmidt 29285d9e75c4SJan Schmidt lowest_level = p->lowest_level; 29295d9e75c4SJan Schmidt WARN_ON(p->nodes[0] != NULL); 29305d9e75c4SJan Schmidt 29315d9e75c4SJan Schmidt if (p->search_commit_root) { 29325d9e75c4SJan Schmidt BUG_ON(time_seq); 29335d9e75c4SJan Schmidt return btrfs_search_slot(NULL, root, key, p, 0, 0); 29345d9e75c4SJan Schmidt } 29355d9e75c4SJan Schmidt 29365d9e75c4SJan Schmidt again: 29375d9e75c4SJan Schmidt b = get_old_root(root, time_seq); 2938315bed43SNikolay Borisov if (!b) { 2939315bed43SNikolay Borisov ret = -EIO; 2940315bed43SNikolay Borisov goto done; 2941315bed43SNikolay Borisov } 29425d9e75c4SJan Schmidt level = btrfs_header_level(b); 29435d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 29445d9e75c4SJan Schmidt 29455d9e75c4SJan Schmidt while (b) { 29465d9e75c4SJan Schmidt level = btrfs_header_level(b); 29475d9e75c4SJan Schmidt p->nodes[level] = b; 29485d9e75c4SJan Schmidt 29495d9e75c4SJan Schmidt /* 29505d9e75c4SJan Schmidt * we have a lock on b and as long as we aren't changing 29515d9e75c4SJan Schmidt * the tree, there is no way to for the items in b to change. 29525d9e75c4SJan Schmidt * It is safe to drop the lock on our parent before we 29535d9e75c4SJan Schmidt * go through the expensive btree search on b. 29545d9e75c4SJan Schmidt */ 29555d9e75c4SJan Schmidt btrfs_unlock_up_safe(p, level + 1); 29565d9e75c4SJan Schmidt 2957d4b4087cSJosef Bacik /* 295801327610SNicholas D Steeves * Since we can unwind ebs we want to do a real search every 2959d4b4087cSJosef Bacik * time. 2960d4b4087cSJosef Bacik */ 2961d4b4087cSJosef Bacik prev_cmp = -1; 2962d7396f07SFilipe David Borba Manana ret = key_search(b, key, level, &prev_cmp, &slot); 29635d9e75c4SJan Schmidt 29645d9e75c4SJan Schmidt if (level != 0) { 29655d9e75c4SJan Schmidt int dec = 0; 29665d9e75c4SJan Schmidt if (ret && slot > 0) { 29675d9e75c4SJan Schmidt dec = 1; 29685d9e75c4SJan Schmidt slot -= 1; 29695d9e75c4SJan Schmidt } 29705d9e75c4SJan Schmidt p->slots[level] = slot; 29715d9e75c4SJan Schmidt unlock_up(p, level, lowest_unlock, 0, NULL); 29725d9e75c4SJan Schmidt 29735d9e75c4SJan Schmidt if (level == lowest_level) { 29745d9e75c4SJan Schmidt if (dec) 29755d9e75c4SJan Schmidt p->slots[level]++; 29765d9e75c4SJan Schmidt goto done; 29775d9e75c4SJan Schmidt } 29785d9e75c4SJan Schmidt 2979d07b8528SLiu Bo err = read_block_for_search(root, p, &b, level, 2980cda79c54SDavid Sterba slot, key); 29815d9e75c4SJan Schmidt if (err == -EAGAIN) 29825d9e75c4SJan Schmidt goto again; 29835d9e75c4SJan Schmidt if (err) { 29845d9e75c4SJan Schmidt ret = err; 29855d9e75c4SJan Schmidt goto done; 29865d9e75c4SJan Schmidt } 29875d9e75c4SJan Schmidt 29885d9e75c4SJan Schmidt level = btrfs_header_level(b); 2989f82c458aSChris Mason err = btrfs_tree_read_lock_atomic(b); 29905d9e75c4SJan Schmidt if (!err) { 29915d9e75c4SJan Schmidt btrfs_set_path_blocking(p); 29925d9e75c4SJan Schmidt btrfs_tree_read_lock(b); 29935d9e75c4SJan Schmidt } 29940b246afaSJeff Mahoney b = tree_mod_log_rewind(fs_info, p, b, time_seq); 2995db7f3436SJosef Bacik if (!b) { 2996db7f3436SJosef Bacik ret = -ENOMEM; 2997db7f3436SJosef Bacik goto done; 2998db7f3436SJosef Bacik } 29995d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 30005d9e75c4SJan Schmidt p->nodes[level] = b; 30015d9e75c4SJan Schmidt } else { 30025d9e75c4SJan Schmidt p->slots[level] = slot; 30035d9e75c4SJan Schmidt unlock_up(p, level, lowest_unlock, 0, NULL); 30045d9e75c4SJan Schmidt goto done; 30055d9e75c4SJan Schmidt } 30065d9e75c4SJan Schmidt } 30075d9e75c4SJan Schmidt ret = 1; 30085d9e75c4SJan Schmidt done: 30095d9e75c4SJan Schmidt if (!p->leave_spinning) 30105d9e75c4SJan Schmidt btrfs_set_path_blocking(p); 30115d9e75c4SJan Schmidt if (ret < 0) 30125d9e75c4SJan Schmidt btrfs_release_path(p); 30135d9e75c4SJan Schmidt 30145d9e75c4SJan Schmidt return ret; 30155d9e75c4SJan Schmidt } 30165d9e75c4SJan Schmidt 30175d9e75c4SJan Schmidt /* 30182f38b3e1SArne Jansen * helper to use instead of search slot if no exact match is needed but 30192f38b3e1SArne Jansen * instead the next or previous item should be returned. 30202f38b3e1SArne Jansen * When find_higher is true, the next higher item is returned, the next lower 30212f38b3e1SArne Jansen * otherwise. 30222f38b3e1SArne Jansen * When return_any and find_higher are both true, and no higher item is found, 30232f38b3e1SArne Jansen * return the next lower instead. 30242f38b3e1SArne Jansen * When return_any is true and find_higher is false, and no lower item is found, 30252f38b3e1SArne Jansen * return the next higher instead. 30262f38b3e1SArne Jansen * It returns 0 if any item is found, 1 if none is found (tree empty), and 30272f38b3e1SArne Jansen * < 0 on error 30282f38b3e1SArne Jansen */ 30292f38b3e1SArne Jansen int btrfs_search_slot_for_read(struct btrfs_root *root, 3030310712b2SOmar Sandoval const struct btrfs_key *key, 3031310712b2SOmar Sandoval struct btrfs_path *p, int find_higher, 3032310712b2SOmar Sandoval int return_any) 30332f38b3e1SArne Jansen { 30342f38b3e1SArne Jansen int ret; 30352f38b3e1SArne Jansen struct extent_buffer *leaf; 30362f38b3e1SArne Jansen 30372f38b3e1SArne Jansen again: 30382f38b3e1SArne Jansen ret = btrfs_search_slot(NULL, root, key, p, 0, 0); 30392f38b3e1SArne Jansen if (ret <= 0) 30402f38b3e1SArne Jansen return ret; 30412f38b3e1SArne Jansen /* 30422f38b3e1SArne Jansen * a return value of 1 means the path is at the position where the 30432f38b3e1SArne Jansen * item should be inserted. Normally this is the next bigger item, 30442f38b3e1SArne Jansen * but in case the previous item is the last in a leaf, path points 30452f38b3e1SArne Jansen * to the first free slot in the previous leaf, i.e. at an invalid 30462f38b3e1SArne Jansen * item. 30472f38b3e1SArne Jansen */ 30482f38b3e1SArne Jansen leaf = p->nodes[0]; 30492f38b3e1SArne Jansen 30502f38b3e1SArne Jansen if (find_higher) { 30512f38b3e1SArne Jansen if (p->slots[0] >= btrfs_header_nritems(leaf)) { 30522f38b3e1SArne Jansen ret = btrfs_next_leaf(root, p); 30532f38b3e1SArne Jansen if (ret <= 0) 30542f38b3e1SArne Jansen return ret; 30552f38b3e1SArne Jansen if (!return_any) 30562f38b3e1SArne Jansen return 1; 30572f38b3e1SArne Jansen /* 30582f38b3e1SArne Jansen * no higher item found, return the next 30592f38b3e1SArne Jansen * lower instead 30602f38b3e1SArne Jansen */ 30612f38b3e1SArne Jansen return_any = 0; 30622f38b3e1SArne Jansen find_higher = 0; 30632f38b3e1SArne Jansen btrfs_release_path(p); 30642f38b3e1SArne Jansen goto again; 30652f38b3e1SArne Jansen } 30662f38b3e1SArne Jansen } else { 30672f38b3e1SArne Jansen if (p->slots[0] == 0) { 30682f38b3e1SArne Jansen ret = btrfs_prev_leaf(root, p); 3069e6793769SArne Jansen if (ret < 0) 30702f38b3e1SArne Jansen return ret; 3071e6793769SArne Jansen if (!ret) { 307223c6bf6aSFilipe David Borba Manana leaf = p->nodes[0]; 307323c6bf6aSFilipe David Borba Manana if (p->slots[0] == btrfs_header_nritems(leaf)) 307423c6bf6aSFilipe David Borba Manana p->slots[0]--; 3075e6793769SArne Jansen return 0; 3076e6793769SArne Jansen } 30772f38b3e1SArne Jansen if (!return_any) 30782f38b3e1SArne Jansen return 1; 30792f38b3e1SArne Jansen /* 30802f38b3e1SArne Jansen * no lower item found, return the next 30812f38b3e1SArne Jansen * higher instead 30822f38b3e1SArne Jansen */ 30832f38b3e1SArne Jansen return_any = 0; 30842f38b3e1SArne Jansen find_higher = 1; 30852f38b3e1SArne Jansen btrfs_release_path(p); 30862f38b3e1SArne Jansen goto again; 3087e6793769SArne Jansen } else { 30882f38b3e1SArne Jansen --p->slots[0]; 30892f38b3e1SArne Jansen } 30902f38b3e1SArne Jansen } 30912f38b3e1SArne Jansen return 0; 30922f38b3e1SArne Jansen } 30932f38b3e1SArne Jansen 30942f38b3e1SArne Jansen /* 309574123bd7SChris Mason * adjust the pointers going up the tree, starting at level 309674123bd7SChris Mason * making sure the right key of each node is points to 'key'. 309774123bd7SChris Mason * This is used after shifting pointers to the left, so it stops 309874123bd7SChris Mason * fixing up pointers when a given leaf/node is not in slot 0 of the 309974123bd7SChris Mason * higher levels 3100aa5d6bedSChris Mason * 310174123bd7SChris Mason */ 3102b167fa91SNikolay Borisov static void fixup_low_keys(struct btrfs_path *path, 31035f39d397SChris Mason struct btrfs_disk_key *key, int level) 3104be0e5c09SChris Mason { 3105be0e5c09SChris Mason int i; 31065f39d397SChris Mason struct extent_buffer *t; 31070e82bcfeSDavid Sterba int ret; 31085f39d397SChris Mason 3109234b63a0SChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 3110be0e5c09SChris Mason int tslot = path->slots[i]; 31110e82bcfeSDavid Sterba 3112eb60ceacSChris Mason if (!path->nodes[i]) 3113be0e5c09SChris Mason break; 31145f39d397SChris Mason t = path->nodes[i]; 31150e82bcfeSDavid Sterba ret = tree_mod_log_insert_key(t, tslot, MOD_LOG_KEY_REPLACE, 31160e82bcfeSDavid Sterba GFP_ATOMIC); 31170e82bcfeSDavid Sterba BUG_ON(ret < 0); 31185f39d397SChris Mason btrfs_set_node_key(t, key, tslot); 3119d6025579SChris Mason btrfs_mark_buffer_dirty(path->nodes[i]); 3120be0e5c09SChris Mason if (tslot != 0) 3121be0e5c09SChris Mason break; 3122be0e5c09SChris Mason } 3123be0e5c09SChris Mason } 3124be0e5c09SChris Mason 312574123bd7SChris Mason /* 312631840ae1SZheng Yan * update item key. 312731840ae1SZheng Yan * 312831840ae1SZheng Yan * This function isn't completely safe. It's the caller's responsibility 312931840ae1SZheng Yan * that the new key won't break the order 313031840ae1SZheng Yan */ 3131b7a0365eSDaniel Dressler void btrfs_set_item_key_safe(struct btrfs_fs_info *fs_info, 3132b7a0365eSDaniel Dressler struct btrfs_path *path, 3133310712b2SOmar Sandoval const struct btrfs_key *new_key) 313431840ae1SZheng Yan { 313531840ae1SZheng Yan struct btrfs_disk_key disk_key; 313631840ae1SZheng Yan struct extent_buffer *eb; 313731840ae1SZheng Yan int slot; 313831840ae1SZheng Yan 313931840ae1SZheng Yan eb = path->nodes[0]; 314031840ae1SZheng Yan slot = path->slots[0]; 314131840ae1SZheng Yan if (slot > 0) { 314231840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot - 1); 3143143bede5SJeff Mahoney BUG_ON(comp_keys(&disk_key, new_key) >= 0); 314431840ae1SZheng Yan } 314531840ae1SZheng Yan if (slot < btrfs_header_nritems(eb) - 1) { 314631840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot + 1); 3147143bede5SJeff Mahoney BUG_ON(comp_keys(&disk_key, new_key) <= 0); 314831840ae1SZheng Yan } 314931840ae1SZheng Yan 315031840ae1SZheng Yan btrfs_cpu_key_to_disk(&disk_key, new_key); 315131840ae1SZheng Yan btrfs_set_item_key(eb, &disk_key, slot); 315231840ae1SZheng Yan btrfs_mark_buffer_dirty(eb); 315331840ae1SZheng Yan if (slot == 0) 3154b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 315531840ae1SZheng Yan } 315631840ae1SZheng Yan 315731840ae1SZheng Yan /* 315874123bd7SChris Mason * try to push data from one node into the next node left in the 315979f95c82SChris Mason * tree. 3160aa5d6bedSChris Mason * 3161aa5d6bedSChris Mason * returns 0 if some ptrs were pushed left, < 0 if there was some horrible 3162aa5d6bedSChris Mason * error, and > 0 if there was no room in the left hand block. 316374123bd7SChris Mason */ 316498ed5174SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 31652ff7e61eSJeff Mahoney struct btrfs_fs_info *fs_info, 31662ff7e61eSJeff Mahoney struct extent_buffer *dst, 3167971a1f66SChris Mason struct extent_buffer *src, int empty) 3168be0e5c09SChris Mason { 3169be0e5c09SChris Mason int push_items = 0; 3170bb803951SChris Mason int src_nritems; 3171bb803951SChris Mason int dst_nritems; 3172aa5d6bedSChris Mason int ret = 0; 3173be0e5c09SChris Mason 31745f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 31755f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 31760b246afaSJeff Mahoney push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems; 31777bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 31787bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 317954aa1f4dSChris Mason 3180bce4eae9SChris Mason if (!empty && src_nritems <= 8) 3181971a1f66SChris Mason return 1; 3182971a1f66SChris Mason 3183d397712bSChris Mason if (push_items <= 0) 3184be0e5c09SChris Mason return 1; 3185be0e5c09SChris Mason 3186bce4eae9SChris Mason if (empty) { 3187971a1f66SChris Mason push_items = min(src_nritems, push_items); 3188bce4eae9SChris Mason if (push_items < src_nritems) { 3189bce4eae9SChris Mason /* leave at least 8 pointers in the node if 3190bce4eae9SChris Mason * we aren't going to empty it 3191bce4eae9SChris Mason */ 3192bce4eae9SChris Mason if (src_nritems - push_items < 8) { 3193bce4eae9SChris Mason if (push_items <= 8) 3194bce4eae9SChris Mason return 1; 3195bce4eae9SChris Mason push_items -= 8; 3196bce4eae9SChris Mason } 3197bce4eae9SChris Mason } 3198bce4eae9SChris Mason } else 3199bce4eae9SChris Mason push_items = min(src_nritems - 8, push_items); 320079f95c82SChris Mason 32010b246afaSJeff Mahoney ret = tree_mod_log_eb_copy(fs_info, dst, src, dst_nritems, 0, 320290f8d62eSJan Schmidt push_items); 32035de865eeSFilipe David Borba Manana if (ret) { 320466642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 32055de865eeSFilipe David Borba Manana return ret; 32065de865eeSFilipe David Borba Manana } 32075f39d397SChris Mason copy_extent_buffer(dst, src, 32085f39d397SChris Mason btrfs_node_key_ptr_offset(dst_nritems), 32095f39d397SChris Mason btrfs_node_key_ptr_offset(0), 3210123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 32115f39d397SChris Mason 3212bb803951SChris Mason if (push_items < src_nritems) { 321357911b8bSJan Schmidt /* 3214bf1d3425SDavid Sterba * Don't call tree_mod_log_insert_move here, key removal was 3215bf1d3425SDavid Sterba * already fully logged by tree_mod_log_eb_copy above. 321657911b8bSJan Schmidt */ 32175f39d397SChris Mason memmove_extent_buffer(src, btrfs_node_key_ptr_offset(0), 32185f39d397SChris Mason btrfs_node_key_ptr_offset(push_items), 3219e2fa7227SChris Mason (src_nritems - push_items) * 3220123abc88SChris Mason sizeof(struct btrfs_key_ptr)); 3221bb803951SChris Mason } 32225f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 32235f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 32245f39d397SChris Mason btrfs_mark_buffer_dirty(src); 32255f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 322631840ae1SZheng Yan 3227bb803951SChris Mason return ret; 3228be0e5c09SChris Mason } 3229be0e5c09SChris Mason 323097571fd0SChris Mason /* 323179f95c82SChris Mason * try to push data from one node into the next node right in the 323279f95c82SChris Mason * tree. 323379f95c82SChris Mason * 323479f95c82SChris Mason * returns 0 if some ptrs were pushed, < 0 if there was some horrible 323579f95c82SChris Mason * error, and > 0 if there was no room in the right hand block. 323679f95c82SChris Mason * 323779f95c82SChris Mason * this will only push up to 1/2 the contents of the left node over 323879f95c82SChris Mason */ 32395f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 32402ff7e61eSJeff Mahoney struct btrfs_fs_info *fs_info, 32415f39d397SChris Mason struct extent_buffer *dst, 32425f39d397SChris Mason struct extent_buffer *src) 324379f95c82SChris Mason { 324479f95c82SChris Mason int push_items = 0; 324579f95c82SChris Mason int max_push; 324679f95c82SChris Mason int src_nritems; 324779f95c82SChris Mason int dst_nritems; 324879f95c82SChris Mason int ret = 0; 324979f95c82SChris Mason 32507bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 32517bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 32527bb86316SChris Mason 32535f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 32545f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 32550b246afaSJeff Mahoney push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems; 3256d397712bSChris Mason if (push_items <= 0) 325779f95c82SChris Mason return 1; 3258bce4eae9SChris Mason 3259d397712bSChris Mason if (src_nritems < 4) 3260bce4eae9SChris Mason return 1; 326179f95c82SChris Mason 326279f95c82SChris Mason max_push = src_nritems / 2 + 1; 326379f95c82SChris Mason /* don't try to empty the node */ 3264d397712bSChris Mason if (max_push >= src_nritems) 326579f95c82SChris Mason return 1; 3266252c38f0SYan 326779f95c82SChris Mason if (max_push < push_items) 326879f95c82SChris Mason push_items = max_push; 326979f95c82SChris Mason 3270bf1d3425SDavid Sterba ret = tree_mod_log_insert_move(dst, push_items, 0, dst_nritems); 3271bf1d3425SDavid Sterba BUG_ON(ret < 0); 32725f39d397SChris Mason memmove_extent_buffer(dst, btrfs_node_key_ptr_offset(push_items), 32735f39d397SChris Mason btrfs_node_key_ptr_offset(0), 32745f39d397SChris Mason (dst_nritems) * 32755f39d397SChris Mason sizeof(struct btrfs_key_ptr)); 3276d6025579SChris Mason 32770b246afaSJeff Mahoney ret = tree_mod_log_eb_copy(fs_info, dst, src, 0, 327890f8d62eSJan Schmidt src_nritems - push_items, push_items); 32795de865eeSFilipe David Borba Manana if (ret) { 328066642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 32815de865eeSFilipe David Borba Manana return ret; 32825de865eeSFilipe David Borba Manana } 32835f39d397SChris Mason copy_extent_buffer(dst, src, 32845f39d397SChris Mason btrfs_node_key_ptr_offset(0), 32855f39d397SChris Mason btrfs_node_key_ptr_offset(src_nritems - push_items), 3286123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 328779f95c82SChris Mason 32885f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 32895f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 329079f95c82SChris Mason 32915f39d397SChris Mason btrfs_mark_buffer_dirty(src); 32925f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 329331840ae1SZheng Yan 329479f95c82SChris Mason return ret; 329579f95c82SChris Mason } 329679f95c82SChris Mason 329779f95c82SChris Mason /* 329897571fd0SChris Mason * helper function to insert a new root level in the tree. 329997571fd0SChris Mason * A new node is allocated, and a single item is inserted to 330097571fd0SChris Mason * point to the existing root 3301aa5d6bedSChris Mason * 3302aa5d6bedSChris Mason * returns zero on success or < 0 on failure. 330397571fd0SChris Mason */ 3304d397712bSChris Mason static noinline int insert_new_root(struct btrfs_trans_handle *trans, 33055f39d397SChris Mason struct btrfs_root *root, 3306fdd99c72SLiu Bo struct btrfs_path *path, int level) 330774123bd7SChris Mason { 33080b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 33097bb86316SChris Mason u64 lower_gen; 33105f39d397SChris Mason struct extent_buffer *lower; 33115f39d397SChris Mason struct extent_buffer *c; 3312925baeddSChris Mason struct extent_buffer *old; 33135f39d397SChris Mason struct btrfs_disk_key lower_key; 3314d9d19a01SDavid Sterba int ret; 33155c680ed6SChris Mason 33165c680ed6SChris Mason BUG_ON(path->nodes[level]); 33175c680ed6SChris Mason BUG_ON(path->nodes[level-1] != root->node); 33185c680ed6SChris Mason 33197bb86316SChris Mason lower = path->nodes[level-1]; 33207bb86316SChris Mason if (level == 1) 33217bb86316SChris Mason btrfs_item_key(lower, &lower_key, 0); 33227bb86316SChris Mason else 33237bb86316SChris Mason btrfs_node_key(lower, &lower_key, 0); 33247bb86316SChris Mason 33254d75f8a9SDavid Sterba c = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 33264d75f8a9SDavid Sterba &lower_key, level, root->node->start, 0); 33275f39d397SChris Mason if (IS_ERR(c)) 33285f39d397SChris Mason return PTR_ERR(c); 3329925baeddSChris Mason 33300b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 3331f0486c68SYan, Zheng 33325f39d397SChris Mason btrfs_set_header_nritems(c, 1); 33335f39d397SChris Mason btrfs_set_node_key(c, &lower_key, 0); 3334db94535dSChris Mason btrfs_set_node_blockptr(c, 0, lower->start); 33357bb86316SChris Mason lower_gen = btrfs_header_generation(lower); 333631840ae1SZheng Yan WARN_ON(lower_gen != trans->transid); 33377bb86316SChris Mason 33387bb86316SChris Mason btrfs_set_node_ptr_generation(c, 0, lower_gen); 33395f39d397SChris Mason 33405f39d397SChris Mason btrfs_mark_buffer_dirty(c); 3341d5719762SChris Mason 3342925baeddSChris Mason old = root->node; 3343d9d19a01SDavid Sterba ret = tree_mod_log_insert_root(root->node, c, 0); 3344d9d19a01SDavid Sterba BUG_ON(ret < 0); 3345240f62c8SChris Mason rcu_assign_pointer(root->node, c); 3346925baeddSChris Mason 3347925baeddSChris Mason /* the super has an extra ref to root->node */ 3348925baeddSChris Mason free_extent_buffer(old); 3349925baeddSChris Mason 33500b86a832SChris Mason add_root_to_dirty_list(root); 33515f39d397SChris Mason extent_buffer_get(c); 33525f39d397SChris Mason path->nodes[level] = c; 335395449a16Schandan path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING; 335474123bd7SChris Mason path->slots[level] = 0; 335574123bd7SChris Mason return 0; 335674123bd7SChris Mason } 33575c680ed6SChris Mason 33585c680ed6SChris Mason /* 33595c680ed6SChris Mason * worker function to insert a single pointer in a node. 33605c680ed6SChris Mason * the node should have enough room for the pointer already 336197571fd0SChris Mason * 33625c680ed6SChris Mason * slot and level indicate where you want the key to go, and 33635c680ed6SChris Mason * blocknr is the block the key points to. 33645c680ed6SChris Mason */ 3365143bede5SJeff Mahoney static void insert_ptr(struct btrfs_trans_handle *trans, 33662ff7e61eSJeff Mahoney struct btrfs_fs_info *fs_info, struct btrfs_path *path, 3367143bede5SJeff Mahoney struct btrfs_disk_key *key, u64 bytenr, 3368c3e06965SJan Schmidt int slot, int level) 33695c680ed6SChris Mason { 33705f39d397SChris Mason struct extent_buffer *lower; 33715c680ed6SChris Mason int nritems; 3372f3ea38daSJan Schmidt int ret; 33735c680ed6SChris Mason 33745c680ed6SChris Mason BUG_ON(!path->nodes[level]); 3375f0486c68SYan, Zheng btrfs_assert_tree_locked(path->nodes[level]); 33765f39d397SChris Mason lower = path->nodes[level]; 33775f39d397SChris Mason nritems = btrfs_header_nritems(lower); 3378c293498bSStoyan Gaydarov BUG_ON(slot > nritems); 33790b246afaSJeff Mahoney BUG_ON(nritems == BTRFS_NODEPTRS_PER_BLOCK(fs_info)); 338074123bd7SChris Mason if (slot != nritems) { 3381bf1d3425SDavid Sterba if (level) { 3382bf1d3425SDavid Sterba ret = tree_mod_log_insert_move(lower, slot + 1, slot, 3383a446a979SDavid Sterba nritems - slot); 3384bf1d3425SDavid Sterba BUG_ON(ret < 0); 3385bf1d3425SDavid Sterba } 33865f39d397SChris Mason memmove_extent_buffer(lower, 33875f39d397SChris Mason btrfs_node_key_ptr_offset(slot + 1), 33885f39d397SChris Mason btrfs_node_key_ptr_offset(slot), 3389123abc88SChris Mason (nritems - slot) * sizeof(struct btrfs_key_ptr)); 339074123bd7SChris Mason } 3391c3e06965SJan Schmidt if (level) { 3392e09c2efeSDavid Sterba ret = tree_mod_log_insert_key(lower, slot, MOD_LOG_KEY_ADD, 3393e09c2efeSDavid Sterba GFP_NOFS); 3394f3ea38daSJan Schmidt BUG_ON(ret < 0); 3395f3ea38daSJan Schmidt } 33965f39d397SChris Mason btrfs_set_node_key(lower, key, slot); 3397db94535dSChris Mason btrfs_set_node_blockptr(lower, slot, bytenr); 339874493f7aSChris Mason WARN_ON(trans->transid == 0); 339974493f7aSChris Mason btrfs_set_node_ptr_generation(lower, slot, trans->transid); 34005f39d397SChris Mason btrfs_set_header_nritems(lower, nritems + 1); 34015f39d397SChris Mason btrfs_mark_buffer_dirty(lower); 340274123bd7SChris Mason } 340374123bd7SChris Mason 340497571fd0SChris Mason /* 340597571fd0SChris Mason * split the node at the specified level in path in two. 340697571fd0SChris Mason * The path is corrected to point to the appropriate node after the split 340797571fd0SChris Mason * 340897571fd0SChris Mason * Before splitting this tries to make some room in the node by pushing 340997571fd0SChris Mason * left and right, if either one works, it returns right away. 3410aa5d6bedSChris Mason * 3411aa5d6bedSChris Mason * returns 0 on success and < 0 on failure 341297571fd0SChris Mason */ 3413e02119d5SChris Mason static noinline int split_node(struct btrfs_trans_handle *trans, 3414e02119d5SChris Mason struct btrfs_root *root, 3415e02119d5SChris Mason struct btrfs_path *path, int level) 3416be0e5c09SChris Mason { 34170b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 34185f39d397SChris Mason struct extent_buffer *c; 34195f39d397SChris Mason struct extent_buffer *split; 34205f39d397SChris Mason struct btrfs_disk_key disk_key; 3421be0e5c09SChris Mason int mid; 34225c680ed6SChris Mason int ret; 34237518a238SChris Mason u32 c_nritems; 3424be0e5c09SChris Mason 34255f39d397SChris Mason c = path->nodes[level]; 34267bb86316SChris Mason WARN_ON(btrfs_header_generation(c) != trans->transid); 34275f39d397SChris Mason if (c == root->node) { 3428d9abbf1cSJan Schmidt /* 342990f8d62eSJan Schmidt * trying to split the root, lets make a new one 343090f8d62eSJan Schmidt * 3431fdd99c72SLiu Bo * tree mod log: We don't log_removal old root in 343290f8d62eSJan Schmidt * insert_new_root, because that root buffer will be kept as a 343390f8d62eSJan Schmidt * normal node. We are going to log removal of half of the 343490f8d62eSJan Schmidt * elements below with tree_mod_log_eb_copy. We're holding a 343590f8d62eSJan Schmidt * tree lock on the buffer, which is why we cannot race with 343690f8d62eSJan Schmidt * other tree_mod_log users. 3437d9abbf1cSJan Schmidt */ 3438fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, level + 1); 34395c680ed6SChris Mason if (ret) 34405c680ed6SChris Mason return ret; 3441b3612421SChris Mason } else { 3442e66f709bSChris Mason ret = push_nodes_for_insert(trans, root, path, level); 34435f39d397SChris Mason c = path->nodes[level]; 34445f39d397SChris Mason if (!ret && btrfs_header_nritems(c) < 34450b246afaSJeff Mahoney BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3) 3446e66f709bSChris Mason return 0; 344754aa1f4dSChris Mason if (ret < 0) 344854aa1f4dSChris Mason return ret; 34495c680ed6SChris Mason } 3450e66f709bSChris Mason 34515f39d397SChris Mason c_nritems = btrfs_header_nritems(c); 34525d4f98a2SYan Zheng mid = (c_nritems + 1) / 2; 34535d4f98a2SYan Zheng btrfs_node_key(c, &disk_key, mid); 34547bb86316SChris Mason 34554d75f8a9SDavid Sterba split = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 34565581a51aSJan Schmidt &disk_key, level, c->start, 0); 34575f39d397SChris Mason if (IS_ERR(split)) 34585f39d397SChris Mason return PTR_ERR(split); 345954aa1f4dSChris Mason 34600b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 3461bc877d28SNikolay Borisov ASSERT(btrfs_header_level(c) == level); 34625f39d397SChris Mason 34630b246afaSJeff Mahoney ret = tree_mod_log_eb_copy(fs_info, split, c, 0, mid, c_nritems - mid); 34645de865eeSFilipe David Borba Manana if (ret) { 346566642832SJeff Mahoney btrfs_abort_transaction(trans, ret); 34665de865eeSFilipe David Borba Manana return ret; 34675de865eeSFilipe David Borba Manana } 34685f39d397SChris Mason copy_extent_buffer(split, c, 34695f39d397SChris Mason btrfs_node_key_ptr_offset(0), 34705f39d397SChris Mason btrfs_node_key_ptr_offset(mid), 3471123abc88SChris Mason (c_nritems - mid) * sizeof(struct btrfs_key_ptr)); 34725f39d397SChris Mason btrfs_set_header_nritems(split, c_nritems - mid); 34735f39d397SChris Mason btrfs_set_header_nritems(c, mid); 3474aa5d6bedSChris Mason ret = 0; 3475aa5d6bedSChris Mason 34765f39d397SChris Mason btrfs_mark_buffer_dirty(c); 34775f39d397SChris Mason btrfs_mark_buffer_dirty(split); 34785f39d397SChris Mason 34792ff7e61eSJeff Mahoney insert_ptr(trans, fs_info, path, &disk_key, split->start, 3480c3e06965SJan Schmidt path->slots[level + 1] + 1, level + 1); 3481aa5d6bedSChris Mason 34825de08d7dSChris Mason if (path->slots[level] >= mid) { 34835c680ed6SChris Mason path->slots[level] -= mid; 3484925baeddSChris Mason btrfs_tree_unlock(c); 34855f39d397SChris Mason free_extent_buffer(c); 34865f39d397SChris Mason path->nodes[level] = split; 34875c680ed6SChris Mason path->slots[level + 1] += 1; 3488eb60ceacSChris Mason } else { 3489925baeddSChris Mason btrfs_tree_unlock(split); 34905f39d397SChris Mason free_extent_buffer(split); 3491be0e5c09SChris Mason } 3492aa5d6bedSChris Mason return ret; 3493be0e5c09SChris Mason } 3494be0e5c09SChris Mason 349574123bd7SChris Mason /* 349674123bd7SChris Mason * how many bytes are required to store the items in a leaf. start 349774123bd7SChris Mason * and nr indicate which items in the leaf to check. This totals up the 349874123bd7SChris Mason * space used both by the item structs and the item data 349974123bd7SChris Mason */ 35005f39d397SChris Mason static int leaf_space_used(struct extent_buffer *l, int start, int nr) 3501be0e5c09SChris Mason { 350241be1f3bSJosef Bacik struct btrfs_item *start_item; 350341be1f3bSJosef Bacik struct btrfs_item *end_item; 350441be1f3bSJosef Bacik struct btrfs_map_token token; 3505be0e5c09SChris Mason int data_len; 35065f39d397SChris Mason int nritems = btrfs_header_nritems(l); 3507d4dbff95SChris Mason int end = min(nritems, start + nr) - 1; 3508be0e5c09SChris Mason 3509be0e5c09SChris Mason if (!nr) 3510be0e5c09SChris Mason return 0; 351141be1f3bSJosef Bacik btrfs_init_map_token(&token); 3512dd3cc16bSRoss Kirk start_item = btrfs_item_nr(start); 3513dd3cc16bSRoss Kirk end_item = btrfs_item_nr(end); 351441be1f3bSJosef Bacik data_len = btrfs_token_item_offset(l, start_item, &token) + 351541be1f3bSJosef Bacik btrfs_token_item_size(l, start_item, &token); 351641be1f3bSJosef Bacik data_len = data_len - btrfs_token_item_offset(l, end_item, &token); 35170783fcfcSChris Mason data_len += sizeof(struct btrfs_item) * nr; 3518d4dbff95SChris Mason WARN_ON(data_len < 0); 3519be0e5c09SChris Mason return data_len; 3520be0e5c09SChris Mason } 3521be0e5c09SChris Mason 352274123bd7SChris Mason /* 3523d4dbff95SChris Mason * The space between the end of the leaf items and 3524d4dbff95SChris Mason * the start of the leaf data. IOW, how much room 3525d4dbff95SChris Mason * the leaf has left for both items and data 3526d4dbff95SChris Mason */ 35272ff7e61eSJeff Mahoney noinline int btrfs_leaf_free_space(struct btrfs_fs_info *fs_info, 3528e02119d5SChris Mason struct extent_buffer *leaf) 3529d4dbff95SChris Mason { 35305f39d397SChris Mason int nritems = btrfs_header_nritems(leaf); 35315f39d397SChris Mason int ret; 35320b246afaSJeff Mahoney 35330b246afaSJeff Mahoney ret = BTRFS_LEAF_DATA_SIZE(fs_info) - leaf_space_used(leaf, 0, nritems); 35345f39d397SChris Mason if (ret < 0) { 35350b246afaSJeff Mahoney btrfs_crit(fs_info, 3536efe120a0SFrank Holton "leaf free space ret %d, leaf data size %lu, used %d nritems %d", 3537da17066cSJeff Mahoney ret, 35380b246afaSJeff Mahoney (unsigned long) BTRFS_LEAF_DATA_SIZE(fs_info), 35395f39d397SChris Mason leaf_space_used(leaf, 0, nritems), nritems); 35405f39d397SChris Mason } 35415f39d397SChris Mason return ret; 3542d4dbff95SChris Mason } 3543d4dbff95SChris Mason 354499d8f83cSChris Mason /* 354599d8f83cSChris Mason * min slot controls the lowest index we're willing to push to the 354699d8f83cSChris Mason * right. We'll push up to and including min_slot, but no lower 354799d8f83cSChris Mason */ 35481e47eef2SDavid Sterba static noinline int __push_leaf_right(struct btrfs_fs_info *fs_info, 354944871b1bSChris Mason struct btrfs_path *path, 355044871b1bSChris Mason int data_size, int empty, 355144871b1bSChris Mason struct extent_buffer *right, 355299d8f83cSChris Mason int free_space, u32 left_nritems, 355399d8f83cSChris Mason u32 min_slot) 355400ec4c51SChris Mason { 35555f39d397SChris Mason struct extent_buffer *left = path->nodes[0]; 355644871b1bSChris Mason struct extent_buffer *upper = path->nodes[1]; 3557cfed81a0SChris Mason struct btrfs_map_token token; 35585f39d397SChris Mason struct btrfs_disk_key disk_key; 355900ec4c51SChris Mason int slot; 356034a38218SChris Mason u32 i; 356100ec4c51SChris Mason int push_space = 0; 356200ec4c51SChris Mason int push_items = 0; 35630783fcfcSChris Mason struct btrfs_item *item; 356434a38218SChris Mason u32 nr; 35657518a238SChris Mason u32 right_nritems; 35665f39d397SChris Mason u32 data_end; 3567db94535dSChris Mason u32 this_item_size; 356800ec4c51SChris Mason 3569cfed81a0SChris Mason btrfs_init_map_token(&token); 3570cfed81a0SChris Mason 357134a38218SChris Mason if (empty) 357234a38218SChris Mason nr = 0; 357334a38218SChris Mason else 357499d8f83cSChris Mason nr = max_t(u32, 1, min_slot); 357534a38218SChris Mason 357631840ae1SZheng Yan if (path->slots[0] >= left_nritems) 357787b29b20SYan Zheng push_space += data_size; 357831840ae1SZheng Yan 357944871b1bSChris Mason slot = path->slots[1]; 358034a38218SChris Mason i = left_nritems - 1; 358134a38218SChris Mason while (i >= nr) { 3582dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 3583db94535dSChris Mason 358431840ae1SZheng Yan if (!empty && push_items > 0) { 358531840ae1SZheng Yan if (path->slots[0] > i) 358631840ae1SZheng Yan break; 358731840ae1SZheng Yan if (path->slots[0] == i) { 35882ff7e61eSJeff Mahoney int space = btrfs_leaf_free_space(fs_info, left); 358931840ae1SZheng Yan if (space + push_space * 2 > free_space) 359031840ae1SZheng Yan break; 359131840ae1SZheng Yan } 359231840ae1SZheng Yan } 359331840ae1SZheng Yan 359400ec4c51SChris Mason if (path->slots[0] == i) 359587b29b20SYan Zheng push_space += data_size; 3596db94535dSChris Mason 3597db94535dSChris Mason this_item_size = btrfs_item_size(left, item); 3598db94535dSChris Mason if (this_item_size + sizeof(*item) + push_space > free_space) 359900ec4c51SChris Mason break; 360031840ae1SZheng Yan 360100ec4c51SChris Mason push_items++; 3602db94535dSChris Mason push_space += this_item_size + sizeof(*item); 360334a38218SChris Mason if (i == 0) 360434a38218SChris Mason break; 360534a38218SChris Mason i--; 3606db94535dSChris Mason } 36075f39d397SChris Mason 3608925baeddSChris Mason if (push_items == 0) 3609925baeddSChris Mason goto out_unlock; 36105f39d397SChris Mason 36116c1500f2SJulia Lawall WARN_ON(!empty && push_items == left_nritems); 36125f39d397SChris Mason 361300ec4c51SChris Mason /* push left to right */ 36145f39d397SChris Mason right_nritems = btrfs_header_nritems(right); 361534a38218SChris Mason 36165f39d397SChris Mason push_space = btrfs_item_end_nr(left, left_nritems - push_items); 36172ff7e61eSJeff Mahoney push_space -= leaf_data_end(fs_info, left); 36185f39d397SChris Mason 361900ec4c51SChris Mason /* make room in the right data area */ 36202ff7e61eSJeff Mahoney data_end = leaf_data_end(fs_info, right); 36215f39d397SChris Mason memmove_extent_buffer(right, 36223d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + data_end - push_space, 36233d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + data_end, 36240b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - data_end); 36255f39d397SChris Mason 362600ec4c51SChris Mason /* copy from the left data area */ 36273d9ec8c4SNikolay Borisov copy_extent_buffer(right, left, BTRFS_LEAF_DATA_OFFSET + 36280b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - push_space, 36293d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + leaf_data_end(fs_info, left), 3630d6025579SChris Mason push_space); 36315f39d397SChris Mason 36325f39d397SChris Mason memmove_extent_buffer(right, btrfs_item_nr_offset(push_items), 36335f39d397SChris Mason btrfs_item_nr_offset(0), 36340783fcfcSChris Mason right_nritems * sizeof(struct btrfs_item)); 36355f39d397SChris Mason 363600ec4c51SChris Mason /* copy the items from left to right */ 36375f39d397SChris Mason copy_extent_buffer(right, left, btrfs_item_nr_offset(0), 36385f39d397SChris Mason btrfs_item_nr_offset(left_nritems - push_items), 36390783fcfcSChris Mason push_items * sizeof(struct btrfs_item)); 364000ec4c51SChris Mason 364100ec4c51SChris Mason /* update the item pointers */ 36427518a238SChris Mason right_nritems += push_items; 36435f39d397SChris Mason btrfs_set_header_nritems(right, right_nritems); 36440b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info); 36457518a238SChris Mason for (i = 0; i < right_nritems; i++) { 3646dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 3647cfed81a0SChris Mason push_space -= btrfs_token_item_size(right, item, &token); 3648cfed81a0SChris Mason btrfs_set_token_item_offset(right, item, push_space, &token); 3649db94535dSChris Mason } 3650db94535dSChris Mason 36517518a238SChris Mason left_nritems -= push_items; 36525f39d397SChris Mason btrfs_set_header_nritems(left, left_nritems); 365300ec4c51SChris Mason 365434a38218SChris Mason if (left_nritems) 36555f39d397SChris Mason btrfs_mark_buffer_dirty(left); 3656f0486c68SYan, Zheng else 36577c302b49SDavid Sterba clean_tree_block(fs_info, left); 3658f0486c68SYan, Zheng 36595f39d397SChris Mason btrfs_mark_buffer_dirty(right); 3660a429e513SChris Mason 36615f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 36625f39d397SChris Mason btrfs_set_node_key(upper, &disk_key, slot + 1); 3663d6025579SChris Mason btrfs_mark_buffer_dirty(upper); 366402217ed2SChris Mason 366500ec4c51SChris Mason /* then fixup the leaf pointer in the path */ 36667518a238SChris Mason if (path->slots[0] >= left_nritems) { 36677518a238SChris Mason path->slots[0] -= left_nritems; 3668925baeddSChris Mason if (btrfs_header_nritems(path->nodes[0]) == 0) 36697c302b49SDavid Sterba clean_tree_block(fs_info, path->nodes[0]); 3670925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 36715f39d397SChris Mason free_extent_buffer(path->nodes[0]); 36725f39d397SChris Mason path->nodes[0] = right; 367300ec4c51SChris Mason path->slots[1] += 1; 367400ec4c51SChris Mason } else { 3675925baeddSChris Mason btrfs_tree_unlock(right); 36765f39d397SChris Mason free_extent_buffer(right); 367700ec4c51SChris Mason } 367800ec4c51SChris Mason return 0; 3679925baeddSChris Mason 3680925baeddSChris Mason out_unlock: 3681925baeddSChris Mason btrfs_tree_unlock(right); 3682925baeddSChris Mason free_extent_buffer(right); 3683925baeddSChris Mason return 1; 368400ec4c51SChris Mason } 3685925baeddSChris Mason 368600ec4c51SChris Mason /* 368744871b1bSChris Mason * push some data in the path leaf to the right, trying to free up at 368874123bd7SChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 368944871b1bSChris Mason * 369044871b1bSChris Mason * returns 1 if the push failed because the other node didn't have enough 369144871b1bSChris Mason * room, 0 if everything worked out and < 0 if there were major errors. 369299d8f83cSChris Mason * 369399d8f83cSChris Mason * this will push starting from min_slot to the end of the leaf. It won't 369499d8f83cSChris Mason * push any slot lower than min_slot 369574123bd7SChris Mason */ 369644871b1bSChris Mason static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root 369799d8f83cSChris Mason *root, struct btrfs_path *path, 369899d8f83cSChris Mason int min_data_size, int data_size, 369999d8f83cSChris Mason int empty, u32 min_slot) 3700be0e5c09SChris Mason { 37012ff7e61eSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 370244871b1bSChris Mason struct extent_buffer *left = path->nodes[0]; 370344871b1bSChris Mason struct extent_buffer *right; 370444871b1bSChris Mason struct extent_buffer *upper; 370544871b1bSChris Mason int slot; 370644871b1bSChris Mason int free_space; 370744871b1bSChris Mason u32 left_nritems; 370844871b1bSChris Mason int ret; 370944871b1bSChris Mason 371044871b1bSChris Mason if (!path->nodes[1]) 371144871b1bSChris Mason return 1; 371244871b1bSChris Mason 371344871b1bSChris Mason slot = path->slots[1]; 371444871b1bSChris Mason upper = path->nodes[1]; 371544871b1bSChris Mason if (slot >= btrfs_header_nritems(upper) - 1) 371644871b1bSChris Mason return 1; 371744871b1bSChris Mason 371844871b1bSChris Mason btrfs_assert_tree_locked(path->nodes[1]); 371944871b1bSChris Mason 37202ff7e61eSJeff Mahoney right = read_node_slot(fs_info, upper, slot + 1); 3721fb770ae4SLiu Bo /* 3722fb770ae4SLiu Bo * slot + 1 is not valid or we fail to read the right node, 3723fb770ae4SLiu Bo * no big deal, just return. 3724fb770ae4SLiu Bo */ 3725fb770ae4SLiu Bo if (IS_ERR(right)) 372691ca338dSTsutomu Itoh return 1; 372791ca338dSTsutomu Itoh 372844871b1bSChris Mason btrfs_tree_lock(right); 372944871b1bSChris Mason btrfs_set_lock_blocking(right); 373044871b1bSChris Mason 37312ff7e61eSJeff Mahoney free_space = btrfs_leaf_free_space(fs_info, right); 373244871b1bSChris Mason if (free_space < data_size) 373344871b1bSChris Mason goto out_unlock; 373444871b1bSChris Mason 373544871b1bSChris Mason /* cow and double check */ 373644871b1bSChris Mason ret = btrfs_cow_block(trans, root, right, upper, 373744871b1bSChris Mason slot + 1, &right); 373844871b1bSChris Mason if (ret) 373944871b1bSChris Mason goto out_unlock; 374044871b1bSChris Mason 37412ff7e61eSJeff Mahoney free_space = btrfs_leaf_free_space(fs_info, right); 374244871b1bSChris Mason if (free_space < data_size) 374344871b1bSChris Mason goto out_unlock; 374444871b1bSChris Mason 374544871b1bSChris Mason left_nritems = btrfs_header_nritems(left); 374644871b1bSChris Mason if (left_nritems == 0) 374744871b1bSChris Mason goto out_unlock; 374844871b1bSChris Mason 37492ef1fed2SFilipe David Borba Manana if (path->slots[0] == left_nritems && !empty) { 37502ef1fed2SFilipe David Borba Manana /* Key greater than all keys in the leaf, right neighbor has 37512ef1fed2SFilipe David Borba Manana * enough room for it and we're not emptying our leaf to delete 37522ef1fed2SFilipe David Borba Manana * it, therefore use right neighbor to insert the new item and 37532ef1fed2SFilipe David Borba Manana * no need to touch/dirty our left leaft. */ 37542ef1fed2SFilipe David Borba Manana btrfs_tree_unlock(left); 37552ef1fed2SFilipe David Borba Manana free_extent_buffer(left); 37562ef1fed2SFilipe David Borba Manana path->nodes[0] = right; 37572ef1fed2SFilipe David Borba Manana path->slots[0] = 0; 37582ef1fed2SFilipe David Borba Manana path->slots[1]++; 37592ef1fed2SFilipe David Borba Manana return 0; 37602ef1fed2SFilipe David Borba Manana } 37612ef1fed2SFilipe David Borba Manana 37621e47eef2SDavid Sterba return __push_leaf_right(fs_info, path, min_data_size, empty, 376399d8f83cSChris Mason right, free_space, left_nritems, min_slot); 376444871b1bSChris Mason out_unlock: 376544871b1bSChris Mason btrfs_tree_unlock(right); 376644871b1bSChris Mason free_extent_buffer(right); 376744871b1bSChris Mason return 1; 376844871b1bSChris Mason } 376944871b1bSChris Mason 377044871b1bSChris Mason /* 377144871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 377244871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 377399d8f83cSChris Mason * 377499d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 377599d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us do all the 377699d8f83cSChris Mason * items 377744871b1bSChris Mason */ 377866cb7ddbSDavid Sterba static noinline int __push_leaf_left(struct btrfs_fs_info *fs_info, 377944871b1bSChris Mason struct btrfs_path *path, int data_size, 378044871b1bSChris Mason int empty, struct extent_buffer *left, 378199d8f83cSChris Mason int free_space, u32 right_nritems, 378299d8f83cSChris Mason u32 max_slot) 378344871b1bSChris Mason { 37845f39d397SChris Mason struct btrfs_disk_key disk_key; 37855f39d397SChris Mason struct extent_buffer *right = path->nodes[0]; 3786be0e5c09SChris Mason int i; 3787be0e5c09SChris Mason int push_space = 0; 3788be0e5c09SChris Mason int push_items = 0; 37890783fcfcSChris Mason struct btrfs_item *item; 37907518a238SChris Mason u32 old_left_nritems; 379134a38218SChris Mason u32 nr; 3792aa5d6bedSChris Mason int ret = 0; 3793db94535dSChris Mason u32 this_item_size; 3794db94535dSChris Mason u32 old_left_item_size; 3795cfed81a0SChris Mason struct btrfs_map_token token; 3796cfed81a0SChris Mason 3797cfed81a0SChris Mason btrfs_init_map_token(&token); 3798be0e5c09SChris Mason 379934a38218SChris Mason if (empty) 380099d8f83cSChris Mason nr = min(right_nritems, max_slot); 380134a38218SChris Mason else 380299d8f83cSChris Mason nr = min(right_nritems - 1, max_slot); 380334a38218SChris Mason 380434a38218SChris Mason for (i = 0; i < nr; i++) { 3805dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 3806db94535dSChris Mason 380731840ae1SZheng Yan if (!empty && push_items > 0) { 380831840ae1SZheng Yan if (path->slots[0] < i) 380931840ae1SZheng Yan break; 381031840ae1SZheng Yan if (path->slots[0] == i) { 38112ff7e61eSJeff Mahoney int space = btrfs_leaf_free_space(fs_info, right); 381231840ae1SZheng Yan if (space + push_space * 2 > free_space) 381331840ae1SZheng Yan break; 381431840ae1SZheng Yan } 381531840ae1SZheng Yan } 381631840ae1SZheng Yan 3817be0e5c09SChris Mason if (path->slots[0] == i) 381887b29b20SYan Zheng push_space += data_size; 3819db94535dSChris Mason 3820db94535dSChris Mason this_item_size = btrfs_item_size(right, item); 3821db94535dSChris Mason if (this_item_size + sizeof(*item) + push_space > free_space) 3822be0e5c09SChris Mason break; 3823db94535dSChris Mason 3824be0e5c09SChris Mason push_items++; 3825db94535dSChris Mason push_space += this_item_size + sizeof(*item); 3826be0e5c09SChris Mason } 3827db94535dSChris Mason 3828be0e5c09SChris Mason if (push_items == 0) { 3829925baeddSChris Mason ret = 1; 3830925baeddSChris Mason goto out; 3831be0e5c09SChris Mason } 3832fae7f21cSDulshani Gunawardhana WARN_ON(!empty && push_items == btrfs_header_nritems(right)); 38335f39d397SChris Mason 3834be0e5c09SChris Mason /* push data from right to left */ 38355f39d397SChris Mason copy_extent_buffer(left, right, 38365f39d397SChris Mason btrfs_item_nr_offset(btrfs_header_nritems(left)), 38375f39d397SChris Mason btrfs_item_nr_offset(0), 38385f39d397SChris Mason push_items * sizeof(struct btrfs_item)); 38395f39d397SChris Mason 38400b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info) - 38415f39d397SChris Mason btrfs_item_offset_nr(right, push_items - 1); 38425f39d397SChris Mason 38433d9ec8c4SNikolay Borisov copy_extent_buffer(left, right, BTRFS_LEAF_DATA_OFFSET + 38442ff7e61eSJeff Mahoney leaf_data_end(fs_info, left) - push_space, 38453d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + 38465f39d397SChris Mason btrfs_item_offset_nr(right, push_items - 1), 3847be0e5c09SChris Mason push_space); 38485f39d397SChris Mason old_left_nritems = btrfs_header_nritems(left); 384987b29b20SYan Zheng BUG_ON(old_left_nritems <= 0); 3850eb60ceacSChris Mason 3851db94535dSChris Mason old_left_item_size = btrfs_item_offset_nr(left, old_left_nritems - 1); 3852be0e5c09SChris Mason for (i = old_left_nritems; i < old_left_nritems + push_items; i++) { 38535f39d397SChris Mason u32 ioff; 3854db94535dSChris Mason 3855dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 3856db94535dSChris Mason 3857cfed81a0SChris Mason ioff = btrfs_token_item_offset(left, item, &token); 3858cfed81a0SChris Mason btrfs_set_token_item_offset(left, item, 38590b246afaSJeff Mahoney ioff - (BTRFS_LEAF_DATA_SIZE(fs_info) - old_left_item_size), 3860cfed81a0SChris Mason &token); 3861be0e5c09SChris Mason } 38625f39d397SChris Mason btrfs_set_header_nritems(left, old_left_nritems + push_items); 3863be0e5c09SChris Mason 3864be0e5c09SChris Mason /* fixup right node */ 386531b1a2bdSJulia Lawall if (push_items > right_nritems) 386631b1a2bdSJulia Lawall WARN(1, KERN_CRIT "push items %d nr %u\n", push_items, 3867d397712bSChris Mason right_nritems); 386834a38218SChris Mason 386934a38218SChris Mason if (push_items < right_nritems) { 38705f39d397SChris Mason push_space = btrfs_item_offset_nr(right, push_items - 1) - 38712ff7e61eSJeff Mahoney leaf_data_end(fs_info, right); 38723d9ec8c4SNikolay Borisov memmove_extent_buffer(right, BTRFS_LEAF_DATA_OFFSET + 38730b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info) - push_space, 38743d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + 38752ff7e61eSJeff Mahoney leaf_data_end(fs_info, right), push_space); 38765f39d397SChris Mason 38775f39d397SChris Mason memmove_extent_buffer(right, btrfs_item_nr_offset(0), 38785f39d397SChris Mason btrfs_item_nr_offset(push_items), 38795f39d397SChris Mason (btrfs_header_nritems(right) - push_items) * 38800783fcfcSChris Mason sizeof(struct btrfs_item)); 388134a38218SChris Mason } 3882eef1c494SYan right_nritems -= push_items; 3883eef1c494SYan btrfs_set_header_nritems(right, right_nritems); 38840b246afaSJeff Mahoney push_space = BTRFS_LEAF_DATA_SIZE(fs_info); 38855f39d397SChris Mason for (i = 0; i < right_nritems; i++) { 3886dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 3887db94535dSChris Mason 3888cfed81a0SChris Mason push_space = push_space - btrfs_token_item_size(right, 3889cfed81a0SChris Mason item, &token); 3890cfed81a0SChris Mason btrfs_set_token_item_offset(right, item, push_space, &token); 3891db94535dSChris Mason } 3892eb60ceacSChris Mason 38935f39d397SChris Mason btrfs_mark_buffer_dirty(left); 389434a38218SChris Mason if (right_nritems) 38955f39d397SChris Mason btrfs_mark_buffer_dirty(right); 3896f0486c68SYan, Zheng else 38977c302b49SDavid Sterba clean_tree_block(fs_info, right); 3898098f59c2SChris Mason 38995f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 3900b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 3901be0e5c09SChris Mason 3902be0e5c09SChris Mason /* then fixup the leaf pointer in the path */ 3903be0e5c09SChris Mason if (path->slots[0] < push_items) { 3904be0e5c09SChris Mason path->slots[0] += old_left_nritems; 3905925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 39065f39d397SChris Mason free_extent_buffer(path->nodes[0]); 39075f39d397SChris Mason path->nodes[0] = left; 3908be0e5c09SChris Mason path->slots[1] -= 1; 3909be0e5c09SChris Mason } else { 3910925baeddSChris Mason btrfs_tree_unlock(left); 39115f39d397SChris Mason free_extent_buffer(left); 3912be0e5c09SChris Mason path->slots[0] -= push_items; 3913be0e5c09SChris Mason } 3914eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 3915aa5d6bedSChris Mason return ret; 3916925baeddSChris Mason out: 3917925baeddSChris Mason btrfs_tree_unlock(left); 3918925baeddSChris Mason free_extent_buffer(left); 3919925baeddSChris Mason return ret; 3920be0e5c09SChris Mason } 3921be0e5c09SChris Mason 392274123bd7SChris Mason /* 392344871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 392444871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 392599d8f83cSChris Mason * 392699d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 392799d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us push all the 392899d8f83cSChris Mason * items 392944871b1bSChris Mason */ 393044871b1bSChris Mason static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root 393199d8f83cSChris Mason *root, struct btrfs_path *path, int min_data_size, 393299d8f83cSChris Mason int data_size, int empty, u32 max_slot) 393344871b1bSChris Mason { 39342ff7e61eSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 393544871b1bSChris Mason struct extent_buffer *right = path->nodes[0]; 393644871b1bSChris Mason struct extent_buffer *left; 393744871b1bSChris Mason int slot; 393844871b1bSChris Mason int free_space; 393944871b1bSChris Mason u32 right_nritems; 394044871b1bSChris Mason int ret = 0; 394144871b1bSChris Mason 394244871b1bSChris Mason slot = path->slots[1]; 394344871b1bSChris Mason if (slot == 0) 394444871b1bSChris Mason return 1; 394544871b1bSChris Mason if (!path->nodes[1]) 394644871b1bSChris Mason return 1; 394744871b1bSChris Mason 394844871b1bSChris Mason right_nritems = btrfs_header_nritems(right); 394944871b1bSChris Mason if (right_nritems == 0) 395044871b1bSChris Mason return 1; 395144871b1bSChris Mason 395244871b1bSChris Mason btrfs_assert_tree_locked(path->nodes[1]); 395344871b1bSChris Mason 39542ff7e61eSJeff Mahoney left = read_node_slot(fs_info, path->nodes[1], slot - 1); 3955fb770ae4SLiu Bo /* 3956fb770ae4SLiu Bo * slot - 1 is not valid or we fail to read the left node, 3957fb770ae4SLiu Bo * no big deal, just return. 3958fb770ae4SLiu Bo */ 3959fb770ae4SLiu Bo if (IS_ERR(left)) 396091ca338dSTsutomu Itoh return 1; 396191ca338dSTsutomu Itoh 396244871b1bSChris Mason btrfs_tree_lock(left); 396344871b1bSChris Mason btrfs_set_lock_blocking(left); 396444871b1bSChris Mason 39652ff7e61eSJeff Mahoney free_space = btrfs_leaf_free_space(fs_info, left); 396644871b1bSChris Mason if (free_space < data_size) { 396744871b1bSChris Mason ret = 1; 396844871b1bSChris Mason goto out; 396944871b1bSChris Mason } 397044871b1bSChris Mason 397144871b1bSChris Mason /* cow and double check */ 397244871b1bSChris Mason ret = btrfs_cow_block(trans, root, left, 397344871b1bSChris Mason path->nodes[1], slot - 1, &left); 397444871b1bSChris Mason if (ret) { 397544871b1bSChris Mason /* we hit -ENOSPC, but it isn't fatal here */ 397679787eaaSJeff Mahoney if (ret == -ENOSPC) 397744871b1bSChris Mason ret = 1; 397844871b1bSChris Mason goto out; 397944871b1bSChris Mason } 398044871b1bSChris Mason 39812ff7e61eSJeff Mahoney free_space = btrfs_leaf_free_space(fs_info, left); 398244871b1bSChris Mason if (free_space < data_size) { 398344871b1bSChris Mason ret = 1; 398444871b1bSChris Mason goto out; 398544871b1bSChris Mason } 398644871b1bSChris Mason 398766cb7ddbSDavid Sterba return __push_leaf_left(fs_info, path, min_data_size, 398899d8f83cSChris Mason empty, left, free_space, right_nritems, 398999d8f83cSChris Mason max_slot); 399044871b1bSChris Mason out: 399144871b1bSChris Mason btrfs_tree_unlock(left); 399244871b1bSChris Mason free_extent_buffer(left); 399344871b1bSChris Mason return ret; 399444871b1bSChris Mason } 399544871b1bSChris Mason 399644871b1bSChris Mason /* 399774123bd7SChris Mason * split the path's leaf in two, making sure there is at least data_size 399874123bd7SChris Mason * available for the resulting leaf level of the path. 399974123bd7SChris Mason */ 4000143bede5SJeff Mahoney static noinline void copy_for_split(struct btrfs_trans_handle *trans, 40012ff7e61eSJeff Mahoney struct btrfs_fs_info *fs_info, 400244871b1bSChris Mason struct btrfs_path *path, 400344871b1bSChris Mason struct extent_buffer *l, 400444871b1bSChris Mason struct extent_buffer *right, 400544871b1bSChris Mason int slot, int mid, int nritems) 4006be0e5c09SChris Mason { 4007be0e5c09SChris Mason int data_copy_size; 4008be0e5c09SChris Mason int rt_data_off; 4009be0e5c09SChris Mason int i; 4010d4dbff95SChris Mason struct btrfs_disk_key disk_key; 4011cfed81a0SChris Mason struct btrfs_map_token token; 4012cfed81a0SChris Mason 4013cfed81a0SChris Mason btrfs_init_map_token(&token); 4014be0e5c09SChris Mason 40155f39d397SChris Mason nritems = nritems - mid; 40165f39d397SChris Mason btrfs_set_header_nritems(right, nritems); 40172ff7e61eSJeff Mahoney data_copy_size = btrfs_item_end_nr(l, mid) - leaf_data_end(fs_info, l); 40185f39d397SChris Mason 40195f39d397SChris Mason copy_extent_buffer(right, l, btrfs_item_nr_offset(0), 40205f39d397SChris Mason btrfs_item_nr_offset(mid), 40215f39d397SChris Mason nritems * sizeof(struct btrfs_item)); 40225f39d397SChris Mason 40235f39d397SChris Mason copy_extent_buffer(right, l, 40243d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + BTRFS_LEAF_DATA_SIZE(fs_info) - 40253d9ec8c4SNikolay Borisov data_copy_size, BTRFS_LEAF_DATA_OFFSET + 40262ff7e61eSJeff Mahoney leaf_data_end(fs_info, l), data_copy_size); 402774123bd7SChris Mason 40280b246afaSJeff Mahoney rt_data_off = BTRFS_LEAF_DATA_SIZE(fs_info) - btrfs_item_end_nr(l, mid); 40295f39d397SChris Mason 40305f39d397SChris Mason for (i = 0; i < nritems; i++) { 4031dd3cc16bSRoss Kirk struct btrfs_item *item = btrfs_item_nr(i); 4032db94535dSChris Mason u32 ioff; 4033db94535dSChris Mason 4034cfed81a0SChris Mason ioff = btrfs_token_item_offset(right, item, &token); 4035cfed81a0SChris Mason btrfs_set_token_item_offset(right, item, 4036cfed81a0SChris Mason ioff + rt_data_off, &token); 40370783fcfcSChris Mason } 403874123bd7SChris Mason 40395f39d397SChris Mason btrfs_set_header_nritems(l, mid); 40405f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 40412ff7e61eSJeff Mahoney insert_ptr(trans, fs_info, path, &disk_key, right->start, 4042c3e06965SJan Schmidt path->slots[1] + 1, 1); 40435f39d397SChris Mason 40445f39d397SChris Mason btrfs_mark_buffer_dirty(right); 40455f39d397SChris Mason btrfs_mark_buffer_dirty(l); 4046eb60ceacSChris Mason BUG_ON(path->slots[0] != slot); 40475f39d397SChris Mason 4048be0e5c09SChris Mason if (mid <= slot) { 4049925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 40505f39d397SChris Mason free_extent_buffer(path->nodes[0]); 40515f39d397SChris Mason path->nodes[0] = right; 4052be0e5c09SChris Mason path->slots[0] -= mid; 4053be0e5c09SChris Mason path->slots[1] += 1; 4054925baeddSChris Mason } else { 4055925baeddSChris Mason btrfs_tree_unlock(right); 40565f39d397SChris Mason free_extent_buffer(right); 4057925baeddSChris Mason } 40585f39d397SChris Mason 4059eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 406044871b1bSChris Mason } 406144871b1bSChris Mason 406244871b1bSChris Mason /* 406399d8f83cSChris Mason * double splits happen when we need to insert a big item in the middle 406499d8f83cSChris Mason * of a leaf. A double split can leave us with 3 mostly empty leaves: 406599d8f83cSChris Mason * leaf: [ slots 0 - N] [ our target ] [ N + 1 - total in leaf ] 406699d8f83cSChris Mason * A B C 406799d8f83cSChris Mason * 406899d8f83cSChris Mason * We avoid this by trying to push the items on either side of our target 406999d8f83cSChris Mason * into the adjacent leaves. If all goes well we can avoid the double split 407099d8f83cSChris Mason * completely. 407199d8f83cSChris Mason */ 407299d8f83cSChris Mason static noinline int push_for_double_split(struct btrfs_trans_handle *trans, 407399d8f83cSChris Mason struct btrfs_root *root, 407499d8f83cSChris Mason struct btrfs_path *path, 407599d8f83cSChris Mason int data_size) 407699d8f83cSChris Mason { 40772ff7e61eSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 407899d8f83cSChris Mason int ret; 407999d8f83cSChris Mason int progress = 0; 408099d8f83cSChris Mason int slot; 408199d8f83cSChris Mason u32 nritems; 40825a4267caSFilipe David Borba Manana int space_needed = data_size; 408399d8f83cSChris Mason 408499d8f83cSChris Mason slot = path->slots[0]; 40855a4267caSFilipe David Borba Manana if (slot < btrfs_header_nritems(path->nodes[0])) 40862ff7e61eSJeff Mahoney space_needed -= btrfs_leaf_free_space(fs_info, path->nodes[0]); 408799d8f83cSChris Mason 408899d8f83cSChris Mason /* 408999d8f83cSChris Mason * try to push all the items after our slot into the 409099d8f83cSChris Mason * right leaf 409199d8f83cSChris Mason */ 40925a4267caSFilipe David Borba Manana ret = push_leaf_right(trans, root, path, 1, space_needed, 0, slot); 409399d8f83cSChris Mason if (ret < 0) 409499d8f83cSChris Mason return ret; 409599d8f83cSChris Mason 409699d8f83cSChris Mason if (ret == 0) 409799d8f83cSChris Mason progress++; 409899d8f83cSChris Mason 409999d8f83cSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 410099d8f83cSChris Mason /* 410199d8f83cSChris Mason * our goal is to get our slot at the start or end of a leaf. If 410299d8f83cSChris Mason * we've done so we're done 410399d8f83cSChris Mason */ 410499d8f83cSChris Mason if (path->slots[0] == 0 || path->slots[0] == nritems) 410599d8f83cSChris Mason return 0; 410699d8f83cSChris Mason 41072ff7e61eSJeff Mahoney if (btrfs_leaf_free_space(fs_info, path->nodes[0]) >= data_size) 410899d8f83cSChris Mason return 0; 410999d8f83cSChris Mason 411099d8f83cSChris Mason /* try to push all the items before our slot into the next leaf */ 411199d8f83cSChris Mason slot = path->slots[0]; 4112263d3995SFilipe Manana space_needed = data_size; 4113263d3995SFilipe Manana if (slot > 0) 4114263d3995SFilipe Manana space_needed -= btrfs_leaf_free_space(fs_info, path->nodes[0]); 41155a4267caSFilipe David Borba Manana ret = push_leaf_left(trans, root, path, 1, space_needed, 0, slot); 411699d8f83cSChris Mason if (ret < 0) 411799d8f83cSChris Mason return ret; 411899d8f83cSChris Mason 411999d8f83cSChris Mason if (ret == 0) 412099d8f83cSChris Mason progress++; 412199d8f83cSChris Mason 412299d8f83cSChris Mason if (progress) 412399d8f83cSChris Mason return 0; 412499d8f83cSChris Mason return 1; 412599d8f83cSChris Mason } 412699d8f83cSChris Mason 412799d8f83cSChris Mason /* 412844871b1bSChris Mason * split the path's leaf in two, making sure there is at least data_size 412944871b1bSChris Mason * available for the resulting leaf level of the path. 413044871b1bSChris Mason * 413144871b1bSChris Mason * returns 0 if all went well and < 0 on failure. 413244871b1bSChris Mason */ 413344871b1bSChris Mason static noinline int split_leaf(struct btrfs_trans_handle *trans, 413444871b1bSChris Mason struct btrfs_root *root, 4135310712b2SOmar Sandoval const struct btrfs_key *ins_key, 413644871b1bSChris Mason struct btrfs_path *path, int data_size, 413744871b1bSChris Mason int extend) 413844871b1bSChris Mason { 41395d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 414044871b1bSChris Mason struct extent_buffer *l; 414144871b1bSChris Mason u32 nritems; 414244871b1bSChris Mason int mid; 414344871b1bSChris Mason int slot; 414444871b1bSChris Mason struct extent_buffer *right; 4145b7a0365eSDaniel Dressler struct btrfs_fs_info *fs_info = root->fs_info; 414644871b1bSChris Mason int ret = 0; 414744871b1bSChris Mason int wret; 41485d4f98a2SYan Zheng int split; 414944871b1bSChris Mason int num_doubles = 0; 415099d8f83cSChris Mason int tried_avoid_double = 0; 415144871b1bSChris Mason 4152a5719521SYan, Zheng l = path->nodes[0]; 4153a5719521SYan, Zheng slot = path->slots[0]; 4154a5719521SYan, Zheng if (extend && data_size + btrfs_item_size_nr(l, slot) + 41550b246afaSJeff Mahoney sizeof(struct btrfs_item) > BTRFS_LEAF_DATA_SIZE(fs_info)) 4156a5719521SYan, Zheng return -EOVERFLOW; 4157a5719521SYan, Zheng 415844871b1bSChris Mason /* first try to make some room by pushing left and right */ 415933157e05SLiu Bo if (data_size && path->nodes[1]) { 41605a4267caSFilipe David Borba Manana int space_needed = data_size; 41615a4267caSFilipe David Borba Manana 41625a4267caSFilipe David Borba Manana if (slot < btrfs_header_nritems(l)) 41632ff7e61eSJeff Mahoney space_needed -= btrfs_leaf_free_space(fs_info, l); 41645a4267caSFilipe David Borba Manana 41655a4267caSFilipe David Borba Manana wret = push_leaf_right(trans, root, path, space_needed, 41665a4267caSFilipe David Borba Manana space_needed, 0, 0); 416744871b1bSChris Mason if (wret < 0) 416844871b1bSChris Mason return wret; 416944871b1bSChris Mason if (wret) { 4170263d3995SFilipe Manana space_needed = data_size; 4171263d3995SFilipe Manana if (slot > 0) 4172263d3995SFilipe Manana space_needed -= btrfs_leaf_free_space(fs_info, 4173263d3995SFilipe Manana l); 41745a4267caSFilipe David Borba Manana wret = push_leaf_left(trans, root, path, space_needed, 41755a4267caSFilipe David Borba Manana space_needed, 0, (u32)-1); 417644871b1bSChris Mason if (wret < 0) 417744871b1bSChris Mason return wret; 417844871b1bSChris Mason } 417944871b1bSChris Mason l = path->nodes[0]; 418044871b1bSChris Mason 418144871b1bSChris Mason /* did the pushes work? */ 41822ff7e61eSJeff Mahoney if (btrfs_leaf_free_space(fs_info, l) >= data_size) 418344871b1bSChris Mason return 0; 418444871b1bSChris Mason } 418544871b1bSChris Mason 418644871b1bSChris Mason if (!path->nodes[1]) { 4187fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, 1); 418844871b1bSChris Mason if (ret) 418944871b1bSChris Mason return ret; 419044871b1bSChris Mason } 419144871b1bSChris Mason again: 41925d4f98a2SYan Zheng split = 1; 419344871b1bSChris Mason l = path->nodes[0]; 419444871b1bSChris Mason slot = path->slots[0]; 419544871b1bSChris Mason nritems = btrfs_header_nritems(l); 419644871b1bSChris Mason mid = (nritems + 1) / 2; 419744871b1bSChris Mason 41985d4f98a2SYan Zheng if (mid <= slot) { 41995d4f98a2SYan Zheng if (nritems == 1 || 42005d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + data_size > 42010b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info)) { 42025d4f98a2SYan Zheng if (slot >= nritems) { 42035d4f98a2SYan Zheng split = 0; 42045d4f98a2SYan Zheng } else { 42055d4f98a2SYan Zheng mid = slot; 42065d4f98a2SYan Zheng if (mid != nritems && 42075d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 42080b246afaSJeff Mahoney data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) { 420999d8f83cSChris Mason if (data_size && !tried_avoid_double) 421099d8f83cSChris Mason goto push_for_double; 42115d4f98a2SYan Zheng split = 2; 42125d4f98a2SYan Zheng } 42135d4f98a2SYan Zheng } 42145d4f98a2SYan Zheng } 42155d4f98a2SYan Zheng } else { 42165d4f98a2SYan Zheng if (leaf_space_used(l, 0, mid) + data_size > 42170b246afaSJeff Mahoney BTRFS_LEAF_DATA_SIZE(fs_info)) { 42185d4f98a2SYan Zheng if (!extend && data_size && slot == 0) { 42195d4f98a2SYan Zheng split = 0; 42205d4f98a2SYan Zheng } else if ((extend || !data_size) && slot == 0) { 42215d4f98a2SYan Zheng mid = 1; 42225d4f98a2SYan Zheng } else { 42235d4f98a2SYan Zheng mid = slot; 42245d4f98a2SYan Zheng if (mid != nritems && 42255d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 42260b246afaSJeff Mahoney data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) { 422799d8f83cSChris Mason if (data_size && !tried_avoid_double) 422899d8f83cSChris Mason goto push_for_double; 42295d4f98a2SYan Zheng split = 2; 42305d4f98a2SYan Zheng } 42315d4f98a2SYan Zheng } 42325d4f98a2SYan Zheng } 42335d4f98a2SYan Zheng } 42345d4f98a2SYan Zheng 42355d4f98a2SYan Zheng if (split == 0) 42365d4f98a2SYan Zheng btrfs_cpu_key_to_disk(&disk_key, ins_key); 42375d4f98a2SYan Zheng else 42385d4f98a2SYan Zheng btrfs_item_key(l, &disk_key, mid); 42395d4f98a2SYan Zheng 42404d75f8a9SDavid Sterba right = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid, 42415581a51aSJan Schmidt &disk_key, 0, l->start, 0); 4242f0486c68SYan, Zheng if (IS_ERR(right)) 424344871b1bSChris Mason return PTR_ERR(right); 4244f0486c68SYan, Zheng 42450b246afaSJeff Mahoney root_add_used(root, fs_info->nodesize); 424644871b1bSChris Mason 42475d4f98a2SYan Zheng if (split == 0) { 424844871b1bSChris Mason if (mid <= slot) { 424944871b1bSChris Mason btrfs_set_header_nritems(right, 0); 42502ff7e61eSJeff Mahoney insert_ptr(trans, fs_info, path, &disk_key, 42512ff7e61eSJeff Mahoney right->start, path->slots[1] + 1, 1); 425244871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 425344871b1bSChris Mason free_extent_buffer(path->nodes[0]); 425444871b1bSChris Mason path->nodes[0] = right; 425544871b1bSChris Mason path->slots[0] = 0; 425644871b1bSChris Mason path->slots[1] += 1; 425744871b1bSChris Mason } else { 425844871b1bSChris Mason btrfs_set_header_nritems(right, 0); 42592ff7e61eSJeff Mahoney insert_ptr(trans, fs_info, path, &disk_key, 42602ff7e61eSJeff Mahoney right->start, path->slots[1], 1); 426144871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 426244871b1bSChris Mason free_extent_buffer(path->nodes[0]); 426344871b1bSChris Mason path->nodes[0] = right; 426444871b1bSChris Mason path->slots[0] = 0; 4265143bede5SJeff Mahoney if (path->slots[1] == 0) 4266b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 42675d4f98a2SYan Zheng } 4268196e0249SLiu Bo /* 4269196e0249SLiu Bo * We create a new leaf 'right' for the required ins_len and 4270196e0249SLiu Bo * we'll do btrfs_mark_buffer_dirty() on this leaf after copying 4271196e0249SLiu Bo * the content of ins_len to 'right'. 4272196e0249SLiu Bo */ 427344871b1bSChris Mason return ret; 427444871b1bSChris Mason } 427544871b1bSChris Mason 42762ff7e61eSJeff Mahoney copy_for_split(trans, fs_info, path, l, right, slot, mid, nritems); 427744871b1bSChris Mason 42785d4f98a2SYan Zheng if (split == 2) { 4279cc0c5538SChris Mason BUG_ON(num_doubles != 0); 4280cc0c5538SChris Mason num_doubles++; 4281cc0c5538SChris Mason goto again; 42823326d1b0SChris Mason } 428344871b1bSChris Mason 4284143bede5SJeff Mahoney return 0; 428599d8f83cSChris Mason 428699d8f83cSChris Mason push_for_double: 428799d8f83cSChris Mason push_for_double_split(trans, root, path, data_size); 428899d8f83cSChris Mason tried_avoid_double = 1; 42892ff7e61eSJeff Mahoney if (btrfs_leaf_free_space(fs_info, path->nodes[0]) >= data_size) 429099d8f83cSChris Mason return 0; 429199d8f83cSChris Mason goto again; 4292be0e5c09SChris Mason } 4293be0e5c09SChris Mason 4294ad48fd75SYan, Zheng static noinline int setup_leaf_for_split(struct btrfs_trans_handle *trans, 4295ad48fd75SYan, Zheng struct btrfs_root *root, 4296ad48fd75SYan, Zheng struct btrfs_path *path, int ins_len) 4297ad48fd75SYan, Zheng { 42982ff7e61eSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 4299ad48fd75SYan, Zheng struct btrfs_key key; 4300ad48fd75SYan, Zheng struct extent_buffer *leaf; 4301ad48fd75SYan, Zheng struct btrfs_file_extent_item *fi; 4302ad48fd75SYan, Zheng u64 extent_len = 0; 4303ad48fd75SYan, Zheng u32 item_size; 4304ad48fd75SYan, Zheng int ret; 4305ad48fd75SYan, Zheng 4306ad48fd75SYan, Zheng leaf = path->nodes[0]; 4307ad48fd75SYan, Zheng btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 4308ad48fd75SYan, Zheng 4309ad48fd75SYan, Zheng BUG_ON(key.type != BTRFS_EXTENT_DATA_KEY && 4310ad48fd75SYan, Zheng key.type != BTRFS_EXTENT_CSUM_KEY); 4311ad48fd75SYan, Zheng 43122ff7e61eSJeff Mahoney if (btrfs_leaf_free_space(fs_info, leaf) >= ins_len) 4313ad48fd75SYan, Zheng return 0; 4314ad48fd75SYan, Zheng 4315ad48fd75SYan, Zheng item_size = btrfs_item_size_nr(leaf, path->slots[0]); 4316ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 4317ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 4318ad48fd75SYan, Zheng struct btrfs_file_extent_item); 4319ad48fd75SYan, Zheng extent_len = btrfs_file_extent_num_bytes(leaf, fi); 4320ad48fd75SYan, Zheng } 4321b3b4aa74SDavid Sterba btrfs_release_path(path); 4322ad48fd75SYan, Zheng 4323ad48fd75SYan, Zheng path->keep_locks = 1; 4324ad48fd75SYan, Zheng path->search_for_split = 1; 4325ad48fd75SYan, Zheng ret = btrfs_search_slot(trans, root, &key, path, 0, 1); 4326ad48fd75SYan, Zheng path->search_for_split = 0; 4327a8df6fe6SFilipe Manana if (ret > 0) 4328a8df6fe6SFilipe Manana ret = -EAGAIN; 4329ad48fd75SYan, Zheng if (ret < 0) 4330ad48fd75SYan, Zheng goto err; 4331ad48fd75SYan, Zheng 4332ad48fd75SYan, Zheng ret = -EAGAIN; 4333ad48fd75SYan, Zheng leaf = path->nodes[0]; 4334a8df6fe6SFilipe Manana /* if our item isn't there, return now */ 4335a8df6fe6SFilipe Manana if (item_size != btrfs_item_size_nr(leaf, path->slots[0])) 4336ad48fd75SYan, Zheng goto err; 4337ad48fd75SYan, Zheng 4338109f6aefSChris Mason /* the leaf has changed, it now has room. return now */ 43392ff7e61eSJeff Mahoney if (btrfs_leaf_free_space(fs_info, path->nodes[0]) >= ins_len) 4340109f6aefSChris Mason goto err; 4341109f6aefSChris Mason 4342ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 4343ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 4344ad48fd75SYan, Zheng struct btrfs_file_extent_item); 4345ad48fd75SYan, Zheng if (extent_len != btrfs_file_extent_num_bytes(leaf, fi)) 4346ad48fd75SYan, Zheng goto err; 4347ad48fd75SYan, Zheng } 4348ad48fd75SYan, Zheng 4349ad48fd75SYan, Zheng btrfs_set_path_blocking(path); 4350ad48fd75SYan, Zheng ret = split_leaf(trans, root, &key, path, ins_len, 1); 4351f0486c68SYan, Zheng if (ret) 4352f0486c68SYan, Zheng goto err; 4353ad48fd75SYan, Zheng 4354ad48fd75SYan, Zheng path->keep_locks = 0; 4355ad48fd75SYan, Zheng btrfs_unlock_up_safe(path, 1); 4356ad48fd75SYan, Zheng return 0; 4357ad48fd75SYan, Zheng err: 4358ad48fd75SYan, Zheng path->keep_locks = 0; 4359ad48fd75SYan, Zheng return ret; 4360ad48fd75SYan, Zheng } 4361ad48fd75SYan, Zheng 43624961e293SDavid Sterba static noinline int split_item(struct btrfs_fs_info *fs_info, 4363459931ecSChris Mason struct btrfs_path *path, 4364310712b2SOmar Sandoval const struct btrfs_key *new_key, 4365459931ecSChris Mason unsigned long split_offset) 4366459931ecSChris Mason { 4367459931ecSChris Mason struct extent_buffer *leaf; 4368459931ecSChris Mason struct btrfs_item *item; 4369459931ecSChris Mason struct btrfs_item *new_item; 4370459931ecSChris Mason int slot; 4371ad48fd75SYan, Zheng char *buf; 4372459931ecSChris Mason u32 nritems; 4373ad48fd75SYan, Zheng u32 item_size; 4374459931ecSChris Mason u32 orig_offset; 4375459931ecSChris Mason struct btrfs_disk_key disk_key; 4376459931ecSChris Mason 4377459931ecSChris Mason leaf = path->nodes[0]; 43782ff7e61eSJeff Mahoney BUG_ON(btrfs_leaf_free_space(fs_info, leaf) < sizeof(struct btrfs_item)); 4379b9473439SChris Mason 4380b4ce94deSChris Mason btrfs_set_path_blocking(path); 4381b4ce94deSChris Mason 4382dd3cc16bSRoss Kirk item = btrfs_item_nr(path->slots[0]); 4383459931ecSChris Mason orig_offset = btrfs_item_offset(leaf, item); 4384459931ecSChris Mason item_size = btrfs_item_size(leaf, item); 4385459931ecSChris Mason 4386459931ecSChris Mason buf = kmalloc(item_size, GFP_NOFS); 4387ad48fd75SYan, Zheng if (!buf) 4388ad48fd75SYan, Zheng return -ENOMEM; 4389ad48fd75SYan, Zheng 4390459931ecSChris Mason read_extent_buffer(leaf, buf, btrfs_item_ptr_offset(leaf, 4391459931ecSChris Mason path->slots[0]), item_size); 4392ad48fd75SYan, Zheng 4393459931ecSChris Mason slot = path->slots[0] + 1; 4394459931ecSChris Mason nritems = btrfs_header_nritems(leaf); 4395459931ecSChris Mason if (slot != nritems) { 4396459931ecSChris Mason /* shift the items */ 4397459931ecSChris Mason memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + 1), 4398459931ecSChris Mason btrfs_item_nr_offset(slot), 4399459931ecSChris Mason (nritems - slot) * sizeof(struct btrfs_item)); 4400459931ecSChris Mason } 4401459931ecSChris Mason 4402459931ecSChris Mason btrfs_cpu_key_to_disk(&disk_key, new_key); 4403459931ecSChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 4404459931ecSChris Mason 4405dd3cc16bSRoss Kirk new_item = btrfs_item_nr(slot); 4406459931ecSChris Mason 4407459931ecSChris Mason btrfs_set_item_offset(leaf, new_item, orig_offset); 4408459931ecSChris Mason btrfs_set_item_size(leaf, new_item, item_size - split_offset); 4409459931ecSChris Mason 4410459931ecSChris Mason btrfs_set_item_offset(leaf, item, 4411459931ecSChris Mason orig_offset + item_size - split_offset); 4412459931ecSChris Mason btrfs_set_item_size(leaf, item, split_offset); 4413459931ecSChris Mason 4414459931ecSChris Mason btrfs_set_header_nritems(leaf, nritems + 1); 4415459931ecSChris Mason 4416459931ecSChris Mason /* write the data for the start of the original item */ 4417459931ecSChris Mason write_extent_buffer(leaf, buf, 4418459931ecSChris Mason btrfs_item_ptr_offset(leaf, path->slots[0]), 4419459931ecSChris Mason split_offset); 4420459931ecSChris Mason 4421459931ecSChris Mason /* write the data for the new item */ 4422459931ecSChris Mason write_extent_buffer(leaf, buf + split_offset, 4423459931ecSChris Mason btrfs_item_ptr_offset(leaf, slot), 4424459931ecSChris Mason item_size - split_offset); 4425459931ecSChris Mason btrfs_mark_buffer_dirty(leaf); 4426459931ecSChris Mason 44272ff7e61eSJeff Mahoney BUG_ON(btrfs_leaf_free_space(fs_info, leaf) < 0); 4428459931ecSChris Mason kfree(buf); 4429ad48fd75SYan, Zheng return 0; 4430ad48fd75SYan, Zheng } 4431ad48fd75SYan, Zheng 4432ad48fd75SYan, Zheng /* 4433ad48fd75SYan, Zheng * This function splits a single item into two items, 4434ad48fd75SYan, Zheng * giving 'new_key' to the new item and splitting the 4435ad48fd75SYan, Zheng * old one at split_offset (from the start of the item). 4436ad48fd75SYan, Zheng * 4437ad48fd75SYan, Zheng * The path may be released by this operation. After 4438ad48fd75SYan, Zheng * the split, the path is pointing to the old item. The 4439ad48fd75SYan, Zheng * new item is going to be in the same node as the old one. 4440ad48fd75SYan, Zheng * 4441ad48fd75SYan, Zheng * Note, the item being split must be smaller enough to live alone on 4442ad48fd75SYan, Zheng * a tree block with room for one extra struct btrfs_item 4443ad48fd75SYan, Zheng * 4444ad48fd75SYan, Zheng * This allows us to split the item in place, keeping a lock on the 4445ad48fd75SYan, Zheng * leaf the entire time. 4446ad48fd75SYan, Zheng */ 4447ad48fd75SYan, Zheng int btrfs_split_item(struct btrfs_trans_handle *trans, 4448ad48fd75SYan, Zheng struct btrfs_root *root, 4449ad48fd75SYan, Zheng struct btrfs_path *path, 4450310712b2SOmar Sandoval const struct btrfs_key *new_key, 4451ad48fd75SYan, Zheng unsigned long split_offset) 4452ad48fd75SYan, Zheng { 4453ad48fd75SYan, Zheng int ret; 4454ad48fd75SYan, Zheng ret = setup_leaf_for_split(trans, root, path, 4455ad48fd75SYan, Zheng sizeof(struct btrfs_item)); 4456ad48fd75SYan, Zheng if (ret) 4457459931ecSChris Mason return ret; 4458ad48fd75SYan, Zheng 44594961e293SDavid Sterba ret = split_item(root->fs_info, path, new_key, split_offset); 4460ad48fd75SYan, Zheng return ret; 4461ad48fd75SYan, Zheng } 4462ad48fd75SYan, Zheng 4463ad48fd75SYan, Zheng /* 4464ad48fd75SYan, Zheng * This function duplicate a item, giving 'new_key' to the new item. 4465ad48fd75SYan, Zheng * It guarantees both items live in the same tree leaf and the new item 4466ad48fd75SYan, Zheng * is contiguous with the original item. 4467ad48fd75SYan, Zheng * 4468ad48fd75SYan, Zheng * This allows us to split file extent in place, keeping a lock on the 4469ad48fd75SYan, Zheng * leaf the entire time. 4470ad48fd75SYan, Zheng */ 4471ad48fd75SYan, Zheng int btrfs_duplicate_item(struct btrfs_trans_handle *trans, 4472ad48fd75SYan, Zheng struct btrfs_root *root, 4473ad48fd75SYan, Zheng struct btrfs_path *path, 4474310712b2SOmar Sandoval const struct btrfs_key *new_key) 4475ad48fd75SYan, Zheng { 4476ad48fd75SYan, Zheng struct extent_buffer *leaf; 4477ad48fd75SYan, Zheng int ret; 4478ad48fd75SYan, Zheng u32 item_size; 4479ad48fd75SYan, Zheng 4480ad48fd75SYan, Zheng leaf = path->nodes[0]; 4481ad48fd75SYan, Zheng item_size = btrfs_item_size_nr(leaf, path->slots[0]); 4482ad48fd75SYan, Zheng ret = setup_leaf_for_split(trans, root, path, 4483ad48fd75SYan, Zheng item_size + sizeof(struct btrfs_item)); 4484ad48fd75SYan, Zheng if (ret) 4485ad48fd75SYan, Zheng return ret; 4486ad48fd75SYan, Zheng 4487ad48fd75SYan, Zheng path->slots[0]++; 4488afe5fea7STsutomu Itoh setup_items_for_insert(root, path, new_key, &item_size, 4489ad48fd75SYan, Zheng item_size, item_size + 4490ad48fd75SYan, Zheng sizeof(struct btrfs_item), 1); 4491ad48fd75SYan, Zheng leaf = path->nodes[0]; 4492ad48fd75SYan, Zheng memcpy_extent_buffer(leaf, 4493ad48fd75SYan, Zheng btrfs_item_ptr_offset(leaf, path->slots[0]), 4494ad48fd75SYan, Zheng btrfs_item_ptr_offset(leaf, path->slots[0] - 1), 4495ad48fd75SYan, Zheng item_size); 4496ad48fd75SYan, Zheng return 0; 4497459931ecSChris Mason } 4498459931ecSChris Mason 4499459931ecSChris Mason /* 4500d352ac68SChris Mason * make the item pointed to by the path smaller. new_size indicates 4501d352ac68SChris Mason * how small to make it, and from_end tells us if we just chop bytes 4502d352ac68SChris Mason * off the end of the item or if we shift the item to chop bytes off 4503d352ac68SChris Mason * the front. 4504d352ac68SChris Mason */ 45052ff7e61eSJeff Mahoney void btrfs_truncate_item(struct btrfs_fs_info *fs_info, 45062ff7e61eSJeff Mahoney struct btrfs_path *path, u32 new_size, int from_end) 4507b18c6685SChris Mason { 4508b18c6685SChris Mason int slot; 45095f39d397SChris Mason struct extent_buffer *leaf; 45105f39d397SChris Mason struct btrfs_item *item; 4511b18c6685SChris Mason u32 nritems; 4512b18c6685SChris Mason unsigned int data_end; 4513b18c6685SChris Mason unsigned int old_data_start; 4514b18c6685SChris Mason unsigned int old_size; 4515b18c6685SChris Mason unsigned int size_diff; 4516b18c6685SChris Mason int i; 4517cfed81a0SChris Mason struct btrfs_map_token token; 4518cfed81a0SChris Mason 4519cfed81a0SChris Mason btrfs_init_map_token(&token); 4520b18c6685SChris Mason 45215f39d397SChris Mason leaf = path->nodes[0]; 4522179e29e4SChris Mason slot = path->slots[0]; 4523179e29e4SChris Mason 4524179e29e4SChris Mason old_size = btrfs_item_size_nr(leaf, slot); 4525179e29e4SChris Mason if (old_size == new_size) 4526143bede5SJeff Mahoney return; 4527b18c6685SChris Mason 45285f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 45292ff7e61eSJeff Mahoney data_end = leaf_data_end(fs_info, leaf); 4530b18c6685SChris Mason 45315f39d397SChris Mason old_data_start = btrfs_item_offset_nr(leaf, slot); 4532179e29e4SChris Mason 4533b18c6685SChris Mason size_diff = old_size - new_size; 4534b18c6685SChris Mason 4535b18c6685SChris Mason BUG_ON(slot < 0); 4536b18c6685SChris Mason BUG_ON(slot >= nritems); 4537b18c6685SChris Mason 4538b18c6685SChris Mason /* 4539b18c6685SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 4540b18c6685SChris Mason */ 4541b18c6685SChris Mason /* first correct the data pointers */ 4542b18c6685SChris Mason for (i = slot; i < nritems; i++) { 45435f39d397SChris Mason u32 ioff; 4544dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 4545db94535dSChris Mason 4546cfed81a0SChris Mason ioff = btrfs_token_item_offset(leaf, item, &token); 4547cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 4548cfed81a0SChris Mason ioff + size_diff, &token); 4549b18c6685SChris Mason } 4550db94535dSChris Mason 4551b18c6685SChris Mason /* shift the data */ 4552179e29e4SChris Mason if (from_end) { 45533d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 45543d9ec8c4SNikolay Borisov data_end + size_diff, BTRFS_LEAF_DATA_OFFSET + 4555b18c6685SChris Mason data_end, old_data_start + new_size - data_end); 4556179e29e4SChris Mason } else { 4557179e29e4SChris Mason struct btrfs_disk_key disk_key; 4558179e29e4SChris Mason u64 offset; 4559179e29e4SChris Mason 4560179e29e4SChris Mason btrfs_item_key(leaf, &disk_key, slot); 4561179e29e4SChris Mason 4562179e29e4SChris Mason if (btrfs_disk_key_type(&disk_key) == BTRFS_EXTENT_DATA_KEY) { 4563179e29e4SChris Mason unsigned long ptr; 4564179e29e4SChris Mason struct btrfs_file_extent_item *fi; 4565179e29e4SChris Mason 4566179e29e4SChris Mason fi = btrfs_item_ptr(leaf, slot, 4567179e29e4SChris Mason struct btrfs_file_extent_item); 4568179e29e4SChris Mason fi = (struct btrfs_file_extent_item *)( 4569179e29e4SChris Mason (unsigned long)fi - size_diff); 4570179e29e4SChris Mason 4571179e29e4SChris Mason if (btrfs_file_extent_type(leaf, fi) == 4572179e29e4SChris Mason BTRFS_FILE_EXTENT_INLINE) { 4573179e29e4SChris Mason ptr = btrfs_item_ptr_offset(leaf, slot); 4574179e29e4SChris Mason memmove_extent_buffer(leaf, ptr, 4575179e29e4SChris Mason (unsigned long)fi, 45767ec20afbSDavid Sterba BTRFS_FILE_EXTENT_INLINE_DATA_START); 4577179e29e4SChris Mason } 4578179e29e4SChris Mason } 4579179e29e4SChris Mason 45803d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 45813d9ec8c4SNikolay Borisov data_end + size_diff, BTRFS_LEAF_DATA_OFFSET + 4582179e29e4SChris Mason data_end, old_data_start - data_end); 4583179e29e4SChris Mason 4584179e29e4SChris Mason offset = btrfs_disk_key_offset(&disk_key); 4585179e29e4SChris Mason btrfs_set_disk_key_offset(&disk_key, offset + size_diff); 4586179e29e4SChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 4587179e29e4SChris Mason if (slot == 0) 4588b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 4589179e29e4SChris Mason } 45905f39d397SChris Mason 4591dd3cc16bSRoss Kirk item = btrfs_item_nr(slot); 45925f39d397SChris Mason btrfs_set_item_size(leaf, item, new_size); 45935f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 4594b18c6685SChris Mason 45952ff7e61eSJeff Mahoney if (btrfs_leaf_free_space(fs_info, leaf) < 0) { 4596a4f78750SDavid Sterba btrfs_print_leaf(leaf); 4597b18c6685SChris Mason BUG(); 45985f39d397SChris Mason } 4599b18c6685SChris Mason } 4600b18c6685SChris Mason 4601d352ac68SChris Mason /* 46028f69dbd2SStefan Behrens * make the item pointed to by the path bigger, data_size is the added size. 4603d352ac68SChris Mason */ 46042ff7e61eSJeff Mahoney void btrfs_extend_item(struct btrfs_fs_info *fs_info, struct btrfs_path *path, 46055f39d397SChris Mason u32 data_size) 46066567e837SChris Mason { 46076567e837SChris Mason int slot; 46085f39d397SChris Mason struct extent_buffer *leaf; 46095f39d397SChris Mason struct btrfs_item *item; 46106567e837SChris Mason u32 nritems; 46116567e837SChris Mason unsigned int data_end; 46126567e837SChris Mason unsigned int old_data; 46136567e837SChris Mason unsigned int old_size; 46146567e837SChris Mason int i; 4615cfed81a0SChris Mason struct btrfs_map_token token; 4616cfed81a0SChris Mason 4617cfed81a0SChris Mason btrfs_init_map_token(&token); 46186567e837SChris Mason 46195f39d397SChris Mason leaf = path->nodes[0]; 46206567e837SChris Mason 46215f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 46222ff7e61eSJeff Mahoney data_end = leaf_data_end(fs_info, leaf); 46236567e837SChris Mason 46242ff7e61eSJeff Mahoney if (btrfs_leaf_free_space(fs_info, leaf) < data_size) { 4625a4f78750SDavid Sterba btrfs_print_leaf(leaf); 46266567e837SChris Mason BUG(); 46275f39d397SChris Mason } 46286567e837SChris Mason slot = path->slots[0]; 46295f39d397SChris Mason old_data = btrfs_item_end_nr(leaf, slot); 46306567e837SChris Mason 46316567e837SChris Mason BUG_ON(slot < 0); 46323326d1b0SChris Mason if (slot >= nritems) { 4633a4f78750SDavid Sterba btrfs_print_leaf(leaf); 46340b246afaSJeff Mahoney btrfs_crit(fs_info, "slot %d too large, nritems %d", 4635d397712bSChris Mason slot, nritems); 46363326d1b0SChris Mason BUG_ON(1); 46373326d1b0SChris Mason } 46386567e837SChris Mason 46396567e837SChris Mason /* 46406567e837SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 46416567e837SChris Mason */ 46426567e837SChris Mason /* first correct the data pointers */ 46436567e837SChris Mason for (i = slot; i < nritems; i++) { 46445f39d397SChris Mason u32 ioff; 4645dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 4646db94535dSChris Mason 4647cfed81a0SChris Mason ioff = btrfs_token_item_offset(leaf, item, &token); 4648cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 4649cfed81a0SChris Mason ioff - data_size, &token); 46506567e837SChris Mason } 46515f39d397SChris Mason 46526567e837SChris Mason /* shift the data */ 46533d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 46543d9ec8c4SNikolay Borisov data_end - data_size, BTRFS_LEAF_DATA_OFFSET + 46556567e837SChris Mason data_end, old_data - data_end); 46565f39d397SChris Mason 46576567e837SChris Mason data_end = old_data; 46585f39d397SChris Mason old_size = btrfs_item_size_nr(leaf, slot); 4659dd3cc16bSRoss Kirk item = btrfs_item_nr(slot); 46605f39d397SChris Mason btrfs_set_item_size(leaf, item, old_size + data_size); 46615f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 46626567e837SChris Mason 46632ff7e61eSJeff Mahoney if (btrfs_leaf_free_space(fs_info, leaf) < 0) { 4664a4f78750SDavid Sterba btrfs_print_leaf(leaf); 46656567e837SChris Mason BUG(); 46665f39d397SChris Mason } 46676567e837SChris Mason } 46686567e837SChris Mason 466974123bd7SChris Mason /* 467044871b1bSChris Mason * this is a helper for btrfs_insert_empty_items, the main goal here is 467144871b1bSChris Mason * to save stack depth by doing the bulk of the work in a function 467244871b1bSChris Mason * that doesn't call btrfs_search_slot 467374123bd7SChris Mason */ 4674afe5fea7STsutomu Itoh void setup_items_for_insert(struct btrfs_root *root, struct btrfs_path *path, 4675310712b2SOmar Sandoval const struct btrfs_key *cpu_key, u32 *data_size, 467644871b1bSChris Mason u32 total_data, u32 total_size, int nr) 4677be0e5c09SChris Mason { 46780b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 46795f39d397SChris Mason struct btrfs_item *item; 46809c58309dSChris Mason int i; 46817518a238SChris Mason u32 nritems; 4682be0e5c09SChris Mason unsigned int data_end; 4683e2fa7227SChris Mason struct btrfs_disk_key disk_key; 468444871b1bSChris Mason struct extent_buffer *leaf; 468544871b1bSChris Mason int slot; 4686cfed81a0SChris Mason struct btrfs_map_token token; 4687cfed81a0SChris Mason 468824cdc847SFilipe Manana if (path->slots[0] == 0) { 468924cdc847SFilipe Manana btrfs_cpu_key_to_disk(&disk_key, cpu_key); 4690b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 469124cdc847SFilipe Manana } 469224cdc847SFilipe Manana btrfs_unlock_up_safe(path, 1); 469324cdc847SFilipe Manana 4694cfed81a0SChris Mason btrfs_init_map_token(&token); 4695e2fa7227SChris Mason 46965f39d397SChris Mason leaf = path->nodes[0]; 469744871b1bSChris Mason slot = path->slots[0]; 469874123bd7SChris Mason 46995f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 47002ff7e61eSJeff Mahoney data_end = leaf_data_end(fs_info, leaf); 4701eb60ceacSChris Mason 47022ff7e61eSJeff Mahoney if (btrfs_leaf_free_space(fs_info, leaf) < total_size) { 4703a4f78750SDavid Sterba btrfs_print_leaf(leaf); 47040b246afaSJeff Mahoney btrfs_crit(fs_info, "not enough freespace need %u have %d", 47052ff7e61eSJeff Mahoney total_size, btrfs_leaf_free_space(fs_info, leaf)); 4706be0e5c09SChris Mason BUG(); 4707d4dbff95SChris Mason } 47085f39d397SChris Mason 4709be0e5c09SChris Mason if (slot != nritems) { 47105f39d397SChris Mason unsigned int old_data = btrfs_item_end_nr(leaf, slot); 4711be0e5c09SChris Mason 47125f39d397SChris Mason if (old_data < data_end) { 4713a4f78750SDavid Sterba btrfs_print_leaf(leaf); 47140b246afaSJeff Mahoney btrfs_crit(fs_info, "slot %d old_data %d data_end %d", 47155f39d397SChris Mason slot, old_data, data_end); 47165f39d397SChris Mason BUG_ON(1); 47175f39d397SChris Mason } 4718be0e5c09SChris Mason /* 4719be0e5c09SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 4720be0e5c09SChris Mason */ 4721be0e5c09SChris Mason /* first correct the data pointers */ 47220783fcfcSChris Mason for (i = slot; i < nritems; i++) { 47235f39d397SChris Mason u32 ioff; 4724db94535dSChris Mason 4725dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 4726cfed81a0SChris Mason ioff = btrfs_token_item_offset(leaf, item, &token); 4727cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 4728cfed81a0SChris Mason ioff - total_data, &token); 47290783fcfcSChris Mason } 4730be0e5c09SChris Mason /* shift the items */ 47319c58309dSChris Mason memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + nr), 47325f39d397SChris Mason btrfs_item_nr_offset(slot), 47330783fcfcSChris Mason (nritems - slot) * sizeof(struct btrfs_item)); 4734be0e5c09SChris Mason 4735be0e5c09SChris Mason /* shift the data */ 47363d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 47373d9ec8c4SNikolay Borisov data_end - total_data, BTRFS_LEAF_DATA_OFFSET + 4738be0e5c09SChris Mason data_end, old_data - data_end); 4739be0e5c09SChris Mason data_end = old_data; 4740be0e5c09SChris Mason } 47415f39d397SChris Mason 474262e2749eSChris Mason /* setup the item for the new data */ 47439c58309dSChris Mason for (i = 0; i < nr; i++) { 47449c58309dSChris Mason btrfs_cpu_key_to_disk(&disk_key, cpu_key + i); 47459c58309dSChris Mason btrfs_set_item_key(leaf, &disk_key, slot + i); 4746dd3cc16bSRoss Kirk item = btrfs_item_nr(slot + i); 4747cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 4748cfed81a0SChris Mason data_end - data_size[i], &token); 47499c58309dSChris Mason data_end -= data_size[i]; 4750cfed81a0SChris Mason btrfs_set_token_item_size(leaf, item, data_size[i], &token); 47519c58309dSChris Mason } 475244871b1bSChris Mason 47539c58309dSChris Mason btrfs_set_header_nritems(leaf, nritems + nr); 4754b9473439SChris Mason btrfs_mark_buffer_dirty(leaf); 4755aa5d6bedSChris Mason 47562ff7e61eSJeff Mahoney if (btrfs_leaf_free_space(fs_info, leaf) < 0) { 4757a4f78750SDavid Sterba btrfs_print_leaf(leaf); 4758be0e5c09SChris Mason BUG(); 47595f39d397SChris Mason } 476044871b1bSChris Mason } 476144871b1bSChris Mason 476244871b1bSChris Mason /* 476344871b1bSChris Mason * Given a key and some data, insert items into the tree. 476444871b1bSChris Mason * This does all the path init required, making room in the tree if needed. 476544871b1bSChris Mason */ 476644871b1bSChris Mason int btrfs_insert_empty_items(struct btrfs_trans_handle *trans, 476744871b1bSChris Mason struct btrfs_root *root, 476844871b1bSChris Mason struct btrfs_path *path, 4769310712b2SOmar Sandoval const struct btrfs_key *cpu_key, u32 *data_size, 477044871b1bSChris Mason int nr) 477144871b1bSChris Mason { 477244871b1bSChris Mason int ret = 0; 477344871b1bSChris Mason int slot; 477444871b1bSChris Mason int i; 477544871b1bSChris Mason u32 total_size = 0; 477644871b1bSChris Mason u32 total_data = 0; 477744871b1bSChris Mason 477844871b1bSChris Mason for (i = 0; i < nr; i++) 477944871b1bSChris Mason total_data += data_size[i]; 478044871b1bSChris Mason 478144871b1bSChris Mason total_size = total_data + (nr * sizeof(struct btrfs_item)); 478244871b1bSChris Mason ret = btrfs_search_slot(trans, root, cpu_key, path, total_size, 1); 478344871b1bSChris Mason if (ret == 0) 478444871b1bSChris Mason return -EEXIST; 478544871b1bSChris Mason if (ret < 0) 4786143bede5SJeff Mahoney return ret; 478744871b1bSChris Mason 478844871b1bSChris Mason slot = path->slots[0]; 478944871b1bSChris Mason BUG_ON(slot < 0); 479044871b1bSChris Mason 4791afe5fea7STsutomu Itoh setup_items_for_insert(root, path, cpu_key, data_size, 479244871b1bSChris Mason total_data, total_size, nr); 4793143bede5SJeff Mahoney return 0; 479462e2749eSChris Mason } 479562e2749eSChris Mason 479662e2749eSChris Mason /* 479762e2749eSChris Mason * Given a key and some data, insert an item into the tree. 479862e2749eSChris Mason * This does all the path init required, making room in the tree if needed. 479962e2749eSChris Mason */ 4800310712b2SOmar Sandoval int btrfs_insert_item(struct btrfs_trans_handle *trans, struct btrfs_root *root, 4801310712b2SOmar Sandoval const struct btrfs_key *cpu_key, void *data, 4802310712b2SOmar Sandoval u32 data_size) 480362e2749eSChris Mason { 480462e2749eSChris Mason int ret = 0; 48052c90e5d6SChris Mason struct btrfs_path *path; 48065f39d397SChris Mason struct extent_buffer *leaf; 48075f39d397SChris Mason unsigned long ptr; 480862e2749eSChris Mason 48092c90e5d6SChris Mason path = btrfs_alloc_path(); 4810db5b493aSTsutomu Itoh if (!path) 4811db5b493aSTsutomu Itoh return -ENOMEM; 48122c90e5d6SChris Mason ret = btrfs_insert_empty_item(trans, root, path, cpu_key, data_size); 481362e2749eSChris Mason if (!ret) { 48145f39d397SChris Mason leaf = path->nodes[0]; 48155f39d397SChris Mason ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); 48165f39d397SChris Mason write_extent_buffer(leaf, data, ptr, data_size); 48175f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 481862e2749eSChris Mason } 48192c90e5d6SChris Mason btrfs_free_path(path); 4820aa5d6bedSChris Mason return ret; 4821be0e5c09SChris Mason } 4822be0e5c09SChris Mason 482374123bd7SChris Mason /* 48245de08d7dSChris Mason * delete the pointer from a given node. 482574123bd7SChris Mason * 4826d352ac68SChris Mason * the tree should have been previously balanced so the deletion does not 4827d352ac68SChris Mason * empty a node. 482874123bd7SChris Mason */ 4829afe5fea7STsutomu Itoh static void del_ptr(struct btrfs_root *root, struct btrfs_path *path, 4830afe5fea7STsutomu Itoh int level, int slot) 4831be0e5c09SChris Mason { 48325f39d397SChris Mason struct extent_buffer *parent = path->nodes[level]; 48337518a238SChris Mason u32 nritems; 4834f3ea38daSJan Schmidt int ret; 4835be0e5c09SChris Mason 48365f39d397SChris Mason nritems = btrfs_header_nritems(parent); 4837be0e5c09SChris Mason if (slot != nritems - 1) { 4838bf1d3425SDavid Sterba if (level) { 4839bf1d3425SDavid Sterba ret = tree_mod_log_insert_move(parent, slot, slot + 1, 4840a446a979SDavid Sterba nritems - slot - 1); 4841bf1d3425SDavid Sterba BUG_ON(ret < 0); 4842bf1d3425SDavid Sterba } 48435f39d397SChris Mason memmove_extent_buffer(parent, 48445f39d397SChris Mason btrfs_node_key_ptr_offset(slot), 48455f39d397SChris Mason btrfs_node_key_ptr_offset(slot + 1), 4846d6025579SChris Mason sizeof(struct btrfs_key_ptr) * 4847d6025579SChris Mason (nritems - slot - 1)); 484857ba86c0SChris Mason } else if (level) { 4849e09c2efeSDavid Sterba ret = tree_mod_log_insert_key(parent, slot, MOD_LOG_KEY_REMOVE, 4850e09c2efeSDavid Sterba GFP_NOFS); 485157ba86c0SChris Mason BUG_ON(ret < 0); 4852be0e5c09SChris Mason } 4853f3ea38daSJan Schmidt 48547518a238SChris Mason nritems--; 48555f39d397SChris Mason btrfs_set_header_nritems(parent, nritems); 48567518a238SChris Mason if (nritems == 0 && parent == root->node) { 48575f39d397SChris Mason BUG_ON(btrfs_header_level(root->node) != 1); 4858eb60ceacSChris Mason /* just turn the root into a leaf and break */ 48595f39d397SChris Mason btrfs_set_header_level(root->node, 0); 4860bb803951SChris Mason } else if (slot == 0) { 48615f39d397SChris Mason struct btrfs_disk_key disk_key; 48625f39d397SChris Mason 48635f39d397SChris Mason btrfs_node_key(parent, &disk_key, 0); 4864b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, level + 1); 4865be0e5c09SChris Mason } 4866d6025579SChris Mason btrfs_mark_buffer_dirty(parent); 4867be0e5c09SChris Mason } 4868be0e5c09SChris Mason 486974123bd7SChris Mason /* 4870323ac95bSChris Mason * a helper function to delete the leaf pointed to by path->slots[1] and 48715d4f98a2SYan Zheng * path->nodes[1]. 4872323ac95bSChris Mason * 4873323ac95bSChris Mason * This deletes the pointer in path->nodes[1] and frees the leaf 4874323ac95bSChris Mason * block extent. zero is returned if it all worked out, < 0 otherwise. 4875323ac95bSChris Mason * 4876323ac95bSChris Mason * The path must have already been setup for deleting the leaf, including 4877323ac95bSChris Mason * all the proper balancing. path->nodes[1] must be locked. 4878323ac95bSChris Mason */ 4879143bede5SJeff Mahoney static noinline void btrfs_del_leaf(struct btrfs_trans_handle *trans, 4880323ac95bSChris Mason struct btrfs_root *root, 48815d4f98a2SYan Zheng struct btrfs_path *path, 48825d4f98a2SYan Zheng struct extent_buffer *leaf) 4883323ac95bSChris Mason { 48845d4f98a2SYan Zheng WARN_ON(btrfs_header_generation(leaf) != trans->transid); 4885afe5fea7STsutomu Itoh del_ptr(root, path, 1, path->slots[1]); 4886323ac95bSChris Mason 48874d081c41SChris Mason /* 48884d081c41SChris Mason * btrfs_free_extent is expensive, we want to make sure we 48894d081c41SChris Mason * aren't holding any locks when we call it 48904d081c41SChris Mason */ 48914d081c41SChris Mason btrfs_unlock_up_safe(path, 0); 48924d081c41SChris Mason 4893f0486c68SYan, Zheng root_sub_used(root, leaf->len); 4894f0486c68SYan, Zheng 48953083ee2eSJosef Bacik extent_buffer_get(leaf); 48965581a51aSJan Schmidt btrfs_free_tree_block(trans, root, leaf, 0, 1); 48973083ee2eSJosef Bacik free_extent_buffer_stale(leaf); 4898323ac95bSChris Mason } 4899323ac95bSChris Mason /* 490074123bd7SChris Mason * delete the item at the leaf level in path. If that empties 490174123bd7SChris Mason * the leaf, remove it from the tree 490274123bd7SChris Mason */ 490385e21bacSChris Mason int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root, 490485e21bacSChris Mason struct btrfs_path *path, int slot, int nr) 4905be0e5c09SChris Mason { 49060b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 49075f39d397SChris Mason struct extent_buffer *leaf; 49085f39d397SChris Mason struct btrfs_item *item; 4909ce0eac2aSAlexandru Moise u32 last_off; 4910ce0eac2aSAlexandru Moise u32 dsize = 0; 4911aa5d6bedSChris Mason int ret = 0; 4912aa5d6bedSChris Mason int wret; 491385e21bacSChris Mason int i; 49147518a238SChris Mason u32 nritems; 4915cfed81a0SChris Mason struct btrfs_map_token token; 4916cfed81a0SChris Mason 4917cfed81a0SChris Mason btrfs_init_map_token(&token); 4918be0e5c09SChris Mason 49195f39d397SChris Mason leaf = path->nodes[0]; 492085e21bacSChris Mason last_off = btrfs_item_offset_nr(leaf, slot + nr - 1); 492185e21bacSChris Mason 492285e21bacSChris Mason for (i = 0; i < nr; i++) 492385e21bacSChris Mason dsize += btrfs_item_size_nr(leaf, slot + i); 492485e21bacSChris Mason 49255f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 4926be0e5c09SChris Mason 492785e21bacSChris Mason if (slot + nr != nritems) { 49282ff7e61eSJeff Mahoney int data_end = leaf_data_end(fs_info, leaf); 49295f39d397SChris Mason 49303d9ec8c4SNikolay Borisov memmove_extent_buffer(leaf, BTRFS_LEAF_DATA_OFFSET + 4931d6025579SChris Mason data_end + dsize, 49323d9ec8c4SNikolay Borisov BTRFS_LEAF_DATA_OFFSET + data_end, 493385e21bacSChris Mason last_off - data_end); 49345f39d397SChris Mason 493585e21bacSChris Mason for (i = slot + nr; i < nritems; i++) { 49365f39d397SChris Mason u32 ioff; 4937db94535dSChris Mason 4938dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 4939cfed81a0SChris Mason ioff = btrfs_token_item_offset(leaf, item, &token); 4940cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 4941cfed81a0SChris Mason ioff + dsize, &token); 49420783fcfcSChris Mason } 4943db94535dSChris Mason 49445f39d397SChris Mason memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot), 494585e21bacSChris Mason btrfs_item_nr_offset(slot + nr), 49460783fcfcSChris Mason sizeof(struct btrfs_item) * 494785e21bacSChris Mason (nritems - slot - nr)); 4948be0e5c09SChris Mason } 494985e21bacSChris Mason btrfs_set_header_nritems(leaf, nritems - nr); 495085e21bacSChris Mason nritems -= nr; 49515f39d397SChris Mason 495274123bd7SChris Mason /* delete the leaf if we've emptied it */ 49537518a238SChris Mason if (nritems == 0) { 49545f39d397SChris Mason if (leaf == root->node) { 49555f39d397SChris Mason btrfs_set_header_level(leaf, 0); 49569a8dd150SChris Mason } else { 4957f0486c68SYan, Zheng btrfs_set_path_blocking(path); 49587c302b49SDavid Sterba clean_tree_block(fs_info, leaf); 4959143bede5SJeff Mahoney btrfs_del_leaf(trans, root, path, leaf); 49609a8dd150SChris Mason } 4961be0e5c09SChris Mason } else { 49627518a238SChris Mason int used = leaf_space_used(leaf, 0, nritems); 4963aa5d6bedSChris Mason if (slot == 0) { 49645f39d397SChris Mason struct btrfs_disk_key disk_key; 49655f39d397SChris Mason 49665f39d397SChris Mason btrfs_item_key(leaf, &disk_key, 0); 4967b167fa91SNikolay Borisov fixup_low_keys(path, &disk_key, 1); 4968aa5d6bedSChris Mason } 4969aa5d6bedSChris Mason 497074123bd7SChris Mason /* delete the leaf if it is mostly empty */ 49710b246afaSJeff Mahoney if (used < BTRFS_LEAF_DATA_SIZE(fs_info) / 3) { 4972be0e5c09SChris Mason /* push_leaf_left fixes the path. 4973be0e5c09SChris Mason * make sure the path still points to our leaf 4974be0e5c09SChris Mason * for possible call to del_ptr below 4975be0e5c09SChris Mason */ 49764920c9acSChris Mason slot = path->slots[1]; 49775f39d397SChris Mason extent_buffer_get(leaf); 49785f39d397SChris Mason 4979b9473439SChris Mason btrfs_set_path_blocking(path); 498099d8f83cSChris Mason wret = push_leaf_left(trans, root, path, 1, 1, 498199d8f83cSChris Mason 1, (u32)-1); 498254aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 4983aa5d6bedSChris Mason ret = wret; 49845f39d397SChris Mason 49855f39d397SChris Mason if (path->nodes[0] == leaf && 49865f39d397SChris Mason btrfs_header_nritems(leaf)) { 498799d8f83cSChris Mason wret = push_leaf_right(trans, root, path, 1, 498899d8f83cSChris Mason 1, 1, 0); 498954aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 4990aa5d6bedSChris Mason ret = wret; 4991aa5d6bedSChris Mason } 49925f39d397SChris Mason 49935f39d397SChris Mason if (btrfs_header_nritems(leaf) == 0) { 4994323ac95bSChris Mason path->slots[1] = slot; 4995143bede5SJeff Mahoney btrfs_del_leaf(trans, root, path, leaf); 49965f39d397SChris Mason free_extent_buffer(leaf); 4997143bede5SJeff Mahoney ret = 0; 49985de08d7dSChris Mason } else { 4999925baeddSChris Mason /* if we're still in the path, make sure 5000925baeddSChris Mason * we're dirty. Otherwise, one of the 5001925baeddSChris Mason * push_leaf functions must have already 5002925baeddSChris Mason * dirtied this buffer 5003925baeddSChris Mason */ 5004925baeddSChris Mason if (path->nodes[0] == leaf) 50055f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 50065f39d397SChris Mason free_extent_buffer(leaf); 5007be0e5c09SChris Mason } 5008d5719762SChris Mason } else { 50095f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 5010be0e5c09SChris Mason } 50119a8dd150SChris Mason } 5012aa5d6bedSChris Mason return ret; 50139a8dd150SChris Mason } 50149a8dd150SChris Mason 501597571fd0SChris Mason /* 5016925baeddSChris Mason * search the tree again to find a leaf with lesser keys 50177bb86316SChris Mason * returns 0 if it found something or 1 if there are no lesser leaves. 50187bb86316SChris Mason * returns < 0 on io errors. 5019d352ac68SChris Mason * 5020d352ac68SChris Mason * This may release the path, and so you may lose any locks held at the 5021d352ac68SChris Mason * time you call it. 50227bb86316SChris Mason */ 502316e7549fSJosef Bacik int btrfs_prev_leaf(struct btrfs_root *root, struct btrfs_path *path) 50247bb86316SChris Mason { 5025925baeddSChris Mason struct btrfs_key key; 5026925baeddSChris Mason struct btrfs_disk_key found_key; 5027925baeddSChris Mason int ret; 50287bb86316SChris Mason 5029925baeddSChris Mason btrfs_item_key_to_cpu(path->nodes[0], &key, 0); 5030925baeddSChris Mason 5031e8b0d724SFilipe David Borba Manana if (key.offset > 0) { 5032925baeddSChris Mason key.offset--; 5033e8b0d724SFilipe David Borba Manana } else if (key.type > 0) { 5034925baeddSChris Mason key.type--; 5035e8b0d724SFilipe David Borba Manana key.offset = (u64)-1; 5036e8b0d724SFilipe David Borba Manana } else if (key.objectid > 0) { 5037925baeddSChris Mason key.objectid--; 5038e8b0d724SFilipe David Borba Manana key.type = (u8)-1; 5039e8b0d724SFilipe David Borba Manana key.offset = (u64)-1; 5040e8b0d724SFilipe David Borba Manana } else { 50417bb86316SChris Mason return 1; 5042e8b0d724SFilipe David Borba Manana } 50437bb86316SChris Mason 5044b3b4aa74SDavid Sterba btrfs_release_path(path); 5045925baeddSChris Mason ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 5046925baeddSChris Mason if (ret < 0) 5047925baeddSChris Mason return ret; 5048925baeddSChris Mason btrfs_item_key(path->nodes[0], &found_key, 0); 5049925baeddSChris Mason ret = comp_keys(&found_key, &key); 5050337c6f68SFilipe Manana /* 5051337c6f68SFilipe Manana * We might have had an item with the previous key in the tree right 5052337c6f68SFilipe Manana * before we released our path. And after we released our path, that 5053337c6f68SFilipe Manana * item might have been pushed to the first slot (0) of the leaf we 5054337c6f68SFilipe Manana * were holding due to a tree balance. Alternatively, an item with the 5055337c6f68SFilipe Manana * previous key can exist as the only element of a leaf (big fat item). 5056337c6f68SFilipe Manana * Therefore account for these 2 cases, so that our callers (like 5057337c6f68SFilipe Manana * btrfs_previous_item) don't miss an existing item with a key matching 5058337c6f68SFilipe Manana * the previous key we computed above. 5059337c6f68SFilipe Manana */ 5060337c6f68SFilipe Manana if (ret <= 0) 50617bb86316SChris Mason return 0; 5062925baeddSChris Mason return 1; 50637bb86316SChris Mason } 50647bb86316SChris Mason 50653f157a2fSChris Mason /* 50663f157a2fSChris Mason * A helper function to walk down the tree starting at min_key, and looking 5067de78b51aSEric Sandeen * for nodes or leaves that are have a minimum transaction id. 5068de78b51aSEric Sandeen * This is used by the btree defrag code, and tree logging 50693f157a2fSChris Mason * 50703f157a2fSChris Mason * This does not cow, but it does stuff the starting key it finds back 50713f157a2fSChris Mason * into min_key, so you can call btrfs_search_slot with cow=1 on the 50723f157a2fSChris Mason * key and get a writable path. 50733f157a2fSChris Mason * 50743f157a2fSChris Mason * This honors path->lowest_level to prevent descent past a given level 50753f157a2fSChris Mason * of the tree. 50763f157a2fSChris Mason * 5077d352ac68SChris Mason * min_trans indicates the oldest transaction that you are interested 5078d352ac68SChris Mason * in walking through. Any nodes or leaves older than min_trans are 5079d352ac68SChris Mason * skipped over (without reading them). 5080d352ac68SChris Mason * 50813f157a2fSChris Mason * returns zero if something useful was found, < 0 on error and 1 if there 50823f157a2fSChris Mason * was nothing in the tree that matched the search criteria. 50833f157a2fSChris Mason */ 50843f157a2fSChris Mason int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key, 5085de78b51aSEric Sandeen struct btrfs_path *path, 50863f157a2fSChris Mason u64 min_trans) 50873f157a2fSChris Mason { 50882ff7e61eSJeff Mahoney struct btrfs_fs_info *fs_info = root->fs_info; 50893f157a2fSChris Mason struct extent_buffer *cur; 50903f157a2fSChris Mason struct btrfs_key found_key; 50913f157a2fSChris Mason int slot; 50929652480bSYan int sret; 50933f157a2fSChris Mason u32 nritems; 50943f157a2fSChris Mason int level; 50953f157a2fSChris Mason int ret = 1; 5096f98de9b9SFilipe Manana int keep_locks = path->keep_locks; 50973f157a2fSChris Mason 5098f98de9b9SFilipe Manana path->keep_locks = 1; 50993f157a2fSChris Mason again: 5100bd681513SChris Mason cur = btrfs_read_lock_root_node(root); 51013f157a2fSChris Mason level = btrfs_header_level(cur); 5102e02119d5SChris Mason WARN_ON(path->nodes[level]); 51033f157a2fSChris Mason path->nodes[level] = cur; 5104bd681513SChris Mason path->locks[level] = BTRFS_READ_LOCK; 51053f157a2fSChris Mason 51063f157a2fSChris Mason if (btrfs_header_generation(cur) < min_trans) { 51073f157a2fSChris Mason ret = 1; 51083f157a2fSChris Mason goto out; 51093f157a2fSChris Mason } 51103f157a2fSChris Mason while (1) { 51113f157a2fSChris Mason nritems = btrfs_header_nritems(cur); 51123f157a2fSChris Mason level = btrfs_header_level(cur); 5113a74b35ecSNikolay Borisov sret = btrfs_bin_search(cur, min_key, level, &slot); 51143f157a2fSChris Mason 5115323ac95bSChris Mason /* at the lowest level, we're done, setup the path and exit */ 5116323ac95bSChris Mason if (level == path->lowest_level) { 5117e02119d5SChris Mason if (slot >= nritems) 5118e02119d5SChris Mason goto find_next_key; 51193f157a2fSChris Mason ret = 0; 51203f157a2fSChris Mason path->slots[level] = slot; 51213f157a2fSChris Mason btrfs_item_key_to_cpu(cur, &found_key, slot); 51223f157a2fSChris Mason goto out; 51233f157a2fSChris Mason } 51249652480bSYan if (sret && slot > 0) 51259652480bSYan slot--; 51263f157a2fSChris Mason /* 5127de78b51aSEric Sandeen * check this node pointer against the min_trans parameters. 5128de78b51aSEric Sandeen * If it is too old, old, skip to the next one. 51293f157a2fSChris Mason */ 51303f157a2fSChris Mason while (slot < nritems) { 51313f157a2fSChris Mason u64 gen; 5132e02119d5SChris Mason 51333f157a2fSChris Mason gen = btrfs_node_ptr_generation(cur, slot); 51343f157a2fSChris Mason if (gen < min_trans) { 51353f157a2fSChris Mason slot++; 51363f157a2fSChris Mason continue; 51373f157a2fSChris Mason } 51383f157a2fSChris Mason break; 51393f157a2fSChris Mason } 5140e02119d5SChris Mason find_next_key: 51413f157a2fSChris Mason /* 51423f157a2fSChris Mason * we didn't find a candidate key in this node, walk forward 51433f157a2fSChris Mason * and find another one 51443f157a2fSChris Mason */ 51453f157a2fSChris Mason if (slot >= nritems) { 5146e02119d5SChris Mason path->slots[level] = slot; 5147b4ce94deSChris Mason btrfs_set_path_blocking(path); 5148e02119d5SChris Mason sret = btrfs_find_next_key(root, path, min_key, level, 5149de78b51aSEric Sandeen min_trans); 5150e02119d5SChris Mason if (sret == 0) { 5151b3b4aa74SDavid Sterba btrfs_release_path(path); 51523f157a2fSChris Mason goto again; 51533f157a2fSChris Mason } else { 51543f157a2fSChris Mason goto out; 51553f157a2fSChris Mason } 51563f157a2fSChris Mason } 51573f157a2fSChris Mason /* save our key for returning back */ 51583f157a2fSChris Mason btrfs_node_key_to_cpu(cur, &found_key, slot); 51593f157a2fSChris Mason path->slots[level] = slot; 51603f157a2fSChris Mason if (level == path->lowest_level) { 51613f157a2fSChris Mason ret = 0; 51623f157a2fSChris Mason goto out; 51633f157a2fSChris Mason } 5164b4ce94deSChris Mason btrfs_set_path_blocking(path); 51652ff7e61eSJeff Mahoney cur = read_node_slot(fs_info, cur, slot); 5166fb770ae4SLiu Bo if (IS_ERR(cur)) { 5167fb770ae4SLiu Bo ret = PTR_ERR(cur); 5168fb770ae4SLiu Bo goto out; 5169fb770ae4SLiu Bo } 51703f157a2fSChris Mason 5171bd681513SChris Mason btrfs_tree_read_lock(cur); 5172b4ce94deSChris Mason 5173bd681513SChris Mason path->locks[level - 1] = BTRFS_READ_LOCK; 51743f157a2fSChris Mason path->nodes[level - 1] = cur; 5175f7c79f30SChris Mason unlock_up(path, level, 1, 0, NULL); 51763f157a2fSChris Mason } 51773f157a2fSChris Mason out: 5178f98de9b9SFilipe Manana path->keep_locks = keep_locks; 5179f98de9b9SFilipe Manana if (ret == 0) { 5180f98de9b9SFilipe Manana btrfs_unlock_up_safe(path, path->lowest_level + 1); 5181b4ce94deSChris Mason btrfs_set_path_blocking(path); 5182f98de9b9SFilipe Manana memcpy(min_key, &found_key, sizeof(found_key)); 5183f98de9b9SFilipe Manana } 51843f157a2fSChris Mason return ret; 51853f157a2fSChris Mason } 51863f157a2fSChris Mason 51872ff7e61eSJeff Mahoney static int tree_move_down(struct btrfs_fs_info *fs_info, 51887069830aSAlexander Block struct btrfs_path *path, 5189ab6a43e1SDavid Sterba int *level) 51907069830aSAlexander Block { 5191fb770ae4SLiu Bo struct extent_buffer *eb; 5192fb770ae4SLiu Bo 519374dd17fbSChris Mason BUG_ON(*level == 0); 51942ff7e61eSJeff Mahoney eb = read_node_slot(fs_info, path->nodes[*level], path->slots[*level]); 5195fb770ae4SLiu Bo if (IS_ERR(eb)) 5196fb770ae4SLiu Bo return PTR_ERR(eb); 5197fb770ae4SLiu Bo 5198fb770ae4SLiu Bo path->nodes[*level - 1] = eb; 51997069830aSAlexander Block path->slots[*level - 1] = 0; 52007069830aSAlexander Block (*level)--; 5201fb770ae4SLiu Bo return 0; 52027069830aSAlexander Block } 52037069830aSAlexander Block 5204f1e30261SDavid Sterba static int tree_move_next_or_upnext(struct btrfs_path *path, 52057069830aSAlexander Block int *level, int root_level) 52067069830aSAlexander Block { 52077069830aSAlexander Block int ret = 0; 52087069830aSAlexander Block int nritems; 52097069830aSAlexander Block nritems = btrfs_header_nritems(path->nodes[*level]); 52107069830aSAlexander Block 52117069830aSAlexander Block path->slots[*level]++; 52127069830aSAlexander Block 521374dd17fbSChris Mason while (path->slots[*level] >= nritems) { 52147069830aSAlexander Block if (*level == root_level) 52157069830aSAlexander Block return -1; 52167069830aSAlexander Block 52177069830aSAlexander Block /* move upnext */ 52187069830aSAlexander Block path->slots[*level] = 0; 52197069830aSAlexander Block free_extent_buffer(path->nodes[*level]); 52207069830aSAlexander Block path->nodes[*level] = NULL; 52217069830aSAlexander Block (*level)++; 52227069830aSAlexander Block path->slots[*level]++; 52237069830aSAlexander Block 52247069830aSAlexander Block nritems = btrfs_header_nritems(path->nodes[*level]); 52257069830aSAlexander Block ret = 1; 52267069830aSAlexander Block } 52277069830aSAlexander Block return ret; 52287069830aSAlexander Block } 52297069830aSAlexander Block 52307069830aSAlexander Block /* 52317069830aSAlexander Block * Returns 1 if it had to move up and next. 0 is returned if it moved only next 52327069830aSAlexander Block * or down. 52337069830aSAlexander Block */ 52342ff7e61eSJeff Mahoney static int tree_advance(struct btrfs_fs_info *fs_info, 52357069830aSAlexander Block struct btrfs_path *path, 52367069830aSAlexander Block int *level, int root_level, 52377069830aSAlexander Block int allow_down, 52387069830aSAlexander Block struct btrfs_key *key) 52397069830aSAlexander Block { 52407069830aSAlexander Block int ret; 52417069830aSAlexander Block 52427069830aSAlexander Block if (*level == 0 || !allow_down) { 5243f1e30261SDavid Sterba ret = tree_move_next_or_upnext(path, level, root_level); 52447069830aSAlexander Block } else { 5245ab6a43e1SDavid Sterba ret = tree_move_down(fs_info, path, level); 52467069830aSAlexander Block } 52477069830aSAlexander Block if (ret >= 0) { 52487069830aSAlexander Block if (*level == 0) 52497069830aSAlexander Block btrfs_item_key_to_cpu(path->nodes[*level], key, 52507069830aSAlexander Block path->slots[*level]); 52517069830aSAlexander Block else 52527069830aSAlexander Block btrfs_node_key_to_cpu(path->nodes[*level], key, 52537069830aSAlexander Block path->slots[*level]); 52547069830aSAlexander Block } 52557069830aSAlexander Block return ret; 52567069830aSAlexander Block } 52577069830aSAlexander Block 52582ff7e61eSJeff Mahoney static int tree_compare_item(struct btrfs_path *left_path, 52597069830aSAlexander Block struct btrfs_path *right_path, 52607069830aSAlexander Block char *tmp_buf) 52617069830aSAlexander Block { 52627069830aSAlexander Block int cmp; 52637069830aSAlexander Block int len1, len2; 52647069830aSAlexander Block unsigned long off1, off2; 52657069830aSAlexander Block 52667069830aSAlexander Block len1 = btrfs_item_size_nr(left_path->nodes[0], left_path->slots[0]); 52677069830aSAlexander Block len2 = btrfs_item_size_nr(right_path->nodes[0], right_path->slots[0]); 52687069830aSAlexander Block if (len1 != len2) 52697069830aSAlexander Block return 1; 52707069830aSAlexander Block 52717069830aSAlexander Block off1 = btrfs_item_ptr_offset(left_path->nodes[0], left_path->slots[0]); 52727069830aSAlexander Block off2 = btrfs_item_ptr_offset(right_path->nodes[0], 52737069830aSAlexander Block right_path->slots[0]); 52747069830aSAlexander Block 52757069830aSAlexander Block read_extent_buffer(left_path->nodes[0], tmp_buf, off1, len1); 52767069830aSAlexander Block 52777069830aSAlexander Block cmp = memcmp_extent_buffer(right_path->nodes[0], tmp_buf, off2, len1); 52787069830aSAlexander Block if (cmp) 52797069830aSAlexander Block return 1; 52807069830aSAlexander Block return 0; 52817069830aSAlexander Block } 52827069830aSAlexander Block 52837069830aSAlexander Block #define ADVANCE 1 52847069830aSAlexander Block #define ADVANCE_ONLY_NEXT -1 52857069830aSAlexander Block 52867069830aSAlexander Block /* 52877069830aSAlexander Block * This function compares two trees and calls the provided callback for 52887069830aSAlexander Block * every changed/new/deleted item it finds. 52897069830aSAlexander Block * If shared tree blocks are encountered, whole subtrees are skipped, making 52907069830aSAlexander Block * the compare pretty fast on snapshotted subvolumes. 52917069830aSAlexander Block * 52927069830aSAlexander Block * This currently works on commit roots only. As commit roots are read only, 52937069830aSAlexander Block * we don't do any locking. The commit roots are protected with transactions. 52947069830aSAlexander Block * Transactions are ended and rejoined when a commit is tried in between. 52957069830aSAlexander Block * 52967069830aSAlexander Block * This function checks for modifications done to the trees while comparing. 52977069830aSAlexander Block * If it detects a change, it aborts immediately. 52987069830aSAlexander Block */ 52997069830aSAlexander Block int btrfs_compare_trees(struct btrfs_root *left_root, 53007069830aSAlexander Block struct btrfs_root *right_root, 53017069830aSAlexander Block btrfs_changed_cb_t changed_cb, void *ctx) 53027069830aSAlexander Block { 53030b246afaSJeff Mahoney struct btrfs_fs_info *fs_info = left_root->fs_info; 53047069830aSAlexander Block int ret; 53057069830aSAlexander Block int cmp; 53067069830aSAlexander Block struct btrfs_path *left_path = NULL; 53077069830aSAlexander Block struct btrfs_path *right_path = NULL; 53087069830aSAlexander Block struct btrfs_key left_key; 53097069830aSAlexander Block struct btrfs_key right_key; 53107069830aSAlexander Block char *tmp_buf = NULL; 53117069830aSAlexander Block int left_root_level; 53127069830aSAlexander Block int right_root_level; 53137069830aSAlexander Block int left_level; 53147069830aSAlexander Block int right_level; 53157069830aSAlexander Block int left_end_reached; 53167069830aSAlexander Block int right_end_reached; 53177069830aSAlexander Block int advance_left; 53187069830aSAlexander Block int advance_right; 53197069830aSAlexander Block u64 left_blockptr; 53207069830aSAlexander Block u64 right_blockptr; 53216baa4293SFilipe Manana u64 left_gen; 53226baa4293SFilipe Manana u64 right_gen; 53237069830aSAlexander Block 53247069830aSAlexander Block left_path = btrfs_alloc_path(); 53257069830aSAlexander Block if (!left_path) { 53267069830aSAlexander Block ret = -ENOMEM; 53277069830aSAlexander Block goto out; 53287069830aSAlexander Block } 53297069830aSAlexander Block right_path = btrfs_alloc_path(); 53307069830aSAlexander Block if (!right_path) { 53317069830aSAlexander Block ret = -ENOMEM; 53327069830aSAlexander Block goto out; 53337069830aSAlexander Block } 53347069830aSAlexander Block 5335752ade68SMichal Hocko tmp_buf = kvmalloc(fs_info->nodesize, GFP_KERNEL); 53367069830aSAlexander Block if (!tmp_buf) { 53377069830aSAlexander Block ret = -ENOMEM; 53387069830aSAlexander Block goto out; 53397069830aSAlexander Block } 53407069830aSAlexander Block 53417069830aSAlexander Block left_path->search_commit_root = 1; 53427069830aSAlexander Block left_path->skip_locking = 1; 53437069830aSAlexander Block right_path->search_commit_root = 1; 53447069830aSAlexander Block right_path->skip_locking = 1; 53457069830aSAlexander Block 53467069830aSAlexander Block /* 53477069830aSAlexander Block * Strategy: Go to the first items of both trees. Then do 53487069830aSAlexander Block * 53497069830aSAlexander Block * If both trees are at level 0 53507069830aSAlexander Block * Compare keys of current items 53517069830aSAlexander Block * If left < right treat left item as new, advance left tree 53527069830aSAlexander Block * and repeat 53537069830aSAlexander Block * If left > right treat right item as deleted, advance right tree 53547069830aSAlexander Block * and repeat 53557069830aSAlexander Block * If left == right do deep compare of items, treat as changed if 53567069830aSAlexander Block * needed, advance both trees and repeat 53577069830aSAlexander Block * If both trees are at the same level but not at level 0 53587069830aSAlexander Block * Compare keys of current nodes/leafs 53597069830aSAlexander Block * If left < right advance left tree and repeat 53607069830aSAlexander Block * If left > right advance right tree and repeat 53617069830aSAlexander Block * If left == right compare blockptrs of the next nodes/leafs 53627069830aSAlexander Block * If they match advance both trees but stay at the same level 53637069830aSAlexander Block * and repeat 53647069830aSAlexander Block * If they don't match advance both trees while allowing to go 53657069830aSAlexander Block * deeper and repeat 53667069830aSAlexander Block * If tree levels are different 53677069830aSAlexander Block * Advance the tree that needs it and repeat 53687069830aSAlexander Block * 53697069830aSAlexander Block * Advancing a tree means: 53707069830aSAlexander Block * If we are at level 0, try to go to the next slot. If that's not 53717069830aSAlexander Block * possible, go one level up and repeat. Stop when we found a level 53727069830aSAlexander Block * where we could go to the next slot. We may at this point be on a 53737069830aSAlexander Block * node or a leaf. 53747069830aSAlexander Block * 53757069830aSAlexander Block * If we are not at level 0 and not on shared tree blocks, go one 53767069830aSAlexander Block * level deeper. 53777069830aSAlexander Block * 53787069830aSAlexander Block * If we are not at level 0 and on shared tree blocks, go one slot to 53797069830aSAlexander Block * the right if possible or go up and right. 53807069830aSAlexander Block */ 53817069830aSAlexander Block 53820b246afaSJeff Mahoney down_read(&fs_info->commit_root_sem); 53837069830aSAlexander Block left_level = btrfs_header_level(left_root->commit_root); 53847069830aSAlexander Block left_root_level = left_level; 53856f2f0b39SRobbie Ko left_path->nodes[left_level] = 53866f2f0b39SRobbie Ko btrfs_clone_extent_buffer(left_root->commit_root); 53876f2f0b39SRobbie Ko if (!left_path->nodes[left_level]) { 53886f2f0b39SRobbie Ko up_read(&fs_info->commit_root_sem); 53896f2f0b39SRobbie Ko ret = -ENOMEM; 53906f2f0b39SRobbie Ko goto out; 53916f2f0b39SRobbie Ko } 53927069830aSAlexander Block 53937069830aSAlexander Block right_level = btrfs_header_level(right_root->commit_root); 53947069830aSAlexander Block right_root_level = right_level; 53956f2f0b39SRobbie Ko right_path->nodes[right_level] = 53966f2f0b39SRobbie Ko btrfs_clone_extent_buffer(right_root->commit_root); 53976f2f0b39SRobbie Ko if (!right_path->nodes[right_level]) { 53986f2f0b39SRobbie Ko up_read(&fs_info->commit_root_sem); 53996f2f0b39SRobbie Ko ret = -ENOMEM; 54006f2f0b39SRobbie Ko goto out; 54016f2f0b39SRobbie Ko } 54020b246afaSJeff Mahoney up_read(&fs_info->commit_root_sem); 54037069830aSAlexander Block 54047069830aSAlexander Block if (left_level == 0) 54057069830aSAlexander Block btrfs_item_key_to_cpu(left_path->nodes[left_level], 54067069830aSAlexander Block &left_key, left_path->slots[left_level]); 54077069830aSAlexander Block else 54087069830aSAlexander Block btrfs_node_key_to_cpu(left_path->nodes[left_level], 54097069830aSAlexander Block &left_key, left_path->slots[left_level]); 54107069830aSAlexander Block if (right_level == 0) 54117069830aSAlexander Block btrfs_item_key_to_cpu(right_path->nodes[right_level], 54127069830aSAlexander Block &right_key, right_path->slots[right_level]); 54137069830aSAlexander Block else 54147069830aSAlexander Block btrfs_node_key_to_cpu(right_path->nodes[right_level], 54157069830aSAlexander Block &right_key, right_path->slots[right_level]); 54167069830aSAlexander Block 54177069830aSAlexander Block left_end_reached = right_end_reached = 0; 54187069830aSAlexander Block advance_left = advance_right = 0; 54197069830aSAlexander Block 54207069830aSAlexander Block while (1) { 54217069830aSAlexander Block if (advance_left && !left_end_reached) { 54222ff7e61eSJeff Mahoney ret = tree_advance(fs_info, left_path, &left_level, 54237069830aSAlexander Block left_root_level, 54247069830aSAlexander Block advance_left != ADVANCE_ONLY_NEXT, 54257069830aSAlexander Block &left_key); 5426fb770ae4SLiu Bo if (ret == -1) 54277069830aSAlexander Block left_end_reached = ADVANCE; 5428fb770ae4SLiu Bo else if (ret < 0) 5429fb770ae4SLiu Bo goto out; 54307069830aSAlexander Block advance_left = 0; 54317069830aSAlexander Block } 54327069830aSAlexander Block if (advance_right && !right_end_reached) { 54332ff7e61eSJeff Mahoney ret = tree_advance(fs_info, right_path, &right_level, 54347069830aSAlexander Block right_root_level, 54357069830aSAlexander Block advance_right != ADVANCE_ONLY_NEXT, 54367069830aSAlexander Block &right_key); 5437fb770ae4SLiu Bo if (ret == -1) 54387069830aSAlexander Block right_end_reached = ADVANCE; 5439fb770ae4SLiu Bo else if (ret < 0) 5440fb770ae4SLiu Bo goto out; 54417069830aSAlexander Block advance_right = 0; 54427069830aSAlexander Block } 54437069830aSAlexander Block 54447069830aSAlexander Block if (left_end_reached && right_end_reached) { 54457069830aSAlexander Block ret = 0; 54467069830aSAlexander Block goto out; 54477069830aSAlexander Block } else if (left_end_reached) { 54487069830aSAlexander Block if (right_level == 0) { 5449ee8c494fSNikolay Borisov ret = changed_cb(left_path, right_path, 54507069830aSAlexander Block &right_key, 54517069830aSAlexander Block BTRFS_COMPARE_TREE_DELETED, 54527069830aSAlexander Block ctx); 54537069830aSAlexander Block if (ret < 0) 54547069830aSAlexander Block goto out; 54557069830aSAlexander Block } 54567069830aSAlexander Block advance_right = ADVANCE; 54577069830aSAlexander Block continue; 54587069830aSAlexander Block } else if (right_end_reached) { 54597069830aSAlexander Block if (left_level == 0) { 5460ee8c494fSNikolay Borisov ret = changed_cb(left_path, right_path, 54617069830aSAlexander Block &left_key, 54627069830aSAlexander Block BTRFS_COMPARE_TREE_NEW, 54637069830aSAlexander Block ctx); 54647069830aSAlexander Block if (ret < 0) 54657069830aSAlexander Block goto out; 54667069830aSAlexander Block } 54677069830aSAlexander Block advance_left = ADVANCE; 54687069830aSAlexander Block continue; 54697069830aSAlexander Block } 54707069830aSAlexander Block 54717069830aSAlexander Block if (left_level == 0 && right_level == 0) { 54727069830aSAlexander Block cmp = btrfs_comp_cpu_keys(&left_key, &right_key); 54737069830aSAlexander Block if (cmp < 0) { 5474ee8c494fSNikolay Borisov ret = changed_cb(left_path, right_path, 54757069830aSAlexander Block &left_key, 54767069830aSAlexander Block BTRFS_COMPARE_TREE_NEW, 54777069830aSAlexander Block ctx); 54787069830aSAlexander Block if (ret < 0) 54797069830aSAlexander Block goto out; 54807069830aSAlexander Block advance_left = ADVANCE; 54817069830aSAlexander Block } else if (cmp > 0) { 5482ee8c494fSNikolay Borisov ret = changed_cb(left_path, right_path, 54837069830aSAlexander Block &right_key, 54847069830aSAlexander Block BTRFS_COMPARE_TREE_DELETED, 54857069830aSAlexander Block ctx); 54867069830aSAlexander Block if (ret < 0) 54877069830aSAlexander Block goto out; 54887069830aSAlexander Block advance_right = ADVANCE; 54897069830aSAlexander Block } else { 5490b99d9a6aSFabian Frederick enum btrfs_compare_tree_result result; 5491ba5e8f2eSJosef Bacik 549274dd17fbSChris Mason WARN_ON(!extent_buffer_uptodate(left_path->nodes[0])); 54932ff7e61eSJeff Mahoney ret = tree_compare_item(left_path, right_path, 54942ff7e61eSJeff Mahoney tmp_buf); 5495ba5e8f2eSJosef Bacik if (ret) 5496b99d9a6aSFabian Frederick result = BTRFS_COMPARE_TREE_CHANGED; 5497ba5e8f2eSJosef Bacik else 5498b99d9a6aSFabian Frederick result = BTRFS_COMPARE_TREE_SAME; 5499ee8c494fSNikolay Borisov ret = changed_cb(left_path, right_path, 5500b99d9a6aSFabian Frederick &left_key, result, ctx); 55017069830aSAlexander Block if (ret < 0) 55027069830aSAlexander Block goto out; 55037069830aSAlexander Block advance_left = ADVANCE; 55047069830aSAlexander Block advance_right = ADVANCE; 55057069830aSAlexander Block } 55067069830aSAlexander Block } else if (left_level == right_level) { 55077069830aSAlexander Block cmp = btrfs_comp_cpu_keys(&left_key, &right_key); 55087069830aSAlexander Block if (cmp < 0) { 55097069830aSAlexander Block advance_left = ADVANCE; 55107069830aSAlexander Block } else if (cmp > 0) { 55117069830aSAlexander Block advance_right = ADVANCE; 55127069830aSAlexander Block } else { 55137069830aSAlexander Block left_blockptr = btrfs_node_blockptr( 55147069830aSAlexander Block left_path->nodes[left_level], 55157069830aSAlexander Block left_path->slots[left_level]); 55167069830aSAlexander Block right_blockptr = btrfs_node_blockptr( 55177069830aSAlexander Block right_path->nodes[right_level], 55187069830aSAlexander Block right_path->slots[right_level]); 55196baa4293SFilipe Manana left_gen = btrfs_node_ptr_generation( 55206baa4293SFilipe Manana left_path->nodes[left_level], 55216baa4293SFilipe Manana left_path->slots[left_level]); 55226baa4293SFilipe Manana right_gen = btrfs_node_ptr_generation( 55236baa4293SFilipe Manana right_path->nodes[right_level], 55246baa4293SFilipe Manana right_path->slots[right_level]); 55256baa4293SFilipe Manana if (left_blockptr == right_blockptr && 55266baa4293SFilipe Manana left_gen == right_gen) { 55277069830aSAlexander Block /* 55287069830aSAlexander Block * As we're on a shared block, don't 55297069830aSAlexander Block * allow to go deeper. 55307069830aSAlexander Block */ 55317069830aSAlexander Block advance_left = ADVANCE_ONLY_NEXT; 55327069830aSAlexander Block advance_right = ADVANCE_ONLY_NEXT; 55337069830aSAlexander Block } else { 55347069830aSAlexander Block advance_left = ADVANCE; 55357069830aSAlexander Block advance_right = ADVANCE; 55367069830aSAlexander Block } 55377069830aSAlexander Block } 55387069830aSAlexander Block } else if (left_level < right_level) { 55397069830aSAlexander Block advance_right = ADVANCE; 55407069830aSAlexander Block } else { 55417069830aSAlexander Block advance_left = ADVANCE; 55427069830aSAlexander Block } 55437069830aSAlexander Block } 55447069830aSAlexander Block 55457069830aSAlexander Block out: 55467069830aSAlexander Block btrfs_free_path(left_path); 55477069830aSAlexander Block btrfs_free_path(right_path); 55488f282f71SDavid Sterba kvfree(tmp_buf); 55497069830aSAlexander Block return ret; 55507069830aSAlexander Block } 55517069830aSAlexander Block 55523f157a2fSChris Mason /* 55533f157a2fSChris Mason * this is similar to btrfs_next_leaf, but does not try to preserve 55543f157a2fSChris Mason * and fixup the path. It looks for and returns the next key in the 5555de78b51aSEric Sandeen * tree based on the current path and the min_trans parameters. 55563f157a2fSChris Mason * 55573f157a2fSChris Mason * 0 is returned if another key is found, < 0 if there are any errors 55583f157a2fSChris Mason * and 1 is returned if there are no higher keys in the tree 55593f157a2fSChris Mason * 55603f157a2fSChris Mason * path->keep_locks should be set to 1 on the search made before 55613f157a2fSChris Mason * calling this function. 55623f157a2fSChris Mason */ 5563e7a84565SChris Mason int btrfs_find_next_key(struct btrfs_root *root, struct btrfs_path *path, 5564de78b51aSEric Sandeen struct btrfs_key *key, int level, u64 min_trans) 5565e7a84565SChris Mason { 5566e7a84565SChris Mason int slot; 5567e7a84565SChris Mason struct extent_buffer *c; 5568e7a84565SChris Mason 5569934d375bSChris Mason WARN_ON(!path->keep_locks); 5570e7a84565SChris Mason while (level < BTRFS_MAX_LEVEL) { 5571e7a84565SChris Mason if (!path->nodes[level]) 5572e7a84565SChris Mason return 1; 5573e7a84565SChris Mason 5574e7a84565SChris Mason slot = path->slots[level] + 1; 5575e7a84565SChris Mason c = path->nodes[level]; 55763f157a2fSChris Mason next: 5577e7a84565SChris Mason if (slot >= btrfs_header_nritems(c)) { 557833c66f43SYan Zheng int ret; 557933c66f43SYan Zheng int orig_lowest; 558033c66f43SYan Zheng struct btrfs_key cur_key; 558133c66f43SYan Zheng if (level + 1 >= BTRFS_MAX_LEVEL || 558233c66f43SYan Zheng !path->nodes[level + 1]) 5583e7a84565SChris Mason return 1; 558433c66f43SYan Zheng 558533c66f43SYan Zheng if (path->locks[level + 1]) { 558633c66f43SYan Zheng level++; 5587e7a84565SChris Mason continue; 5588e7a84565SChris Mason } 558933c66f43SYan Zheng 559033c66f43SYan Zheng slot = btrfs_header_nritems(c) - 1; 559133c66f43SYan Zheng if (level == 0) 559233c66f43SYan Zheng btrfs_item_key_to_cpu(c, &cur_key, slot); 559333c66f43SYan Zheng else 559433c66f43SYan Zheng btrfs_node_key_to_cpu(c, &cur_key, slot); 559533c66f43SYan Zheng 559633c66f43SYan Zheng orig_lowest = path->lowest_level; 5597b3b4aa74SDavid Sterba btrfs_release_path(path); 559833c66f43SYan Zheng path->lowest_level = level; 559933c66f43SYan Zheng ret = btrfs_search_slot(NULL, root, &cur_key, path, 560033c66f43SYan Zheng 0, 0); 560133c66f43SYan Zheng path->lowest_level = orig_lowest; 560233c66f43SYan Zheng if (ret < 0) 560333c66f43SYan Zheng return ret; 560433c66f43SYan Zheng 560533c66f43SYan Zheng c = path->nodes[level]; 560633c66f43SYan Zheng slot = path->slots[level]; 560733c66f43SYan Zheng if (ret == 0) 560833c66f43SYan Zheng slot++; 560933c66f43SYan Zheng goto next; 561033c66f43SYan Zheng } 561133c66f43SYan Zheng 5612e7a84565SChris Mason if (level == 0) 5613e7a84565SChris Mason btrfs_item_key_to_cpu(c, key, slot); 56143f157a2fSChris Mason else { 56153f157a2fSChris Mason u64 gen = btrfs_node_ptr_generation(c, slot); 56163f157a2fSChris Mason 56173f157a2fSChris Mason if (gen < min_trans) { 56183f157a2fSChris Mason slot++; 56193f157a2fSChris Mason goto next; 56203f157a2fSChris Mason } 5621e7a84565SChris Mason btrfs_node_key_to_cpu(c, key, slot); 56223f157a2fSChris Mason } 5623e7a84565SChris Mason return 0; 5624e7a84565SChris Mason } 5625e7a84565SChris Mason return 1; 5626e7a84565SChris Mason } 5627e7a84565SChris Mason 56287bb86316SChris Mason /* 5629925baeddSChris Mason * search the tree again to find a leaf with greater keys 56300f70abe2SChris Mason * returns 0 if it found something or 1 if there are no greater leaves. 56310f70abe2SChris Mason * returns < 0 on io errors. 563297571fd0SChris Mason */ 5633234b63a0SChris Mason int btrfs_next_leaf(struct btrfs_root *root, struct btrfs_path *path) 5634d97e63b6SChris Mason { 56353d7806ecSJan Schmidt return btrfs_next_old_leaf(root, path, 0); 56363d7806ecSJan Schmidt } 56373d7806ecSJan Schmidt 56383d7806ecSJan Schmidt int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path, 56393d7806ecSJan Schmidt u64 time_seq) 56403d7806ecSJan Schmidt { 5641d97e63b6SChris Mason int slot; 56428e73f275SChris Mason int level; 56435f39d397SChris Mason struct extent_buffer *c; 56448e73f275SChris Mason struct extent_buffer *next; 5645925baeddSChris Mason struct btrfs_key key; 5646925baeddSChris Mason u32 nritems; 5647925baeddSChris Mason int ret; 56488e73f275SChris Mason int old_spinning = path->leave_spinning; 5649bd681513SChris Mason int next_rw_lock = 0; 5650925baeddSChris Mason 5651925baeddSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 5652d397712bSChris Mason if (nritems == 0) 5653925baeddSChris Mason return 1; 5654925baeddSChris Mason 56558e73f275SChris Mason btrfs_item_key_to_cpu(path->nodes[0], &key, nritems - 1); 56568e73f275SChris Mason again: 56578e73f275SChris Mason level = 1; 56588e73f275SChris Mason next = NULL; 5659bd681513SChris Mason next_rw_lock = 0; 5660b3b4aa74SDavid Sterba btrfs_release_path(path); 56618e73f275SChris Mason 5662a2135011SChris Mason path->keep_locks = 1; 56638e73f275SChris Mason path->leave_spinning = 1; 56648e73f275SChris Mason 56653d7806ecSJan Schmidt if (time_seq) 56663d7806ecSJan Schmidt ret = btrfs_search_old_slot(root, &key, path, time_seq); 56673d7806ecSJan Schmidt else 5668925baeddSChris Mason ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 5669925baeddSChris Mason path->keep_locks = 0; 5670925baeddSChris Mason 5671925baeddSChris Mason if (ret < 0) 5672925baeddSChris Mason return ret; 5673925baeddSChris Mason 5674a2135011SChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 5675168fd7d2SChris Mason /* 5676168fd7d2SChris Mason * by releasing the path above we dropped all our locks. A balance 5677168fd7d2SChris Mason * could have added more items next to the key that used to be 5678168fd7d2SChris Mason * at the very end of the block. So, check again here and 5679168fd7d2SChris Mason * advance the path if there are now more items available. 5680168fd7d2SChris Mason */ 5681a2135011SChris Mason if (nritems > 0 && path->slots[0] < nritems - 1) { 5682e457afecSYan Zheng if (ret == 0) 5683168fd7d2SChris Mason path->slots[0]++; 56848e73f275SChris Mason ret = 0; 5685925baeddSChris Mason goto done; 5686925baeddSChris Mason } 56870b43e04fSLiu Bo /* 56880b43e04fSLiu Bo * So the above check misses one case: 56890b43e04fSLiu Bo * - after releasing the path above, someone has removed the item that 56900b43e04fSLiu Bo * used to be at the very end of the block, and balance between leafs 56910b43e04fSLiu Bo * gets another one with bigger key.offset to replace it. 56920b43e04fSLiu Bo * 56930b43e04fSLiu Bo * This one should be returned as well, or we can get leaf corruption 56940b43e04fSLiu Bo * later(esp. in __btrfs_drop_extents()). 56950b43e04fSLiu Bo * 56960b43e04fSLiu Bo * And a bit more explanation about this check, 56970b43e04fSLiu Bo * with ret > 0, the key isn't found, the path points to the slot 56980b43e04fSLiu Bo * where it should be inserted, so the path->slots[0] item must be the 56990b43e04fSLiu Bo * bigger one. 57000b43e04fSLiu Bo */ 57010b43e04fSLiu Bo if (nritems > 0 && ret > 0 && path->slots[0] == nritems - 1) { 57020b43e04fSLiu Bo ret = 0; 57030b43e04fSLiu Bo goto done; 57040b43e04fSLiu Bo } 5705d97e63b6SChris Mason 5706234b63a0SChris Mason while (level < BTRFS_MAX_LEVEL) { 57078e73f275SChris Mason if (!path->nodes[level]) { 57088e73f275SChris Mason ret = 1; 57098e73f275SChris Mason goto done; 57108e73f275SChris Mason } 57115f39d397SChris Mason 5712d97e63b6SChris Mason slot = path->slots[level] + 1; 5713d97e63b6SChris Mason c = path->nodes[level]; 57145f39d397SChris Mason if (slot >= btrfs_header_nritems(c)) { 5715d97e63b6SChris Mason level++; 57168e73f275SChris Mason if (level == BTRFS_MAX_LEVEL) { 57178e73f275SChris Mason ret = 1; 57188e73f275SChris Mason goto done; 57198e73f275SChris Mason } 5720d97e63b6SChris Mason continue; 5721d97e63b6SChris Mason } 57225f39d397SChris Mason 5723925baeddSChris Mason if (next) { 5724bd681513SChris Mason btrfs_tree_unlock_rw(next, next_rw_lock); 57255f39d397SChris Mason free_extent_buffer(next); 5726925baeddSChris Mason } 57275f39d397SChris Mason 57288e73f275SChris Mason next = c; 5729bd681513SChris Mason next_rw_lock = path->locks[level]; 5730d07b8528SLiu Bo ret = read_block_for_search(root, path, &next, level, 5731cda79c54SDavid Sterba slot, &key); 57328e73f275SChris Mason if (ret == -EAGAIN) 57338e73f275SChris Mason goto again; 57345f39d397SChris Mason 573576a05b35SChris Mason if (ret < 0) { 5736b3b4aa74SDavid Sterba btrfs_release_path(path); 573776a05b35SChris Mason goto done; 573876a05b35SChris Mason } 573976a05b35SChris Mason 57405cd57b2cSChris Mason if (!path->skip_locking) { 5741bd681513SChris Mason ret = btrfs_try_tree_read_lock(next); 5742d42244a0SJan Schmidt if (!ret && time_seq) { 5743d42244a0SJan Schmidt /* 5744d42244a0SJan Schmidt * If we don't get the lock, we may be racing 5745d42244a0SJan Schmidt * with push_leaf_left, holding that lock while 5746d42244a0SJan Schmidt * itself waiting for the leaf we've currently 5747d42244a0SJan Schmidt * locked. To solve this situation, we give up 5748d42244a0SJan Schmidt * on our lock and cycle. 5749d42244a0SJan Schmidt */ 5750cf538830SJan Schmidt free_extent_buffer(next); 5751d42244a0SJan Schmidt btrfs_release_path(path); 5752d42244a0SJan Schmidt cond_resched(); 5753d42244a0SJan Schmidt goto again; 5754d42244a0SJan Schmidt } 57558e73f275SChris Mason if (!ret) { 57568e73f275SChris Mason btrfs_set_path_blocking(path); 5757bd681513SChris Mason btrfs_tree_read_lock(next); 57588e73f275SChris Mason } 5759bd681513SChris Mason next_rw_lock = BTRFS_READ_LOCK; 5760bd681513SChris Mason } 5761d97e63b6SChris Mason break; 5762d97e63b6SChris Mason } 5763d97e63b6SChris Mason path->slots[level] = slot; 5764d97e63b6SChris Mason while (1) { 5765d97e63b6SChris Mason level--; 5766d97e63b6SChris Mason c = path->nodes[level]; 5767925baeddSChris Mason if (path->locks[level]) 5768bd681513SChris Mason btrfs_tree_unlock_rw(c, path->locks[level]); 57698e73f275SChris Mason 57705f39d397SChris Mason free_extent_buffer(c); 5771d97e63b6SChris Mason path->nodes[level] = next; 5772d97e63b6SChris Mason path->slots[level] = 0; 5773a74a4b97SChris Mason if (!path->skip_locking) 5774bd681513SChris Mason path->locks[level] = next_rw_lock; 5775d97e63b6SChris Mason if (!level) 5776d97e63b6SChris Mason break; 5777b4ce94deSChris Mason 5778d07b8528SLiu Bo ret = read_block_for_search(root, path, &next, level, 5779cda79c54SDavid Sterba 0, &key); 57808e73f275SChris Mason if (ret == -EAGAIN) 57818e73f275SChris Mason goto again; 57828e73f275SChris Mason 578376a05b35SChris Mason if (ret < 0) { 5784b3b4aa74SDavid Sterba btrfs_release_path(path); 578576a05b35SChris Mason goto done; 578676a05b35SChris Mason } 578776a05b35SChris Mason 57885cd57b2cSChris Mason if (!path->skip_locking) { 5789bd681513SChris Mason ret = btrfs_try_tree_read_lock(next); 57908e73f275SChris Mason if (!ret) { 57918e73f275SChris Mason btrfs_set_path_blocking(path); 5792bd681513SChris Mason btrfs_tree_read_lock(next); 57938e73f275SChris Mason } 5794bd681513SChris Mason next_rw_lock = BTRFS_READ_LOCK; 5795bd681513SChris Mason } 5796d97e63b6SChris Mason } 57978e73f275SChris Mason ret = 0; 5798925baeddSChris Mason done: 5799f7c79f30SChris Mason unlock_up(path, 0, 1, 0, NULL); 58008e73f275SChris Mason path->leave_spinning = old_spinning; 58018e73f275SChris Mason if (!old_spinning) 58028e73f275SChris Mason btrfs_set_path_blocking(path); 58038e73f275SChris Mason 58048e73f275SChris Mason return ret; 5805d97e63b6SChris Mason } 58060b86a832SChris Mason 58073f157a2fSChris Mason /* 58083f157a2fSChris Mason * this uses btrfs_prev_leaf to walk backwards in the tree, and keeps 58093f157a2fSChris Mason * searching until it gets past min_objectid or finds an item of 'type' 58103f157a2fSChris Mason * 58113f157a2fSChris Mason * returns 0 if something is found, 1 if nothing was found and < 0 on error 58123f157a2fSChris Mason */ 58130b86a832SChris Mason int btrfs_previous_item(struct btrfs_root *root, 58140b86a832SChris Mason struct btrfs_path *path, u64 min_objectid, 58150b86a832SChris Mason int type) 58160b86a832SChris Mason { 58170b86a832SChris Mason struct btrfs_key found_key; 58180b86a832SChris Mason struct extent_buffer *leaf; 5819e02119d5SChris Mason u32 nritems; 58200b86a832SChris Mason int ret; 58210b86a832SChris Mason 58220b86a832SChris Mason while (1) { 58230b86a832SChris Mason if (path->slots[0] == 0) { 5824b4ce94deSChris Mason btrfs_set_path_blocking(path); 58250b86a832SChris Mason ret = btrfs_prev_leaf(root, path); 58260b86a832SChris Mason if (ret != 0) 58270b86a832SChris Mason return ret; 58280b86a832SChris Mason } else { 58290b86a832SChris Mason path->slots[0]--; 58300b86a832SChris Mason } 58310b86a832SChris Mason leaf = path->nodes[0]; 5832e02119d5SChris Mason nritems = btrfs_header_nritems(leaf); 5833e02119d5SChris Mason if (nritems == 0) 5834e02119d5SChris Mason return 1; 5835e02119d5SChris Mason if (path->slots[0] == nritems) 5836e02119d5SChris Mason path->slots[0]--; 5837e02119d5SChris Mason 58380b86a832SChris Mason btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 5839e02119d5SChris Mason if (found_key.objectid < min_objectid) 5840e02119d5SChris Mason break; 58410a4eefbbSYan Zheng if (found_key.type == type) 58420a4eefbbSYan Zheng return 0; 5843e02119d5SChris Mason if (found_key.objectid == min_objectid && 5844e02119d5SChris Mason found_key.type < type) 5845e02119d5SChris Mason break; 58460b86a832SChris Mason } 58470b86a832SChris Mason return 1; 58480b86a832SChris Mason } 5849ade2e0b3SWang Shilong 5850ade2e0b3SWang Shilong /* 5851ade2e0b3SWang Shilong * search in extent tree to find a previous Metadata/Data extent item with 5852ade2e0b3SWang Shilong * min objecitd. 5853ade2e0b3SWang Shilong * 5854ade2e0b3SWang Shilong * returns 0 if something is found, 1 if nothing was found and < 0 on error 5855ade2e0b3SWang Shilong */ 5856ade2e0b3SWang Shilong int btrfs_previous_extent_item(struct btrfs_root *root, 5857ade2e0b3SWang Shilong struct btrfs_path *path, u64 min_objectid) 5858ade2e0b3SWang Shilong { 5859ade2e0b3SWang Shilong struct btrfs_key found_key; 5860ade2e0b3SWang Shilong struct extent_buffer *leaf; 5861ade2e0b3SWang Shilong u32 nritems; 5862ade2e0b3SWang Shilong int ret; 5863ade2e0b3SWang Shilong 5864ade2e0b3SWang Shilong while (1) { 5865ade2e0b3SWang Shilong if (path->slots[0] == 0) { 5866ade2e0b3SWang Shilong btrfs_set_path_blocking(path); 5867ade2e0b3SWang Shilong ret = btrfs_prev_leaf(root, path); 5868ade2e0b3SWang Shilong if (ret != 0) 5869ade2e0b3SWang Shilong return ret; 5870ade2e0b3SWang Shilong } else { 5871ade2e0b3SWang Shilong path->slots[0]--; 5872ade2e0b3SWang Shilong } 5873ade2e0b3SWang Shilong leaf = path->nodes[0]; 5874ade2e0b3SWang Shilong nritems = btrfs_header_nritems(leaf); 5875ade2e0b3SWang Shilong if (nritems == 0) 5876ade2e0b3SWang Shilong return 1; 5877ade2e0b3SWang Shilong if (path->slots[0] == nritems) 5878ade2e0b3SWang Shilong path->slots[0]--; 5879ade2e0b3SWang Shilong 5880ade2e0b3SWang Shilong btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 5881ade2e0b3SWang Shilong if (found_key.objectid < min_objectid) 5882ade2e0b3SWang Shilong break; 5883ade2e0b3SWang Shilong if (found_key.type == BTRFS_EXTENT_ITEM_KEY || 5884ade2e0b3SWang Shilong found_key.type == BTRFS_METADATA_ITEM_KEY) 5885ade2e0b3SWang Shilong return 0; 5886ade2e0b3SWang Shilong if (found_key.objectid == min_objectid && 5887ade2e0b3SWang Shilong found_key.type < BTRFS_EXTENT_ITEM_KEY) 5888ade2e0b3SWang Shilong break; 5889ade2e0b3SWang Shilong } 5890ade2e0b3SWang Shilong return 1; 5891ade2e0b3SWang Shilong } 5892