16cbd5570SChris Mason /* 2d352ac68SChris Mason * Copyright (C) 2007,2008 Oracle. All rights reserved. 36cbd5570SChris Mason * 46cbd5570SChris Mason * This program is free software; you can redistribute it and/or 56cbd5570SChris Mason * modify it under the terms of the GNU General Public 66cbd5570SChris Mason * License v2 as published by the Free Software Foundation. 76cbd5570SChris Mason * 86cbd5570SChris Mason * This program is distributed in the hope that it will be useful, 96cbd5570SChris Mason * but WITHOUT ANY WARRANTY; without even the implied warranty of 106cbd5570SChris Mason * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 116cbd5570SChris Mason * General Public License for more details. 126cbd5570SChris Mason * 136cbd5570SChris Mason * You should have received a copy of the GNU General Public 146cbd5570SChris Mason * License along with this program; if not, write to the 156cbd5570SChris Mason * Free Software Foundation, Inc., 59 Temple Place - Suite 330, 166cbd5570SChris Mason * Boston, MA 021110-1307, USA. 176cbd5570SChris Mason */ 186cbd5570SChris Mason 19a6b6e75eSChris Mason #include <linux/sched.h> 205a0e3ad6STejun Heo #include <linux/slab.h> 21bd989ba3SJan Schmidt #include <linux/rbtree.h> 22eb60ceacSChris Mason #include "ctree.h" 23eb60ceacSChris Mason #include "disk-io.h" 247f5c1516SChris Mason #include "transaction.h" 255f39d397SChris Mason #include "print-tree.h" 26925baeddSChris Mason #include "locking.h" 279a8dd150SChris Mason 28e089f05cSChris Mason static int split_node(struct btrfs_trans_handle *trans, struct btrfs_root 29e089f05cSChris Mason *root, struct btrfs_path *path, int level); 30e089f05cSChris Mason static int split_leaf(struct btrfs_trans_handle *trans, struct btrfs_root 31d4dbff95SChris Mason *root, struct btrfs_key *ins_key, 32cc0c5538SChris Mason struct btrfs_path *path, int data_size, int extend); 335f39d397SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 345f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *dst, 35971a1f66SChris Mason struct extent_buffer *src, int empty); 365f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 375f39d397SChris Mason struct btrfs_root *root, 385f39d397SChris Mason struct extent_buffer *dst_buf, 395f39d397SChris Mason struct extent_buffer *src_buf); 40afe5fea7STsutomu Itoh static void del_ptr(struct btrfs_root *root, struct btrfs_path *path, 41afe5fea7STsutomu Itoh int level, int slot); 42f230475eSJan Schmidt static void tree_mod_log_free_eb(struct btrfs_fs_info *fs_info, 43f230475eSJan Schmidt struct extent_buffer *eb); 4448a3b636SEric Sandeen static int btrfs_prev_leaf(struct btrfs_root *root, struct btrfs_path *path); 45d97e63b6SChris Mason 462c90e5d6SChris Mason struct btrfs_path *btrfs_alloc_path(void) 472c90e5d6SChris Mason { 48df24a2b9SChris Mason struct btrfs_path *path; 49e00f7308SJeff Mahoney path = kmem_cache_zalloc(btrfs_path_cachep, GFP_NOFS); 50df24a2b9SChris Mason return path; 512c90e5d6SChris Mason } 522c90e5d6SChris Mason 53b4ce94deSChris Mason /* 54b4ce94deSChris Mason * set all locked nodes in the path to blocking locks. This should 55b4ce94deSChris Mason * be done before scheduling 56b4ce94deSChris Mason */ 57b4ce94deSChris Mason noinline void btrfs_set_path_blocking(struct btrfs_path *p) 58b4ce94deSChris Mason { 59b4ce94deSChris Mason int i; 60b4ce94deSChris Mason for (i = 0; i < BTRFS_MAX_LEVEL; i++) { 61bd681513SChris Mason if (!p->nodes[i] || !p->locks[i]) 62bd681513SChris Mason continue; 63bd681513SChris Mason btrfs_set_lock_blocking_rw(p->nodes[i], p->locks[i]); 64bd681513SChris Mason if (p->locks[i] == BTRFS_READ_LOCK) 65bd681513SChris Mason p->locks[i] = BTRFS_READ_LOCK_BLOCKING; 66bd681513SChris Mason else if (p->locks[i] == BTRFS_WRITE_LOCK) 67bd681513SChris Mason p->locks[i] = BTRFS_WRITE_LOCK_BLOCKING; 68b4ce94deSChris Mason } 69b4ce94deSChris Mason } 70b4ce94deSChris Mason 71b4ce94deSChris Mason /* 72b4ce94deSChris Mason * reset all the locked nodes in the patch to spinning locks. 734008c04aSChris Mason * 744008c04aSChris Mason * held is used to keep lockdep happy, when lockdep is enabled 754008c04aSChris Mason * we set held to a blocking lock before we go around and 764008c04aSChris Mason * retake all the spinlocks in the path. You can safely use NULL 774008c04aSChris Mason * for held 78b4ce94deSChris Mason */ 794008c04aSChris Mason noinline void btrfs_clear_path_blocking(struct btrfs_path *p, 80bd681513SChris Mason struct extent_buffer *held, int held_rw) 81b4ce94deSChris Mason { 82b4ce94deSChris Mason int i; 834008c04aSChris Mason 844008c04aSChris Mason #ifdef CONFIG_DEBUG_LOCK_ALLOC 854008c04aSChris Mason /* lockdep really cares that we take all of these spinlocks 864008c04aSChris Mason * in the right order. If any of the locks in the path are not 874008c04aSChris Mason * currently blocking, it is going to complain. So, make really 884008c04aSChris Mason * really sure by forcing the path to blocking before we clear 894008c04aSChris Mason * the path blocking. 904008c04aSChris Mason */ 91bd681513SChris Mason if (held) { 92bd681513SChris Mason btrfs_set_lock_blocking_rw(held, held_rw); 93bd681513SChris Mason if (held_rw == BTRFS_WRITE_LOCK) 94bd681513SChris Mason held_rw = BTRFS_WRITE_LOCK_BLOCKING; 95bd681513SChris Mason else if (held_rw == BTRFS_READ_LOCK) 96bd681513SChris Mason held_rw = BTRFS_READ_LOCK_BLOCKING; 97bd681513SChris Mason } 984008c04aSChris Mason btrfs_set_path_blocking(p); 994008c04aSChris Mason #endif 1004008c04aSChris Mason 1014008c04aSChris Mason for (i = BTRFS_MAX_LEVEL - 1; i >= 0; i--) { 102bd681513SChris Mason if (p->nodes[i] && p->locks[i]) { 103bd681513SChris Mason btrfs_clear_lock_blocking_rw(p->nodes[i], p->locks[i]); 104bd681513SChris Mason if (p->locks[i] == BTRFS_WRITE_LOCK_BLOCKING) 105bd681513SChris Mason p->locks[i] = BTRFS_WRITE_LOCK; 106bd681513SChris Mason else if (p->locks[i] == BTRFS_READ_LOCK_BLOCKING) 107bd681513SChris Mason p->locks[i] = BTRFS_READ_LOCK; 108bd681513SChris Mason } 109b4ce94deSChris Mason } 1104008c04aSChris Mason 1114008c04aSChris Mason #ifdef CONFIG_DEBUG_LOCK_ALLOC 1124008c04aSChris Mason if (held) 113bd681513SChris Mason btrfs_clear_lock_blocking_rw(held, held_rw); 1144008c04aSChris Mason #endif 115b4ce94deSChris Mason } 116b4ce94deSChris Mason 117d352ac68SChris Mason /* this also releases the path */ 1182c90e5d6SChris Mason void btrfs_free_path(struct btrfs_path *p) 1192c90e5d6SChris Mason { 120ff175d57SJesper Juhl if (!p) 121ff175d57SJesper Juhl return; 122b3b4aa74SDavid Sterba btrfs_release_path(p); 1232c90e5d6SChris Mason kmem_cache_free(btrfs_path_cachep, p); 1242c90e5d6SChris Mason } 1252c90e5d6SChris Mason 126d352ac68SChris Mason /* 127d352ac68SChris Mason * path release drops references on the extent buffers in the path 128d352ac68SChris Mason * and it drops any locks held by this path 129d352ac68SChris Mason * 130d352ac68SChris Mason * It is safe to call this on paths that no locks or extent buffers held. 131d352ac68SChris Mason */ 132b3b4aa74SDavid Sterba noinline void btrfs_release_path(struct btrfs_path *p) 133eb60ceacSChris Mason { 134eb60ceacSChris Mason int i; 135a2135011SChris Mason 136234b63a0SChris Mason for (i = 0; i < BTRFS_MAX_LEVEL; i++) { 1373f157a2fSChris Mason p->slots[i] = 0; 138eb60ceacSChris Mason if (!p->nodes[i]) 139925baeddSChris Mason continue; 140925baeddSChris Mason if (p->locks[i]) { 141bd681513SChris Mason btrfs_tree_unlock_rw(p->nodes[i], p->locks[i]); 142925baeddSChris Mason p->locks[i] = 0; 143925baeddSChris Mason } 1445f39d397SChris Mason free_extent_buffer(p->nodes[i]); 1453f157a2fSChris Mason p->nodes[i] = NULL; 146eb60ceacSChris Mason } 147eb60ceacSChris Mason } 148eb60ceacSChris Mason 149d352ac68SChris Mason /* 150d352ac68SChris Mason * safely gets a reference on the root node of a tree. A lock 151d352ac68SChris Mason * is not taken, so a concurrent writer may put a different node 152d352ac68SChris Mason * at the root of the tree. See btrfs_lock_root_node for the 153d352ac68SChris Mason * looping required. 154d352ac68SChris Mason * 155d352ac68SChris Mason * The extent buffer returned by this has a reference taken, so 156d352ac68SChris Mason * it won't disappear. It may stop being the root of the tree 157d352ac68SChris Mason * at any time because there are no locks held. 158d352ac68SChris Mason */ 159925baeddSChris Mason struct extent_buffer *btrfs_root_node(struct btrfs_root *root) 160925baeddSChris Mason { 161925baeddSChris Mason struct extent_buffer *eb; 162240f62c8SChris Mason 1633083ee2eSJosef Bacik while (1) { 164240f62c8SChris Mason rcu_read_lock(); 165240f62c8SChris Mason eb = rcu_dereference(root->node); 1663083ee2eSJosef Bacik 1673083ee2eSJosef Bacik /* 1683083ee2eSJosef Bacik * RCU really hurts here, we could free up the root node because 1693083ee2eSJosef Bacik * it was cow'ed but we may not get the new root node yet so do 1703083ee2eSJosef Bacik * the inc_not_zero dance and if it doesn't work then 1713083ee2eSJosef Bacik * synchronize_rcu and try again. 1723083ee2eSJosef Bacik */ 1733083ee2eSJosef Bacik if (atomic_inc_not_zero(&eb->refs)) { 174240f62c8SChris Mason rcu_read_unlock(); 1753083ee2eSJosef Bacik break; 1763083ee2eSJosef Bacik } 1773083ee2eSJosef Bacik rcu_read_unlock(); 1783083ee2eSJosef Bacik synchronize_rcu(); 1793083ee2eSJosef Bacik } 180925baeddSChris Mason return eb; 181925baeddSChris Mason } 182925baeddSChris Mason 183d352ac68SChris Mason /* loop around taking references on and locking the root node of the 184d352ac68SChris Mason * tree until you end up with a lock on the root. A locked buffer 185d352ac68SChris Mason * is returned, with a reference held. 186d352ac68SChris Mason */ 187925baeddSChris Mason struct extent_buffer *btrfs_lock_root_node(struct btrfs_root *root) 188925baeddSChris Mason { 189925baeddSChris Mason struct extent_buffer *eb; 190925baeddSChris Mason 191925baeddSChris Mason while (1) { 192925baeddSChris Mason eb = btrfs_root_node(root); 193925baeddSChris Mason btrfs_tree_lock(eb); 194240f62c8SChris Mason if (eb == root->node) 195925baeddSChris Mason break; 196925baeddSChris Mason btrfs_tree_unlock(eb); 197925baeddSChris Mason free_extent_buffer(eb); 198925baeddSChris Mason } 199925baeddSChris Mason return eb; 200925baeddSChris Mason } 201925baeddSChris Mason 202bd681513SChris Mason /* loop around taking references on and locking the root node of the 203bd681513SChris Mason * tree until you end up with a lock on the root. A locked buffer 204bd681513SChris Mason * is returned, with a reference held. 205bd681513SChris Mason */ 20648a3b636SEric Sandeen static struct extent_buffer *btrfs_read_lock_root_node(struct btrfs_root *root) 207bd681513SChris Mason { 208bd681513SChris Mason struct extent_buffer *eb; 209bd681513SChris Mason 210bd681513SChris Mason while (1) { 211bd681513SChris Mason eb = btrfs_root_node(root); 212bd681513SChris Mason btrfs_tree_read_lock(eb); 213bd681513SChris Mason if (eb == root->node) 214bd681513SChris Mason break; 215bd681513SChris Mason btrfs_tree_read_unlock(eb); 216bd681513SChris Mason free_extent_buffer(eb); 217bd681513SChris Mason } 218bd681513SChris Mason return eb; 219bd681513SChris Mason } 220bd681513SChris Mason 221d352ac68SChris Mason /* cowonly root (everything not a reference counted cow subvolume), just get 222d352ac68SChris Mason * put onto a simple dirty list. transaction.c walks this to make sure they 223d352ac68SChris Mason * get properly updated on disk. 224d352ac68SChris Mason */ 2250b86a832SChris Mason static void add_root_to_dirty_list(struct btrfs_root *root) 2260b86a832SChris Mason { 227e5846fc6SChris Mason spin_lock(&root->fs_info->trans_lock); 2280b86a832SChris Mason if (root->track_dirty && list_empty(&root->dirty_list)) { 2290b86a832SChris Mason list_add(&root->dirty_list, 2300b86a832SChris Mason &root->fs_info->dirty_cowonly_roots); 2310b86a832SChris Mason } 232e5846fc6SChris Mason spin_unlock(&root->fs_info->trans_lock); 2330b86a832SChris Mason } 2340b86a832SChris Mason 235d352ac68SChris Mason /* 236d352ac68SChris Mason * used by snapshot creation to make a copy of a root for a tree with 237d352ac68SChris Mason * a given objectid. The buffer with the new root node is returned in 238d352ac68SChris Mason * cow_ret, and this func returns zero on success or a negative error code. 239d352ac68SChris Mason */ 240be20aa9dSChris Mason int btrfs_copy_root(struct btrfs_trans_handle *trans, 241be20aa9dSChris Mason struct btrfs_root *root, 242be20aa9dSChris Mason struct extent_buffer *buf, 243be20aa9dSChris Mason struct extent_buffer **cow_ret, u64 new_root_objectid) 244be20aa9dSChris Mason { 245be20aa9dSChris Mason struct extent_buffer *cow; 246be20aa9dSChris Mason int ret = 0; 247be20aa9dSChris Mason int level; 2485d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 249be20aa9dSChris Mason 250be20aa9dSChris Mason WARN_ON(root->ref_cows && trans->transid != 251be20aa9dSChris Mason root->fs_info->running_transaction->transid); 252be20aa9dSChris Mason WARN_ON(root->ref_cows && trans->transid != root->last_trans); 253be20aa9dSChris Mason 254be20aa9dSChris Mason level = btrfs_header_level(buf); 2555d4f98a2SYan Zheng if (level == 0) 2565d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 2575d4f98a2SYan Zheng else 2585d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 25931840ae1SZheng Yan 2605d4f98a2SYan Zheng cow = btrfs_alloc_free_block(trans, root, buf->len, 0, 2615d4f98a2SYan Zheng new_root_objectid, &disk_key, level, 2625581a51aSJan Schmidt buf->start, 0); 2635d4f98a2SYan Zheng if (IS_ERR(cow)) 264be20aa9dSChris Mason return PTR_ERR(cow); 265be20aa9dSChris Mason 266be20aa9dSChris Mason copy_extent_buffer(cow, buf, 0, 0, cow->len); 267be20aa9dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 268be20aa9dSChris Mason btrfs_set_header_generation(cow, trans->transid); 2695d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 2705d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 2715d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 2725d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 2735d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 2745d4f98a2SYan Zheng else 275be20aa9dSChris Mason btrfs_set_header_owner(cow, new_root_objectid); 276be20aa9dSChris Mason 2772b82032cSYan Zheng write_extent_buffer(cow, root->fs_info->fsid, 2782b82032cSYan Zheng (unsigned long)btrfs_header_fsid(cow), 2792b82032cSYan Zheng BTRFS_FSID_SIZE); 2802b82032cSYan Zheng 281be20aa9dSChris Mason WARN_ON(btrfs_header_generation(buf) > trans->transid); 2825d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 28366d7e7f0SArne Jansen ret = btrfs_inc_ref(trans, root, cow, 1, 1); 2845d4f98a2SYan Zheng else 28566d7e7f0SArne Jansen ret = btrfs_inc_ref(trans, root, cow, 0, 1); 2864aec2b52SChris Mason 287be20aa9dSChris Mason if (ret) 288be20aa9dSChris Mason return ret; 289be20aa9dSChris Mason 290be20aa9dSChris Mason btrfs_mark_buffer_dirty(cow); 291be20aa9dSChris Mason *cow_ret = cow; 292be20aa9dSChris Mason return 0; 293be20aa9dSChris Mason } 294be20aa9dSChris Mason 295bd989ba3SJan Schmidt enum mod_log_op { 296bd989ba3SJan Schmidt MOD_LOG_KEY_REPLACE, 297bd989ba3SJan Schmidt MOD_LOG_KEY_ADD, 298bd989ba3SJan Schmidt MOD_LOG_KEY_REMOVE, 299bd989ba3SJan Schmidt MOD_LOG_KEY_REMOVE_WHILE_FREEING, 300bd989ba3SJan Schmidt MOD_LOG_KEY_REMOVE_WHILE_MOVING, 301bd989ba3SJan Schmidt MOD_LOG_MOVE_KEYS, 302bd989ba3SJan Schmidt MOD_LOG_ROOT_REPLACE, 303bd989ba3SJan Schmidt }; 304bd989ba3SJan Schmidt 305bd989ba3SJan Schmidt struct tree_mod_move { 306bd989ba3SJan Schmidt int dst_slot; 307bd989ba3SJan Schmidt int nr_items; 308bd989ba3SJan Schmidt }; 309bd989ba3SJan Schmidt 310bd989ba3SJan Schmidt struct tree_mod_root { 311bd989ba3SJan Schmidt u64 logical; 312bd989ba3SJan Schmidt u8 level; 313bd989ba3SJan Schmidt }; 314bd989ba3SJan Schmidt 315bd989ba3SJan Schmidt struct tree_mod_elem { 316bd989ba3SJan Schmidt struct rb_node node; 317bd989ba3SJan Schmidt u64 index; /* shifted logical */ 318097b8a7cSJan Schmidt u64 seq; 319bd989ba3SJan Schmidt enum mod_log_op op; 320bd989ba3SJan Schmidt 321bd989ba3SJan Schmidt /* this is used for MOD_LOG_KEY_* and MOD_LOG_MOVE_KEYS operations */ 322bd989ba3SJan Schmidt int slot; 323bd989ba3SJan Schmidt 324bd989ba3SJan Schmidt /* this is used for MOD_LOG_KEY* and MOD_LOG_ROOT_REPLACE */ 325bd989ba3SJan Schmidt u64 generation; 326bd989ba3SJan Schmidt 327bd989ba3SJan Schmidt /* those are used for op == MOD_LOG_KEY_{REPLACE,REMOVE} */ 328bd989ba3SJan Schmidt struct btrfs_disk_key key; 329bd989ba3SJan Schmidt u64 blockptr; 330bd989ba3SJan Schmidt 331bd989ba3SJan Schmidt /* this is used for op == MOD_LOG_MOVE_KEYS */ 332bd989ba3SJan Schmidt struct tree_mod_move move; 333bd989ba3SJan Schmidt 334bd989ba3SJan Schmidt /* this is used for op == MOD_LOG_ROOT_REPLACE */ 335bd989ba3SJan Schmidt struct tree_mod_root old_root; 336bd989ba3SJan Schmidt }; 337bd989ba3SJan Schmidt 338097b8a7cSJan Schmidt static inline void tree_mod_log_read_lock(struct btrfs_fs_info *fs_info) 339bd989ba3SJan Schmidt { 340097b8a7cSJan Schmidt read_lock(&fs_info->tree_mod_log_lock); 341bd989ba3SJan Schmidt } 342bd989ba3SJan Schmidt 343097b8a7cSJan Schmidt static inline void tree_mod_log_read_unlock(struct btrfs_fs_info *fs_info) 344097b8a7cSJan Schmidt { 345097b8a7cSJan Schmidt read_unlock(&fs_info->tree_mod_log_lock); 346097b8a7cSJan Schmidt } 347097b8a7cSJan Schmidt 348097b8a7cSJan Schmidt static inline void tree_mod_log_write_lock(struct btrfs_fs_info *fs_info) 349097b8a7cSJan Schmidt { 350097b8a7cSJan Schmidt write_lock(&fs_info->tree_mod_log_lock); 351097b8a7cSJan Schmidt } 352097b8a7cSJan Schmidt 353097b8a7cSJan Schmidt static inline void tree_mod_log_write_unlock(struct btrfs_fs_info *fs_info) 354097b8a7cSJan Schmidt { 355097b8a7cSJan Schmidt write_unlock(&fs_info->tree_mod_log_lock); 356097b8a7cSJan Schmidt } 357097b8a7cSJan Schmidt 358097b8a7cSJan Schmidt /* 359fc36ed7eSJan Schmidt * Increment the upper half of tree_mod_seq, set lower half zero. 360fc36ed7eSJan Schmidt * 361fc36ed7eSJan Schmidt * Must be called with fs_info->tree_mod_seq_lock held. 362fc36ed7eSJan Schmidt */ 363fc36ed7eSJan Schmidt static inline u64 btrfs_inc_tree_mod_seq_major(struct btrfs_fs_info *fs_info) 364fc36ed7eSJan Schmidt { 365fc36ed7eSJan Schmidt u64 seq = atomic64_read(&fs_info->tree_mod_seq); 366fc36ed7eSJan Schmidt seq &= 0xffffffff00000000ull; 367fc36ed7eSJan Schmidt seq += 1ull << 32; 368fc36ed7eSJan Schmidt atomic64_set(&fs_info->tree_mod_seq, seq); 369fc36ed7eSJan Schmidt return seq; 370fc36ed7eSJan Schmidt } 371fc36ed7eSJan Schmidt 372fc36ed7eSJan Schmidt /* 373fc36ed7eSJan Schmidt * Increment the lower half of tree_mod_seq. 374fc36ed7eSJan Schmidt * 375fc36ed7eSJan Schmidt * Must be called with fs_info->tree_mod_seq_lock held. The way major numbers 376fc36ed7eSJan Schmidt * are generated should not technically require a spin lock here. (Rationale: 377fc36ed7eSJan Schmidt * incrementing the minor while incrementing the major seq number is between its 378fc36ed7eSJan Schmidt * atomic64_read and atomic64_set calls doesn't duplicate sequence numbers, it 379fc36ed7eSJan Schmidt * just returns a unique sequence number as usual.) We have decided to leave 380fc36ed7eSJan Schmidt * that requirement in here and rethink it once we notice it really imposes a 381fc36ed7eSJan Schmidt * problem on some workload. 382fc36ed7eSJan Schmidt */ 383fc36ed7eSJan Schmidt static inline u64 btrfs_inc_tree_mod_seq_minor(struct btrfs_fs_info *fs_info) 384fc36ed7eSJan Schmidt { 385fc36ed7eSJan Schmidt return atomic64_inc_return(&fs_info->tree_mod_seq); 386fc36ed7eSJan Schmidt } 387fc36ed7eSJan Schmidt 388fc36ed7eSJan Schmidt /* 389fc36ed7eSJan Schmidt * return the last minor in the previous major tree_mod_seq number 390fc36ed7eSJan Schmidt */ 391fc36ed7eSJan Schmidt u64 btrfs_tree_mod_seq_prev(u64 seq) 392fc36ed7eSJan Schmidt { 393fc36ed7eSJan Schmidt return (seq & 0xffffffff00000000ull) - 1ull; 394fc36ed7eSJan Schmidt } 395fc36ed7eSJan Schmidt 396fc36ed7eSJan Schmidt /* 397097b8a7cSJan Schmidt * This adds a new blocker to the tree mod log's blocker list if the @elem 398097b8a7cSJan Schmidt * passed does not already have a sequence number set. So when a caller expects 399097b8a7cSJan Schmidt * to record tree modifications, it should ensure to set elem->seq to zero 400097b8a7cSJan Schmidt * before calling btrfs_get_tree_mod_seq. 401097b8a7cSJan Schmidt * Returns a fresh, unused tree log modification sequence number, even if no new 402097b8a7cSJan Schmidt * blocker was added. 403097b8a7cSJan Schmidt */ 404097b8a7cSJan Schmidt u64 btrfs_get_tree_mod_seq(struct btrfs_fs_info *fs_info, 405bd989ba3SJan Schmidt struct seq_list *elem) 406bd989ba3SJan Schmidt { 407097b8a7cSJan Schmidt u64 seq; 408097b8a7cSJan Schmidt 409097b8a7cSJan Schmidt tree_mod_log_write_lock(fs_info); 410bd989ba3SJan Schmidt spin_lock(&fs_info->tree_mod_seq_lock); 411097b8a7cSJan Schmidt if (!elem->seq) { 412fc36ed7eSJan Schmidt elem->seq = btrfs_inc_tree_mod_seq_major(fs_info); 413097b8a7cSJan Schmidt list_add_tail(&elem->list, &fs_info->tree_mod_seq_list); 414097b8a7cSJan Schmidt } 415fc36ed7eSJan Schmidt seq = btrfs_inc_tree_mod_seq_minor(fs_info); 416bd989ba3SJan Schmidt spin_unlock(&fs_info->tree_mod_seq_lock); 417097b8a7cSJan Schmidt tree_mod_log_write_unlock(fs_info); 418097b8a7cSJan Schmidt 419097b8a7cSJan Schmidt return seq; 420bd989ba3SJan Schmidt } 421bd989ba3SJan Schmidt 422bd989ba3SJan Schmidt void btrfs_put_tree_mod_seq(struct btrfs_fs_info *fs_info, 423bd989ba3SJan Schmidt struct seq_list *elem) 424bd989ba3SJan Schmidt { 425bd989ba3SJan Schmidt struct rb_root *tm_root; 426bd989ba3SJan Schmidt struct rb_node *node; 427bd989ba3SJan Schmidt struct rb_node *next; 428bd989ba3SJan Schmidt struct seq_list *cur_elem; 429bd989ba3SJan Schmidt struct tree_mod_elem *tm; 430bd989ba3SJan Schmidt u64 min_seq = (u64)-1; 431bd989ba3SJan Schmidt u64 seq_putting = elem->seq; 432bd989ba3SJan Schmidt 433bd989ba3SJan Schmidt if (!seq_putting) 434bd989ba3SJan Schmidt return; 435bd989ba3SJan Schmidt 436bd989ba3SJan Schmidt spin_lock(&fs_info->tree_mod_seq_lock); 437bd989ba3SJan Schmidt list_del(&elem->list); 438097b8a7cSJan Schmidt elem->seq = 0; 439bd989ba3SJan Schmidt 440bd989ba3SJan Schmidt list_for_each_entry(cur_elem, &fs_info->tree_mod_seq_list, list) { 441097b8a7cSJan Schmidt if (cur_elem->seq < min_seq) { 442bd989ba3SJan Schmidt if (seq_putting > cur_elem->seq) { 443bd989ba3SJan Schmidt /* 444bd989ba3SJan Schmidt * blocker with lower sequence number exists, we 445bd989ba3SJan Schmidt * cannot remove anything from the log 446bd989ba3SJan Schmidt */ 447097b8a7cSJan Schmidt spin_unlock(&fs_info->tree_mod_seq_lock); 448097b8a7cSJan Schmidt return; 449bd989ba3SJan Schmidt } 450bd989ba3SJan Schmidt min_seq = cur_elem->seq; 451bd989ba3SJan Schmidt } 452bd989ba3SJan Schmidt } 453097b8a7cSJan Schmidt spin_unlock(&fs_info->tree_mod_seq_lock); 454097b8a7cSJan Schmidt 455097b8a7cSJan Schmidt /* 456bd989ba3SJan Schmidt * anything that's lower than the lowest existing (read: blocked) 457bd989ba3SJan Schmidt * sequence number can be removed from the tree. 458bd989ba3SJan Schmidt */ 459097b8a7cSJan Schmidt tree_mod_log_write_lock(fs_info); 460bd989ba3SJan Schmidt tm_root = &fs_info->tree_mod_log; 461bd989ba3SJan Schmidt for (node = rb_first(tm_root); node; node = next) { 462bd989ba3SJan Schmidt next = rb_next(node); 463bd989ba3SJan Schmidt tm = container_of(node, struct tree_mod_elem, node); 464097b8a7cSJan Schmidt if (tm->seq > min_seq) 465bd989ba3SJan Schmidt continue; 466bd989ba3SJan Schmidt rb_erase(node, tm_root); 467bd989ba3SJan Schmidt kfree(tm); 468bd989ba3SJan Schmidt } 469097b8a7cSJan Schmidt tree_mod_log_write_unlock(fs_info); 470bd989ba3SJan Schmidt } 471bd989ba3SJan Schmidt 472bd989ba3SJan Schmidt /* 473bd989ba3SJan Schmidt * key order of the log: 474bd989ba3SJan Schmidt * index -> sequence 475bd989ba3SJan Schmidt * 476bd989ba3SJan Schmidt * the index is the shifted logical of the *new* root node for root replace 477bd989ba3SJan Schmidt * operations, or the shifted logical of the affected block for all other 478bd989ba3SJan Schmidt * operations. 479bd989ba3SJan Schmidt */ 480bd989ba3SJan Schmidt static noinline int 481bd989ba3SJan Schmidt __tree_mod_log_insert(struct btrfs_fs_info *fs_info, struct tree_mod_elem *tm) 482bd989ba3SJan Schmidt { 483bd989ba3SJan Schmidt struct rb_root *tm_root; 484bd989ba3SJan Schmidt struct rb_node **new; 485bd989ba3SJan Schmidt struct rb_node *parent = NULL; 486bd989ba3SJan Schmidt struct tree_mod_elem *cur; 487c8cc6341SJosef Bacik int ret = 0; 488bd989ba3SJan Schmidt 489c8cc6341SJosef Bacik BUG_ON(!tm); 490c8cc6341SJosef Bacik 491c8cc6341SJosef Bacik tree_mod_log_write_lock(fs_info); 492c8cc6341SJosef Bacik if (list_empty(&fs_info->tree_mod_seq_list)) { 493c8cc6341SJosef Bacik tree_mod_log_write_unlock(fs_info); 494c8cc6341SJosef Bacik /* 495c8cc6341SJosef Bacik * Ok we no longer care about logging modifications, free up tm 496c8cc6341SJosef Bacik * and return 0. Any callers shouldn't be using tm after 497c8cc6341SJosef Bacik * calling tree_mod_log_insert, but if they do we can just 498c8cc6341SJosef Bacik * change this to return a special error code to let the callers 499c8cc6341SJosef Bacik * do their own thing. 500c8cc6341SJosef Bacik */ 501c8cc6341SJosef Bacik kfree(tm); 502c8cc6341SJosef Bacik return 0; 503c8cc6341SJosef Bacik } 504c8cc6341SJosef Bacik 505c8cc6341SJosef Bacik spin_lock(&fs_info->tree_mod_seq_lock); 506c8cc6341SJosef Bacik tm->seq = btrfs_inc_tree_mod_seq_minor(fs_info); 507c8cc6341SJosef Bacik spin_unlock(&fs_info->tree_mod_seq_lock); 508bd989ba3SJan Schmidt 509bd989ba3SJan Schmidt tm_root = &fs_info->tree_mod_log; 510bd989ba3SJan Schmidt new = &tm_root->rb_node; 511bd989ba3SJan Schmidt while (*new) { 512bd989ba3SJan Schmidt cur = container_of(*new, struct tree_mod_elem, node); 513bd989ba3SJan Schmidt parent = *new; 514bd989ba3SJan Schmidt if (cur->index < tm->index) 515bd989ba3SJan Schmidt new = &((*new)->rb_left); 516bd989ba3SJan Schmidt else if (cur->index > tm->index) 517bd989ba3SJan Schmidt new = &((*new)->rb_right); 518097b8a7cSJan Schmidt else if (cur->seq < tm->seq) 519bd989ba3SJan Schmidt new = &((*new)->rb_left); 520097b8a7cSJan Schmidt else if (cur->seq > tm->seq) 521bd989ba3SJan Schmidt new = &((*new)->rb_right); 522bd989ba3SJan Schmidt else { 523c8cc6341SJosef Bacik ret = -EEXIST; 524bd989ba3SJan Schmidt kfree(tm); 525c8cc6341SJosef Bacik goto out; 526bd989ba3SJan Schmidt } 527bd989ba3SJan Schmidt } 528bd989ba3SJan Schmidt 529bd989ba3SJan Schmidt rb_link_node(&tm->node, parent, new); 530bd989ba3SJan Schmidt rb_insert_color(&tm->node, tm_root); 531c8cc6341SJosef Bacik out: 532c8cc6341SJosef Bacik tree_mod_log_write_unlock(fs_info); 533c8cc6341SJosef Bacik return ret; 534bd989ba3SJan Schmidt } 535bd989ba3SJan Schmidt 536097b8a7cSJan Schmidt /* 537097b8a7cSJan Schmidt * Determines if logging can be omitted. Returns 1 if it can. Otherwise, it 538097b8a7cSJan Schmidt * returns zero with the tree_mod_log_lock acquired. The caller must hold 539097b8a7cSJan Schmidt * this until all tree mod log insertions are recorded in the rb tree and then 540097b8a7cSJan Schmidt * call tree_mod_log_write_unlock() to release. 541097b8a7cSJan Schmidt */ 542e9b7fd4dSJan Schmidt static inline int tree_mod_dont_log(struct btrfs_fs_info *fs_info, 543e9b7fd4dSJan Schmidt struct extent_buffer *eb) { 544e9b7fd4dSJan Schmidt smp_mb(); 545e9b7fd4dSJan Schmidt if (list_empty(&(fs_info)->tree_mod_seq_list)) 546e9b7fd4dSJan Schmidt return 1; 547097b8a7cSJan Schmidt if (eb && btrfs_header_level(eb) == 0) 548e9b7fd4dSJan Schmidt return 1; 549e9b7fd4dSJan Schmidt return 0; 550e9b7fd4dSJan Schmidt } 551e9b7fd4dSJan Schmidt 552097b8a7cSJan Schmidt static inline int 553097b8a7cSJan Schmidt __tree_mod_log_insert_key(struct btrfs_fs_info *fs_info, 554bd989ba3SJan Schmidt struct extent_buffer *eb, int slot, 555bd989ba3SJan Schmidt enum mod_log_op op, gfp_t flags) 556bd989ba3SJan Schmidt { 557097b8a7cSJan Schmidt struct tree_mod_elem *tm; 558bd989ba3SJan Schmidt 559c8cc6341SJosef Bacik tm = kzalloc(sizeof(*tm), flags); 560c8cc6341SJosef Bacik if (!tm) 561c8cc6341SJosef Bacik return -ENOMEM; 562bd989ba3SJan Schmidt 563bd989ba3SJan Schmidt tm->index = eb->start >> PAGE_CACHE_SHIFT; 564bd989ba3SJan Schmidt if (op != MOD_LOG_KEY_ADD) { 565bd989ba3SJan Schmidt btrfs_node_key(eb, &tm->key, slot); 566bd989ba3SJan Schmidt tm->blockptr = btrfs_node_blockptr(eb, slot); 567bd989ba3SJan Schmidt } 568bd989ba3SJan Schmidt tm->op = op; 569bd989ba3SJan Schmidt tm->slot = slot; 570bd989ba3SJan Schmidt tm->generation = btrfs_node_ptr_generation(eb, slot); 571bd989ba3SJan Schmidt 572097b8a7cSJan Schmidt return __tree_mod_log_insert(fs_info, tm); 573097b8a7cSJan Schmidt } 574097b8a7cSJan Schmidt 575097b8a7cSJan Schmidt static noinline int 576c8cc6341SJosef Bacik tree_mod_log_insert_key(struct btrfs_fs_info *fs_info, 577097b8a7cSJan Schmidt struct extent_buffer *eb, int slot, 578097b8a7cSJan Schmidt enum mod_log_op op, gfp_t flags) 579097b8a7cSJan Schmidt { 580097b8a7cSJan Schmidt if (tree_mod_dont_log(fs_info, eb)) 581097b8a7cSJan Schmidt return 0; 582097b8a7cSJan Schmidt 583c8cc6341SJosef Bacik return __tree_mod_log_insert_key(fs_info, eb, slot, op, flags); 584097b8a7cSJan Schmidt } 585097b8a7cSJan Schmidt 586097b8a7cSJan Schmidt static noinline int 587bd989ba3SJan Schmidt tree_mod_log_insert_move(struct btrfs_fs_info *fs_info, 588bd989ba3SJan Schmidt struct extent_buffer *eb, int dst_slot, int src_slot, 589bd989ba3SJan Schmidt int nr_items, gfp_t flags) 590bd989ba3SJan Schmidt { 591bd989ba3SJan Schmidt struct tree_mod_elem *tm; 592bd989ba3SJan Schmidt int ret; 593bd989ba3SJan Schmidt int i; 594bd989ba3SJan Schmidt 595f395694cSJan Schmidt if (tree_mod_dont_log(fs_info, eb)) 596f395694cSJan Schmidt return 0; 597bd989ba3SJan Schmidt 59801763a2eSJan Schmidt /* 59901763a2eSJan Schmidt * When we override something during the move, we log these removals. 60001763a2eSJan Schmidt * This can only happen when we move towards the beginning of the 60101763a2eSJan Schmidt * buffer, i.e. dst_slot < src_slot. 60201763a2eSJan Schmidt */ 603bd989ba3SJan Schmidt for (i = 0; i + dst_slot < src_slot && i < nr_items; i++) { 604c8cc6341SJosef Bacik ret = __tree_mod_log_insert_key(fs_info, eb, i + dst_slot, 605c8cc6341SJosef Bacik MOD_LOG_KEY_REMOVE_WHILE_MOVING, GFP_NOFS); 606bd989ba3SJan Schmidt BUG_ON(ret < 0); 607bd989ba3SJan Schmidt } 608bd989ba3SJan Schmidt 609c8cc6341SJosef Bacik tm = kzalloc(sizeof(*tm), flags); 610c8cc6341SJosef Bacik if (!tm) 611c8cc6341SJosef Bacik return -ENOMEM; 612f395694cSJan Schmidt 613bd989ba3SJan Schmidt tm->index = eb->start >> PAGE_CACHE_SHIFT; 614bd989ba3SJan Schmidt tm->slot = src_slot; 615bd989ba3SJan Schmidt tm->move.dst_slot = dst_slot; 616bd989ba3SJan Schmidt tm->move.nr_items = nr_items; 617bd989ba3SJan Schmidt tm->op = MOD_LOG_MOVE_KEYS; 618bd989ba3SJan Schmidt 619c8cc6341SJosef Bacik return __tree_mod_log_insert(fs_info, tm); 620bd989ba3SJan Schmidt } 621bd989ba3SJan Schmidt 622097b8a7cSJan Schmidt static inline void 623097b8a7cSJan Schmidt __tree_mod_log_free_eb(struct btrfs_fs_info *fs_info, struct extent_buffer *eb) 624097b8a7cSJan Schmidt { 625097b8a7cSJan Schmidt int i; 626097b8a7cSJan Schmidt u32 nritems; 627097b8a7cSJan Schmidt int ret; 628097b8a7cSJan Schmidt 629b12a3b1eSChris Mason if (btrfs_header_level(eb) == 0) 630b12a3b1eSChris Mason return; 631b12a3b1eSChris Mason 632097b8a7cSJan Schmidt nritems = btrfs_header_nritems(eb); 633097b8a7cSJan Schmidt for (i = nritems - 1; i >= 0; i--) { 634c8cc6341SJosef Bacik ret = __tree_mod_log_insert_key(fs_info, eb, i, 635c8cc6341SJosef Bacik MOD_LOG_KEY_REMOVE_WHILE_FREEING, GFP_NOFS); 636097b8a7cSJan Schmidt BUG_ON(ret < 0); 637097b8a7cSJan Schmidt } 638097b8a7cSJan Schmidt } 639097b8a7cSJan Schmidt 640bd989ba3SJan Schmidt static noinline int 641bd989ba3SJan Schmidt tree_mod_log_insert_root(struct btrfs_fs_info *fs_info, 642bd989ba3SJan Schmidt struct extent_buffer *old_root, 64390f8d62eSJan Schmidt struct extent_buffer *new_root, gfp_t flags, 64490f8d62eSJan Schmidt int log_removal) 645bd989ba3SJan Schmidt { 646bd989ba3SJan Schmidt struct tree_mod_elem *tm; 647bd989ba3SJan Schmidt 648097b8a7cSJan Schmidt if (tree_mod_dont_log(fs_info, NULL)) 649097b8a7cSJan Schmidt return 0; 650097b8a7cSJan Schmidt 65190f8d62eSJan Schmidt if (log_removal) 652d9abbf1cSJan Schmidt __tree_mod_log_free_eb(fs_info, old_root); 653d9abbf1cSJan Schmidt 654c8cc6341SJosef Bacik tm = kzalloc(sizeof(*tm), flags); 655c8cc6341SJosef Bacik if (!tm) 656c8cc6341SJosef Bacik return -ENOMEM; 657bd989ba3SJan Schmidt 658bd989ba3SJan Schmidt tm->index = new_root->start >> PAGE_CACHE_SHIFT; 659bd989ba3SJan Schmidt tm->old_root.logical = old_root->start; 660bd989ba3SJan Schmidt tm->old_root.level = btrfs_header_level(old_root); 661bd989ba3SJan Schmidt tm->generation = btrfs_header_generation(old_root); 662bd989ba3SJan Schmidt tm->op = MOD_LOG_ROOT_REPLACE; 663bd989ba3SJan Schmidt 664c8cc6341SJosef Bacik return __tree_mod_log_insert(fs_info, tm); 665bd989ba3SJan Schmidt } 666bd989ba3SJan Schmidt 667bd989ba3SJan Schmidt static struct tree_mod_elem * 668bd989ba3SJan Schmidt __tree_mod_log_search(struct btrfs_fs_info *fs_info, u64 start, u64 min_seq, 669bd989ba3SJan Schmidt int smallest) 670bd989ba3SJan Schmidt { 671bd989ba3SJan Schmidt struct rb_root *tm_root; 672bd989ba3SJan Schmidt struct rb_node *node; 673bd989ba3SJan Schmidt struct tree_mod_elem *cur = NULL; 674bd989ba3SJan Schmidt struct tree_mod_elem *found = NULL; 675bd989ba3SJan Schmidt u64 index = start >> PAGE_CACHE_SHIFT; 676bd989ba3SJan Schmidt 677097b8a7cSJan Schmidt tree_mod_log_read_lock(fs_info); 678bd989ba3SJan Schmidt tm_root = &fs_info->tree_mod_log; 679bd989ba3SJan Schmidt node = tm_root->rb_node; 680bd989ba3SJan Schmidt while (node) { 681bd989ba3SJan Schmidt cur = container_of(node, struct tree_mod_elem, node); 682bd989ba3SJan Schmidt if (cur->index < index) { 683bd989ba3SJan Schmidt node = node->rb_left; 684bd989ba3SJan Schmidt } else if (cur->index > index) { 685bd989ba3SJan Schmidt node = node->rb_right; 686097b8a7cSJan Schmidt } else if (cur->seq < min_seq) { 687bd989ba3SJan Schmidt node = node->rb_left; 688bd989ba3SJan Schmidt } else if (!smallest) { 689bd989ba3SJan Schmidt /* we want the node with the highest seq */ 690bd989ba3SJan Schmidt if (found) 691097b8a7cSJan Schmidt BUG_ON(found->seq > cur->seq); 692bd989ba3SJan Schmidt found = cur; 693bd989ba3SJan Schmidt node = node->rb_left; 694097b8a7cSJan Schmidt } else if (cur->seq > min_seq) { 695bd989ba3SJan Schmidt /* we want the node with the smallest seq */ 696bd989ba3SJan Schmidt if (found) 697097b8a7cSJan Schmidt BUG_ON(found->seq < cur->seq); 698bd989ba3SJan Schmidt found = cur; 699bd989ba3SJan Schmidt node = node->rb_right; 700bd989ba3SJan Schmidt } else { 701bd989ba3SJan Schmidt found = cur; 702bd989ba3SJan Schmidt break; 703bd989ba3SJan Schmidt } 704bd989ba3SJan Schmidt } 705097b8a7cSJan Schmidt tree_mod_log_read_unlock(fs_info); 706bd989ba3SJan Schmidt 707bd989ba3SJan Schmidt return found; 708bd989ba3SJan Schmidt } 709bd989ba3SJan Schmidt 710bd989ba3SJan Schmidt /* 711bd989ba3SJan Schmidt * this returns the element from the log with the smallest time sequence 712bd989ba3SJan Schmidt * value that's in the log (the oldest log item). any element with a time 713bd989ba3SJan Schmidt * sequence lower than min_seq will be ignored. 714bd989ba3SJan Schmidt */ 715bd989ba3SJan Schmidt static struct tree_mod_elem * 716bd989ba3SJan Schmidt tree_mod_log_search_oldest(struct btrfs_fs_info *fs_info, u64 start, 717bd989ba3SJan Schmidt u64 min_seq) 718bd989ba3SJan Schmidt { 719bd989ba3SJan Schmidt return __tree_mod_log_search(fs_info, start, min_seq, 1); 720bd989ba3SJan Schmidt } 721bd989ba3SJan Schmidt 722bd989ba3SJan Schmidt /* 723bd989ba3SJan Schmidt * this returns the element from the log with the largest time sequence 724bd989ba3SJan Schmidt * value that's in the log (the most recent log item). any element with 725bd989ba3SJan Schmidt * a time sequence lower than min_seq will be ignored. 726bd989ba3SJan Schmidt */ 727bd989ba3SJan Schmidt static struct tree_mod_elem * 728bd989ba3SJan Schmidt tree_mod_log_search(struct btrfs_fs_info *fs_info, u64 start, u64 min_seq) 729bd989ba3SJan Schmidt { 730bd989ba3SJan Schmidt return __tree_mod_log_search(fs_info, start, min_seq, 0); 731bd989ba3SJan Schmidt } 732bd989ba3SJan Schmidt 733097b8a7cSJan Schmidt static noinline void 734bd989ba3SJan Schmidt tree_mod_log_eb_copy(struct btrfs_fs_info *fs_info, struct extent_buffer *dst, 735bd989ba3SJan Schmidt struct extent_buffer *src, unsigned long dst_offset, 73690f8d62eSJan Schmidt unsigned long src_offset, int nr_items) 737bd989ba3SJan Schmidt { 738bd989ba3SJan Schmidt int ret; 739bd989ba3SJan Schmidt int i; 740bd989ba3SJan Schmidt 741e9b7fd4dSJan Schmidt if (tree_mod_dont_log(fs_info, NULL)) 742bd989ba3SJan Schmidt return; 743bd989ba3SJan Schmidt 744c8cc6341SJosef Bacik if (btrfs_header_level(dst) == 0 && btrfs_header_level(src) == 0) 745bd989ba3SJan Schmidt return; 746bd989ba3SJan Schmidt 747bd989ba3SJan Schmidt for (i = 0; i < nr_items; i++) { 748c8cc6341SJosef Bacik ret = __tree_mod_log_insert_key(fs_info, src, 749097b8a7cSJan Schmidt i + src_offset, 750c8cc6341SJosef Bacik MOD_LOG_KEY_REMOVE, GFP_NOFS); 751bd989ba3SJan Schmidt BUG_ON(ret < 0); 752c8cc6341SJosef Bacik ret = __tree_mod_log_insert_key(fs_info, dst, 753097b8a7cSJan Schmidt i + dst_offset, 754c8cc6341SJosef Bacik MOD_LOG_KEY_ADD, 755c8cc6341SJosef Bacik GFP_NOFS); 756bd989ba3SJan Schmidt BUG_ON(ret < 0); 757bd989ba3SJan Schmidt } 758bd989ba3SJan Schmidt } 759bd989ba3SJan Schmidt 760bd989ba3SJan Schmidt static inline void 761bd989ba3SJan Schmidt tree_mod_log_eb_move(struct btrfs_fs_info *fs_info, struct extent_buffer *dst, 762bd989ba3SJan Schmidt int dst_offset, int src_offset, int nr_items) 763bd989ba3SJan Schmidt { 764bd989ba3SJan Schmidt int ret; 765bd989ba3SJan Schmidt ret = tree_mod_log_insert_move(fs_info, dst, dst_offset, src_offset, 766bd989ba3SJan Schmidt nr_items, GFP_NOFS); 767bd989ba3SJan Schmidt BUG_ON(ret < 0); 768bd989ba3SJan Schmidt } 769bd989ba3SJan Schmidt 770097b8a7cSJan Schmidt static noinline void 771bd989ba3SJan Schmidt tree_mod_log_set_node_key(struct btrfs_fs_info *fs_info, 77232adf090SLiu Bo struct extent_buffer *eb, int slot, int atomic) 773bd989ba3SJan Schmidt { 774bd989ba3SJan Schmidt int ret; 775bd989ba3SJan Schmidt 776c8cc6341SJosef Bacik ret = __tree_mod_log_insert_key(fs_info, eb, slot, 777bd989ba3SJan Schmidt MOD_LOG_KEY_REPLACE, 778bd989ba3SJan Schmidt atomic ? GFP_ATOMIC : GFP_NOFS); 779bd989ba3SJan Schmidt BUG_ON(ret < 0); 780bd989ba3SJan Schmidt } 781bd989ba3SJan Schmidt 782097b8a7cSJan Schmidt static noinline void 783097b8a7cSJan Schmidt tree_mod_log_free_eb(struct btrfs_fs_info *fs_info, struct extent_buffer *eb) 784bd989ba3SJan Schmidt { 785e9b7fd4dSJan Schmidt if (tree_mod_dont_log(fs_info, eb)) 786bd989ba3SJan Schmidt return; 787097b8a7cSJan Schmidt __tree_mod_log_free_eb(fs_info, eb); 788bd989ba3SJan Schmidt } 789bd989ba3SJan Schmidt 790097b8a7cSJan Schmidt static noinline void 791bd989ba3SJan Schmidt tree_mod_log_set_root_pointer(struct btrfs_root *root, 79290f8d62eSJan Schmidt struct extent_buffer *new_root_node, 79390f8d62eSJan Schmidt int log_removal) 794bd989ba3SJan Schmidt { 795bd989ba3SJan Schmidt int ret; 796bd989ba3SJan Schmidt ret = tree_mod_log_insert_root(root->fs_info, root->node, 79790f8d62eSJan Schmidt new_root_node, GFP_NOFS, log_removal); 798bd989ba3SJan Schmidt BUG_ON(ret < 0); 799bd989ba3SJan Schmidt } 800bd989ba3SJan Schmidt 801d352ac68SChris Mason /* 8025d4f98a2SYan Zheng * check if the tree block can be shared by multiple trees 8035d4f98a2SYan Zheng */ 8045d4f98a2SYan Zheng int btrfs_block_can_be_shared(struct btrfs_root *root, 8055d4f98a2SYan Zheng struct extent_buffer *buf) 8065d4f98a2SYan Zheng { 8075d4f98a2SYan Zheng /* 8085d4f98a2SYan Zheng * Tree blocks not in refernece counted trees and tree roots 8095d4f98a2SYan Zheng * are never shared. If a block was allocated after the last 8105d4f98a2SYan Zheng * snapshot and the block was not allocated by tree relocation, 8115d4f98a2SYan Zheng * we know the block is not shared. 8125d4f98a2SYan Zheng */ 8135d4f98a2SYan Zheng if (root->ref_cows && 8145d4f98a2SYan Zheng buf != root->node && buf != root->commit_root && 8155d4f98a2SYan Zheng (btrfs_header_generation(buf) <= 8165d4f98a2SYan Zheng btrfs_root_last_snapshot(&root->root_item) || 8175d4f98a2SYan Zheng btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC))) 8185d4f98a2SYan Zheng return 1; 8195d4f98a2SYan Zheng #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 8205d4f98a2SYan Zheng if (root->ref_cows && 8215d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 8225d4f98a2SYan Zheng return 1; 8235d4f98a2SYan Zheng #endif 8245d4f98a2SYan Zheng return 0; 8255d4f98a2SYan Zheng } 8265d4f98a2SYan Zheng 8275d4f98a2SYan Zheng static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans, 8285d4f98a2SYan Zheng struct btrfs_root *root, 8295d4f98a2SYan Zheng struct extent_buffer *buf, 830f0486c68SYan, Zheng struct extent_buffer *cow, 831f0486c68SYan, Zheng int *last_ref) 8325d4f98a2SYan Zheng { 8335d4f98a2SYan Zheng u64 refs; 8345d4f98a2SYan Zheng u64 owner; 8355d4f98a2SYan Zheng u64 flags; 8365d4f98a2SYan Zheng u64 new_flags = 0; 8375d4f98a2SYan Zheng int ret; 8385d4f98a2SYan Zheng 8395d4f98a2SYan Zheng /* 8405d4f98a2SYan Zheng * Backrefs update rules: 8415d4f98a2SYan Zheng * 8425d4f98a2SYan Zheng * Always use full backrefs for extent pointers in tree block 8435d4f98a2SYan Zheng * allocated by tree relocation. 8445d4f98a2SYan Zheng * 8455d4f98a2SYan Zheng * If a shared tree block is no longer referenced by its owner 8465d4f98a2SYan Zheng * tree (btrfs_header_owner(buf) == root->root_key.objectid), 8475d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 8485d4f98a2SYan Zheng * 8495d4f98a2SYan Zheng * If a tree block is been relocating 8505d4f98a2SYan Zheng * (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID), 8515d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 8525d4f98a2SYan Zheng * The reason for this is some operations (such as drop tree) 8535d4f98a2SYan Zheng * are only allowed for blocks use full backrefs. 8545d4f98a2SYan Zheng */ 8555d4f98a2SYan Zheng 8565d4f98a2SYan Zheng if (btrfs_block_can_be_shared(root, buf)) { 8575d4f98a2SYan Zheng ret = btrfs_lookup_extent_info(trans, root, buf->start, 8583173a18fSJosef Bacik btrfs_header_level(buf), 1, 8593173a18fSJosef Bacik &refs, &flags); 860be1a5564SMark Fasheh if (ret) 861be1a5564SMark Fasheh return ret; 862e5df9573SMark Fasheh if (refs == 0) { 863e5df9573SMark Fasheh ret = -EROFS; 864e5df9573SMark Fasheh btrfs_std_error(root->fs_info, ret); 865e5df9573SMark Fasheh return ret; 866e5df9573SMark Fasheh } 8675d4f98a2SYan Zheng } else { 8685d4f98a2SYan Zheng refs = 1; 8695d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 8705d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 8715d4f98a2SYan Zheng flags = BTRFS_BLOCK_FLAG_FULL_BACKREF; 8725d4f98a2SYan Zheng else 8735d4f98a2SYan Zheng flags = 0; 8745d4f98a2SYan Zheng } 8755d4f98a2SYan Zheng 8765d4f98a2SYan Zheng owner = btrfs_header_owner(buf); 8775d4f98a2SYan Zheng BUG_ON(owner == BTRFS_TREE_RELOC_OBJECTID && 8785d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)); 8795d4f98a2SYan Zheng 8805d4f98a2SYan Zheng if (refs > 1) { 8815d4f98a2SYan Zheng if ((owner == root->root_key.objectid || 8825d4f98a2SYan Zheng root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && 8835d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) { 88466d7e7f0SArne Jansen ret = btrfs_inc_ref(trans, root, buf, 1, 1); 88579787eaaSJeff Mahoney BUG_ON(ret); /* -ENOMEM */ 8865d4f98a2SYan Zheng 8875d4f98a2SYan Zheng if (root->root_key.objectid == 8885d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) { 88966d7e7f0SArne Jansen ret = btrfs_dec_ref(trans, root, buf, 0, 1); 89079787eaaSJeff Mahoney BUG_ON(ret); /* -ENOMEM */ 89166d7e7f0SArne Jansen ret = btrfs_inc_ref(trans, root, cow, 1, 1); 89279787eaaSJeff Mahoney BUG_ON(ret); /* -ENOMEM */ 8935d4f98a2SYan Zheng } 8945d4f98a2SYan Zheng new_flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF; 8955d4f98a2SYan Zheng } else { 8965d4f98a2SYan Zheng 8975d4f98a2SYan Zheng if (root->root_key.objectid == 8985d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 89966d7e7f0SArne Jansen ret = btrfs_inc_ref(trans, root, cow, 1, 1); 9005d4f98a2SYan Zheng else 90166d7e7f0SArne Jansen ret = btrfs_inc_ref(trans, root, cow, 0, 1); 90279787eaaSJeff Mahoney BUG_ON(ret); /* -ENOMEM */ 9035d4f98a2SYan Zheng } 9045d4f98a2SYan Zheng if (new_flags != 0) { 905b1c79e09SJosef Bacik int level = btrfs_header_level(buf); 906b1c79e09SJosef Bacik 9075d4f98a2SYan Zheng ret = btrfs_set_disk_extent_flags(trans, root, 9085d4f98a2SYan Zheng buf->start, 9095d4f98a2SYan Zheng buf->len, 910b1c79e09SJosef Bacik new_flags, level, 0); 911be1a5564SMark Fasheh if (ret) 912be1a5564SMark Fasheh return ret; 9135d4f98a2SYan Zheng } 9145d4f98a2SYan Zheng } else { 9155d4f98a2SYan Zheng if (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) { 9165d4f98a2SYan Zheng if (root->root_key.objectid == 9175d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 91866d7e7f0SArne Jansen ret = btrfs_inc_ref(trans, root, cow, 1, 1); 9195d4f98a2SYan Zheng else 92066d7e7f0SArne Jansen ret = btrfs_inc_ref(trans, root, cow, 0, 1); 92179787eaaSJeff Mahoney BUG_ON(ret); /* -ENOMEM */ 92266d7e7f0SArne Jansen ret = btrfs_dec_ref(trans, root, buf, 1, 1); 92379787eaaSJeff Mahoney BUG_ON(ret); /* -ENOMEM */ 9245d4f98a2SYan Zheng } 9255d4f98a2SYan Zheng clean_tree_block(trans, root, buf); 926f0486c68SYan, Zheng *last_ref = 1; 9275d4f98a2SYan Zheng } 9285d4f98a2SYan Zheng return 0; 9295d4f98a2SYan Zheng } 9305d4f98a2SYan Zheng 9315d4f98a2SYan Zheng /* 932d397712bSChris Mason * does the dirty work in cow of a single block. The parent block (if 933d397712bSChris Mason * supplied) is updated to point to the new cow copy. The new buffer is marked 934d397712bSChris Mason * dirty and returned locked. If you modify the block it needs to be marked 935d397712bSChris Mason * dirty again. 936d352ac68SChris Mason * 937d352ac68SChris Mason * search_start -- an allocation hint for the new block 938d352ac68SChris Mason * 939d397712bSChris Mason * empty_size -- a hint that you plan on doing more cow. This is the size in 940d397712bSChris Mason * bytes the allocator should try to find free next to the block it returns. 941d397712bSChris Mason * This is just a hint and may be ignored by the allocator. 942d352ac68SChris Mason */ 943d397712bSChris Mason static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans, 9445f39d397SChris Mason struct btrfs_root *root, 9455f39d397SChris Mason struct extent_buffer *buf, 9465f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 9475f39d397SChris Mason struct extent_buffer **cow_ret, 9489fa8cfe7SChris Mason u64 search_start, u64 empty_size) 9496702ed49SChris Mason { 9505d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 9515f39d397SChris Mason struct extent_buffer *cow; 952be1a5564SMark Fasheh int level, ret; 953f0486c68SYan, Zheng int last_ref = 0; 954925baeddSChris Mason int unlock_orig = 0; 9555d4f98a2SYan Zheng u64 parent_start; 9566702ed49SChris Mason 957925baeddSChris Mason if (*cow_ret == buf) 958925baeddSChris Mason unlock_orig = 1; 959925baeddSChris Mason 960b9447ef8SChris Mason btrfs_assert_tree_locked(buf); 961925baeddSChris Mason 9627bb86316SChris Mason WARN_ON(root->ref_cows && trans->transid != 9637bb86316SChris Mason root->fs_info->running_transaction->transid); 9646702ed49SChris Mason WARN_ON(root->ref_cows && trans->transid != root->last_trans); 9655f39d397SChris Mason 9667bb86316SChris Mason level = btrfs_header_level(buf); 96731840ae1SZheng Yan 9685d4f98a2SYan Zheng if (level == 0) 9695d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 9705d4f98a2SYan Zheng else 9715d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 9725d4f98a2SYan Zheng 9735d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) { 9745d4f98a2SYan Zheng if (parent) 9755d4f98a2SYan Zheng parent_start = parent->start; 9765d4f98a2SYan Zheng else 9775d4f98a2SYan Zheng parent_start = 0; 9785d4f98a2SYan Zheng } else 9795d4f98a2SYan Zheng parent_start = 0; 9805d4f98a2SYan Zheng 9815d4f98a2SYan Zheng cow = btrfs_alloc_free_block(trans, root, buf->len, parent_start, 9825d4f98a2SYan Zheng root->root_key.objectid, &disk_key, 9835581a51aSJan Schmidt level, search_start, empty_size); 9846702ed49SChris Mason if (IS_ERR(cow)) 9856702ed49SChris Mason return PTR_ERR(cow); 9866702ed49SChris Mason 987b4ce94deSChris Mason /* cow is set to blocking by btrfs_init_new_buffer */ 988b4ce94deSChris Mason 9895f39d397SChris Mason copy_extent_buffer(cow, buf, 0, 0, cow->len); 990db94535dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 9915f39d397SChris Mason btrfs_set_header_generation(cow, trans->transid); 9925d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 9935d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 9945d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 9955d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) 9965d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 9975d4f98a2SYan Zheng else 9985f39d397SChris Mason btrfs_set_header_owner(cow, root->root_key.objectid); 9996702ed49SChris Mason 10002b82032cSYan Zheng write_extent_buffer(cow, root->fs_info->fsid, 10012b82032cSYan Zheng (unsigned long)btrfs_header_fsid(cow), 10022b82032cSYan Zheng BTRFS_FSID_SIZE); 10032b82032cSYan Zheng 1004be1a5564SMark Fasheh ret = update_ref_for_cow(trans, root, buf, cow, &last_ref); 1005b68dc2a9SMark Fasheh if (ret) { 100679787eaaSJeff Mahoney btrfs_abort_transaction(trans, root, ret); 1007b68dc2a9SMark Fasheh return ret; 1008b68dc2a9SMark Fasheh } 10091a40e23bSZheng Yan 10103fd0a558SYan, Zheng if (root->ref_cows) 10113fd0a558SYan, Zheng btrfs_reloc_cow_block(trans, root, buf, cow); 10123fd0a558SYan, Zheng 10136702ed49SChris Mason if (buf == root->node) { 1014925baeddSChris Mason WARN_ON(parent && parent != buf); 10155d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 10165d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 10175d4f98a2SYan Zheng parent_start = buf->start; 10185d4f98a2SYan Zheng else 10195d4f98a2SYan Zheng parent_start = 0; 1020925baeddSChris Mason 10215f39d397SChris Mason extent_buffer_get(cow); 102290f8d62eSJan Schmidt tree_mod_log_set_root_pointer(root, cow, 1); 1023240f62c8SChris Mason rcu_assign_pointer(root->node, cow); 1024925baeddSChris Mason 1025f0486c68SYan, Zheng btrfs_free_tree_block(trans, root, buf, parent_start, 10265581a51aSJan Schmidt last_ref); 10275f39d397SChris Mason free_extent_buffer(buf); 10280b86a832SChris Mason add_root_to_dirty_list(root); 10296702ed49SChris Mason } else { 10305d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) 10315d4f98a2SYan Zheng parent_start = parent->start; 10325d4f98a2SYan Zheng else 10335d4f98a2SYan Zheng parent_start = 0; 10345d4f98a2SYan Zheng 10355d4f98a2SYan Zheng WARN_ON(trans->transid != btrfs_header_generation(parent)); 1036f230475eSJan Schmidt tree_mod_log_insert_key(root->fs_info, parent, parent_slot, 1037c8cc6341SJosef Bacik MOD_LOG_KEY_REPLACE, GFP_NOFS); 10385f39d397SChris Mason btrfs_set_node_blockptr(parent, parent_slot, 1039db94535dSChris Mason cow->start); 104074493f7aSChris Mason btrfs_set_node_ptr_generation(parent, parent_slot, 104174493f7aSChris Mason trans->transid); 10426702ed49SChris Mason btrfs_mark_buffer_dirty(parent); 10437fb7d76fSJosef Bacik if (last_ref) 1044d9abbf1cSJan Schmidt tree_mod_log_free_eb(root->fs_info, buf); 1045f0486c68SYan, Zheng btrfs_free_tree_block(trans, root, buf, parent_start, 10465581a51aSJan Schmidt last_ref); 10476702ed49SChris Mason } 1048925baeddSChris Mason if (unlock_orig) 1049925baeddSChris Mason btrfs_tree_unlock(buf); 10503083ee2eSJosef Bacik free_extent_buffer_stale(buf); 10516702ed49SChris Mason btrfs_mark_buffer_dirty(cow); 10526702ed49SChris Mason *cow_ret = cow; 10536702ed49SChris Mason return 0; 10546702ed49SChris Mason } 10556702ed49SChris Mason 10565d9e75c4SJan Schmidt /* 10575d9e75c4SJan Schmidt * returns the logical address of the oldest predecessor of the given root. 10585d9e75c4SJan Schmidt * entries older than time_seq are ignored. 10595d9e75c4SJan Schmidt */ 10605d9e75c4SJan Schmidt static struct tree_mod_elem * 10615d9e75c4SJan Schmidt __tree_mod_log_oldest_root(struct btrfs_fs_info *fs_info, 106230b0463aSJan Schmidt struct extent_buffer *eb_root, u64 time_seq) 10635d9e75c4SJan Schmidt { 10645d9e75c4SJan Schmidt struct tree_mod_elem *tm; 10655d9e75c4SJan Schmidt struct tree_mod_elem *found = NULL; 106630b0463aSJan Schmidt u64 root_logical = eb_root->start; 10675d9e75c4SJan Schmidt int looped = 0; 10685d9e75c4SJan Schmidt 10695d9e75c4SJan Schmidt if (!time_seq) 10705d9e75c4SJan Schmidt return 0; 10715d9e75c4SJan Schmidt 10725d9e75c4SJan Schmidt /* 10735d9e75c4SJan Schmidt * the very last operation that's logged for a root is the replacement 10745d9e75c4SJan Schmidt * operation (if it is replaced at all). this has the index of the *new* 10755d9e75c4SJan Schmidt * root, making it the very first operation that's logged for this root. 10765d9e75c4SJan Schmidt */ 10775d9e75c4SJan Schmidt while (1) { 10785d9e75c4SJan Schmidt tm = tree_mod_log_search_oldest(fs_info, root_logical, 10795d9e75c4SJan Schmidt time_seq); 10805d9e75c4SJan Schmidt if (!looped && !tm) 10815d9e75c4SJan Schmidt return 0; 10825d9e75c4SJan Schmidt /* 108328da9fb4SJan Schmidt * if there are no tree operation for the oldest root, we simply 108428da9fb4SJan Schmidt * return it. this should only happen if that (old) root is at 108528da9fb4SJan Schmidt * level 0. 10865d9e75c4SJan Schmidt */ 108728da9fb4SJan Schmidt if (!tm) 108828da9fb4SJan Schmidt break; 10895d9e75c4SJan Schmidt 109028da9fb4SJan Schmidt /* 109128da9fb4SJan Schmidt * if there's an operation that's not a root replacement, we 109228da9fb4SJan Schmidt * found the oldest version of our root. normally, we'll find a 109328da9fb4SJan Schmidt * MOD_LOG_KEY_REMOVE_WHILE_FREEING operation here. 109428da9fb4SJan Schmidt */ 10955d9e75c4SJan Schmidt if (tm->op != MOD_LOG_ROOT_REPLACE) 10965d9e75c4SJan Schmidt break; 10975d9e75c4SJan Schmidt 10985d9e75c4SJan Schmidt found = tm; 10995d9e75c4SJan Schmidt root_logical = tm->old_root.logical; 11005d9e75c4SJan Schmidt looped = 1; 11015d9e75c4SJan Schmidt } 11025d9e75c4SJan Schmidt 1103a95236d9SJan Schmidt /* if there's no old root to return, return what we found instead */ 1104a95236d9SJan Schmidt if (!found) 1105a95236d9SJan Schmidt found = tm; 1106a95236d9SJan Schmidt 11075d9e75c4SJan Schmidt return found; 11085d9e75c4SJan Schmidt } 11095d9e75c4SJan Schmidt 11105d9e75c4SJan Schmidt /* 11115d9e75c4SJan Schmidt * tm is a pointer to the first operation to rewind within eb. then, all 11125d9e75c4SJan Schmidt * previous operations will be rewinded (until we reach something older than 11135d9e75c4SJan Schmidt * time_seq). 11145d9e75c4SJan Schmidt */ 11155d9e75c4SJan Schmidt static void 1116f1ca7e98SJosef Bacik __tree_mod_log_rewind(struct btrfs_fs_info *fs_info, struct extent_buffer *eb, 1117f1ca7e98SJosef Bacik u64 time_seq, struct tree_mod_elem *first_tm) 11185d9e75c4SJan Schmidt { 11195d9e75c4SJan Schmidt u32 n; 11205d9e75c4SJan Schmidt struct rb_node *next; 11215d9e75c4SJan Schmidt struct tree_mod_elem *tm = first_tm; 11225d9e75c4SJan Schmidt unsigned long o_dst; 11235d9e75c4SJan Schmidt unsigned long o_src; 11245d9e75c4SJan Schmidt unsigned long p_size = sizeof(struct btrfs_key_ptr); 11255d9e75c4SJan Schmidt 11265d9e75c4SJan Schmidt n = btrfs_header_nritems(eb); 1127f1ca7e98SJosef Bacik tree_mod_log_read_lock(fs_info); 1128097b8a7cSJan Schmidt while (tm && tm->seq >= time_seq) { 11295d9e75c4SJan Schmidt /* 11305d9e75c4SJan Schmidt * all the operations are recorded with the operator used for 11315d9e75c4SJan Schmidt * the modification. as we're going backwards, we do the 11325d9e75c4SJan Schmidt * opposite of each operation here. 11335d9e75c4SJan Schmidt */ 11345d9e75c4SJan Schmidt switch (tm->op) { 11355d9e75c4SJan Schmidt case MOD_LOG_KEY_REMOVE_WHILE_FREEING: 11365d9e75c4SJan Schmidt BUG_ON(tm->slot < n); 11371c697d4aSEric Sandeen /* Fallthrough */ 113895c80bb1SLiu Bo case MOD_LOG_KEY_REMOVE_WHILE_MOVING: 11394c3e6969SChris Mason case MOD_LOG_KEY_REMOVE: 11405d9e75c4SJan Schmidt btrfs_set_node_key(eb, &tm->key, tm->slot); 11415d9e75c4SJan Schmidt btrfs_set_node_blockptr(eb, tm->slot, tm->blockptr); 11425d9e75c4SJan Schmidt btrfs_set_node_ptr_generation(eb, tm->slot, 11435d9e75c4SJan Schmidt tm->generation); 11444c3e6969SChris Mason n++; 11455d9e75c4SJan Schmidt break; 11465d9e75c4SJan Schmidt case MOD_LOG_KEY_REPLACE: 11475d9e75c4SJan Schmidt BUG_ON(tm->slot >= n); 11485d9e75c4SJan Schmidt btrfs_set_node_key(eb, &tm->key, tm->slot); 11495d9e75c4SJan Schmidt btrfs_set_node_blockptr(eb, tm->slot, tm->blockptr); 11505d9e75c4SJan Schmidt btrfs_set_node_ptr_generation(eb, tm->slot, 11515d9e75c4SJan Schmidt tm->generation); 11525d9e75c4SJan Schmidt break; 11535d9e75c4SJan Schmidt case MOD_LOG_KEY_ADD: 115419956c7eSJan Schmidt /* if a move operation is needed it's in the log */ 11555d9e75c4SJan Schmidt n--; 11565d9e75c4SJan Schmidt break; 11575d9e75c4SJan Schmidt case MOD_LOG_MOVE_KEYS: 1158c3193108SJan Schmidt o_dst = btrfs_node_key_ptr_offset(tm->slot); 1159c3193108SJan Schmidt o_src = btrfs_node_key_ptr_offset(tm->move.dst_slot); 1160c3193108SJan Schmidt memmove_extent_buffer(eb, o_dst, o_src, 11615d9e75c4SJan Schmidt tm->move.nr_items * p_size); 11625d9e75c4SJan Schmidt break; 11635d9e75c4SJan Schmidt case MOD_LOG_ROOT_REPLACE: 11645d9e75c4SJan Schmidt /* 11655d9e75c4SJan Schmidt * this operation is special. for roots, this must be 11665d9e75c4SJan Schmidt * handled explicitly before rewinding. 11675d9e75c4SJan Schmidt * for non-roots, this operation may exist if the node 11685d9e75c4SJan Schmidt * was a root: root A -> child B; then A gets empty and 11695d9e75c4SJan Schmidt * B is promoted to the new root. in the mod log, we'll 11705d9e75c4SJan Schmidt * have a root-replace operation for B, a tree block 11715d9e75c4SJan Schmidt * that is no root. we simply ignore that operation. 11725d9e75c4SJan Schmidt */ 11735d9e75c4SJan Schmidt break; 11745d9e75c4SJan Schmidt } 11755d9e75c4SJan Schmidt next = rb_next(&tm->node); 11765d9e75c4SJan Schmidt if (!next) 11775d9e75c4SJan Schmidt break; 11785d9e75c4SJan Schmidt tm = container_of(next, struct tree_mod_elem, node); 11795d9e75c4SJan Schmidt if (tm->index != first_tm->index) 11805d9e75c4SJan Schmidt break; 11815d9e75c4SJan Schmidt } 1182f1ca7e98SJosef Bacik tree_mod_log_read_unlock(fs_info); 11835d9e75c4SJan Schmidt btrfs_set_header_nritems(eb, n); 11845d9e75c4SJan Schmidt } 11855d9e75c4SJan Schmidt 118647fb091fSJan Schmidt /* 118747fb091fSJan Schmidt * Called with eb read locked. If the buffer cannot be rewinded, the same buffer 118847fb091fSJan Schmidt * is returned. If rewind operations happen, a fresh buffer is returned. The 118947fb091fSJan Schmidt * returned buffer is always read-locked. If the returned buffer is not the 119047fb091fSJan Schmidt * input buffer, the lock on the input buffer is released and the input buffer 119147fb091fSJan Schmidt * is freed (its refcount is decremented). 119247fb091fSJan Schmidt */ 11935d9e75c4SJan Schmidt static struct extent_buffer * 11945d9e75c4SJan Schmidt tree_mod_log_rewind(struct btrfs_fs_info *fs_info, struct extent_buffer *eb, 11955d9e75c4SJan Schmidt u64 time_seq) 11965d9e75c4SJan Schmidt { 11975d9e75c4SJan Schmidt struct extent_buffer *eb_rewin; 11985d9e75c4SJan Schmidt struct tree_mod_elem *tm; 11995d9e75c4SJan Schmidt 12005d9e75c4SJan Schmidt if (!time_seq) 12015d9e75c4SJan Schmidt return eb; 12025d9e75c4SJan Schmidt 12035d9e75c4SJan Schmidt if (btrfs_header_level(eb) == 0) 12045d9e75c4SJan Schmidt return eb; 12055d9e75c4SJan Schmidt 12065d9e75c4SJan Schmidt tm = tree_mod_log_search(fs_info, eb->start, time_seq); 12075d9e75c4SJan Schmidt if (!tm) 12085d9e75c4SJan Schmidt return eb; 12095d9e75c4SJan Schmidt 12105d9e75c4SJan Schmidt if (tm->op == MOD_LOG_KEY_REMOVE_WHILE_FREEING) { 12115d9e75c4SJan Schmidt BUG_ON(tm->slot != 0); 12125d9e75c4SJan Schmidt eb_rewin = alloc_dummy_extent_buffer(eb->start, 12135d9e75c4SJan Schmidt fs_info->tree_root->nodesize); 1214db7f3436SJosef Bacik if (!eb_rewin) { 1215db7f3436SJosef Bacik btrfs_tree_read_unlock(eb); 1216db7f3436SJosef Bacik free_extent_buffer(eb); 1217db7f3436SJosef Bacik return NULL; 1218db7f3436SJosef Bacik } 12195d9e75c4SJan Schmidt btrfs_set_header_bytenr(eb_rewin, eb->start); 12205d9e75c4SJan Schmidt btrfs_set_header_backref_rev(eb_rewin, 12215d9e75c4SJan Schmidt btrfs_header_backref_rev(eb)); 12225d9e75c4SJan Schmidt btrfs_set_header_owner(eb_rewin, btrfs_header_owner(eb)); 1223c3193108SJan Schmidt btrfs_set_header_level(eb_rewin, btrfs_header_level(eb)); 12245d9e75c4SJan Schmidt } else { 12255d9e75c4SJan Schmidt eb_rewin = btrfs_clone_extent_buffer(eb); 1226db7f3436SJosef Bacik if (!eb_rewin) { 1227db7f3436SJosef Bacik btrfs_tree_read_unlock(eb); 1228db7f3436SJosef Bacik free_extent_buffer(eb); 1229db7f3436SJosef Bacik return NULL; 1230db7f3436SJosef Bacik } 12315d9e75c4SJan Schmidt } 12325d9e75c4SJan Schmidt 123347fb091fSJan Schmidt btrfs_tree_read_unlock(eb); 12345d9e75c4SJan Schmidt free_extent_buffer(eb); 12355d9e75c4SJan Schmidt 123647fb091fSJan Schmidt extent_buffer_get(eb_rewin); 123747fb091fSJan Schmidt btrfs_tree_read_lock(eb_rewin); 1238f1ca7e98SJosef Bacik __tree_mod_log_rewind(fs_info, eb_rewin, time_seq, tm); 123957911b8bSJan Schmidt WARN_ON(btrfs_header_nritems(eb_rewin) > 12402a745b14SArne Jansen BTRFS_NODEPTRS_PER_BLOCK(fs_info->tree_root)); 12415d9e75c4SJan Schmidt 12425d9e75c4SJan Schmidt return eb_rewin; 12435d9e75c4SJan Schmidt } 12445d9e75c4SJan Schmidt 12458ba97a15SJan Schmidt /* 12468ba97a15SJan Schmidt * get_old_root() rewinds the state of @root's root node to the given @time_seq 12478ba97a15SJan Schmidt * value. If there are no changes, the current root->root_node is returned. If 12488ba97a15SJan Schmidt * anything changed in between, there's a fresh buffer allocated on which the 12498ba97a15SJan Schmidt * rewind operations are done. In any case, the returned buffer is read locked. 12508ba97a15SJan Schmidt * Returns NULL on error (with no locks held). 12518ba97a15SJan Schmidt */ 12525d9e75c4SJan Schmidt static inline struct extent_buffer * 12535d9e75c4SJan Schmidt get_old_root(struct btrfs_root *root, u64 time_seq) 12545d9e75c4SJan Schmidt { 12555d9e75c4SJan Schmidt struct tree_mod_elem *tm; 125630b0463aSJan Schmidt struct extent_buffer *eb = NULL; 125730b0463aSJan Schmidt struct extent_buffer *eb_root; 12587bfdcf7fSLiu Bo struct extent_buffer *old; 1259a95236d9SJan Schmidt struct tree_mod_root *old_root = NULL; 12604325edd0SChris Mason u64 old_generation = 0; 1261a95236d9SJan Schmidt u64 logical; 1262834328a8SJan Schmidt u32 blocksize; 12635d9e75c4SJan Schmidt 126430b0463aSJan Schmidt eb_root = btrfs_read_lock_root_node(root); 126530b0463aSJan Schmidt tm = __tree_mod_log_oldest_root(root->fs_info, eb_root, time_seq); 12665d9e75c4SJan Schmidt if (!tm) 126730b0463aSJan Schmidt return eb_root; 12685d9e75c4SJan Schmidt 1269a95236d9SJan Schmidt if (tm->op == MOD_LOG_ROOT_REPLACE) { 12705d9e75c4SJan Schmidt old_root = &tm->old_root; 12715d9e75c4SJan Schmidt old_generation = tm->generation; 1272a95236d9SJan Schmidt logical = old_root->logical; 1273a95236d9SJan Schmidt } else { 127430b0463aSJan Schmidt logical = eb_root->start; 1275a95236d9SJan Schmidt } 12765d9e75c4SJan Schmidt 1277a95236d9SJan Schmidt tm = tree_mod_log_search(root->fs_info, logical, time_seq); 1278834328a8SJan Schmidt if (old_root && tm && tm->op != MOD_LOG_KEY_REMOVE_WHILE_FREEING) { 127930b0463aSJan Schmidt btrfs_tree_read_unlock(eb_root); 128030b0463aSJan Schmidt free_extent_buffer(eb_root); 1281834328a8SJan Schmidt blocksize = btrfs_level_size(root, old_root->level); 12827bfdcf7fSLiu Bo old = read_tree_block(root, logical, blocksize, 0); 1283416bc658SJosef Bacik if (!old || !extent_buffer_uptodate(old)) { 1284416bc658SJosef Bacik free_extent_buffer(old); 1285834328a8SJan Schmidt pr_warn("btrfs: failed to read tree block %llu from get_old_root\n", 1286834328a8SJan Schmidt logical); 1287834328a8SJan Schmidt WARN_ON(1); 1288834328a8SJan Schmidt } else { 12897bfdcf7fSLiu Bo eb = btrfs_clone_extent_buffer(old); 12907bfdcf7fSLiu Bo free_extent_buffer(old); 1291834328a8SJan Schmidt } 1292834328a8SJan Schmidt } else if (old_root) { 129330b0463aSJan Schmidt btrfs_tree_read_unlock(eb_root); 129430b0463aSJan Schmidt free_extent_buffer(eb_root); 129528da9fb4SJan Schmidt eb = alloc_dummy_extent_buffer(logical, root->nodesize); 1296834328a8SJan Schmidt } else { 129730b0463aSJan Schmidt eb = btrfs_clone_extent_buffer(eb_root); 129830b0463aSJan Schmidt btrfs_tree_read_unlock(eb_root); 129930b0463aSJan Schmidt free_extent_buffer(eb_root); 1300834328a8SJan Schmidt } 1301834328a8SJan Schmidt 13028ba97a15SJan Schmidt if (!eb) 13038ba97a15SJan Schmidt return NULL; 1304d6381084SJan Schmidt extent_buffer_get(eb); 13058ba97a15SJan Schmidt btrfs_tree_read_lock(eb); 1306a95236d9SJan Schmidt if (old_root) { 13075d9e75c4SJan Schmidt btrfs_set_header_bytenr(eb, eb->start); 13085d9e75c4SJan Schmidt btrfs_set_header_backref_rev(eb, BTRFS_MIXED_BACKREF_REV); 130930b0463aSJan Schmidt btrfs_set_header_owner(eb, btrfs_header_owner(eb_root)); 13105d9e75c4SJan Schmidt btrfs_set_header_level(eb, old_root->level); 13115d9e75c4SJan Schmidt btrfs_set_header_generation(eb, old_generation); 1312a95236d9SJan Schmidt } 131328da9fb4SJan Schmidt if (tm) 1314f1ca7e98SJosef Bacik __tree_mod_log_rewind(root->fs_info, eb, time_seq, tm); 131528da9fb4SJan Schmidt else 131628da9fb4SJan Schmidt WARN_ON(btrfs_header_level(eb) != 0); 131757911b8bSJan Schmidt WARN_ON(btrfs_header_nritems(eb) > BTRFS_NODEPTRS_PER_BLOCK(root)); 13185d9e75c4SJan Schmidt 13195d9e75c4SJan Schmidt return eb; 13205d9e75c4SJan Schmidt } 13215d9e75c4SJan Schmidt 13225b6602e7SJan Schmidt int btrfs_old_root_level(struct btrfs_root *root, u64 time_seq) 13235b6602e7SJan Schmidt { 13245b6602e7SJan Schmidt struct tree_mod_elem *tm; 13255b6602e7SJan Schmidt int level; 132630b0463aSJan Schmidt struct extent_buffer *eb_root = btrfs_root_node(root); 13275b6602e7SJan Schmidt 132830b0463aSJan Schmidt tm = __tree_mod_log_oldest_root(root->fs_info, eb_root, time_seq); 13295b6602e7SJan Schmidt if (tm && tm->op == MOD_LOG_ROOT_REPLACE) { 13305b6602e7SJan Schmidt level = tm->old_root.level; 13315b6602e7SJan Schmidt } else { 133230b0463aSJan Schmidt level = btrfs_header_level(eb_root); 13335b6602e7SJan Schmidt } 133430b0463aSJan Schmidt free_extent_buffer(eb_root); 13355b6602e7SJan Schmidt 13365b6602e7SJan Schmidt return level; 13375b6602e7SJan Schmidt } 13385b6602e7SJan Schmidt 13395d4f98a2SYan Zheng static inline int should_cow_block(struct btrfs_trans_handle *trans, 13405d4f98a2SYan Zheng struct btrfs_root *root, 13415d4f98a2SYan Zheng struct extent_buffer *buf) 13425d4f98a2SYan Zheng { 1343f1ebcc74SLiu Bo /* ensure we can see the force_cow */ 1344f1ebcc74SLiu Bo smp_rmb(); 1345f1ebcc74SLiu Bo 1346f1ebcc74SLiu Bo /* 1347f1ebcc74SLiu Bo * We do not need to cow a block if 1348f1ebcc74SLiu Bo * 1) this block is not created or changed in this transaction; 1349f1ebcc74SLiu Bo * 2) this block does not belong to TREE_RELOC tree; 1350f1ebcc74SLiu Bo * 3) the root is not forced COW. 1351f1ebcc74SLiu Bo * 1352f1ebcc74SLiu Bo * What is forced COW: 1353f1ebcc74SLiu Bo * when we create snapshot during commiting the transaction, 1354f1ebcc74SLiu Bo * after we've finished coping src root, we must COW the shared 1355f1ebcc74SLiu Bo * block to ensure the metadata consistency. 1356f1ebcc74SLiu Bo */ 13575d4f98a2SYan Zheng if (btrfs_header_generation(buf) == trans->transid && 13585d4f98a2SYan Zheng !btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN) && 13595d4f98a2SYan Zheng !(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID && 1360f1ebcc74SLiu Bo btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)) && 1361f1ebcc74SLiu Bo !root->force_cow) 13625d4f98a2SYan Zheng return 0; 13635d4f98a2SYan Zheng return 1; 13645d4f98a2SYan Zheng } 13655d4f98a2SYan Zheng 1366d352ac68SChris Mason /* 1367d352ac68SChris Mason * cows a single block, see __btrfs_cow_block for the real work. 1368d352ac68SChris Mason * This version of it has extra checks so that a block isn't cow'd more than 1369d352ac68SChris Mason * once per transaction, as long as it hasn't been written yet 1370d352ac68SChris Mason */ 1371d397712bSChris Mason noinline int btrfs_cow_block(struct btrfs_trans_handle *trans, 13725f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *buf, 13735f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 13749fa8cfe7SChris Mason struct extent_buffer **cow_ret) 137502217ed2SChris Mason { 13766702ed49SChris Mason u64 search_start; 1377f510cfecSChris Mason int ret; 1378dc17ff8fSChris Mason 137931b1a2bdSJulia Lawall if (trans->transaction != root->fs_info->running_transaction) 138031b1a2bdSJulia Lawall WARN(1, KERN_CRIT "trans %llu running %llu\n", 1381d397712bSChris Mason (unsigned long long)trans->transid, 1382d397712bSChris Mason (unsigned long long) 1383ccd467d6SChris Mason root->fs_info->running_transaction->transid); 138431b1a2bdSJulia Lawall 138531b1a2bdSJulia Lawall if (trans->transid != root->fs_info->generation) 138631b1a2bdSJulia Lawall WARN(1, KERN_CRIT "trans %llu running %llu\n", 1387d397712bSChris Mason (unsigned long long)trans->transid, 1388d397712bSChris Mason (unsigned long long)root->fs_info->generation); 1389dc17ff8fSChris Mason 13905d4f98a2SYan Zheng if (!should_cow_block(trans, root, buf)) { 139102217ed2SChris Mason *cow_ret = buf; 139202217ed2SChris Mason return 0; 139302217ed2SChris Mason } 1394c487685dSChris Mason 13950b86a832SChris Mason search_start = buf->start & ~((u64)(1024 * 1024 * 1024) - 1); 1396b4ce94deSChris Mason 1397b4ce94deSChris Mason if (parent) 1398b4ce94deSChris Mason btrfs_set_lock_blocking(parent); 1399b4ce94deSChris Mason btrfs_set_lock_blocking(buf); 1400b4ce94deSChris Mason 1401f510cfecSChris Mason ret = __btrfs_cow_block(trans, root, buf, parent, 14029fa8cfe7SChris Mason parent_slot, cow_ret, search_start, 0); 14031abe9b8aSliubo 14041abe9b8aSliubo trace_btrfs_cow_block(root, buf, *cow_ret); 14051abe9b8aSliubo 1406f510cfecSChris Mason return ret; 14072c90e5d6SChris Mason } 14086702ed49SChris Mason 1409d352ac68SChris Mason /* 1410d352ac68SChris Mason * helper function for defrag to decide if two blocks pointed to by a 1411d352ac68SChris Mason * node are actually close by 1412d352ac68SChris Mason */ 14136b80053dSChris Mason static int close_blocks(u64 blocknr, u64 other, u32 blocksize) 14146702ed49SChris Mason { 14156b80053dSChris Mason if (blocknr < other && other - (blocknr + blocksize) < 32768) 14166702ed49SChris Mason return 1; 14176b80053dSChris Mason if (blocknr > other && blocknr - (other + blocksize) < 32768) 14186702ed49SChris Mason return 1; 141902217ed2SChris Mason return 0; 142002217ed2SChris Mason } 142102217ed2SChris Mason 1422081e9573SChris Mason /* 1423081e9573SChris Mason * compare two keys in a memcmp fashion 1424081e9573SChris Mason */ 1425081e9573SChris Mason static int comp_keys(struct btrfs_disk_key *disk, struct btrfs_key *k2) 1426081e9573SChris Mason { 1427081e9573SChris Mason struct btrfs_key k1; 1428081e9573SChris Mason 1429081e9573SChris Mason btrfs_disk_key_to_cpu(&k1, disk); 1430081e9573SChris Mason 143120736abaSDiego Calleja return btrfs_comp_cpu_keys(&k1, k2); 1432081e9573SChris Mason } 1433081e9573SChris Mason 1434f3465ca4SJosef Bacik /* 1435f3465ca4SJosef Bacik * same as comp_keys only with two btrfs_key's 1436f3465ca4SJosef Bacik */ 14375d4f98a2SYan Zheng int btrfs_comp_cpu_keys(struct btrfs_key *k1, struct btrfs_key *k2) 1438f3465ca4SJosef Bacik { 1439f3465ca4SJosef Bacik if (k1->objectid > k2->objectid) 1440f3465ca4SJosef Bacik return 1; 1441f3465ca4SJosef Bacik if (k1->objectid < k2->objectid) 1442f3465ca4SJosef Bacik return -1; 1443f3465ca4SJosef Bacik if (k1->type > k2->type) 1444f3465ca4SJosef Bacik return 1; 1445f3465ca4SJosef Bacik if (k1->type < k2->type) 1446f3465ca4SJosef Bacik return -1; 1447f3465ca4SJosef Bacik if (k1->offset > k2->offset) 1448f3465ca4SJosef Bacik return 1; 1449f3465ca4SJosef Bacik if (k1->offset < k2->offset) 1450f3465ca4SJosef Bacik return -1; 1451f3465ca4SJosef Bacik return 0; 1452f3465ca4SJosef Bacik } 1453081e9573SChris Mason 1454d352ac68SChris Mason /* 1455d352ac68SChris Mason * this is used by the defrag code to go through all the 1456d352ac68SChris Mason * leaves pointed to by a node and reallocate them so that 1457d352ac68SChris Mason * disk order is close to key order 1458d352ac68SChris Mason */ 14596702ed49SChris Mason int btrfs_realloc_node(struct btrfs_trans_handle *trans, 14605f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *parent, 1461de78b51aSEric Sandeen int start_slot, u64 *last_ret, 1462a6b6e75eSChris Mason struct btrfs_key *progress) 14636702ed49SChris Mason { 14646b80053dSChris Mason struct extent_buffer *cur; 14656702ed49SChris Mason u64 blocknr; 1466ca7a79adSChris Mason u64 gen; 1467e9d0b13bSChris Mason u64 search_start = *last_ret; 1468e9d0b13bSChris Mason u64 last_block = 0; 14696702ed49SChris Mason u64 other; 14706702ed49SChris Mason u32 parent_nritems; 14716702ed49SChris Mason int end_slot; 14726702ed49SChris Mason int i; 14736702ed49SChris Mason int err = 0; 1474f2183bdeSChris Mason int parent_level; 14756b80053dSChris Mason int uptodate; 14766b80053dSChris Mason u32 blocksize; 1477081e9573SChris Mason int progress_passed = 0; 1478081e9573SChris Mason struct btrfs_disk_key disk_key; 14796702ed49SChris Mason 14805708b959SChris Mason parent_level = btrfs_header_level(parent); 14815708b959SChris Mason 14826c1500f2SJulia Lawall WARN_ON(trans->transaction != root->fs_info->running_transaction); 14836c1500f2SJulia Lawall WARN_ON(trans->transid != root->fs_info->generation); 148486479a04SChris Mason 14856b80053dSChris Mason parent_nritems = btrfs_header_nritems(parent); 14866b80053dSChris Mason blocksize = btrfs_level_size(root, parent_level - 1); 14876702ed49SChris Mason end_slot = parent_nritems; 14886702ed49SChris Mason 14896702ed49SChris Mason if (parent_nritems == 1) 14906702ed49SChris Mason return 0; 14916702ed49SChris Mason 1492b4ce94deSChris Mason btrfs_set_lock_blocking(parent); 1493b4ce94deSChris Mason 14946702ed49SChris Mason for (i = start_slot; i < end_slot; i++) { 14956702ed49SChris Mason int close = 1; 1496a6b6e75eSChris Mason 1497081e9573SChris Mason btrfs_node_key(parent, &disk_key, i); 1498081e9573SChris Mason if (!progress_passed && comp_keys(&disk_key, progress) < 0) 1499081e9573SChris Mason continue; 1500081e9573SChris Mason 1501081e9573SChris Mason progress_passed = 1; 15026b80053dSChris Mason blocknr = btrfs_node_blockptr(parent, i); 1503ca7a79adSChris Mason gen = btrfs_node_ptr_generation(parent, i); 1504e9d0b13bSChris Mason if (last_block == 0) 1505e9d0b13bSChris Mason last_block = blocknr; 15065708b959SChris Mason 15076702ed49SChris Mason if (i > 0) { 15086b80053dSChris Mason other = btrfs_node_blockptr(parent, i - 1); 15096b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 15106702ed49SChris Mason } 15110ef3e66bSChris Mason if (!close && i < end_slot - 2) { 15126b80053dSChris Mason other = btrfs_node_blockptr(parent, i + 1); 15136b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 15146702ed49SChris Mason } 1515e9d0b13bSChris Mason if (close) { 1516e9d0b13bSChris Mason last_block = blocknr; 15176702ed49SChris Mason continue; 1518e9d0b13bSChris Mason } 15196702ed49SChris Mason 15206b80053dSChris Mason cur = btrfs_find_tree_block(root, blocknr, blocksize); 15216b80053dSChris Mason if (cur) 1522b9fab919SChris Mason uptodate = btrfs_buffer_uptodate(cur, gen, 0); 15236b80053dSChris Mason else 15246b80053dSChris Mason uptodate = 0; 15255708b959SChris Mason if (!cur || !uptodate) { 15266b80053dSChris Mason if (!cur) { 15276b80053dSChris Mason cur = read_tree_block(root, blocknr, 1528ca7a79adSChris Mason blocksize, gen); 1529416bc658SJosef Bacik if (!cur || !extent_buffer_uptodate(cur)) { 1530416bc658SJosef Bacik free_extent_buffer(cur); 153197d9a8a4STsutomu Itoh return -EIO; 1532416bc658SJosef Bacik } 15336b80053dSChris Mason } else if (!uptodate) { 1534018642a1STsutomu Itoh err = btrfs_read_buffer(cur, gen); 1535018642a1STsutomu Itoh if (err) { 1536018642a1STsutomu Itoh free_extent_buffer(cur); 1537018642a1STsutomu Itoh return err; 1538018642a1STsutomu Itoh } 15396702ed49SChris Mason } 1540f2183bdeSChris Mason } 1541e9d0b13bSChris Mason if (search_start == 0) 15426b80053dSChris Mason search_start = last_block; 1543e9d0b13bSChris Mason 1544e7a84565SChris Mason btrfs_tree_lock(cur); 1545b4ce94deSChris Mason btrfs_set_lock_blocking(cur); 15466b80053dSChris Mason err = __btrfs_cow_block(trans, root, cur, parent, i, 1547e7a84565SChris Mason &cur, search_start, 15486b80053dSChris Mason min(16 * blocksize, 15499fa8cfe7SChris Mason (end_slot - i) * blocksize)); 1550252c38f0SYan if (err) { 1551e7a84565SChris Mason btrfs_tree_unlock(cur); 15526b80053dSChris Mason free_extent_buffer(cur); 15536702ed49SChris Mason break; 1554252c38f0SYan } 1555e7a84565SChris Mason search_start = cur->start; 1556e7a84565SChris Mason last_block = cur->start; 1557f2183bdeSChris Mason *last_ret = search_start; 1558e7a84565SChris Mason btrfs_tree_unlock(cur); 1559e7a84565SChris Mason free_extent_buffer(cur); 15606702ed49SChris Mason } 15616702ed49SChris Mason return err; 15626702ed49SChris Mason } 15636702ed49SChris Mason 156474123bd7SChris Mason /* 156574123bd7SChris Mason * The leaf data grows from end-to-front in the node. 156674123bd7SChris Mason * this returns the address of the start of the last item, 156774123bd7SChris Mason * which is the stop of the leaf data stack 156874123bd7SChris Mason */ 1569123abc88SChris Mason static inline unsigned int leaf_data_end(struct btrfs_root *root, 15705f39d397SChris Mason struct extent_buffer *leaf) 1571be0e5c09SChris Mason { 15725f39d397SChris Mason u32 nr = btrfs_header_nritems(leaf); 1573be0e5c09SChris Mason if (nr == 0) 1574123abc88SChris Mason return BTRFS_LEAF_DATA_SIZE(root); 15755f39d397SChris Mason return btrfs_item_offset_nr(leaf, nr - 1); 1576be0e5c09SChris Mason } 1577be0e5c09SChris Mason 1578aa5d6bedSChris Mason 157974123bd7SChris Mason /* 15805f39d397SChris Mason * search for key in the extent_buffer. The items start at offset p, 15815f39d397SChris Mason * and they are item_size apart. There are 'max' items in p. 15825f39d397SChris Mason * 158374123bd7SChris Mason * the slot in the array is returned via slot, and it points to 158474123bd7SChris Mason * the place where you would insert key if it is not found in 158574123bd7SChris Mason * the array. 158674123bd7SChris Mason * 158774123bd7SChris Mason * slot may point to max if the key is bigger than all of the keys 158874123bd7SChris Mason */ 1589e02119d5SChris Mason static noinline int generic_bin_search(struct extent_buffer *eb, 1590e02119d5SChris Mason unsigned long p, 15915f39d397SChris Mason int item_size, struct btrfs_key *key, 1592be0e5c09SChris Mason int max, int *slot) 1593be0e5c09SChris Mason { 1594be0e5c09SChris Mason int low = 0; 1595be0e5c09SChris Mason int high = max; 1596be0e5c09SChris Mason int mid; 1597be0e5c09SChris Mason int ret; 1598479965d6SChris Mason struct btrfs_disk_key *tmp = NULL; 15995f39d397SChris Mason struct btrfs_disk_key unaligned; 16005f39d397SChris Mason unsigned long offset; 16015f39d397SChris Mason char *kaddr = NULL; 16025f39d397SChris Mason unsigned long map_start = 0; 16035f39d397SChris Mason unsigned long map_len = 0; 1604479965d6SChris Mason int err; 1605be0e5c09SChris Mason 1606be0e5c09SChris Mason while (low < high) { 1607be0e5c09SChris Mason mid = (low + high) / 2; 16085f39d397SChris Mason offset = p + mid * item_size; 16095f39d397SChris Mason 1610a6591715SChris Mason if (!kaddr || offset < map_start || 16115f39d397SChris Mason (offset + sizeof(struct btrfs_disk_key)) > 16125f39d397SChris Mason map_start + map_len) { 1613934d375bSChris Mason 1614934d375bSChris Mason err = map_private_extent_buffer(eb, offset, 1615479965d6SChris Mason sizeof(struct btrfs_disk_key), 1616a6591715SChris Mason &kaddr, &map_start, &map_len); 16175f39d397SChris Mason 1618479965d6SChris Mason if (!err) { 1619479965d6SChris Mason tmp = (struct btrfs_disk_key *)(kaddr + offset - 1620479965d6SChris Mason map_start); 1621479965d6SChris Mason } else { 16225f39d397SChris Mason read_extent_buffer(eb, &unaligned, 16235f39d397SChris Mason offset, sizeof(unaligned)); 16245f39d397SChris Mason tmp = &unaligned; 1625479965d6SChris Mason } 1626479965d6SChris Mason 16275f39d397SChris Mason } else { 16285f39d397SChris Mason tmp = (struct btrfs_disk_key *)(kaddr + offset - 16295f39d397SChris Mason map_start); 16305f39d397SChris Mason } 1631be0e5c09SChris Mason ret = comp_keys(tmp, key); 1632be0e5c09SChris Mason 1633be0e5c09SChris Mason if (ret < 0) 1634be0e5c09SChris Mason low = mid + 1; 1635be0e5c09SChris Mason else if (ret > 0) 1636be0e5c09SChris Mason high = mid; 1637be0e5c09SChris Mason else { 1638be0e5c09SChris Mason *slot = mid; 1639be0e5c09SChris Mason return 0; 1640be0e5c09SChris Mason } 1641be0e5c09SChris Mason } 1642be0e5c09SChris Mason *slot = low; 1643be0e5c09SChris Mason return 1; 1644be0e5c09SChris Mason } 1645be0e5c09SChris Mason 164697571fd0SChris Mason /* 164797571fd0SChris Mason * simple bin_search frontend that does the right thing for 164897571fd0SChris Mason * leaves vs nodes 164997571fd0SChris Mason */ 16505f39d397SChris Mason static int bin_search(struct extent_buffer *eb, struct btrfs_key *key, 16515f39d397SChris Mason int level, int *slot) 1652be0e5c09SChris Mason { 1653f775738fSWang Sheng-Hui if (level == 0) 16545f39d397SChris Mason return generic_bin_search(eb, 16555f39d397SChris Mason offsetof(struct btrfs_leaf, items), 16560783fcfcSChris Mason sizeof(struct btrfs_item), 16575f39d397SChris Mason key, btrfs_header_nritems(eb), 16587518a238SChris Mason slot); 1659f775738fSWang Sheng-Hui else 16605f39d397SChris Mason return generic_bin_search(eb, 16615f39d397SChris Mason offsetof(struct btrfs_node, ptrs), 1662123abc88SChris Mason sizeof(struct btrfs_key_ptr), 16635f39d397SChris Mason key, btrfs_header_nritems(eb), 16647518a238SChris Mason slot); 1665be0e5c09SChris Mason } 1666be0e5c09SChris Mason 16675d4f98a2SYan Zheng int btrfs_bin_search(struct extent_buffer *eb, struct btrfs_key *key, 16685d4f98a2SYan Zheng int level, int *slot) 16695d4f98a2SYan Zheng { 16705d4f98a2SYan Zheng return bin_search(eb, key, level, slot); 16715d4f98a2SYan Zheng } 16725d4f98a2SYan Zheng 1673f0486c68SYan, Zheng static void root_add_used(struct btrfs_root *root, u32 size) 1674f0486c68SYan, Zheng { 1675f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 1676f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 1677f0486c68SYan, Zheng btrfs_root_used(&root->root_item) + size); 1678f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 1679f0486c68SYan, Zheng } 1680f0486c68SYan, Zheng 1681f0486c68SYan, Zheng static void root_sub_used(struct btrfs_root *root, u32 size) 1682f0486c68SYan, Zheng { 1683f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 1684f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 1685f0486c68SYan, Zheng btrfs_root_used(&root->root_item) - size); 1686f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 1687f0486c68SYan, Zheng } 1688f0486c68SYan, Zheng 1689d352ac68SChris Mason /* given a node and slot number, this reads the blocks it points to. The 1690d352ac68SChris Mason * extent buffer is returned with a reference taken (but unlocked). 1691d352ac68SChris Mason * NULL is returned on error. 1692d352ac68SChris Mason */ 1693e02119d5SChris Mason static noinline struct extent_buffer *read_node_slot(struct btrfs_root *root, 16945f39d397SChris Mason struct extent_buffer *parent, int slot) 1695bb803951SChris Mason { 1696ca7a79adSChris Mason int level = btrfs_header_level(parent); 1697416bc658SJosef Bacik struct extent_buffer *eb; 1698416bc658SJosef Bacik 1699bb803951SChris Mason if (slot < 0) 1700bb803951SChris Mason return NULL; 17015f39d397SChris Mason if (slot >= btrfs_header_nritems(parent)) 1702bb803951SChris Mason return NULL; 1703ca7a79adSChris Mason 1704ca7a79adSChris Mason BUG_ON(level == 0); 1705ca7a79adSChris Mason 1706416bc658SJosef Bacik eb = read_tree_block(root, btrfs_node_blockptr(parent, slot), 1707ca7a79adSChris Mason btrfs_level_size(root, level - 1), 1708ca7a79adSChris Mason btrfs_node_ptr_generation(parent, slot)); 1709416bc658SJosef Bacik if (eb && !extent_buffer_uptodate(eb)) { 1710416bc658SJosef Bacik free_extent_buffer(eb); 1711416bc658SJosef Bacik eb = NULL; 1712416bc658SJosef Bacik } 1713416bc658SJosef Bacik 1714416bc658SJosef Bacik return eb; 1715bb803951SChris Mason } 1716bb803951SChris Mason 1717d352ac68SChris Mason /* 1718d352ac68SChris Mason * node level balancing, used to make sure nodes are in proper order for 1719d352ac68SChris Mason * item deletion. We balance from the top down, so we have to make sure 1720d352ac68SChris Mason * that a deletion won't leave an node completely empty later on. 1721d352ac68SChris Mason */ 1722e02119d5SChris Mason static noinline int balance_level(struct btrfs_trans_handle *trans, 172398ed5174SChris Mason struct btrfs_root *root, 172498ed5174SChris Mason struct btrfs_path *path, int level) 1725bb803951SChris Mason { 17265f39d397SChris Mason struct extent_buffer *right = NULL; 17275f39d397SChris Mason struct extent_buffer *mid; 17285f39d397SChris Mason struct extent_buffer *left = NULL; 17295f39d397SChris Mason struct extent_buffer *parent = NULL; 1730bb803951SChris Mason int ret = 0; 1731bb803951SChris Mason int wret; 1732bb803951SChris Mason int pslot; 1733bb803951SChris Mason int orig_slot = path->slots[level]; 173479f95c82SChris Mason u64 orig_ptr; 1735bb803951SChris Mason 1736bb803951SChris Mason if (level == 0) 1737bb803951SChris Mason return 0; 1738bb803951SChris Mason 17395f39d397SChris Mason mid = path->nodes[level]; 1740b4ce94deSChris Mason 1741bd681513SChris Mason WARN_ON(path->locks[level] != BTRFS_WRITE_LOCK && 1742bd681513SChris Mason path->locks[level] != BTRFS_WRITE_LOCK_BLOCKING); 17437bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 17447bb86316SChris Mason 17451d4f8a0cSChris Mason orig_ptr = btrfs_node_blockptr(mid, orig_slot); 174679f95c82SChris Mason 1747a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 17485f39d397SChris Mason parent = path->nodes[level + 1]; 1749bb803951SChris Mason pslot = path->slots[level + 1]; 1750a05a9bb1SLi Zefan } 1751bb803951SChris Mason 175240689478SChris Mason /* 175340689478SChris Mason * deal with the case where there is only one pointer in the root 175440689478SChris Mason * by promoting the node below to a root 175540689478SChris Mason */ 17565f39d397SChris Mason if (!parent) { 17575f39d397SChris Mason struct extent_buffer *child; 1758bb803951SChris Mason 17595f39d397SChris Mason if (btrfs_header_nritems(mid) != 1) 1760bb803951SChris Mason return 0; 1761bb803951SChris Mason 1762bb803951SChris Mason /* promote the child to a root */ 17635f39d397SChris Mason child = read_node_slot(root, mid, 0); 1764305a26afSMark Fasheh if (!child) { 1765305a26afSMark Fasheh ret = -EROFS; 1766305a26afSMark Fasheh btrfs_std_error(root->fs_info, ret); 1767305a26afSMark Fasheh goto enospc; 1768305a26afSMark Fasheh } 1769305a26afSMark Fasheh 1770925baeddSChris Mason btrfs_tree_lock(child); 1771b4ce94deSChris Mason btrfs_set_lock_blocking(child); 17729fa8cfe7SChris Mason ret = btrfs_cow_block(trans, root, child, mid, 0, &child); 1773f0486c68SYan, Zheng if (ret) { 1774f0486c68SYan, Zheng btrfs_tree_unlock(child); 1775f0486c68SYan, Zheng free_extent_buffer(child); 1776f0486c68SYan, Zheng goto enospc; 1777f0486c68SYan, Zheng } 17782f375ab9SYan 177990f8d62eSJan Schmidt tree_mod_log_set_root_pointer(root, child, 1); 1780240f62c8SChris Mason rcu_assign_pointer(root->node, child); 1781925baeddSChris Mason 17820b86a832SChris Mason add_root_to_dirty_list(root); 1783925baeddSChris Mason btrfs_tree_unlock(child); 1784b4ce94deSChris Mason 1785925baeddSChris Mason path->locks[level] = 0; 1786bb803951SChris Mason path->nodes[level] = NULL; 17875f39d397SChris Mason clean_tree_block(trans, root, mid); 1788925baeddSChris Mason btrfs_tree_unlock(mid); 1789bb803951SChris Mason /* once for the path */ 17905f39d397SChris Mason free_extent_buffer(mid); 1791f0486c68SYan, Zheng 1792f0486c68SYan, Zheng root_sub_used(root, mid->len); 17935581a51aSJan Schmidt btrfs_free_tree_block(trans, root, mid, 0, 1); 1794bb803951SChris Mason /* once for the root ptr */ 17953083ee2eSJosef Bacik free_extent_buffer_stale(mid); 1796f0486c68SYan, Zheng return 0; 1797bb803951SChris Mason } 17985f39d397SChris Mason if (btrfs_header_nritems(mid) > 1799123abc88SChris Mason BTRFS_NODEPTRS_PER_BLOCK(root) / 4) 1800bb803951SChris Mason return 0; 1801bb803951SChris Mason 18025f39d397SChris Mason left = read_node_slot(root, parent, pslot - 1); 18035f39d397SChris Mason if (left) { 1804925baeddSChris Mason btrfs_tree_lock(left); 1805b4ce94deSChris Mason btrfs_set_lock_blocking(left); 18065f39d397SChris Mason wret = btrfs_cow_block(trans, root, left, 18079fa8cfe7SChris Mason parent, pslot - 1, &left); 180854aa1f4dSChris Mason if (wret) { 180954aa1f4dSChris Mason ret = wret; 181054aa1f4dSChris Mason goto enospc; 181154aa1f4dSChris Mason } 18122cc58cf2SChris Mason } 18135f39d397SChris Mason right = read_node_slot(root, parent, pslot + 1); 18145f39d397SChris Mason if (right) { 1815925baeddSChris Mason btrfs_tree_lock(right); 1816b4ce94deSChris Mason btrfs_set_lock_blocking(right); 18175f39d397SChris Mason wret = btrfs_cow_block(trans, root, right, 18189fa8cfe7SChris Mason parent, pslot + 1, &right); 18192cc58cf2SChris Mason if (wret) { 18202cc58cf2SChris Mason ret = wret; 18212cc58cf2SChris Mason goto enospc; 18222cc58cf2SChris Mason } 18232cc58cf2SChris Mason } 18242cc58cf2SChris Mason 18252cc58cf2SChris Mason /* first, try to make some room in the middle buffer */ 18265f39d397SChris Mason if (left) { 18275f39d397SChris Mason orig_slot += btrfs_header_nritems(left); 1828bce4eae9SChris Mason wret = push_node_left(trans, root, left, mid, 1); 182979f95c82SChris Mason if (wret < 0) 183079f95c82SChris Mason ret = wret; 1831bb803951SChris Mason } 183279f95c82SChris Mason 183379f95c82SChris Mason /* 183479f95c82SChris Mason * then try to empty the right most buffer into the middle 183579f95c82SChris Mason */ 18365f39d397SChris Mason if (right) { 1837971a1f66SChris Mason wret = push_node_left(trans, root, mid, right, 1); 183854aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 183979f95c82SChris Mason ret = wret; 18405f39d397SChris Mason if (btrfs_header_nritems(right) == 0) { 18415f39d397SChris Mason clean_tree_block(trans, root, right); 1842925baeddSChris Mason btrfs_tree_unlock(right); 1843afe5fea7STsutomu Itoh del_ptr(root, path, level + 1, pslot + 1); 1844f0486c68SYan, Zheng root_sub_used(root, right->len); 18455581a51aSJan Schmidt btrfs_free_tree_block(trans, root, right, 0, 1); 18463083ee2eSJosef Bacik free_extent_buffer_stale(right); 1847f0486c68SYan, Zheng right = NULL; 1848bb803951SChris Mason } else { 18495f39d397SChris Mason struct btrfs_disk_key right_key; 18505f39d397SChris Mason btrfs_node_key(right, &right_key, 0); 1851f230475eSJan Schmidt tree_mod_log_set_node_key(root->fs_info, parent, 185232adf090SLiu Bo pslot + 1, 0); 18535f39d397SChris Mason btrfs_set_node_key(parent, &right_key, pslot + 1); 18545f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 1855bb803951SChris Mason } 1856bb803951SChris Mason } 18575f39d397SChris Mason if (btrfs_header_nritems(mid) == 1) { 185879f95c82SChris Mason /* 185979f95c82SChris Mason * we're not allowed to leave a node with one item in the 186079f95c82SChris Mason * tree during a delete. A deletion from lower in the tree 186179f95c82SChris Mason * could try to delete the only pointer in this node. 186279f95c82SChris Mason * So, pull some keys from the left. 186379f95c82SChris Mason * There has to be a left pointer at this point because 186479f95c82SChris Mason * otherwise we would have pulled some pointers from the 186579f95c82SChris Mason * right 186679f95c82SChris Mason */ 1867305a26afSMark Fasheh if (!left) { 1868305a26afSMark Fasheh ret = -EROFS; 1869305a26afSMark Fasheh btrfs_std_error(root->fs_info, ret); 1870305a26afSMark Fasheh goto enospc; 1871305a26afSMark Fasheh } 18725f39d397SChris Mason wret = balance_node_right(trans, root, mid, left); 187354aa1f4dSChris Mason if (wret < 0) { 187479f95c82SChris Mason ret = wret; 187554aa1f4dSChris Mason goto enospc; 187654aa1f4dSChris Mason } 1877bce4eae9SChris Mason if (wret == 1) { 1878bce4eae9SChris Mason wret = push_node_left(trans, root, left, mid, 1); 1879bce4eae9SChris Mason if (wret < 0) 1880bce4eae9SChris Mason ret = wret; 1881bce4eae9SChris Mason } 188279f95c82SChris Mason BUG_ON(wret == 1); 188379f95c82SChris Mason } 18845f39d397SChris Mason if (btrfs_header_nritems(mid) == 0) { 18855f39d397SChris Mason clean_tree_block(trans, root, mid); 1886925baeddSChris Mason btrfs_tree_unlock(mid); 1887afe5fea7STsutomu Itoh del_ptr(root, path, level + 1, pslot); 1888f0486c68SYan, Zheng root_sub_used(root, mid->len); 18895581a51aSJan Schmidt btrfs_free_tree_block(trans, root, mid, 0, 1); 18903083ee2eSJosef Bacik free_extent_buffer_stale(mid); 1891f0486c68SYan, Zheng mid = NULL; 189279f95c82SChris Mason } else { 189379f95c82SChris Mason /* update the parent key to reflect our changes */ 18945f39d397SChris Mason struct btrfs_disk_key mid_key; 18955f39d397SChris Mason btrfs_node_key(mid, &mid_key, 0); 189632adf090SLiu Bo tree_mod_log_set_node_key(root->fs_info, parent, 1897f230475eSJan Schmidt pslot, 0); 18985f39d397SChris Mason btrfs_set_node_key(parent, &mid_key, pslot); 18995f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 190079f95c82SChris Mason } 1901bb803951SChris Mason 190279f95c82SChris Mason /* update the path */ 19035f39d397SChris Mason if (left) { 19045f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 19055f39d397SChris Mason extent_buffer_get(left); 1906925baeddSChris Mason /* left was locked after cow */ 19075f39d397SChris Mason path->nodes[level] = left; 1908bb803951SChris Mason path->slots[level + 1] -= 1; 1909bb803951SChris Mason path->slots[level] = orig_slot; 1910925baeddSChris Mason if (mid) { 1911925baeddSChris Mason btrfs_tree_unlock(mid); 19125f39d397SChris Mason free_extent_buffer(mid); 1913925baeddSChris Mason } 1914bb803951SChris Mason } else { 19155f39d397SChris Mason orig_slot -= btrfs_header_nritems(left); 1916bb803951SChris Mason path->slots[level] = orig_slot; 1917bb803951SChris Mason } 1918bb803951SChris Mason } 191979f95c82SChris Mason /* double check we haven't messed things up */ 1920e20d96d6SChris Mason if (orig_ptr != 19215f39d397SChris Mason btrfs_node_blockptr(path->nodes[level], path->slots[level])) 192279f95c82SChris Mason BUG(); 192354aa1f4dSChris Mason enospc: 1924925baeddSChris Mason if (right) { 1925925baeddSChris Mason btrfs_tree_unlock(right); 19265f39d397SChris Mason free_extent_buffer(right); 1927925baeddSChris Mason } 1928925baeddSChris Mason if (left) { 1929925baeddSChris Mason if (path->nodes[level] != left) 1930925baeddSChris Mason btrfs_tree_unlock(left); 19315f39d397SChris Mason free_extent_buffer(left); 1932925baeddSChris Mason } 1933bb803951SChris Mason return ret; 1934bb803951SChris Mason } 1935bb803951SChris Mason 1936d352ac68SChris Mason /* Node balancing for insertion. Here we only split or push nodes around 1937d352ac68SChris Mason * when they are completely full. This is also done top down, so we 1938d352ac68SChris Mason * have to be pessimistic. 1939d352ac68SChris Mason */ 1940d397712bSChris Mason static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans, 1941e66f709bSChris Mason struct btrfs_root *root, 1942e66f709bSChris Mason struct btrfs_path *path, int level) 1943e66f709bSChris Mason { 19445f39d397SChris Mason struct extent_buffer *right = NULL; 19455f39d397SChris Mason struct extent_buffer *mid; 19465f39d397SChris Mason struct extent_buffer *left = NULL; 19475f39d397SChris Mason struct extent_buffer *parent = NULL; 1948e66f709bSChris Mason int ret = 0; 1949e66f709bSChris Mason int wret; 1950e66f709bSChris Mason int pslot; 1951e66f709bSChris Mason int orig_slot = path->slots[level]; 1952e66f709bSChris Mason 1953e66f709bSChris Mason if (level == 0) 1954e66f709bSChris Mason return 1; 1955e66f709bSChris Mason 19565f39d397SChris Mason mid = path->nodes[level]; 19577bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 1958e66f709bSChris Mason 1959a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 19605f39d397SChris Mason parent = path->nodes[level + 1]; 1961e66f709bSChris Mason pslot = path->slots[level + 1]; 1962a05a9bb1SLi Zefan } 1963e66f709bSChris Mason 19645f39d397SChris Mason if (!parent) 1965e66f709bSChris Mason return 1; 1966e66f709bSChris Mason 19675f39d397SChris Mason left = read_node_slot(root, parent, pslot - 1); 1968e66f709bSChris Mason 1969e66f709bSChris Mason /* first, try to make some room in the middle buffer */ 19705f39d397SChris Mason if (left) { 1971e66f709bSChris Mason u32 left_nr; 1972925baeddSChris Mason 1973925baeddSChris Mason btrfs_tree_lock(left); 1974b4ce94deSChris Mason btrfs_set_lock_blocking(left); 1975b4ce94deSChris Mason 19765f39d397SChris Mason left_nr = btrfs_header_nritems(left); 197733ade1f8SChris Mason if (left_nr >= BTRFS_NODEPTRS_PER_BLOCK(root) - 1) { 197833ade1f8SChris Mason wret = 1; 197933ade1f8SChris Mason } else { 19805f39d397SChris Mason ret = btrfs_cow_block(trans, root, left, parent, 19819fa8cfe7SChris Mason pslot - 1, &left); 198254aa1f4dSChris Mason if (ret) 198354aa1f4dSChris Mason wret = 1; 198454aa1f4dSChris Mason else { 198554aa1f4dSChris Mason wret = push_node_left(trans, root, 1986971a1f66SChris Mason left, mid, 0); 198754aa1f4dSChris Mason } 198833ade1f8SChris Mason } 1989e66f709bSChris Mason if (wret < 0) 1990e66f709bSChris Mason ret = wret; 1991e66f709bSChris Mason if (wret == 0) { 19925f39d397SChris Mason struct btrfs_disk_key disk_key; 1993e66f709bSChris Mason orig_slot += left_nr; 19945f39d397SChris Mason btrfs_node_key(mid, &disk_key, 0); 1995f230475eSJan Schmidt tree_mod_log_set_node_key(root->fs_info, parent, 199632adf090SLiu Bo pslot, 0); 19975f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot); 19985f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 19995f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 20005f39d397SChris Mason path->nodes[level] = left; 2001e66f709bSChris Mason path->slots[level + 1] -= 1; 2002e66f709bSChris Mason path->slots[level] = orig_slot; 2003925baeddSChris Mason btrfs_tree_unlock(mid); 20045f39d397SChris Mason free_extent_buffer(mid); 2005e66f709bSChris Mason } else { 2006e66f709bSChris Mason orig_slot -= 20075f39d397SChris Mason btrfs_header_nritems(left); 2008e66f709bSChris Mason path->slots[level] = orig_slot; 2009925baeddSChris Mason btrfs_tree_unlock(left); 20105f39d397SChris Mason free_extent_buffer(left); 2011e66f709bSChris Mason } 2012e66f709bSChris Mason return 0; 2013e66f709bSChris Mason } 2014925baeddSChris Mason btrfs_tree_unlock(left); 20155f39d397SChris Mason free_extent_buffer(left); 2016e66f709bSChris Mason } 20175f39d397SChris Mason right = read_node_slot(root, parent, pslot + 1); 2018e66f709bSChris Mason 2019e66f709bSChris Mason /* 2020e66f709bSChris Mason * then try to empty the right most buffer into the middle 2021e66f709bSChris Mason */ 20225f39d397SChris Mason if (right) { 202333ade1f8SChris Mason u32 right_nr; 2024b4ce94deSChris Mason 2025925baeddSChris Mason btrfs_tree_lock(right); 2026b4ce94deSChris Mason btrfs_set_lock_blocking(right); 2027b4ce94deSChris Mason 20285f39d397SChris Mason right_nr = btrfs_header_nritems(right); 202933ade1f8SChris Mason if (right_nr >= BTRFS_NODEPTRS_PER_BLOCK(root) - 1) { 203033ade1f8SChris Mason wret = 1; 203133ade1f8SChris Mason } else { 20325f39d397SChris Mason ret = btrfs_cow_block(trans, root, right, 20335f39d397SChris Mason parent, pslot + 1, 20349fa8cfe7SChris Mason &right); 203554aa1f4dSChris Mason if (ret) 203654aa1f4dSChris Mason wret = 1; 203754aa1f4dSChris Mason else { 203833ade1f8SChris Mason wret = balance_node_right(trans, root, 20395f39d397SChris Mason right, mid); 204033ade1f8SChris Mason } 204154aa1f4dSChris Mason } 2042e66f709bSChris Mason if (wret < 0) 2043e66f709bSChris Mason ret = wret; 2044e66f709bSChris Mason if (wret == 0) { 20455f39d397SChris Mason struct btrfs_disk_key disk_key; 20465f39d397SChris Mason 20475f39d397SChris Mason btrfs_node_key(right, &disk_key, 0); 2048f230475eSJan Schmidt tree_mod_log_set_node_key(root->fs_info, parent, 204932adf090SLiu Bo pslot + 1, 0); 20505f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot + 1); 20515f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 20525f39d397SChris Mason 20535f39d397SChris Mason if (btrfs_header_nritems(mid) <= orig_slot) { 20545f39d397SChris Mason path->nodes[level] = right; 2055e66f709bSChris Mason path->slots[level + 1] += 1; 2056e66f709bSChris Mason path->slots[level] = orig_slot - 20575f39d397SChris Mason btrfs_header_nritems(mid); 2058925baeddSChris Mason btrfs_tree_unlock(mid); 20595f39d397SChris Mason free_extent_buffer(mid); 2060e66f709bSChris Mason } else { 2061925baeddSChris Mason btrfs_tree_unlock(right); 20625f39d397SChris Mason free_extent_buffer(right); 2063e66f709bSChris Mason } 2064e66f709bSChris Mason return 0; 2065e66f709bSChris Mason } 2066925baeddSChris Mason btrfs_tree_unlock(right); 20675f39d397SChris Mason free_extent_buffer(right); 2068e66f709bSChris Mason } 2069e66f709bSChris Mason return 1; 2070e66f709bSChris Mason } 2071e66f709bSChris Mason 207274123bd7SChris Mason /* 2073d352ac68SChris Mason * readahead one full node of leaves, finding things that are close 2074d352ac68SChris Mason * to the block in 'slot', and triggering ra on them. 20753c69faecSChris Mason */ 2076c8c42864SChris Mason static void reada_for_search(struct btrfs_root *root, 2077e02119d5SChris Mason struct btrfs_path *path, 207801f46658SChris Mason int level, int slot, u64 objectid) 20793c69faecSChris Mason { 20805f39d397SChris Mason struct extent_buffer *node; 208101f46658SChris Mason struct btrfs_disk_key disk_key; 20823c69faecSChris Mason u32 nritems; 20833c69faecSChris Mason u64 search; 2084a7175319SChris Mason u64 target; 20856b80053dSChris Mason u64 nread = 0; 2086cb25c2eaSJosef Bacik u64 gen; 20873c69faecSChris Mason int direction = path->reada; 20885f39d397SChris Mason struct extent_buffer *eb; 20896b80053dSChris Mason u32 nr; 20906b80053dSChris Mason u32 blocksize; 20916b80053dSChris Mason u32 nscan = 0; 2092db94535dSChris Mason 2093a6b6e75eSChris Mason if (level != 1) 20943c69faecSChris Mason return; 20953c69faecSChris Mason 20966702ed49SChris Mason if (!path->nodes[level]) 20976702ed49SChris Mason return; 20986702ed49SChris Mason 20995f39d397SChris Mason node = path->nodes[level]; 2100925baeddSChris Mason 21013c69faecSChris Mason search = btrfs_node_blockptr(node, slot); 21026b80053dSChris Mason blocksize = btrfs_level_size(root, level - 1); 21036b80053dSChris Mason eb = btrfs_find_tree_block(root, search, blocksize); 21045f39d397SChris Mason if (eb) { 21055f39d397SChris Mason free_extent_buffer(eb); 21063c69faecSChris Mason return; 21073c69faecSChris Mason } 21083c69faecSChris Mason 2109a7175319SChris Mason target = search; 21106b80053dSChris Mason 21115f39d397SChris Mason nritems = btrfs_header_nritems(node); 21126b80053dSChris Mason nr = slot; 211325b8b936SJosef Bacik 21143c69faecSChris Mason while (1) { 21156b80053dSChris Mason if (direction < 0) { 21166b80053dSChris Mason if (nr == 0) 21173c69faecSChris Mason break; 21186b80053dSChris Mason nr--; 21196b80053dSChris Mason } else if (direction > 0) { 21206b80053dSChris Mason nr++; 21216b80053dSChris Mason if (nr >= nritems) 21226b80053dSChris Mason break; 21233c69faecSChris Mason } 212401f46658SChris Mason if (path->reada < 0 && objectid) { 212501f46658SChris Mason btrfs_node_key(node, &disk_key, nr); 212601f46658SChris Mason if (btrfs_disk_key_objectid(&disk_key) != objectid) 212701f46658SChris Mason break; 212801f46658SChris Mason } 21296b80053dSChris Mason search = btrfs_node_blockptr(node, nr); 2130a7175319SChris Mason if ((search <= target && target - search <= 65536) || 2131a7175319SChris Mason (search > target && search - target <= 65536)) { 2132cb25c2eaSJosef Bacik gen = btrfs_node_ptr_generation(node, nr); 2133cb25c2eaSJosef Bacik readahead_tree_block(root, search, blocksize, gen); 21346b80053dSChris Mason nread += blocksize; 21353c69faecSChris Mason } 21366b80053dSChris Mason nscan++; 2137a7175319SChris Mason if ((nread > 65536 || nscan > 32)) 21386b80053dSChris Mason break; 21393c69faecSChris Mason } 21403c69faecSChris Mason } 2141925baeddSChris Mason 21420b08851fSJosef Bacik static noinline void reada_for_balance(struct btrfs_root *root, 2143b4ce94deSChris Mason struct btrfs_path *path, int level) 2144b4ce94deSChris Mason { 2145b4ce94deSChris Mason int slot; 2146b4ce94deSChris Mason int nritems; 2147b4ce94deSChris Mason struct extent_buffer *parent; 2148b4ce94deSChris Mason struct extent_buffer *eb; 2149b4ce94deSChris Mason u64 gen; 2150b4ce94deSChris Mason u64 block1 = 0; 2151b4ce94deSChris Mason u64 block2 = 0; 2152b4ce94deSChris Mason int blocksize; 2153b4ce94deSChris Mason 21548c594ea8SChris Mason parent = path->nodes[level + 1]; 2155b4ce94deSChris Mason if (!parent) 21560b08851fSJosef Bacik return; 2157b4ce94deSChris Mason 2158b4ce94deSChris Mason nritems = btrfs_header_nritems(parent); 21598c594ea8SChris Mason slot = path->slots[level + 1]; 2160b4ce94deSChris Mason blocksize = btrfs_level_size(root, level); 2161b4ce94deSChris Mason 2162b4ce94deSChris Mason if (slot > 0) { 2163b4ce94deSChris Mason block1 = btrfs_node_blockptr(parent, slot - 1); 2164b4ce94deSChris Mason gen = btrfs_node_ptr_generation(parent, slot - 1); 2165b4ce94deSChris Mason eb = btrfs_find_tree_block(root, block1, blocksize); 2166b9fab919SChris Mason /* 2167b9fab919SChris Mason * if we get -eagain from btrfs_buffer_uptodate, we 2168b9fab919SChris Mason * don't want to return eagain here. That will loop 2169b9fab919SChris Mason * forever 2170b9fab919SChris Mason */ 2171b9fab919SChris Mason if (eb && btrfs_buffer_uptodate(eb, gen, 1) != 0) 2172b4ce94deSChris Mason block1 = 0; 2173b4ce94deSChris Mason free_extent_buffer(eb); 2174b4ce94deSChris Mason } 21758c594ea8SChris Mason if (slot + 1 < nritems) { 2176b4ce94deSChris Mason block2 = btrfs_node_blockptr(parent, slot + 1); 2177b4ce94deSChris Mason gen = btrfs_node_ptr_generation(parent, slot + 1); 2178b4ce94deSChris Mason eb = btrfs_find_tree_block(root, block2, blocksize); 2179b9fab919SChris Mason if (eb && btrfs_buffer_uptodate(eb, gen, 1) != 0) 2180b4ce94deSChris Mason block2 = 0; 2181b4ce94deSChris Mason free_extent_buffer(eb); 2182b4ce94deSChris Mason } 21838c594ea8SChris Mason 2184b4ce94deSChris Mason if (block1) 2185b4ce94deSChris Mason readahead_tree_block(root, block1, blocksize, 0); 2186b4ce94deSChris Mason if (block2) 2187b4ce94deSChris Mason readahead_tree_block(root, block2, blocksize, 0); 2188b4ce94deSChris Mason } 2189b4ce94deSChris Mason 2190b4ce94deSChris Mason 2191b4ce94deSChris Mason /* 2192d397712bSChris Mason * when we walk down the tree, it is usually safe to unlock the higher layers 2193d397712bSChris Mason * in the tree. The exceptions are when our path goes through slot 0, because 2194d397712bSChris Mason * operations on the tree might require changing key pointers higher up in the 2195d397712bSChris Mason * tree. 2196d352ac68SChris Mason * 2197d397712bSChris Mason * callers might also have set path->keep_locks, which tells this code to keep 2198d397712bSChris Mason * the lock if the path points to the last slot in the block. This is part of 2199d397712bSChris Mason * walking through the tree, and selecting the next slot in the higher block. 2200d352ac68SChris Mason * 2201d397712bSChris Mason * lowest_unlock sets the lowest level in the tree we're allowed to unlock. so 2202d397712bSChris Mason * if lowest_unlock is 1, level 0 won't be unlocked 2203d352ac68SChris Mason */ 2204e02119d5SChris Mason static noinline void unlock_up(struct btrfs_path *path, int level, 2205f7c79f30SChris Mason int lowest_unlock, int min_write_lock_level, 2206f7c79f30SChris Mason int *write_lock_level) 2207925baeddSChris Mason { 2208925baeddSChris Mason int i; 2209925baeddSChris Mason int skip_level = level; 2210051e1b9fSChris Mason int no_skips = 0; 2211925baeddSChris Mason struct extent_buffer *t; 2212925baeddSChris Mason 2213925baeddSChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 2214925baeddSChris Mason if (!path->nodes[i]) 2215925baeddSChris Mason break; 2216925baeddSChris Mason if (!path->locks[i]) 2217925baeddSChris Mason break; 2218051e1b9fSChris Mason if (!no_skips && path->slots[i] == 0) { 2219925baeddSChris Mason skip_level = i + 1; 2220925baeddSChris Mason continue; 2221925baeddSChris Mason } 2222051e1b9fSChris Mason if (!no_skips && path->keep_locks) { 2223925baeddSChris Mason u32 nritems; 2224925baeddSChris Mason t = path->nodes[i]; 2225925baeddSChris Mason nritems = btrfs_header_nritems(t); 2226051e1b9fSChris Mason if (nritems < 1 || path->slots[i] >= nritems - 1) { 2227925baeddSChris Mason skip_level = i + 1; 2228925baeddSChris Mason continue; 2229925baeddSChris Mason } 2230925baeddSChris Mason } 2231051e1b9fSChris Mason if (skip_level < i && i >= lowest_unlock) 2232051e1b9fSChris Mason no_skips = 1; 2233051e1b9fSChris Mason 2234925baeddSChris Mason t = path->nodes[i]; 2235925baeddSChris Mason if (i >= lowest_unlock && i > skip_level && path->locks[i]) { 2236bd681513SChris Mason btrfs_tree_unlock_rw(t, path->locks[i]); 2237925baeddSChris Mason path->locks[i] = 0; 2238f7c79f30SChris Mason if (write_lock_level && 2239f7c79f30SChris Mason i > min_write_lock_level && 2240f7c79f30SChris Mason i <= *write_lock_level) { 2241f7c79f30SChris Mason *write_lock_level = i - 1; 2242f7c79f30SChris Mason } 2243925baeddSChris Mason } 2244925baeddSChris Mason } 2245925baeddSChris Mason } 2246925baeddSChris Mason 22473c69faecSChris Mason /* 2248b4ce94deSChris Mason * This releases any locks held in the path starting at level and 2249b4ce94deSChris Mason * going all the way up to the root. 2250b4ce94deSChris Mason * 2251b4ce94deSChris Mason * btrfs_search_slot will keep the lock held on higher nodes in a few 2252b4ce94deSChris Mason * corner cases, such as COW of the block at slot zero in the node. This 2253b4ce94deSChris Mason * ignores those rules, and it should only be called when there are no 2254b4ce94deSChris Mason * more updates to be done higher up in the tree. 2255b4ce94deSChris Mason */ 2256b4ce94deSChris Mason noinline void btrfs_unlock_up_safe(struct btrfs_path *path, int level) 2257b4ce94deSChris Mason { 2258b4ce94deSChris Mason int i; 2259b4ce94deSChris Mason 226009a2a8f9SJosef Bacik if (path->keep_locks) 2261b4ce94deSChris Mason return; 2262b4ce94deSChris Mason 2263b4ce94deSChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 2264b4ce94deSChris Mason if (!path->nodes[i]) 226512f4daccSChris Mason continue; 2266b4ce94deSChris Mason if (!path->locks[i]) 226712f4daccSChris Mason continue; 2268bd681513SChris Mason btrfs_tree_unlock_rw(path->nodes[i], path->locks[i]); 2269b4ce94deSChris Mason path->locks[i] = 0; 2270b4ce94deSChris Mason } 2271b4ce94deSChris Mason } 2272b4ce94deSChris Mason 2273b4ce94deSChris Mason /* 2274c8c42864SChris Mason * helper function for btrfs_search_slot. The goal is to find a block 2275c8c42864SChris Mason * in cache without setting the path to blocking. If we find the block 2276c8c42864SChris Mason * we return zero and the path is unchanged. 2277c8c42864SChris Mason * 2278c8c42864SChris Mason * If we can't find the block, we set the path blocking and do some 2279c8c42864SChris Mason * reada. -EAGAIN is returned and the search must be repeated. 2280c8c42864SChris Mason */ 2281c8c42864SChris Mason static int 2282c8c42864SChris Mason read_block_for_search(struct btrfs_trans_handle *trans, 2283c8c42864SChris Mason struct btrfs_root *root, struct btrfs_path *p, 2284c8c42864SChris Mason struct extent_buffer **eb_ret, int level, int slot, 22855d9e75c4SJan Schmidt struct btrfs_key *key, u64 time_seq) 2286c8c42864SChris Mason { 2287c8c42864SChris Mason u64 blocknr; 2288c8c42864SChris Mason u64 gen; 2289c8c42864SChris Mason u32 blocksize; 2290c8c42864SChris Mason struct extent_buffer *b = *eb_ret; 2291c8c42864SChris Mason struct extent_buffer *tmp; 229276a05b35SChris Mason int ret; 2293c8c42864SChris Mason 2294c8c42864SChris Mason blocknr = btrfs_node_blockptr(b, slot); 2295c8c42864SChris Mason gen = btrfs_node_ptr_generation(b, slot); 2296c8c42864SChris Mason blocksize = btrfs_level_size(root, level - 1); 2297c8c42864SChris Mason 2298c8c42864SChris Mason tmp = btrfs_find_tree_block(root, blocknr, blocksize); 2299cb44921aSChris Mason if (tmp) { 2300b9fab919SChris Mason /* first we do an atomic uptodate check */ 2301b9fab919SChris Mason if (btrfs_buffer_uptodate(tmp, gen, 1) > 0) { 2302c8c42864SChris Mason *eb_ret = tmp; 2303c8c42864SChris Mason return 0; 2304c8c42864SChris Mason } 2305bdf7c00eSJosef Bacik 2306cb44921aSChris Mason /* the pages were up to date, but we failed 2307cb44921aSChris Mason * the generation number check. Do a full 2308cb44921aSChris Mason * read for the generation number that is correct. 2309cb44921aSChris Mason * We must do this without dropping locks so 2310cb44921aSChris Mason * we can trust our generation number 2311cb44921aSChris Mason */ 2312bd681513SChris Mason btrfs_set_path_blocking(p); 2313bd681513SChris Mason 2314b9fab919SChris Mason /* now we're allowed to do a blocking uptodate check */ 2315bdf7c00eSJosef Bacik ret = btrfs_read_buffer(tmp, gen); 2316bdf7c00eSJosef Bacik if (!ret) { 2317cb44921aSChris Mason *eb_ret = tmp; 2318cb44921aSChris Mason return 0; 2319cb44921aSChris Mason } 2320cb44921aSChris Mason free_extent_buffer(tmp); 2321b3b4aa74SDavid Sterba btrfs_release_path(p); 2322cb44921aSChris Mason return -EIO; 2323cb44921aSChris Mason } 2324c8c42864SChris Mason 2325c8c42864SChris Mason /* 2326c8c42864SChris Mason * reduce lock contention at high levels 2327c8c42864SChris Mason * of the btree by dropping locks before 232876a05b35SChris Mason * we read. Don't release the lock on the current 232976a05b35SChris Mason * level because we need to walk this node to figure 233076a05b35SChris Mason * out which blocks to read. 2331c8c42864SChris Mason */ 23328c594ea8SChris Mason btrfs_unlock_up_safe(p, level + 1); 23338c594ea8SChris Mason btrfs_set_path_blocking(p); 23348c594ea8SChris Mason 2335c8c42864SChris Mason free_extent_buffer(tmp); 2336c8c42864SChris Mason if (p->reada) 2337c8c42864SChris Mason reada_for_search(root, p, level, slot, key->objectid); 2338c8c42864SChris Mason 2339b3b4aa74SDavid Sterba btrfs_release_path(p); 234076a05b35SChris Mason 234176a05b35SChris Mason ret = -EAGAIN; 23425bdd3536SYan, Zheng tmp = read_tree_block(root, blocknr, blocksize, 0); 234376a05b35SChris Mason if (tmp) { 234476a05b35SChris Mason /* 234576a05b35SChris Mason * If the read above didn't mark this buffer up to date, 234676a05b35SChris Mason * it will never end up being up to date. Set ret to EIO now 234776a05b35SChris Mason * and give up so that our caller doesn't loop forever 234876a05b35SChris Mason * on our EAGAINs. 234976a05b35SChris Mason */ 2350b9fab919SChris Mason if (!btrfs_buffer_uptodate(tmp, 0, 0)) 235176a05b35SChris Mason ret = -EIO; 2352c8c42864SChris Mason free_extent_buffer(tmp); 235376a05b35SChris Mason } 235476a05b35SChris Mason return ret; 2355c8c42864SChris Mason } 2356c8c42864SChris Mason 2357c8c42864SChris Mason /* 2358c8c42864SChris Mason * helper function for btrfs_search_slot. This does all of the checks 2359c8c42864SChris Mason * for node-level blocks and does any balancing required based on 2360c8c42864SChris Mason * the ins_len. 2361c8c42864SChris Mason * 2362c8c42864SChris Mason * If no extra work was required, zero is returned. If we had to 2363c8c42864SChris Mason * drop the path, -EAGAIN is returned and btrfs_search_slot must 2364c8c42864SChris Mason * start over 2365c8c42864SChris Mason */ 2366c8c42864SChris Mason static int 2367c8c42864SChris Mason setup_nodes_for_search(struct btrfs_trans_handle *trans, 2368c8c42864SChris Mason struct btrfs_root *root, struct btrfs_path *p, 2369bd681513SChris Mason struct extent_buffer *b, int level, int ins_len, 2370bd681513SChris Mason int *write_lock_level) 2371c8c42864SChris Mason { 2372c8c42864SChris Mason int ret; 2373c8c42864SChris Mason if ((p->search_for_split || ins_len > 0) && btrfs_header_nritems(b) >= 2374c8c42864SChris Mason BTRFS_NODEPTRS_PER_BLOCK(root) - 3) { 2375c8c42864SChris Mason int sret; 2376c8c42864SChris Mason 2377bd681513SChris Mason if (*write_lock_level < level + 1) { 2378bd681513SChris Mason *write_lock_level = level + 1; 2379bd681513SChris Mason btrfs_release_path(p); 2380bd681513SChris Mason goto again; 2381bd681513SChris Mason } 2382bd681513SChris Mason 2383c8c42864SChris Mason btrfs_set_path_blocking(p); 23840b08851fSJosef Bacik reada_for_balance(root, p, level); 2385c8c42864SChris Mason sret = split_node(trans, root, p, level); 2386bd681513SChris Mason btrfs_clear_path_blocking(p, NULL, 0); 2387c8c42864SChris Mason 2388c8c42864SChris Mason BUG_ON(sret > 0); 2389c8c42864SChris Mason if (sret) { 2390c8c42864SChris Mason ret = sret; 2391c8c42864SChris Mason goto done; 2392c8c42864SChris Mason } 2393c8c42864SChris Mason b = p->nodes[level]; 2394c8c42864SChris Mason } else if (ins_len < 0 && btrfs_header_nritems(b) < 2395cfbb9308SChris Mason BTRFS_NODEPTRS_PER_BLOCK(root) / 2) { 2396c8c42864SChris Mason int sret; 2397c8c42864SChris Mason 2398bd681513SChris Mason if (*write_lock_level < level + 1) { 2399bd681513SChris Mason *write_lock_level = level + 1; 2400bd681513SChris Mason btrfs_release_path(p); 2401bd681513SChris Mason goto again; 2402bd681513SChris Mason } 2403bd681513SChris Mason 2404c8c42864SChris Mason btrfs_set_path_blocking(p); 24050b08851fSJosef Bacik reada_for_balance(root, p, level); 2406c8c42864SChris Mason sret = balance_level(trans, root, p, level); 2407bd681513SChris Mason btrfs_clear_path_blocking(p, NULL, 0); 2408c8c42864SChris Mason 2409c8c42864SChris Mason if (sret) { 2410c8c42864SChris Mason ret = sret; 2411c8c42864SChris Mason goto done; 2412c8c42864SChris Mason } 2413c8c42864SChris Mason b = p->nodes[level]; 2414c8c42864SChris Mason if (!b) { 2415b3b4aa74SDavid Sterba btrfs_release_path(p); 2416c8c42864SChris Mason goto again; 2417c8c42864SChris Mason } 2418c8c42864SChris Mason BUG_ON(btrfs_header_nritems(b) == 1); 2419c8c42864SChris Mason } 2420c8c42864SChris Mason return 0; 2421c8c42864SChris Mason 2422c8c42864SChris Mason again: 2423c8c42864SChris Mason ret = -EAGAIN; 2424c8c42864SChris Mason done: 2425c8c42864SChris Mason return ret; 2426c8c42864SChris Mason } 2427c8c42864SChris Mason 2428c8c42864SChris Mason /* 242974123bd7SChris Mason * look for key in the tree. path is filled in with nodes along the way 243074123bd7SChris Mason * if key is found, we return zero and you can find the item in the leaf 243174123bd7SChris Mason * level of the path (level 0) 243274123bd7SChris Mason * 243374123bd7SChris Mason * If the key isn't found, the path points to the slot where it should 2434aa5d6bedSChris Mason * be inserted, and 1 is returned. If there are other errors during the 2435aa5d6bedSChris Mason * search a negative error number is returned. 243697571fd0SChris Mason * 243797571fd0SChris Mason * if ins_len > 0, nodes and leaves will be split as we walk down the 243897571fd0SChris Mason * tree. if ins_len < 0, nodes will be merged as we walk down the tree (if 243997571fd0SChris Mason * possible) 244074123bd7SChris Mason */ 2441e089f05cSChris Mason int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root 2442e089f05cSChris Mason *root, struct btrfs_key *key, struct btrfs_path *p, int 2443e089f05cSChris Mason ins_len, int cow) 2444be0e5c09SChris Mason { 24455f39d397SChris Mason struct extent_buffer *b; 2446be0e5c09SChris Mason int slot; 2447be0e5c09SChris Mason int ret; 244833c66f43SYan Zheng int err; 2449be0e5c09SChris Mason int level; 2450925baeddSChris Mason int lowest_unlock = 1; 2451bd681513SChris Mason int root_lock; 2452bd681513SChris Mason /* everything at write_lock_level or lower must be write locked */ 2453bd681513SChris Mason int write_lock_level = 0; 24549f3a7427SChris Mason u8 lowest_level = 0; 2455f7c79f30SChris Mason int min_write_lock_level; 24569f3a7427SChris Mason 24576702ed49SChris Mason lowest_level = p->lowest_level; 2458323ac95bSChris Mason WARN_ON(lowest_level && ins_len > 0); 245922b0ebdaSChris Mason WARN_ON(p->nodes[0] != NULL); 246025179201SJosef Bacik 2461bd681513SChris Mason if (ins_len < 0) { 2462925baeddSChris Mason lowest_unlock = 2; 246365b51a00SChris Mason 2464bd681513SChris Mason /* when we are removing items, we might have to go up to level 2465bd681513SChris Mason * two as we update tree pointers Make sure we keep write 2466bd681513SChris Mason * for those levels as well 2467bd681513SChris Mason */ 2468bd681513SChris Mason write_lock_level = 2; 2469bd681513SChris Mason } else if (ins_len > 0) { 2470bd681513SChris Mason /* 2471bd681513SChris Mason * for inserting items, make sure we have a write lock on 2472bd681513SChris Mason * level 1 so we can update keys 2473bd681513SChris Mason */ 2474bd681513SChris Mason write_lock_level = 1; 2475bd681513SChris Mason } 2476bd681513SChris Mason 2477bd681513SChris Mason if (!cow) 2478bd681513SChris Mason write_lock_level = -1; 2479bd681513SChris Mason 248009a2a8f9SJosef Bacik if (cow && (p->keep_locks || p->lowest_level)) 2481bd681513SChris Mason write_lock_level = BTRFS_MAX_LEVEL; 2482bd681513SChris Mason 2483f7c79f30SChris Mason min_write_lock_level = write_lock_level; 2484f7c79f30SChris Mason 2485bb803951SChris Mason again: 2486bd681513SChris Mason /* 2487bd681513SChris Mason * we try very hard to do read locks on the root 2488bd681513SChris Mason */ 2489bd681513SChris Mason root_lock = BTRFS_READ_LOCK; 2490bd681513SChris Mason level = 0; 24915d4f98a2SYan Zheng if (p->search_commit_root) { 2492bd681513SChris Mason /* 2493bd681513SChris Mason * the commit roots are read only 2494bd681513SChris Mason * so we always do read locks 2495bd681513SChris Mason */ 24965d4f98a2SYan Zheng b = root->commit_root; 24975d4f98a2SYan Zheng extent_buffer_get(b); 2498bd681513SChris Mason level = btrfs_header_level(b); 24995d4f98a2SYan Zheng if (!p->skip_locking) 2500bd681513SChris Mason btrfs_tree_read_lock(b); 25015d4f98a2SYan Zheng } else { 2502bd681513SChris Mason if (p->skip_locking) { 25035cd57b2cSChris Mason b = btrfs_root_node(root); 2504bd681513SChris Mason level = btrfs_header_level(b); 2505bd681513SChris Mason } else { 2506bd681513SChris Mason /* we don't know the level of the root node 2507bd681513SChris Mason * until we actually have it read locked 2508bd681513SChris Mason */ 2509bd681513SChris Mason b = btrfs_read_lock_root_node(root); 2510bd681513SChris Mason level = btrfs_header_level(b); 2511bd681513SChris Mason if (level <= write_lock_level) { 2512bd681513SChris Mason /* whoops, must trade for write lock */ 2513bd681513SChris Mason btrfs_tree_read_unlock(b); 2514bd681513SChris Mason free_extent_buffer(b); 2515925baeddSChris Mason b = btrfs_lock_root_node(root); 2516bd681513SChris Mason root_lock = BTRFS_WRITE_LOCK; 2517bd681513SChris Mason 2518bd681513SChris Mason /* the level might have changed, check again */ 2519bd681513SChris Mason level = btrfs_header_level(b); 25205d4f98a2SYan Zheng } 2521bd681513SChris Mason } 2522bd681513SChris Mason } 2523bd681513SChris Mason p->nodes[level] = b; 2524bd681513SChris Mason if (!p->skip_locking) 2525bd681513SChris Mason p->locks[level] = root_lock; 2526925baeddSChris Mason 2527eb60ceacSChris Mason while (b) { 25285f39d397SChris Mason level = btrfs_header_level(b); 252965b51a00SChris Mason 253065b51a00SChris Mason /* 253165b51a00SChris Mason * setup the path here so we can release it under lock 253265b51a00SChris Mason * contention with the cow code 253365b51a00SChris Mason */ 253402217ed2SChris Mason if (cow) { 2535c8c42864SChris Mason /* 2536c8c42864SChris Mason * if we don't really need to cow this block 2537c8c42864SChris Mason * then we don't want to set the path blocking, 2538c8c42864SChris Mason * so we test it here 2539c8c42864SChris Mason */ 25405d4f98a2SYan Zheng if (!should_cow_block(trans, root, b)) 254165b51a00SChris Mason goto cow_done; 25425d4f98a2SYan Zheng 2543b4ce94deSChris Mason btrfs_set_path_blocking(p); 2544b4ce94deSChris Mason 2545bd681513SChris Mason /* 2546bd681513SChris Mason * must have write locks on this node and the 2547bd681513SChris Mason * parent 2548bd681513SChris Mason */ 25495124e00eSJosef Bacik if (level > write_lock_level || 25505124e00eSJosef Bacik (level + 1 > write_lock_level && 25515124e00eSJosef Bacik level + 1 < BTRFS_MAX_LEVEL && 25525124e00eSJosef Bacik p->nodes[level + 1])) { 2553bd681513SChris Mason write_lock_level = level + 1; 2554bd681513SChris Mason btrfs_release_path(p); 2555bd681513SChris Mason goto again; 2556bd681513SChris Mason } 2557bd681513SChris Mason 255833c66f43SYan Zheng err = btrfs_cow_block(trans, root, b, 2559e20d96d6SChris Mason p->nodes[level + 1], 25609fa8cfe7SChris Mason p->slots[level + 1], &b); 256133c66f43SYan Zheng if (err) { 256233c66f43SYan Zheng ret = err; 256365b51a00SChris Mason goto done; 256454aa1f4dSChris Mason } 256502217ed2SChris Mason } 256665b51a00SChris Mason cow_done: 256702217ed2SChris Mason BUG_ON(!cow && ins_len); 256865b51a00SChris Mason 2569eb60ceacSChris Mason p->nodes[level] = b; 2570bd681513SChris Mason btrfs_clear_path_blocking(p, NULL, 0); 2571b4ce94deSChris Mason 2572b4ce94deSChris Mason /* 2573b4ce94deSChris Mason * we have a lock on b and as long as we aren't changing 2574b4ce94deSChris Mason * the tree, there is no way to for the items in b to change. 2575b4ce94deSChris Mason * It is safe to drop the lock on our parent before we 2576b4ce94deSChris Mason * go through the expensive btree search on b. 2577b4ce94deSChris Mason * 2578b4ce94deSChris Mason * If cow is true, then we might be changing slot zero, 2579b4ce94deSChris Mason * which may require changing the parent. So, we can't 2580b4ce94deSChris Mason * drop the lock until after we know which slot we're 2581b4ce94deSChris Mason * operating on. 2582b4ce94deSChris Mason */ 2583b4ce94deSChris Mason if (!cow) 2584b4ce94deSChris Mason btrfs_unlock_up_safe(p, level + 1); 2585b4ce94deSChris Mason 25865f39d397SChris Mason ret = bin_search(b, key, level, &slot); 2587b4ce94deSChris Mason 25885f39d397SChris Mason if (level != 0) { 258933c66f43SYan Zheng int dec = 0; 259033c66f43SYan Zheng if (ret && slot > 0) { 259133c66f43SYan Zheng dec = 1; 2592be0e5c09SChris Mason slot -= 1; 259333c66f43SYan Zheng } 2594be0e5c09SChris Mason p->slots[level] = slot; 259533c66f43SYan Zheng err = setup_nodes_for_search(trans, root, p, b, level, 2596bd681513SChris Mason ins_len, &write_lock_level); 259733c66f43SYan Zheng if (err == -EAGAIN) 2598b4ce94deSChris Mason goto again; 259933c66f43SYan Zheng if (err) { 260033c66f43SYan Zheng ret = err; 260165b51a00SChris Mason goto done; 260233c66f43SYan Zheng } 26035c680ed6SChris Mason b = p->nodes[level]; 26045c680ed6SChris Mason slot = p->slots[level]; 2605b4ce94deSChris Mason 2606bd681513SChris Mason /* 2607bd681513SChris Mason * slot 0 is special, if we change the key 2608bd681513SChris Mason * we have to update the parent pointer 2609bd681513SChris Mason * which means we must have a write lock 2610bd681513SChris Mason * on the parent 2611bd681513SChris Mason */ 2612bd681513SChris Mason if (slot == 0 && cow && 2613bd681513SChris Mason write_lock_level < level + 1) { 2614bd681513SChris Mason write_lock_level = level + 1; 2615bd681513SChris Mason btrfs_release_path(p); 2616bd681513SChris Mason goto again; 2617bd681513SChris Mason } 2618bd681513SChris Mason 2619f7c79f30SChris Mason unlock_up(p, level, lowest_unlock, 2620f7c79f30SChris Mason min_write_lock_level, &write_lock_level); 2621f9efa9c7SChris Mason 2622925baeddSChris Mason if (level == lowest_level) { 262333c66f43SYan Zheng if (dec) 262433c66f43SYan Zheng p->slots[level]++; 26255b21f2edSZheng Yan goto done; 2626925baeddSChris Mason } 2627ca7a79adSChris Mason 262833c66f43SYan Zheng err = read_block_for_search(trans, root, p, 26295d9e75c4SJan Schmidt &b, level, slot, key, 0); 263033c66f43SYan Zheng if (err == -EAGAIN) 2631051e1b9fSChris Mason goto again; 263233c66f43SYan Zheng if (err) { 263333c66f43SYan Zheng ret = err; 263476a05b35SChris Mason goto done; 263533c66f43SYan Zheng } 263676a05b35SChris Mason 2637b4ce94deSChris Mason if (!p->skip_locking) { 2638bd681513SChris Mason level = btrfs_header_level(b); 2639bd681513SChris Mason if (level <= write_lock_level) { 2640bd681513SChris Mason err = btrfs_try_tree_write_lock(b); 264133c66f43SYan Zheng if (!err) { 2642b4ce94deSChris Mason btrfs_set_path_blocking(p); 2643925baeddSChris Mason btrfs_tree_lock(b); 2644bd681513SChris Mason btrfs_clear_path_blocking(p, b, 2645bd681513SChris Mason BTRFS_WRITE_LOCK); 2646b4ce94deSChris Mason } 2647bd681513SChris Mason p->locks[level] = BTRFS_WRITE_LOCK; 2648bd681513SChris Mason } else { 2649bd681513SChris Mason err = btrfs_try_tree_read_lock(b); 2650bd681513SChris Mason if (!err) { 2651bd681513SChris Mason btrfs_set_path_blocking(p); 2652bd681513SChris Mason btrfs_tree_read_lock(b); 2653bd681513SChris Mason btrfs_clear_path_blocking(p, b, 2654bd681513SChris Mason BTRFS_READ_LOCK); 2655bd681513SChris Mason } 2656bd681513SChris Mason p->locks[level] = BTRFS_READ_LOCK; 2657bd681513SChris Mason } 2658bd681513SChris Mason p->nodes[level] = b; 2659b4ce94deSChris Mason } 2660be0e5c09SChris Mason } else { 2661be0e5c09SChris Mason p->slots[level] = slot; 266287b29b20SYan Zheng if (ins_len > 0 && 266387b29b20SYan Zheng btrfs_leaf_free_space(root, b) < ins_len) { 2664bd681513SChris Mason if (write_lock_level < 1) { 2665bd681513SChris Mason write_lock_level = 1; 2666bd681513SChris Mason btrfs_release_path(p); 2667bd681513SChris Mason goto again; 2668bd681513SChris Mason } 2669bd681513SChris Mason 2670b4ce94deSChris Mason btrfs_set_path_blocking(p); 267133c66f43SYan Zheng err = split_leaf(trans, root, key, 2672cc0c5538SChris Mason p, ins_len, ret == 0); 2673bd681513SChris Mason btrfs_clear_path_blocking(p, NULL, 0); 2674b4ce94deSChris Mason 267533c66f43SYan Zheng BUG_ON(err > 0); 267633c66f43SYan Zheng if (err) { 267733c66f43SYan Zheng ret = err; 267865b51a00SChris Mason goto done; 267965b51a00SChris Mason } 26805c680ed6SChris Mason } 2681459931ecSChris Mason if (!p->search_for_split) 2682f7c79f30SChris Mason unlock_up(p, level, lowest_unlock, 2683f7c79f30SChris Mason min_write_lock_level, &write_lock_level); 268465b51a00SChris Mason goto done; 268565b51a00SChris Mason } 268665b51a00SChris Mason } 268765b51a00SChris Mason ret = 1; 268865b51a00SChris Mason done: 2689b4ce94deSChris Mason /* 2690b4ce94deSChris Mason * we don't really know what they plan on doing with the path 2691b4ce94deSChris Mason * from here on, so for now just mark it as blocking 2692b4ce94deSChris Mason */ 2693b9473439SChris Mason if (!p->leave_spinning) 2694b4ce94deSChris Mason btrfs_set_path_blocking(p); 269576a05b35SChris Mason if (ret < 0) 2696b3b4aa74SDavid Sterba btrfs_release_path(p); 2697be0e5c09SChris Mason return ret; 2698be0e5c09SChris Mason } 2699be0e5c09SChris Mason 270074123bd7SChris Mason /* 27015d9e75c4SJan Schmidt * Like btrfs_search_slot, this looks for a key in the given tree. It uses the 27025d9e75c4SJan Schmidt * current state of the tree together with the operations recorded in the tree 27035d9e75c4SJan Schmidt * modification log to search for the key in a previous version of this tree, as 27045d9e75c4SJan Schmidt * denoted by the time_seq parameter. 27055d9e75c4SJan Schmidt * 27065d9e75c4SJan Schmidt * Naturally, there is no support for insert, delete or cow operations. 27075d9e75c4SJan Schmidt * 27085d9e75c4SJan Schmidt * The resulting path and return value will be set up as if we called 27095d9e75c4SJan Schmidt * btrfs_search_slot at that point in time with ins_len and cow both set to 0. 27105d9e75c4SJan Schmidt */ 27115d9e75c4SJan Schmidt int btrfs_search_old_slot(struct btrfs_root *root, struct btrfs_key *key, 27125d9e75c4SJan Schmidt struct btrfs_path *p, u64 time_seq) 27135d9e75c4SJan Schmidt { 27145d9e75c4SJan Schmidt struct extent_buffer *b; 27155d9e75c4SJan Schmidt int slot; 27165d9e75c4SJan Schmidt int ret; 27175d9e75c4SJan Schmidt int err; 27185d9e75c4SJan Schmidt int level; 27195d9e75c4SJan Schmidt int lowest_unlock = 1; 27205d9e75c4SJan Schmidt u8 lowest_level = 0; 27215d9e75c4SJan Schmidt 27225d9e75c4SJan Schmidt lowest_level = p->lowest_level; 27235d9e75c4SJan Schmidt WARN_ON(p->nodes[0] != NULL); 27245d9e75c4SJan Schmidt 27255d9e75c4SJan Schmidt if (p->search_commit_root) { 27265d9e75c4SJan Schmidt BUG_ON(time_seq); 27275d9e75c4SJan Schmidt return btrfs_search_slot(NULL, root, key, p, 0, 0); 27285d9e75c4SJan Schmidt } 27295d9e75c4SJan Schmidt 27305d9e75c4SJan Schmidt again: 27315d9e75c4SJan Schmidt b = get_old_root(root, time_seq); 27325d9e75c4SJan Schmidt level = btrfs_header_level(b); 27335d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 27345d9e75c4SJan Schmidt 27355d9e75c4SJan Schmidt while (b) { 27365d9e75c4SJan Schmidt level = btrfs_header_level(b); 27375d9e75c4SJan Schmidt p->nodes[level] = b; 27385d9e75c4SJan Schmidt btrfs_clear_path_blocking(p, NULL, 0); 27395d9e75c4SJan Schmidt 27405d9e75c4SJan Schmidt /* 27415d9e75c4SJan Schmidt * we have a lock on b and as long as we aren't changing 27425d9e75c4SJan Schmidt * the tree, there is no way to for the items in b to change. 27435d9e75c4SJan Schmidt * It is safe to drop the lock on our parent before we 27445d9e75c4SJan Schmidt * go through the expensive btree search on b. 27455d9e75c4SJan Schmidt */ 27465d9e75c4SJan Schmidt btrfs_unlock_up_safe(p, level + 1); 27475d9e75c4SJan Schmidt 27485d9e75c4SJan Schmidt ret = bin_search(b, key, level, &slot); 27495d9e75c4SJan Schmidt 27505d9e75c4SJan Schmidt if (level != 0) { 27515d9e75c4SJan Schmidt int dec = 0; 27525d9e75c4SJan Schmidt if (ret && slot > 0) { 27535d9e75c4SJan Schmidt dec = 1; 27545d9e75c4SJan Schmidt slot -= 1; 27555d9e75c4SJan Schmidt } 27565d9e75c4SJan Schmidt p->slots[level] = slot; 27575d9e75c4SJan Schmidt unlock_up(p, level, lowest_unlock, 0, NULL); 27585d9e75c4SJan Schmidt 27595d9e75c4SJan Schmidt if (level == lowest_level) { 27605d9e75c4SJan Schmidt if (dec) 27615d9e75c4SJan Schmidt p->slots[level]++; 27625d9e75c4SJan Schmidt goto done; 27635d9e75c4SJan Schmidt } 27645d9e75c4SJan Schmidt 27655d9e75c4SJan Schmidt err = read_block_for_search(NULL, root, p, &b, level, 27665d9e75c4SJan Schmidt slot, key, time_seq); 27675d9e75c4SJan Schmidt if (err == -EAGAIN) 27685d9e75c4SJan Schmidt goto again; 27695d9e75c4SJan Schmidt if (err) { 27705d9e75c4SJan Schmidt ret = err; 27715d9e75c4SJan Schmidt goto done; 27725d9e75c4SJan Schmidt } 27735d9e75c4SJan Schmidt 27745d9e75c4SJan Schmidt level = btrfs_header_level(b); 27755d9e75c4SJan Schmidt err = btrfs_try_tree_read_lock(b); 27765d9e75c4SJan Schmidt if (!err) { 27775d9e75c4SJan Schmidt btrfs_set_path_blocking(p); 27785d9e75c4SJan Schmidt btrfs_tree_read_lock(b); 27795d9e75c4SJan Schmidt btrfs_clear_path_blocking(p, b, 27805d9e75c4SJan Schmidt BTRFS_READ_LOCK); 27815d9e75c4SJan Schmidt } 278247fb091fSJan Schmidt b = tree_mod_log_rewind(root->fs_info, b, time_seq); 2783db7f3436SJosef Bacik if (!b) { 2784db7f3436SJosef Bacik ret = -ENOMEM; 2785db7f3436SJosef Bacik goto done; 2786db7f3436SJosef Bacik } 27875d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 27885d9e75c4SJan Schmidt p->nodes[level] = b; 27895d9e75c4SJan Schmidt } else { 27905d9e75c4SJan Schmidt p->slots[level] = slot; 27915d9e75c4SJan Schmidt unlock_up(p, level, lowest_unlock, 0, NULL); 27925d9e75c4SJan Schmidt goto done; 27935d9e75c4SJan Schmidt } 27945d9e75c4SJan Schmidt } 27955d9e75c4SJan Schmidt ret = 1; 27965d9e75c4SJan Schmidt done: 27975d9e75c4SJan Schmidt if (!p->leave_spinning) 27985d9e75c4SJan Schmidt btrfs_set_path_blocking(p); 27995d9e75c4SJan Schmidt if (ret < 0) 28005d9e75c4SJan Schmidt btrfs_release_path(p); 28015d9e75c4SJan Schmidt 28025d9e75c4SJan Schmidt return ret; 28035d9e75c4SJan Schmidt } 28045d9e75c4SJan Schmidt 28055d9e75c4SJan Schmidt /* 28062f38b3e1SArne Jansen * helper to use instead of search slot if no exact match is needed but 28072f38b3e1SArne Jansen * instead the next or previous item should be returned. 28082f38b3e1SArne Jansen * When find_higher is true, the next higher item is returned, the next lower 28092f38b3e1SArne Jansen * otherwise. 28102f38b3e1SArne Jansen * When return_any and find_higher are both true, and no higher item is found, 28112f38b3e1SArne Jansen * return the next lower instead. 28122f38b3e1SArne Jansen * When return_any is true and find_higher is false, and no lower item is found, 28132f38b3e1SArne Jansen * return the next higher instead. 28142f38b3e1SArne Jansen * It returns 0 if any item is found, 1 if none is found (tree empty), and 28152f38b3e1SArne Jansen * < 0 on error 28162f38b3e1SArne Jansen */ 28172f38b3e1SArne Jansen int btrfs_search_slot_for_read(struct btrfs_root *root, 28182f38b3e1SArne Jansen struct btrfs_key *key, struct btrfs_path *p, 28192f38b3e1SArne Jansen int find_higher, int return_any) 28202f38b3e1SArne Jansen { 28212f38b3e1SArne Jansen int ret; 28222f38b3e1SArne Jansen struct extent_buffer *leaf; 28232f38b3e1SArne Jansen 28242f38b3e1SArne Jansen again: 28252f38b3e1SArne Jansen ret = btrfs_search_slot(NULL, root, key, p, 0, 0); 28262f38b3e1SArne Jansen if (ret <= 0) 28272f38b3e1SArne Jansen return ret; 28282f38b3e1SArne Jansen /* 28292f38b3e1SArne Jansen * a return value of 1 means the path is at the position where the 28302f38b3e1SArne Jansen * item should be inserted. Normally this is the next bigger item, 28312f38b3e1SArne Jansen * but in case the previous item is the last in a leaf, path points 28322f38b3e1SArne Jansen * to the first free slot in the previous leaf, i.e. at an invalid 28332f38b3e1SArne Jansen * item. 28342f38b3e1SArne Jansen */ 28352f38b3e1SArne Jansen leaf = p->nodes[0]; 28362f38b3e1SArne Jansen 28372f38b3e1SArne Jansen if (find_higher) { 28382f38b3e1SArne Jansen if (p->slots[0] >= btrfs_header_nritems(leaf)) { 28392f38b3e1SArne Jansen ret = btrfs_next_leaf(root, p); 28402f38b3e1SArne Jansen if (ret <= 0) 28412f38b3e1SArne Jansen return ret; 28422f38b3e1SArne Jansen if (!return_any) 28432f38b3e1SArne Jansen return 1; 28442f38b3e1SArne Jansen /* 28452f38b3e1SArne Jansen * no higher item found, return the next 28462f38b3e1SArne Jansen * lower instead 28472f38b3e1SArne Jansen */ 28482f38b3e1SArne Jansen return_any = 0; 28492f38b3e1SArne Jansen find_higher = 0; 28502f38b3e1SArne Jansen btrfs_release_path(p); 28512f38b3e1SArne Jansen goto again; 28522f38b3e1SArne Jansen } 28532f38b3e1SArne Jansen } else { 28542f38b3e1SArne Jansen if (p->slots[0] == 0) { 28552f38b3e1SArne Jansen ret = btrfs_prev_leaf(root, p); 2856e6793769SArne Jansen if (ret < 0) 28572f38b3e1SArne Jansen return ret; 2858e6793769SArne Jansen if (!ret) { 2859e6793769SArne Jansen p->slots[0] = btrfs_header_nritems(leaf) - 1; 2860e6793769SArne Jansen return 0; 2861e6793769SArne Jansen } 28622f38b3e1SArne Jansen if (!return_any) 28632f38b3e1SArne Jansen return 1; 28642f38b3e1SArne Jansen /* 28652f38b3e1SArne Jansen * no lower item found, return the next 28662f38b3e1SArne Jansen * higher instead 28672f38b3e1SArne Jansen */ 28682f38b3e1SArne Jansen return_any = 0; 28692f38b3e1SArne Jansen find_higher = 1; 28702f38b3e1SArne Jansen btrfs_release_path(p); 28712f38b3e1SArne Jansen goto again; 2872e6793769SArne Jansen } else { 28732f38b3e1SArne Jansen --p->slots[0]; 28742f38b3e1SArne Jansen } 28752f38b3e1SArne Jansen } 28762f38b3e1SArne Jansen return 0; 28772f38b3e1SArne Jansen } 28782f38b3e1SArne Jansen 28792f38b3e1SArne Jansen /* 288074123bd7SChris Mason * adjust the pointers going up the tree, starting at level 288174123bd7SChris Mason * making sure the right key of each node is points to 'key'. 288274123bd7SChris Mason * This is used after shifting pointers to the left, so it stops 288374123bd7SChris Mason * fixing up pointers when a given leaf/node is not in slot 0 of the 288474123bd7SChris Mason * higher levels 2885aa5d6bedSChris Mason * 288674123bd7SChris Mason */ 2887d6a0a126STsutomu Itoh static void fixup_low_keys(struct btrfs_root *root, struct btrfs_path *path, 28885f39d397SChris Mason struct btrfs_disk_key *key, int level) 2889be0e5c09SChris Mason { 2890be0e5c09SChris Mason int i; 28915f39d397SChris Mason struct extent_buffer *t; 28925f39d397SChris Mason 2893234b63a0SChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 2894be0e5c09SChris Mason int tslot = path->slots[i]; 2895eb60ceacSChris Mason if (!path->nodes[i]) 2896be0e5c09SChris Mason break; 28975f39d397SChris Mason t = path->nodes[i]; 289832adf090SLiu Bo tree_mod_log_set_node_key(root->fs_info, t, tslot, 1); 28995f39d397SChris Mason btrfs_set_node_key(t, key, tslot); 2900d6025579SChris Mason btrfs_mark_buffer_dirty(path->nodes[i]); 2901be0e5c09SChris Mason if (tslot != 0) 2902be0e5c09SChris Mason break; 2903be0e5c09SChris Mason } 2904be0e5c09SChris Mason } 2905be0e5c09SChris Mason 290674123bd7SChris Mason /* 290731840ae1SZheng Yan * update item key. 290831840ae1SZheng Yan * 290931840ae1SZheng Yan * This function isn't completely safe. It's the caller's responsibility 291031840ae1SZheng Yan * that the new key won't break the order 291131840ae1SZheng Yan */ 2912afe5fea7STsutomu Itoh void btrfs_set_item_key_safe(struct btrfs_root *root, struct btrfs_path *path, 291331840ae1SZheng Yan struct btrfs_key *new_key) 291431840ae1SZheng Yan { 291531840ae1SZheng Yan struct btrfs_disk_key disk_key; 291631840ae1SZheng Yan struct extent_buffer *eb; 291731840ae1SZheng Yan int slot; 291831840ae1SZheng Yan 291931840ae1SZheng Yan eb = path->nodes[0]; 292031840ae1SZheng Yan slot = path->slots[0]; 292131840ae1SZheng Yan if (slot > 0) { 292231840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot - 1); 2923143bede5SJeff Mahoney BUG_ON(comp_keys(&disk_key, new_key) >= 0); 292431840ae1SZheng Yan } 292531840ae1SZheng Yan if (slot < btrfs_header_nritems(eb) - 1) { 292631840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot + 1); 2927143bede5SJeff Mahoney BUG_ON(comp_keys(&disk_key, new_key) <= 0); 292831840ae1SZheng Yan } 292931840ae1SZheng Yan 293031840ae1SZheng Yan btrfs_cpu_key_to_disk(&disk_key, new_key); 293131840ae1SZheng Yan btrfs_set_item_key(eb, &disk_key, slot); 293231840ae1SZheng Yan btrfs_mark_buffer_dirty(eb); 293331840ae1SZheng Yan if (slot == 0) 2934d6a0a126STsutomu Itoh fixup_low_keys(root, path, &disk_key, 1); 293531840ae1SZheng Yan } 293631840ae1SZheng Yan 293731840ae1SZheng Yan /* 293874123bd7SChris Mason * try to push data from one node into the next node left in the 293979f95c82SChris Mason * tree. 2940aa5d6bedSChris Mason * 2941aa5d6bedSChris Mason * returns 0 if some ptrs were pushed left, < 0 if there was some horrible 2942aa5d6bedSChris Mason * error, and > 0 if there was no room in the left hand block. 294374123bd7SChris Mason */ 294498ed5174SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 294598ed5174SChris Mason struct btrfs_root *root, struct extent_buffer *dst, 2946971a1f66SChris Mason struct extent_buffer *src, int empty) 2947be0e5c09SChris Mason { 2948be0e5c09SChris Mason int push_items = 0; 2949bb803951SChris Mason int src_nritems; 2950bb803951SChris Mason int dst_nritems; 2951aa5d6bedSChris Mason int ret = 0; 2952be0e5c09SChris Mason 29535f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 29545f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 2955123abc88SChris Mason push_items = BTRFS_NODEPTRS_PER_BLOCK(root) - dst_nritems; 29567bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 29577bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 295854aa1f4dSChris Mason 2959bce4eae9SChris Mason if (!empty && src_nritems <= 8) 2960971a1f66SChris Mason return 1; 2961971a1f66SChris Mason 2962d397712bSChris Mason if (push_items <= 0) 2963be0e5c09SChris Mason return 1; 2964be0e5c09SChris Mason 2965bce4eae9SChris Mason if (empty) { 2966971a1f66SChris Mason push_items = min(src_nritems, push_items); 2967bce4eae9SChris Mason if (push_items < src_nritems) { 2968bce4eae9SChris Mason /* leave at least 8 pointers in the node if 2969bce4eae9SChris Mason * we aren't going to empty it 2970bce4eae9SChris Mason */ 2971bce4eae9SChris Mason if (src_nritems - push_items < 8) { 2972bce4eae9SChris Mason if (push_items <= 8) 2973bce4eae9SChris Mason return 1; 2974bce4eae9SChris Mason push_items -= 8; 2975bce4eae9SChris Mason } 2976bce4eae9SChris Mason } 2977bce4eae9SChris Mason } else 2978bce4eae9SChris Mason push_items = min(src_nritems - 8, push_items); 297979f95c82SChris Mason 2980f230475eSJan Schmidt tree_mod_log_eb_copy(root->fs_info, dst, src, dst_nritems, 0, 298190f8d62eSJan Schmidt push_items); 29825f39d397SChris Mason copy_extent_buffer(dst, src, 29835f39d397SChris Mason btrfs_node_key_ptr_offset(dst_nritems), 29845f39d397SChris Mason btrfs_node_key_ptr_offset(0), 2985123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 29865f39d397SChris Mason 2987bb803951SChris Mason if (push_items < src_nritems) { 298857911b8bSJan Schmidt /* 298957911b8bSJan Schmidt * don't call tree_mod_log_eb_move here, key removal was already 299057911b8bSJan Schmidt * fully logged by tree_mod_log_eb_copy above. 299157911b8bSJan Schmidt */ 29925f39d397SChris Mason memmove_extent_buffer(src, btrfs_node_key_ptr_offset(0), 29935f39d397SChris Mason btrfs_node_key_ptr_offset(push_items), 2994e2fa7227SChris Mason (src_nritems - push_items) * 2995123abc88SChris Mason sizeof(struct btrfs_key_ptr)); 2996bb803951SChris Mason } 29975f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 29985f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 29995f39d397SChris Mason btrfs_mark_buffer_dirty(src); 30005f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 300131840ae1SZheng Yan 3002bb803951SChris Mason return ret; 3003be0e5c09SChris Mason } 3004be0e5c09SChris Mason 300597571fd0SChris Mason /* 300679f95c82SChris Mason * try to push data from one node into the next node right in the 300779f95c82SChris Mason * tree. 300879f95c82SChris Mason * 300979f95c82SChris Mason * returns 0 if some ptrs were pushed, < 0 if there was some horrible 301079f95c82SChris Mason * error, and > 0 if there was no room in the right hand block. 301179f95c82SChris Mason * 301279f95c82SChris Mason * this will only push up to 1/2 the contents of the left node over 301379f95c82SChris Mason */ 30145f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 30155f39d397SChris Mason struct btrfs_root *root, 30165f39d397SChris Mason struct extent_buffer *dst, 30175f39d397SChris Mason struct extent_buffer *src) 301879f95c82SChris Mason { 301979f95c82SChris Mason int push_items = 0; 302079f95c82SChris Mason int max_push; 302179f95c82SChris Mason int src_nritems; 302279f95c82SChris Mason int dst_nritems; 302379f95c82SChris Mason int ret = 0; 302479f95c82SChris Mason 30257bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 30267bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 30277bb86316SChris Mason 30285f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 30295f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 3030123abc88SChris Mason push_items = BTRFS_NODEPTRS_PER_BLOCK(root) - dst_nritems; 3031d397712bSChris Mason if (push_items <= 0) 303279f95c82SChris Mason return 1; 3033bce4eae9SChris Mason 3034d397712bSChris Mason if (src_nritems < 4) 3035bce4eae9SChris Mason return 1; 303679f95c82SChris Mason 303779f95c82SChris Mason max_push = src_nritems / 2 + 1; 303879f95c82SChris Mason /* don't try to empty the node */ 3039d397712bSChris Mason if (max_push >= src_nritems) 304079f95c82SChris Mason return 1; 3041252c38f0SYan 304279f95c82SChris Mason if (max_push < push_items) 304379f95c82SChris Mason push_items = max_push; 304479f95c82SChris Mason 3045f230475eSJan Schmidt tree_mod_log_eb_move(root->fs_info, dst, push_items, 0, dst_nritems); 30465f39d397SChris Mason memmove_extent_buffer(dst, btrfs_node_key_ptr_offset(push_items), 30475f39d397SChris Mason btrfs_node_key_ptr_offset(0), 30485f39d397SChris Mason (dst_nritems) * 30495f39d397SChris Mason sizeof(struct btrfs_key_ptr)); 3050d6025579SChris Mason 3051f230475eSJan Schmidt tree_mod_log_eb_copy(root->fs_info, dst, src, 0, 305290f8d62eSJan Schmidt src_nritems - push_items, push_items); 30535f39d397SChris Mason copy_extent_buffer(dst, src, 30545f39d397SChris Mason btrfs_node_key_ptr_offset(0), 30555f39d397SChris Mason btrfs_node_key_ptr_offset(src_nritems - push_items), 3056123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 305779f95c82SChris Mason 30585f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 30595f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 306079f95c82SChris Mason 30615f39d397SChris Mason btrfs_mark_buffer_dirty(src); 30625f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 306331840ae1SZheng Yan 306479f95c82SChris Mason return ret; 306579f95c82SChris Mason } 306679f95c82SChris Mason 306779f95c82SChris Mason /* 306897571fd0SChris Mason * helper function to insert a new root level in the tree. 306997571fd0SChris Mason * A new node is allocated, and a single item is inserted to 307097571fd0SChris Mason * point to the existing root 3071aa5d6bedSChris Mason * 3072aa5d6bedSChris Mason * returns zero on success or < 0 on failure. 307397571fd0SChris Mason */ 3074d397712bSChris Mason static noinline int insert_new_root(struct btrfs_trans_handle *trans, 30755f39d397SChris Mason struct btrfs_root *root, 3076fdd99c72SLiu Bo struct btrfs_path *path, int level) 307774123bd7SChris Mason { 30787bb86316SChris Mason u64 lower_gen; 30795f39d397SChris Mason struct extent_buffer *lower; 30805f39d397SChris Mason struct extent_buffer *c; 3081925baeddSChris Mason struct extent_buffer *old; 30825f39d397SChris Mason struct btrfs_disk_key lower_key; 30835c680ed6SChris Mason 30845c680ed6SChris Mason BUG_ON(path->nodes[level]); 30855c680ed6SChris Mason BUG_ON(path->nodes[level-1] != root->node); 30865c680ed6SChris Mason 30877bb86316SChris Mason lower = path->nodes[level-1]; 30887bb86316SChris Mason if (level == 1) 30897bb86316SChris Mason btrfs_item_key(lower, &lower_key, 0); 30907bb86316SChris Mason else 30917bb86316SChris Mason btrfs_node_key(lower, &lower_key, 0); 30927bb86316SChris Mason 309331840ae1SZheng Yan c = btrfs_alloc_free_block(trans, root, root->nodesize, 0, 30945d4f98a2SYan Zheng root->root_key.objectid, &lower_key, 30955581a51aSJan Schmidt level, root->node->start, 0); 30965f39d397SChris Mason if (IS_ERR(c)) 30975f39d397SChris Mason return PTR_ERR(c); 3098925baeddSChris Mason 3099f0486c68SYan, Zheng root_add_used(root, root->nodesize); 3100f0486c68SYan, Zheng 31015d4f98a2SYan Zheng memset_extent_buffer(c, 0, 0, sizeof(struct btrfs_header)); 31025f39d397SChris Mason btrfs_set_header_nritems(c, 1); 31035f39d397SChris Mason btrfs_set_header_level(c, level); 3104db94535dSChris Mason btrfs_set_header_bytenr(c, c->start); 31055f39d397SChris Mason btrfs_set_header_generation(c, trans->transid); 31065d4f98a2SYan Zheng btrfs_set_header_backref_rev(c, BTRFS_MIXED_BACKREF_REV); 31075f39d397SChris Mason btrfs_set_header_owner(c, root->root_key.objectid); 3108d5719762SChris Mason 31095f39d397SChris Mason write_extent_buffer(c, root->fs_info->fsid, 31105f39d397SChris Mason (unsigned long)btrfs_header_fsid(c), 31115f39d397SChris Mason BTRFS_FSID_SIZE); 3112e17cade2SChris Mason 3113e17cade2SChris Mason write_extent_buffer(c, root->fs_info->chunk_tree_uuid, 3114e17cade2SChris Mason (unsigned long)btrfs_header_chunk_tree_uuid(c), 3115e17cade2SChris Mason BTRFS_UUID_SIZE); 3116e17cade2SChris Mason 31175f39d397SChris Mason btrfs_set_node_key(c, &lower_key, 0); 3118db94535dSChris Mason btrfs_set_node_blockptr(c, 0, lower->start); 31197bb86316SChris Mason lower_gen = btrfs_header_generation(lower); 312031840ae1SZheng Yan WARN_ON(lower_gen != trans->transid); 31217bb86316SChris Mason 31227bb86316SChris Mason btrfs_set_node_ptr_generation(c, 0, lower_gen); 31235f39d397SChris Mason 31245f39d397SChris Mason btrfs_mark_buffer_dirty(c); 3125d5719762SChris Mason 3126925baeddSChris Mason old = root->node; 3127fdd99c72SLiu Bo tree_mod_log_set_root_pointer(root, c, 0); 3128240f62c8SChris Mason rcu_assign_pointer(root->node, c); 3129925baeddSChris Mason 3130925baeddSChris Mason /* the super has an extra ref to root->node */ 3131925baeddSChris Mason free_extent_buffer(old); 3132925baeddSChris Mason 31330b86a832SChris Mason add_root_to_dirty_list(root); 31345f39d397SChris Mason extent_buffer_get(c); 31355f39d397SChris Mason path->nodes[level] = c; 3136bd681513SChris Mason path->locks[level] = BTRFS_WRITE_LOCK; 313774123bd7SChris Mason path->slots[level] = 0; 313874123bd7SChris Mason return 0; 313974123bd7SChris Mason } 31405c680ed6SChris Mason 31415c680ed6SChris Mason /* 31425c680ed6SChris Mason * worker function to insert a single pointer in a node. 31435c680ed6SChris Mason * the node should have enough room for the pointer already 314497571fd0SChris Mason * 31455c680ed6SChris Mason * slot and level indicate where you want the key to go, and 31465c680ed6SChris Mason * blocknr is the block the key points to. 31475c680ed6SChris Mason */ 3148143bede5SJeff Mahoney static void insert_ptr(struct btrfs_trans_handle *trans, 3149143bede5SJeff Mahoney struct btrfs_root *root, struct btrfs_path *path, 3150143bede5SJeff Mahoney struct btrfs_disk_key *key, u64 bytenr, 3151c3e06965SJan Schmidt int slot, int level) 31525c680ed6SChris Mason { 31535f39d397SChris Mason struct extent_buffer *lower; 31545c680ed6SChris Mason int nritems; 3155f3ea38daSJan Schmidt int ret; 31565c680ed6SChris Mason 31575c680ed6SChris Mason BUG_ON(!path->nodes[level]); 3158f0486c68SYan, Zheng btrfs_assert_tree_locked(path->nodes[level]); 31595f39d397SChris Mason lower = path->nodes[level]; 31605f39d397SChris Mason nritems = btrfs_header_nritems(lower); 3161c293498bSStoyan Gaydarov BUG_ON(slot > nritems); 3162143bede5SJeff Mahoney BUG_ON(nritems == BTRFS_NODEPTRS_PER_BLOCK(root)); 316374123bd7SChris Mason if (slot != nritems) { 3164c3e06965SJan Schmidt if (level) 3165f3ea38daSJan Schmidt tree_mod_log_eb_move(root->fs_info, lower, slot + 1, 3166f3ea38daSJan Schmidt slot, nritems - slot); 31675f39d397SChris Mason memmove_extent_buffer(lower, 31685f39d397SChris Mason btrfs_node_key_ptr_offset(slot + 1), 31695f39d397SChris Mason btrfs_node_key_ptr_offset(slot), 3170123abc88SChris Mason (nritems - slot) * sizeof(struct btrfs_key_ptr)); 317174123bd7SChris Mason } 3172c3e06965SJan Schmidt if (level) { 3173f3ea38daSJan Schmidt ret = tree_mod_log_insert_key(root->fs_info, lower, slot, 3174c8cc6341SJosef Bacik MOD_LOG_KEY_ADD, GFP_NOFS); 3175f3ea38daSJan Schmidt BUG_ON(ret < 0); 3176f3ea38daSJan Schmidt } 31775f39d397SChris Mason btrfs_set_node_key(lower, key, slot); 3178db94535dSChris Mason btrfs_set_node_blockptr(lower, slot, bytenr); 317974493f7aSChris Mason WARN_ON(trans->transid == 0); 318074493f7aSChris Mason btrfs_set_node_ptr_generation(lower, slot, trans->transid); 31815f39d397SChris Mason btrfs_set_header_nritems(lower, nritems + 1); 31825f39d397SChris Mason btrfs_mark_buffer_dirty(lower); 318374123bd7SChris Mason } 318474123bd7SChris Mason 318597571fd0SChris Mason /* 318697571fd0SChris Mason * split the node at the specified level in path in two. 318797571fd0SChris Mason * The path is corrected to point to the appropriate node after the split 318897571fd0SChris Mason * 318997571fd0SChris Mason * Before splitting this tries to make some room in the node by pushing 319097571fd0SChris Mason * left and right, if either one works, it returns right away. 3191aa5d6bedSChris Mason * 3192aa5d6bedSChris Mason * returns 0 on success and < 0 on failure 319397571fd0SChris Mason */ 3194e02119d5SChris Mason static noinline int split_node(struct btrfs_trans_handle *trans, 3195e02119d5SChris Mason struct btrfs_root *root, 3196e02119d5SChris Mason struct btrfs_path *path, int level) 3197be0e5c09SChris Mason { 31985f39d397SChris Mason struct extent_buffer *c; 31995f39d397SChris Mason struct extent_buffer *split; 32005f39d397SChris Mason struct btrfs_disk_key disk_key; 3201be0e5c09SChris Mason int mid; 32025c680ed6SChris Mason int ret; 32037518a238SChris Mason u32 c_nritems; 3204be0e5c09SChris Mason 32055f39d397SChris Mason c = path->nodes[level]; 32067bb86316SChris Mason WARN_ON(btrfs_header_generation(c) != trans->transid); 32075f39d397SChris Mason if (c == root->node) { 3208d9abbf1cSJan Schmidt /* 320990f8d62eSJan Schmidt * trying to split the root, lets make a new one 321090f8d62eSJan Schmidt * 3211fdd99c72SLiu Bo * tree mod log: We don't log_removal old root in 321290f8d62eSJan Schmidt * insert_new_root, because that root buffer will be kept as a 321390f8d62eSJan Schmidt * normal node. We are going to log removal of half of the 321490f8d62eSJan Schmidt * elements below with tree_mod_log_eb_copy. We're holding a 321590f8d62eSJan Schmidt * tree lock on the buffer, which is why we cannot race with 321690f8d62eSJan Schmidt * other tree_mod_log users. 3217d9abbf1cSJan Schmidt */ 3218fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, level + 1); 32195c680ed6SChris Mason if (ret) 32205c680ed6SChris Mason return ret; 3221b3612421SChris Mason } else { 3222e66f709bSChris Mason ret = push_nodes_for_insert(trans, root, path, level); 32235f39d397SChris Mason c = path->nodes[level]; 32245f39d397SChris Mason if (!ret && btrfs_header_nritems(c) < 3225c448acf0SChris Mason BTRFS_NODEPTRS_PER_BLOCK(root) - 3) 3226e66f709bSChris Mason return 0; 322754aa1f4dSChris Mason if (ret < 0) 322854aa1f4dSChris Mason return ret; 32295c680ed6SChris Mason } 3230e66f709bSChris Mason 32315f39d397SChris Mason c_nritems = btrfs_header_nritems(c); 32325d4f98a2SYan Zheng mid = (c_nritems + 1) / 2; 32335d4f98a2SYan Zheng btrfs_node_key(c, &disk_key, mid); 32347bb86316SChris Mason 32355d4f98a2SYan Zheng split = btrfs_alloc_free_block(trans, root, root->nodesize, 0, 32367bb86316SChris Mason root->root_key.objectid, 32375581a51aSJan Schmidt &disk_key, level, c->start, 0); 32385f39d397SChris Mason if (IS_ERR(split)) 32395f39d397SChris Mason return PTR_ERR(split); 324054aa1f4dSChris Mason 3241f0486c68SYan, Zheng root_add_used(root, root->nodesize); 3242f0486c68SYan, Zheng 32435d4f98a2SYan Zheng memset_extent_buffer(split, 0, 0, sizeof(struct btrfs_header)); 32445f39d397SChris Mason btrfs_set_header_level(split, btrfs_header_level(c)); 3245db94535dSChris Mason btrfs_set_header_bytenr(split, split->start); 32465f39d397SChris Mason btrfs_set_header_generation(split, trans->transid); 32475d4f98a2SYan Zheng btrfs_set_header_backref_rev(split, BTRFS_MIXED_BACKREF_REV); 32485f39d397SChris Mason btrfs_set_header_owner(split, root->root_key.objectid); 32495f39d397SChris Mason write_extent_buffer(split, root->fs_info->fsid, 32505f39d397SChris Mason (unsigned long)btrfs_header_fsid(split), 32515f39d397SChris Mason BTRFS_FSID_SIZE); 3252e17cade2SChris Mason write_extent_buffer(split, root->fs_info->chunk_tree_uuid, 3253e17cade2SChris Mason (unsigned long)btrfs_header_chunk_tree_uuid(split), 3254e17cade2SChris Mason BTRFS_UUID_SIZE); 32555f39d397SChris Mason 325690f8d62eSJan Schmidt tree_mod_log_eb_copy(root->fs_info, split, c, 0, mid, c_nritems - mid); 32575f39d397SChris Mason copy_extent_buffer(split, c, 32585f39d397SChris Mason btrfs_node_key_ptr_offset(0), 32595f39d397SChris Mason btrfs_node_key_ptr_offset(mid), 3260123abc88SChris Mason (c_nritems - mid) * sizeof(struct btrfs_key_ptr)); 32615f39d397SChris Mason btrfs_set_header_nritems(split, c_nritems - mid); 32625f39d397SChris Mason btrfs_set_header_nritems(c, mid); 3263aa5d6bedSChris Mason ret = 0; 3264aa5d6bedSChris Mason 32655f39d397SChris Mason btrfs_mark_buffer_dirty(c); 32665f39d397SChris Mason btrfs_mark_buffer_dirty(split); 32675f39d397SChris Mason 3268143bede5SJeff Mahoney insert_ptr(trans, root, path, &disk_key, split->start, 3269c3e06965SJan Schmidt path->slots[level + 1] + 1, level + 1); 3270aa5d6bedSChris Mason 32715de08d7dSChris Mason if (path->slots[level] >= mid) { 32725c680ed6SChris Mason path->slots[level] -= mid; 3273925baeddSChris Mason btrfs_tree_unlock(c); 32745f39d397SChris Mason free_extent_buffer(c); 32755f39d397SChris Mason path->nodes[level] = split; 32765c680ed6SChris Mason path->slots[level + 1] += 1; 3277eb60ceacSChris Mason } else { 3278925baeddSChris Mason btrfs_tree_unlock(split); 32795f39d397SChris Mason free_extent_buffer(split); 3280be0e5c09SChris Mason } 3281aa5d6bedSChris Mason return ret; 3282be0e5c09SChris Mason } 3283be0e5c09SChris Mason 328474123bd7SChris Mason /* 328574123bd7SChris Mason * how many bytes are required to store the items in a leaf. start 328674123bd7SChris Mason * and nr indicate which items in the leaf to check. This totals up the 328774123bd7SChris Mason * space used both by the item structs and the item data 328874123bd7SChris Mason */ 32895f39d397SChris Mason static int leaf_space_used(struct extent_buffer *l, int start, int nr) 3290be0e5c09SChris Mason { 329141be1f3bSJosef Bacik struct btrfs_item *start_item; 329241be1f3bSJosef Bacik struct btrfs_item *end_item; 329341be1f3bSJosef Bacik struct btrfs_map_token token; 3294be0e5c09SChris Mason int data_len; 32955f39d397SChris Mason int nritems = btrfs_header_nritems(l); 3296d4dbff95SChris Mason int end = min(nritems, start + nr) - 1; 3297be0e5c09SChris Mason 3298be0e5c09SChris Mason if (!nr) 3299be0e5c09SChris Mason return 0; 330041be1f3bSJosef Bacik btrfs_init_map_token(&token); 330141be1f3bSJosef Bacik start_item = btrfs_item_nr(l, start); 330241be1f3bSJosef Bacik end_item = btrfs_item_nr(l, end); 330341be1f3bSJosef Bacik data_len = btrfs_token_item_offset(l, start_item, &token) + 330441be1f3bSJosef Bacik btrfs_token_item_size(l, start_item, &token); 330541be1f3bSJosef Bacik data_len = data_len - btrfs_token_item_offset(l, end_item, &token); 33060783fcfcSChris Mason data_len += sizeof(struct btrfs_item) * nr; 3307d4dbff95SChris Mason WARN_ON(data_len < 0); 3308be0e5c09SChris Mason return data_len; 3309be0e5c09SChris Mason } 3310be0e5c09SChris Mason 331174123bd7SChris Mason /* 3312d4dbff95SChris Mason * The space between the end of the leaf items and 3313d4dbff95SChris Mason * the start of the leaf data. IOW, how much room 3314d4dbff95SChris Mason * the leaf has left for both items and data 3315d4dbff95SChris Mason */ 3316d397712bSChris Mason noinline int btrfs_leaf_free_space(struct btrfs_root *root, 3317e02119d5SChris Mason struct extent_buffer *leaf) 3318d4dbff95SChris Mason { 33195f39d397SChris Mason int nritems = btrfs_header_nritems(leaf); 33205f39d397SChris Mason int ret; 33215f39d397SChris Mason ret = BTRFS_LEAF_DATA_SIZE(root) - leaf_space_used(leaf, 0, nritems); 33225f39d397SChris Mason if (ret < 0) { 3323d397712bSChris Mason printk(KERN_CRIT "leaf free space ret %d, leaf data size %lu, " 3324d397712bSChris Mason "used %d nritems %d\n", 3325ae2f5411SJens Axboe ret, (unsigned long) BTRFS_LEAF_DATA_SIZE(root), 33265f39d397SChris Mason leaf_space_used(leaf, 0, nritems), nritems); 33275f39d397SChris Mason } 33285f39d397SChris Mason return ret; 3329d4dbff95SChris Mason } 3330d4dbff95SChris Mason 333199d8f83cSChris Mason /* 333299d8f83cSChris Mason * min slot controls the lowest index we're willing to push to the 333399d8f83cSChris Mason * right. We'll push up to and including min_slot, but no lower 333499d8f83cSChris Mason */ 333544871b1bSChris Mason static noinline int __push_leaf_right(struct btrfs_trans_handle *trans, 333644871b1bSChris Mason struct btrfs_root *root, 333744871b1bSChris Mason struct btrfs_path *path, 333844871b1bSChris Mason int data_size, int empty, 333944871b1bSChris Mason struct extent_buffer *right, 334099d8f83cSChris Mason int free_space, u32 left_nritems, 334199d8f83cSChris Mason u32 min_slot) 334200ec4c51SChris Mason { 33435f39d397SChris Mason struct extent_buffer *left = path->nodes[0]; 334444871b1bSChris Mason struct extent_buffer *upper = path->nodes[1]; 3345cfed81a0SChris Mason struct btrfs_map_token token; 33465f39d397SChris Mason struct btrfs_disk_key disk_key; 334700ec4c51SChris Mason int slot; 334834a38218SChris Mason u32 i; 334900ec4c51SChris Mason int push_space = 0; 335000ec4c51SChris Mason int push_items = 0; 33510783fcfcSChris Mason struct btrfs_item *item; 335234a38218SChris Mason u32 nr; 33537518a238SChris Mason u32 right_nritems; 33545f39d397SChris Mason u32 data_end; 3355db94535dSChris Mason u32 this_item_size; 335600ec4c51SChris Mason 3357cfed81a0SChris Mason btrfs_init_map_token(&token); 3358cfed81a0SChris Mason 335934a38218SChris Mason if (empty) 336034a38218SChris Mason nr = 0; 336134a38218SChris Mason else 336299d8f83cSChris Mason nr = max_t(u32, 1, min_slot); 336334a38218SChris Mason 336431840ae1SZheng Yan if (path->slots[0] >= left_nritems) 336587b29b20SYan Zheng push_space += data_size; 336631840ae1SZheng Yan 336744871b1bSChris Mason slot = path->slots[1]; 336834a38218SChris Mason i = left_nritems - 1; 336934a38218SChris Mason while (i >= nr) { 33705f39d397SChris Mason item = btrfs_item_nr(left, i); 3371db94535dSChris Mason 337231840ae1SZheng Yan if (!empty && push_items > 0) { 337331840ae1SZheng Yan if (path->slots[0] > i) 337431840ae1SZheng Yan break; 337531840ae1SZheng Yan if (path->slots[0] == i) { 337631840ae1SZheng Yan int space = btrfs_leaf_free_space(root, left); 337731840ae1SZheng Yan if (space + push_space * 2 > free_space) 337831840ae1SZheng Yan break; 337931840ae1SZheng Yan } 338031840ae1SZheng Yan } 338131840ae1SZheng Yan 338200ec4c51SChris Mason if (path->slots[0] == i) 338387b29b20SYan Zheng push_space += data_size; 3384db94535dSChris Mason 3385db94535dSChris Mason this_item_size = btrfs_item_size(left, item); 3386db94535dSChris Mason if (this_item_size + sizeof(*item) + push_space > free_space) 338700ec4c51SChris Mason break; 338831840ae1SZheng Yan 338900ec4c51SChris Mason push_items++; 3390db94535dSChris Mason push_space += this_item_size + sizeof(*item); 339134a38218SChris Mason if (i == 0) 339234a38218SChris Mason break; 339334a38218SChris Mason i--; 3394db94535dSChris Mason } 33955f39d397SChris Mason 3396925baeddSChris Mason if (push_items == 0) 3397925baeddSChris Mason goto out_unlock; 33985f39d397SChris Mason 33996c1500f2SJulia Lawall WARN_ON(!empty && push_items == left_nritems); 34005f39d397SChris Mason 340100ec4c51SChris Mason /* push left to right */ 34025f39d397SChris Mason right_nritems = btrfs_header_nritems(right); 340334a38218SChris Mason 34045f39d397SChris Mason push_space = btrfs_item_end_nr(left, left_nritems - push_items); 3405123abc88SChris Mason push_space -= leaf_data_end(root, left); 34065f39d397SChris Mason 340700ec4c51SChris Mason /* make room in the right data area */ 34085f39d397SChris Mason data_end = leaf_data_end(root, right); 34095f39d397SChris Mason memmove_extent_buffer(right, 34105f39d397SChris Mason btrfs_leaf_data(right) + data_end - push_space, 34115f39d397SChris Mason btrfs_leaf_data(right) + data_end, 34125f39d397SChris Mason BTRFS_LEAF_DATA_SIZE(root) - data_end); 34135f39d397SChris Mason 341400ec4c51SChris Mason /* copy from the left data area */ 34155f39d397SChris Mason copy_extent_buffer(right, left, btrfs_leaf_data(right) + 3416d6025579SChris Mason BTRFS_LEAF_DATA_SIZE(root) - push_space, 3417d6025579SChris Mason btrfs_leaf_data(left) + leaf_data_end(root, left), 3418d6025579SChris Mason push_space); 34195f39d397SChris Mason 34205f39d397SChris Mason memmove_extent_buffer(right, btrfs_item_nr_offset(push_items), 34215f39d397SChris Mason btrfs_item_nr_offset(0), 34220783fcfcSChris Mason right_nritems * sizeof(struct btrfs_item)); 34235f39d397SChris Mason 342400ec4c51SChris Mason /* copy the items from left to right */ 34255f39d397SChris Mason copy_extent_buffer(right, left, btrfs_item_nr_offset(0), 34265f39d397SChris Mason btrfs_item_nr_offset(left_nritems - push_items), 34270783fcfcSChris Mason push_items * sizeof(struct btrfs_item)); 342800ec4c51SChris Mason 342900ec4c51SChris Mason /* update the item pointers */ 34307518a238SChris Mason right_nritems += push_items; 34315f39d397SChris Mason btrfs_set_header_nritems(right, right_nritems); 3432123abc88SChris Mason push_space = BTRFS_LEAF_DATA_SIZE(root); 34337518a238SChris Mason for (i = 0; i < right_nritems; i++) { 34345f39d397SChris Mason item = btrfs_item_nr(right, i); 3435cfed81a0SChris Mason push_space -= btrfs_token_item_size(right, item, &token); 3436cfed81a0SChris Mason btrfs_set_token_item_offset(right, item, push_space, &token); 3437db94535dSChris Mason } 3438db94535dSChris Mason 34397518a238SChris Mason left_nritems -= push_items; 34405f39d397SChris Mason btrfs_set_header_nritems(left, left_nritems); 344100ec4c51SChris Mason 344234a38218SChris Mason if (left_nritems) 34435f39d397SChris Mason btrfs_mark_buffer_dirty(left); 3444f0486c68SYan, Zheng else 3445f0486c68SYan, Zheng clean_tree_block(trans, root, left); 3446f0486c68SYan, Zheng 34475f39d397SChris Mason btrfs_mark_buffer_dirty(right); 3448a429e513SChris Mason 34495f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 34505f39d397SChris Mason btrfs_set_node_key(upper, &disk_key, slot + 1); 3451d6025579SChris Mason btrfs_mark_buffer_dirty(upper); 345202217ed2SChris Mason 345300ec4c51SChris Mason /* then fixup the leaf pointer in the path */ 34547518a238SChris Mason if (path->slots[0] >= left_nritems) { 34557518a238SChris Mason path->slots[0] -= left_nritems; 3456925baeddSChris Mason if (btrfs_header_nritems(path->nodes[0]) == 0) 3457925baeddSChris Mason clean_tree_block(trans, root, path->nodes[0]); 3458925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 34595f39d397SChris Mason free_extent_buffer(path->nodes[0]); 34605f39d397SChris Mason path->nodes[0] = right; 346100ec4c51SChris Mason path->slots[1] += 1; 346200ec4c51SChris Mason } else { 3463925baeddSChris Mason btrfs_tree_unlock(right); 34645f39d397SChris Mason free_extent_buffer(right); 346500ec4c51SChris Mason } 346600ec4c51SChris Mason return 0; 3467925baeddSChris Mason 3468925baeddSChris Mason out_unlock: 3469925baeddSChris Mason btrfs_tree_unlock(right); 3470925baeddSChris Mason free_extent_buffer(right); 3471925baeddSChris Mason return 1; 347200ec4c51SChris Mason } 3473925baeddSChris Mason 347400ec4c51SChris Mason /* 347544871b1bSChris Mason * push some data in the path leaf to the right, trying to free up at 347674123bd7SChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 347744871b1bSChris Mason * 347844871b1bSChris Mason * returns 1 if the push failed because the other node didn't have enough 347944871b1bSChris Mason * room, 0 if everything worked out and < 0 if there were major errors. 348099d8f83cSChris Mason * 348199d8f83cSChris Mason * this will push starting from min_slot to the end of the leaf. It won't 348299d8f83cSChris Mason * push any slot lower than min_slot 348374123bd7SChris Mason */ 348444871b1bSChris Mason static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root 348599d8f83cSChris Mason *root, struct btrfs_path *path, 348699d8f83cSChris Mason int min_data_size, int data_size, 348799d8f83cSChris Mason int empty, u32 min_slot) 3488be0e5c09SChris Mason { 348944871b1bSChris Mason struct extent_buffer *left = path->nodes[0]; 349044871b1bSChris Mason struct extent_buffer *right; 349144871b1bSChris Mason struct extent_buffer *upper; 349244871b1bSChris Mason int slot; 349344871b1bSChris Mason int free_space; 349444871b1bSChris Mason u32 left_nritems; 349544871b1bSChris Mason int ret; 349644871b1bSChris Mason 349744871b1bSChris Mason if (!path->nodes[1]) 349844871b1bSChris Mason return 1; 349944871b1bSChris Mason 350044871b1bSChris Mason slot = path->slots[1]; 350144871b1bSChris Mason upper = path->nodes[1]; 350244871b1bSChris Mason if (slot >= btrfs_header_nritems(upper) - 1) 350344871b1bSChris Mason return 1; 350444871b1bSChris Mason 350544871b1bSChris Mason btrfs_assert_tree_locked(path->nodes[1]); 350644871b1bSChris Mason 350744871b1bSChris Mason right = read_node_slot(root, upper, slot + 1); 350891ca338dSTsutomu Itoh if (right == NULL) 350991ca338dSTsutomu Itoh return 1; 351091ca338dSTsutomu Itoh 351144871b1bSChris Mason btrfs_tree_lock(right); 351244871b1bSChris Mason btrfs_set_lock_blocking(right); 351344871b1bSChris Mason 351444871b1bSChris Mason free_space = btrfs_leaf_free_space(root, right); 351544871b1bSChris Mason if (free_space < data_size) 351644871b1bSChris Mason goto out_unlock; 351744871b1bSChris Mason 351844871b1bSChris Mason /* cow and double check */ 351944871b1bSChris Mason ret = btrfs_cow_block(trans, root, right, upper, 352044871b1bSChris Mason slot + 1, &right); 352144871b1bSChris Mason if (ret) 352244871b1bSChris Mason goto out_unlock; 352344871b1bSChris Mason 352444871b1bSChris Mason free_space = btrfs_leaf_free_space(root, right); 352544871b1bSChris Mason if (free_space < data_size) 352644871b1bSChris Mason goto out_unlock; 352744871b1bSChris Mason 352844871b1bSChris Mason left_nritems = btrfs_header_nritems(left); 352944871b1bSChris Mason if (left_nritems == 0) 353044871b1bSChris Mason goto out_unlock; 353144871b1bSChris Mason 353299d8f83cSChris Mason return __push_leaf_right(trans, root, path, min_data_size, empty, 353399d8f83cSChris Mason right, free_space, left_nritems, min_slot); 353444871b1bSChris Mason out_unlock: 353544871b1bSChris Mason btrfs_tree_unlock(right); 353644871b1bSChris Mason free_extent_buffer(right); 353744871b1bSChris Mason return 1; 353844871b1bSChris Mason } 353944871b1bSChris Mason 354044871b1bSChris Mason /* 354144871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 354244871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 354399d8f83cSChris Mason * 354499d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 354599d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us do all the 354699d8f83cSChris Mason * items 354744871b1bSChris Mason */ 354844871b1bSChris Mason static noinline int __push_leaf_left(struct btrfs_trans_handle *trans, 354944871b1bSChris Mason struct btrfs_root *root, 355044871b1bSChris Mason struct btrfs_path *path, int data_size, 355144871b1bSChris Mason int empty, struct extent_buffer *left, 355299d8f83cSChris Mason int free_space, u32 right_nritems, 355399d8f83cSChris Mason u32 max_slot) 355444871b1bSChris Mason { 35555f39d397SChris Mason struct btrfs_disk_key disk_key; 35565f39d397SChris Mason struct extent_buffer *right = path->nodes[0]; 3557be0e5c09SChris Mason int i; 3558be0e5c09SChris Mason int push_space = 0; 3559be0e5c09SChris Mason int push_items = 0; 35600783fcfcSChris Mason struct btrfs_item *item; 35617518a238SChris Mason u32 old_left_nritems; 356234a38218SChris Mason u32 nr; 3563aa5d6bedSChris Mason int ret = 0; 3564db94535dSChris Mason u32 this_item_size; 3565db94535dSChris Mason u32 old_left_item_size; 3566cfed81a0SChris Mason struct btrfs_map_token token; 3567cfed81a0SChris Mason 3568cfed81a0SChris Mason btrfs_init_map_token(&token); 3569be0e5c09SChris Mason 357034a38218SChris Mason if (empty) 357199d8f83cSChris Mason nr = min(right_nritems, max_slot); 357234a38218SChris Mason else 357399d8f83cSChris Mason nr = min(right_nritems - 1, max_slot); 357434a38218SChris Mason 357534a38218SChris Mason for (i = 0; i < nr; i++) { 35765f39d397SChris Mason item = btrfs_item_nr(right, i); 3577db94535dSChris Mason 357831840ae1SZheng Yan if (!empty && push_items > 0) { 357931840ae1SZheng Yan if (path->slots[0] < i) 358031840ae1SZheng Yan break; 358131840ae1SZheng Yan if (path->slots[0] == i) { 358231840ae1SZheng Yan int space = btrfs_leaf_free_space(root, right); 358331840ae1SZheng Yan if (space + push_space * 2 > free_space) 358431840ae1SZheng Yan break; 358531840ae1SZheng Yan } 358631840ae1SZheng Yan } 358731840ae1SZheng Yan 3588be0e5c09SChris Mason if (path->slots[0] == i) 358987b29b20SYan Zheng push_space += data_size; 3590db94535dSChris Mason 3591db94535dSChris Mason this_item_size = btrfs_item_size(right, item); 3592db94535dSChris Mason if (this_item_size + sizeof(*item) + push_space > free_space) 3593be0e5c09SChris Mason break; 3594db94535dSChris Mason 3595be0e5c09SChris Mason push_items++; 3596db94535dSChris Mason push_space += this_item_size + sizeof(*item); 3597be0e5c09SChris Mason } 3598db94535dSChris Mason 3599be0e5c09SChris Mason if (push_items == 0) { 3600925baeddSChris Mason ret = 1; 3601925baeddSChris Mason goto out; 3602be0e5c09SChris Mason } 360334a38218SChris Mason if (!empty && push_items == btrfs_header_nritems(right)) 3604a429e513SChris Mason WARN_ON(1); 36055f39d397SChris Mason 3606be0e5c09SChris Mason /* push data from right to left */ 36075f39d397SChris Mason copy_extent_buffer(left, right, 36085f39d397SChris Mason btrfs_item_nr_offset(btrfs_header_nritems(left)), 36095f39d397SChris Mason btrfs_item_nr_offset(0), 36105f39d397SChris Mason push_items * sizeof(struct btrfs_item)); 36115f39d397SChris Mason 3612123abc88SChris Mason push_space = BTRFS_LEAF_DATA_SIZE(root) - 36135f39d397SChris Mason btrfs_item_offset_nr(right, push_items - 1); 36145f39d397SChris Mason 36155f39d397SChris Mason copy_extent_buffer(left, right, btrfs_leaf_data(left) + 3616d6025579SChris Mason leaf_data_end(root, left) - push_space, 3617123abc88SChris Mason btrfs_leaf_data(right) + 36185f39d397SChris Mason btrfs_item_offset_nr(right, push_items - 1), 3619be0e5c09SChris Mason push_space); 36205f39d397SChris Mason old_left_nritems = btrfs_header_nritems(left); 362187b29b20SYan Zheng BUG_ON(old_left_nritems <= 0); 3622eb60ceacSChris Mason 3623db94535dSChris Mason old_left_item_size = btrfs_item_offset_nr(left, old_left_nritems - 1); 3624be0e5c09SChris Mason for (i = old_left_nritems; i < old_left_nritems + push_items; i++) { 36255f39d397SChris Mason u32 ioff; 3626db94535dSChris Mason 36275f39d397SChris Mason item = btrfs_item_nr(left, i); 3628db94535dSChris Mason 3629cfed81a0SChris Mason ioff = btrfs_token_item_offset(left, item, &token); 3630cfed81a0SChris Mason btrfs_set_token_item_offset(left, item, 3631cfed81a0SChris Mason ioff - (BTRFS_LEAF_DATA_SIZE(root) - old_left_item_size), 3632cfed81a0SChris Mason &token); 3633be0e5c09SChris Mason } 36345f39d397SChris Mason btrfs_set_header_nritems(left, old_left_nritems + push_items); 3635be0e5c09SChris Mason 3636be0e5c09SChris Mason /* fixup right node */ 363731b1a2bdSJulia Lawall if (push_items > right_nritems) 363831b1a2bdSJulia Lawall WARN(1, KERN_CRIT "push items %d nr %u\n", push_items, 3639d397712bSChris Mason right_nritems); 364034a38218SChris Mason 364134a38218SChris Mason if (push_items < right_nritems) { 36425f39d397SChris Mason push_space = btrfs_item_offset_nr(right, push_items - 1) - 3643123abc88SChris Mason leaf_data_end(root, right); 36445f39d397SChris Mason memmove_extent_buffer(right, btrfs_leaf_data(right) + 3645d6025579SChris Mason BTRFS_LEAF_DATA_SIZE(root) - push_space, 3646d6025579SChris Mason btrfs_leaf_data(right) + 3647123abc88SChris Mason leaf_data_end(root, right), push_space); 36485f39d397SChris Mason 36495f39d397SChris Mason memmove_extent_buffer(right, btrfs_item_nr_offset(0), 36505f39d397SChris Mason btrfs_item_nr_offset(push_items), 36515f39d397SChris Mason (btrfs_header_nritems(right) - push_items) * 36520783fcfcSChris Mason sizeof(struct btrfs_item)); 365334a38218SChris Mason } 3654eef1c494SYan right_nritems -= push_items; 3655eef1c494SYan btrfs_set_header_nritems(right, right_nritems); 3656123abc88SChris Mason push_space = BTRFS_LEAF_DATA_SIZE(root); 36575f39d397SChris Mason for (i = 0; i < right_nritems; i++) { 36585f39d397SChris Mason item = btrfs_item_nr(right, i); 3659db94535dSChris Mason 3660cfed81a0SChris Mason push_space = push_space - btrfs_token_item_size(right, 3661cfed81a0SChris Mason item, &token); 3662cfed81a0SChris Mason btrfs_set_token_item_offset(right, item, push_space, &token); 3663db94535dSChris Mason } 3664eb60ceacSChris Mason 36655f39d397SChris Mason btrfs_mark_buffer_dirty(left); 366634a38218SChris Mason if (right_nritems) 36675f39d397SChris Mason btrfs_mark_buffer_dirty(right); 3668f0486c68SYan, Zheng else 3669f0486c68SYan, Zheng clean_tree_block(trans, root, right); 3670098f59c2SChris Mason 36715f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 3672d6a0a126STsutomu Itoh fixup_low_keys(root, path, &disk_key, 1); 3673be0e5c09SChris Mason 3674be0e5c09SChris Mason /* then fixup the leaf pointer in the path */ 3675be0e5c09SChris Mason if (path->slots[0] < push_items) { 3676be0e5c09SChris Mason path->slots[0] += old_left_nritems; 3677925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 36785f39d397SChris Mason free_extent_buffer(path->nodes[0]); 36795f39d397SChris Mason path->nodes[0] = left; 3680be0e5c09SChris Mason path->slots[1] -= 1; 3681be0e5c09SChris Mason } else { 3682925baeddSChris Mason btrfs_tree_unlock(left); 36835f39d397SChris Mason free_extent_buffer(left); 3684be0e5c09SChris Mason path->slots[0] -= push_items; 3685be0e5c09SChris Mason } 3686eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 3687aa5d6bedSChris Mason return ret; 3688925baeddSChris Mason out: 3689925baeddSChris Mason btrfs_tree_unlock(left); 3690925baeddSChris Mason free_extent_buffer(left); 3691925baeddSChris Mason return ret; 3692be0e5c09SChris Mason } 3693be0e5c09SChris Mason 369474123bd7SChris Mason /* 369544871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 369644871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 369799d8f83cSChris Mason * 369899d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 369999d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us push all the 370099d8f83cSChris Mason * items 370144871b1bSChris Mason */ 370244871b1bSChris Mason static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root 370399d8f83cSChris Mason *root, struct btrfs_path *path, int min_data_size, 370499d8f83cSChris Mason int data_size, int empty, u32 max_slot) 370544871b1bSChris Mason { 370644871b1bSChris Mason struct extent_buffer *right = path->nodes[0]; 370744871b1bSChris Mason struct extent_buffer *left; 370844871b1bSChris Mason int slot; 370944871b1bSChris Mason int free_space; 371044871b1bSChris Mason u32 right_nritems; 371144871b1bSChris Mason int ret = 0; 371244871b1bSChris Mason 371344871b1bSChris Mason slot = path->slots[1]; 371444871b1bSChris Mason if (slot == 0) 371544871b1bSChris Mason return 1; 371644871b1bSChris Mason if (!path->nodes[1]) 371744871b1bSChris Mason return 1; 371844871b1bSChris Mason 371944871b1bSChris Mason right_nritems = btrfs_header_nritems(right); 372044871b1bSChris Mason if (right_nritems == 0) 372144871b1bSChris Mason return 1; 372244871b1bSChris Mason 372344871b1bSChris Mason btrfs_assert_tree_locked(path->nodes[1]); 372444871b1bSChris Mason 372544871b1bSChris Mason left = read_node_slot(root, path->nodes[1], slot - 1); 372691ca338dSTsutomu Itoh if (left == NULL) 372791ca338dSTsutomu Itoh return 1; 372891ca338dSTsutomu Itoh 372944871b1bSChris Mason btrfs_tree_lock(left); 373044871b1bSChris Mason btrfs_set_lock_blocking(left); 373144871b1bSChris Mason 373244871b1bSChris Mason free_space = btrfs_leaf_free_space(root, left); 373344871b1bSChris Mason if (free_space < data_size) { 373444871b1bSChris Mason ret = 1; 373544871b1bSChris Mason goto out; 373644871b1bSChris Mason } 373744871b1bSChris Mason 373844871b1bSChris Mason /* cow and double check */ 373944871b1bSChris Mason ret = btrfs_cow_block(trans, root, left, 374044871b1bSChris Mason path->nodes[1], slot - 1, &left); 374144871b1bSChris Mason if (ret) { 374244871b1bSChris Mason /* we hit -ENOSPC, but it isn't fatal here */ 374379787eaaSJeff Mahoney if (ret == -ENOSPC) 374444871b1bSChris Mason ret = 1; 374544871b1bSChris Mason goto out; 374644871b1bSChris Mason } 374744871b1bSChris Mason 374844871b1bSChris Mason free_space = btrfs_leaf_free_space(root, left); 374944871b1bSChris Mason if (free_space < data_size) { 375044871b1bSChris Mason ret = 1; 375144871b1bSChris Mason goto out; 375244871b1bSChris Mason } 375344871b1bSChris Mason 375499d8f83cSChris Mason return __push_leaf_left(trans, root, path, min_data_size, 375599d8f83cSChris Mason empty, left, free_space, right_nritems, 375699d8f83cSChris Mason max_slot); 375744871b1bSChris Mason out: 375844871b1bSChris Mason btrfs_tree_unlock(left); 375944871b1bSChris Mason free_extent_buffer(left); 376044871b1bSChris Mason return ret; 376144871b1bSChris Mason } 376244871b1bSChris Mason 376344871b1bSChris Mason /* 376474123bd7SChris Mason * split the path's leaf in two, making sure there is at least data_size 376574123bd7SChris Mason * available for the resulting leaf level of the path. 376674123bd7SChris Mason */ 3767143bede5SJeff Mahoney static noinline void copy_for_split(struct btrfs_trans_handle *trans, 3768e02119d5SChris Mason struct btrfs_root *root, 376944871b1bSChris Mason struct btrfs_path *path, 377044871b1bSChris Mason struct extent_buffer *l, 377144871b1bSChris Mason struct extent_buffer *right, 377244871b1bSChris Mason int slot, int mid, int nritems) 3773be0e5c09SChris Mason { 3774be0e5c09SChris Mason int data_copy_size; 3775be0e5c09SChris Mason int rt_data_off; 3776be0e5c09SChris Mason int i; 3777d4dbff95SChris Mason struct btrfs_disk_key disk_key; 3778cfed81a0SChris Mason struct btrfs_map_token token; 3779cfed81a0SChris Mason 3780cfed81a0SChris Mason btrfs_init_map_token(&token); 3781be0e5c09SChris Mason 37825f39d397SChris Mason nritems = nritems - mid; 37835f39d397SChris Mason btrfs_set_header_nritems(right, nritems); 37845f39d397SChris Mason data_copy_size = btrfs_item_end_nr(l, mid) - leaf_data_end(root, l); 37855f39d397SChris Mason 37865f39d397SChris Mason copy_extent_buffer(right, l, btrfs_item_nr_offset(0), 37875f39d397SChris Mason btrfs_item_nr_offset(mid), 37885f39d397SChris Mason nritems * sizeof(struct btrfs_item)); 37895f39d397SChris Mason 37905f39d397SChris Mason copy_extent_buffer(right, l, 3791d6025579SChris Mason btrfs_leaf_data(right) + BTRFS_LEAF_DATA_SIZE(root) - 3792123abc88SChris Mason data_copy_size, btrfs_leaf_data(l) + 3793123abc88SChris Mason leaf_data_end(root, l), data_copy_size); 379474123bd7SChris Mason 37955f39d397SChris Mason rt_data_off = BTRFS_LEAF_DATA_SIZE(root) - 37965f39d397SChris Mason btrfs_item_end_nr(l, mid); 37975f39d397SChris Mason 37985f39d397SChris Mason for (i = 0; i < nritems; i++) { 37995f39d397SChris Mason struct btrfs_item *item = btrfs_item_nr(right, i); 3800db94535dSChris Mason u32 ioff; 3801db94535dSChris Mason 3802cfed81a0SChris Mason ioff = btrfs_token_item_offset(right, item, &token); 3803cfed81a0SChris Mason btrfs_set_token_item_offset(right, item, 3804cfed81a0SChris Mason ioff + rt_data_off, &token); 38050783fcfcSChris Mason } 380674123bd7SChris Mason 38075f39d397SChris Mason btrfs_set_header_nritems(l, mid); 38085f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 3809143bede5SJeff Mahoney insert_ptr(trans, root, path, &disk_key, right->start, 3810c3e06965SJan Schmidt path->slots[1] + 1, 1); 38115f39d397SChris Mason 38125f39d397SChris Mason btrfs_mark_buffer_dirty(right); 38135f39d397SChris Mason btrfs_mark_buffer_dirty(l); 3814eb60ceacSChris Mason BUG_ON(path->slots[0] != slot); 38155f39d397SChris Mason 3816be0e5c09SChris Mason if (mid <= slot) { 3817925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 38185f39d397SChris Mason free_extent_buffer(path->nodes[0]); 38195f39d397SChris Mason path->nodes[0] = right; 3820be0e5c09SChris Mason path->slots[0] -= mid; 3821be0e5c09SChris Mason path->slots[1] += 1; 3822925baeddSChris Mason } else { 3823925baeddSChris Mason btrfs_tree_unlock(right); 38245f39d397SChris Mason free_extent_buffer(right); 3825925baeddSChris Mason } 38265f39d397SChris Mason 3827eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 382844871b1bSChris Mason } 382944871b1bSChris Mason 383044871b1bSChris Mason /* 383199d8f83cSChris Mason * double splits happen when we need to insert a big item in the middle 383299d8f83cSChris Mason * of a leaf. A double split can leave us with 3 mostly empty leaves: 383399d8f83cSChris Mason * leaf: [ slots 0 - N] [ our target ] [ N + 1 - total in leaf ] 383499d8f83cSChris Mason * A B C 383599d8f83cSChris Mason * 383699d8f83cSChris Mason * We avoid this by trying to push the items on either side of our target 383799d8f83cSChris Mason * into the adjacent leaves. If all goes well we can avoid the double split 383899d8f83cSChris Mason * completely. 383999d8f83cSChris Mason */ 384099d8f83cSChris Mason static noinline int push_for_double_split(struct btrfs_trans_handle *trans, 384199d8f83cSChris Mason struct btrfs_root *root, 384299d8f83cSChris Mason struct btrfs_path *path, 384399d8f83cSChris Mason int data_size) 384499d8f83cSChris Mason { 384599d8f83cSChris Mason int ret; 384699d8f83cSChris Mason int progress = 0; 384799d8f83cSChris Mason int slot; 384899d8f83cSChris Mason u32 nritems; 384999d8f83cSChris Mason 385099d8f83cSChris Mason slot = path->slots[0]; 385199d8f83cSChris Mason 385299d8f83cSChris Mason /* 385399d8f83cSChris Mason * try to push all the items after our slot into the 385499d8f83cSChris Mason * right leaf 385599d8f83cSChris Mason */ 385699d8f83cSChris Mason ret = push_leaf_right(trans, root, path, 1, data_size, 0, slot); 385799d8f83cSChris Mason if (ret < 0) 385899d8f83cSChris Mason return ret; 385999d8f83cSChris Mason 386099d8f83cSChris Mason if (ret == 0) 386199d8f83cSChris Mason progress++; 386299d8f83cSChris Mason 386399d8f83cSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 386499d8f83cSChris Mason /* 386599d8f83cSChris Mason * our goal is to get our slot at the start or end of a leaf. If 386699d8f83cSChris Mason * we've done so we're done 386799d8f83cSChris Mason */ 386899d8f83cSChris Mason if (path->slots[0] == 0 || path->slots[0] == nritems) 386999d8f83cSChris Mason return 0; 387099d8f83cSChris Mason 387199d8f83cSChris Mason if (btrfs_leaf_free_space(root, path->nodes[0]) >= data_size) 387299d8f83cSChris Mason return 0; 387399d8f83cSChris Mason 387499d8f83cSChris Mason /* try to push all the items before our slot into the next leaf */ 387599d8f83cSChris Mason slot = path->slots[0]; 387699d8f83cSChris Mason ret = push_leaf_left(trans, root, path, 1, data_size, 0, slot); 387799d8f83cSChris Mason if (ret < 0) 387899d8f83cSChris Mason return ret; 387999d8f83cSChris Mason 388099d8f83cSChris Mason if (ret == 0) 388199d8f83cSChris Mason progress++; 388299d8f83cSChris Mason 388399d8f83cSChris Mason if (progress) 388499d8f83cSChris Mason return 0; 388599d8f83cSChris Mason return 1; 388699d8f83cSChris Mason } 388799d8f83cSChris Mason 388899d8f83cSChris Mason /* 388944871b1bSChris Mason * split the path's leaf in two, making sure there is at least data_size 389044871b1bSChris Mason * available for the resulting leaf level of the path. 389144871b1bSChris Mason * 389244871b1bSChris Mason * returns 0 if all went well and < 0 on failure. 389344871b1bSChris Mason */ 389444871b1bSChris Mason static noinline int split_leaf(struct btrfs_trans_handle *trans, 389544871b1bSChris Mason struct btrfs_root *root, 389644871b1bSChris Mason struct btrfs_key *ins_key, 389744871b1bSChris Mason struct btrfs_path *path, int data_size, 389844871b1bSChris Mason int extend) 389944871b1bSChris Mason { 39005d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 390144871b1bSChris Mason struct extent_buffer *l; 390244871b1bSChris Mason u32 nritems; 390344871b1bSChris Mason int mid; 390444871b1bSChris Mason int slot; 390544871b1bSChris Mason struct extent_buffer *right; 390644871b1bSChris Mason int ret = 0; 390744871b1bSChris Mason int wret; 39085d4f98a2SYan Zheng int split; 390944871b1bSChris Mason int num_doubles = 0; 391099d8f83cSChris Mason int tried_avoid_double = 0; 391144871b1bSChris Mason 3912a5719521SYan, Zheng l = path->nodes[0]; 3913a5719521SYan, Zheng slot = path->slots[0]; 3914a5719521SYan, Zheng if (extend && data_size + btrfs_item_size_nr(l, slot) + 3915a5719521SYan, Zheng sizeof(struct btrfs_item) > BTRFS_LEAF_DATA_SIZE(root)) 3916a5719521SYan, Zheng return -EOVERFLOW; 3917a5719521SYan, Zheng 391844871b1bSChris Mason /* first try to make some room by pushing left and right */ 391933157e05SLiu Bo if (data_size && path->nodes[1]) { 392099d8f83cSChris Mason wret = push_leaf_right(trans, root, path, data_size, 392199d8f83cSChris Mason data_size, 0, 0); 392244871b1bSChris Mason if (wret < 0) 392344871b1bSChris Mason return wret; 392444871b1bSChris Mason if (wret) { 392599d8f83cSChris Mason wret = push_leaf_left(trans, root, path, data_size, 392699d8f83cSChris Mason data_size, 0, (u32)-1); 392744871b1bSChris Mason if (wret < 0) 392844871b1bSChris Mason return wret; 392944871b1bSChris Mason } 393044871b1bSChris Mason l = path->nodes[0]; 393144871b1bSChris Mason 393244871b1bSChris Mason /* did the pushes work? */ 393344871b1bSChris Mason if (btrfs_leaf_free_space(root, l) >= data_size) 393444871b1bSChris Mason return 0; 393544871b1bSChris Mason } 393644871b1bSChris Mason 393744871b1bSChris Mason if (!path->nodes[1]) { 3938fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, 1); 393944871b1bSChris Mason if (ret) 394044871b1bSChris Mason return ret; 394144871b1bSChris Mason } 394244871b1bSChris Mason again: 39435d4f98a2SYan Zheng split = 1; 394444871b1bSChris Mason l = path->nodes[0]; 394544871b1bSChris Mason slot = path->slots[0]; 394644871b1bSChris Mason nritems = btrfs_header_nritems(l); 394744871b1bSChris Mason mid = (nritems + 1) / 2; 394844871b1bSChris Mason 39495d4f98a2SYan Zheng if (mid <= slot) { 39505d4f98a2SYan Zheng if (nritems == 1 || 39515d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + data_size > 39525d4f98a2SYan Zheng BTRFS_LEAF_DATA_SIZE(root)) { 39535d4f98a2SYan Zheng if (slot >= nritems) { 39545d4f98a2SYan Zheng split = 0; 39555d4f98a2SYan Zheng } else { 39565d4f98a2SYan Zheng mid = slot; 39575d4f98a2SYan Zheng if (mid != nritems && 39585d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 39595d4f98a2SYan Zheng data_size > BTRFS_LEAF_DATA_SIZE(root)) { 396099d8f83cSChris Mason if (data_size && !tried_avoid_double) 396199d8f83cSChris Mason goto push_for_double; 39625d4f98a2SYan Zheng split = 2; 39635d4f98a2SYan Zheng } 39645d4f98a2SYan Zheng } 39655d4f98a2SYan Zheng } 39665d4f98a2SYan Zheng } else { 39675d4f98a2SYan Zheng if (leaf_space_used(l, 0, mid) + data_size > 39685d4f98a2SYan Zheng BTRFS_LEAF_DATA_SIZE(root)) { 39695d4f98a2SYan Zheng if (!extend && data_size && slot == 0) { 39705d4f98a2SYan Zheng split = 0; 39715d4f98a2SYan Zheng } else if ((extend || !data_size) && slot == 0) { 39725d4f98a2SYan Zheng mid = 1; 39735d4f98a2SYan Zheng } else { 39745d4f98a2SYan Zheng mid = slot; 39755d4f98a2SYan Zheng if (mid != nritems && 39765d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 39775d4f98a2SYan Zheng data_size > BTRFS_LEAF_DATA_SIZE(root)) { 397899d8f83cSChris Mason if (data_size && !tried_avoid_double) 397999d8f83cSChris Mason goto push_for_double; 39805d4f98a2SYan Zheng split = 2 ; 39815d4f98a2SYan Zheng } 39825d4f98a2SYan Zheng } 39835d4f98a2SYan Zheng } 39845d4f98a2SYan Zheng } 39855d4f98a2SYan Zheng 39865d4f98a2SYan Zheng if (split == 0) 39875d4f98a2SYan Zheng btrfs_cpu_key_to_disk(&disk_key, ins_key); 39885d4f98a2SYan Zheng else 39895d4f98a2SYan Zheng btrfs_item_key(l, &disk_key, mid); 39905d4f98a2SYan Zheng 39915d4f98a2SYan Zheng right = btrfs_alloc_free_block(trans, root, root->leafsize, 0, 399244871b1bSChris Mason root->root_key.objectid, 39935581a51aSJan Schmidt &disk_key, 0, l->start, 0); 3994f0486c68SYan, Zheng if (IS_ERR(right)) 399544871b1bSChris Mason return PTR_ERR(right); 3996f0486c68SYan, Zheng 3997f0486c68SYan, Zheng root_add_used(root, root->leafsize); 399844871b1bSChris Mason 399944871b1bSChris Mason memset_extent_buffer(right, 0, 0, sizeof(struct btrfs_header)); 400044871b1bSChris Mason btrfs_set_header_bytenr(right, right->start); 400144871b1bSChris Mason btrfs_set_header_generation(right, trans->transid); 40025d4f98a2SYan Zheng btrfs_set_header_backref_rev(right, BTRFS_MIXED_BACKREF_REV); 400344871b1bSChris Mason btrfs_set_header_owner(right, root->root_key.objectid); 400444871b1bSChris Mason btrfs_set_header_level(right, 0); 400544871b1bSChris Mason write_extent_buffer(right, root->fs_info->fsid, 400644871b1bSChris Mason (unsigned long)btrfs_header_fsid(right), 400744871b1bSChris Mason BTRFS_FSID_SIZE); 400844871b1bSChris Mason 400944871b1bSChris Mason write_extent_buffer(right, root->fs_info->chunk_tree_uuid, 401044871b1bSChris Mason (unsigned long)btrfs_header_chunk_tree_uuid(right), 401144871b1bSChris Mason BTRFS_UUID_SIZE); 401244871b1bSChris Mason 40135d4f98a2SYan Zheng if (split == 0) { 401444871b1bSChris Mason if (mid <= slot) { 401544871b1bSChris Mason btrfs_set_header_nritems(right, 0); 4016143bede5SJeff Mahoney insert_ptr(trans, root, path, &disk_key, right->start, 4017c3e06965SJan Schmidt path->slots[1] + 1, 1); 401844871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 401944871b1bSChris Mason free_extent_buffer(path->nodes[0]); 402044871b1bSChris Mason path->nodes[0] = right; 402144871b1bSChris Mason path->slots[0] = 0; 402244871b1bSChris Mason path->slots[1] += 1; 402344871b1bSChris Mason } else { 402444871b1bSChris Mason btrfs_set_header_nritems(right, 0); 4025143bede5SJeff Mahoney insert_ptr(trans, root, path, &disk_key, right->start, 4026c3e06965SJan Schmidt path->slots[1], 1); 402744871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 402844871b1bSChris Mason free_extent_buffer(path->nodes[0]); 402944871b1bSChris Mason path->nodes[0] = right; 403044871b1bSChris Mason path->slots[0] = 0; 4031143bede5SJeff Mahoney if (path->slots[1] == 0) 4032d6a0a126STsutomu Itoh fixup_low_keys(root, path, &disk_key, 1); 40335d4f98a2SYan Zheng } 403444871b1bSChris Mason btrfs_mark_buffer_dirty(right); 403544871b1bSChris Mason return ret; 403644871b1bSChris Mason } 403744871b1bSChris Mason 4038143bede5SJeff Mahoney copy_for_split(trans, root, path, l, right, slot, mid, nritems); 403944871b1bSChris Mason 40405d4f98a2SYan Zheng if (split == 2) { 4041cc0c5538SChris Mason BUG_ON(num_doubles != 0); 4042cc0c5538SChris Mason num_doubles++; 4043cc0c5538SChris Mason goto again; 40443326d1b0SChris Mason } 404544871b1bSChris Mason 4046143bede5SJeff Mahoney return 0; 404799d8f83cSChris Mason 404899d8f83cSChris Mason push_for_double: 404999d8f83cSChris Mason push_for_double_split(trans, root, path, data_size); 405099d8f83cSChris Mason tried_avoid_double = 1; 405199d8f83cSChris Mason if (btrfs_leaf_free_space(root, path->nodes[0]) >= data_size) 405299d8f83cSChris Mason return 0; 405399d8f83cSChris Mason goto again; 4054be0e5c09SChris Mason } 4055be0e5c09SChris Mason 4056ad48fd75SYan, Zheng static noinline int setup_leaf_for_split(struct btrfs_trans_handle *trans, 4057ad48fd75SYan, Zheng struct btrfs_root *root, 4058ad48fd75SYan, Zheng struct btrfs_path *path, int ins_len) 4059ad48fd75SYan, Zheng { 4060ad48fd75SYan, Zheng struct btrfs_key key; 4061ad48fd75SYan, Zheng struct extent_buffer *leaf; 4062ad48fd75SYan, Zheng struct btrfs_file_extent_item *fi; 4063ad48fd75SYan, Zheng u64 extent_len = 0; 4064ad48fd75SYan, Zheng u32 item_size; 4065ad48fd75SYan, Zheng int ret; 4066ad48fd75SYan, Zheng 4067ad48fd75SYan, Zheng leaf = path->nodes[0]; 4068ad48fd75SYan, Zheng btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 4069ad48fd75SYan, Zheng 4070ad48fd75SYan, Zheng BUG_ON(key.type != BTRFS_EXTENT_DATA_KEY && 4071ad48fd75SYan, Zheng key.type != BTRFS_EXTENT_CSUM_KEY); 4072ad48fd75SYan, Zheng 4073ad48fd75SYan, Zheng if (btrfs_leaf_free_space(root, leaf) >= ins_len) 4074ad48fd75SYan, Zheng return 0; 4075ad48fd75SYan, Zheng 4076ad48fd75SYan, Zheng item_size = btrfs_item_size_nr(leaf, path->slots[0]); 4077ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 4078ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 4079ad48fd75SYan, Zheng struct btrfs_file_extent_item); 4080ad48fd75SYan, Zheng extent_len = btrfs_file_extent_num_bytes(leaf, fi); 4081ad48fd75SYan, Zheng } 4082b3b4aa74SDavid Sterba btrfs_release_path(path); 4083ad48fd75SYan, Zheng 4084ad48fd75SYan, Zheng path->keep_locks = 1; 4085ad48fd75SYan, Zheng path->search_for_split = 1; 4086ad48fd75SYan, Zheng ret = btrfs_search_slot(trans, root, &key, path, 0, 1); 4087ad48fd75SYan, Zheng path->search_for_split = 0; 4088ad48fd75SYan, Zheng if (ret < 0) 4089ad48fd75SYan, Zheng goto err; 4090ad48fd75SYan, Zheng 4091ad48fd75SYan, Zheng ret = -EAGAIN; 4092ad48fd75SYan, Zheng leaf = path->nodes[0]; 4093ad48fd75SYan, Zheng /* if our item isn't there or got smaller, return now */ 4094ad48fd75SYan, Zheng if (ret > 0 || item_size != btrfs_item_size_nr(leaf, path->slots[0])) 4095ad48fd75SYan, Zheng goto err; 4096ad48fd75SYan, Zheng 4097109f6aefSChris Mason /* the leaf has changed, it now has room. return now */ 4098109f6aefSChris Mason if (btrfs_leaf_free_space(root, path->nodes[0]) >= ins_len) 4099109f6aefSChris Mason goto err; 4100109f6aefSChris Mason 4101ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 4102ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 4103ad48fd75SYan, Zheng struct btrfs_file_extent_item); 4104ad48fd75SYan, Zheng if (extent_len != btrfs_file_extent_num_bytes(leaf, fi)) 4105ad48fd75SYan, Zheng goto err; 4106ad48fd75SYan, Zheng } 4107ad48fd75SYan, Zheng 4108ad48fd75SYan, Zheng btrfs_set_path_blocking(path); 4109ad48fd75SYan, Zheng ret = split_leaf(trans, root, &key, path, ins_len, 1); 4110f0486c68SYan, Zheng if (ret) 4111f0486c68SYan, Zheng goto err; 4112ad48fd75SYan, Zheng 4113ad48fd75SYan, Zheng path->keep_locks = 0; 4114ad48fd75SYan, Zheng btrfs_unlock_up_safe(path, 1); 4115ad48fd75SYan, Zheng return 0; 4116ad48fd75SYan, Zheng err: 4117ad48fd75SYan, Zheng path->keep_locks = 0; 4118ad48fd75SYan, Zheng return ret; 4119ad48fd75SYan, Zheng } 4120ad48fd75SYan, Zheng 4121ad48fd75SYan, Zheng static noinline int split_item(struct btrfs_trans_handle *trans, 4122459931ecSChris Mason struct btrfs_root *root, 4123459931ecSChris Mason struct btrfs_path *path, 4124459931ecSChris Mason struct btrfs_key *new_key, 4125459931ecSChris Mason unsigned long split_offset) 4126459931ecSChris Mason { 4127459931ecSChris Mason struct extent_buffer *leaf; 4128459931ecSChris Mason struct btrfs_item *item; 4129459931ecSChris Mason struct btrfs_item *new_item; 4130459931ecSChris Mason int slot; 4131ad48fd75SYan, Zheng char *buf; 4132459931ecSChris Mason u32 nritems; 4133ad48fd75SYan, Zheng u32 item_size; 4134459931ecSChris Mason u32 orig_offset; 4135459931ecSChris Mason struct btrfs_disk_key disk_key; 4136459931ecSChris Mason 4137459931ecSChris Mason leaf = path->nodes[0]; 4138b9473439SChris Mason BUG_ON(btrfs_leaf_free_space(root, leaf) < sizeof(struct btrfs_item)); 4139b9473439SChris Mason 4140b4ce94deSChris Mason btrfs_set_path_blocking(path); 4141b4ce94deSChris Mason 4142459931ecSChris Mason item = btrfs_item_nr(leaf, path->slots[0]); 4143459931ecSChris Mason orig_offset = btrfs_item_offset(leaf, item); 4144459931ecSChris Mason item_size = btrfs_item_size(leaf, item); 4145459931ecSChris Mason 4146459931ecSChris Mason buf = kmalloc(item_size, GFP_NOFS); 4147ad48fd75SYan, Zheng if (!buf) 4148ad48fd75SYan, Zheng return -ENOMEM; 4149ad48fd75SYan, Zheng 4150459931ecSChris Mason read_extent_buffer(leaf, buf, btrfs_item_ptr_offset(leaf, 4151459931ecSChris Mason path->slots[0]), item_size); 4152ad48fd75SYan, Zheng 4153459931ecSChris Mason slot = path->slots[0] + 1; 4154459931ecSChris Mason nritems = btrfs_header_nritems(leaf); 4155459931ecSChris Mason if (slot != nritems) { 4156459931ecSChris Mason /* shift the items */ 4157459931ecSChris Mason memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + 1), 4158459931ecSChris Mason btrfs_item_nr_offset(slot), 4159459931ecSChris Mason (nritems - slot) * sizeof(struct btrfs_item)); 4160459931ecSChris Mason } 4161459931ecSChris Mason 4162459931ecSChris Mason btrfs_cpu_key_to_disk(&disk_key, new_key); 4163459931ecSChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 4164459931ecSChris Mason 4165459931ecSChris Mason new_item = btrfs_item_nr(leaf, slot); 4166459931ecSChris Mason 4167459931ecSChris Mason btrfs_set_item_offset(leaf, new_item, orig_offset); 4168459931ecSChris Mason btrfs_set_item_size(leaf, new_item, item_size - split_offset); 4169459931ecSChris Mason 4170459931ecSChris Mason btrfs_set_item_offset(leaf, item, 4171459931ecSChris Mason orig_offset + item_size - split_offset); 4172459931ecSChris Mason btrfs_set_item_size(leaf, item, split_offset); 4173459931ecSChris Mason 4174459931ecSChris Mason btrfs_set_header_nritems(leaf, nritems + 1); 4175459931ecSChris Mason 4176459931ecSChris Mason /* write the data for the start of the original item */ 4177459931ecSChris Mason write_extent_buffer(leaf, buf, 4178459931ecSChris Mason btrfs_item_ptr_offset(leaf, path->slots[0]), 4179459931ecSChris Mason split_offset); 4180459931ecSChris Mason 4181459931ecSChris Mason /* write the data for the new item */ 4182459931ecSChris Mason write_extent_buffer(leaf, buf + split_offset, 4183459931ecSChris Mason btrfs_item_ptr_offset(leaf, slot), 4184459931ecSChris Mason item_size - split_offset); 4185459931ecSChris Mason btrfs_mark_buffer_dirty(leaf); 4186459931ecSChris Mason 4187ad48fd75SYan, Zheng BUG_ON(btrfs_leaf_free_space(root, leaf) < 0); 4188459931ecSChris Mason kfree(buf); 4189ad48fd75SYan, Zheng return 0; 4190ad48fd75SYan, Zheng } 4191ad48fd75SYan, Zheng 4192ad48fd75SYan, Zheng /* 4193ad48fd75SYan, Zheng * This function splits a single item into two items, 4194ad48fd75SYan, Zheng * giving 'new_key' to the new item and splitting the 4195ad48fd75SYan, Zheng * old one at split_offset (from the start of the item). 4196ad48fd75SYan, Zheng * 4197ad48fd75SYan, Zheng * The path may be released by this operation. After 4198ad48fd75SYan, Zheng * the split, the path is pointing to the old item. The 4199ad48fd75SYan, Zheng * new item is going to be in the same node as the old one. 4200ad48fd75SYan, Zheng * 4201ad48fd75SYan, Zheng * Note, the item being split must be smaller enough to live alone on 4202ad48fd75SYan, Zheng * a tree block with room for one extra struct btrfs_item 4203ad48fd75SYan, Zheng * 4204ad48fd75SYan, Zheng * This allows us to split the item in place, keeping a lock on the 4205ad48fd75SYan, Zheng * leaf the entire time. 4206ad48fd75SYan, Zheng */ 4207ad48fd75SYan, Zheng int btrfs_split_item(struct btrfs_trans_handle *trans, 4208ad48fd75SYan, Zheng struct btrfs_root *root, 4209ad48fd75SYan, Zheng struct btrfs_path *path, 4210ad48fd75SYan, Zheng struct btrfs_key *new_key, 4211ad48fd75SYan, Zheng unsigned long split_offset) 4212ad48fd75SYan, Zheng { 4213ad48fd75SYan, Zheng int ret; 4214ad48fd75SYan, Zheng ret = setup_leaf_for_split(trans, root, path, 4215ad48fd75SYan, Zheng sizeof(struct btrfs_item)); 4216ad48fd75SYan, Zheng if (ret) 4217459931ecSChris Mason return ret; 4218ad48fd75SYan, Zheng 4219ad48fd75SYan, Zheng ret = split_item(trans, root, path, new_key, split_offset); 4220ad48fd75SYan, Zheng return ret; 4221ad48fd75SYan, Zheng } 4222ad48fd75SYan, Zheng 4223ad48fd75SYan, Zheng /* 4224ad48fd75SYan, Zheng * This function duplicate a item, giving 'new_key' to the new item. 4225ad48fd75SYan, Zheng * It guarantees both items live in the same tree leaf and the new item 4226ad48fd75SYan, Zheng * is contiguous with the original item. 4227ad48fd75SYan, Zheng * 4228ad48fd75SYan, Zheng * This allows us to split file extent in place, keeping a lock on the 4229ad48fd75SYan, Zheng * leaf the entire time. 4230ad48fd75SYan, Zheng */ 4231ad48fd75SYan, Zheng int btrfs_duplicate_item(struct btrfs_trans_handle *trans, 4232ad48fd75SYan, Zheng struct btrfs_root *root, 4233ad48fd75SYan, Zheng struct btrfs_path *path, 4234ad48fd75SYan, Zheng struct btrfs_key *new_key) 4235ad48fd75SYan, Zheng { 4236ad48fd75SYan, Zheng struct extent_buffer *leaf; 4237ad48fd75SYan, Zheng int ret; 4238ad48fd75SYan, Zheng u32 item_size; 4239ad48fd75SYan, Zheng 4240ad48fd75SYan, Zheng leaf = path->nodes[0]; 4241ad48fd75SYan, Zheng item_size = btrfs_item_size_nr(leaf, path->slots[0]); 4242ad48fd75SYan, Zheng ret = setup_leaf_for_split(trans, root, path, 4243ad48fd75SYan, Zheng item_size + sizeof(struct btrfs_item)); 4244ad48fd75SYan, Zheng if (ret) 4245ad48fd75SYan, Zheng return ret; 4246ad48fd75SYan, Zheng 4247ad48fd75SYan, Zheng path->slots[0]++; 4248afe5fea7STsutomu Itoh setup_items_for_insert(root, path, new_key, &item_size, 4249ad48fd75SYan, Zheng item_size, item_size + 4250ad48fd75SYan, Zheng sizeof(struct btrfs_item), 1); 4251ad48fd75SYan, Zheng leaf = path->nodes[0]; 4252ad48fd75SYan, Zheng memcpy_extent_buffer(leaf, 4253ad48fd75SYan, Zheng btrfs_item_ptr_offset(leaf, path->slots[0]), 4254ad48fd75SYan, Zheng btrfs_item_ptr_offset(leaf, path->slots[0] - 1), 4255ad48fd75SYan, Zheng item_size); 4256ad48fd75SYan, Zheng return 0; 4257459931ecSChris Mason } 4258459931ecSChris Mason 4259459931ecSChris Mason /* 4260d352ac68SChris Mason * make the item pointed to by the path smaller. new_size indicates 4261d352ac68SChris Mason * how small to make it, and from_end tells us if we just chop bytes 4262d352ac68SChris Mason * off the end of the item or if we shift the item to chop bytes off 4263d352ac68SChris Mason * the front. 4264d352ac68SChris Mason */ 4265afe5fea7STsutomu Itoh void btrfs_truncate_item(struct btrfs_root *root, struct btrfs_path *path, 4266179e29e4SChris Mason u32 new_size, int from_end) 4267b18c6685SChris Mason { 4268b18c6685SChris Mason int slot; 42695f39d397SChris Mason struct extent_buffer *leaf; 42705f39d397SChris Mason struct btrfs_item *item; 4271b18c6685SChris Mason u32 nritems; 4272b18c6685SChris Mason unsigned int data_end; 4273b18c6685SChris Mason unsigned int old_data_start; 4274b18c6685SChris Mason unsigned int old_size; 4275b18c6685SChris Mason unsigned int size_diff; 4276b18c6685SChris Mason int i; 4277cfed81a0SChris Mason struct btrfs_map_token token; 4278cfed81a0SChris Mason 4279cfed81a0SChris Mason btrfs_init_map_token(&token); 4280b18c6685SChris Mason 42815f39d397SChris Mason leaf = path->nodes[0]; 4282179e29e4SChris Mason slot = path->slots[0]; 4283179e29e4SChris Mason 4284179e29e4SChris Mason old_size = btrfs_item_size_nr(leaf, slot); 4285179e29e4SChris Mason if (old_size == new_size) 4286143bede5SJeff Mahoney return; 4287b18c6685SChris Mason 42885f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 4289b18c6685SChris Mason data_end = leaf_data_end(root, leaf); 4290b18c6685SChris Mason 42915f39d397SChris Mason old_data_start = btrfs_item_offset_nr(leaf, slot); 4292179e29e4SChris Mason 4293b18c6685SChris Mason size_diff = old_size - new_size; 4294b18c6685SChris Mason 4295b18c6685SChris Mason BUG_ON(slot < 0); 4296b18c6685SChris Mason BUG_ON(slot >= nritems); 4297b18c6685SChris Mason 4298b18c6685SChris Mason /* 4299b18c6685SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 4300b18c6685SChris Mason */ 4301b18c6685SChris Mason /* first correct the data pointers */ 4302b18c6685SChris Mason for (i = slot; i < nritems; i++) { 43035f39d397SChris Mason u32 ioff; 43045f39d397SChris Mason item = btrfs_item_nr(leaf, i); 4305db94535dSChris Mason 4306cfed81a0SChris Mason ioff = btrfs_token_item_offset(leaf, item, &token); 4307cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 4308cfed81a0SChris Mason ioff + size_diff, &token); 4309b18c6685SChris Mason } 4310db94535dSChris Mason 4311b18c6685SChris Mason /* shift the data */ 4312179e29e4SChris Mason if (from_end) { 43135f39d397SChris Mason memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) + 4314b18c6685SChris Mason data_end + size_diff, btrfs_leaf_data(leaf) + 4315b18c6685SChris Mason data_end, old_data_start + new_size - data_end); 4316179e29e4SChris Mason } else { 4317179e29e4SChris Mason struct btrfs_disk_key disk_key; 4318179e29e4SChris Mason u64 offset; 4319179e29e4SChris Mason 4320179e29e4SChris Mason btrfs_item_key(leaf, &disk_key, slot); 4321179e29e4SChris Mason 4322179e29e4SChris Mason if (btrfs_disk_key_type(&disk_key) == BTRFS_EXTENT_DATA_KEY) { 4323179e29e4SChris Mason unsigned long ptr; 4324179e29e4SChris Mason struct btrfs_file_extent_item *fi; 4325179e29e4SChris Mason 4326179e29e4SChris Mason fi = btrfs_item_ptr(leaf, slot, 4327179e29e4SChris Mason struct btrfs_file_extent_item); 4328179e29e4SChris Mason fi = (struct btrfs_file_extent_item *)( 4329179e29e4SChris Mason (unsigned long)fi - size_diff); 4330179e29e4SChris Mason 4331179e29e4SChris Mason if (btrfs_file_extent_type(leaf, fi) == 4332179e29e4SChris Mason BTRFS_FILE_EXTENT_INLINE) { 4333179e29e4SChris Mason ptr = btrfs_item_ptr_offset(leaf, slot); 4334179e29e4SChris Mason memmove_extent_buffer(leaf, ptr, 4335179e29e4SChris Mason (unsigned long)fi, 4336179e29e4SChris Mason offsetof(struct btrfs_file_extent_item, 4337179e29e4SChris Mason disk_bytenr)); 4338179e29e4SChris Mason } 4339179e29e4SChris Mason } 4340179e29e4SChris Mason 4341179e29e4SChris Mason memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) + 4342179e29e4SChris Mason data_end + size_diff, btrfs_leaf_data(leaf) + 4343179e29e4SChris Mason data_end, old_data_start - data_end); 4344179e29e4SChris Mason 4345179e29e4SChris Mason offset = btrfs_disk_key_offset(&disk_key); 4346179e29e4SChris Mason btrfs_set_disk_key_offset(&disk_key, offset + size_diff); 4347179e29e4SChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 4348179e29e4SChris Mason if (slot == 0) 4349d6a0a126STsutomu Itoh fixup_low_keys(root, path, &disk_key, 1); 4350179e29e4SChris Mason } 43515f39d397SChris Mason 43525f39d397SChris Mason item = btrfs_item_nr(leaf, slot); 43535f39d397SChris Mason btrfs_set_item_size(leaf, item, new_size); 43545f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 4355b18c6685SChris Mason 43565f39d397SChris Mason if (btrfs_leaf_free_space(root, leaf) < 0) { 43575f39d397SChris Mason btrfs_print_leaf(root, leaf); 4358b18c6685SChris Mason BUG(); 43595f39d397SChris Mason } 4360b18c6685SChris Mason } 4361b18c6685SChris Mason 4362d352ac68SChris Mason /* 43638f69dbd2SStefan Behrens * make the item pointed to by the path bigger, data_size is the added size. 4364d352ac68SChris Mason */ 43654b90c680STsutomu Itoh void btrfs_extend_item(struct btrfs_root *root, struct btrfs_path *path, 43665f39d397SChris Mason u32 data_size) 43676567e837SChris Mason { 43686567e837SChris Mason int slot; 43695f39d397SChris Mason struct extent_buffer *leaf; 43705f39d397SChris Mason struct btrfs_item *item; 43716567e837SChris Mason u32 nritems; 43726567e837SChris Mason unsigned int data_end; 43736567e837SChris Mason unsigned int old_data; 43746567e837SChris Mason unsigned int old_size; 43756567e837SChris Mason int i; 4376cfed81a0SChris Mason struct btrfs_map_token token; 4377cfed81a0SChris Mason 4378cfed81a0SChris Mason btrfs_init_map_token(&token); 43796567e837SChris Mason 43805f39d397SChris Mason leaf = path->nodes[0]; 43816567e837SChris Mason 43825f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 43836567e837SChris Mason data_end = leaf_data_end(root, leaf); 43846567e837SChris Mason 43855f39d397SChris Mason if (btrfs_leaf_free_space(root, leaf) < data_size) { 43865f39d397SChris Mason btrfs_print_leaf(root, leaf); 43876567e837SChris Mason BUG(); 43885f39d397SChris Mason } 43896567e837SChris Mason slot = path->slots[0]; 43905f39d397SChris Mason old_data = btrfs_item_end_nr(leaf, slot); 43916567e837SChris Mason 43926567e837SChris Mason BUG_ON(slot < 0); 43933326d1b0SChris Mason if (slot >= nritems) { 43943326d1b0SChris Mason btrfs_print_leaf(root, leaf); 4395d397712bSChris Mason printk(KERN_CRIT "slot %d too large, nritems %d\n", 4396d397712bSChris Mason slot, nritems); 43973326d1b0SChris Mason BUG_ON(1); 43983326d1b0SChris Mason } 43996567e837SChris Mason 44006567e837SChris Mason /* 44016567e837SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 44026567e837SChris Mason */ 44036567e837SChris Mason /* first correct the data pointers */ 44046567e837SChris Mason for (i = slot; i < nritems; i++) { 44055f39d397SChris Mason u32 ioff; 44065f39d397SChris Mason item = btrfs_item_nr(leaf, i); 4407db94535dSChris Mason 4408cfed81a0SChris Mason ioff = btrfs_token_item_offset(leaf, item, &token); 4409cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 4410cfed81a0SChris Mason ioff - data_size, &token); 44116567e837SChris Mason } 44125f39d397SChris Mason 44136567e837SChris Mason /* shift the data */ 44145f39d397SChris Mason memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) + 44156567e837SChris Mason data_end - data_size, btrfs_leaf_data(leaf) + 44166567e837SChris Mason data_end, old_data - data_end); 44175f39d397SChris Mason 44186567e837SChris Mason data_end = old_data; 44195f39d397SChris Mason old_size = btrfs_item_size_nr(leaf, slot); 44205f39d397SChris Mason item = btrfs_item_nr(leaf, slot); 44215f39d397SChris Mason btrfs_set_item_size(leaf, item, old_size + data_size); 44225f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 44236567e837SChris Mason 44245f39d397SChris Mason if (btrfs_leaf_free_space(root, leaf) < 0) { 44255f39d397SChris Mason btrfs_print_leaf(root, leaf); 44266567e837SChris Mason BUG(); 44275f39d397SChris Mason } 44286567e837SChris Mason } 44296567e837SChris Mason 443074123bd7SChris Mason /* 443144871b1bSChris Mason * this is a helper for btrfs_insert_empty_items, the main goal here is 443244871b1bSChris Mason * to save stack depth by doing the bulk of the work in a function 443344871b1bSChris Mason * that doesn't call btrfs_search_slot 443474123bd7SChris Mason */ 4435afe5fea7STsutomu Itoh void setup_items_for_insert(struct btrfs_root *root, struct btrfs_path *path, 44369c58309dSChris Mason struct btrfs_key *cpu_key, u32 *data_size, 443744871b1bSChris Mason u32 total_data, u32 total_size, int nr) 4438be0e5c09SChris Mason { 44395f39d397SChris Mason struct btrfs_item *item; 44409c58309dSChris Mason int i; 44417518a238SChris Mason u32 nritems; 4442be0e5c09SChris Mason unsigned int data_end; 4443e2fa7227SChris Mason struct btrfs_disk_key disk_key; 444444871b1bSChris Mason struct extent_buffer *leaf; 444544871b1bSChris Mason int slot; 4446cfed81a0SChris Mason struct btrfs_map_token token; 4447cfed81a0SChris Mason 4448cfed81a0SChris Mason btrfs_init_map_token(&token); 4449e2fa7227SChris Mason 44505f39d397SChris Mason leaf = path->nodes[0]; 445144871b1bSChris Mason slot = path->slots[0]; 445274123bd7SChris Mason 44535f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 4454123abc88SChris Mason data_end = leaf_data_end(root, leaf); 4455eb60ceacSChris Mason 4456f25956ccSChris Mason if (btrfs_leaf_free_space(root, leaf) < total_size) { 44573326d1b0SChris Mason btrfs_print_leaf(root, leaf); 4458d397712bSChris Mason printk(KERN_CRIT "not enough freespace need %u have %d\n", 44599c58309dSChris Mason total_size, btrfs_leaf_free_space(root, leaf)); 4460be0e5c09SChris Mason BUG(); 4461d4dbff95SChris Mason } 44625f39d397SChris Mason 4463be0e5c09SChris Mason if (slot != nritems) { 44645f39d397SChris Mason unsigned int old_data = btrfs_item_end_nr(leaf, slot); 4465be0e5c09SChris Mason 44665f39d397SChris Mason if (old_data < data_end) { 44675f39d397SChris Mason btrfs_print_leaf(root, leaf); 4468d397712bSChris Mason printk(KERN_CRIT "slot %d old_data %d data_end %d\n", 44695f39d397SChris Mason slot, old_data, data_end); 44705f39d397SChris Mason BUG_ON(1); 44715f39d397SChris Mason } 4472be0e5c09SChris Mason /* 4473be0e5c09SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 4474be0e5c09SChris Mason */ 4475be0e5c09SChris Mason /* first correct the data pointers */ 44760783fcfcSChris Mason for (i = slot; i < nritems; i++) { 44775f39d397SChris Mason u32 ioff; 4478db94535dSChris Mason 44795f39d397SChris Mason item = btrfs_item_nr(leaf, i); 4480cfed81a0SChris Mason ioff = btrfs_token_item_offset(leaf, item, &token); 4481cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 4482cfed81a0SChris Mason ioff - total_data, &token); 44830783fcfcSChris Mason } 4484be0e5c09SChris Mason /* shift the items */ 44859c58309dSChris Mason memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + nr), 44865f39d397SChris Mason btrfs_item_nr_offset(slot), 44870783fcfcSChris Mason (nritems - slot) * sizeof(struct btrfs_item)); 4488be0e5c09SChris Mason 4489be0e5c09SChris Mason /* shift the data */ 44905f39d397SChris Mason memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) + 44919c58309dSChris Mason data_end - total_data, btrfs_leaf_data(leaf) + 4492be0e5c09SChris Mason data_end, old_data - data_end); 4493be0e5c09SChris Mason data_end = old_data; 4494be0e5c09SChris Mason } 44955f39d397SChris Mason 449662e2749eSChris Mason /* setup the item for the new data */ 44979c58309dSChris Mason for (i = 0; i < nr; i++) { 44989c58309dSChris Mason btrfs_cpu_key_to_disk(&disk_key, cpu_key + i); 44999c58309dSChris Mason btrfs_set_item_key(leaf, &disk_key, slot + i); 45009c58309dSChris Mason item = btrfs_item_nr(leaf, slot + i); 4501cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 4502cfed81a0SChris Mason data_end - data_size[i], &token); 45039c58309dSChris Mason data_end -= data_size[i]; 4504cfed81a0SChris Mason btrfs_set_token_item_size(leaf, item, data_size[i], &token); 45059c58309dSChris Mason } 450644871b1bSChris Mason 45079c58309dSChris Mason btrfs_set_header_nritems(leaf, nritems + nr); 4508aa5d6bedSChris Mason 45095a01a2e3SChris Mason if (slot == 0) { 45105a01a2e3SChris Mason btrfs_cpu_key_to_disk(&disk_key, cpu_key); 4511d6a0a126STsutomu Itoh fixup_low_keys(root, path, &disk_key, 1); 45125a01a2e3SChris Mason } 4513b9473439SChris Mason btrfs_unlock_up_safe(path, 1); 4514b9473439SChris Mason btrfs_mark_buffer_dirty(leaf); 4515aa5d6bedSChris Mason 45165f39d397SChris Mason if (btrfs_leaf_free_space(root, leaf) < 0) { 45175f39d397SChris Mason btrfs_print_leaf(root, leaf); 4518be0e5c09SChris Mason BUG(); 45195f39d397SChris Mason } 452044871b1bSChris Mason } 452144871b1bSChris Mason 452244871b1bSChris Mason /* 452344871b1bSChris Mason * Given a key and some data, insert items into the tree. 452444871b1bSChris Mason * This does all the path init required, making room in the tree if needed. 452544871b1bSChris Mason */ 452644871b1bSChris Mason int btrfs_insert_empty_items(struct btrfs_trans_handle *trans, 452744871b1bSChris Mason struct btrfs_root *root, 452844871b1bSChris Mason struct btrfs_path *path, 452944871b1bSChris Mason struct btrfs_key *cpu_key, u32 *data_size, 453044871b1bSChris Mason int nr) 453144871b1bSChris Mason { 453244871b1bSChris Mason int ret = 0; 453344871b1bSChris Mason int slot; 453444871b1bSChris Mason int i; 453544871b1bSChris Mason u32 total_size = 0; 453644871b1bSChris Mason u32 total_data = 0; 453744871b1bSChris Mason 453844871b1bSChris Mason for (i = 0; i < nr; i++) 453944871b1bSChris Mason total_data += data_size[i]; 454044871b1bSChris Mason 454144871b1bSChris Mason total_size = total_data + (nr * sizeof(struct btrfs_item)); 454244871b1bSChris Mason ret = btrfs_search_slot(trans, root, cpu_key, path, total_size, 1); 454344871b1bSChris Mason if (ret == 0) 454444871b1bSChris Mason return -EEXIST; 454544871b1bSChris Mason if (ret < 0) 4546143bede5SJeff Mahoney return ret; 454744871b1bSChris Mason 454844871b1bSChris Mason slot = path->slots[0]; 454944871b1bSChris Mason BUG_ON(slot < 0); 455044871b1bSChris Mason 4551afe5fea7STsutomu Itoh setup_items_for_insert(root, path, cpu_key, data_size, 455244871b1bSChris Mason total_data, total_size, nr); 4553143bede5SJeff Mahoney return 0; 455462e2749eSChris Mason } 455562e2749eSChris Mason 455662e2749eSChris Mason /* 455762e2749eSChris Mason * Given a key and some data, insert an item into the tree. 455862e2749eSChris Mason * This does all the path init required, making room in the tree if needed. 455962e2749eSChris Mason */ 4560e089f05cSChris Mason int btrfs_insert_item(struct btrfs_trans_handle *trans, struct btrfs_root 4561e089f05cSChris Mason *root, struct btrfs_key *cpu_key, void *data, u32 4562e089f05cSChris Mason data_size) 456362e2749eSChris Mason { 456462e2749eSChris Mason int ret = 0; 45652c90e5d6SChris Mason struct btrfs_path *path; 45665f39d397SChris Mason struct extent_buffer *leaf; 45675f39d397SChris Mason unsigned long ptr; 456862e2749eSChris Mason 45692c90e5d6SChris Mason path = btrfs_alloc_path(); 4570db5b493aSTsutomu Itoh if (!path) 4571db5b493aSTsutomu Itoh return -ENOMEM; 45722c90e5d6SChris Mason ret = btrfs_insert_empty_item(trans, root, path, cpu_key, data_size); 457362e2749eSChris Mason if (!ret) { 45745f39d397SChris Mason leaf = path->nodes[0]; 45755f39d397SChris Mason ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); 45765f39d397SChris Mason write_extent_buffer(leaf, data, ptr, data_size); 45775f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 457862e2749eSChris Mason } 45792c90e5d6SChris Mason btrfs_free_path(path); 4580aa5d6bedSChris Mason return ret; 4581be0e5c09SChris Mason } 4582be0e5c09SChris Mason 458374123bd7SChris Mason /* 45845de08d7dSChris Mason * delete the pointer from a given node. 458574123bd7SChris Mason * 4586d352ac68SChris Mason * the tree should have been previously balanced so the deletion does not 4587d352ac68SChris Mason * empty a node. 458874123bd7SChris Mason */ 4589afe5fea7STsutomu Itoh static void del_ptr(struct btrfs_root *root, struct btrfs_path *path, 4590afe5fea7STsutomu Itoh int level, int slot) 4591be0e5c09SChris Mason { 45925f39d397SChris Mason struct extent_buffer *parent = path->nodes[level]; 45937518a238SChris Mason u32 nritems; 4594f3ea38daSJan Schmidt int ret; 4595be0e5c09SChris Mason 45965f39d397SChris Mason nritems = btrfs_header_nritems(parent); 4597be0e5c09SChris Mason if (slot != nritems - 1) { 45980e411eceSLiu Bo if (level) 4599f3ea38daSJan Schmidt tree_mod_log_eb_move(root->fs_info, parent, slot, 4600f3ea38daSJan Schmidt slot + 1, nritems - slot - 1); 46015f39d397SChris Mason memmove_extent_buffer(parent, 46025f39d397SChris Mason btrfs_node_key_ptr_offset(slot), 46035f39d397SChris Mason btrfs_node_key_ptr_offset(slot + 1), 4604d6025579SChris Mason sizeof(struct btrfs_key_ptr) * 4605d6025579SChris Mason (nritems - slot - 1)); 460657ba86c0SChris Mason } else if (level) { 460757ba86c0SChris Mason ret = tree_mod_log_insert_key(root->fs_info, parent, slot, 4608c8cc6341SJosef Bacik MOD_LOG_KEY_REMOVE, GFP_NOFS); 460957ba86c0SChris Mason BUG_ON(ret < 0); 4610be0e5c09SChris Mason } 4611f3ea38daSJan Schmidt 46127518a238SChris Mason nritems--; 46135f39d397SChris Mason btrfs_set_header_nritems(parent, nritems); 46147518a238SChris Mason if (nritems == 0 && parent == root->node) { 46155f39d397SChris Mason BUG_ON(btrfs_header_level(root->node) != 1); 4616eb60ceacSChris Mason /* just turn the root into a leaf and break */ 46175f39d397SChris Mason btrfs_set_header_level(root->node, 0); 4618bb803951SChris Mason } else if (slot == 0) { 46195f39d397SChris Mason struct btrfs_disk_key disk_key; 46205f39d397SChris Mason 46215f39d397SChris Mason btrfs_node_key(parent, &disk_key, 0); 4622d6a0a126STsutomu Itoh fixup_low_keys(root, path, &disk_key, level + 1); 4623be0e5c09SChris Mason } 4624d6025579SChris Mason btrfs_mark_buffer_dirty(parent); 4625be0e5c09SChris Mason } 4626be0e5c09SChris Mason 462774123bd7SChris Mason /* 4628323ac95bSChris Mason * a helper function to delete the leaf pointed to by path->slots[1] and 46295d4f98a2SYan Zheng * path->nodes[1]. 4630323ac95bSChris Mason * 4631323ac95bSChris Mason * This deletes the pointer in path->nodes[1] and frees the leaf 4632323ac95bSChris Mason * block extent. zero is returned if it all worked out, < 0 otherwise. 4633323ac95bSChris Mason * 4634323ac95bSChris Mason * The path must have already been setup for deleting the leaf, including 4635323ac95bSChris Mason * all the proper balancing. path->nodes[1] must be locked. 4636323ac95bSChris Mason */ 4637143bede5SJeff Mahoney static noinline void btrfs_del_leaf(struct btrfs_trans_handle *trans, 4638323ac95bSChris Mason struct btrfs_root *root, 46395d4f98a2SYan Zheng struct btrfs_path *path, 46405d4f98a2SYan Zheng struct extent_buffer *leaf) 4641323ac95bSChris Mason { 46425d4f98a2SYan Zheng WARN_ON(btrfs_header_generation(leaf) != trans->transid); 4643afe5fea7STsutomu Itoh del_ptr(root, path, 1, path->slots[1]); 4644323ac95bSChris Mason 46454d081c41SChris Mason /* 46464d081c41SChris Mason * btrfs_free_extent is expensive, we want to make sure we 46474d081c41SChris Mason * aren't holding any locks when we call it 46484d081c41SChris Mason */ 46494d081c41SChris Mason btrfs_unlock_up_safe(path, 0); 46504d081c41SChris Mason 4651f0486c68SYan, Zheng root_sub_used(root, leaf->len); 4652f0486c68SYan, Zheng 46533083ee2eSJosef Bacik extent_buffer_get(leaf); 46545581a51aSJan Schmidt btrfs_free_tree_block(trans, root, leaf, 0, 1); 46553083ee2eSJosef Bacik free_extent_buffer_stale(leaf); 4656323ac95bSChris Mason } 4657323ac95bSChris Mason /* 465874123bd7SChris Mason * delete the item at the leaf level in path. If that empties 465974123bd7SChris Mason * the leaf, remove it from the tree 466074123bd7SChris Mason */ 466185e21bacSChris Mason int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root, 466285e21bacSChris Mason struct btrfs_path *path, int slot, int nr) 4663be0e5c09SChris Mason { 46645f39d397SChris Mason struct extent_buffer *leaf; 46655f39d397SChris Mason struct btrfs_item *item; 466685e21bacSChris Mason int last_off; 466785e21bacSChris Mason int dsize = 0; 4668aa5d6bedSChris Mason int ret = 0; 4669aa5d6bedSChris Mason int wret; 467085e21bacSChris Mason int i; 46717518a238SChris Mason u32 nritems; 4672cfed81a0SChris Mason struct btrfs_map_token token; 4673cfed81a0SChris Mason 4674cfed81a0SChris Mason btrfs_init_map_token(&token); 4675be0e5c09SChris Mason 46765f39d397SChris Mason leaf = path->nodes[0]; 467785e21bacSChris Mason last_off = btrfs_item_offset_nr(leaf, slot + nr - 1); 467885e21bacSChris Mason 467985e21bacSChris Mason for (i = 0; i < nr; i++) 468085e21bacSChris Mason dsize += btrfs_item_size_nr(leaf, slot + i); 468185e21bacSChris Mason 46825f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 4683be0e5c09SChris Mason 468485e21bacSChris Mason if (slot + nr != nritems) { 4685123abc88SChris Mason int data_end = leaf_data_end(root, leaf); 46865f39d397SChris Mason 46875f39d397SChris Mason memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) + 4688d6025579SChris Mason data_end + dsize, 4689123abc88SChris Mason btrfs_leaf_data(leaf) + data_end, 469085e21bacSChris Mason last_off - data_end); 46915f39d397SChris Mason 469285e21bacSChris Mason for (i = slot + nr; i < nritems; i++) { 46935f39d397SChris Mason u32 ioff; 4694db94535dSChris Mason 46955f39d397SChris Mason item = btrfs_item_nr(leaf, i); 4696cfed81a0SChris Mason ioff = btrfs_token_item_offset(leaf, item, &token); 4697cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 4698cfed81a0SChris Mason ioff + dsize, &token); 46990783fcfcSChris Mason } 4700db94535dSChris Mason 47015f39d397SChris Mason memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot), 470285e21bacSChris Mason btrfs_item_nr_offset(slot + nr), 47030783fcfcSChris Mason sizeof(struct btrfs_item) * 470485e21bacSChris Mason (nritems - slot - nr)); 4705be0e5c09SChris Mason } 470685e21bacSChris Mason btrfs_set_header_nritems(leaf, nritems - nr); 470785e21bacSChris Mason nritems -= nr; 47085f39d397SChris Mason 470974123bd7SChris Mason /* delete the leaf if we've emptied it */ 47107518a238SChris Mason if (nritems == 0) { 47115f39d397SChris Mason if (leaf == root->node) { 47125f39d397SChris Mason btrfs_set_header_level(leaf, 0); 47139a8dd150SChris Mason } else { 4714f0486c68SYan, Zheng btrfs_set_path_blocking(path); 4715f0486c68SYan, Zheng clean_tree_block(trans, root, leaf); 4716143bede5SJeff Mahoney btrfs_del_leaf(trans, root, path, leaf); 47179a8dd150SChris Mason } 4718be0e5c09SChris Mason } else { 47197518a238SChris Mason int used = leaf_space_used(leaf, 0, nritems); 4720aa5d6bedSChris Mason if (slot == 0) { 47215f39d397SChris Mason struct btrfs_disk_key disk_key; 47225f39d397SChris Mason 47235f39d397SChris Mason btrfs_item_key(leaf, &disk_key, 0); 4724d6a0a126STsutomu Itoh fixup_low_keys(root, path, &disk_key, 1); 4725aa5d6bedSChris Mason } 4726aa5d6bedSChris Mason 472774123bd7SChris Mason /* delete the leaf if it is mostly empty */ 4728d717aa1dSYan Zheng if (used < BTRFS_LEAF_DATA_SIZE(root) / 3) { 4729be0e5c09SChris Mason /* push_leaf_left fixes the path. 4730be0e5c09SChris Mason * make sure the path still points to our leaf 4731be0e5c09SChris Mason * for possible call to del_ptr below 4732be0e5c09SChris Mason */ 47334920c9acSChris Mason slot = path->slots[1]; 47345f39d397SChris Mason extent_buffer_get(leaf); 47355f39d397SChris Mason 4736b9473439SChris Mason btrfs_set_path_blocking(path); 473799d8f83cSChris Mason wret = push_leaf_left(trans, root, path, 1, 1, 473899d8f83cSChris Mason 1, (u32)-1); 473954aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 4740aa5d6bedSChris Mason ret = wret; 47415f39d397SChris Mason 47425f39d397SChris Mason if (path->nodes[0] == leaf && 47435f39d397SChris Mason btrfs_header_nritems(leaf)) { 474499d8f83cSChris Mason wret = push_leaf_right(trans, root, path, 1, 474599d8f83cSChris Mason 1, 1, 0); 474654aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 4747aa5d6bedSChris Mason ret = wret; 4748aa5d6bedSChris Mason } 47495f39d397SChris Mason 47505f39d397SChris Mason if (btrfs_header_nritems(leaf) == 0) { 4751323ac95bSChris Mason path->slots[1] = slot; 4752143bede5SJeff Mahoney btrfs_del_leaf(trans, root, path, leaf); 47535f39d397SChris Mason free_extent_buffer(leaf); 4754143bede5SJeff Mahoney ret = 0; 47555de08d7dSChris Mason } else { 4756925baeddSChris Mason /* if we're still in the path, make sure 4757925baeddSChris Mason * we're dirty. Otherwise, one of the 4758925baeddSChris Mason * push_leaf functions must have already 4759925baeddSChris Mason * dirtied this buffer 4760925baeddSChris Mason */ 4761925baeddSChris Mason if (path->nodes[0] == leaf) 47625f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 47635f39d397SChris Mason free_extent_buffer(leaf); 4764be0e5c09SChris Mason } 4765d5719762SChris Mason } else { 47665f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 4767be0e5c09SChris Mason } 47689a8dd150SChris Mason } 4769aa5d6bedSChris Mason return ret; 47709a8dd150SChris Mason } 47719a8dd150SChris Mason 477297571fd0SChris Mason /* 4773925baeddSChris Mason * search the tree again to find a leaf with lesser keys 47747bb86316SChris Mason * returns 0 if it found something or 1 if there are no lesser leaves. 47757bb86316SChris Mason * returns < 0 on io errors. 4776d352ac68SChris Mason * 4777d352ac68SChris Mason * This may release the path, and so you may lose any locks held at the 4778d352ac68SChris Mason * time you call it. 47797bb86316SChris Mason */ 47807bb86316SChris Mason int btrfs_prev_leaf(struct btrfs_root *root, struct btrfs_path *path) 47817bb86316SChris Mason { 4782925baeddSChris Mason struct btrfs_key key; 4783925baeddSChris Mason struct btrfs_disk_key found_key; 4784925baeddSChris Mason int ret; 47857bb86316SChris Mason 4786925baeddSChris Mason btrfs_item_key_to_cpu(path->nodes[0], &key, 0); 4787925baeddSChris Mason 4788925baeddSChris Mason if (key.offset > 0) 4789925baeddSChris Mason key.offset--; 4790925baeddSChris Mason else if (key.type > 0) 4791925baeddSChris Mason key.type--; 4792925baeddSChris Mason else if (key.objectid > 0) 4793925baeddSChris Mason key.objectid--; 4794925baeddSChris Mason else 47957bb86316SChris Mason return 1; 47967bb86316SChris Mason 4797b3b4aa74SDavid Sterba btrfs_release_path(path); 4798925baeddSChris Mason ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 4799925baeddSChris Mason if (ret < 0) 4800925baeddSChris Mason return ret; 4801925baeddSChris Mason btrfs_item_key(path->nodes[0], &found_key, 0); 4802925baeddSChris Mason ret = comp_keys(&found_key, &key); 4803925baeddSChris Mason if (ret < 0) 48047bb86316SChris Mason return 0; 4805925baeddSChris Mason return 1; 48067bb86316SChris Mason } 48077bb86316SChris Mason 48083f157a2fSChris Mason /* 48093f157a2fSChris Mason * A helper function to walk down the tree starting at min_key, and looking 4810de78b51aSEric Sandeen * for nodes or leaves that are have a minimum transaction id. 4811de78b51aSEric Sandeen * This is used by the btree defrag code, and tree logging 48123f157a2fSChris Mason * 48133f157a2fSChris Mason * This does not cow, but it does stuff the starting key it finds back 48143f157a2fSChris Mason * into min_key, so you can call btrfs_search_slot with cow=1 on the 48153f157a2fSChris Mason * key and get a writable path. 48163f157a2fSChris Mason * 48173f157a2fSChris Mason * This does lock as it descends, and path->keep_locks should be set 48183f157a2fSChris Mason * to 1 by the caller. 48193f157a2fSChris Mason * 48203f157a2fSChris Mason * This honors path->lowest_level to prevent descent past a given level 48213f157a2fSChris Mason * of the tree. 48223f157a2fSChris Mason * 4823d352ac68SChris Mason * min_trans indicates the oldest transaction that you are interested 4824d352ac68SChris Mason * in walking through. Any nodes or leaves older than min_trans are 4825d352ac68SChris Mason * skipped over (without reading them). 4826d352ac68SChris Mason * 48273f157a2fSChris Mason * returns zero if something useful was found, < 0 on error and 1 if there 48283f157a2fSChris Mason * was nothing in the tree that matched the search criteria. 48293f157a2fSChris Mason */ 48303f157a2fSChris Mason int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key, 4831e02119d5SChris Mason struct btrfs_key *max_key, 4832de78b51aSEric Sandeen struct btrfs_path *path, 48333f157a2fSChris Mason u64 min_trans) 48343f157a2fSChris Mason { 48353f157a2fSChris Mason struct extent_buffer *cur; 48363f157a2fSChris Mason struct btrfs_key found_key; 48373f157a2fSChris Mason int slot; 48389652480bSYan int sret; 48393f157a2fSChris Mason u32 nritems; 48403f157a2fSChris Mason int level; 48413f157a2fSChris Mason int ret = 1; 48423f157a2fSChris Mason 4843934d375bSChris Mason WARN_ON(!path->keep_locks); 48443f157a2fSChris Mason again: 4845bd681513SChris Mason cur = btrfs_read_lock_root_node(root); 48463f157a2fSChris Mason level = btrfs_header_level(cur); 4847e02119d5SChris Mason WARN_ON(path->nodes[level]); 48483f157a2fSChris Mason path->nodes[level] = cur; 4849bd681513SChris Mason path->locks[level] = BTRFS_READ_LOCK; 48503f157a2fSChris Mason 48513f157a2fSChris Mason if (btrfs_header_generation(cur) < min_trans) { 48523f157a2fSChris Mason ret = 1; 48533f157a2fSChris Mason goto out; 48543f157a2fSChris Mason } 48553f157a2fSChris Mason while (1) { 48563f157a2fSChris Mason nritems = btrfs_header_nritems(cur); 48573f157a2fSChris Mason level = btrfs_header_level(cur); 48589652480bSYan sret = bin_search(cur, min_key, level, &slot); 48593f157a2fSChris Mason 4860323ac95bSChris Mason /* at the lowest level, we're done, setup the path and exit */ 4861323ac95bSChris Mason if (level == path->lowest_level) { 4862e02119d5SChris Mason if (slot >= nritems) 4863e02119d5SChris Mason goto find_next_key; 48643f157a2fSChris Mason ret = 0; 48653f157a2fSChris Mason path->slots[level] = slot; 48663f157a2fSChris Mason btrfs_item_key_to_cpu(cur, &found_key, slot); 48673f157a2fSChris Mason goto out; 48683f157a2fSChris Mason } 48699652480bSYan if (sret && slot > 0) 48709652480bSYan slot--; 48713f157a2fSChris Mason /* 4872de78b51aSEric Sandeen * check this node pointer against the min_trans parameters. 4873de78b51aSEric Sandeen * If it is too old, old, skip to the next one. 48743f157a2fSChris Mason */ 48753f157a2fSChris Mason while (slot < nritems) { 48763f157a2fSChris Mason u64 blockptr; 48773f157a2fSChris Mason u64 gen; 4878e02119d5SChris Mason 48793f157a2fSChris Mason blockptr = btrfs_node_blockptr(cur, slot); 48803f157a2fSChris Mason gen = btrfs_node_ptr_generation(cur, slot); 48813f157a2fSChris Mason if (gen < min_trans) { 48823f157a2fSChris Mason slot++; 48833f157a2fSChris Mason continue; 48843f157a2fSChris Mason } 48853f157a2fSChris Mason break; 48863f157a2fSChris Mason } 4887e02119d5SChris Mason find_next_key: 48883f157a2fSChris Mason /* 48893f157a2fSChris Mason * we didn't find a candidate key in this node, walk forward 48903f157a2fSChris Mason * and find another one 48913f157a2fSChris Mason */ 48923f157a2fSChris Mason if (slot >= nritems) { 4893e02119d5SChris Mason path->slots[level] = slot; 4894b4ce94deSChris Mason btrfs_set_path_blocking(path); 4895e02119d5SChris Mason sret = btrfs_find_next_key(root, path, min_key, level, 4896de78b51aSEric Sandeen min_trans); 4897e02119d5SChris Mason if (sret == 0) { 4898b3b4aa74SDavid Sterba btrfs_release_path(path); 48993f157a2fSChris Mason goto again; 49003f157a2fSChris Mason } else { 49013f157a2fSChris Mason goto out; 49023f157a2fSChris Mason } 49033f157a2fSChris Mason } 49043f157a2fSChris Mason /* save our key for returning back */ 49053f157a2fSChris Mason btrfs_node_key_to_cpu(cur, &found_key, slot); 49063f157a2fSChris Mason path->slots[level] = slot; 49073f157a2fSChris Mason if (level == path->lowest_level) { 49083f157a2fSChris Mason ret = 0; 4909f7c79f30SChris Mason unlock_up(path, level, 1, 0, NULL); 49103f157a2fSChris Mason goto out; 49113f157a2fSChris Mason } 4912b4ce94deSChris Mason btrfs_set_path_blocking(path); 49133f157a2fSChris Mason cur = read_node_slot(root, cur, slot); 491479787eaaSJeff Mahoney BUG_ON(!cur); /* -ENOMEM */ 49153f157a2fSChris Mason 4916bd681513SChris Mason btrfs_tree_read_lock(cur); 4917b4ce94deSChris Mason 4918bd681513SChris Mason path->locks[level - 1] = BTRFS_READ_LOCK; 49193f157a2fSChris Mason path->nodes[level - 1] = cur; 4920f7c79f30SChris Mason unlock_up(path, level, 1, 0, NULL); 4921bd681513SChris Mason btrfs_clear_path_blocking(path, NULL, 0); 49223f157a2fSChris Mason } 49233f157a2fSChris Mason out: 49243f157a2fSChris Mason if (ret == 0) 49253f157a2fSChris Mason memcpy(min_key, &found_key, sizeof(found_key)); 4926b4ce94deSChris Mason btrfs_set_path_blocking(path); 49273f157a2fSChris Mason return ret; 49283f157a2fSChris Mason } 49293f157a2fSChris Mason 49307069830aSAlexander Block static void tree_move_down(struct btrfs_root *root, 49317069830aSAlexander Block struct btrfs_path *path, 49327069830aSAlexander Block int *level, int root_level) 49337069830aSAlexander Block { 493474dd17fbSChris Mason BUG_ON(*level == 0); 49357069830aSAlexander Block path->nodes[*level - 1] = read_node_slot(root, path->nodes[*level], 49367069830aSAlexander Block path->slots[*level]); 49377069830aSAlexander Block path->slots[*level - 1] = 0; 49387069830aSAlexander Block (*level)--; 49397069830aSAlexander Block } 49407069830aSAlexander Block 49417069830aSAlexander Block static int tree_move_next_or_upnext(struct btrfs_root *root, 49427069830aSAlexander Block struct btrfs_path *path, 49437069830aSAlexander Block int *level, int root_level) 49447069830aSAlexander Block { 49457069830aSAlexander Block int ret = 0; 49467069830aSAlexander Block int nritems; 49477069830aSAlexander Block nritems = btrfs_header_nritems(path->nodes[*level]); 49487069830aSAlexander Block 49497069830aSAlexander Block path->slots[*level]++; 49507069830aSAlexander Block 495174dd17fbSChris Mason while (path->slots[*level] >= nritems) { 49527069830aSAlexander Block if (*level == root_level) 49537069830aSAlexander Block return -1; 49547069830aSAlexander Block 49557069830aSAlexander Block /* move upnext */ 49567069830aSAlexander Block path->slots[*level] = 0; 49577069830aSAlexander Block free_extent_buffer(path->nodes[*level]); 49587069830aSAlexander Block path->nodes[*level] = NULL; 49597069830aSAlexander Block (*level)++; 49607069830aSAlexander Block path->slots[*level]++; 49617069830aSAlexander Block 49627069830aSAlexander Block nritems = btrfs_header_nritems(path->nodes[*level]); 49637069830aSAlexander Block ret = 1; 49647069830aSAlexander Block } 49657069830aSAlexander Block return ret; 49667069830aSAlexander Block } 49677069830aSAlexander Block 49687069830aSAlexander Block /* 49697069830aSAlexander Block * Returns 1 if it had to move up and next. 0 is returned if it moved only next 49707069830aSAlexander Block * or down. 49717069830aSAlexander Block */ 49727069830aSAlexander Block static int tree_advance(struct btrfs_root *root, 49737069830aSAlexander Block struct btrfs_path *path, 49747069830aSAlexander Block int *level, int root_level, 49757069830aSAlexander Block int allow_down, 49767069830aSAlexander Block struct btrfs_key *key) 49777069830aSAlexander Block { 49787069830aSAlexander Block int ret; 49797069830aSAlexander Block 49807069830aSAlexander Block if (*level == 0 || !allow_down) { 49817069830aSAlexander Block ret = tree_move_next_or_upnext(root, path, level, root_level); 49827069830aSAlexander Block } else { 49837069830aSAlexander Block tree_move_down(root, path, level, root_level); 49847069830aSAlexander Block ret = 0; 49857069830aSAlexander Block } 49867069830aSAlexander Block if (ret >= 0) { 49877069830aSAlexander Block if (*level == 0) 49887069830aSAlexander Block btrfs_item_key_to_cpu(path->nodes[*level], key, 49897069830aSAlexander Block path->slots[*level]); 49907069830aSAlexander Block else 49917069830aSAlexander Block btrfs_node_key_to_cpu(path->nodes[*level], key, 49927069830aSAlexander Block path->slots[*level]); 49937069830aSAlexander Block } 49947069830aSAlexander Block return ret; 49957069830aSAlexander Block } 49967069830aSAlexander Block 49977069830aSAlexander Block static int tree_compare_item(struct btrfs_root *left_root, 49987069830aSAlexander Block struct btrfs_path *left_path, 49997069830aSAlexander Block struct btrfs_path *right_path, 50007069830aSAlexander Block char *tmp_buf) 50017069830aSAlexander Block { 50027069830aSAlexander Block int cmp; 50037069830aSAlexander Block int len1, len2; 50047069830aSAlexander Block unsigned long off1, off2; 50057069830aSAlexander Block 50067069830aSAlexander Block len1 = btrfs_item_size_nr(left_path->nodes[0], left_path->slots[0]); 50077069830aSAlexander Block len2 = btrfs_item_size_nr(right_path->nodes[0], right_path->slots[0]); 50087069830aSAlexander Block if (len1 != len2) 50097069830aSAlexander Block return 1; 50107069830aSAlexander Block 50117069830aSAlexander Block off1 = btrfs_item_ptr_offset(left_path->nodes[0], left_path->slots[0]); 50127069830aSAlexander Block off2 = btrfs_item_ptr_offset(right_path->nodes[0], 50137069830aSAlexander Block right_path->slots[0]); 50147069830aSAlexander Block 50157069830aSAlexander Block read_extent_buffer(left_path->nodes[0], tmp_buf, off1, len1); 50167069830aSAlexander Block 50177069830aSAlexander Block cmp = memcmp_extent_buffer(right_path->nodes[0], tmp_buf, off2, len1); 50187069830aSAlexander Block if (cmp) 50197069830aSAlexander Block return 1; 50207069830aSAlexander Block return 0; 50217069830aSAlexander Block } 50227069830aSAlexander Block 50237069830aSAlexander Block #define ADVANCE 1 50247069830aSAlexander Block #define ADVANCE_ONLY_NEXT -1 50257069830aSAlexander Block 50267069830aSAlexander Block /* 50277069830aSAlexander Block * This function compares two trees and calls the provided callback for 50287069830aSAlexander Block * every changed/new/deleted item it finds. 50297069830aSAlexander Block * If shared tree blocks are encountered, whole subtrees are skipped, making 50307069830aSAlexander Block * the compare pretty fast on snapshotted subvolumes. 50317069830aSAlexander Block * 50327069830aSAlexander Block * This currently works on commit roots only. As commit roots are read only, 50337069830aSAlexander Block * we don't do any locking. The commit roots are protected with transactions. 50347069830aSAlexander Block * Transactions are ended and rejoined when a commit is tried in between. 50357069830aSAlexander Block * 50367069830aSAlexander Block * This function checks for modifications done to the trees while comparing. 50377069830aSAlexander Block * If it detects a change, it aborts immediately. 50387069830aSAlexander Block */ 50397069830aSAlexander Block int btrfs_compare_trees(struct btrfs_root *left_root, 50407069830aSAlexander Block struct btrfs_root *right_root, 50417069830aSAlexander Block btrfs_changed_cb_t changed_cb, void *ctx) 50427069830aSAlexander Block { 50437069830aSAlexander Block int ret; 50447069830aSAlexander Block int cmp; 50457069830aSAlexander Block struct btrfs_trans_handle *trans = NULL; 50467069830aSAlexander Block struct btrfs_path *left_path = NULL; 50477069830aSAlexander Block struct btrfs_path *right_path = NULL; 50487069830aSAlexander Block struct btrfs_key left_key; 50497069830aSAlexander Block struct btrfs_key right_key; 50507069830aSAlexander Block char *tmp_buf = NULL; 50517069830aSAlexander Block int left_root_level; 50527069830aSAlexander Block int right_root_level; 50537069830aSAlexander Block int left_level; 50547069830aSAlexander Block int right_level; 50557069830aSAlexander Block int left_end_reached; 50567069830aSAlexander Block int right_end_reached; 50577069830aSAlexander Block int advance_left; 50587069830aSAlexander Block int advance_right; 50597069830aSAlexander Block u64 left_blockptr; 50607069830aSAlexander Block u64 right_blockptr; 50617069830aSAlexander Block u64 left_start_ctransid; 50627069830aSAlexander Block u64 right_start_ctransid; 50637069830aSAlexander Block u64 ctransid; 50647069830aSAlexander Block 50657069830aSAlexander Block left_path = btrfs_alloc_path(); 50667069830aSAlexander Block if (!left_path) { 50677069830aSAlexander Block ret = -ENOMEM; 50687069830aSAlexander Block goto out; 50697069830aSAlexander Block } 50707069830aSAlexander Block right_path = btrfs_alloc_path(); 50717069830aSAlexander Block if (!right_path) { 50727069830aSAlexander Block ret = -ENOMEM; 50737069830aSAlexander Block goto out; 50747069830aSAlexander Block } 50757069830aSAlexander Block 50767069830aSAlexander Block tmp_buf = kmalloc(left_root->leafsize, GFP_NOFS); 50777069830aSAlexander Block if (!tmp_buf) { 50787069830aSAlexander Block ret = -ENOMEM; 50797069830aSAlexander Block goto out; 50807069830aSAlexander Block } 50817069830aSAlexander Block 50827069830aSAlexander Block left_path->search_commit_root = 1; 50837069830aSAlexander Block left_path->skip_locking = 1; 50847069830aSAlexander Block right_path->search_commit_root = 1; 50857069830aSAlexander Block right_path->skip_locking = 1; 50867069830aSAlexander Block 50875f3ab90aSAnand Jain spin_lock(&left_root->root_item_lock); 50887069830aSAlexander Block left_start_ctransid = btrfs_root_ctransid(&left_root->root_item); 50895f3ab90aSAnand Jain spin_unlock(&left_root->root_item_lock); 50907069830aSAlexander Block 50915f3ab90aSAnand Jain spin_lock(&right_root->root_item_lock); 50927069830aSAlexander Block right_start_ctransid = btrfs_root_ctransid(&right_root->root_item); 50935f3ab90aSAnand Jain spin_unlock(&right_root->root_item_lock); 50947069830aSAlexander Block 50957069830aSAlexander Block trans = btrfs_join_transaction(left_root); 50967069830aSAlexander Block if (IS_ERR(trans)) { 50977069830aSAlexander Block ret = PTR_ERR(trans); 50987069830aSAlexander Block trans = NULL; 50997069830aSAlexander Block goto out; 51007069830aSAlexander Block } 51017069830aSAlexander Block 51027069830aSAlexander Block /* 51037069830aSAlexander Block * Strategy: Go to the first items of both trees. Then do 51047069830aSAlexander Block * 51057069830aSAlexander Block * If both trees are at level 0 51067069830aSAlexander Block * Compare keys of current items 51077069830aSAlexander Block * If left < right treat left item as new, advance left tree 51087069830aSAlexander Block * and repeat 51097069830aSAlexander Block * If left > right treat right item as deleted, advance right tree 51107069830aSAlexander Block * and repeat 51117069830aSAlexander Block * If left == right do deep compare of items, treat as changed if 51127069830aSAlexander Block * needed, advance both trees and repeat 51137069830aSAlexander Block * If both trees are at the same level but not at level 0 51147069830aSAlexander Block * Compare keys of current nodes/leafs 51157069830aSAlexander Block * If left < right advance left tree and repeat 51167069830aSAlexander Block * If left > right advance right tree and repeat 51177069830aSAlexander Block * If left == right compare blockptrs of the next nodes/leafs 51187069830aSAlexander Block * If they match advance both trees but stay at the same level 51197069830aSAlexander Block * and repeat 51207069830aSAlexander Block * If they don't match advance both trees while allowing to go 51217069830aSAlexander Block * deeper and repeat 51227069830aSAlexander Block * If tree levels are different 51237069830aSAlexander Block * Advance the tree that needs it and repeat 51247069830aSAlexander Block * 51257069830aSAlexander Block * Advancing a tree means: 51267069830aSAlexander Block * If we are at level 0, try to go to the next slot. If that's not 51277069830aSAlexander Block * possible, go one level up and repeat. Stop when we found a level 51287069830aSAlexander Block * where we could go to the next slot. We may at this point be on a 51297069830aSAlexander Block * node or a leaf. 51307069830aSAlexander Block * 51317069830aSAlexander Block * If we are not at level 0 and not on shared tree blocks, go one 51327069830aSAlexander Block * level deeper. 51337069830aSAlexander Block * 51347069830aSAlexander Block * If we are not at level 0 and on shared tree blocks, go one slot to 51357069830aSAlexander Block * the right if possible or go up and right. 51367069830aSAlexander Block */ 51377069830aSAlexander Block 51387069830aSAlexander Block left_level = btrfs_header_level(left_root->commit_root); 51397069830aSAlexander Block left_root_level = left_level; 51407069830aSAlexander Block left_path->nodes[left_level] = left_root->commit_root; 51417069830aSAlexander Block extent_buffer_get(left_path->nodes[left_level]); 51427069830aSAlexander Block 51437069830aSAlexander Block right_level = btrfs_header_level(right_root->commit_root); 51447069830aSAlexander Block right_root_level = right_level; 51457069830aSAlexander Block right_path->nodes[right_level] = right_root->commit_root; 51467069830aSAlexander Block extent_buffer_get(right_path->nodes[right_level]); 51477069830aSAlexander Block 51487069830aSAlexander Block if (left_level == 0) 51497069830aSAlexander Block btrfs_item_key_to_cpu(left_path->nodes[left_level], 51507069830aSAlexander Block &left_key, left_path->slots[left_level]); 51517069830aSAlexander Block else 51527069830aSAlexander Block btrfs_node_key_to_cpu(left_path->nodes[left_level], 51537069830aSAlexander Block &left_key, left_path->slots[left_level]); 51547069830aSAlexander Block if (right_level == 0) 51557069830aSAlexander Block btrfs_item_key_to_cpu(right_path->nodes[right_level], 51567069830aSAlexander Block &right_key, right_path->slots[right_level]); 51577069830aSAlexander Block else 51587069830aSAlexander Block btrfs_node_key_to_cpu(right_path->nodes[right_level], 51597069830aSAlexander Block &right_key, right_path->slots[right_level]); 51607069830aSAlexander Block 51617069830aSAlexander Block left_end_reached = right_end_reached = 0; 51627069830aSAlexander Block advance_left = advance_right = 0; 51637069830aSAlexander Block 51647069830aSAlexander Block while (1) { 51657069830aSAlexander Block /* 51667069830aSAlexander Block * We need to make sure the transaction does not get committed 51677069830aSAlexander Block * while we do anything on commit roots. This means, we need to 51687069830aSAlexander Block * join and leave transactions for every item that we process. 51697069830aSAlexander Block */ 51707069830aSAlexander Block if (trans && btrfs_should_end_transaction(trans, left_root)) { 51717069830aSAlexander Block btrfs_release_path(left_path); 51727069830aSAlexander Block btrfs_release_path(right_path); 51737069830aSAlexander Block 51747069830aSAlexander Block ret = btrfs_end_transaction(trans, left_root); 51757069830aSAlexander Block trans = NULL; 51767069830aSAlexander Block if (ret < 0) 51777069830aSAlexander Block goto out; 51787069830aSAlexander Block } 51797069830aSAlexander Block /* now rejoin the transaction */ 51807069830aSAlexander Block if (!trans) { 51817069830aSAlexander Block trans = btrfs_join_transaction(left_root); 51827069830aSAlexander Block if (IS_ERR(trans)) { 51837069830aSAlexander Block ret = PTR_ERR(trans); 51847069830aSAlexander Block trans = NULL; 51857069830aSAlexander Block goto out; 51867069830aSAlexander Block } 51877069830aSAlexander Block 51885f3ab90aSAnand Jain spin_lock(&left_root->root_item_lock); 51897069830aSAlexander Block ctransid = btrfs_root_ctransid(&left_root->root_item); 51905f3ab90aSAnand Jain spin_unlock(&left_root->root_item_lock); 51917069830aSAlexander Block if (ctransid != left_start_ctransid) 51927069830aSAlexander Block left_start_ctransid = 0; 51937069830aSAlexander Block 51945f3ab90aSAnand Jain spin_lock(&right_root->root_item_lock); 51957069830aSAlexander Block ctransid = btrfs_root_ctransid(&right_root->root_item); 51965f3ab90aSAnand Jain spin_unlock(&right_root->root_item_lock); 51977069830aSAlexander Block if (ctransid != right_start_ctransid) 51987069830aSAlexander Block right_start_ctransid = 0; 51997069830aSAlexander Block 52007069830aSAlexander Block if (!left_start_ctransid || !right_start_ctransid) { 52017069830aSAlexander Block WARN(1, KERN_WARNING 52027069830aSAlexander Block "btrfs: btrfs_compare_tree detected " 52037069830aSAlexander Block "a change in one of the trees while " 52047069830aSAlexander Block "iterating. This is probably a " 52057069830aSAlexander Block "bug.\n"); 52067069830aSAlexander Block ret = -EIO; 52077069830aSAlexander Block goto out; 52087069830aSAlexander Block } 52097069830aSAlexander Block 52107069830aSAlexander Block /* 52117069830aSAlexander Block * the commit root may have changed, so start again 52127069830aSAlexander Block * where we stopped 52137069830aSAlexander Block */ 52147069830aSAlexander Block left_path->lowest_level = left_level; 52157069830aSAlexander Block right_path->lowest_level = right_level; 52167069830aSAlexander Block ret = btrfs_search_slot(NULL, left_root, 52177069830aSAlexander Block &left_key, left_path, 0, 0); 52187069830aSAlexander Block if (ret < 0) 52197069830aSAlexander Block goto out; 52207069830aSAlexander Block ret = btrfs_search_slot(NULL, right_root, 52217069830aSAlexander Block &right_key, right_path, 0, 0); 52227069830aSAlexander Block if (ret < 0) 52237069830aSAlexander Block goto out; 52247069830aSAlexander Block } 52257069830aSAlexander Block 52267069830aSAlexander Block if (advance_left && !left_end_reached) { 52277069830aSAlexander Block ret = tree_advance(left_root, left_path, &left_level, 52287069830aSAlexander Block left_root_level, 52297069830aSAlexander Block advance_left != ADVANCE_ONLY_NEXT, 52307069830aSAlexander Block &left_key); 52317069830aSAlexander Block if (ret < 0) 52327069830aSAlexander Block left_end_reached = ADVANCE; 52337069830aSAlexander Block advance_left = 0; 52347069830aSAlexander Block } 52357069830aSAlexander Block if (advance_right && !right_end_reached) { 52367069830aSAlexander Block ret = tree_advance(right_root, right_path, &right_level, 52377069830aSAlexander Block right_root_level, 52387069830aSAlexander Block advance_right != ADVANCE_ONLY_NEXT, 52397069830aSAlexander Block &right_key); 52407069830aSAlexander Block if (ret < 0) 52417069830aSAlexander Block right_end_reached = ADVANCE; 52427069830aSAlexander Block advance_right = 0; 52437069830aSAlexander Block } 52447069830aSAlexander Block 52457069830aSAlexander Block if (left_end_reached && right_end_reached) { 52467069830aSAlexander Block ret = 0; 52477069830aSAlexander Block goto out; 52487069830aSAlexander Block } else if (left_end_reached) { 52497069830aSAlexander Block if (right_level == 0) { 52507069830aSAlexander Block ret = changed_cb(left_root, right_root, 52517069830aSAlexander Block left_path, right_path, 52527069830aSAlexander Block &right_key, 52537069830aSAlexander Block BTRFS_COMPARE_TREE_DELETED, 52547069830aSAlexander Block ctx); 52557069830aSAlexander Block if (ret < 0) 52567069830aSAlexander Block goto out; 52577069830aSAlexander Block } 52587069830aSAlexander Block advance_right = ADVANCE; 52597069830aSAlexander Block continue; 52607069830aSAlexander Block } else if (right_end_reached) { 52617069830aSAlexander Block if (left_level == 0) { 52627069830aSAlexander Block ret = changed_cb(left_root, right_root, 52637069830aSAlexander Block left_path, right_path, 52647069830aSAlexander Block &left_key, 52657069830aSAlexander Block BTRFS_COMPARE_TREE_NEW, 52667069830aSAlexander Block ctx); 52677069830aSAlexander Block if (ret < 0) 52687069830aSAlexander Block goto out; 52697069830aSAlexander Block } 52707069830aSAlexander Block advance_left = ADVANCE; 52717069830aSAlexander Block continue; 52727069830aSAlexander Block } 52737069830aSAlexander Block 52747069830aSAlexander Block if (left_level == 0 && right_level == 0) { 52757069830aSAlexander Block cmp = btrfs_comp_cpu_keys(&left_key, &right_key); 52767069830aSAlexander Block if (cmp < 0) { 52777069830aSAlexander Block ret = changed_cb(left_root, right_root, 52787069830aSAlexander Block left_path, right_path, 52797069830aSAlexander Block &left_key, 52807069830aSAlexander Block BTRFS_COMPARE_TREE_NEW, 52817069830aSAlexander Block ctx); 52827069830aSAlexander Block if (ret < 0) 52837069830aSAlexander Block goto out; 52847069830aSAlexander Block advance_left = ADVANCE; 52857069830aSAlexander Block } else if (cmp > 0) { 52867069830aSAlexander Block ret = changed_cb(left_root, right_root, 52877069830aSAlexander Block left_path, right_path, 52887069830aSAlexander Block &right_key, 52897069830aSAlexander Block BTRFS_COMPARE_TREE_DELETED, 52907069830aSAlexander Block ctx); 52917069830aSAlexander Block if (ret < 0) 52927069830aSAlexander Block goto out; 52937069830aSAlexander Block advance_right = ADVANCE; 52947069830aSAlexander Block } else { 529574dd17fbSChris Mason WARN_ON(!extent_buffer_uptodate(left_path->nodes[0])); 52967069830aSAlexander Block ret = tree_compare_item(left_root, left_path, 52977069830aSAlexander Block right_path, tmp_buf); 52987069830aSAlexander Block if (ret) { 529974dd17fbSChris Mason WARN_ON(!extent_buffer_uptodate(left_path->nodes[0])); 53007069830aSAlexander Block ret = changed_cb(left_root, right_root, 53017069830aSAlexander Block left_path, right_path, 53027069830aSAlexander Block &left_key, 53037069830aSAlexander Block BTRFS_COMPARE_TREE_CHANGED, 53047069830aSAlexander Block ctx); 53057069830aSAlexander Block if (ret < 0) 53067069830aSAlexander Block goto out; 53077069830aSAlexander Block } 53087069830aSAlexander Block advance_left = ADVANCE; 53097069830aSAlexander Block advance_right = ADVANCE; 53107069830aSAlexander Block } 53117069830aSAlexander Block } else if (left_level == right_level) { 53127069830aSAlexander Block cmp = btrfs_comp_cpu_keys(&left_key, &right_key); 53137069830aSAlexander Block if (cmp < 0) { 53147069830aSAlexander Block advance_left = ADVANCE; 53157069830aSAlexander Block } else if (cmp > 0) { 53167069830aSAlexander Block advance_right = ADVANCE; 53177069830aSAlexander Block } else { 53187069830aSAlexander Block left_blockptr = btrfs_node_blockptr( 53197069830aSAlexander Block left_path->nodes[left_level], 53207069830aSAlexander Block left_path->slots[left_level]); 53217069830aSAlexander Block right_blockptr = btrfs_node_blockptr( 53227069830aSAlexander Block right_path->nodes[right_level], 53237069830aSAlexander Block right_path->slots[right_level]); 53247069830aSAlexander Block if (left_blockptr == right_blockptr) { 53257069830aSAlexander Block /* 53267069830aSAlexander Block * As we're on a shared block, don't 53277069830aSAlexander Block * allow to go deeper. 53287069830aSAlexander Block */ 53297069830aSAlexander Block advance_left = ADVANCE_ONLY_NEXT; 53307069830aSAlexander Block advance_right = ADVANCE_ONLY_NEXT; 53317069830aSAlexander Block } else { 53327069830aSAlexander Block advance_left = ADVANCE; 53337069830aSAlexander Block advance_right = ADVANCE; 53347069830aSAlexander Block } 53357069830aSAlexander Block } 53367069830aSAlexander Block } else if (left_level < right_level) { 53377069830aSAlexander Block advance_right = ADVANCE; 53387069830aSAlexander Block } else { 53397069830aSAlexander Block advance_left = ADVANCE; 53407069830aSAlexander Block } 53417069830aSAlexander Block } 53427069830aSAlexander Block 53437069830aSAlexander Block out: 53447069830aSAlexander Block btrfs_free_path(left_path); 53457069830aSAlexander Block btrfs_free_path(right_path); 53467069830aSAlexander Block kfree(tmp_buf); 53477069830aSAlexander Block 53487069830aSAlexander Block if (trans) { 53497069830aSAlexander Block if (!ret) 53507069830aSAlexander Block ret = btrfs_end_transaction(trans, left_root); 53517069830aSAlexander Block else 53527069830aSAlexander Block btrfs_end_transaction(trans, left_root); 53537069830aSAlexander Block } 53547069830aSAlexander Block 53557069830aSAlexander Block return ret; 53567069830aSAlexander Block } 53577069830aSAlexander Block 53583f157a2fSChris Mason /* 53593f157a2fSChris Mason * this is similar to btrfs_next_leaf, but does not try to preserve 53603f157a2fSChris Mason * and fixup the path. It looks for and returns the next key in the 5361de78b51aSEric Sandeen * tree based on the current path and the min_trans parameters. 53623f157a2fSChris Mason * 53633f157a2fSChris Mason * 0 is returned if another key is found, < 0 if there are any errors 53643f157a2fSChris Mason * and 1 is returned if there are no higher keys in the tree 53653f157a2fSChris Mason * 53663f157a2fSChris Mason * path->keep_locks should be set to 1 on the search made before 53673f157a2fSChris Mason * calling this function. 53683f157a2fSChris Mason */ 5369e7a84565SChris Mason int btrfs_find_next_key(struct btrfs_root *root, struct btrfs_path *path, 5370de78b51aSEric Sandeen struct btrfs_key *key, int level, u64 min_trans) 5371e7a84565SChris Mason { 5372e7a84565SChris Mason int slot; 5373e7a84565SChris Mason struct extent_buffer *c; 5374e7a84565SChris Mason 5375934d375bSChris Mason WARN_ON(!path->keep_locks); 5376e7a84565SChris Mason while (level < BTRFS_MAX_LEVEL) { 5377e7a84565SChris Mason if (!path->nodes[level]) 5378e7a84565SChris Mason return 1; 5379e7a84565SChris Mason 5380e7a84565SChris Mason slot = path->slots[level] + 1; 5381e7a84565SChris Mason c = path->nodes[level]; 53823f157a2fSChris Mason next: 5383e7a84565SChris Mason if (slot >= btrfs_header_nritems(c)) { 538433c66f43SYan Zheng int ret; 538533c66f43SYan Zheng int orig_lowest; 538633c66f43SYan Zheng struct btrfs_key cur_key; 538733c66f43SYan Zheng if (level + 1 >= BTRFS_MAX_LEVEL || 538833c66f43SYan Zheng !path->nodes[level + 1]) 5389e7a84565SChris Mason return 1; 539033c66f43SYan Zheng 539133c66f43SYan Zheng if (path->locks[level + 1]) { 539233c66f43SYan Zheng level++; 5393e7a84565SChris Mason continue; 5394e7a84565SChris Mason } 539533c66f43SYan Zheng 539633c66f43SYan Zheng slot = btrfs_header_nritems(c) - 1; 539733c66f43SYan Zheng if (level == 0) 539833c66f43SYan Zheng btrfs_item_key_to_cpu(c, &cur_key, slot); 539933c66f43SYan Zheng else 540033c66f43SYan Zheng btrfs_node_key_to_cpu(c, &cur_key, slot); 540133c66f43SYan Zheng 540233c66f43SYan Zheng orig_lowest = path->lowest_level; 5403b3b4aa74SDavid Sterba btrfs_release_path(path); 540433c66f43SYan Zheng path->lowest_level = level; 540533c66f43SYan Zheng ret = btrfs_search_slot(NULL, root, &cur_key, path, 540633c66f43SYan Zheng 0, 0); 540733c66f43SYan Zheng path->lowest_level = orig_lowest; 540833c66f43SYan Zheng if (ret < 0) 540933c66f43SYan Zheng return ret; 541033c66f43SYan Zheng 541133c66f43SYan Zheng c = path->nodes[level]; 541233c66f43SYan Zheng slot = path->slots[level]; 541333c66f43SYan Zheng if (ret == 0) 541433c66f43SYan Zheng slot++; 541533c66f43SYan Zheng goto next; 541633c66f43SYan Zheng } 541733c66f43SYan Zheng 5418e7a84565SChris Mason if (level == 0) 5419e7a84565SChris Mason btrfs_item_key_to_cpu(c, key, slot); 54203f157a2fSChris Mason else { 54213f157a2fSChris Mason u64 gen = btrfs_node_ptr_generation(c, slot); 54223f157a2fSChris Mason 54233f157a2fSChris Mason if (gen < min_trans) { 54243f157a2fSChris Mason slot++; 54253f157a2fSChris Mason goto next; 54263f157a2fSChris Mason } 5427e7a84565SChris Mason btrfs_node_key_to_cpu(c, key, slot); 54283f157a2fSChris Mason } 5429e7a84565SChris Mason return 0; 5430e7a84565SChris Mason } 5431e7a84565SChris Mason return 1; 5432e7a84565SChris Mason } 5433e7a84565SChris Mason 54347bb86316SChris Mason /* 5435925baeddSChris Mason * search the tree again to find a leaf with greater keys 54360f70abe2SChris Mason * returns 0 if it found something or 1 if there are no greater leaves. 54370f70abe2SChris Mason * returns < 0 on io errors. 543897571fd0SChris Mason */ 5439234b63a0SChris Mason int btrfs_next_leaf(struct btrfs_root *root, struct btrfs_path *path) 5440d97e63b6SChris Mason { 54413d7806ecSJan Schmidt return btrfs_next_old_leaf(root, path, 0); 54423d7806ecSJan Schmidt } 54433d7806ecSJan Schmidt 54443d7806ecSJan Schmidt int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path, 54453d7806ecSJan Schmidt u64 time_seq) 54463d7806ecSJan Schmidt { 5447d97e63b6SChris Mason int slot; 54488e73f275SChris Mason int level; 54495f39d397SChris Mason struct extent_buffer *c; 54508e73f275SChris Mason struct extent_buffer *next; 5451925baeddSChris Mason struct btrfs_key key; 5452925baeddSChris Mason u32 nritems; 5453925baeddSChris Mason int ret; 54548e73f275SChris Mason int old_spinning = path->leave_spinning; 5455bd681513SChris Mason int next_rw_lock = 0; 5456925baeddSChris Mason 5457925baeddSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 5458d397712bSChris Mason if (nritems == 0) 5459925baeddSChris Mason return 1; 5460925baeddSChris Mason 54618e73f275SChris Mason btrfs_item_key_to_cpu(path->nodes[0], &key, nritems - 1); 54628e73f275SChris Mason again: 54638e73f275SChris Mason level = 1; 54648e73f275SChris Mason next = NULL; 5465bd681513SChris Mason next_rw_lock = 0; 5466b3b4aa74SDavid Sterba btrfs_release_path(path); 54678e73f275SChris Mason 5468a2135011SChris Mason path->keep_locks = 1; 54698e73f275SChris Mason path->leave_spinning = 1; 54708e73f275SChris Mason 54713d7806ecSJan Schmidt if (time_seq) 54723d7806ecSJan Schmidt ret = btrfs_search_old_slot(root, &key, path, time_seq); 54733d7806ecSJan Schmidt else 5474925baeddSChris Mason ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 5475925baeddSChris Mason path->keep_locks = 0; 5476925baeddSChris Mason 5477925baeddSChris Mason if (ret < 0) 5478925baeddSChris Mason return ret; 5479925baeddSChris Mason 5480a2135011SChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 5481168fd7d2SChris Mason /* 5482168fd7d2SChris Mason * by releasing the path above we dropped all our locks. A balance 5483168fd7d2SChris Mason * could have added more items next to the key that used to be 5484168fd7d2SChris Mason * at the very end of the block. So, check again here and 5485168fd7d2SChris Mason * advance the path if there are now more items available. 5486168fd7d2SChris Mason */ 5487a2135011SChris Mason if (nritems > 0 && path->slots[0] < nritems - 1) { 5488e457afecSYan Zheng if (ret == 0) 5489168fd7d2SChris Mason path->slots[0]++; 54908e73f275SChris Mason ret = 0; 5491925baeddSChris Mason goto done; 5492925baeddSChris Mason } 5493d97e63b6SChris Mason 5494234b63a0SChris Mason while (level < BTRFS_MAX_LEVEL) { 54958e73f275SChris Mason if (!path->nodes[level]) { 54968e73f275SChris Mason ret = 1; 54978e73f275SChris Mason goto done; 54988e73f275SChris Mason } 54995f39d397SChris Mason 5500d97e63b6SChris Mason slot = path->slots[level] + 1; 5501d97e63b6SChris Mason c = path->nodes[level]; 55025f39d397SChris Mason if (slot >= btrfs_header_nritems(c)) { 5503d97e63b6SChris Mason level++; 55048e73f275SChris Mason if (level == BTRFS_MAX_LEVEL) { 55058e73f275SChris Mason ret = 1; 55068e73f275SChris Mason goto done; 55078e73f275SChris Mason } 5508d97e63b6SChris Mason continue; 5509d97e63b6SChris Mason } 55105f39d397SChris Mason 5511925baeddSChris Mason if (next) { 5512bd681513SChris Mason btrfs_tree_unlock_rw(next, next_rw_lock); 55135f39d397SChris Mason free_extent_buffer(next); 5514925baeddSChris Mason } 55155f39d397SChris Mason 55168e73f275SChris Mason next = c; 5517bd681513SChris Mason next_rw_lock = path->locks[level]; 55188e73f275SChris Mason ret = read_block_for_search(NULL, root, path, &next, level, 55195d9e75c4SJan Schmidt slot, &key, 0); 55208e73f275SChris Mason if (ret == -EAGAIN) 55218e73f275SChris Mason goto again; 55225f39d397SChris Mason 552376a05b35SChris Mason if (ret < 0) { 5524b3b4aa74SDavid Sterba btrfs_release_path(path); 552576a05b35SChris Mason goto done; 552676a05b35SChris Mason } 552776a05b35SChris Mason 55285cd57b2cSChris Mason if (!path->skip_locking) { 5529bd681513SChris Mason ret = btrfs_try_tree_read_lock(next); 5530d42244a0SJan Schmidt if (!ret && time_seq) { 5531d42244a0SJan Schmidt /* 5532d42244a0SJan Schmidt * If we don't get the lock, we may be racing 5533d42244a0SJan Schmidt * with push_leaf_left, holding that lock while 5534d42244a0SJan Schmidt * itself waiting for the leaf we've currently 5535d42244a0SJan Schmidt * locked. To solve this situation, we give up 5536d42244a0SJan Schmidt * on our lock and cycle. 5537d42244a0SJan Schmidt */ 5538cf538830SJan Schmidt free_extent_buffer(next); 5539d42244a0SJan Schmidt btrfs_release_path(path); 5540d42244a0SJan Schmidt cond_resched(); 5541d42244a0SJan Schmidt goto again; 5542d42244a0SJan Schmidt } 55438e73f275SChris Mason if (!ret) { 55448e73f275SChris Mason btrfs_set_path_blocking(path); 5545bd681513SChris Mason btrfs_tree_read_lock(next); 5546bd681513SChris Mason btrfs_clear_path_blocking(path, next, 5547bd681513SChris Mason BTRFS_READ_LOCK); 55488e73f275SChris Mason } 5549bd681513SChris Mason next_rw_lock = BTRFS_READ_LOCK; 5550bd681513SChris Mason } 5551d97e63b6SChris Mason break; 5552d97e63b6SChris Mason } 5553d97e63b6SChris Mason path->slots[level] = slot; 5554d97e63b6SChris Mason while (1) { 5555d97e63b6SChris Mason level--; 5556d97e63b6SChris Mason c = path->nodes[level]; 5557925baeddSChris Mason if (path->locks[level]) 5558bd681513SChris Mason btrfs_tree_unlock_rw(c, path->locks[level]); 55598e73f275SChris Mason 55605f39d397SChris Mason free_extent_buffer(c); 5561d97e63b6SChris Mason path->nodes[level] = next; 5562d97e63b6SChris Mason path->slots[level] = 0; 5563a74a4b97SChris Mason if (!path->skip_locking) 5564bd681513SChris Mason path->locks[level] = next_rw_lock; 5565d97e63b6SChris Mason if (!level) 5566d97e63b6SChris Mason break; 5567b4ce94deSChris Mason 55688e73f275SChris Mason ret = read_block_for_search(NULL, root, path, &next, level, 55695d9e75c4SJan Schmidt 0, &key, 0); 55708e73f275SChris Mason if (ret == -EAGAIN) 55718e73f275SChris Mason goto again; 55728e73f275SChris Mason 557376a05b35SChris Mason if (ret < 0) { 5574b3b4aa74SDavid Sterba btrfs_release_path(path); 557576a05b35SChris Mason goto done; 557676a05b35SChris Mason } 557776a05b35SChris Mason 55785cd57b2cSChris Mason if (!path->skip_locking) { 5579bd681513SChris Mason ret = btrfs_try_tree_read_lock(next); 55808e73f275SChris Mason if (!ret) { 55818e73f275SChris Mason btrfs_set_path_blocking(path); 5582bd681513SChris Mason btrfs_tree_read_lock(next); 5583bd681513SChris Mason btrfs_clear_path_blocking(path, next, 5584bd681513SChris Mason BTRFS_READ_LOCK); 55858e73f275SChris Mason } 5586bd681513SChris Mason next_rw_lock = BTRFS_READ_LOCK; 5587bd681513SChris Mason } 5588d97e63b6SChris Mason } 55898e73f275SChris Mason ret = 0; 5590925baeddSChris Mason done: 5591f7c79f30SChris Mason unlock_up(path, 0, 1, 0, NULL); 55928e73f275SChris Mason path->leave_spinning = old_spinning; 55938e73f275SChris Mason if (!old_spinning) 55948e73f275SChris Mason btrfs_set_path_blocking(path); 55958e73f275SChris Mason 55968e73f275SChris Mason return ret; 5597d97e63b6SChris Mason } 55980b86a832SChris Mason 55993f157a2fSChris Mason /* 56003f157a2fSChris Mason * this uses btrfs_prev_leaf to walk backwards in the tree, and keeps 56013f157a2fSChris Mason * searching until it gets past min_objectid or finds an item of 'type' 56023f157a2fSChris Mason * 56033f157a2fSChris Mason * returns 0 if something is found, 1 if nothing was found and < 0 on error 56043f157a2fSChris Mason */ 56050b86a832SChris Mason int btrfs_previous_item(struct btrfs_root *root, 56060b86a832SChris Mason struct btrfs_path *path, u64 min_objectid, 56070b86a832SChris Mason int type) 56080b86a832SChris Mason { 56090b86a832SChris Mason struct btrfs_key found_key; 56100b86a832SChris Mason struct extent_buffer *leaf; 5611e02119d5SChris Mason u32 nritems; 56120b86a832SChris Mason int ret; 56130b86a832SChris Mason 56140b86a832SChris Mason while (1) { 56150b86a832SChris Mason if (path->slots[0] == 0) { 5616b4ce94deSChris Mason btrfs_set_path_blocking(path); 56170b86a832SChris Mason ret = btrfs_prev_leaf(root, path); 56180b86a832SChris Mason if (ret != 0) 56190b86a832SChris Mason return ret; 56200b86a832SChris Mason } else { 56210b86a832SChris Mason path->slots[0]--; 56220b86a832SChris Mason } 56230b86a832SChris Mason leaf = path->nodes[0]; 5624e02119d5SChris Mason nritems = btrfs_header_nritems(leaf); 5625e02119d5SChris Mason if (nritems == 0) 5626e02119d5SChris Mason return 1; 5627e02119d5SChris Mason if (path->slots[0] == nritems) 5628e02119d5SChris Mason path->slots[0]--; 5629e02119d5SChris Mason 56300b86a832SChris Mason btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 5631e02119d5SChris Mason if (found_key.objectid < min_objectid) 5632e02119d5SChris Mason break; 56330a4eefbbSYan Zheng if (found_key.type == type) 56340a4eefbbSYan Zheng return 0; 5635e02119d5SChris Mason if (found_key.objectid == min_objectid && 5636e02119d5SChris Mason found_key.type < type) 5637e02119d5SChris Mason break; 56380b86a832SChris Mason } 56390b86a832SChris Mason return 1; 56400b86a832SChris Mason } 5641