16cbd5570SChris Mason /* 2d352ac68SChris Mason * Copyright (C) 2007,2008 Oracle. All rights reserved. 36cbd5570SChris Mason * 46cbd5570SChris Mason * This program is free software; you can redistribute it and/or 56cbd5570SChris Mason * modify it under the terms of the GNU General Public 66cbd5570SChris Mason * License v2 as published by the Free Software Foundation. 76cbd5570SChris Mason * 86cbd5570SChris Mason * This program is distributed in the hope that it will be useful, 96cbd5570SChris Mason * but WITHOUT ANY WARRANTY; without even the implied warranty of 106cbd5570SChris Mason * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 116cbd5570SChris Mason * General Public License for more details. 126cbd5570SChris Mason * 136cbd5570SChris Mason * You should have received a copy of the GNU General Public 146cbd5570SChris Mason * License along with this program; if not, write to the 156cbd5570SChris Mason * Free Software Foundation, Inc., 59 Temple Place - Suite 330, 166cbd5570SChris Mason * Boston, MA 021110-1307, USA. 176cbd5570SChris Mason */ 186cbd5570SChris Mason 19a6b6e75eSChris Mason #include <linux/sched.h> 205a0e3ad6STejun Heo #include <linux/slab.h> 21bd989ba3SJan Schmidt #include <linux/rbtree.h> 22eb60ceacSChris Mason #include "ctree.h" 23eb60ceacSChris Mason #include "disk-io.h" 247f5c1516SChris Mason #include "transaction.h" 255f39d397SChris Mason #include "print-tree.h" 26925baeddSChris Mason #include "locking.h" 279a8dd150SChris Mason 28e089f05cSChris Mason static int split_node(struct btrfs_trans_handle *trans, struct btrfs_root 29e089f05cSChris Mason *root, struct btrfs_path *path, int level); 30e089f05cSChris Mason static int split_leaf(struct btrfs_trans_handle *trans, struct btrfs_root 31d4dbff95SChris Mason *root, struct btrfs_key *ins_key, 32cc0c5538SChris Mason struct btrfs_path *path, int data_size, int extend); 335f39d397SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 345f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *dst, 35971a1f66SChris Mason struct extent_buffer *src, int empty); 365f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 375f39d397SChris Mason struct btrfs_root *root, 385f39d397SChris Mason struct extent_buffer *dst_buf, 395f39d397SChris Mason struct extent_buffer *src_buf); 40afe5fea7STsutomu Itoh static void del_ptr(struct btrfs_root *root, struct btrfs_path *path, 41afe5fea7STsutomu Itoh int level, int slot); 425de865eeSFilipe David Borba Manana static int tree_mod_log_free_eb(struct btrfs_fs_info *fs_info, 43f230475eSJan Schmidt struct extent_buffer *eb); 44d97e63b6SChris Mason 452c90e5d6SChris Mason struct btrfs_path *btrfs_alloc_path(void) 462c90e5d6SChris Mason { 47df24a2b9SChris Mason struct btrfs_path *path; 48e00f7308SJeff Mahoney path = kmem_cache_zalloc(btrfs_path_cachep, GFP_NOFS); 49df24a2b9SChris Mason return path; 502c90e5d6SChris Mason } 512c90e5d6SChris Mason 52b4ce94deSChris Mason /* 53b4ce94deSChris Mason * set all locked nodes in the path to blocking locks. This should 54b4ce94deSChris Mason * be done before scheduling 55b4ce94deSChris Mason */ 56b4ce94deSChris Mason noinline void btrfs_set_path_blocking(struct btrfs_path *p) 57b4ce94deSChris Mason { 58b4ce94deSChris Mason int i; 59b4ce94deSChris Mason for (i = 0; i < BTRFS_MAX_LEVEL; i++) { 60bd681513SChris Mason if (!p->nodes[i] || !p->locks[i]) 61bd681513SChris Mason continue; 62bd681513SChris Mason btrfs_set_lock_blocking_rw(p->nodes[i], p->locks[i]); 63bd681513SChris Mason if (p->locks[i] == BTRFS_READ_LOCK) 64bd681513SChris Mason p->locks[i] = BTRFS_READ_LOCK_BLOCKING; 65bd681513SChris Mason else if (p->locks[i] == BTRFS_WRITE_LOCK) 66bd681513SChris Mason p->locks[i] = BTRFS_WRITE_LOCK_BLOCKING; 67b4ce94deSChris Mason } 68b4ce94deSChris Mason } 69b4ce94deSChris Mason 70b4ce94deSChris Mason /* 71b4ce94deSChris Mason * reset all the locked nodes in the patch to spinning locks. 724008c04aSChris Mason * 734008c04aSChris Mason * held is used to keep lockdep happy, when lockdep is enabled 744008c04aSChris Mason * we set held to a blocking lock before we go around and 754008c04aSChris Mason * retake all the spinlocks in the path. You can safely use NULL 764008c04aSChris Mason * for held 77b4ce94deSChris Mason */ 784008c04aSChris Mason noinline void btrfs_clear_path_blocking(struct btrfs_path *p, 79bd681513SChris Mason struct extent_buffer *held, int held_rw) 80b4ce94deSChris Mason { 81b4ce94deSChris Mason int i; 824008c04aSChris Mason 834008c04aSChris Mason #ifdef CONFIG_DEBUG_LOCK_ALLOC 844008c04aSChris Mason /* lockdep really cares that we take all of these spinlocks 854008c04aSChris Mason * in the right order. If any of the locks in the path are not 864008c04aSChris Mason * currently blocking, it is going to complain. So, make really 874008c04aSChris Mason * really sure by forcing the path to blocking before we clear 884008c04aSChris Mason * the path blocking. 894008c04aSChris Mason */ 90bd681513SChris Mason if (held) { 91bd681513SChris Mason btrfs_set_lock_blocking_rw(held, held_rw); 92bd681513SChris Mason if (held_rw == BTRFS_WRITE_LOCK) 93bd681513SChris Mason held_rw = BTRFS_WRITE_LOCK_BLOCKING; 94bd681513SChris Mason else if (held_rw == BTRFS_READ_LOCK) 95bd681513SChris Mason held_rw = BTRFS_READ_LOCK_BLOCKING; 96bd681513SChris Mason } 974008c04aSChris Mason btrfs_set_path_blocking(p); 984008c04aSChris Mason #endif 994008c04aSChris Mason 1004008c04aSChris Mason for (i = BTRFS_MAX_LEVEL - 1; i >= 0; i--) { 101bd681513SChris Mason if (p->nodes[i] && p->locks[i]) { 102bd681513SChris Mason btrfs_clear_lock_blocking_rw(p->nodes[i], p->locks[i]); 103bd681513SChris Mason if (p->locks[i] == BTRFS_WRITE_LOCK_BLOCKING) 104bd681513SChris Mason p->locks[i] = BTRFS_WRITE_LOCK; 105bd681513SChris Mason else if (p->locks[i] == BTRFS_READ_LOCK_BLOCKING) 106bd681513SChris Mason p->locks[i] = BTRFS_READ_LOCK; 107bd681513SChris Mason } 108b4ce94deSChris Mason } 1094008c04aSChris Mason 1104008c04aSChris Mason #ifdef CONFIG_DEBUG_LOCK_ALLOC 1114008c04aSChris Mason if (held) 112bd681513SChris Mason btrfs_clear_lock_blocking_rw(held, held_rw); 1134008c04aSChris Mason #endif 114b4ce94deSChris Mason } 115b4ce94deSChris Mason 116d352ac68SChris Mason /* this also releases the path */ 1172c90e5d6SChris Mason void btrfs_free_path(struct btrfs_path *p) 1182c90e5d6SChris Mason { 119ff175d57SJesper Juhl if (!p) 120ff175d57SJesper Juhl return; 121b3b4aa74SDavid Sterba btrfs_release_path(p); 1222c90e5d6SChris Mason kmem_cache_free(btrfs_path_cachep, p); 1232c90e5d6SChris Mason } 1242c90e5d6SChris Mason 125d352ac68SChris Mason /* 126d352ac68SChris Mason * path release drops references on the extent buffers in the path 127d352ac68SChris Mason * and it drops any locks held by this path 128d352ac68SChris Mason * 129d352ac68SChris Mason * It is safe to call this on paths that no locks or extent buffers held. 130d352ac68SChris Mason */ 131b3b4aa74SDavid Sterba noinline void btrfs_release_path(struct btrfs_path *p) 132eb60ceacSChris Mason { 133eb60ceacSChris Mason int i; 134a2135011SChris Mason 135234b63a0SChris Mason for (i = 0; i < BTRFS_MAX_LEVEL; i++) { 1363f157a2fSChris Mason p->slots[i] = 0; 137eb60ceacSChris Mason if (!p->nodes[i]) 138925baeddSChris Mason continue; 139925baeddSChris Mason if (p->locks[i]) { 140bd681513SChris Mason btrfs_tree_unlock_rw(p->nodes[i], p->locks[i]); 141925baeddSChris Mason p->locks[i] = 0; 142925baeddSChris Mason } 1435f39d397SChris Mason free_extent_buffer(p->nodes[i]); 1443f157a2fSChris Mason p->nodes[i] = NULL; 145eb60ceacSChris Mason } 146eb60ceacSChris Mason } 147eb60ceacSChris Mason 148d352ac68SChris Mason /* 149d352ac68SChris Mason * safely gets a reference on the root node of a tree. A lock 150d352ac68SChris Mason * is not taken, so a concurrent writer may put a different node 151d352ac68SChris Mason * at the root of the tree. See btrfs_lock_root_node for the 152d352ac68SChris Mason * looping required. 153d352ac68SChris Mason * 154d352ac68SChris Mason * The extent buffer returned by this has a reference taken, so 155d352ac68SChris Mason * it won't disappear. It may stop being the root of the tree 156d352ac68SChris Mason * at any time because there are no locks held. 157d352ac68SChris Mason */ 158925baeddSChris Mason struct extent_buffer *btrfs_root_node(struct btrfs_root *root) 159925baeddSChris Mason { 160925baeddSChris Mason struct extent_buffer *eb; 161240f62c8SChris Mason 1623083ee2eSJosef Bacik while (1) { 163240f62c8SChris Mason rcu_read_lock(); 164240f62c8SChris Mason eb = rcu_dereference(root->node); 1653083ee2eSJosef Bacik 1663083ee2eSJosef Bacik /* 1673083ee2eSJosef Bacik * RCU really hurts here, we could free up the root node because 1683083ee2eSJosef Bacik * it was cow'ed but we may not get the new root node yet so do 1693083ee2eSJosef Bacik * the inc_not_zero dance and if it doesn't work then 1703083ee2eSJosef Bacik * synchronize_rcu and try again. 1713083ee2eSJosef Bacik */ 1723083ee2eSJosef Bacik if (atomic_inc_not_zero(&eb->refs)) { 173240f62c8SChris Mason rcu_read_unlock(); 1743083ee2eSJosef Bacik break; 1753083ee2eSJosef Bacik } 1763083ee2eSJosef Bacik rcu_read_unlock(); 1773083ee2eSJosef Bacik synchronize_rcu(); 1783083ee2eSJosef Bacik } 179925baeddSChris Mason return eb; 180925baeddSChris Mason } 181925baeddSChris Mason 182d352ac68SChris Mason /* loop around taking references on and locking the root node of the 183d352ac68SChris Mason * tree until you end up with a lock on the root. A locked buffer 184d352ac68SChris Mason * is returned, with a reference held. 185d352ac68SChris Mason */ 186925baeddSChris Mason struct extent_buffer *btrfs_lock_root_node(struct btrfs_root *root) 187925baeddSChris Mason { 188925baeddSChris Mason struct extent_buffer *eb; 189925baeddSChris Mason 190925baeddSChris Mason while (1) { 191925baeddSChris Mason eb = btrfs_root_node(root); 192925baeddSChris Mason btrfs_tree_lock(eb); 193240f62c8SChris Mason if (eb == root->node) 194925baeddSChris Mason break; 195925baeddSChris Mason btrfs_tree_unlock(eb); 196925baeddSChris Mason free_extent_buffer(eb); 197925baeddSChris Mason } 198925baeddSChris Mason return eb; 199925baeddSChris Mason } 200925baeddSChris Mason 201bd681513SChris Mason /* loop around taking references on and locking the root node of the 202bd681513SChris Mason * tree until you end up with a lock on the root. A locked buffer 203bd681513SChris Mason * is returned, with a reference held. 204bd681513SChris Mason */ 20548a3b636SEric Sandeen static struct extent_buffer *btrfs_read_lock_root_node(struct btrfs_root *root) 206bd681513SChris Mason { 207bd681513SChris Mason struct extent_buffer *eb; 208bd681513SChris Mason 209bd681513SChris Mason while (1) { 210bd681513SChris Mason eb = btrfs_root_node(root); 211bd681513SChris Mason btrfs_tree_read_lock(eb); 212bd681513SChris Mason if (eb == root->node) 213bd681513SChris Mason break; 214bd681513SChris Mason btrfs_tree_read_unlock(eb); 215bd681513SChris Mason free_extent_buffer(eb); 216bd681513SChris Mason } 217bd681513SChris Mason return eb; 218bd681513SChris Mason } 219bd681513SChris Mason 220d352ac68SChris Mason /* cowonly root (everything not a reference counted cow subvolume), just get 221d352ac68SChris Mason * put onto a simple dirty list. transaction.c walks this to make sure they 222d352ac68SChris Mason * get properly updated on disk. 223d352ac68SChris Mason */ 2240b86a832SChris Mason static void add_root_to_dirty_list(struct btrfs_root *root) 2250b86a832SChris Mason { 226e5846fc6SChris Mason spin_lock(&root->fs_info->trans_lock); 22727cdeb70SMiao Xie if (test_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state) && 22827cdeb70SMiao Xie list_empty(&root->dirty_list)) { 2290b86a832SChris Mason list_add(&root->dirty_list, 2300b86a832SChris Mason &root->fs_info->dirty_cowonly_roots); 2310b86a832SChris Mason } 232e5846fc6SChris Mason spin_unlock(&root->fs_info->trans_lock); 2330b86a832SChris Mason } 2340b86a832SChris Mason 235d352ac68SChris Mason /* 236d352ac68SChris Mason * used by snapshot creation to make a copy of a root for a tree with 237d352ac68SChris Mason * a given objectid. The buffer with the new root node is returned in 238d352ac68SChris Mason * cow_ret, and this func returns zero on success or a negative error code. 239d352ac68SChris Mason */ 240be20aa9dSChris Mason int btrfs_copy_root(struct btrfs_trans_handle *trans, 241be20aa9dSChris Mason struct btrfs_root *root, 242be20aa9dSChris Mason struct extent_buffer *buf, 243be20aa9dSChris Mason struct extent_buffer **cow_ret, u64 new_root_objectid) 244be20aa9dSChris Mason { 245be20aa9dSChris Mason struct extent_buffer *cow; 246be20aa9dSChris Mason int ret = 0; 247be20aa9dSChris Mason int level; 2485d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 249be20aa9dSChris Mason 25027cdeb70SMiao Xie WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) && 25127cdeb70SMiao Xie trans->transid != root->fs_info->running_transaction->transid); 25227cdeb70SMiao Xie WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) && 25327cdeb70SMiao Xie trans->transid != root->last_trans); 254be20aa9dSChris Mason 255be20aa9dSChris Mason level = btrfs_header_level(buf); 2565d4f98a2SYan Zheng if (level == 0) 2575d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 2585d4f98a2SYan Zheng else 2595d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 26031840ae1SZheng Yan 2615d4f98a2SYan Zheng cow = btrfs_alloc_free_block(trans, root, buf->len, 0, 2625d4f98a2SYan Zheng new_root_objectid, &disk_key, level, 2635581a51aSJan Schmidt buf->start, 0); 2645d4f98a2SYan Zheng if (IS_ERR(cow)) 265be20aa9dSChris Mason return PTR_ERR(cow); 266be20aa9dSChris Mason 267be20aa9dSChris Mason copy_extent_buffer(cow, buf, 0, 0, cow->len); 268be20aa9dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 269be20aa9dSChris Mason btrfs_set_header_generation(cow, trans->transid); 2705d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 2715d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 2725d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 2735d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 2745d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 2755d4f98a2SYan Zheng else 276be20aa9dSChris Mason btrfs_set_header_owner(cow, new_root_objectid); 277be20aa9dSChris Mason 2780a4e5586SRoss Kirk write_extent_buffer(cow, root->fs_info->fsid, btrfs_header_fsid(), 2792b82032cSYan Zheng BTRFS_FSID_SIZE); 2802b82032cSYan Zheng 281be20aa9dSChris Mason WARN_ON(btrfs_header_generation(buf) > trans->transid); 2825d4f98a2SYan Zheng if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) 28366d7e7f0SArne Jansen ret = btrfs_inc_ref(trans, root, cow, 1, 1); 2845d4f98a2SYan Zheng else 28566d7e7f0SArne Jansen ret = btrfs_inc_ref(trans, root, cow, 0, 1); 2864aec2b52SChris Mason 287be20aa9dSChris Mason if (ret) 288be20aa9dSChris Mason return ret; 289be20aa9dSChris Mason 290be20aa9dSChris Mason btrfs_mark_buffer_dirty(cow); 291be20aa9dSChris Mason *cow_ret = cow; 292be20aa9dSChris Mason return 0; 293be20aa9dSChris Mason } 294be20aa9dSChris Mason 295bd989ba3SJan Schmidt enum mod_log_op { 296bd989ba3SJan Schmidt MOD_LOG_KEY_REPLACE, 297bd989ba3SJan Schmidt MOD_LOG_KEY_ADD, 298bd989ba3SJan Schmidt MOD_LOG_KEY_REMOVE, 299bd989ba3SJan Schmidt MOD_LOG_KEY_REMOVE_WHILE_FREEING, 300bd989ba3SJan Schmidt MOD_LOG_KEY_REMOVE_WHILE_MOVING, 301bd989ba3SJan Schmidt MOD_LOG_MOVE_KEYS, 302bd989ba3SJan Schmidt MOD_LOG_ROOT_REPLACE, 303bd989ba3SJan Schmidt }; 304bd989ba3SJan Schmidt 305bd989ba3SJan Schmidt struct tree_mod_move { 306bd989ba3SJan Schmidt int dst_slot; 307bd989ba3SJan Schmidt int nr_items; 308bd989ba3SJan Schmidt }; 309bd989ba3SJan Schmidt 310bd989ba3SJan Schmidt struct tree_mod_root { 311bd989ba3SJan Schmidt u64 logical; 312bd989ba3SJan Schmidt u8 level; 313bd989ba3SJan Schmidt }; 314bd989ba3SJan Schmidt 315bd989ba3SJan Schmidt struct tree_mod_elem { 316bd989ba3SJan Schmidt struct rb_node node; 317bd989ba3SJan Schmidt u64 index; /* shifted logical */ 318097b8a7cSJan Schmidt u64 seq; 319bd989ba3SJan Schmidt enum mod_log_op op; 320bd989ba3SJan Schmidt 321bd989ba3SJan Schmidt /* this is used for MOD_LOG_KEY_* and MOD_LOG_MOVE_KEYS operations */ 322bd989ba3SJan Schmidt int slot; 323bd989ba3SJan Schmidt 324bd989ba3SJan Schmidt /* this is used for MOD_LOG_KEY* and MOD_LOG_ROOT_REPLACE */ 325bd989ba3SJan Schmidt u64 generation; 326bd989ba3SJan Schmidt 327bd989ba3SJan Schmidt /* those are used for op == MOD_LOG_KEY_{REPLACE,REMOVE} */ 328bd989ba3SJan Schmidt struct btrfs_disk_key key; 329bd989ba3SJan Schmidt u64 blockptr; 330bd989ba3SJan Schmidt 331bd989ba3SJan Schmidt /* this is used for op == MOD_LOG_MOVE_KEYS */ 332bd989ba3SJan Schmidt struct tree_mod_move move; 333bd989ba3SJan Schmidt 334bd989ba3SJan Schmidt /* this is used for op == MOD_LOG_ROOT_REPLACE */ 335bd989ba3SJan Schmidt struct tree_mod_root old_root; 336bd989ba3SJan Schmidt }; 337bd989ba3SJan Schmidt 338097b8a7cSJan Schmidt static inline void tree_mod_log_read_lock(struct btrfs_fs_info *fs_info) 339bd989ba3SJan Schmidt { 340097b8a7cSJan Schmidt read_lock(&fs_info->tree_mod_log_lock); 341bd989ba3SJan Schmidt } 342bd989ba3SJan Schmidt 343097b8a7cSJan Schmidt static inline void tree_mod_log_read_unlock(struct btrfs_fs_info *fs_info) 344097b8a7cSJan Schmidt { 345097b8a7cSJan Schmidt read_unlock(&fs_info->tree_mod_log_lock); 346097b8a7cSJan Schmidt } 347097b8a7cSJan Schmidt 348097b8a7cSJan Schmidt static inline void tree_mod_log_write_lock(struct btrfs_fs_info *fs_info) 349097b8a7cSJan Schmidt { 350097b8a7cSJan Schmidt write_lock(&fs_info->tree_mod_log_lock); 351097b8a7cSJan Schmidt } 352097b8a7cSJan Schmidt 353097b8a7cSJan Schmidt static inline void tree_mod_log_write_unlock(struct btrfs_fs_info *fs_info) 354097b8a7cSJan Schmidt { 355097b8a7cSJan Schmidt write_unlock(&fs_info->tree_mod_log_lock); 356097b8a7cSJan Schmidt } 357097b8a7cSJan Schmidt 358097b8a7cSJan Schmidt /* 359fc36ed7eSJan Schmidt * Increment the upper half of tree_mod_seq, set lower half zero. 360fc36ed7eSJan Schmidt * 361fc36ed7eSJan Schmidt * Must be called with fs_info->tree_mod_seq_lock held. 362fc36ed7eSJan Schmidt */ 363fc36ed7eSJan Schmidt static inline u64 btrfs_inc_tree_mod_seq_major(struct btrfs_fs_info *fs_info) 364fc36ed7eSJan Schmidt { 365fc36ed7eSJan Schmidt u64 seq = atomic64_read(&fs_info->tree_mod_seq); 366fc36ed7eSJan Schmidt seq &= 0xffffffff00000000ull; 367fc36ed7eSJan Schmidt seq += 1ull << 32; 368fc36ed7eSJan Schmidt atomic64_set(&fs_info->tree_mod_seq, seq); 369fc36ed7eSJan Schmidt return seq; 370fc36ed7eSJan Schmidt } 371fc36ed7eSJan Schmidt 372fc36ed7eSJan Schmidt /* 373fc36ed7eSJan Schmidt * Increment the lower half of tree_mod_seq. 374fc36ed7eSJan Schmidt * 375fc36ed7eSJan Schmidt * Must be called with fs_info->tree_mod_seq_lock held. The way major numbers 376fc36ed7eSJan Schmidt * are generated should not technically require a spin lock here. (Rationale: 377fc36ed7eSJan Schmidt * incrementing the minor while incrementing the major seq number is between its 378fc36ed7eSJan Schmidt * atomic64_read and atomic64_set calls doesn't duplicate sequence numbers, it 379fc36ed7eSJan Schmidt * just returns a unique sequence number as usual.) We have decided to leave 380fc36ed7eSJan Schmidt * that requirement in here and rethink it once we notice it really imposes a 381fc36ed7eSJan Schmidt * problem on some workload. 382fc36ed7eSJan Schmidt */ 383fc36ed7eSJan Schmidt static inline u64 btrfs_inc_tree_mod_seq_minor(struct btrfs_fs_info *fs_info) 384fc36ed7eSJan Schmidt { 385fc36ed7eSJan Schmidt return atomic64_inc_return(&fs_info->tree_mod_seq); 386fc36ed7eSJan Schmidt } 387fc36ed7eSJan Schmidt 388fc36ed7eSJan Schmidt /* 389fc36ed7eSJan Schmidt * return the last minor in the previous major tree_mod_seq number 390fc36ed7eSJan Schmidt */ 391fc36ed7eSJan Schmidt u64 btrfs_tree_mod_seq_prev(u64 seq) 392fc36ed7eSJan Schmidt { 393fc36ed7eSJan Schmidt return (seq & 0xffffffff00000000ull) - 1ull; 394fc36ed7eSJan Schmidt } 395fc36ed7eSJan Schmidt 396fc36ed7eSJan Schmidt /* 397097b8a7cSJan Schmidt * This adds a new blocker to the tree mod log's blocker list if the @elem 398097b8a7cSJan Schmidt * passed does not already have a sequence number set. So when a caller expects 399097b8a7cSJan Schmidt * to record tree modifications, it should ensure to set elem->seq to zero 400097b8a7cSJan Schmidt * before calling btrfs_get_tree_mod_seq. 401097b8a7cSJan Schmidt * Returns a fresh, unused tree log modification sequence number, even if no new 402097b8a7cSJan Schmidt * blocker was added. 403097b8a7cSJan Schmidt */ 404097b8a7cSJan Schmidt u64 btrfs_get_tree_mod_seq(struct btrfs_fs_info *fs_info, 405bd989ba3SJan Schmidt struct seq_list *elem) 406bd989ba3SJan Schmidt { 407097b8a7cSJan Schmidt u64 seq; 408097b8a7cSJan Schmidt 409097b8a7cSJan Schmidt tree_mod_log_write_lock(fs_info); 410bd989ba3SJan Schmidt spin_lock(&fs_info->tree_mod_seq_lock); 411097b8a7cSJan Schmidt if (!elem->seq) { 412fc36ed7eSJan Schmidt elem->seq = btrfs_inc_tree_mod_seq_major(fs_info); 413097b8a7cSJan Schmidt list_add_tail(&elem->list, &fs_info->tree_mod_seq_list); 414097b8a7cSJan Schmidt } 415fc36ed7eSJan Schmidt seq = btrfs_inc_tree_mod_seq_minor(fs_info); 416bd989ba3SJan Schmidt spin_unlock(&fs_info->tree_mod_seq_lock); 417097b8a7cSJan Schmidt tree_mod_log_write_unlock(fs_info); 418097b8a7cSJan Schmidt 419097b8a7cSJan Schmidt return seq; 420bd989ba3SJan Schmidt } 421bd989ba3SJan Schmidt 422bd989ba3SJan Schmidt void btrfs_put_tree_mod_seq(struct btrfs_fs_info *fs_info, 423bd989ba3SJan Schmidt struct seq_list *elem) 424bd989ba3SJan Schmidt { 425bd989ba3SJan Schmidt struct rb_root *tm_root; 426bd989ba3SJan Schmidt struct rb_node *node; 427bd989ba3SJan Schmidt struct rb_node *next; 428bd989ba3SJan Schmidt struct seq_list *cur_elem; 429bd989ba3SJan Schmidt struct tree_mod_elem *tm; 430bd989ba3SJan Schmidt u64 min_seq = (u64)-1; 431bd989ba3SJan Schmidt u64 seq_putting = elem->seq; 432bd989ba3SJan Schmidt 433bd989ba3SJan Schmidt if (!seq_putting) 434bd989ba3SJan Schmidt return; 435bd989ba3SJan Schmidt 436bd989ba3SJan Schmidt spin_lock(&fs_info->tree_mod_seq_lock); 437bd989ba3SJan Schmidt list_del(&elem->list); 438097b8a7cSJan Schmidt elem->seq = 0; 439bd989ba3SJan Schmidt 440bd989ba3SJan Schmidt list_for_each_entry(cur_elem, &fs_info->tree_mod_seq_list, list) { 441097b8a7cSJan Schmidt if (cur_elem->seq < min_seq) { 442bd989ba3SJan Schmidt if (seq_putting > cur_elem->seq) { 443bd989ba3SJan Schmidt /* 444bd989ba3SJan Schmidt * blocker with lower sequence number exists, we 445bd989ba3SJan Schmidt * cannot remove anything from the log 446bd989ba3SJan Schmidt */ 447097b8a7cSJan Schmidt spin_unlock(&fs_info->tree_mod_seq_lock); 448097b8a7cSJan Schmidt return; 449bd989ba3SJan Schmidt } 450bd989ba3SJan Schmidt min_seq = cur_elem->seq; 451bd989ba3SJan Schmidt } 452bd989ba3SJan Schmidt } 453097b8a7cSJan Schmidt spin_unlock(&fs_info->tree_mod_seq_lock); 454097b8a7cSJan Schmidt 455097b8a7cSJan Schmidt /* 456bd989ba3SJan Schmidt * anything that's lower than the lowest existing (read: blocked) 457bd989ba3SJan Schmidt * sequence number can be removed from the tree. 458bd989ba3SJan Schmidt */ 459097b8a7cSJan Schmidt tree_mod_log_write_lock(fs_info); 460bd989ba3SJan Schmidt tm_root = &fs_info->tree_mod_log; 461bd989ba3SJan Schmidt for (node = rb_first(tm_root); node; node = next) { 462bd989ba3SJan Schmidt next = rb_next(node); 463bd989ba3SJan Schmidt tm = container_of(node, struct tree_mod_elem, node); 464097b8a7cSJan Schmidt if (tm->seq > min_seq) 465bd989ba3SJan Schmidt continue; 466bd989ba3SJan Schmidt rb_erase(node, tm_root); 467bd989ba3SJan Schmidt kfree(tm); 468bd989ba3SJan Schmidt } 469097b8a7cSJan Schmidt tree_mod_log_write_unlock(fs_info); 470bd989ba3SJan Schmidt } 471bd989ba3SJan Schmidt 472bd989ba3SJan Schmidt /* 473bd989ba3SJan Schmidt * key order of the log: 474bd989ba3SJan Schmidt * index -> sequence 475bd989ba3SJan Schmidt * 476bd989ba3SJan Schmidt * the index is the shifted logical of the *new* root node for root replace 477bd989ba3SJan Schmidt * operations, or the shifted logical of the affected block for all other 478bd989ba3SJan Schmidt * operations. 4795de865eeSFilipe David Borba Manana * 4805de865eeSFilipe David Borba Manana * Note: must be called with write lock (tree_mod_log_write_lock). 481bd989ba3SJan Schmidt */ 482bd989ba3SJan Schmidt static noinline int 483bd989ba3SJan Schmidt __tree_mod_log_insert(struct btrfs_fs_info *fs_info, struct tree_mod_elem *tm) 484bd989ba3SJan Schmidt { 485bd989ba3SJan Schmidt struct rb_root *tm_root; 486bd989ba3SJan Schmidt struct rb_node **new; 487bd989ba3SJan Schmidt struct rb_node *parent = NULL; 488bd989ba3SJan Schmidt struct tree_mod_elem *cur; 489bd989ba3SJan Schmidt 490c8cc6341SJosef Bacik BUG_ON(!tm); 491c8cc6341SJosef Bacik 492c8cc6341SJosef Bacik spin_lock(&fs_info->tree_mod_seq_lock); 493c8cc6341SJosef Bacik tm->seq = btrfs_inc_tree_mod_seq_minor(fs_info); 494c8cc6341SJosef Bacik spin_unlock(&fs_info->tree_mod_seq_lock); 495bd989ba3SJan Schmidt 496bd989ba3SJan Schmidt tm_root = &fs_info->tree_mod_log; 497bd989ba3SJan Schmidt new = &tm_root->rb_node; 498bd989ba3SJan Schmidt while (*new) { 499bd989ba3SJan Schmidt cur = container_of(*new, struct tree_mod_elem, node); 500bd989ba3SJan Schmidt parent = *new; 501bd989ba3SJan Schmidt if (cur->index < tm->index) 502bd989ba3SJan Schmidt new = &((*new)->rb_left); 503bd989ba3SJan Schmidt else if (cur->index > tm->index) 504bd989ba3SJan Schmidt new = &((*new)->rb_right); 505097b8a7cSJan Schmidt else if (cur->seq < tm->seq) 506bd989ba3SJan Schmidt new = &((*new)->rb_left); 507097b8a7cSJan Schmidt else if (cur->seq > tm->seq) 508bd989ba3SJan Schmidt new = &((*new)->rb_right); 5095de865eeSFilipe David Borba Manana else 5105de865eeSFilipe David Borba Manana return -EEXIST; 511bd989ba3SJan Schmidt } 512bd989ba3SJan Schmidt 513bd989ba3SJan Schmidt rb_link_node(&tm->node, parent, new); 514bd989ba3SJan Schmidt rb_insert_color(&tm->node, tm_root); 5155de865eeSFilipe David Borba Manana return 0; 516bd989ba3SJan Schmidt } 517bd989ba3SJan Schmidt 518097b8a7cSJan Schmidt /* 519097b8a7cSJan Schmidt * Determines if logging can be omitted. Returns 1 if it can. Otherwise, it 520097b8a7cSJan Schmidt * returns zero with the tree_mod_log_lock acquired. The caller must hold 521097b8a7cSJan Schmidt * this until all tree mod log insertions are recorded in the rb tree and then 522097b8a7cSJan Schmidt * call tree_mod_log_write_unlock() to release. 523097b8a7cSJan Schmidt */ 524e9b7fd4dSJan Schmidt static inline int tree_mod_dont_log(struct btrfs_fs_info *fs_info, 525e9b7fd4dSJan Schmidt struct extent_buffer *eb) { 526e9b7fd4dSJan Schmidt smp_mb(); 527e9b7fd4dSJan Schmidt if (list_empty(&(fs_info)->tree_mod_seq_list)) 528e9b7fd4dSJan Schmidt return 1; 529097b8a7cSJan Schmidt if (eb && btrfs_header_level(eb) == 0) 530e9b7fd4dSJan Schmidt return 1; 5315de865eeSFilipe David Borba Manana 5325de865eeSFilipe David Borba Manana tree_mod_log_write_lock(fs_info); 5335de865eeSFilipe David Borba Manana if (list_empty(&(fs_info)->tree_mod_seq_list)) { 5345de865eeSFilipe David Borba Manana tree_mod_log_write_unlock(fs_info); 5355de865eeSFilipe David Borba Manana return 1; 5365de865eeSFilipe David Borba Manana } 5375de865eeSFilipe David Borba Manana 538e9b7fd4dSJan Schmidt return 0; 539e9b7fd4dSJan Schmidt } 540e9b7fd4dSJan Schmidt 5415de865eeSFilipe David Borba Manana /* Similar to tree_mod_dont_log, but doesn't acquire any locks. */ 5425de865eeSFilipe David Borba Manana static inline int tree_mod_need_log(const struct btrfs_fs_info *fs_info, 5435de865eeSFilipe David Borba Manana struct extent_buffer *eb) 5445de865eeSFilipe David Borba Manana { 5455de865eeSFilipe David Borba Manana smp_mb(); 5465de865eeSFilipe David Borba Manana if (list_empty(&(fs_info)->tree_mod_seq_list)) 5475de865eeSFilipe David Borba Manana return 0; 5485de865eeSFilipe David Borba Manana if (eb && btrfs_header_level(eb) == 0) 5495de865eeSFilipe David Borba Manana return 0; 5505de865eeSFilipe David Borba Manana 5515de865eeSFilipe David Borba Manana return 1; 5525de865eeSFilipe David Borba Manana } 5535de865eeSFilipe David Borba Manana 5545de865eeSFilipe David Borba Manana static struct tree_mod_elem * 5555de865eeSFilipe David Borba Manana alloc_tree_mod_elem(struct extent_buffer *eb, int slot, 556bd989ba3SJan Schmidt enum mod_log_op op, gfp_t flags) 557bd989ba3SJan Schmidt { 558097b8a7cSJan Schmidt struct tree_mod_elem *tm; 559bd989ba3SJan Schmidt 560c8cc6341SJosef Bacik tm = kzalloc(sizeof(*tm), flags); 561c8cc6341SJosef Bacik if (!tm) 5625de865eeSFilipe David Borba Manana return NULL; 563bd989ba3SJan Schmidt 564bd989ba3SJan Schmidt tm->index = eb->start >> PAGE_CACHE_SHIFT; 565bd989ba3SJan Schmidt if (op != MOD_LOG_KEY_ADD) { 566bd989ba3SJan Schmidt btrfs_node_key(eb, &tm->key, slot); 567bd989ba3SJan Schmidt tm->blockptr = btrfs_node_blockptr(eb, slot); 568bd989ba3SJan Schmidt } 569bd989ba3SJan Schmidt tm->op = op; 570bd989ba3SJan Schmidt tm->slot = slot; 571bd989ba3SJan Schmidt tm->generation = btrfs_node_ptr_generation(eb, slot); 5725de865eeSFilipe David Borba Manana RB_CLEAR_NODE(&tm->node); 573bd989ba3SJan Schmidt 5745de865eeSFilipe David Borba Manana return tm; 575097b8a7cSJan Schmidt } 576097b8a7cSJan Schmidt 577097b8a7cSJan Schmidt static noinline int 578c8cc6341SJosef Bacik tree_mod_log_insert_key(struct btrfs_fs_info *fs_info, 579097b8a7cSJan Schmidt struct extent_buffer *eb, int slot, 580097b8a7cSJan Schmidt enum mod_log_op op, gfp_t flags) 581097b8a7cSJan Schmidt { 5825de865eeSFilipe David Borba Manana struct tree_mod_elem *tm; 5835de865eeSFilipe David Borba Manana int ret; 5845de865eeSFilipe David Borba Manana 5855de865eeSFilipe David Borba Manana if (!tree_mod_need_log(fs_info, eb)) 586097b8a7cSJan Schmidt return 0; 587097b8a7cSJan Schmidt 5885de865eeSFilipe David Borba Manana tm = alloc_tree_mod_elem(eb, slot, op, flags); 5895de865eeSFilipe David Borba Manana if (!tm) 5905de865eeSFilipe David Borba Manana return -ENOMEM; 5915de865eeSFilipe David Borba Manana 5925de865eeSFilipe David Borba Manana if (tree_mod_dont_log(fs_info, eb)) { 5935de865eeSFilipe David Borba Manana kfree(tm); 5945de865eeSFilipe David Borba Manana return 0; 5955de865eeSFilipe David Borba Manana } 5965de865eeSFilipe David Borba Manana 5975de865eeSFilipe David Borba Manana ret = __tree_mod_log_insert(fs_info, tm); 5985de865eeSFilipe David Borba Manana tree_mod_log_write_unlock(fs_info); 5995de865eeSFilipe David Borba Manana if (ret) 6005de865eeSFilipe David Borba Manana kfree(tm); 6015de865eeSFilipe David Borba Manana 6025de865eeSFilipe David Borba Manana return ret; 603097b8a7cSJan Schmidt } 604097b8a7cSJan Schmidt 605097b8a7cSJan Schmidt static noinline int 606bd989ba3SJan Schmidt tree_mod_log_insert_move(struct btrfs_fs_info *fs_info, 607bd989ba3SJan Schmidt struct extent_buffer *eb, int dst_slot, int src_slot, 608bd989ba3SJan Schmidt int nr_items, gfp_t flags) 609bd989ba3SJan Schmidt { 6105de865eeSFilipe David Borba Manana struct tree_mod_elem *tm = NULL; 6115de865eeSFilipe David Borba Manana struct tree_mod_elem **tm_list = NULL; 6125de865eeSFilipe David Borba Manana int ret = 0; 613bd989ba3SJan Schmidt int i; 6145de865eeSFilipe David Borba Manana int locked = 0; 615bd989ba3SJan Schmidt 6165de865eeSFilipe David Borba Manana if (!tree_mod_need_log(fs_info, eb)) 617f395694cSJan Schmidt return 0; 618bd989ba3SJan Schmidt 6195de865eeSFilipe David Borba Manana tm_list = kzalloc(nr_items * sizeof(struct tree_mod_elem *), flags); 6205de865eeSFilipe David Borba Manana if (!tm_list) 6215de865eeSFilipe David Borba Manana return -ENOMEM; 622bd989ba3SJan Schmidt 623c8cc6341SJosef Bacik tm = kzalloc(sizeof(*tm), flags); 6245de865eeSFilipe David Borba Manana if (!tm) { 6255de865eeSFilipe David Borba Manana ret = -ENOMEM; 6265de865eeSFilipe David Borba Manana goto free_tms; 6275de865eeSFilipe David Borba Manana } 628f395694cSJan Schmidt 629bd989ba3SJan Schmidt tm->index = eb->start >> PAGE_CACHE_SHIFT; 630bd989ba3SJan Schmidt tm->slot = src_slot; 631bd989ba3SJan Schmidt tm->move.dst_slot = dst_slot; 632bd989ba3SJan Schmidt tm->move.nr_items = nr_items; 633bd989ba3SJan Schmidt tm->op = MOD_LOG_MOVE_KEYS; 634bd989ba3SJan Schmidt 6355de865eeSFilipe David Borba Manana for (i = 0; i + dst_slot < src_slot && i < nr_items; i++) { 6365de865eeSFilipe David Borba Manana tm_list[i] = alloc_tree_mod_elem(eb, i + dst_slot, 6375de865eeSFilipe David Borba Manana MOD_LOG_KEY_REMOVE_WHILE_MOVING, flags); 6385de865eeSFilipe David Borba Manana if (!tm_list[i]) { 6395de865eeSFilipe David Borba Manana ret = -ENOMEM; 6405de865eeSFilipe David Borba Manana goto free_tms; 6415de865eeSFilipe David Borba Manana } 642bd989ba3SJan Schmidt } 643bd989ba3SJan Schmidt 6445de865eeSFilipe David Borba Manana if (tree_mod_dont_log(fs_info, eb)) 6455de865eeSFilipe David Borba Manana goto free_tms; 6465de865eeSFilipe David Borba Manana locked = 1; 6475de865eeSFilipe David Borba Manana 6485de865eeSFilipe David Borba Manana /* 6495de865eeSFilipe David Borba Manana * When we override something during the move, we log these removals. 6505de865eeSFilipe David Borba Manana * This can only happen when we move towards the beginning of the 6515de865eeSFilipe David Borba Manana * buffer, i.e. dst_slot < src_slot. 6525de865eeSFilipe David Borba Manana */ 6535de865eeSFilipe David Borba Manana for (i = 0; i + dst_slot < src_slot && i < nr_items; i++) { 6545de865eeSFilipe David Borba Manana ret = __tree_mod_log_insert(fs_info, tm_list[i]); 6555de865eeSFilipe David Borba Manana if (ret) 6565de865eeSFilipe David Borba Manana goto free_tms; 6575de865eeSFilipe David Borba Manana } 6585de865eeSFilipe David Borba Manana 6595de865eeSFilipe David Borba Manana ret = __tree_mod_log_insert(fs_info, tm); 6605de865eeSFilipe David Borba Manana if (ret) 6615de865eeSFilipe David Borba Manana goto free_tms; 6625de865eeSFilipe David Borba Manana tree_mod_log_write_unlock(fs_info); 6635de865eeSFilipe David Borba Manana kfree(tm_list); 6645de865eeSFilipe David Borba Manana 6655de865eeSFilipe David Borba Manana return 0; 6665de865eeSFilipe David Borba Manana free_tms: 6675de865eeSFilipe David Borba Manana for (i = 0; i < nr_items; i++) { 6685de865eeSFilipe David Borba Manana if (tm_list[i] && !RB_EMPTY_NODE(&tm_list[i]->node)) 6695de865eeSFilipe David Borba Manana rb_erase(&tm_list[i]->node, &fs_info->tree_mod_log); 6705de865eeSFilipe David Borba Manana kfree(tm_list[i]); 6715de865eeSFilipe David Borba Manana } 6725de865eeSFilipe David Borba Manana if (locked) 6735de865eeSFilipe David Borba Manana tree_mod_log_write_unlock(fs_info); 6745de865eeSFilipe David Borba Manana kfree(tm_list); 6755de865eeSFilipe David Borba Manana kfree(tm); 6765de865eeSFilipe David Borba Manana 6775de865eeSFilipe David Borba Manana return ret; 6785de865eeSFilipe David Borba Manana } 6795de865eeSFilipe David Borba Manana 6805de865eeSFilipe David Borba Manana static inline int 6815de865eeSFilipe David Borba Manana __tree_mod_log_free_eb(struct btrfs_fs_info *fs_info, 6825de865eeSFilipe David Borba Manana struct tree_mod_elem **tm_list, 6835de865eeSFilipe David Borba Manana int nritems) 684097b8a7cSJan Schmidt { 6855de865eeSFilipe David Borba Manana int i, j; 686097b8a7cSJan Schmidt int ret; 687097b8a7cSJan Schmidt 688097b8a7cSJan Schmidt for (i = nritems - 1; i >= 0; i--) { 6895de865eeSFilipe David Borba Manana ret = __tree_mod_log_insert(fs_info, tm_list[i]); 6905de865eeSFilipe David Borba Manana if (ret) { 6915de865eeSFilipe David Borba Manana for (j = nritems - 1; j > i; j--) 6925de865eeSFilipe David Borba Manana rb_erase(&tm_list[j]->node, 6935de865eeSFilipe David Borba Manana &fs_info->tree_mod_log); 6945de865eeSFilipe David Borba Manana return ret; 695097b8a7cSJan Schmidt } 696097b8a7cSJan Schmidt } 697097b8a7cSJan Schmidt 6985de865eeSFilipe David Borba Manana return 0; 6995de865eeSFilipe David Borba Manana } 7005de865eeSFilipe David Borba Manana 701bd989ba3SJan Schmidt static noinline int 702bd989ba3SJan Schmidt tree_mod_log_insert_root(struct btrfs_fs_info *fs_info, 703bd989ba3SJan Schmidt struct extent_buffer *old_root, 70490f8d62eSJan Schmidt struct extent_buffer *new_root, gfp_t flags, 70590f8d62eSJan Schmidt int log_removal) 706bd989ba3SJan Schmidt { 7075de865eeSFilipe David Borba Manana struct tree_mod_elem *tm = NULL; 7085de865eeSFilipe David Borba Manana struct tree_mod_elem **tm_list = NULL; 7095de865eeSFilipe David Borba Manana int nritems = 0; 7105de865eeSFilipe David Borba Manana int ret = 0; 7115de865eeSFilipe David Borba Manana int i; 712bd989ba3SJan Schmidt 7135de865eeSFilipe David Borba Manana if (!tree_mod_need_log(fs_info, NULL)) 714097b8a7cSJan Schmidt return 0; 715097b8a7cSJan Schmidt 7165de865eeSFilipe David Borba Manana if (log_removal && btrfs_header_level(old_root) > 0) { 7175de865eeSFilipe David Borba Manana nritems = btrfs_header_nritems(old_root); 7185de865eeSFilipe David Borba Manana tm_list = kzalloc(nritems * sizeof(struct tree_mod_elem *), 7195de865eeSFilipe David Borba Manana flags); 7205de865eeSFilipe David Borba Manana if (!tm_list) { 7215de865eeSFilipe David Borba Manana ret = -ENOMEM; 7225de865eeSFilipe David Borba Manana goto free_tms; 7235de865eeSFilipe David Borba Manana } 7245de865eeSFilipe David Borba Manana for (i = 0; i < nritems; i++) { 7255de865eeSFilipe David Borba Manana tm_list[i] = alloc_tree_mod_elem(old_root, i, 7265de865eeSFilipe David Borba Manana MOD_LOG_KEY_REMOVE_WHILE_FREEING, flags); 7275de865eeSFilipe David Borba Manana if (!tm_list[i]) { 7285de865eeSFilipe David Borba Manana ret = -ENOMEM; 7295de865eeSFilipe David Borba Manana goto free_tms; 7305de865eeSFilipe David Borba Manana } 7315de865eeSFilipe David Borba Manana } 7325de865eeSFilipe David Borba Manana } 733d9abbf1cSJan Schmidt 734c8cc6341SJosef Bacik tm = kzalloc(sizeof(*tm), flags); 7355de865eeSFilipe David Borba Manana if (!tm) { 7365de865eeSFilipe David Borba Manana ret = -ENOMEM; 7375de865eeSFilipe David Borba Manana goto free_tms; 7385de865eeSFilipe David Borba Manana } 739bd989ba3SJan Schmidt 740bd989ba3SJan Schmidt tm->index = new_root->start >> PAGE_CACHE_SHIFT; 741bd989ba3SJan Schmidt tm->old_root.logical = old_root->start; 742bd989ba3SJan Schmidt tm->old_root.level = btrfs_header_level(old_root); 743bd989ba3SJan Schmidt tm->generation = btrfs_header_generation(old_root); 744bd989ba3SJan Schmidt tm->op = MOD_LOG_ROOT_REPLACE; 745bd989ba3SJan Schmidt 7465de865eeSFilipe David Borba Manana if (tree_mod_dont_log(fs_info, NULL)) 7475de865eeSFilipe David Borba Manana goto free_tms; 7485de865eeSFilipe David Borba Manana 7495de865eeSFilipe David Borba Manana if (tm_list) 7505de865eeSFilipe David Borba Manana ret = __tree_mod_log_free_eb(fs_info, tm_list, nritems); 7515de865eeSFilipe David Borba Manana if (!ret) 7525de865eeSFilipe David Borba Manana ret = __tree_mod_log_insert(fs_info, tm); 7535de865eeSFilipe David Borba Manana 7545de865eeSFilipe David Borba Manana tree_mod_log_write_unlock(fs_info); 7555de865eeSFilipe David Borba Manana if (ret) 7565de865eeSFilipe David Borba Manana goto free_tms; 7575de865eeSFilipe David Borba Manana kfree(tm_list); 7585de865eeSFilipe David Borba Manana 7595de865eeSFilipe David Borba Manana return ret; 7605de865eeSFilipe David Borba Manana 7615de865eeSFilipe David Borba Manana free_tms: 7625de865eeSFilipe David Borba Manana if (tm_list) { 7635de865eeSFilipe David Borba Manana for (i = 0; i < nritems; i++) 7645de865eeSFilipe David Borba Manana kfree(tm_list[i]); 7655de865eeSFilipe David Borba Manana kfree(tm_list); 7665de865eeSFilipe David Borba Manana } 7675de865eeSFilipe David Borba Manana kfree(tm); 7685de865eeSFilipe David Borba Manana 7695de865eeSFilipe David Borba Manana return ret; 770bd989ba3SJan Schmidt } 771bd989ba3SJan Schmidt 772bd989ba3SJan Schmidt static struct tree_mod_elem * 773bd989ba3SJan Schmidt __tree_mod_log_search(struct btrfs_fs_info *fs_info, u64 start, u64 min_seq, 774bd989ba3SJan Schmidt int smallest) 775bd989ba3SJan Schmidt { 776bd989ba3SJan Schmidt struct rb_root *tm_root; 777bd989ba3SJan Schmidt struct rb_node *node; 778bd989ba3SJan Schmidt struct tree_mod_elem *cur = NULL; 779bd989ba3SJan Schmidt struct tree_mod_elem *found = NULL; 780bd989ba3SJan Schmidt u64 index = start >> PAGE_CACHE_SHIFT; 781bd989ba3SJan Schmidt 782097b8a7cSJan Schmidt tree_mod_log_read_lock(fs_info); 783bd989ba3SJan Schmidt tm_root = &fs_info->tree_mod_log; 784bd989ba3SJan Schmidt node = tm_root->rb_node; 785bd989ba3SJan Schmidt while (node) { 786bd989ba3SJan Schmidt cur = container_of(node, struct tree_mod_elem, node); 787bd989ba3SJan Schmidt if (cur->index < index) { 788bd989ba3SJan Schmidt node = node->rb_left; 789bd989ba3SJan Schmidt } else if (cur->index > index) { 790bd989ba3SJan Schmidt node = node->rb_right; 791097b8a7cSJan Schmidt } else if (cur->seq < min_seq) { 792bd989ba3SJan Schmidt node = node->rb_left; 793bd989ba3SJan Schmidt } else if (!smallest) { 794bd989ba3SJan Schmidt /* we want the node with the highest seq */ 795bd989ba3SJan Schmidt if (found) 796097b8a7cSJan Schmidt BUG_ON(found->seq > cur->seq); 797bd989ba3SJan Schmidt found = cur; 798bd989ba3SJan Schmidt node = node->rb_left; 799097b8a7cSJan Schmidt } else if (cur->seq > min_seq) { 800bd989ba3SJan Schmidt /* we want the node with the smallest seq */ 801bd989ba3SJan Schmidt if (found) 802097b8a7cSJan Schmidt BUG_ON(found->seq < cur->seq); 803bd989ba3SJan Schmidt found = cur; 804bd989ba3SJan Schmidt node = node->rb_right; 805bd989ba3SJan Schmidt } else { 806bd989ba3SJan Schmidt found = cur; 807bd989ba3SJan Schmidt break; 808bd989ba3SJan Schmidt } 809bd989ba3SJan Schmidt } 810097b8a7cSJan Schmidt tree_mod_log_read_unlock(fs_info); 811bd989ba3SJan Schmidt 812bd989ba3SJan Schmidt return found; 813bd989ba3SJan Schmidt } 814bd989ba3SJan Schmidt 815bd989ba3SJan Schmidt /* 816bd989ba3SJan Schmidt * this returns the element from the log with the smallest time sequence 817bd989ba3SJan Schmidt * value that's in the log (the oldest log item). any element with a time 818bd989ba3SJan Schmidt * sequence lower than min_seq will be ignored. 819bd989ba3SJan Schmidt */ 820bd989ba3SJan Schmidt static struct tree_mod_elem * 821bd989ba3SJan Schmidt tree_mod_log_search_oldest(struct btrfs_fs_info *fs_info, u64 start, 822bd989ba3SJan Schmidt u64 min_seq) 823bd989ba3SJan Schmidt { 824bd989ba3SJan Schmidt return __tree_mod_log_search(fs_info, start, min_seq, 1); 825bd989ba3SJan Schmidt } 826bd989ba3SJan Schmidt 827bd989ba3SJan Schmidt /* 828bd989ba3SJan Schmidt * this returns the element from the log with the largest time sequence 829bd989ba3SJan Schmidt * value that's in the log (the most recent log item). any element with 830bd989ba3SJan Schmidt * a time sequence lower than min_seq will be ignored. 831bd989ba3SJan Schmidt */ 832bd989ba3SJan Schmidt static struct tree_mod_elem * 833bd989ba3SJan Schmidt tree_mod_log_search(struct btrfs_fs_info *fs_info, u64 start, u64 min_seq) 834bd989ba3SJan Schmidt { 835bd989ba3SJan Schmidt return __tree_mod_log_search(fs_info, start, min_seq, 0); 836bd989ba3SJan Schmidt } 837bd989ba3SJan Schmidt 8385de865eeSFilipe David Borba Manana static noinline int 839bd989ba3SJan Schmidt tree_mod_log_eb_copy(struct btrfs_fs_info *fs_info, struct extent_buffer *dst, 840bd989ba3SJan Schmidt struct extent_buffer *src, unsigned long dst_offset, 84190f8d62eSJan Schmidt unsigned long src_offset, int nr_items) 842bd989ba3SJan Schmidt { 8435de865eeSFilipe David Borba Manana int ret = 0; 8445de865eeSFilipe David Borba Manana struct tree_mod_elem **tm_list = NULL; 8455de865eeSFilipe David Borba Manana struct tree_mod_elem **tm_list_add, **tm_list_rem; 846bd989ba3SJan Schmidt int i; 8475de865eeSFilipe David Borba Manana int locked = 0; 848bd989ba3SJan Schmidt 8495de865eeSFilipe David Borba Manana if (!tree_mod_need_log(fs_info, NULL)) 8505de865eeSFilipe David Borba Manana return 0; 851bd989ba3SJan Schmidt 852c8cc6341SJosef Bacik if (btrfs_header_level(dst) == 0 && btrfs_header_level(src) == 0) 8535de865eeSFilipe David Borba Manana return 0; 8545de865eeSFilipe David Borba Manana 8555de865eeSFilipe David Borba Manana tm_list = kzalloc(nr_items * 2 * sizeof(struct tree_mod_elem *), 8565de865eeSFilipe David Borba Manana GFP_NOFS); 8575de865eeSFilipe David Borba Manana if (!tm_list) 8585de865eeSFilipe David Borba Manana return -ENOMEM; 8595de865eeSFilipe David Borba Manana 8605de865eeSFilipe David Borba Manana tm_list_add = tm_list; 8615de865eeSFilipe David Borba Manana tm_list_rem = tm_list + nr_items; 8625de865eeSFilipe David Borba Manana for (i = 0; i < nr_items; i++) { 8635de865eeSFilipe David Borba Manana tm_list_rem[i] = alloc_tree_mod_elem(src, i + src_offset, 8645de865eeSFilipe David Borba Manana MOD_LOG_KEY_REMOVE, GFP_NOFS); 8655de865eeSFilipe David Borba Manana if (!tm_list_rem[i]) { 8665de865eeSFilipe David Borba Manana ret = -ENOMEM; 8675de865eeSFilipe David Borba Manana goto free_tms; 8685de865eeSFilipe David Borba Manana } 8695de865eeSFilipe David Borba Manana 8705de865eeSFilipe David Borba Manana tm_list_add[i] = alloc_tree_mod_elem(dst, i + dst_offset, 8715de865eeSFilipe David Borba Manana MOD_LOG_KEY_ADD, GFP_NOFS); 8725de865eeSFilipe David Borba Manana if (!tm_list_add[i]) { 8735de865eeSFilipe David Borba Manana ret = -ENOMEM; 8745de865eeSFilipe David Borba Manana goto free_tms; 8755de865eeSFilipe David Borba Manana } 8765de865eeSFilipe David Borba Manana } 8775de865eeSFilipe David Borba Manana 8785de865eeSFilipe David Borba Manana if (tree_mod_dont_log(fs_info, NULL)) 8795de865eeSFilipe David Borba Manana goto free_tms; 8805de865eeSFilipe David Borba Manana locked = 1; 881bd989ba3SJan Schmidt 882bd989ba3SJan Schmidt for (i = 0; i < nr_items; i++) { 8835de865eeSFilipe David Borba Manana ret = __tree_mod_log_insert(fs_info, tm_list_rem[i]); 8845de865eeSFilipe David Borba Manana if (ret) 8855de865eeSFilipe David Borba Manana goto free_tms; 8865de865eeSFilipe David Borba Manana ret = __tree_mod_log_insert(fs_info, tm_list_add[i]); 8875de865eeSFilipe David Borba Manana if (ret) 8885de865eeSFilipe David Borba Manana goto free_tms; 889bd989ba3SJan Schmidt } 8905de865eeSFilipe David Borba Manana 8915de865eeSFilipe David Borba Manana tree_mod_log_write_unlock(fs_info); 8925de865eeSFilipe David Borba Manana kfree(tm_list); 8935de865eeSFilipe David Borba Manana 8945de865eeSFilipe David Borba Manana return 0; 8955de865eeSFilipe David Borba Manana 8965de865eeSFilipe David Borba Manana free_tms: 8975de865eeSFilipe David Borba Manana for (i = 0; i < nr_items * 2; i++) { 8985de865eeSFilipe David Borba Manana if (tm_list[i] && !RB_EMPTY_NODE(&tm_list[i]->node)) 8995de865eeSFilipe David Borba Manana rb_erase(&tm_list[i]->node, &fs_info->tree_mod_log); 9005de865eeSFilipe David Borba Manana kfree(tm_list[i]); 9015de865eeSFilipe David Borba Manana } 9025de865eeSFilipe David Borba Manana if (locked) 9035de865eeSFilipe David Borba Manana tree_mod_log_write_unlock(fs_info); 9045de865eeSFilipe David Borba Manana kfree(tm_list); 9055de865eeSFilipe David Borba Manana 9065de865eeSFilipe David Borba Manana return ret; 907bd989ba3SJan Schmidt } 908bd989ba3SJan Schmidt 909bd989ba3SJan Schmidt static inline void 910bd989ba3SJan Schmidt tree_mod_log_eb_move(struct btrfs_fs_info *fs_info, struct extent_buffer *dst, 911bd989ba3SJan Schmidt int dst_offset, int src_offset, int nr_items) 912bd989ba3SJan Schmidt { 913bd989ba3SJan Schmidt int ret; 914bd989ba3SJan Schmidt ret = tree_mod_log_insert_move(fs_info, dst, dst_offset, src_offset, 915bd989ba3SJan Schmidt nr_items, GFP_NOFS); 916bd989ba3SJan Schmidt BUG_ON(ret < 0); 917bd989ba3SJan Schmidt } 918bd989ba3SJan Schmidt 919097b8a7cSJan Schmidt static noinline void 920bd989ba3SJan Schmidt tree_mod_log_set_node_key(struct btrfs_fs_info *fs_info, 92132adf090SLiu Bo struct extent_buffer *eb, int slot, int atomic) 922bd989ba3SJan Schmidt { 923bd989ba3SJan Schmidt int ret; 924bd989ba3SJan Schmidt 92578357766SFilipe David Borba Manana ret = tree_mod_log_insert_key(fs_info, eb, slot, 926bd989ba3SJan Schmidt MOD_LOG_KEY_REPLACE, 927bd989ba3SJan Schmidt atomic ? GFP_ATOMIC : GFP_NOFS); 928bd989ba3SJan Schmidt BUG_ON(ret < 0); 929bd989ba3SJan Schmidt } 930bd989ba3SJan Schmidt 9315de865eeSFilipe David Borba Manana static noinline int 932097b8a7cSJan Schmidt tree_mod_log_free_eb(struct btrfs_fs_info *fs_info, struct extent_buffer *eb) 933bd989ba3SJan Schmidt { 9345de865eeSFilipe David Borba Manana struct tree_mod_elem **tm_list = NULL; 9355de865eeSFilipe David Borba Manana int nritems = 0; 9365de865eeSFilipe David Borba Manana int i; 9375de865eeSFilipe David Borba Manana int ret = 0; 9385de865eeSFilipe David Borba Manana 9395de865eeSFilipe David Borba Manana if (btrfs_header_level(eb) == 0) 9405de865eeSFilipe David Borba Manana return 0; 9415de865eeSFilipe David Borba Manana 9425de865eeSFilipe David Borba Manana if (!tree_mod_need_log(fs_info, NULL)) 9435de865eeSFilipe David Borba Manana return 0; 9445de865eeSFilipe David Borba Manana 9455de865eeSFilipe David Borba Manana nritems = btrfs_header_nritems(eb); 9465de865eeSFilipe David Borba Manana tm_list = kzalloc(nritems * sizeof(struct tree_mod_elem *), 9475de865eeSFilipe David Borba Manana GFP_NOFS); 9485de865eeSFilipe David Borba Manana if (!tm_list) 9495de865eeSFilipe David Borba Manana return -ENOMEM; 9505de865eeSFilipe David Borba Manana 9515de865eeSFilipe David Borba Manana for (i = 0; i < nritems; i++) { 9525de865eeSFilipe David Borba Manana tm_list[i] = alloc_tree_mod_elem(eb, i, 9535de865eeSFilipe David Borba Manana MOD_LOG_KEY_REMOVE_WHILE_FREEING, GFP_NOFS); 9545de865eeSFilipe David Borba Manana if (!tm_list[i]) { 9555de865eeSFilipe David Borba Manana ret = -ENOMEM; 9565de865eeSFilipe David Borba Manana goto free_tms; 9575de865eeSFilipe David Borba Manana } 9585de865eeSFilipe David Borba Manana } 9595de865eeSFilipe David Borba Manana 960e9b7fd4dSJan Schmidt if (tree_mod_dont_log(fs_info, eb)) 9615de865eeSFilipe David Borba Manana goto free_tms; 9625de865eeSFilipe David Borba Manana 9635de865eeSFilipe David Borba Manana ret = __tree_mod_log_free_eb(fs_info, tm_list, nritems); 9645de865eeSFilipe David Borba Manana tree_mod_log_write_unlock(fs_info); 9655de865eeSFilipe David Borba Manana if (ret) 9665de865eeSFilipe David Borba Manana goto free_tms; 9675de865eeSFilipe David Borba Manana kfree(tm_list); 9685de865eeSFilipe David Borba Manana 9695de865eeSFilipe David Borba Manana return 0; 9705de865eeSFilipe David Borba Manana 9715de865eeSFilipe David Borba Manana free_tms: 9725de865eeSFilipe David Borba Manana for (i = 0; i < nritems; i++) 9735de865eeSFilipe David Borba Manana kfree(tm_list[i]); 9745de865eeSFilipe David Borba Manana kfree(tm_list); 9755de865eeSFilipe David Borba Manana 9765de865eeSFilipe David Borba Manana return ret; 977bd989ba3SJan Schmidt } 978bd989ba3SJan Schmidt 979097b8a7cSJan Schmidt static noinline void 980bd989ba3SJan Schmidt tree_mod_log_set_root_pointer(struct btrfs_root *root, 98190f8d62eSJan Schmidt struct extent_buffer *new_root_node, 98290f8d62eSJan Schmidt int log_removal) 983bd989ba3SJan Schmidt { 984bd989ba3SJan Schmidt int ret; 985bd989ba3SJan Schmidt ret = tree_mod_log_insert_root(root->fs_info, root->node, 98690f8d62eSJan Schmidt new_root_node, GFP_NOFS, log_removal); 987bd989ba3SJan Schmidt BUG_ON(ret < 0); 988bd989ba3SJan Schmidt } 989bd989ba3SJan Schmidt 990d352ac68SChris Mason /* 9915d4f98a2SYan Zheng * check if the tree block can be shared by multiple trees 9925d4f98a2SYan Zheng */ 9935d4f98a2SYan Zheng int btrfs_block_can_be_shared(struct btrfs_root *root, 9945d4f98a2SYan Zheng struct extent_buffer *buf) 9955d4f98a2SYan Zheng { 9965d4f98a2SYan Zheng /* 9975d4f98a2SYan Zheng * Tree blocks not in refernece counted trees and tree roots 9985d4f98a2SYan Zheng * are never shared. If a block was allocated after the last 9995d4f98a2SYan Zheng * snapshot and the block was not allocated by tree relocation, 10005d4f98a2SYan Zheng * we know the block is not shared. 10015d4f98a2SYan Zheng */ 100227cdeb70SMiao Xie if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) && 10035d4f98a2SYan Zheng buf != root->node && buf != root->commit_root && 10045d4f98a2SYan Zheng (btrfs_header_generation(buf) <= 10055d4f98a2SYan Zheng btrfs_root_last_snapshot(&root->root_item) || 10065d4f98a2SYan Zheng btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC))) 10075d4f98a2SYan Zheng return 1; 10085d4f98a2SYan Zheng #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 100927cdeb70SMiao Xie if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) && 10105d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 10115d4f98a2SYan Zheng return 1; 10125d4f98a2SYan Zheng #endif 10135d4f98a2SYan Zheng return 0; 10145d4f98a2SYan Zheng } 10155d4f98a2SYan Zheng 10165d4f98a2SYan Zheng static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans, 10175d4f98a2SYan Zheng struct btrfs_root *root, 10185d4f98a2SYan Zheng struct extent_buffer *buf, 1019f0486c68SYan, Zheng struct extent_buffer *cow, 1020f0486c68SYan, Zheng int *last_ref) 10215d4f98a2SYan Zheng { 10225d4f98a2SYan Zheng u64 refs; 10235d4f98a2SYan Zheng u64 owner; 10245d4f98a2SYan Zheng u64 flags; 10255d4f98a2SYan Zheng u64 new_flags = 0; 10265d4f98a2SYan Zheng int ret; 10275d4f98a2SYan Zheng 10285d4f98a2SYan Zheng /* 10295d4f98a2SYan Zheng * Backrefs update rules: 10305d4f98a2SYan Zheng * 10315d4f98a2SYan Zheng * Always use full backrefs for extent pointers in tree block 10325d4f98a2SYan Zheng * allocated by tree relocation. 10335d4f98a2SYan Zheng * 10345d4f98a2SYan Zheng * If a shared tree block is no longer referenced by its owner 10355d4f98a2SYan Zheng * tree (btrfs_header_owner(buf) == root->root_key.objectid), 10365d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 10375d4f98a2SYan Zheng * 10385d4f98a2SYan Zheng * If a tree block is been relocating 10395d4f98a2SYan Zheng * (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID), 10405d4f98a2SYan Zheng * use full backrefs for extent pointers in tree block. 10415d4f98a2SYan Zheng * The reason for this is some operations (such as drop tree) 10425d4f98a2SYan Zheng * are only allowed for blocks use full backrefs. 10435d4f98a2SYan Zheng */ 10445d4f98a2SYan Zheng 10455d4f98a2SYan Zheng if (btrfs_block_can_be_shared(root, buf)) { 10465d4f98a2SYan Zheng ret = btrfs_lookup_extent_info(trans, root, buf->start, 10473173a18fSJosef Bacik btrfs_header_level(buf), 1, 10483173a18fSJosef Bacik &refs, &flags); 1049be1a5564SMark Fasheh if (ret) 1050be1a5564SMark Fasheh return ret; 1051e5df9573SMark Fasheh if (refs == 0) { 1052e5df9573SMark Fasheh ret = -EROFS; 1053e5df9573SMark Fasheh btrfs_std_error(root->fs_info, ret); 1054e5df9573SMark Fasheh return ret; 1055e5df9573SMark Fasheh } 10565d4f98a2SYan Zheng } else { 10575d4f98a2SYan Zheng refs = 1; 10585d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 10595d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 10605d4f98a2SYan Zheng flags = BTRFS_BLOCK_FLAG_FULL_BACKREF; 10615d4f98a2SYan Zheng else 10625d4f98a2SYan Zheng flags = 0; 10635d4f98a2SYan Zheng } 10645d4f98a2SYan Zheng 10655d4f98a2SYan Zheng owner = btrfs_header_owner(buf); 10665d4f98a2SYan Zheng BUG_ON(owner == BTRFS_TREE_RELOC_OBJECTID && 10675d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)); 10685d4f98a2SYan Zheng 10695d4f98a2SYan Zheng if (refs > 1) { 10705d4f98a2SYan Zheng if ((owner == root->root_key.objectid || 10715d4f98a2SYan Zheng root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) && 10725d4f98a2SYan Zheng !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) { 107366d7e7f0SArne Jansen ret = btrfs_inc_ref(trans, root, buf, 1, 1); 107479787eaaSJeff Mahoney BUG_ON(ret); /* -ENOMEM */ 10755d4f98a2SYan Zheng 10765d4f98a2SYan Zheng if (root->root_key.objectid == 10775d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) { 107866d7e7f0SArne Jansen ret = btrfs_dec_ref(trans, root, buf, 0, 1); 107979787eaaSJeff Mahoney BUG_ON(ret); /* -ENOMEM */ 108066d7e7f0SArne Jansen ret = btrfs_inc_ref(trans, root, cow, 1, 1); 108179787eaaSJeff Mahoney BUG_ON(ret); /* -ENOMEM */ 10825d4f98a2SYan Zheng } 10835d4f98a2SYan Zheng new_flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF; 10845d4f98a2SYan Zheng } else { 10855d4f98a2SYan Zheng 10865d4f98a2SYan Zheng if (root->root_key.objectid == 10875d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 108866d7e7f0SArne Jansen ret = btrfs_inc_ref(trans, root, cow, 1, 1); 10895d4f98a2SYan Zheng else 109066d7e7f0SArne Jansen ret = btrfs_inc_ref(trans, root, cow, 0, 1); 109179787eaaSJeff Mahoney BUG_ON(ret); /* -ENOMEM */ 10925d4f98a2SYan Zheng } 10935d4f98a2SYan Zheng if (new_flags != 0) { 1094b1c79e09SJosef Bacik int level = btrfs_header_level(buf); 1095b1c79e09SJosef Bacik 10965d4f98a2SYan Zheng ret = btrfs_set_disk_extent_flags(trans, root, 10975d4f98a2SYan Zheng buf->start, 10985d4f98a2SYan Zheng buf->len, 1099b1c79e09SJosef Bacik new_flags, level, 0); 1100be1a5564SMark Fasheh if (ret) 1101be1a5564SMark Fasheh return ret; 11025d4f98a2SYan Zheng } 11035d4f98a2SYan Zheng } else { 11045d4f98a2SYan Zheng if (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) { 11055d4f98a2SYan Zheng if (root->root_key.objectid == 11065d4f98a2SYan Zheng BTRFS_TREE_RELOC_OBJECTID) 110766d7e7f0SArne Jansen ret = btrfs_inc_ref(trans, root, cow, 1, 1); 11085d4f98a2SYan Zheng else 110966d7e7f0SArne Jansen ret = btrfs_inc_ref(trans, root, cow, 0, 1); 111079787eaaSJeff Mahoney BUG_ON(ret); /* -ENOMEM */ 111166d7e7f0SArne Jansen ret = btrfs_dec_ref(trans, root, buf, 1, 1); 111279787eaaSJeff Mahoney BUG_ON(ret); /* -ENOMEM */ 11135d4f98a2SYan Zheng } 11145d4f98a2SYan Zheng clean_tree_block(trans, root, buf); 1115f0486c68SYan, Zheng *last_ref = 1; 11165d4f98a2SYan Zheng } 11175d4f98a2SYan Zheng return 0; 11185d4f98a2SYan Zheng } 11195d4f98a2SYan Zheng 11205d4f98a2SYan Zheng /* 1121d397712bSChris Mason * does the dirty work in cow of a single block. The parent block (if 1122d397712bSChris Mason * supplied) is updated to point to the new cow copy. The new buffer is marked 1123d397712bSChris Mason * dirty and returned locked. If you modify the block it needs to be marked 1124d397712bSChris Mason * dirty again. 1125d352ac68SChris Mason * 1126d352ac68SChris Mason * search_start -- an allocation hint for the new block 1127d352ac68SChris Mason * 1128d397712bSChris Mason * empty_size -- a hint that you plan on doing more cow. This is the size in 1129d397712bSChris Mason * bytes the allocator should try to find free next to the block it returns. 1130d397712bSChris Mason * This is just a hint and may be ignored by the allocator. 1131d352ac68SChris Mason */ 1132d397712bSChris Mason static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans, 11335f39d397SChris Mason struct btrfs_root *root, 11345f39d397SChris Mason struct extent_buffer *buf, 11355f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 11365f39d397SChris Mason struct extent_buffer **cow_ret, 11379fa8cfe7SChris Mason u64 search_start, u64 empty_size) 11386702ed49SChris Mason { 11395d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 11405f39d397SChris Mason struct extent_buffer *cow; 1141be1a5564SMark Fasheh int level, ret; 1142f0486c68SYan, Zheng int last_ref = 0; 1143925baeddSChris Mason int unlock_orig = 0; 11445d4f98a2SYan Zheng u64 parent_start; 11456702ed49SChris Mason 1146925baeddSChris Mason if (*cow_ret == buf) 1147925baeddSChris Mason unlock_orig = 1; 1148925baeddSChris Mason 1149b9447ef8SChris Mason btrfs_assert_tree_locked(buf); 1150925baeddSChris Mason 115127cdeb70SMiao Xie WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) && 115227cdeb70SMiao Xie trans->transid != root->fs_info->running_transaction->transid); 115327cdeb70SMiao Xie WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) && 115427cdeb70SMiao Xie trans->transid != root->last_trans); 11555f39d397SChris Mason 11567bb86316SChris Mason level = btrfs_header_level(buf); 115731840ae1SZheng Yan 11585d4f98a2SYan Zheng if (level == 0) 11595d4f98a2SYan Zheng btrfs_item_key(buf, &disk_key, 0); 11605d4f98a2SYan Zheng else 11615d4f98a2SYan Zheng btrfs_node_key(buf, &disk_key, 0); 11625d4f98a2SYan Zheng 11635d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) { 11645d4f98a2SYan Zheng if (parent) 11655d4f98a2SYan Zheng parent_start = parent->start; 11665d4f98a2SYan Zheng else 11675d4f98a2SYan Zheng parent_start = 0; 11685d4f98a2SYan Zheng } else 11695d4f98a2SYan Zheng parent_start = 0; 11705d4f98a2SYan Zheng 11715d4f98a2SYan Zheng cow = btrfs_alloc_free_block(trans, root, buf->len, parent_start, 11725d4f98a2SYan Zheng root->root_key.objectid, &disk_key, 11735581a51aSJan Schmidt level, search_start, empty_size); 11746702ed49SChris Mason if (IS_ERR(cow)) 11756702ed49SChris Mason return PTR_ERR(cow); 11766702ed49SChris Mason 1177b4ce94deSChris Mason /* cow is set to blocking by btrfs_init_new_buffer */ 1178b4ce94deSChris Mason 11795f39d397SChris Mason copy_extent_buffer(cow, buf, 0, 0, cow->len); 1180db94535dSChris Mason btrfs_set_header_bytenr(cow, cow->start); 11815f39d397SChris Mason btrfs_set_header_generation(cow, trans->transid); 11825d4f98a2SYan Zheng btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); 11835d4f98a2SYan Zheng btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | 11845d4f98a2SYan Zheng BTRFS_HEADER_FLAG_RELOC); 11855d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) 11865d4f98a2SYan Zheng btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); 11875d4f98a2SYan Zheng else 11885f39d397SChris Mason btrfs_set_header_owner(cow, root->root_key.objectid); 11896702ed49SChris Mason 11900a4e5586SRoss Kirk write_extent_buffer(cow, root->fs_info->fsid, btrfs_header_fsid(), 11912b82032cSYan Zheng BTRFS_FSID_SIZE); 11922b82032cSYan Zheng 1193be1a5564SMark Fasheh ret = update_ref_for_cow(trans, root, buf, cow, &last_ref); 1194b68dc2a9SMark Fasheh if (ret) { 119579787eaaSJeff Mahoney btrfs_abort_transaction(trans, root, ret); 1196b68dc2a9SMark Fasheh return ret; 1197b68dc2a9SMark Fasheh } 11981a40e23bSZheng Yan 119927cdeb70SMiao Xie if (test_bit(BTRFS_ROOT_REF_COWS, &root->state)) { 120083d4cfd4SJosef Bacik ret = btrfs_reloc_cow_block(trans, root, buf, cow); 120183d4cfd4SJosef Bacik if (ret) 120283d4cfd4SJosef Bacik return ret; 120383d4cfd4SJosef Bacik } 12043fd0a558SYan, Zheng 12056702ed49SChris Mason if (buf == root->node) { 1206925baeddSChris Mason WARN_ON(parent && parent != buf); 12075d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || 12085d4f98a2SYan Zheng btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) 12095d4f98a2SYan Zheng parent_start = buf->start; 12105d4f98a2SYan Zheng else 12115d4f98a2SYan Zheng parent_start = 0; 1212925baeddSChris Mason 12135f39d397SChris Mason extent_buffer_get(cow); 121490f8d62eSJan Schmidt tree_mod_log_set_root_pointer(root, cow, 1); 1215240f62c8SChris Mason rcu_assign_pointer(root->node, cow); 1216925baeddSChris Mason 1217f0486c68SYan, Zheng btrfs_free_tree_block(trans, root, buf, parent_start, 12185581a51aSJan Schmidt last_ref); 12195f39d397SChris Mason free_extent_buffer(buf); 12200b86a832SChris Mason add_root_to_dirty_list(root); 12216702ed49SChris Mason } else { 12225d4f98a2SYan Zheng if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) 12235d4f98a2SYan Zheng parent_start = parent->start; 12245d4f98a2SYan Zheng else 12255d4f98a2SYan Zheng parent_start = 0; 12265d4f98a2SYan Zheng 12275d4f98a2SYan Zheng WARN_ON(trans->transid != btrfs_header_generation(parent)); 1228f230475eSJan Schmidt tree_mod_log_insert_key(root->fs_info, parent, parent_slot, 1229c8cc6341SJosef Bacik MOD_LOG_KEY_REPLACE, GFP_NOFS); 12305f39d397SChris Mason btrfs_set_node_blockptr(parent, parent_slot, 1231db94535dSChris Mason cow->start); 123274493f7aSChris Mason btrfs_set_node_ptr_generation(parent, parent_slot, 123374493f7aSChris Mason trans->transid); 12346702ed49SChris Mason btrfs_mark_buffer_dirty(parent); 12355de865eeSFilipe David Borba Manana if (last_ref) { 12365de865eeSFilipe David Borba Manana ret = tree_mod_log_free_eb(root->fs_info, buf); 12375de865eeSFilipe David Borba Manana if (ret) { 12385de865eeSFilipe David Borba Manana btrfs_abort_transaction(trans, root, ret); 12395de865eeSFilipe David Borba Manana return ret; 12405de865eeSFilipe David Borba Manana } 12415de865eeSFilipe David Borba Manana } 1242f0486c68SYan, Zheng btrfs_free_tree_block(trans, root, buf, parent_start, 12435581a51aSJan Schmidt last_ref); 12446702ed49SChris Mason } 1245925baeddSChris Mason if (unlock_orig) 1246925baeddSChris Mason btrfs_tree_unlock(buf); 12473083ee2eSJosef Bacik free_extent_buffer_stale(buf); 12486702ed49SChris Mason btrfs_mark_buffer_dirty(cow); 12496702ed49SChris Mason *cow_ret = cow; 12506702ed49SChris Mason return 0; 12516702ed49SChris Mason } 12526702ed49SChris Mason 12535d9e75c4SJan Schmidt /* 12545d9e75c4SJan Schmidt * returns the logical address of the oldest predecessor of the given root. 12555d9e75c4SJan Schmidt * entries older than time_seq are ignored. 12565d9e75c4SJan Schmidt */ 12575d9e75c4SJan Schmidt static struct tree_mod_elem * 12585d9e75c4SJan Schmidt __tree_mod_log_oldest_root(struct btrfs_fs_info *fs_info, 125930b0463aSJan Schmidt struct extent_buffer *eb_root, u64 time_seq) 12605d9e75c4SJan Schmidt { 12615d9e75c4SJan Schmidt struct tree_mod_elem *tm; 12625d9e75c4SJan Schmidt struct tree_mod_elem *found = NULL; 126330b0463aSJan Schmidt u64 root_logical = eb_root->start; 12645d9e75c4SJan Schmidt int looped = 0; 12655d9e75c4SJan Schmidt 12665d9e75c4SJan Schmidt if (!time_seq) 126735a3621bSStefan Behrens return NULL; 12685d9e75c4SJan Schmidt 12695d9e75c4SJan Schmidt /* 12705d9e75c4SJan Schmidt * the very last operation that's logged for a root is the replacement 12715d9e75c4SJan Schmidt * operation (if it is replaced at all). this has the index of the *new* 12725d9e75c4SJan Schmidt * root, making it the very first operation that's logged for this root. 12735d9e75c4SJan Schmidt */ 12745d9e75c4SJan Schmidt while (1) { 12755d9e75c4SJan Schmidt tm = tree_mod_log_search_oldest(fs_info, root_logical, 12765d9e75c4SJan Schmidt time_seq); 12775d9e75c4SJan Schmidt if (!looped && !tm) 127835a3621bSStefan Behrens return NULL; 12795d9e75c4SJan Schmidt /* 128028da9fb4SJan Schmidt * if there are no tree operation for the oldest root, we simply 128128da9fb4SJan Schmidt * return it. this should only happen if that (old) root is at 128228da9fb4SJan Schmidt * level 0. 12835d9e75c4SJan Schmidt */ 128428da9fb4SJan Schmidt if (!tm) 128528da9fb4SJan Schmidt break; 12865d9e75c4SJan Schmidt 128728da9fb4SJan Schmidt /* 128828da9fb4SJan Schmidt * if there's an operation that's not a root replacement, we 128928da9fb4SJan Schmidt * found the oldest version of our root. normally, we'll find a 129028da9fb4SJan Schmidt * MOD_LOG_KEY_REMOVE_WHILE_FREEING operation here. 129128da9fb4SJan Schmidt */ 12925d9e75c4SJan Schmidt if (tm->op != MOD_LOG_ROOT_REPLACE) 12935d9e75c4SJan Schmidt break; 12945d9e75c4SJan Schmidt 12955d9e75c4SJan Schmidt found = tm; 12965d9e75c4SJan Schmidt root_logical = tm->old_root.logical; 12975d9e75c4SJan Schmidt looped = 1; 12985d9e75c4SJan Schmidt } 12995d9e75c4SJan Schmidt 1300a95236d9SJan Schmidt /* if there's no old root to return, return what we found instead */ 1301a95236d9SJan Schmidt if (!found) 1302a95236d9SJan Schmidt found = tm; 1303a95236d9SJan Schmidt 13045d9e75c4SJan Schmidt return found; 13055d9e75c4SJan Schmidt } 13065d9e75c4SJan Schmidt 13075d9e75c4SJan Schmidt /* 13085d9e75c4SJan Schmidt * tm is a pointer to the first operation to rewind within eb. then, all 13095d9e75c4SJan Schmidt * previous operations will be rewinded (until we reach something older than 13105d9e75c4SJan Schmidt * time_seq). 13115d9e75c4SJan Schmidt */ 13125d9e75c4SJan Schmidt static void 1313f1ca7e98SJosef Bacik __tree_mod_log_rewind(struct btrfs_fs_info *fs_info, struct extent_buffer *eb, 1314f1ca7e98SJosef Bacik u64 time_seq, struct tree_mod_elem *first_tm) 13155d9e75c4SJan Schmidt { 13165d9e75c4SJan Schmidt u32 n; 13175d9e75c4SJan Schmidt struct rb_node *next; 13185d9e75c4SJan Schmidt struct tree_mod_elem *tm = first_tm; 13195d9e75c4SJan Schmidt unsigned long o_dst; 13205d9e75c4SJan Schmidt unsigned long o_src; 13215d9e75c4SJan Schmidt unsigned long p_size = sizeof(struct btrfs_key_ptr); 13225d9e75c4SJan Schmidt 13235d9e75c4SJan Schmidt n = btrfs_header_nritems(eb); 1324f1ca7e98SJosef Bacik tree_mod_log_read_lock(fs_info); 1325097b8a7cSJan Schmidt while (tm && tm->seq >= time_seq) { 13265d9e75c4SJan Schmidt /* 13275d9e75c4SJan Schmidt * all the operations are recorded with the operator used for 13285d9e75c4SJan Schmidt * the modification. as we're going backwards, we do the 13295d9e75c4SJan Schmidt * opposite of each operation here. 13305d9e75c4SJan Schmidt */ 13315d9e75c4SJan Schmidt switch (tm->op) { 13325d9e75c4SJan Schmidt case MOD_LOG_KEY_REMOVE_WHILE_FREEING: 13335d9e75c4SJan Schmidt BUG_ON(tm->slot < n); 13341c697d4aSEric Sandeen /* Fallthrough */ 133595c80bb1SLiu Bo case MOD_LOG_KEY_REMOVE_WHILE_MOVING: 13364c3e6969SChris Mason case MOD_LOG_KEY_REMOVE: 13375d9e75c4SJan Schmidt btrfs_set_node_key(eb, &tm->key, tm->slot); 13385d9e75c4SJan Schmidt btrfs_set_node_blockptr(eb, tm->slot, tm->blockptr); 13395d9e75c4SJan Schmidt btrfs_set_node_ptr_generation(eb, tm->slot, 13405d9e75c4SJan Schmidt tm->generation); 13414c3e6969SChris Mason n++; 13425d9e75c4SJan Schmidt break; 13435d9e75c4SJan Schmidt case MOD_LOG_KEY_REPLACE: 13445d9e75c4SJan Schmidt BUG_ON(tm->slot >= n); 13455d9e75c4SJan Schmidt btrfs_set_node_key(eb, &tm->key, tm->slot); 13465d9e75c4SJan Schmidt btrfs_set_node_blockptr(eb, tm->slot, tm->blockptr); 13475d9e75c4SJan Schmidt btrfs_set_node_ptr_generation(eb, tm->slot, 13485d9e75c4SJan Schmidt tm->generation); 13495d9e75c4SJan Schmidt break; 13505d9e75c4SJan Schmidt case MOD_LOG_KEY_ADD: 135119956c7eSJan Schmidt /* if a move operation is needed it's in the log */ 13525d9e75c4SJan Schmidt n--; 13535d9e75c4SJan Schmidt break; 13545d9e75c4SJan Schmidt case MOD_LOG_MOVE_KEYS: 1355c3193108SJan Schmidt o_dst = btrfs_node_key_ptr_offset(tm->slot); 1356c3193108SJan Schmidt o_src = btrfs_node_key_ptr_offset(tm->move.dst_slot); 1357c3193108SJan Schmidt memmove_extent_buffer(eb, o_dst, o_src, 13585d9e75c4SJan Schmidt tm->move.nr_items * p_size); 13595d9e75c4SJan Schmidt break; 13605d9e75c4SJan Schmidt case MOD_LOG_ROOT_REPLACE: 13615d9e75c4SJan Schmidt /* 13625d9e75c4SJan Schmidt * this operation is special. for roots, this must be 13635d9e75c4SJan Schmidt * handled explicitly before rewinding. 13645d9e75c4SJan Schmidt * for non-roots, this operation may exist if the node 13655d9e75c4SJan Schmidt * was a root: root A -> child B; then A gets empty and 13665d9e75c4SJan Schmidt * B is promoted to the new root. in the mod log, we'll 13675d9e75c4SJan Schmidt * have a root-replace operation for B, a tree block 13685d9e75c4SJan Schmidt * that is no root. we simply ignore that operation. 13695d9e75c4SJan Schmidt */ 13705d9e75c4SJan Schmidt break; 13715d9e75c4SJan Schmidt } 13725d9e75c4SJan Schmidt next = rb_next(&tm->node); 13735d9e75c4SJan Schmidt if (!next) 13745d9e75c4SJan Schmidt break; 13755d9e75c4SJan Schmidt tm = container_of(next, struct tree_mod_elem, node); 13765d9e75c4SJan Schmidt if (tm->index != first_tm->index) 13775d9e75c4SJan Schmidt break; 13785d9e75c4SJan Schmidt } 1379f1ca7e98SJosef Bacik tree_mod_log_read_unlock(fs_info); 13805d9e75c4SJan Schmidt btrfs_set_header_nritems(eb, n); 13815d9e75c4SJan Schmidt } 13825d9e75c4SJan Schmidt 138347fb091fSJan Schmidt /* 138447fb091fSJan Schmidt * Called with eb read locked. If the buffer cannot be rewinded, the same buffer 138547fb091fSJan Schmidt * is returned. If rewind operations happen, a fresh buffer is returned. The 138647fb091fSJan Schmidt * returned buffer is always read-locked. If the returned buffer is not the 138747fb091fSJan Schmidt * input buffer, the lock on the input buffer is released and the input buffer 138847fb091fSJan Schmidt * is freed (its refcount is decremented). 138947fb091fSJan Schmidt */ 13905d9e75c4SJan Schmidt static struct extent_buffer * 13919ec72677SJosef Bacik tree_mod_log_rewind(struct btrfs_fs_info *fs_info, struct btrfs_path *path, 13929ec72677SJosef Bacik struct extent_buffer *eb, u64 time_seq) 13935d9e75c4SJan Schmidt { 13945d9e75c4SJan Schmidt struct extent_buffer *eb_rewin; 13955d9e75c4SJan Schmidt struct tree_mod_elem *tm; 13965d9e75c4SJan Schmidt 13975d9e75c4SJan Schmidt if (!time_seq) 13985d9e75c4SJan Schmidt return eb; 13995d9e75c4SJan Schmidt 14005d9e75c4SJan Schmidt if (btrfs_header_level(eb) == 0) 14015d9e75c4SJan Schmidt return eb; 14025d9e75c4SJan Schmidt 14035d9e75c4SJan Schmidt tm = tree_mod_log_search(fs_info, eb->start, time_seq); 14045d9e75c4SJan Schmidt if (!tm) 14055d9e75c4SJan Schmidt return eb; 14065d9e75c4SJan Schmidt 14079ec72677SJosef Bacik btrfs_set_path_blocking(path); 14089ec72677SJosef Bacik btrfs_set_lock_blocking_rw(eb, BTRFS_READ_LOCK); 14099ec72677SJosef Bacik 14105d9e75c4SJan Schmidt if (tm->op == MOD_LOG_KEY_REMOVE_WHILE_FREEING) { 14115d9e75c4SJan Schmidt BUG_ON(tm->slot != 0); 14125d9e75c4SJan Schmidt eb_rewin = alloc_dummy_extent_buffer(eb->start, 14135d9e75c4SJan Schmidt fs_info->tree_root->nodesize); 1414db7f3436SJosef Bacik if (!eb_rewin) { 14159ec72677SJosef Bacik btrfs_tree_read_unlock_blocking(eb); 1416db7f3436SJosef Bacik free_extent_buffer(eb); 1417db7f3436SJosef Bacik return NULL; 1418db7f3436SJosef Bacik } 14195d9e75c4SJan Schmidt btrfs_set_header_bytenr(eb_rewin, eb->start); 14205d9e75c4SJan Schmidt btrfs_set_header_backref_rev(eb_rewin, 14215d9e75c4SJan Schmidt btrfs_header_backref_rev(eb)); 14225d9e75c4SJan Schmidt btrfs_set_header_owner(eb_rewin, btrfs_header_owner(eb)); 1423c3193108SJan Schmidt btrfs_set_header_level(eb_rewin, btrfs_header_level(eb)); 14245d9e75c4SJan Schmidt } else { 14255d9e75c4SJan Schmidt eb_rewin = btrfs_clone_extent_buffer(eb); 1426db7f3436SJosef Bacik if (!eb_rewin) { 14279ec72677SJosef Bacik btrfs_tree_read_unlock_blocking(eb); 1428db7f3436SJosef Bacik free_extent_buffer(eb); 1429db7f3436SJosef Bacik return NULL; 1430db7f3436SJosef Bacik } 14315d9e75c4SJan Schmidt } 14325d9e75c4SJan Schmidt 14339ec72677SJosef Bacik btrfs_clear_path_blocking(path, NULL, BTRFS_READ_LOCK); 14349ec72677SJosef Bacik btrfs_tree_read_unlock_blocking(eb); 14355d9e75c4SJan Schmidt free_extent_buffer(eb); 14365d9e75c4SJan Schmidt 143747fb091fSJan Schmidt extent_buffer_get(eb_rewin); 143847fb091fSJan Schmidt btrfs_tree_read_lock(eb_rewin); 1439f1ca7e98SJosef Bacik __tree_mod_log_rewind(fs_info, eb_rewin, time_seq, tm); 144057911b8bSJan Schmidt WARN_ON(btrfs_header_nritems(eb_rewin) > 14412a745b14SArne Jansen BTRFS_NODEPTRS_PER_BLOCK(fs_info->tree_root)); 14425d9e75c4SJan Schmidt 14435d9e75c4SJan Schmidt return eb_rewin; 14445d9e75c4SJan Schmidt } 14455d9e75c4SJan Schmidt 14468ba97a15SJan Schmidt /* 14478ba97a15SJan Schmidt * get_old_root() rewinds the state of @root's root node to the given @time_seq 14488ba97a15SJan Schmidt * value. If there are no changes, the current root->root_node is returned. If 14498ba97a15SJan Schmidt * anything changed in between, there's a fresh buffer allocated on which the 14508ba97a15SJan Schmidt * rewind operations are done. In any case, the returned buffer is read locked. 14518ba97a15SJan Schmidt * Returns NULL on error (with no locks held). 14528ba97a15SJan Schmidt */ 14535d9e75c4SJan Schmidt static inline struct extent_buffer * 14545d9e75c4SJan Schmidt get_old_root(struct btrfs_root *root, u64 time_seq) 14555d9e75c4SJan Schmidt { 14565d9e75c4SJan Schmidt struct tree_mod_elem *tm; 145730b0463aSJan Schmidt struct extent_buffer *eb = NULL; 145830b0463aSJan Schmidt struct extent_buffer *eb_root; 14597bfdcf7fSLiu Bo struct extent_buffer *old; 1460a95236d9SJan Schmidt struct tree_mod_root *old_root = NULL; 14614325edd0SChris Mason u64 old_generation = 0; 1462a95236d9SJan Schmidt u64 logical; 1463834328a8SJan Schmidt u32 blocksize; 14645d9e75c4SJan Schmidt 146530b0463aSJan Schmidt eb_root = btrfs_read_lock_root_node(root); 146630b0463aSJan Schmidt tm = __tree_mod_log_oldest_root(root->fs_info, eb_root, time_seq); 14675d9e75c4SJan Schmidt if (!tm) 146830b0463aSJan Schmidt return eb_root; 14695d9e75c4SJan Schmidt 1470a95236d9SJan Schmidt if (tm->op == MOD_LOG_ROOT_REPLACE) { 14715d9e75c4SJan Schmidt old_root = &tm->old_root; 14725d9e75c4SJan Schmidt old_generation = tm->generation; 1473a95236d9SJan Schmidt logical = old_root->logical; 1474a95236d9SJan Schmidt } else { 147530b0463aSJan Schmidt logical = eb_root->start; 1476a95236d9SJan Schmidt } 14775d9e75c4SJan Schmidt 1478a95236d9SJan Schmidt tm = tree_mod_log_search(root->fs_info, logical, time_seq); 1479834328a8SJan Schmidt if (old_root && tm && tm->op != MOD_LOG_KEY_REMOVE_WHILE_FREEING) { 148030b0463aSJan Schmidt btrfs_tree_read_unlock(eb_root); 148130b0463aSJan Schmidt free_extent_buffer(eb_root); 1482834328a8SJan Schmidt blocksize = btrfs_level_size(root, old_root->level); 14837bfdcf7fSLiu Bo old = read_tree_block(root, logical, blocksize, 0); 1484fae7f21cSDulshani Gunawardhana if (WARN_ON(!old || !extent_buffer_uptodate(old))) { 1485416bc658SJosef Bacik free_extent_buffer(old); 1486efe120a0SFrank Holton btrfs_warn(root->fs_info, 1487efe120a0SFrank Holton "failed to read tree block %llu from get_old_root", logical); 1488834328a8SJan Schmidt } else { 14897bfdcf7fSLiu Bo eb = btrfs_clone_extent_buffer(old); 14907bfdcf7fSLiu Bo free_extent_buffer(old); 1491834328a8SJan Schmidt } 1492834328a8SJan Schmidt } else if (old_root) { 149330b0463aSJan Schmidt btrfs_tree_read_unlock(eb_root); 149430b0463aSJan Schmidt free_extent_buffer(eb_root); 149528da9fb4SJan Schmidt eb = alloc_dummy_extent_buffer(logical, root->nodesize); 1496834328a8SJan Schmidt } else { 14979ec72677SJosef Bacik btrfs_set_lock_blocking_rw(eb_root, BTRFS_READ_LOCK); 149830b0463aSJan Schmidt eb = btrfs_clone_extent_buffer(eb_root); 14999ec72677SJosef Bacik btrfs_tree_read_unlock_blocking(eb_root); 150030b0463aSJan Schmidt free_extent_buffer(eb_root); 1501834328a8SJan Schmidt } 1502834328a8SJan Schmidt 15038ba97a15SJan Schmidt if (!eb) 15048ba97a15SJan Schmidt return NULL; 1505d6381084SJan Schmidt extent_buffer_get(eb); 15068ba97a15SJan Schmidt btrfs_tree_read_lock(eb); 1507a95236d9SJan Schmidt if (old_root) { 15085d9e75c4SJan Schmidt btrfs_set_header_bytenr(eb, eb->start); 15095d9e75c4SJan Schmidt btrfs_set_header_backref_rev(eb, BTRFS_MIXED_BACKREF_REV); 151030b0463aSJan Schmidt btrfs_set_header_owner(eb, btrfs_header_owner(eb_root)); 15115d9e75c4SJan Schmidt btrfs_set_header_level(eb, old_root->level); 15125d9e75c4SJan Schmidt btrfs_set_header_generation(eb, old_generation); 1513a95236d9SJan Schmidt } 151428da9fb4SJan Schmidt if (tm) 1515f1ca7e98SJosef Bacik __tree_mod_log_rewind(root->fs_info, eb, time_seq, tm); 151628da9fb4SJan Schmidt else 151728da9fb4SJan Schmidt WARN_ON(btrfs_header_level(eb) != 0); 151857911b8bSJan Schmidt WARN_ON(btrfs_header_nritems(eb) > BTRFS_NODEPTRS_PER_BLOCK(root)); 15195d9e75c4SJan Schmidt 15205d9e75c4SJan Schmidt return eb; 15215d9e75c4SJan Schmidt } 15225d9e75c4SJan Schmidt 15235b6602e7SJan Schmidt int btrfs_old_root_level(struct btrfs_root *root, u64 time_seq) 15245b6602e7SJan Schmidt { 15255b6602e7SJan Schmidt struct tree_mod_elem *tm; 15265b6602e7SJan Schmidt int level; 152730b0463aSJan Schmidt struct extent_buffer *eb_root = btrfs_root_node(root); 15285b6602e7SJan Schmidt 152930b0463aSJan Schmidt tm = __tree_mod_log_oldest_root(root->fs_info, eb_root, time_seq); 15305b6602e7SJan Schmidt if (tm && tm->op == MOD_LOG_ROOT_REPLACE) { 15315b6602e7SJan Schmidt level = tm->old_root.level; 15325b6602e7SJan Schmidt } else { 153330b0463aSJan Schmidt level = btrfs_header_level(eb_root); 15345b6602e7SJan Schmidt } 153530b0463aSJan Schmidt free_extent_buffer(eb_root); 15365b6602e7SJan Schmidt 15375b6602e7SJan Schmidt return level; 15385b6602e7SJan Schmidt } 15395b6602e7SJan Schmidt 15405d4f98a2SYan Zheng static inline int should_cow_block(struct btrfs_trans_handle *trans, 15415d4f98a2SYan Zheng struct btrfs_root *root, 15425d4f98a2SYan Zheng struct extent_buffer *buf) 15435d4f98a2SYan Zheng { 1544f1ebcc74SLiu Bo /* ensure we can see the force_cow */ 1545f1ebcc74SLiu Bo smp_rmb(); 1546f1ebcc74SLiu Bo 1547f1ebcc74SLiu Bo /* 1548f1ebcc74SLiu Bo * We do not need to cow a block if 1549f1ebcc74SLiu Bo * 1) this block is not created or changed in this transaction; 1550f1ebcc74SLiu Bo * 2) this block does not belong to TREE_RELOC tree; 1551f1ebcc74SLiu Bo * 3) the root is not forced COW. 1552f1ebcc74SLiu Bo * 1553f1ebcc74SLiu Bo * What is forced COW: 1554f1ebcc74SLiu Bo * when we create snapshot during commiting the transaction, 1555f1ebcc74SLiu Bo * after we've finished coping src root, we must COW the shared 1556f1ebcc74SLiu Bo * block to ensure the metadata consistency. 1557f1ebcc74SLiu Bo */ 15585d4f98a2SYan Zheng if (btrfs_header_generation(buf) == trans->transid && 15595d4f98a2SYan Zheng !btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN) && 15605d4f98a2SYan Zheng !(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID && 1561f1ebcc74SLiu Bo btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)) && 156227cdeb70SMiao Xie !test_bit(BTRFS_ROOT_FORCE_COW, &root->state)) 15635d4f98a2SYan Zheng return 0; 15645d4f98a2SYan Zheng return 1; 15655d4f98a2SYan Zheng } 15665d4f98a2SYan Zheng 1567d352ac68SChris Mason /* 1568d352ac68SChris Mason * cows a single block, see __btrfs_cow_block for the real work. 1569d352ac68SChris Mason * This version of it has extra checks so that a block isn't cow'd more than 1570d352ac68SChris Mason * once per transaction, as long as it hasn't been written yet 1571d352ac68SChris Mason */ 1572d397712bSChris Mason noinline int btrfs_cow_block(struct btrfs_trans_handle *trans, 15735f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *buf, 15745f39d397SChris Mason struct extent_buffer *parent, int parent_slot, 15759fa8cfe7SChris Mason struct extent_buffer **cow_ret) 157602217ed2SChris Mason { 15776702ed49SChris Mason u64 search_start; 1578f510cfecSChris Mason int ret; 1579dc17ff8fSChris Mason 158031b1a2bdSJulia Lawall if (trans->transaction != root->fs_info->running_transaction) 158131b1a2bdSJulia Lawall WARN(1, KERN_CRIT "trans %llu running %llu\n", 1582c1c9ff7cSGeert Uytterhoeven trans->transid, 1583ccd467d6SChris Mason root->fs_info->running_transaction->transid); 158431b1a2bdSJulia Lawall 158531b1a2bdSJulia Lawall if (trans->transid != root->fs_info->generation) 158631b1a2bdSJulia Lawall WARN(1, KERN_CRIT "trans %llu running %llu\n", 1587c1c9ff7cSGeert Uytterhoeven trans->transid, root->fs_info->generation); 1588dc17ff8fSChris Mason 15895d4f98a2SYan Zheng if (!should_cow_block(trans, root, buf)) { 159002217ed2SChris Mason *cow_ret = buf; 159102217ed2SChris Mason return 0; 159202217ed2SChris Mason } 1593c487685dSChris Mason 15940b86a832SChris Mason search_start = buf->start & ~((u64)(1024 * 1024 * 1024) - 1); 1595b4ce94deSChris Mason 1596b4ce94deSChris Mason if (parent) 1597b4ce94deSChris Mason btrfs_set_lock_blocking(parent); 1598b4ce94deSChris Mason btrfs_set_lock_blocking(buf); 1599b4ce94deSChris Mason 1600f510cfecSChris Mason ret = __btrfs_cow_block(trans, root, buf, parent, 16019fa8cfe7SChris Mason parent_slot, cow_ret, search_start, 0); 16021abe9b8aSliubo 16031abe9b8aSliubo trace_btrfs_cow_block(root, buf, *cow_ret); 16041abe9b8aSliubo 1605f510cfecSChris Mason return ret; 16062c90e5d6SChris Mason } 16076702ed49SChris Mason 1608d352ac68SChris Mason /* 1609d352ac68SChris Mason * helper function for defrag to decide if two blocks pointed to by a 1610d352ac68SChris Mason * node are actually close by 1611d352ac68SChris Mason */ 16126b80053dSChris Mason static int close_blocks(u64 blocknr, u64 other, u32 blocksize) 16136702ed49SChris Mason { 16146b80053dSChris Mason if (blocknr < other && other - (blocknr + blocksize) < 32768) 16156702ed49SChris Mason return 1; 16166b80053dSChris Mason if (blocknr > other && blocknr - (other + blocksize) < 32768) 16176702ed49SChris Mason return 1; 161802217ed2SChris Mason return 0; 161902217ed2SChris Mason } 162002217ed2SChris Mason 1621081e9573SChris Mason /* 1622081e9573SChris Mason * compare two keys in a memcmp fashion 1623081e9573SChris Mason */ 1624081e9573SChris Mason static int comp_keys(struct btrfs_disk_key *disk, struct btrfs_key *k2) 1625081e9573SChris Mason { 1626081e9573SChris Mason struct btrfs_key k1; 1627081e9573SChris Mason 1628081e9573SChris Mason btrfs_disk_key_to_cpu(&k1, disk); 1629081e9573SChris Mason 163020736abaSDiego Calleja return btrfs_comp_cpu_keys(&k1, k2); 1631081e9573SChris Mason } 1632081e9573SChris Mason 1633f3465ca4SJosef Bacik /* 1634f3465ca4SJosef Bacik * same as comp_keys only with two btrfs_key's 1635f3465ca4SJosef Bacik */ 16365d4f98a2SYan Zheng int btrfs_comp_cpu_keys(struct btrfs_key *k1, struct btrfs_key *k2) 1637f3465ca4SJosef Bacik { 1638f3465ca4SJosef Bacik if (k1->objectid > k2->objectid) 1639f3465ca4SJosef Bacik return 1; 1640f3465ca4SJosef Bacik if (k1->objectid < k2->objectid) 1641f3465ca4SJosef Bacik return -1; 1642f3465ca4SJosef Bacik if (k1->type > k2->type) 1643f3465ca4SJosef Bacik return 1; 1644f3465ca4SJosef Bacik if (k1->type < k2->type) 1645f3465ca4SJosef Bacik return -1; 1646f3465ca4SJosef Bacik if (k1->offset > k2->offset) 1647f3465ca4SJosef Bacik return 1; 1648f3465ca4SJosef Bacik if (k1->offset < k2->offset) 1649f3465ca4SJosef Bacik return -1; 1650f3465ca4SJosef Bacik return 0; 1651f3465ca4SJosef Bacik } 1652081e9573SChris Mason 1653d352ac68SChris Mason /* 1654d352ac68SChris Mason * this is used by the defrag code to go through all the 1655d352ac68SChris Mason * leaves pointed to by a node and reallocate them so that 1656d352ac68SChris Mason * disk order is close to key order 1657d352ac68SChris Mason */ 16586702ed49SChris Mason int btrfs_realloc_node(struct btrfs_trans_handle *trans, 16595f39d397SChris Mason struct btrfs_root *root, struct extent_buffer *parent, 1660de78b51aSEric Sandeen int start_slot, u64 *last_ret, 1661a6b6e75eSChris Mason struct btrfs_key *progress) 16626702ed49SChris Mason { 16636b80053dSChris Mason struct extent_buffer *cur; 16646702ed49SChris Mason u64 blocknr; 1665ca7a79adSChris Mason u64 gen; 1666e9d0b13bSChris Mason u64 search_start = *last_ret; 1667e9d0b13bSChris Mason u64 last_block = 0; 16686702ed49SChris Mason u64 other; 16696702ed49SChris Mason u32 parent_nritems; 16706702ed49SChris Mason int end_slot; 16716702ed49SChris Mason int i; 16726702ed49SChris Mason int err = 0; 1673f2183bdeSChris Mason int parent_level; 16746b80053dSChris Mason int uptodate; 16756b80053dSChris Mason u32 blocksize; 1676081e9573SChris Mason int progress_passed = 0; 1677081e9573SChris Mason struct btrfs_disk_key disk_key; 16786702ed49SChris Mason 16795708b959SChris Mason parent_level = btrfs_header_level(parent); 16805708b959SChris Mason 16816c1500f2SJulia Lawall WARN_ON(trans->transaction != root->fs_info->running_transaction); 16826c1500f2SJulia Lawall WARN_ON(trans->transid != root->fs_info->generation); 168386479a04SChris Mason 16846b80053dSChris Mason parent_nritems = btrfs_header_nritems(parent); 16856b80053dSChris Mason blocksize = btrfs_level_size(root, parent_level - 1); 16866702ed49SChris Mason end_slot = parent_nritems; 16876702ed49SChris Mason 16886702ed49SChris Mason if (parent_nritems == 1) 16896702ed49SChris Mason return 0; 16906702ed49SChris Mason 1691b4ce94deSChris Mason btrfs_set_lock_blocking(parent); 1692b4ce94deSChris Mason 16936702ed49SChris Mason for (i = start_slot; i < end_slot; i++) { 16946702ed49SChris Mason int close = 1; 1695a6b6e75eSChris Mason 1696081e9573SChris Mason btrfs_node_key(parent, &disk_key, i); 1697081e9573SChris Mason if (!progress_passed && comp_keys(&disk_key, progress) < 0) 1698081e9573SChris Mason continue; 1699081e9573SChris Mason 1700081e9573SChris Mason progress_passed = 1; 17016b80053dSChris Mason blocknr = btrfs_node_blockptr(parent, i); 1702ca7a79adSChris Mason gen = btrfs_node_ptr_generation(parent, i); 1703e9d0b13bSChris Mason if (last_block == 0) 1704e9d0b13bSChris Mason last_block = blocknr; 17055708b959SChris Mason 17066702ed49SChris Mason if (i > 0) { 17076b80053dSChris Mason other = btrfs_node_blockptr(parent, i - 1); 17086b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 17096702ed49SChris Mason } 17100ef3e66bSChris Mason if (!close && i < end_slot - 2) { 17116b80053dSChris Mason other = btrfs_node_blockptr(parent, i + 1); 17126b80053dSChris Mason close = close_blocks(blocknr, other, blocksize); 17136702ed49SChris Mason } 1714e9d0b13bSChris Mason if (close) { 1715e9d0b13bSChris Mason last_block = blocknr; 17166702ed49SChris Mason continue; 1717e9d0b13bSChris Mason } 17186702ed49SChris Mason 17196b80053dSChris Mason cur = btrfs_find_tree_block(root, blocknr, blocksize); 17206b80053dSChris Mason if (cur) 1721b9fab919SChris Mason uptodate = btrfs_buffer_uptodate(cur, gen, 0); 17226b80053dSChris Mason else 17236b80053dSChris Mason uptodate = 0; 17245708b959SChris Mason if (!cur || !uptodate) { 17256b80053dSChris Mason if (!cur) { 17266b80053dSChris Mason cur = read_tree_block(root, blocknr, 1727ca7a79adSChris Mason blocksize, gen); 1728416bc658SJosef Bacik if (!cur || !extent_buffer_uptodate(cur)) { 1729416bc658SJosef Bacik free_extent_buffer(cur); 173097d9a8a4STsutomu Itoh return -EIO; 1731416bc658SJosef Bacik } 17326b80053dSChris Mason } else if (!uptodate) { 1733018642a1STsutomu Itoh err = btrfs_read_buffer(cur, gen); 1734018642a1STsutomu Itoh if (err) { 1735018642a1STsutomu Itoh free_extent_buffer(cur); 1736018642a1STsutomu Itoh return err; 1737018642a1STsutomu Itoh } 17386702ed49SChris Mason } 1739f2183bdeSChris Mason } 1740e9d0b13bSChris Mason if (search_start == 0) 17416b80053dSChris Mason search_start = last_block; 1742e9d0b13bSChris Mason 1743e7a84565SChris Mason btrfs_tree_lock(cur); 1744b4ce94deSChris Mason btrfs_set_lock_blocking(cur); 17456b80053dSChris Mason err = __btrfs_cow_block(trans, root, cur, parent, i, 1746e7a84565SChris Mason &cur, search_start, 17476b80053dSChris Mason min(16 * blocksize, 17489fa8cfe7SChris Mason (end_slot - i) * blocksize)); 1749252c38f0SYan if (err) { 1750e7a84565SChris Mason btrfs_tree_unlock(cur); 17516b80053dSChris Mason free_extent_buffer(cur); 17526702ed49SChris Mason break; 1753252c38f0SYan } 1754e7a84565SChris Mason search_start = cur->start; 1755e7a84565SChris Mason last_block = cur->start; 1756f2183bdeSChris Mason *last_ret = search_start; 1757e7a84565SChris Mason btrfs_tree_unlock(cur); 1758e7a84565SChris Mason free_extent_buffer(cur); 17596702ed49SChris Mason } 17606702ed49SChris Mason return err; 17616702ed49SChris Mason } 17626702ed49SChris Mason 176374123bd7SChris Mason /* 176474123bd7SChris Mason * The leaf data grows from end-to-front in the node. 176574123bd7SChris Mason * this returns the address of the start of the last item, 176674123bd7SChris Mason * which is the stop of the leaf data stack 176774123bd7SChris Mason */ 1768123abc88SChris Mason static inline unsigned int leaf_data_end(struct btrfs_root *root, 17695f39d397SChris Mason struct extent_buffer *leaf) 1770be0e5c09SChris Mason { 17715f39d397SChris Mason u32 nr = btrfs_header_nritems(leaf); 1772be0e5c09SChris Mason if (nr == 0) 1773123abc88SChris Mason return BTRFS_LEAF_DATA_SIZE(root); 17745f39d397SChris Mason return btrfs_item_offset_nr(leaf, nr - 1); 1775be0e5c09SChris Mason } 1776be0e5c09SChris Mason 1777aa5d6bedSChris Mason 177874123bd7SChris Mason /* 17795f39d397SChris Mason * search for key in the extent_buffer. The items start at offset p, 17805f39d397SChris Mason * and they are item_size apart. There are 'max' items in p. 17815f39d397SChris Mason * 178274123bd7SChris Mason * the slot in the array is returned via slot, and it points to 178374123bd7SChris Mason * the place where you would insert key if it is not found in 178474123bd7SChris Mason * the array. 178574123bd7SChris Mason * 178674123bd7SChris Mason * slot may point to max if the key is bigger than all of the keys 178774123bd7SChris Mason */ 1788e02119d5SChris Mason static noinline int generic_bin_search(struct extent_buffer *eb, 1789e02119d5SChris Mason unsigned long p, 17905f39d397SChris Mason int item_size, struct btrfs_key *key, 1791be0e5c09SChris Mason int max, int *slot) 1792be0e5c09SChris Mason { 1793be0e5c09SChris Mason int low = 0; 1794be0e5c09SChris Mason int high = max; 1795be0e5c09SChris Mason int mid; 1796be0e5c09SChris Mason int ret; 1797479965d6SChris Mason struct btrfs_disk_key *tmp = NULL; 17985f39d397SChris Mason struct btrfs_disk_key unaligned; 17995f39d397SChris Mason unsigned long offset; 18005f39d397SChris Mason char *kaddr = NULL; 18015f39d397SChris Mason unsigned long map_start = 0; 18025f39d397SChris Mason unsigned long map_len = 0; 1803479965d6SChris Mason int err; 1804be0e5c09SChris Mason 1805be0e5c09SChris Mason while (low < high) { 1806be0e5c09SChris Mason mid = (low + high) / 2; 18075f39d397SChris Mason offset = p + mid * item_size; 18085f39d397SChris Mason 1809a6591715SChris Mason if (!kaddr || offset < map_start || 18105f39d397SChris Mason (offset + sizeof(struct btrfs_disk_key)) > 18115f39d397SChris Mason map_start + map_len) { 1812934d375bSChris Mason 1813934d375bSChris Mason err = map_private_extent_buffer(eb, offset, 1814479965d6SChris Mason sizeof(struct btrfs_disk_key), 1815a6591715SChris Mason &kaddr, &map_start, &map_len); 18165f39d397SChris Mason 1817479965d6SChris Mason if (!err) { 1818479965d6SChris Mason tmp = (struct btrfs_disk_key *)(kaddr + offset - 1819479965d6SChris Mason map_start); 1820479965d6SChris Mason } else { 18215f39d397SChris Mason read_extent_buffer(eb, &unaligned, 18225f39d397SChris Mason offset, sizeof(unaligned)); 18235f39d397SChris Mason tmp = &unaligned; 1824479965d6SChris Mason } 1825479965d6SChris Mason 18265f39d397SChris Mason } else { 18275f39d397SChris Mason tmp = (struct btrfs_disk_key *)(kaddr + offset - 18285f39d397SChris Mason map_start); 18295f39d397SChris Mason } 1830be0e5c09SChris Mason ret = comp_keys(tmp, key); 1831be0e5c09SChris Mason 1832be0e5c09SChris Mason if (ret < 0) 1833be0e5c09SChris Mason low = mid + 1; 1834be0e5c09SChris Mason else if (ret > 0) 1835be0e5c09SChris Mason high = mid; 1836be0e5c09SChris Mason else { 1837be0e5c09SChris Mason *slot = mid; 1838be0e5c09SChris Mason return 0; 1839be0e5c09SChris Mason } 1840be0e5c09SChris Mason } 1841be0e5c09SChris Mason *slot = low; 1842be0e5c09SChris Mason return 1; 1843be0e5c09SChris Mason } 1844be0e5c09SChris Mason 184597571fd0SChris Mason /* 184697571fd0SChris Mason * simple bin_search frontend that does the right thing for 184797571fd0SChris Mason * leaves vs nodes 184897571fd0SChris Mason */ 18495f39d397SChris Mason static int bin_search(struct extent_buffer *eb, struct btrfs_key *key, 18505f39d397SChris Mason int level, int *slot) 1851be0e5c09SChris Mason { 1852f775738fSWang Sheng-Hui if (level == 0) 18535f39d397SChris Mason return generic_bin_search(eb, 18545f39d397SChris Mason offsetof(struct btrfs_leaf, items), 18550783fcfcSChris Mason sizeof(struct btrfs_item), 18565f39d397SChris Mason key, btrfs_header_nritems(eb), 18577518a238SChris Mason slot); 1858f775738fSWang Sheng-Hui else 18595f39d397SChris Mason return generic_bin_search(eb, 18605f39d397SChris Mason offsetof(struct btrfs_node, ptrs), 1861123abc88SChris Mason sizeof(struct btrfs_key_ptr), 18625f39d397SChris Mason key, btrfs_header_nritems(eb), 18637518a238SChris Mason slot); 1864be0e5c09SChris Mason } 1865be0e5c09SChris Mason 18665d4f98a2SYan Zheng int btrfs_bin_search(struct extent_buffer *eb, struct btrfs_key *key, 18675d4f98a2SYan Zheng int level, int *slot) 18685d4f98a2SYan Zheng { 18695d4f98a2SYan Zheng return bin_search(eb, key, level, slot); 18705d4f98a2SYan Zheng } 18715d4f98a2SYan Zheng 1872f0486c68SYan, Zheng static void root_add_used(struct btrfs_root *root, u32 size) 1873f0486c68SYan, Zheng { 1874f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 1875f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 1876f0486c68SYan, Zheng btrfs_root_used(&root->root_item) + size); 1877f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 1878f0486c68SYan, Zheng } 1879f0486c68SYan, Zheng 1880f0486c68SYan, Zheng static void root_sub_used(struct btrfs_root *root, u32 size) 1881f0486c68SYan, Zheng { 1882f0486c68SYan, Zheng spin_lock(&root->accounting_lock); 1883f0486c68SYan, Zheng btrfs_set_root_used(&root->root_item, 1884f0486c68SYan, Zheng btrfs_root_used(&root->root_item) - size); 1885f0486c68SYan, Zheng spin_unlock(&root->accounting_lock); 1886f0486c68SYan, Zheng } 1887f0486c68SYan, Zheng 1888d352ac68SChris Mason /* given a node and slot number, this reads the blocks it points to. The 1889d352ac68SChris Mason * extent buffer is returned with a reference taken (but unlocked). 1890d352ac68SChris Mason * NULL is returned on error. 1891d352ac68SChris Mason */ 1892e02119d5SChris Mason static noinline struct extent_buffer *read_node_slot(struct btrfs_root *root, 18935f39d397SChris Mason struct extent_buffer *parent, int slot) 1894bb803951SChris Mason { 1895ca7a79adSChris Mason int level = btrfs_header_level(parent); 1896416bc658SJosef Bacik struct extent_buffer *eb; 1897416bc658SJosef Bacik 1898bb803951SChris Mason if (slot < 0) 1899bb803951SChris Mason return NULL; 19005f39d397SChris Mason if (slot >= btrfs_header_nritems(parent)) 1901bb803951SChris Mason return NULL; 1902ca7a79adSChris Mason 1903ca7a79adSChris Mason BUG_ON(level == 0); 1904ca7a79adSChris Mason 1905416bc658SJosef Bacik eb = read_tree_block(root, btrfs_node_blockptr(parent, slot), 1906ca7a79adSChris Mason btrfs_level_size(root, level - 1), 1907ca7a79adSChris Mason btrfs_node_ptr_generation(parent, slot)); 1908416bc658SJosef Bacik if (eb && !extent_buffer_uptodate(eb)) { 1909416bc658SJosef Bacik free_extent_buffer(eb); 1910416bc658SJosef Bacik eb = NULL; 1911416bc658SJosef Bacik } 1912416bc658SJosef Bacik 1913416bc658SJosef Bacik return eb; 1914bb803951SChris Mason } 1915bb803951SChris Mason 1916d352ac68SChris Mason /* 1917d352ac68SChris Mason * node level balancing, used to make sure nodes are in proper order for 1918d352ac68SChris Mason * item deletion. We balance from the top down, so we have to make sure 1919d352ac68SChris Mason * that a deletion won't leave an node completely empty later on. 1920d352ac68SChris Mason */ 1921e02119d5SChris Mason static noinline int balance_level(struct btrfs_trans_handle *trans, 192298ed5174SChris Mason struct btrfs_root *root, 192398ed5174SChris Mason struct btrfs_path *path, int level) 1924bb803951SChris Mason { 19255f39d397SChris Mason struct extent_buffer *right = NULL; 19265f39d397SChris Mason struct extent_buffer *mid; 19275f39d397SChris Mason struct extent_buffer *left = NULL; 19285f39d397SChris Mason struct extent_buffer *parent = NULL; 1929bb803951SChris Mason int ret = 0; 1930bb803951SChris Mason int wret; 1931bb803951SChris Mason int pslot; 1932bb803951SChris Mason int orig_slot = path->slots[level]; 193379f95c82SChris Mason u64 orig_ptr; 1934bb803951SChris Mason 1935bb803951SChris Mason if (level == 0) 1936bb803951SChris Mason return 0; 1937bb803951SChris Mason 19385f39d397SChris Mason mid = path->nodes[level]; 1939b4ce94deSChris Mason 1940bd681513SChris Mason WARN_ON(path->locks[level] != BTRFS_WRITE_LOCK && 1941bd681513SChris Mason path->locks[level] != BTRFS_WRITE_LOCK_BLOCKING); 19427bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 19437bb86316SChris Mason 19441d4f8a0cSChris Mason orig_ptr = btrfs_node_blockptr(mid, orig_slot); 194579f95c82SChris Mason 1946a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 19475f39d397SChris Mason parent = path->nodes[level + 1]; 1948bb803951SChris Mason pslot = path->slots[level + 1]; 1949a05a9bb1SLi Zefan } 1950bb803951SChris Mason 195140689478SChris Mason /* 195240689478SChris Mason * deal with the case where there is only one pointer in the root 195340689478SChris Mason * by promoting the node below to a root 195440689478SChris Mason */ 19555f39d397SChris Mason if (!parent) { 19565f39d397SChris Mason struct extent_buffer *child; 1957bb803951SChris Mason 19585f39d397SChris Mason if (btrfs_header_nritems(mid) != 1) 1959bb803951SChris Mason return 0; 1960bb803951SChris Mason 1961bb803951SChris Mason /* promote the child to a root */ 19625f39d397SChris Mason child = read_node_slot(root, mid, 0); 1963305a26afSMark Fasheh if (!child) { 1964305a26afSMark Fasheh ret = -EROFS; 1965305a26afSMark Fasheh btrfs_std_error(root->fs_info, ret); 1966305a26afSMark Fasheh goto enospc; 1967305a26afSMark Fasheh } 1968305a26afSMark Fasheh 1969925baeddSChris Mason btrfs_tree_lock(child); 1970b4ce94deSChris Mason btrfs_set_lock_blocking(child); 19719fa8cfe7SChris Mason ret = btrfs_cow_block(trans, root, child, mid, 0, &child); 1972f0486c68SYan, Zheng if (ret) { 1973f0486c68SYan, Zheng btrfs_tree_unlock(child); 1974f0486c68SYan, Zheng free_extent_buffer(child); 1975f0486c68SYan, Zheng goto enospc; 1976f0486c68SYan, Zheng } 19772f375ab9SYan 197890f8d62eSJan Schmidt tree_mod_log_set_root_pointer(root, child, 1); 1979240f62c8SChris Mason rcu_assign_pointer(root->node, child); 1980925baeddSChris Mason 19810b86a832SChris Mason add_root_to_dirty_list(root); 1982925baeddSChris Mason btrfs_tree_unlock(child); 1983b4ce94deSChris Mason 1984925baeddSChris Mason path->locks[level] = 0; 1985bb803951SChris Mason path->nodes[level] = NULL; 19865f39d397SChris Mason clean_tree_block(trans, root, mid); 1987925baeddSChris Mason btrfs_tree_unlock(mid); 1988bb803951SChris Mason /* once for the path */ 19895f39d397SChris Mason free_extent_buffer(mid); 1990f0486c68SYan, Zheng 1991f0486c68SYan, Zheng root_sub_used(root, mid->len); 19925581a51aSJan Schmidt btrfs_free_tree_block(trans, root, mid, 0, 1); 1993bb803951SChris Mason /* once for the root ptr */ 19943083ee2eSJosef Bacik free_extent_buffer_stale(mid); 1995f0486c68SYan, Zheng return 0; 1996bb803951SChris Mason } 19975f39d397SChris Mason if (btrfs_header_nritems(mid) > 1998123abc88SChris Mason BTRFS_NODEPTRS_PER_BLOCK(root) / 4) 1999bb803951SChris Mason return 0; 2000bb803951SChris Mason 20015f39d397SChris Mason left = read_node_slot(root, parent, pslot - 1); 20025f39d397SChris Mason if (left) { 2003925baeddSChris Mason btrfs_tree_lock(left); 2004b4ce94deSChris Mason btrfs_set_lock_blocking(left); 20055f39d397SChris Mason wret = btrfs_cow_block(trans, root, left, 20069fa8cfe7SChris Mason parent, pslot - 1, &left); 200754aa1f4dSChris Mason if (wret) { 200854aa1f4dSChris Mason ret = wret; 200954aa1f4dSChris Mason goto enospc; 201054aa1f4dSChris Mason } 20112cc58cf2SChris Mason } 20125f39d397SChris Mason right = read_node_slot(root, parent, pslot + 1); 20135f39d397SChris Mason if (right) { 2014925baeddSChris Mason btrfs_tree_lock(right); 2015b4ce94deSChris Mason btrfs_set_lock_blocking(right); 20165f39d397SChris Mason wret = btrfs_cow_block(trans, root, right, 20179fa8cfe7SChris Mason parent, pslot + 1, &right); 20182cc58cf2SChris Mason if (wret) { 20192cc58cf2SChris Mason ret = wret; 20202cc58cf2SChris Mason goto enospc; 20212cc58cf2SChris Mason } 20222cc58cf2SChris Mason } 20232cc58cf2SChris Mason 20242cc58cf2SChris Mason /* first, try to make some room in the middle buffer */ 20255f39d397SChris Mason if (left) { 20265f39d397SChris Mason orig_slot += btrfs_header_nritems(left); 2027bce4eae9SChris Mason wret = push_node_left(trans, root, left, mid, 1); 202879f95c82SChris Mason if (wret < 0) 202979f95c82SChris Mason ret = wret; 2030bb803951SChris Mason } 203179f95c82SChris Mason 203279f95c82SChris Mason /* 203379f95c82SChris Mason * then try to empty the right most buffer into the middle 203479f95c82SChris Mason */ 20355f39d397SChris Mason if (right) { 2036971a1f66SChris Mason wret = push_node_left(trans, root, mid, right, 1); 203754aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 203879f95c82SChris Mason ret = wret; 20395f39d397SChris Mason if (btrfs_header_nritems(right) == 0) { 20405f39d397SChris Mason clean_tree_block(trans, root, right); 2041925baeddSChris Mason btrfs_tree_unlock(right); 2042afe5fea7STsutomu Itoh del_ptr(root, path, level + 1, pslot + 1); 2043f0486c68SYan, Zheng root_sub_used(root, right->len); 20445581a51aSJan Schmidt btrfs_free_tree_block(trans, root, right, 0, 1); 20453083ee2eSJosef Bacik free_extent_buffer_stale(right); 2046f0486c68SYan, Zheng right = NULL; 2047bb803951SChris Mason } else { 20485f39d397SChris Mason struct btrfs_disk_key right_key; 20495f39d397SChris Mason btrfs_node_key(right, &right_key, 0); 2050f230475eSJan Schmidt tree_mod_log_set_node_key(root->fs_info, parent, 205132adf090SLiu Bo pslot + 1, 0); 20525f39d397SChris Mason btrfs_set_node_key(parent, &right_key, pslot + 1); 20535f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 2054bb803951SChris Mason } 2055bb803951SChris Mason } 20565f39d397SChris Mason if (btrfs_header_nritems(mid) == 1) { 205779f95c82SChris Mason /* 205879f95c82SChris Mason * we're not allowed to leave a node with one item in the 205979f95c82SChris Mason * tree during a delete. A deletion from lower in the tree 206079f95c82SChris Mason * could try to delete the only pointer in this node. 206179f95c82SChris Mason * So, pull some keys from the left. 206279f95c82SChris Mason * There has to be a left pointer at this point because 206379f95c82SChris Mason * otherwise we would have pulled some pointers from the 206479f95c82SChris Mason * right 206579f95c82SChris Mason */ 2066305a26afSMark Fasheh if (!left) { 2067305a26afSMark Fasheh ret = -EROFS; 2068305a26afSMark Fasheh btrfs_std_error(root->fs_info, ret); 2069305a26afSMark Fasheh goto enospc; 2070305a26afSMark Fasheh } 20715f39d397SChris Mason wret = balance_node_right(trans, root, mid, left); 207254aa1f4dSChris Mason if (wret < 0) { 207379f95c82SChris Mason ret = wret; 207454aa1f4dSChris Mason goto enospc; 207554aa1f4dSChris Mason } 2076bce4eae9SChris Mason if (wret == 1) { 2077bce4eae9SChris Mason wret = push_node_left(trans, root, left, mid, 1); 2078bce4eae9SChris Mason if (wret < 0) 2079bce4eae9SChris Mason ret = wret; 2080bce4eae9SChris Mason } 208179f95c82SChris Mason BUG_ON(wret == 1); 208279f95c82SChris Mason } 20835f39d397SChris Mason if (btrfs_header_nritems(mid) == 0) { 20845f39d397SChris Mason clean_tree_block(trans, root, mid); 2085925baeddSChris Mason btrfs_tree_unlock(mid); 2086afe5fea7STsutomu Itoh del_ptr(root, path, level + 1, pslot); 2087f0486c68SYan, Zheng root_sub_used(root, mid->len); 20885581a51aSJan Schmidt btrfs_free_tree_block(trans, root, mid, 0, 1); 20893083ee2eSJosef Bacik free_extent_buffer_stale(mid); 2090f0486c68SYan, Zheng mid = NULL; 209179f95c82SChris Mason } else { 209279f95c82SChris Mason /* update the parent key to reflect our changes */ 20935f39d397SChris Mason struct btrfs_disk_key mid_key; 20945f39d397SChris Mason btrfs_node_key(mid, &mid_key, 0); 209532adf090SLiu Bo tree_mod_log_set_node_key(root->fs_info, parent, 2096f230475eSJan Schmidt pslot, 0); 20975f39d397SChris Mason btrfs_set_node_key(parent, &mid_key, pslot); 20985f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 209979f95c82SChris Mason } 2100bb803951SChris Mason 210179f95c82SChris Mason /* update the path */ 21025f39d397SChris Mason if (left) { 21035f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 21045f39d397SChris Mason extent_buffer_get(left); 2105925baeddSChris Mason /* left was locked after cow */ 21065f39d397SChris Mason path->nodes[level] = left; 2107bb803951SChris Mason path->slots[level + 1] -= 1; 2108bb803951SChris Mason path->slots[level] = orig_slot; 2109925baeddSChris Mason if (mid) { 2110925baeddSChris Mason btrfs_tree_unlock(mid); 21115f39d397SChris Mason free_extent_buffer(mid); 2112925baeddSChris Mason } 2113bb803951SChris Mason } else { 21145f39d397SChris Mason orig_slot -= btrfs_header_nritems(left); 2115bb803951SChris Mason path->slots[level] = orig_slot; 2116bb803951SChris Mason } 2117bb803951SChris Mason } 211879f95c82SChris Mason /* double check we haven't messed things up */ 2119e20d96d6SChris Mason if (orig_ptr != 21205f39d397SChris Mason btrfs_node_blockptr(path->nodes[level], path->slots[level])) 212179f95c82SChris Mason BUG(); 212254aa1f4dSChris Mason enospc: 2123925baeddSChris Mason if (right) { 2124925baeddSChris Mason btrfs_tree_unlock(right); 21255f39d397SChris Mason free_extent_buffer(right); 2126925baeddSChris Mason } 2127925baeddSChris Mason if (left) { 2128925baeddSChris Mason if (path->nodes[level] != left) 2129925baeddSChris Mason btrfs_tree_unlock(left); 21305f39d397SChris Mason free_extent_buffer(left); 2131925baeddSChris Mason } 2132bb803951SChris Mason return ret; 2133bb803951SChris Mason } 2134bb803951SChris Mason 2135d352ac68SChris Mason /* Node balancing for insertion. Here we only split or push nodes around 2136d352ac68SChris Mason * when they are completely full. This is also done top down, so we 2137d352ac68SChris Mason * have to be pessimistic. 2138d352ac68SChris Mason */ 2139d397712bSChris Mason static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans, 2140e66f709bSChris Mason struct btrfs_root *root, 2141e66f709bSChris Mason struct btrfs_path *path, int level) 2142e66f709bSChris Mason { 21435f39d397SChris Mason struct extent_buffer *right = NULL; 21445f39d397SChris Mason struct extent_buffer *mid; 21455f39d397SChris Mason struct extent_buffer *left = NULL; 21465f39d397SChris Mason struct extent_buffer *parent = NULL; 2147e66f709bSChris Mason int ret = 0; 2148e66f709bSChris Mason int wret; 2149e66f709bSChris Mason int pslot; 2150e66f709bSChris Mason int orig_slot = path->slots[level]; 2151e66f709bSChris Mason 2152e66f709bSChris Mason if (level == 0) 2153e66f709bSChris Mason return 1; 2154e66f709bSChris Mason 21555f39d397SChris Mason mid = path->nodes[level]; 21567bb86316SChris Mason WARN_ON(btrfs_header_generation(mid) != trans->transid); 2157e66f709bSChris Mason 2158a05a9bb1SLi Zefan if (level < BTRFS_MAX_LEVEL - 1) { 21595f39d397SChris Mason parent = path->nodes[level + 1]; 2160e66f709bSChris Mason pslot = path->slots[level + 1]; 2161a05a9bb1SLi Zefan } 2162e66f709bSChris Mason 21635f39d397SChris Mason if (!parent) 2164e66f709bSChris Mason return 1; 2165e66f709bSChris Mason 21665f39d397SChris Mason left = read_node_slot(root, parent, pslot - 1); 2167e66f709bSChris Mason 2168e66f709bSChris Mason /* first, try to make some room in the middle buffer */ 21695f39d397SChris Mason if (left) { 2170e66f709bSChris Mason u32 left_nr; 2171925baeddSChris Mason 2172925baeddSChris Mason btrfs_tree_lock(left); 2173b4ce94deSChris Mason btrfs_set_lock_blocking(left); 2174b4ce94deSChris Mason 21755f39d397SChris Mason left_nr = btrfs_header_nritems(left); 217633ade1f8SChris Mason if (left_nr >= BTRFS_NODEPTRS_PER_BLOCK(root) - 1) { 217733ade1f8SChris Mason wret = 1; 217833ade1f8SChris Mason } else { 21795f39d397SChris Mason ret = btrfs_cow_block(trans, root, left, parent, 21809fa8cfe7SChris Mason pslot - 1, &left); 218154aa1f4dSChris Mason if (ret) 218254aa1f4dSChris Mason wret = 1; 218354aa1f4dSChris Mason else { 218454aa1f4dSChris Mason wret = push_node_left(trans, root, 2185971a1f66SChris Mason left, mid, 0); 218654aa1f4dSChris Mason } 218733ade1f8SChris Mason } 2188e66f709bSChris Mason if (wret < 0) 2189e66f709bSChris Mason ret = wret; 2190e66f709bSChris Mason if (wret == 0) { 21915f39d397SChris Mason struct btrfs_disk_key disk_key; 2192e66f709bSChris Mason orig_slot += left_nr; 21935f39d397SChris Mason btrfs_node_key(mid, &disk_key, 0); 2194f230475eSJan Schmidt tree_mod_log_set_node_key(root->fs_info, parent, 219532adf090SLiu Bo pslot, 0); 21965f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot); 21975f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 21985f39d397SChris Mason if (btrfs_header_nritems(left) > orig_slot) { 21995f39d397SChris Mason path->nodes[level] = left; 2200e66f709bSChris Mason path->slots[level + 1] -= 1; 2201e66f709bSChris Mason path->slots[level] = orig_slot; 2202925baeddSChris Mason btrfs_tree_unlock(mid); 22035f39d397SChris Mason free_extent_buffer(mid); 2204e66f709bSChris Mason } else { 2205e66f709bSChris Mason orig_slot -= 22065f39d397SChris Mason btrfs_header_nritems(left); 2207e66f709bSChris Mason path->slots[level] = orig_slot; 2208925baeddSChris Mason btrfs_tree_unlock(left); 22095f39d397SChris Mason free_extent_buffer(left); 2210e66f709bSChris Mason } 2211e66f709bSChris Mason return 0; 2212e66f709bSChris Mason } 2213925baeddSChris Mason btrfs_tree_unlock(left); 22145f39d397SChris Mason free_extent_buffer(left); 2215e66f709bSChris Mason } 22165f39d397SChris Mason right = read_node_slot(root, parent, pslot + 1); 2217e66f709bSChris Mason 2218e66f709bSChris Mason /* 2219e66f709bSChris Mason * then try to empty the right most buffer into the middle 2220e66f709bSChris Mason */ 22215f39d397SChris Mason if (right) { 222233ade1f8SChris Mason u32 right_nr; 2223b4ce94deSChris Mason 2224925baeddSChris Mason btrfs_tree_lock(right); 2225b4ce94deSChris Mason btrfs_set_lock_blocking(right); 2226b4ce94deSChris Mason 22275f39d397SChris Mason right_nr = btrfs_header_nritems(right); 222833ade1f8SChris Mason if (right_nr >= BTRFS_NODEPTRS_PER_BLOCK(root) - 1) { 222933ade1f8SChris Mason wret = 1; 223033ade1f8SChris Mason } else { 22315f39d397SChris Mason ret = btrfs_cow_block(trans, root, right, 22325f39d397SChris Mason parent, pslot + 1, 22339fa8cfe7SChris Mason &right); 223454aa1f4dSChris Mason if (ret) 223554aa1f4dSChris Mason wret = 1; 223654aa1f4dSChris Mason else { 223733ade1f8SChris Mason wret = balance_node_right(trans, root, 22385f39d397SChris Mason right, mid); 223933ade1f8SChris Mason } 224054aa1f4dSChris Mason } 2241e66f709bSChris Mason if (wret < 0) 2242e66f709bSChris Mason ret = wret; 2243e66f709bSChris Mason if (wret == 0) { 22445f39d397SChris Mason struct btrfs_disk_key disk_key; 22455f39d397SChris Mason 22465f39d397SChris Mason btrfs_node_key(right, &disk_key, 0); 2247f230475eSJan Schmidt tree_mod_log_set_node_key(root->fs_info, parent, 224832adf090SLiu Bo pslot + 1, 0); 22495f39d397SChris Mason btrfs_set_node_key(parent, &disk_key, pslot + 1); 22505f39d397SChris Mason btrfs_mark_buffer_dirty(parent); 22515f39d397SChris Mason 22525f39d397SChris Mason if (btrfs_header_nritems(mid) <= orig_slot) { 22535f39d397SChris Mason path->nodes[level] = right; 2254e66f709bSChris Mason path->slots[level + 1] += 1; 2255e66f709bSChris Mason path->slots[level] = orig_slot - 22565f39d397SChris Mason btrfs_header_nritems(mid); 2257925baeddSChris Mason btrfs_tree_unlock(mid); 22585f39d397SChris Mason free_extent_buffer(mid); 2259e66f709bSChris Mason } else { 2260925baeddSChris Mason btrfs_tree_unlock(right); 22615f39d397SChris Mason free_extent_buffer(right); 2262e66f709bSChris Mason } 2263e66f709bSChris Mason return 0; 2264e66f709bSChris Mason } 2265925baeddSChris Mason btrfs_tree_unlock(right); 22665f39d397SChris Mason free_extent_buffer(right); 2267e66f709bSChris Mason } 2268e66f709bSChris Mason return 1; 2269e66f709bSChris Mason } 2270e66f709bSChris Mason 227174123bd7SChris Mason /* 2272d352ac68SChris Mason * readahead one full node of leaves, finding things that are close 2273d352ac68SChris Mason * to the block in 'slot', and triggering ra on them. 22743c69faecSChris Mason */ 2275c8c42864SChris Mason static void reada_for_search(struct btrfs_root *root, 2276e02119d5SChris Mason struct btrfs_path *path, 227701f46658SChris Mason int level, int slot, u64 objectid) 22783c69faecSChris Mason { 22795f39d397SChris Mason struct extent_buffer *node; 228001f46658SChris Mason struct btrfs_disk_key disk_key; 22813c69faecSChris Mason u32 nritems; 22823c69faecSChris Mason u64 search; 2283a7175319SChris Mason u64 target; 22846b80053dSChris Mason u64 nread = 0; 2285cb25c2eaSJosef Bacik u64 gen; 22863c69faecSChris Mason int direction = path->reada; 22875f39d397SChris Mason struct extent_buffer *eb; 22886b80053dSChris Mason u32 nr; 22896b80053dSChris Mason u32 blocksize; 22906b80053dSChris Mason u32 nscan = 0; 2291db94535dSChris Mason 2292a6b6e75eSChris Mason if (level != 1) 22933c69faecSChris Mason return; 22943c69faecSChris Mason 22956702ed49SChris Mason if (!path->nodes[level]) 22966702ed49SChris Mason return; 22976702ed49SChris Mason 22985f39d397SChris Mason node = path->nodes[level]; 2299925baeddSChris Mason 23003c69faecSChris Mason search = btrfs_node_blockptr(node, slot); 23016b80053dSChris Mason blocksize = btrfs_level_size(root, level - 1); 23026b80053dSChris Mason eb = btrfs_find_tree_block(root, search, blocksize); 23035f39d397SChris Mason if (eb) { 23045f39d397SChris Mason free_extent_buffer(eb); 23053c69faecSChris Mason return; 23063c69faecSChris Mason } 23073c69faecSChris Mason 2308a7175319SChris Mason target = search; 23096b80053dSChris Mason 23105f39d397SChris Mason nritems = btrfs_header_nritems(node); 23116b80053dSChris Mason nr = slot; 231225b8b936SJosef Bacik 23133c69faecSChris Mason while (1) { 23146b80053dSChris Mason if (direction < 0) { 23156b80053dSChris Mason if (nr == 0) 23163c69faecSChris Mason break; 23176b80053dSChris Mason nr--; 23186b80053dSChris Mason } else if (direction > 0) { 23196b80053dSChris Mason nr++; 23206b80053dSChris Mason if (nr >= nritems) 23216b80053dSChris Mason break; 23223c69faecSChris Mason } 232301f46658SChris Mason if (path->reada < 0 && objectid) { 232401f46658SChris Mason btrfs_node_key(node, &disk_key, nr); 232501f46658SChris Mason if (btrfs_disk_key_objectid(&disk_key) != objectid) 232601f46658SChris Mason break; 232701f46658SChris Mason } 23286b80053dSChris Mason search = btrfs_node_blockptr(node, nr); 2329a7175319SChris Mason if ((search <= target && target - search <= 65536) || 2330a7175319SChris Mason (search > target && search - target <= 65536)) { 2331cb25c2eaSJosef Bacik gen = btrfs_node_ptr_generation(node, nr); 2332cb25c2eaSJosef Bacik readahead_tree_block(root, search, blocksize, gen); 23336b80053dSChris Mason nread += blocksize; 23343c69faecSChris Mason } 23356b80053dSChris Mason nscan++; 2336a7175319SChris Mason if ((nread > 65536 || nscan > 32)) 23376b80053dSChris Mason break; 23383c69faecSChris Mason } 23393c69faecSChris Mason } 2340925baeddSChris Mason 23410b08851fSJosef Bacik static noinline void reada_for_balance(struct btrfs_root *root, 2342b4ce94deSChris Mason struct btrfs_path *path, int level) 2343b4ce94deSChris Mason { 2344b4ce94deSChris Mason int slot; 2345b4ce94deSChris Mason int nritems; 2346b4ce94deSChris Mason struct extent_buffer *parent; 2347b4ce94deSChris Mason struct extent_buffer *eb; 2348b4ce94deSChris Mason u64 gen; 2349b4ce94deSChris Mason u64 block1 = 0; 2350b4ce94deSChris Mason u64 block2 = 0; 2351b4ce94deSChris Mason int blocksize; 2352b4ce94deSChris Mason 23538c594ea8SChris Mason parent = path->nodes[level + 1]; 2354b4ce94deSChris Mason if (!parent) 23550b08851fSJosef Bacik return; 2356b4ce94deSChris Mason 2357b4ce94deSChris Mason nritems = btrfs_header_nritems(parent); 23588c594ea8SChris Mason slot = path->slots[level + 1]; 2359b4ce94deSChris Mason blocksize = btrfs_level_size(root, level); 2360b4ce94deSChris Mason 2361b4ce94deSChris Mason if (slot > 0) { 2362b4ce94deSChris Mason block1 = btrfs_node_blockptr(parent, slot - 1); 2363b4ce94deSChris Mason gen = btrfs_node_ptr_generation(parent, slot - 1); 2364b4ce94deSChris Mason eb = btrfs_find_tree_block(root, block1, blocksize); 2365b9fab919SChris Mason /* 2366b9fab919SChris Mason * if we get -eagain from btrfs_buffer_uptodate, we 2367b9fab919SChris Mason * don't want to return eagain here. That will loop 2368b9fab919SChris Mason * forever 2369b9fab919SChris Mason */ 2370b9fab919SChris Mason if (eb && btrfs_buffer_uptodate(eb, gen, 1) != 0) 2371b4ce94deSChris Mason block1 = 0; 2372b4ce94deSChris Mason free_extent_buffer(eb); 2373b4ce94deSChris Mason } 23748c594ea8SChris Mason if (slot + 1 < nritems) { 2375b4ce94deSChris Mason block2 = btrfs_node_blockptr(parent, slot + 1); 2376b4ce94deSChris Mason gen = btrfs_node_ptr_generation(parent, slot + 1); 2377b4ce94deSChris Mason eb = btrfs_find_tree_block(root, block2, blocksize); 2378b9fab919SChris Mason if (eb && btrfs_buffer_uptodate(eb, gen, 1) != 0) 2379b4ce94deSChris Mason block2 = 0; 2380b4ce94deSChris Mason free_extent_buffer(eb); 2381b4ce94deSChris Mason } 23828c594ea8SChris Mason 2383b4ce94deSChris Mason if (block1) 2384b4ce94deSChris Mason readahead_tree_block(root, block1, blocksize, 0); 2385b4ce94deSChris Mason if (block2) 2386b4ce94deSChris Mason readahead_tree_block(root, block2, blocksize, 0); 2387b4ce94deSChris Mason } 2388b4ce94deSChris Mason 2389b4ce94deSChris Mason 2390b4ce94deSChris Mason /* 2391d397712bSChris Mason * when we walk down the tree, it is usually safe to unlock the higher layers 2392d397712bSChris Mason * in the tree. The exceptions are when our path goes through slot 0, because 2393d397712bSChris Mason * operations on the tree might require changing key pointers higher up in the 2394d397712bSChris Mason * tree. 2395d352ac68SChris Mason * 2396d397712bSChris Mason * callers might also have set path->keep_locks, which tells this code to keep 2397d397712bSChris Mason * the lock if the path points to the last slot in the block. This is part of 2398d397712bSChris Mason * walking through the tree, and selecting the next slot in the higher block. 2399d352ac68SChris Mason * 2400d397712bSChris Mason * lowest_unlock sets the lowest level in the tree we're allowed to unlock. so 2401d397712bSChris Mason * if lowest_unlock is 1, level 0 won't be unlocked 2402d352ac68SChris Mason */ 2403e02119d5SChris Mason static noinline void unlock_up(struct btrfs_path *path, int level, 2404f7c79f30SChris Mason int lowest_unlock, int min_write_lock_level, 2405f7c79f30SChris Mason int *write_lock_level) 2406925baeddSChris Mason { 2407925baeddSChris Mason int i; 2408925baeddSChris Mason int skip_level = level; 2409051e1b9fSChris Mason int no_skips = 0; 2410925baeddSChris Mason struct extent_buffer *t; 2411925baeddSChris Mason 2412925baeddSChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 2413925baeddSChris Mason if (!path->nodes[i]) 2414925baeddSChris Mason break; 2415925baeddSChris Mason if (!path->locks[i]) 2416925baeddSChris Mason break; 2417051e1b9fSChris Mason if (!no_skips && path->slots[i] == 0) { 2418925baeddSChris Mason skip_level = i + 1; 2419925baeddSChris Mason continue; 2420925baeddSChris Mason } 2421051e1b9fSChris Mason if (!no_skips && path->keep_locks) { 2422925baeddSChris Mason u32 nritems; 2423925baeddSChris Mason t = path->nodes[i]; 2424925baeddSChris Mason nritems = btrfs_header_nritems(t); 2425051e1b9fSChris Mason if (nritems < 1 || path->slots[i] >= nritems - 1) { 2426925baeddSChris Mason skip_level = i + 1; 2427925baeddSChris Mason continue; 2428925baeddSChris Mason } 2429925baeddSChris Mason } 2430051e1b9fSChris Mason if (skip_level < i && i >= lowest_unlock) 2431051e1b9fSChris Mason no_skips = 1; 2432051e1b9fSChris Mason 2433925baeddSChris Mason t = path->nodes[i]; 2434925baeddSChris Mason if (i >= lowest_unlock && i > skip_level && path->locks[i]) { 2435bd681513SChris Mason btrfs_tree_unlock_rw(t, path->locks[i]); 2436925baeddSChris Mason path->locks[i] = 0; 2437f7c79f30SChris Mason if (write_lock_level && 2438f7c79f30SChris Mason i > min_write_lock_level && 2439f7c79f30SChris Mason i <= *write_lock_level) { 2440f7c79f30SChris Mason *write_lock_level = i - 1; 2441f7c79f30SChris Mason } 2442925baeddSChris Mason } 2443925baeddSChris Mason } 2444925baeddSChris Mason } 2445925baeddSChris Mason 24463c69faecSChris Mason /* 2447b4ce94deSChris Mason * This releases any locks held in the path starting at level and 2448b4ce94deSChris Mason * going all the way up to the root. 2449b4ce94deSChris Mason * 2450b4ce94deSChris Mason * btrfs_search_slot will keep the lock held on higher nodes in a few 2451b4ce94deSChris Mason * corner cases, such as COW of the block at slot zero in the node. This 2452b4ce94deSChris Mason * ignores those rules, and it should only be called when there are no 2453b4ce94deSChris Mason * more updates to be done higher up in the tree. 2454b4ce94deSChris Mason */ 2455b4ce94deSChris Mason noinline void btrfs_unlock_up_safe(struct btrfs_path *path, int level) 2456b4ce94deSChris Mason { 2457b4ce94deSChris Mason int i; 2458b4ce94deSChris Mason 245909a2a8f9SJosef Bacik if (path->keep_locks) 2460b4ce94deSChris Mason return; 2461b4ce94deSChris Mason 2462b4ce94deSChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 2463b4ce94deSChris Mason if (!path->nodes[i]) 246412f4daccSChris Mason continue; 2465b4ce94deSChris Mason if (!path->locks[i]) 246612f4daccSChris Mason continue; 2467bd681513SChris Mason btrfs_tree_unlock_rw(path->nodes[i], path->locks[i]); 2468b4ce94deSChris Mason path->locks[i] = 0; 2469b4ce94deSChris Mason } 2470b4ce94deSChris Mason } 2471b4ce94deSChris Mason 2472b4ce94deSChris Mason /* 2473c8c42864SChris Mason * helper function for btrfs_search_slot. The goal is to find a block 2474c8c42864SChris Mason * in cache without setting the path to blocking. If we find the block 2475c8c42864SChris Mason * we return zero and the path is unchanged. 2476c8c42864SChris Mason * 2477c8c42864SChris Mason * If we can't find the block, we set the path blocking and do some 2478c8c42864SChris Mason * reada. -EAGAIN is returned and the search must be repeated. 2479c8c42864SChris Mason */ 2480c8c42864SChris Mason static int 2481c8c42864SChris Mason read_block_for_search(struct btrfs_trans_handle *trans, 2482c8c42864SChris Mason struct btrfs_root *root, struct btrfs_path *p, 2483c8c42864SChris Mason struct extent_buffer **eb_ret, int level, int slot, 24845d9e75c4SJan Schmidt struct btrfs_key *key, u64 time_seq) 2485c8c42864SChris Mason { 2486c8c42864SChris Mason u64 blocknr; 2487c8c42864SChris Mason u64 gen; 2488c8c42864SChris Mason u32 blocksize; 2489c8c42864SChris Mason struct extent_buffer *b = *eb_ret; 2490c8c42864SChris Mason struct extent_buffer *tmp; 249176a05b35SChris Mason int ret; 2492c8c42864SChris Mason 2493c8c42864SChris Mason blocknr = btrfs_node_blockptr(b, slot); 2494c8c42864SChris Mason gen = btrfs_node_ptr_generation(b, slot); 2495c8c42864SChris Mason blocksize = btrfs_level_size(root, level - 1); 2496c8c42864SChris Mason 2497c8c42864SChris Mason tmp = btrfs_find_tree_block(root, blocknr, blocksize); 2498cb44921aSChris Mason if (tmp) { 2499b9fab919SChris Mason /* first we do an atomic uptodate check */ 2500b9fab919SChris Mason if (btrfs_buffer_uptodate(tmp, gen, 1) > 0) { 2501c8c42864SChris Mason *eb_ret = tmp; 2502c8c42864SChris Mason return 0; 2503c8c42864SChris Mason } 2504bdf7c00eSJosef Bacik 2505cb44921aSChris Mason /* the pages were up to date, but we failed 2506cb44921aSChris Mason * the generation number check. Do a full 2507cb44921aSChris Mason * read for the generation number that is correct. 2508cb44921aSChris Mason * We must do this without dropping locks so 2509cb44921aSChris Mason * we can trust our generation number 2510cb44921aSChris Mason */ 2511bd681513SChris Mason btrfs_set_path_blocking(p); 2512bd681513SChris Mason 2513b9fab919SChris Mason /* now we're allowed to do a blocking uptodate check */ 2514bdf7c00eSJosef Bacik ret = btrfs_read_buffer(tmp, gen); 2515bdf7c00eSJosef Bacik if (!ret) { 2516cb44921aSChris Mason *eb_ret = tmp; 2517cb44921aSChris Mason return 0; 2518cb44921aSChris Mason } 2519cb44921aSChris Mason free_extent_buffer(tmp); 2520b3b4aa74SDavid Sterba btrfs_release_path(p); 2521cb44921aSChris Mason return -EIO; 2522cb44921aSChris Mason } 2523c8c42864SChris Mason 2524c8c42864SChris Mason /* 2525c8c42864SChris Mason * reduce lock contention at high levels 2526c8c42864SChris Mason * of the btree by dropping locks before 252776a05b35SChris Mason * we read. Don't release the lock on the current 252876a05b35SChris Mason * level because we need to walk this node to figure 252976a05b35SChris Mason * out which blocks to read. 2530c8c42864SChris Mason */ 25318c594ea8SChris Mason btrfs_unlock_up_safe(p, level + 1); 25328c594ea8SChris Mason btrfs_set_path_blocking(p); 25338c594ea8SChris Mason 2534c8c42864SChris Mason free_extent_buffer(tmp); 2535c8c42864SChris Mason if (p->reada) 2536c8c42864SChris Mason reada_for_search(root, p, level, slot, key->objectid); 2537c8c42864SChris Mason 2538b3b4aa74SDavid Sterba btrfs_release_path(p); 253976a05b35SChris Mason 254076a05b35SChris Mason ret = -EAGAIN; 25415bdd3536SYan, Zheng tmp = read_tree_block(root, blocknr, blocksize, 0); 254276a05b35SChris Mason if (tmp) { 254376a05b35SChris Mason /* 254476a05b35SChris Mason * If the read above didn't mark this buffer up to date, 254576a05b35SChris Mason * it will never end up being up to date. Set ret to EIO now 254676a05b35SChris Mason * and give up so that our caller doesn't loop forever 254776a05b35SChris Mason * on our EAGAINs. 254876a05b35SChris Mason */ 2549b9fab919SChris Mason if (!btrfs_buffer_uptodate(tmp, 0, 0)) 255076a05b35SChris Mason ret = -EIO; 2551c8c42864SChris Mason free_extent_buffer(tmp); 255276a05b35SChris Mason } 255376a05b35SChris Mason return ret; 2554c8c42864SChris Mason } 2555c8c42864SChris Mason 2556c8c42864SChris Mason /* 2557c8c42864SChris Mason * helper function for btrfs_search_slot. This does all of the checks 2558c8c42864SChris Mason * for node-level blocks and does any balancing required based on 2559c8c42864SChris Mason * the ins_len. 2560c8c42864SChris Mason * 2561c8c42864SChris Mason * If no extra work was required, zero is returned. If we had to 2562c8c42864SChris Mason * drop the path, -EAGAIN is returned and btrfs_search_slot must 2563c8c42864SChris Mason * start over 2564c8c42864SChris Mason */ 2565c8c42864SChris Mason static int 2566c8c42864SChris Mason setup_nodes_for_search(struct btrfs_trans_handle *trans, 2567c8c42864SChris Mason struct btrfs_root *root, struct btrfs_path *p, 2568bd681513SChris Mason struct extent_buffer *b, int level, int ins_len, 2569bd681513SChris Mason int *write_lock_level) 2570c8c42864SChris Mason { 2571c8c42864SChris Mason int ret; 2572c8c42864SChris Mason if ((p->search_for_split || ins_len > 0) && btrfs_header_nritems(b) >= 2573c8c42864SChris Mason BTRFS_NODEPTRS_PER_BLOCK(root) - 3) { 2574c8c42864SChris Mason int sret; 2575c8c42864SChris Mason 2576bd681513SChris Mason if (*write_lock_level < level + 1) { 2577bd681513SChris Mason *write_lock_level = level + 1; 2578bd681513SChris Mason btrfs_release_path(p); 2579bd681513SChris Mason goto again; 2580bd681513SChris Mason } 2581bd681513SChris Mason 2582c8c42864SChris Mason btrfs_set_path_blocking(p); 25830b08851fSJosef Bacik reada_for_balance(root, p, level); 2584c8c42864SChris Mason sret = split_node(trans, root, p, level); 2585bd681513SChris Mason btrfs_clear_path_blocking(p, NULL, 0); 2586c8c42864SChris Mason 2587c8c42864SChris Mason BUG_ON(sret > 0); 2588c8c42864SChris Mason if (sret) { 2589c8c42864SChris Mason ret = sret; 2590c8c42864SChris Mason goto done; 2591c8c42864SChris Mason } 2592c8c42864SChris Mason b = p->nodes[level]; 2593c8c42864SChris Mason } else if (ins_len < 0 && btrfs_header_nritems(b) < 2594cfbb9308SChris Mason BTRFS_NODEPTRS_PER_BLOCK(root) / 2) { 2595c8c42864SChris Mason int sret; 2596c8c42864SChris Mason 2597bd681513SChris Mason if (*write_lock_level < level + 1) { 2598bd681513SChris Mason *write_lock_level = level + 1; 2599bd681513SChris Mason btrfs_release_path(p); 2600bd681513SChris Mason goto again; 2601bd681513SChris Mason } 2602bd681513SChris Mason 2603c8c42864SChris Mason btrfs_set_path_blocking(p); 26040b08851fSJosef Bacik reada_for_balance(root, p, level); 2605c8c42864SChris Mason sret = balance_level(trans, root, p, level); 2606bd681513SChris Mason btrfs_clear_path_blocking(p, NULL, 0); 2607c8c42864SChris Mason 2608c8c42864SChris Mason if (sret) { 2609c8c42864SChris Mason ret = sret; 2610c8c42864SChris Mason goto done; 2611c8c42864SChris Mason } 2612c8c42864SChris Mason b = p->nodes[level]; 2613c8c42864SChris Mason if (!b) { 2614b3b4aa74SDavid Sterba btrfs_release_path(p); 2615c8c42864SChris Mason goto again; 2616c8c42864SChris Mason } 2617c8c42864SChris Mason BUG_ON(btrfs_header_nritems(b) == 1); 2618c8c42864SChris Mason } 2619c8c42864SChris Mason return 0; 2620c8c42864SChris Mason 2621c8c42864SChris Mason again: 2622c8c42864SChris Mason ret = -EAGAIN; 2623c8c42864SChris Mason done: 2624c8c42864SChris Mason return ret; 2625c8c42864SChris Mason } 2626c8c42864SChris Mason 2627d7396f07SFilipe David Borba Manana static void key_search_validate(struct extent_buffer *b, 2628d7396f07SFilipe David Borba Manana struct btrfs_key *key, 2629d7396f07SFilipe David Borba Manana int level) 2630d7396f07SFilipe David Borba Manana { 2631d7396f07SFilipe David Borba Manana #ifdef CONFIG_BTRFS_ASSERT 2632d7396f07SFilipe David Borba Manana struct btrfs_disk_key disk_key; 2633d7396f07SFilipe David Borba Manana 2634d7396f07SFilipe David Borba Manana btrfs_cpu_key_to_disk(&disk_key, key); 2635d7396f07SFilipe David Borba Manana 2636d7396f07SFilipe David Borba Manana if (level == 0) 2637d7396f07SFilipe David Borba Manana ASSERT(!memcmp_extent_buffer(b, &disk_key, 2638d7396f07SFilipe David Borba Manana offsetof(struct btrfs_leaf, items[0].key), 2639d7396f07SFilipe David Borba Manana sizeof(disk_key))); 2640d7396f07SFilipe David Borba Manana else 2641d7396f07SFilipe David Borba Manana ASSERT(!memcmp_extent_buffer(b, &disk_key, 2642d7396f07SFilipe David Borba Manana offsetof(struct btrfs_node, ptrs[0].key), 2643d7396f07SFilipe David Borba Manana sizeof(disk_key))); 2644d7396f07SFilipe David Borba Manana #endif 2645d7396f07SFilipe David Borba Manana } 2646d7396f07SFilipe David Borba Manana 2647d7396f07SFilipe David Borba Manana static int key_search(struct extent_buffer *b, struct btrfs_key *key, 2648d7396f07SFilipe David Borba Manana int level, int *prev_cmp, int *slot) 2649d7396f07SFilipe David Borba Manana { 2650d7396f07SFilipe David Borba Manana if (*prev_cmp != 0) { 2651d7396f07SFilipe David Borba Manana *prev_cmp = bin_search(b, key, level, slot); 2652d7396f07SFilipe David Borba Manana return *prev_cmp; 2653d7396f07SFilipe David Borba Manana } 2654d7396f07SFilipe David Borba Manana 2655d7396f07SFilipe David Borba Manana key_search_validate(b, key, level); 2656d7396f07SFilipe David Borba Manana *slot = 0; 2657d7396f07SFilipe David Borba Manana 2658d7396f07SFilipe David Borba Manana return 0; 2659d7396f07SFilipe David Borba Manana } 2660d7396f07SFilipe David Borba Manana 26613f870c28SKelley Nielsen int btrfs_find_item(struct btrfs_root *fs_root, struct btrfs_path *found_path, 2662e33d5c3dSKelley Nielsen u64 iobjectid, u64 ioff, u8 key_type, 2663e33d5c3dSKelley Nielsen struct btrfs_key *found_key) 2664e33d5c3dSKelley Nielsen { 2665e33d5c3dSKelley Nielsen int ret; 2666e33d5c3dSKelley Nielsen struct btrfs_key key; 2667e33d5c3dSKelley Nielsen struct extent_buffer *eb; 26683f870c28SKelley Nielsen struct btrfs_path *path; 2669e33d5c3dSKelley Nielsen 2670e33d5c3dSKelley Nielsen key.type = key_type; 2671e33d5c3dSKelley Nielsen key.objectid = iobjectid; 2672e33d5c3dSKelley Nielsen key.offset = ioff; 2673e33d5c3dSKelley Nielsen 26743f870c28SKelley Nielsen if (found_path == NULL) { 26753f870c28SKelley Nielsen path = btrfs_alloc_path(); 26763f870c28SKelley Nielsen if (!path) 26773f870c28SKelley Nielsen return -ENOMEM; 26783f870c28SKelley Nielsen } else 26793f870c28SKelley Nielsen path = found_path; 26803f870c28SKelley Nielsen 2681e33d5c3dSKelley Nielsen ret = btrfs_search_slot(NULL, fs_root, &key, path, 0, 0); 26823f870c28SKelley Nielsen if ((ret < 0) || (found_key == NULL)) { 26833f870c28SKelley Nielsen if (path != found_path) 26843f870c28SKelley Nielsen btrfs_free_path(path); 2685e33d5c3dSKelley Nielsen return ret; 26863f870c28SKelley Nielsen } 2687e33d5c3dSKelley Nielsen 2688e33d5c3dSKelley Nielsen eb = path->nodes[0]; 2689e33d5c3dSKelley Nielsen if (ret && path->slots[0] >= btrfs_header_nritems(eb)) { 2690e33d5c3dSKelley Nielsen ret = btrfs_next_leaf(fs_root, path); 2691e33d5c3dSKelley Nielsen if (ret) 2692e33d5c3dSKelley Nielsen return ret; 2693e33d5c3dSKelley Nielsen eb = path->nodes[0]; 2694e33d5c3dSKelley Nielsen } 2695e33d5c3dSKelley Nielsen 2696e33d5c3dSKelley Nielsen btrfs_item_key_to_cpu(eb, found_key, path->slots[0]); 2697e33d5c3dSKelley Nielsen if (found_key->type != key.type || 2698e33d5c3dSKelley Nielsen found_key->objectid != key.objectid) 2699e33d5c3dSKelley Nielsen return 1; 2700e33d5c3dSKelley Nielsen 2701e33d5c3dSKelley Nielsen return 0; 2702e33d5c3dSKelley Nielsen } 2703e33d5c3dSKelley Nielsen 2704c8c42864SChris Mason /* 270574123bd7SChris Mason * look for key in the tree. path is filled in with nodes along the way 270674123bd7SChris Mason * if key is found, we return zero and you can find the item in the leaf 270774123bd7SChris Mason * level of the path (level 0) 270874123bd7SChris Mason * 270974123bd7SChris Mason * If the key isn't found, the path points to the slot where it should 2710aa5d6bedSChris Mason * be inserted, and 1 is returned. If there are other errors during the 2711aa5d6bedSChris Mason * search a negative error number is returned. 271297571fd0SChris Mason * 271397571fd0SChris Mason * if ins_len > 0, nodes and leaves will be split as we walk down the 271497571fd0SChris Mason * tree. if ins_len < 0, nodes will be merged as we walk down the tree (if 271597571fd0SChris Mason * possible) 271674123bd7SChris Mason */ 2717e089f05cSChris Mason int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root 2718e089f05cSChris Mason *root, struct btrfs_key *key, struct btrfs_path *p, int 2719e089f05cSChris Mason ins_len, int cow) 2720be0e5c09SChris Mason { 27215f39d397SChris Mason struct extent_buffer *b; 2722be0e5c09SChris Mason int slot; 2723be0e5c09SChris Mason int ret; 272433c66f43SYan Zheng int err; 2725be0e5c09SChris Mason int level; 2726925baeddSChris Mason int lowest_unlock = 1; 2727bd681513SChris Mason int root_lock; 2728bd681513SChris Mason /* everything at write_lock_level or lower must be write locked */ 2729bd681513SChris Mason int write_lock_level = 0; 27309f3a7427SChris Mason u8 lowest_level = 0; 2731f7c79f30SChris Mason int min_write_lock_level; 2732d7396f07SFilipe David Borba Manana int prev_cmp; 27339f3a7427SChris Mason 27346702ed49SChris Mason lowest_level = p->lowest_level; 2735323ac95bSChris Mason WARN_ON(lowest_level && ins_len > 0); 273622b0ebdaSChris Mason WARN_ON(p->nodes[0] != NULL); 2737eb653de1SFilipe David Borba Manana BUG_ON(!cow && ins_len); 273825179201SJosef Bacik 2739bd681513SChris Mason if (ins_len < 0) { 2740925baeddSChris Mason lowest_unlock = 2; 274165b51a00SChris Mason 2742bd681513SChris Mason /* when we are removing items, we might have to go up to level 2743bd681513SChris Mason * two as we update tree pointers Make sure we keep write 2744bd681513SChris Mason * for those levels as well 2745bd681513SChris Mason */ 2746bd681513SChris Mason write_lock_level = 2; 2747bd681513SChris Mason } else if (ins_len > 0) { 2748bd681513SChris Mason /* 2749bd681513SChris Mason * for inserting items, make sure we have a write lock on 2750bd681513SChris Mason * level 1 so we can update keys 2751bd681513SChris Mason */ 2752bd681513SChris Mason write_lock_level = 1; 2753bd681513SChris Mason } 2754bd681513SChris Mason 2755bd681513SChris Mason if (!cow) 2756bd681513SChris Mason write_lock_level = -1; 2757bd681513SChris Mason 275809a2a8f9SJosef Bacik if (cow && (p->keep_locks || p->lowest_level)) 2759bd681513SChris Mason write_lock_level = BTRFS_MAX_LEVEL; 2760bd681513SChris Mason 2761f7c79f30SChris Mason min_write_lock_level = write_lock_level; 2762f7c79f30SChris Mason 2763bb803951SChris Mason again: 2764d7396f07SFilipe David Borba Manana prev_cmp = -1; 2765bd681513SChris Mason /* 2766bd681513SChris Mason * we try very hard to do read locks on the root 2767bd681513SChris Mason */ 2768bd681513SChris Mason root_lock = BTRFS_READ_LOCK; 2769bd681513SChris Mason level = 0; 27705d4f98a2SYan Zheng if (p->search_commit_root) { 2771bd681513SChris Mason /* 2772bd681513SChris Mason * the commit roots are read only 2773bd681513SChris Mason * so we always do read locks 2774bd681513SChris Mason */ 27753f8a18ccSJosef Bacik if (p->need_commit_sem) 27763f8a18ccSJosef Bacik down_read(&root->fs_info->commit_root_sem); 27775d4f98a2SYan Zheng b = root->commit_root; 27785d4f98a2SYan Zheng extent_buffer_get(b); 2779bd681513SChris Mason level = btrfs_header_level(b); 27803f8a18ccSJosef Bacik if (p->need_commit_sem) 27813f8a18ccSJosef Bacik up_read(&root->fs_info->commit_root_sem); 27825d4f98a2SYan Zheng if (!p->skip_locking) 2783bd681513SChris Mason btrfs_tree_read_lock(b); 27845d4f98a2SYan Zheng } else { 2785bd681513SChris Mason if (p->skip_locking) { 27865cd57b2cSChris Mason b = btrfs_root_node(root); 2787bd681513SChris Mason level = btrfs_header_level(b); 2788bd681513SChris Mason } else { 2789bd681513SChris Mason /* we don't know the level of the root node 2790bd681513SChris Mason * until we actually have it read locked 2791bd681513SChris Mason */ 2792bd681513SChris Mason b = btrfs_read_lock_root_node(root); 2793bd681513SChris Mason level = btrfs_header_level(b); 2794bd681513SChris Mason if (level <= write_lock_level) { 2795bd681513SChris Mason /* whoops, must trade for write lock */ 2796bd681513SChris Mason btrfs_tree_read_unlock(b); 2797bd681513SChris Mason free_extent_buffer(b); 2798925baeddSChris Mason b = btrfs_lock_root_node(root); 2799bd681513SChris Mason root_lock = BTRFS_WRITE_LOCK; 2800bd681513SChris Mason 2801bd681513SChris Mason /* the level might have changed, check again */ 2802bd681513SChris Mason level = btrfs_header_level(b); 28035d4f98a2SYan Zheng } 2804bd681513SChris Mason } 2805bd681513SChris Mason } 2806bd681513SChris Mason p->nodes[level] = b; 2807bd681513SChris Mason if (!p->skip_locking) 2808bd681513SChris Mason p->locks[level] = root_lock; 2809925baeddSChris Mason 2810eb60ceacSChris Mason while (b) { 28115f39d397SChris Mason level = btrfs_header_level(b); 281265b51a00SChris Mason 281365b51a00SChris Mason /* 281465b51a00SChris Mason * setup the path here so we can release it under lock 281565b51a00SChris Mason * contention with the cow code 281665b51a00SChris Mason */ 281702217ed2SChris Mason if (cow) { 2818c8c42864SChris Mason /* 2819c8c42864SChris Mason * if we don't really need to cow this block 2820c8c42864SChris Mason * then we don't want to set the path blocking, 2821c8c42864SChris Mason * so we test it here 2822c8c42864SChris Mason */ 28235d4f98a2SYan Zheng if (!should_cow_block(trans, root, b)) 282465b51a00SChris Mason goto cow_done; 28255d4f98a2SYan Zheng 2826b4ce94deSChris Mason btrfs_set_path_blocking(p); 2827b4ce94deSChris Mason 2828bd681513SChris Mason /* 2829bd681513SChris Mason * must have write locks on this node and the 2830bd681513SChris Mason * parent 2831bd681513SChris Mason */ 28325124e00eSJosef Bacik if (level > write_lock_level || 28335124e00eSJosef Bacik (level + 1 > write_lock_level && 28345124e00eSJosef Bacik level + 1 < BTRFS_MAX_LEVEL && 28355124e00eSJosef Bacik p->nodes[level + 1])) { 2836bd681513SChris Mason write_lock_level = level + 1; 2837bd681513SChris Mason btrfs_release_path(p); 2838bd681513SChris Mason goto again; 2839bd681513SChris Mason } 2840bd681513SChris Mason 284133c66f43SYan Zheng err = btrfs_cow_block(trans, root, b, 2842e20d96d6SChris Mason p->nodes[level + 1], 28439fa8cfe7SChris Mason p->slots[level + 1], &b); 284433c66f43SYan Zheng if (err) { 284533c66f43SYan Zheng ret = err; 284665b51a00SChris Mason goto done; 284754aa1f4dSChris Mason } 284802217ed2SChris Mason } 284965b51a00SChris Mason cow_done: 2850eb60ceacSChris Mason p->nodes[level] = b; 2851bd681513SChris Mason btrfs_clear_path_blocking(p, NULL, 0); 2852b4ce94deSChris Mason 2853b4ce94deSChris Mason /* 2854b4ce94deSChris Mason * we have a lock on b and as long as we aren't changing 2855b4ce94deSChris Mason * the tree, there is no way to for the items in b to change. 2856b4ce94deSChris Mason * It is safe to drop the lock on our parent before we 2857b4ce94deSChris Mason * go through the expensive btree search on b. 2858b4ce94deSChris Mason * 2859eb653de1SFilipe David Borba Manana * If we're inserting or deleting (ins_len != 0), then we might 2860eb653de1SFilipe David Borba Manana * be changing slot zero, which may require changing the parent. 2861eb653de1SFilipe David Borba Manana * So, we can't drop the lock until after we know which slot 2862eb653de1SFilipe David Borba Manana * we're operating on. 2863b4ce94deSChris Mason */ 2864eb653de1SFilipe David Borba Manana if (!ins_len && !p->keep_locks) { 2865eb653de1SFilipe David Borba Manana int u = level + 1; 2866eb653de1SFilipe David Borba Manana 2867eb653de1SFilipe David Borba Manana if (u < BTRFS_MAX_LEVEL && p->locks[u]) { 2868eb653de1SFilipe David Borba Manana btrfs_tree_unlock_rw(p->nodes[u], p->locks[u]); 2869eb653de1SFilipe David Borba Manana p->locks[u] = 0; 2870eb653de1SFilipe David Borba Manana } 2871eb653de1SFilipe David Borba Manana } 2872b4ce94deSChris Mason 2873d7396f07SFilipe David Borba Manana ret = key_search(b, key, level, &prev_cmp, &slot); 2874b4ce94deSChris Mason 28755f39d397SChris Mason if (level != 0) { 287633c66f43SYan Zheng int dec = 0; 287733c66f43SYan Zheng if (ret && slot > 0) { 287833c66f43SYan Zheng dec = 1; 2879be0e5c09SChris Mason slot -= 1; 288033c66f43SYan Zheng } 2881be0e5c09SChris Mason p->slots[level] = slot; 288233c66f43SYan Zheng err = setup_nodes_for_search(trans, root, p, b, level, 2883bd681513SChris Mason ins_len, &write_lock_level); 288433c66f43SYan Zheng if (err == -EAGAIN) 2885b4ce94deSChris Mason goto again; 288633c66f43SYan Zheng if (err) { 288733c66f43SYan Zheng ret = err; 288865b51a00SChris Mason goto done; 288933c66f43SYan Zheng } 28905c680ed6SChris Mason b = p->nodes[level]; 28915c680ed6SChris Mason slot = p->slots[level]; 2892b4ce94deSChris Mason 2893bd681513SChris Mason /* 2894bd681513SChris Mason * slot 0 is special, if we change the key 2895bd681513SChris Mason * we have to update the parent pointer 2896bd681513SChris Mason * which means we must have a write lock 2897bd681513SChris Mason * on the parent 2898bd681513SChris Mason */ 2899eb653de1SFilipe David Borba Manana if (slot == 0 && ins_len && 2900bd681513SChris Mason write_lock_level < level + 1) { 2901bd681513SChris Mason write_lock_level = level + 1; 2902bd681513SChris Mason btrfs_release_path(p); 2903bd681513SChris Mason goto again; 2904bd681513SChris Mason } 2905bd681513SChris Mason 2906f7c79f30SChris Mason unlock_up(p, level, lowest_unlock, 2907f7c79f30SChris Mason min_write_lock_level, &write_lock_level); 2908f9efa9c7SChris Mason 2909925baeddSChris Mason if (level == lowest_level) { 291033c66f43SYan Zheng if (dec) 291133c66f43SYan Zheng p->slots[level]++; 29125b21f2edSZheng Yan goto done; 2913925baeddSChris Mason } 2914ca7a79adSChris Mason 291533c66f43SYan Zheng err = read_block_for_search(trans, root, p, 29165d9e75c4SJan Schmidt &b, level, slot, key, 0); 291733c66f43SYan Zheng if (err == -EAGAIN) 2918051e1b9fSChris Mason goto again; 291933c66f43SYan Zheng if (err) { 292033c66f43SYan Zheng ret = err; 292176a05b35SChris Mason goto done; 292233c66f43SYan Zheng } 292376a05b35SChris Mason 2924b4ce94deSChris Mason if (!p->skip_locking) { 2925bd681513SChris Mason level = btrfs_header_level(b); 2926bd681513SChris Mason if (level <= write_lock_level) { 2927bd681513SChris Mason err = btrfs_try_tree_write_lock(b); 292833c66f43SYan Zheng if (!err) { 2929b4ce94deSChris Mason btrfs_set_path_blocking(p); 2930925baeddSChris Mason btrfs_tree_lock(b); 2931bd681513SChris Mason btrfs_clear_path_blocking(p, b, 2932bd681513SChris Mason BTRFS_WRITE_LOCK); 2933b4ce94deSChris Mason } 2934bd681513SChris Mason p->locks[level] = BTRFS_WRITE_LOCK; 2935bd681513SChris Mason } else { 2936bd681513SChris Mason err = btrfs_try_tree_read_lock(b); 2937bd681513SChris Mason if (!err) { 2938bd681513SChris Mason btrfs_set_path_blocking(p); 2939bd681513SChris Mason btrfs_tree_read_lock(b); 2940bd681513SChris Mason btrfs_clear_path_blocking(p, b, 2941bd681513SChris Mason BTRFS_READ_LOCK); 2942bd681513SChris Mason } 2943bd681513SChris Mason p->locks[level] = BTRFS_READ_LOCK; 2944bd681513SChris Mason } 2945bd681513SChris Mason p->nodes[level] = b; 2946b4ce94deSChris Mason } 2947be0e5c09SChris Mason } else { 2948be0e5c09SChris Mason p->slots[level] = slot; 294987b29b20SYan Zheng if (ins_len > 0 && 295087b29b20SYan Zheng btrfs_leaf_free_space(root, b) < ins_len) { 2951bd681513SChris Mason if (write_lock_level < 1) { 2952bd681513SChris Mason write_lock_level = 1; 2953bd681513SChris Mason btrfs_release_path(p); 2954bd681513SChris Mason goto again; 2955bd681513SChris Mason } 2956bd681513SChris Mason 2957b4ce94deSChris Mason btrfs_set_path_blocking(p); 295833c66f43SYan Zheng err = split_leaf(trans, root, key, 2959cc0c5538SChris Mason p, ins_len, ret == 0); 2960bd681513SChris Mason btrfs_clear_path_blocking(p, NULL, 0); 2961b4ce94deSChris Mason 296233c66f43SYan Zheng BUG_ON(err > 0); 296333c66f43SYan Zheng if (err) { 296433c66f43SYan Zheng ret = err; 296565b51a00SChris Mason goto done; 296665b51a00SChris Mason } 29675c680ed6SChris Mason } 2968459931ecSChris Mason if (!p->search_for_split) 2969f7c79f30SChris Mason unlock_up(p, level, lowest_unlock, 2970f7c79f30SChris Mason min_write_lock_level, &write_lock_level); 297165b51a00SChris Mason goto done; 297265b51a00SChris Mason } 297365b51a00SChris Mason } 297465b51a00SChris Mason ret = 1; 297565b51a00SChris Mason done: 2976b4ce94deSChris Mason /* 2977b4ce94deSChris Mason * we don't really know what they plan on doing with the path 2978b4ce94deSChris Mason * from here on, so for now just mark it as blocking 2979b4ce94deSChris Mason */ 2980b9473439SChris Mason if (!p->leave_spinning) 2981b4ce94deSChris Mason btrfs_set_path_blocking(p); 298276a05b35SChris Mason if (ret < 0) 2983b3b4aa74SDavid Sterba btrfs_release_path(p); 2984be0e5c09SChris Mason return ret; 2985be0e5c09SChris Mason } 2986be0e5c09SChris Mason 298774123bd7SChris Mason /* 29885d9e75c4SJan Schmidt * Like btrfs_search_slot, this looks for a key in the given tree. It uses the 29895d9e75c4SJan Schmidt * current state of the tree together with the operations recorded in the tree 29905d9e75c4SJan Schmidt * modification log to search for the key in a previous version of this tree, as 29915d9e75c4SJan Schmidt * denoted by the time_seq parameter. 29925d9e75c4SJan Schmidt * 29935d9e75c4SJan Schmidt * Naturally, there is no support for insert, delete or cow operations. 29945d9e75c4SJan Schmidt * 29955d9e75c4SJan Schmidt * The resulting path and return value will be set up as if we called 29965d9e75c4SJan Schmidt * btrfs_search_slot at that point in time with ins_len and cow both set to 0. 29975d9e75c4SJan Schmidt */ 29985d9e75c4SJan Schmidt int btrfs_search_old_slot(struct btrfs_root *root, struct btrfs_key *key, 29995d9e75c4SJan Schmidt struct btrfs_path *p, u64 time_seq) 30005d9e75c4SJan Schmidt { 30015d9e75c4SJan Schmidt struct extent_buffer *b; 30025d9e75c4SJan Schmidt int slot; 30035d9e75c4SJan Schmidt int ret; 30045d9e75c4SJan Schmidt int err; 30055d9e75c4SJan Schmidt int level; 30065d9e75c4SJan Schmidt int lowest_unlock = 1; 30075d9e75c4SJan Schmidt u8 lowest_level = 0; 3008d4b4087cSJosef Bacik int prev_cmp = -1; 30095d9e75c4SJan Schmidt 30105d9e75c4SJan Schmidt lowest_level = p->lowest_level; 30115d9e75c4SJan Schmidt WARN_ON(p->nodes[0] != NULL); 30125d9e75c4SJan Schmidt 30135d9e75c4SJan Schmidt if (p->search_commit_root) { 30145d9e75c4SJan Schmidt BUG_ON(time_seq); 30155d9e75c4SJan Schmidt return btrfs_search_slot(NULL, root, key, p, 0, 0); 30165d9e75c4SJan Schmidt } 30175d9e75c4SJan Schmidt 30185d9e75c4SJan Schmidt again: 30195d9e75c4SJan Schmidt b = get_old_root(root, time_seq); 30205d9e75c4SJan Schmidt level = btrfs_header_level(b); 30215d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 30225d9e75c4SJan Schmidt 30235d9e75c4SJan Schmidt while (b) { 30245d9e75c4SJan Schmidt level = btrfs_header_level(b); 30255d9e75c4SJan Schmidt p->nodes[level] = b; 30265d9e75c4SJan Schmidt btrfs_clear_path_blocking(p, NULL, 0); 30275d9e75c4SJan Schmidt 30285d9e75c4SJan Schmidt /* 30295d9e75c4SJan Schmidt * we have a lock on b and as long as we aren't changing 30305d9e75c4SJan Schmidt * the tree, there is no way to for the items in b to change. 30315d9e75c4SJan Schmidt * It is safe to drop the lock on our parent before we 30325d9e75c4SJan Schmidt * go through the expensive btree search on b. 30335d9e75c4SJan Schmidt */ 30345d9e75c4SJan Schmidt btrfs_unlock_up_safe(p, level + 1); 30355d9e75c4SJan Schmidt 3036d4b4087cSJosef Bacik /* 3037d4b4087cSJosef Bacik * Since we can unwind eb's we want to do a real search every 3038d4b4087cSJosef Bacik * time. 3039d4b4087cSJosef Bacik */ 3040d4b4087cSJosef Bacik prev_cmp = -1; 3041d7396f07SFilipe David Borba Manana ret = key_search(b, key, level, &prev_cmp, &slot); 30425d9e75c4SJan Schmidt 30435d9e75c4SJan Schmidt if (level != 0) { 30445d9e75c4SJan Schmidt int dec = 0; 30455d9e75c4SJan Schmidt if (ret && slot > 0) { 30465d9e75c4SJan Schmidt dec = 1; 30475d9e75c4SJan Schmidt slot -= 1; 30485d9e75c4SJan Schmidt } 30495d9e75c4SJan Schmidt p->slots[level] = slot; 30505d9e75c4SJan Schmidt unlock_up(p, level, lowest_unlock, 0, NULL); 30515d9e75c4SJan Schmidt 30525d9e75c4SJan Schmidt if (level == lowest_level) { 30535d9e75c4SJan Schmidt if (dec) 30545d9e75c4SJan Schmidt p->slots[level]++; 30555d9e75c4SJan Schmidt goto done; 30565d9e75c4SJan Schmidt } 30575d9e75c4SJan Schmidt 30585d9e75c4SJan Schmidt err = read_block_for_search(NULL, root, p, &b, level, 30595d9e75c4SJan Schmidt slot, key, time_seq); 30605d9e75c4SJan Schmidt if (err == -EAGAIN) 30615d9e75c4SJan Schmidt goto again; 30625d9e75c4SJan Schmidt if (err) { 30635d9e75c4SJan Schmidt ret = err; 30645d9e75c4SJan Schmidt goto done; 30655d9e75c4SJan Schmidt } 30665d9e75c4SJan Schmidt 30675d9e75c4SJan Schmidt level = btrfs_header_level(b); 30685d9e75c4SJan Schmidt err = btrfs_try_tree_read_lock(b); 30695d9e75c4SJan Schmidt if (!err) { 30705d9e75c4SJan Schmidt btrfs_set_path_blocking(p); 30715d9e75c4SJan Schmidt btrfs_tree_read_lock(b); 30725d9e75c4SJan Schmidt btrfs_clear_path_blocking(p, b, 30735d9e75c4SJan Schmidt BTRFS_READ_LOCK); 30745d9e75c4SJan Schmidt } 30759ec72677SJosef Bacik b = tree_mod_log_rewind(root->fs_info, p, b, time_seq); 3076db7f3436SJosef Bacik if (!b) { 3077db7f3436SJosef Bacik ret = -ENOMEM; 3078db7f3436SJosef Bacik goto done; 3079db7f3436SJosef Bacik } 30805d9e75c4SJan Schmidt p->locks[level] = BTRFS_READ_LOCK; 30815d9e75c4SJan Schmidt p->nodes[level] = b; 30825d9e75c4SJan Schmidt } else { 30835d9e75c4SJan Schmidt p->slots[level] = slot; 30845d9e75c4SJan Schmidt unlock_up(p, level, lowest_unlock, 0, NULL); 30855d9e75c4SJan Schmidt goto done; 30865d9e75c4SJan Schmidt } 30875d9e75c4SJan Schmidt } 30885d9e75c4SJan Schmidt ret = 1; 30895d9e75c4SJan Schmidt done: 30905d9e75c4SJan Schmidt if (!p->leave_spinning) 30915d9e75c4SJan Schmidt btrfs_set_path_blocking(p); 30925d9e75c4SJan Schmidt if (ret < 0) 30935d9e75c4SJan Schmidt btrfs_release_path(p); 30945d9e75c4SJan Schmidt 30955d9e75c4SJan Schmidt return ret; 30965d9e75c4SJan Schmidt } 30975d9e75c4SJan Schmidt 30985d9e75c4SJan Schmidt /* 30992f38b3e1SArne Jansen * helper to use instead of search slot if no exact match is needed but 31002f38b3e1SArne Jansen * instead the next or previous item should be returned. 31012f38b3e1SArne Jansen * When find_higher is true, the next higher item is returned, the next lower 31022f38b3e1SArne Jansen * otherwise. 31032f38b3e1SArne Jansen * When return_any and find_higher are both true, and no higher item is found, 31042f38b3e1SArne Jansen * return the next lower instead. 31052f38b3e1SArne Jansen * When return_any is true and find_higher is false, and no lower item is found, 31062f38b3e1SArne Jansen * return the next higher instead. 31072f38b3e1SArne Jansen * It returns 0 if any item is found, 1 if none is found (tree empty), and 31082f38b3e1SArne Jansen * < 0 on error 31092f38b3e1SArne Jansen */ 31102f38b3e1SArne Jansen int btrfs_search_slot_for_read(struct btrfs_root *root, 31112f38b3e1SArne Jansen struct btrfs_key *key, struct btrfs_path *p, 31122f38b3e1SArne Jansen int find_higher, int return_any) 31132f38b3e1SArne Jansen { 31142f38b3e1SArne Jansen int ret; 31152f38b3e1SArne Jansen struct extent_buffer *leaf; 31162f38b3e1SArne Jansen 31172f38b3e1SArne Jansen again: 31182f38b3e1SArne Jansen ret = btrfs_search_slot(NULL, root, key, p, 0, 0); 31192f38b3e1SArne Jansen if (ret <= 0) 31202f38b3e1SArne Jansen return ret; 31212f38b3e1SArne Jansen /* 31222f38b3e1SArne Jansen * a return value of 1 means the path is at the position where the 31232f38b3e1SArne Jansen * item should be inserted. Normally this is the next bigger item, 31242f38b3e1SArne Jansen * but in case the previous item is the last in a leaf, path points 31252f38b3e1SArne Jansen * to the first free slot in the previous leaf, i.e. at an invalid 31262f38b3e1SArne Jansen * item. 31272f38b3e1SArne Jansen */ 31282f38b3e1SArne Jansen leaf = p->nodes[0]; 31292f38b3e1SArne Jansen 31302f38b3e1SArne Jansen if (find_higher) { 31312f38b3e1SArne Jansen if (p->slots[0] >= btrfs_header_nritems(leaf)) { 31322f38b3e1SArne Jansen ret = btrfs_next_leaf(root, p); 31332f38b3e1SArne Jansen if (ret <= 0) 31342f38b3e1SArne Jansen return ret; 31352f38b3e1SArne Jansen if (!return_any) 31362f38b3e1SArne Jansen return 1; 31372f38b3e1SArne Jansen /* 31382f38b3e1SArne Jansen * no higher item found, return the next 31392f38b3e1SArne Jansen * lower instead 31402f38b3e1SArne Jansen */ 31412f38b3e1SArne Jansen return_any = 0; 31422f38b3e1SArne Jansen find_higher = 0; 31432f38b3e1SArne Jansen btrfs_release_path(p); 31442f38b3e1SArne Jansen goto again; 31452f38b3e1SArne Jansen } 31462f38b3e1SArne Jansen } else { 31472f38b3e1SArne Jansen if (p->slots[0] == 0) { 31482f38b3e1SArne Jansen ret = btrfs_prev_leaf(root, p); 3149e6793769SArne Jansen if (ret < 0) 31502f38b3e1SArne Jansen return ret; 3151e6793769SArne Jansen if (!ret) { 315223c6bf6aSFilipe David Borba Manana leaf = p->nodes[0]; 315323c6bf6aSFilipe David Borba Manana if (p->slots[0] == btrfs_header_nritems(leaf)) 315423c6bf6aSFilipe David Borba Manana p->slots[0]--; 3155e6793769SArne Jansen return 0; 3156e6793769SArne Jansen } 31572f38b3e1SArne Jansen if (!return_any) 31582f38b3e1SArne Jansen return 1; 31592f38b3e1SArne Jansen /* 31602f38b3e1SArne Jansen * no lower item found, return the next 31612f38b3e1SArne Jansen * higher instead 31622f38b3e1SArne Jansen */ 31632f38b3e1SArne Jansen return_any = 0; 31642f38b3e1SArne Jansen find_higher = 1; 31652f38b3e1SArne Jansen btrfs_release_path(p); 31662f38b3e1SArne Jansen goto again; 3167e6793769SArne Jansen } else { 31682f38b3e1SArne Jansen --p->slots[0]; 31692f38b3e1SArne Jansen } 31702f38b3e1SArne Jansen } 31712f38b3e1SArne Jansen return 0; 31722f38b3e1SArne Jansen } 31732f38b3e1SArne Jansen 31742f38b3e1SArne Jansen /* 317574123bd7SChris Mason * adjust the pointers going up the tree, starting at level 317674123bd7SChris Mason * making sure the right key of each node is points to 'key'. 317774123bd7SChris Mason * This is used after shifting pointers to the left, so it stops 317874123bd7SChris Mason * fixing up pointers when a given leaf/node is not in slot 0 of the 317974123bd7SChris Mason * higher levels 3180aa5d6bedSChris Mason * 318174123bd7SChris Mason */ 3182d6a0a126STsutomu Itoh static void fixup_low_keys(struct btrfs_root *root, struct btrfs_path *path, 31835f39d397SChris Mason struct btrfs_disk_key *key, int level) 3184be0e5c09SChris Mason { 3185be0e5c09SChris Mason int i; 31865f39d397SChris Mason struct extent_buffer *t; 31875f39d397SChris Mason 3188234b63a0SChris Mason for (i = level; i < BTRFS_MAX_LEVEL; i++) { 3189be0e5c09SChris Mason int tslot = path->slots[i]; 3190eb60ceacSChris Mason if (!path->nodes[i]) 3191be0e5c09SChris Mason break; 31925f39d397SChris Mason t = path->nodes[i]; 319332adf090SLiu Bo tree_mod_log_set_node_key(root->fs_info, t, tslot, 1); 31945f39d397SChris Mason btrfs_set_node_key(t, key, tslot); 3195d6025579SChris Mason btrfs_mark_buffer_dirty(path->nodes[i]); 3196be0e5c09SChris Mason if (tslot != 0) 3197be0e5c09SChris Mason break; 3198be0e5c09SChris Mason } 3199be0e5c09SChris Mason } 3200be0e5c09SChris Mason 320174123bd7SChris Mason /* 320231840ae1SZheng Yan * update item key. 320331840ae1SZheng Yan * 320431840ae1SZheng Yan * This function isn't completely safe. It's the caller's responsibility 320531840ae1SZheng Yan * that the new key won't break the order 320631840ae1SZheng Yan */ 3207afe5fea7STsutomu Itoh void btrfs_set_item_key_safe(struct btrfs_root *root, struct btrfs_path *path, 320831840ae1SZheng Yan struct btrfs_key *new_key) 320931840ae1SZheng Yan { 321031840ae1SZheng Yan struct btrfs_disk_key disk_key; 321131840ae1SZheng Yan struct extent_buffer *eb; 321231840ae1SZheng Yan int slot; 321331840ae1SZheng Yan 321431840ae1SZheng Yan eb = path->nodes[0]; 321531840ae1SZheng Yan slot = path->slots[0]; 321631840ae1SZheng Yan if (slot > 0) { 321731840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot - 1); 3218143bede5SJeff Mahoney BUG_ON(comp_keys(&disk_key, new_key) >= 0); 321931840ae1SZheng Yan } 322031840ae1SZheng Yan if (slot < btrfs_header_nritems(eb) - 1) { 322131840ae1SZheng Yan btrfs_item_key(eb, &disk_key, slot + 1); 3222143bede5SJeff Mahoney BUG_ON(comp_keys(&disk_key, new_key) <= 0); 322331840ae1SZheng Yan } 322431840ae1SZheng Yan 322531840ae1SZheng Yan btrfs_cpu_key_to_disk(&disk_key, new_key); 322631840ae1SZheng Yan btrfs_set_item_key(eb, &disk_key, slot); 322731840ae1SZheng Yan btrfs_mark_buffer_dirty(eb); 322831840ae1SZheng Yan if (slot == 0) 3229d6a0a126STsutomu Itoh fixup_low_keys(root, path, &disk_key, 1); 323031840ae1SZheng Yan } 323131840ae1SZheng Yan 323231840ae1SZheng Yan /* 323374123bd7SChris Mason * try to push data from one node into the next node left in the 323479f95c82SChris Mason * tree. 3235aa5d6bedSChris Mason * 3236aa5d6bedSChris Mason * returns 0 if some ptrs were pushed left, < 0 if there was some horrible 3237aa5d6bedSChris Mason * error, and > 0 if there was no room in the left hand block. 323874123bd7SChris Mason */ 323998ed5174SChris Mason static int push_node_left(struct btrfs_trans_handle *trans, 324098ed5174SChris Mason struct btrfs_root *root, struct extent_buffer *dst, 3241971a1f66SChris Mason struct extent_buffer *src, int empty) 3242be0e5c09SChris Mason { 3243be0e5c09SChris Mason int push_items = 0; 3244bb803951SChris Mason int src_nritems; 3245bb803951SChris Mason int dst_nritems; 3246aa5d6bedSChris Mason int ret = 0; 3247be0e5c09SChris Mason 32485f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 32495f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 3250123abc88SChris Mason push_items = BTRFS_NODEPTRS_PER_BLOCK(root) - dst_nritems; 32517bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 32527bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 325354aa1f4dSChris Mason 3254bce4eae9SChris Mason if (!empty && src_nritems <= 8) 3255971a1f66SChris Mason return 1; 3256971a1f66SChris Mason 3257d397712bSChris Mason if (push_items <= 0) 3258be0e5c09SChris Mason return 1; 3259be0e5c09SChris Mason 3260bce4eae9SChris Mason if (empty) { 3261971a1f66SChris Mason push_items = min(src_nritems, push_items); 3262bce4eae9SChris Mason if (push_items < src_nritems) { 3263bce4eae9SChris Mason /* leave at least 8 pointers in the node if 3264bce4eae9SChris Mason * we aren't going to empty it 3265bce4eae9SChris Mason */ 3266bce4eae9SChris Mason if (src_nritems - push_items < 8) { 3267bce4eae9SChris Mason if (push_items <= 8) 3268bce4eae9SChris Mason return 1; 3269bce4eae9SChris Mason push_items -= 8; 3270bce4eae9SChris Mason } 3271bce4eae9SChris Mason } 3272bce4eae9SChris Mason } else 3273bce4eae9SChris Mason push_items = min(src_nritems - 8, push_items); 327479f95c82SChris Mason 32755de865eeSFilipe David Borba Manana ret = tree_mod_log_eb_copy(root->fs_info, dst, src, dst_nritems, 0, 327690f8d62eSJan Schmidt push_items); 32775de865eeSFilipe David Borba Manana if (ret) { 32785de865eeSFilipe David Borba Manana btrfs_abort_transaction(trans, root, ret); 32795de865eeSFilipe David Borba Manana return ret; 32805de865eeSFilipe David Borba Manana } 32815f39d397SChris Mason copy_extent_buffer(dst, src, 32825f39d397SChris Mason btrfs_node_key_ptr_offset(dst_nritems), 32835f39d397SChris Mason btrfs_node_key_ptr_offset(0), 3284123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 32855f39d397SChris Mason 3286bb803951SChris Mason if (push_items < src_nritems) { 328757911b8bSJan Schmidt /* 328857911b8bSJan Schmidt * don't call tree_mod_log_eb_move here, key removal was already 328957911b8bSJan Schmidt * fully logged by tree_mod_log_eb_copy above. 329057911b8bSJan Schmidt */ 32915f39d397SChris Mason memmove_extent_buffer(src, btrfs_node_key_ptr_offset(0), 32925f39d397SChris Mason btrfs_node_key_ptr_offset(push_items), 3293e2fa7227SChris Mason (src_nritems - push_items) * 3294123abc88SChris Mason sizeof(struct btrfs_key_ptr)); 3295bb803951SChris Mason } 32965f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 32975f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 32985f39d397SChris Mason btrfs_mark_buffer_dirty(src); 32995f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 330031840ae1SZheng Yan 3301bb803951SChris Mason return ret; 3302be0e5c09SChris Mason } 3303be0e5c09SChris Mason 330497571fd0SChris Mason /* 330579f95c82SChris Mason * try to push data from one node into the next node right in the 330679f95c82SChris Mason * tree. 330779f95c82SChris Mason * 330879f95c82SChris Mason * returns 0 if some ptrs were pushed, < 0 if there was some horrible 330979f95c82SChris Mason * error, and > 0 if there was no room in the right hand block. 331079f95c82SChris Mason * 331179f95c82SChris Mason * this will only push up to 1/2 the contents of the left node over 331279f95c82SChris Mason */ 33135f39d397SChris Mason static int balance_node_right(struct btrfs_trans_handle *trans, 33145f39d397SChris Mason struct btrfs_root *root, 33155f39d397SChris Mason struct extent_buffer *dst, 33165f39d397SChris Mason struct extent_buffer *src) 331779f95c82SChris Mason { 331879f95c82SChris Mason int push_items = 0; 331979f95c82SChris Mason int max_push; 332079f95c82SChris Mason int src_nritems; 332179f95c82SChris Mason int dst_nritems; 332279f95c82SChris Mason int ret = 0; 332379f95c82SChris Mason 33247bb86316SChris Mason WARN_ON(btrfs_header_generation(src) != trans->transid); 33257bb86316SChris Mason WARN_ON(btrfs_header_generation(dst) != trans->transid); 33267bb86316SChris Mason 33275f39d397SChris Mason src_nritems = btrfs_header_nritems(src); 33285f39d397SChris Mason dst_nritems = btrfs_header_nritems(dst); 3329123abc88SChris Mason push_items = BTRFS_NODEPTRS_PER_BLOCK(root) - dst_nritems; 3330d397712bSChris Mason if (push_items <= 0) 333179f95c82SChris Mason return 1; 3332bce4eae9SChris Mason 3333d397712bSChris Mason if (src_nritems < 4) 3334bce4eae9SChris Mason return 1; 333579f95c82SChris Mason 333679f95c82SChris Mason max_push = src_nritems / 2 + 1; 333779f95c82SChris Mason /* don't try to empty the node */ 3338d397712bSChris Mason if (max_push >= src_nritems) 333979f95c82SChris Mason return 1; 3340252c38f0SYan 334179f95c82SChris Mason if (max_push < push_items) 334279f95c82SChris Mason push_items = max_push; 334379f95c82SChris Mason 3344f230475eSJan Schmidt tree_mod_log_eb_move(root->fs_info, dst, push_items, 0, dst_nritems); 33455f39d397SChris Mason memmove_extent_buffer(dst, btrfs_node_key_ptr_offset(push_items), 33465f39d397SChris Mason btrfs_node_key_ptr_offset(0), 33475f39d397SChris Mason (dst_nritems) * 33485f39d397SChris Mason sizeof(struct btrfs_key_ptr)); 3349d6025579SChris Mason 33505de865eeSFilipe David Borba Manana ret = tree_mod_log_eb_copy(root->fs_info, dst, src, 0, 335190f8d62eSJan Schmidt src_nritems - push_items, push_items); 33525de865eeSFilipe David Borba Manana if (ret) { 33535de865eeSFilipe David Borba Manana btrfs_abort_transaction(trans, root, ret); 33545de865eeSFilipe David Borba Manana return ret; 33555de865eeSFilipe David Borba Manana } 33565f39d397SChris Mason copy_extent_buffer(dst, src, 33575f39d397SChris Mason btrfs_node_key_ptr_offset(0), 33585f39d397SChris Mason btrfs_node_key_ptr_offset(src_nritems - push_items), 3359123abc88SChris Mason push_items * sizeof(struct btrfs_key_ptr)); 336079f95c82SChris Mason 33615f39d397SChris Mason btrfs_set_header_nritems(src, src_nritems - push_items); 33625f39d397SChris Mason btrfs_set_header_nritems(dst, dst_nritems + push_items); 336379f95c82SChris Mason 33645f39d397SChris Mason btrfs_mark_buffer_dirty(src); 33655f39d397SChris Mason btrfs_mark_buffer_dirty(dst); 336631840ae1SZheng Yan 336779f95c82SChris Mason return ret; 336879f95c82SChris Mason } 336979f95c82SChris Mason 337079f95c82SChris Mason /* 337197571fd0SChris Mason * helper function to insert a new root level in the tree. 337297571fd0SChris Mason * A new node is allocated, and a single item is inserted to 337397571fd0SChris Mason * point to the existing root 3374aa5d6bedSChris Mason * 3375aa5d6bedSChris Mason * returns zero on success or < 0 on failure. 337697571fd0SChris Mason */ 3377d397712bSChris Mason static noinline int insert_new_root(struct btrfs_trans_handle *trans, 33785f39d397SChris Mason struct btrfs_root *root, 3379fdd99c72SLiu Bo struct btrfs_path *path, int level) 338074123bd7SChris Mason { 33817bb86316SChris Mason u64 lower_gen; 33825f39d397SChris Mason struct extent_buffer *lower; 33835f39d397SChris Mason struct extent_buffer *c; 3384925baeddSChris Mason struct extent_buffer *old; 33855f39d397SChris Mason struct btrfs_disk_key lower_key; 33865c680ed6SChris Mason 33875c680ed6SChris Mason BUG_ON(path->nodes[level]); 33885c680ed6SChris Mason BUG_ON(path->nodes[level-1] != root->node); 33895c680ed6SChris Mason 33907bb86316SChris Mason lower = path->nodes[level-1]; 33917bb86316SChris Mason if (level == 1) 33927bb86316SChris Mason btrfs_item_key(lower, &lower_key, 0); 33937bb86316SChris Mason else 33947bb86316SChris Mason btrfs_node_key(lower, &lower_key, 0); 33957bb86316SChris Mason 339631840ae1SZheng Yan c = btrfs_alloc_free_block(trans, root, root->nodesize, 0, 33975d4f98a2SYan Zheng root->root_key.objectid, &lower_key, 33985581a51aSJan Schmidt level, root->node->start, 0); 33995f39d397SChris Mason if (IS_ERR(c)) 34005f39d397SChris Mason return PTR_ERR(c); 3401925baeddSChris Mason 3402f0486c68SYan, Zheng root_add_used(root, root->nodesize); 3403f0486c68SYan, Zheng 34045d4f98a2SYan Zheng memset_extent_buffer(c, 0, 0, sizeof(struct btrfs_header)); 34055f39d397SChris Mason btrfs_set_header_nritems(c, 1); 34065f39d397SChris Mason btrfs_set_header_level(c, level); 3407db94535dSChris Mason btrfs_set_header_bytenr(c, c->start); 34085f39d397SChris Mason btrfs_set_header_generation(c, trans->transid); 34095d4f98a2SYan Zheng btrfs_set_header_backref_rev(c, BTRFS_MIXED_BACKREF_REV); 34105f39d397SChris Mason btrfs_set_header_owner(c, root->root_key.objectid); 3411d5719762SChris Mason 34120a4e5586SRoss Kirk write_extent_buffer(c, root->fs_info->fsid, btrfs_header_fsid(), 34135f39d397SChris Mason BTRFS_FSID_SIZE); 3414e17cade2SChris Mason 3415e17cade2SChris Mason write_extent_buffer(c, root->fs_info->chunk_tree_uuid, 3416b308bc2fSGeert Uytterhoeven btrfs_header_chunk_tree_uuid(c), BTRFS_UUID_SIZE); 3417e17cade2SChris Mason 34185f39d397SChris Mason btrfs_set_node_key(c, &lower_key, 0); 3419db94535dSChris Mason btrfs_set_node_blockptr(c, 0, lower->start); 34207bb86316SChris Mason lower_gen = btrfs_header_generation(lower); 342131840ae1SZheng Yan WARN_ON(lower_gen != trans->transid); 34227bb86316SChris Mason 34237bb86316SChris Mason btrfs_set_node_ptr_generation(c, 0, lower_gen); 34245f39d397SChris Mason 34255f39d397SChris Mason btrfs_mark_buffer_dirty(c); 3426d5719762SChris Mason 3427925baeddSChris Mason old = root->node; 3428fdd99c72SLiu Bo tree_mod_log_set_root_pointer(root, c, 0); 3429240f62c8SChris Mason rcu_assign_pointer(root->node, c); 3430925baeddSChris Mason 3431925baeddSChris Mason /* the super has an extra ref to root->node */ 3432925baeddSChris Mason free_extent_buffer(old); 3433925baeddSChris Mason 34340b86a832SChris Mason add_root_to_dirty_list(root); 34355f39d397SChris Mason extent_buffer_get(c); 34365f39d397SChris Mason path->nodes[level] = c; 3437bd681513SChris Mason path->locks[level] = BTRFS_WRITE_LOCK; 343874123bd7SChris Mason path->slots[level] = 0; 343974123bd7SChris Mason return 0; 344074123bd7SChris Mason } 34415c680ed6SChris Mason 34425c680ed6SChris Mason /* 34435c680ed6SChris Mason * worker function to insert a single pointer in a node. 34445c680ed6SChris Mason * the node should have enough room for the pointer already 344597571fd0SChris Mason * 34465c680ed6SChris Mason * slot and level indicate where you want the key to go, and 34475c680ed6SChris Mason * blocknr is the block the key points to. 34485c680ed6SChris Mason */ 3449143bede5SJeff Mahoney static void insert_ptr(struct btrfs_trans_handle *trans, 3450143bede5SJeff Mahoney struct btrfs_root *root, struct btrfs_path *path, 3451143bede5SJeff Mahoney struct btrfs_disk_key *key, u64 bytenr, 3452c3e06965SJan Schmidt int slot, int level) 34535c680ed6SChris Mason { 34545f39d397SChris Mason struct extent_buffer *lower; 34555c680ed6SChris Mason int nritems; 3456f3ea38daSJan Schmidt int ret; 34575c680ed6SChris Mason 34585c680ed6SChris Mason BUG_ON(!path->nodes[level]); 3459f0486c68SYan, Zheng btrfs_assert_tree_locked(path->nodes[level]); 34605f39d397SChris Mason lower = path->nodes[level]; 34615f39d397SChris Mason nritems = btrfs_header_nritems(lower); 3462c293498bSStoyan Gaydarov BUG_ON(slot > nritems); 3463143bede5SJeff Mahoney BUG_ON(nritems == BTRFS_NODEPTRS_PER_BLOCK(root)); 346474123bd7SChris Mason if (slot != nritems) { 3465c3e06965SJan Schmidt if (level) 3466f3ea38daSJan Schmidt tree_mod_log_eb_move(root->fs_info, lower, slot + 1, 3467f3ea38daSJan Schmidt slot, nritems - slot); 34685f39d397SChris Mason memmove_extent_buffer(lower, 34695f39d397SChris Mason btrfs_node_key_ptr_offset(slot + 1), 34705f39d397SChris Mason btrfs_node_key_ptr_offset(slot), 3471123abc88SChris Mason (nritems - slot) * sizeof(struct btrfs_key_ptr)); 347274123bd7SChris Mason } 3473c3e06965SJan Schmidt if (level) { 3474f3ea38daSJan Schmidt ret = tree_mod_log_insert_key(root->fs_info, lower, slot, 3475c8cc6341SJosef Bacik MOD_LOG_KEY_ADD, GFP_NOFS); 3476f3ea38daSJan Schmidt BUG_ON(ret < 0); 3477f3ea38daSJan Schmidt } 34785f39d397SChris Mason btrfs_set_node_key(lower, key, slot); 3479db94535dSChris Mason btrfs_set_node_blockptr(lower, slot, bytenr); 348074493f7aSChris Mason WARN_ON(trans->transid == 0); 348174493f7aSChris Mason btrfs_set_node_ptr_generation(lower, slot, trans->transid); 34825f39d397SChris Mason btrfs_set_header_nritems(lower, nritems + 1); 34835f39d397SChris Mason btrfs_mark_buffer_dirty(lower); 348474123bd7SChris Mason } 348574123bd7SChris Mason 348697571fd0SChris Mason /* 348797571fd0SChris Mason * split the node at the specified level in path in two. 348897571fd0SChris Mason * The path is corrected to point to the appropriate node after the split 348997571fd0SChris Mason * 349097571fd0SChris Mason * Before splitting this tries to make some room in the node by pushing 349197571fd0SChris Mason * left and right, if either one works, it returns right away. 3492aa5d6bedSChris Mason * 3493aa5d6bedSChris Mason * returns 0 on success and < 0 on failure 349497571fd0SChris Mason */ 3495e02119d5SChris Mason static noinline int split_node(struct btrfs_trans_handle *trans, 3496e02119d5SChris Mason struct btrfs_root *root, 3497e02119d5SChris Mason struct btrfs_path *path, int level) 3498be0e5c09SChris Mason { 34995f39d397SChris Mason struct extent_buffer *c; 35005f39d397SChris Mason struct extent_buffer *split; 35015f39d397SChris Mason struct btrfs_disk_key disk_key; 3502be0e5c09SChris Mason int mid; 35035c680ed6SChris Mason int ret; 35047518a238SChris Mason u32 c_nritems; 3505be0e5c09SChris Mason 35065f39d397SChris Mason c = path->nodes[level]; 35077bb86316SChris Mason WARN_ON(btrfs_header_generation(c) != trans->transid); 35085f39d397SChris Mason if (c == root->node) { 3509d9abbf1cSJan Schmidt /* 351090f8d62eSJan Schmidt * trying to split the root, lets make a new one 351190f8d62eSJan Schmidt * 3512fdd99c72SLiu Bo * tree mod log: We don't log_removal old root in 351390f8d62eSJan Schmidt * insert_new_root, because that root buffer will be kept as a 351490f8d62eSJan Schmidt * normal node. We are going to log removal of half of the 351590f8d62eSJan Schmidt * elements below with tree_mod_log_eb_copy. We're holding a 351690f8d62eSJan Schmidt * tree lock on the buffer, which is why we cannot race with 351790f8d62eSJan Schmidt * other tree_mod_log users. 3518d9abbf1cSJan Schmidt */ 3519fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, level + 1); 35205c680ed6SChris Mason if (ret) 35215c680ed6SChris Mason return ret; 3522b3612421SChris Mason } else { 3523e66f709bSChris Mason ret = push_nodes_for_insert(trans, root, path, level); 35245f39d397SChris Mason c = path->nodes[level]; 35255f39d397SChris Mason if (!ret && btrfs_header_nritems(c) < 3526c448acf0SChris Mason BTRFS_NODEPTRS_PER_BLOCK(root) - 3) 3527e66f709bSChris Mason return 0; 352854aa1f4dSChris Mason if (ret < 0) 352954aa1f4dSChris Mason return ret; 35305c680ed6SChris Mason } 3531e66f709bSChris Mason 35325f39d397SChris Mason c_nritems = btrfs_header_nritems(c); 35335d4f98a2SYan Zheng mid = (c_nritems + 1) / 2; 35345d4f98a2SYan Zheng btrfs_node_key(c, &disk_key, mid); 35357bb86316SChris Mason 35365d4f98a2SYan Zheng split = btrfs_alloc_free_block(trans, root, root->nodesize, 0, 35377bb86316SChris Mason root->root_key.objectid, 35385581a51aSJan Schmidt &disk_key, level, c->start, 0); 35395f39d397SChris Mason if (IS_ERR(split)) 35405f39d397SChris Mason return PTR_ERR(split); 354154aa1f4dSChris Mason 3542f0486c68SYan, Zheng root_add_used(root, root->nodesize); 3543f0486c68SYan, Zheng 35445d4f98a2SYan Zheng memset_extent_buffer(split, 0, 0, sizeof(struct btrfs_header)); 35455f39d397SChris Mason btrfs_set_header_level(split, btrfs_header_level(c)); 3546db94535dSChris Mason btrfs_set_header_bytenr(split, split->start); 35475f39d397SChris Mason btrfs_set_header_generation(split, trans->transid); 35485d4f98a2SYan Zheng btrfs_set_header_backref_rev(split, BTRFS_MIXED_BACKREF_REV); 35495f39d397SChris Mason btrfs_set_header_owner(split, root->root_key.objectid); 35505f39d397SChris Mason write_extent_buffer(split, root->fs_info->fsid, 35510a4e5586SRoss Kirk btrfs_header_fsid(), BTRFS_FSID_SIZE); 3552e17cade2SChris Mason write_extent_buffer(split, root->fs_info->chunk_tree_uuid, 3553b308bc2fSGeert Uytterhoeven btrfs_header_chunk_tree_uuid(split), 3554e17cade2SChris Mason BTRFS_UUID_SIZE); 35555f39d397SChris Mason 35565de865eeSFilipe David Borba Manana ret = tree_mod_log_eb_copy(root->fs_info, split, c, 0, 35575de865eeSFilipe David Borba Manana mid, c_nritems - mid); 35585de865eeSFilipe David Borba Manana if (ret) { 35595de865eeSFilipe David Borba Manana btrfs_abort_transaction(trans, root, ret); 35605de865eeSFilipe David Borba Manana return ret; 35615de865eeSFilipe David Borba Manana } 35625f39d397SChris Mason copy_extent_buffer(split, c, 35635f39d397SChris Mason btrfs_node_key_ptr_offset(0), 35645f39d397SChris Mason btrfs_node_key_ptr_offset(mid), 3565123abc88SChris Mason (c_nritems - mid) * sizeof(struct btrfs_key_ptr)); 35665f39d397SChris Mason btrfs_set_header_nritems(split, c_nritems - mid); 35675f39d397SChris Mason btrfs_set_header_nritems(c, mid); 3568aa5d6bedSChris Mason ret = 0; 3569aa5d6bedSChris Mason 35705f39d397SChris Mason btrfs_mark_buffer_dirty(c); 35715f39d397SChris Mason btrfs_mark_buffer_dirty(split); 35725f39d397SChris Mason 3573143bede5SJeff Mahoney insert_ptr(trans, root, path, &disk_key, split->start, 3574c3e06965SJan Schmidt path->slots[level + 1] + 1, level + 1); 3575aa5d6bedSChris Mason 35765de08d7dSChris Mason if (path->slots[level] >= mid) { 35775c680ed6SChris Mason path->slots[level] -= mid; 3578925baeddSChris Mason btrfs_tree_unlock(c); 35795f39d397SChris Mason free_extent_buffer(c); 35805f39d397SChris Mason path->nodes[level] = split; 35815c680ed6SChris Mason path->slots[level + 1] += 1; 3582eb60ceacSChris Mason } else { 3583925baeddSChris Mason btrfs_tree_unlock(split); 35845f39d397SChris Mason free_extent_buffer(split); 3585be0e5c09SChris Mason } 3586aa5d6bedSChris Mason return ret; 3587be0e5c09SChris Mason } 3588be0e5c09SChris Mason 358974123bd7SChris Mason /* 359074123bd7SChris Mason * how many bytes are required to store the items in a leaf. start 359174123bd7SChris Mason * and nr indicate which items in the leaf to check. This totals up the 359274123bd7SChris Mason * space used both by the item structs and the item data 359374123bd7SChris Mason */ 35945f39d397SChris Mason static int leaf_space_used(struct extent_buffer *l, int start, int nr) 3595be0e5c09SChris Mason { 359641be1f3bSJosef Bacik struct btrfs_item *start_item; 359741be1f3bSJosef Bacik struct btrfs_item *end_item; 359841be1f3bSJosef Bacik struct btrfs_map_token token; 3599be0e5c09SChris Mason int data_len; 36005f39d397SChris Mason int nritems = btrfs_header_nritems(l); 3601d4dbff95SChris Mason int end = min(nritems, start + nr) - 1; 3602be0e5c09SChris Mason 3603be0e5c09SChris Mason if (!nr) 3604be0e5c09SChris Mason return 0; 360541be1f3bSJosef Bacik btrfs_init_map_token(&token); 3606dd3cc16bSRoss Kirk start_item = btrfs_item_nr(start); 3607dd3cc16bSRoss Kirk end_item = btrfs_item_nr(end); 360841be1f3bSJosef Bacik data_len = btrfs_token_item_offset(l, start_item, &token) + 360941be1f3bSJosef Bacik btrfs_token_item_size(l, start_item, &token); 361041be1f3bSJosef Bacik data_len = data_len - btrfs_token_item_offset(l, end_item, &token); 36110783fcfcSChris Mason data_len += sizeof(struct btrfs_item) * nr; 3612d4dbff95SChris Mason WARN_ON(data_len < 0); 3613be0e5c09SChris Mason return data_len; 3614be0e5c09SChris Mason } 3615be0e5c09SChris Mason 361674123bd7SChris Mason /* 3617d4dbff95SChris Mason * The space between the end of the leaf items and 3618d4dbff95SChris Mason * the start of the leaf data. IOW, how much room 3619d4dbff95SChris Mason * the leaf has left for both items and data 3620d4dbff95SChris Mason */ 3621d397712bSChris Mason noinline int btrfs_leaf_free_space(struct btrfs_root *root, 3622e02119d5SChris Mason struct extent_buffer *leaf) 3623d4dbff95SChris Mason { 36245f39d397SChris Mason int nritems = btrfs_header_nritems(leaf); 36255f39d397SChris Mason int ret; 36265f39d397SChris Mason ret = BTRFS_LEAF_DATA_SIZE(root) - leaf_space_used(leaf, 0, nritems); 36275f39d397SChris Mason if (ret < 0) { 3628efe120a0SFrank Holton btrfs_crit(root->fs_info, 3629efe120a0SFrank Holton "leaf free space ret %d, leaf data size %lu, used %d nritems %d", 3630ae2f5411SJens Axboe ret, (unsigned long) BTRFS_LEAF_DATA_SIZE(root), 36315f39d397SChris Mason leaf_space_used(leaf, 0, nritems), nritems); 36325f39d397SChris Mason } 36335f39d397SChris Mason return ret; 3634d4dbff95SChris Mason } 3635d4dbff95SChris Mason 363699d8f83cSChris Mason /* 363799d8f83cSChris Mason * min slot controls the lowest index we're willing to push to the 363899d8f83cSChris Mason * right. We'll push up to and including min_slot, but no lower 363999d8f83cSChris Mason */ 364044871b1bSChris Mason static noinline int __push_leaf_right(struct btrfs_trans_handle *trans, 364144871b1bSChris Mason struct btrfs_root *root, 364244871b1bSChris Mason struct btrfs_path *path, 364344871b1bSChris Mason int data_size, int empty, 364444871b1bSChris Mason struct extent_buffer *right, 364599d8f83cSChris Mason int free_space, u32 left_nritems, 364699d8f83cSChris Mason u32 min_slot) 364700ec4c51SChris Mason { 36485f39d397SChris Mason struct extent_buffer *left = path->nodes[0]; 364944871b1bSChris Mason struct extent_buffer *upper = path->nodes[1]; 3650cfed81a0SChris Mason struct btrfs_map_token token; 36515f39d397SChris Mason struct btrfs_disk_key disk_key; 365200ec4c51SChris Mason int slot; 365334a38218SChris Mason u32 i; 365400ec4c51SChris Mason int push_space = 0; 365500ec4c51SChris Mason int push_items = 0; 36560783fcfcSChris Mason struct btrfs_item *item; 365734a38218SChris Mason u32 nr; 36587518a238SChris Mason u32 right_nritems; 36595f39d397SChris Mason u32 data_end; 3660db94535dSChris Mason u32 this_item_size; 366100ec4c51SChris Mason 3662cfed81a0SChris Mason btrfs_init_map_token(&token); 3663cfed81a0SChris Mason 366434a38218SChris Mason if (empty) 366534a38218SChris Mason nr = 0; 366634a38218SChris Mason else 366799d8f83cSChris Mason nr = max_t(u32, 1, min_slot); 366834a38218SChris Mason 366931840ae1SZheng Yan if (path->slots[0] >= left_nritems) 367087b29b20SYan Zheng push_space += data_size; 367131840ae1SZheng Yan 367244871b1bSChris Mason slot = path->slots[1]; 367334a38218SChris Mason i = left_nritems - 1; 367434a38218SChris Mason while (i >= nr) { 3675dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 3676db94535dSChris Mason 367731840ae1SZheng Yan if (!empty && push_items > 0) { 367831840ae1SZheng Yan if (path->slots[0] > i) 367931840ae1SZheng Yan break; 368031840ae1SZheng Yan if (path->slots[0] == i) { 368131840ae1SZheng Yan int space = btrfs_leaf_free_space(root, left); 368231840ae1SZheng Yan if (space + push_space * 2 > free_space) 368331840ae1SZheng Yan break; 368431840ae1SZheng Yan } 368531840ae1SZheng Yan } 368631840ae1SZheng Yan 368700ec4c51SChris Mason if (path->slots[0] == i) 368887b29b20SYan Zheng push_space += data_size; 3689db94535dSChris Mason 3690db94535dSChris Mason this_item_size = btrfs_item_size(left, item); 3691db94535dSChris Mason if (this_item_size + sizeof(*item) + push_space > free_space) 369200ec4c51SChris Mason break; 369331840ae1SZheng Yan 369400ec4c51SChris Mason push_items++; 3695db94535dSChris Mason push_space += this_item_size + sizeof(*item); 369634a38218SChris Mason if (i == 0) 369734a38218SChris Mason break; 369834a38218SChris Mason i--; 3699db94535dSChris Mason } 37005f39d397SChris Mason 3701925baeddSChris Mason if (push_items == 0) 3702925baeddSChris Mason goto out_unlock; 37035f39d397SChris Mason 37046c1500f2SJulia Lawall WARN_ON(!empty && push_items == left_nritems); 37055f39d397SChris Mason 370600ec4c51SChris Mason /* push left to right */ 37075f39d397SChris Mason right_nritems = btrfs_header_nritems(right); 370834a38218SChris Mason 37095f39d397SChris Mason push_space = btrfs_item_end_nr(left, left_nritems - push_items); 3710123abc88SChris Mason push_space -= leaf_data_end(root, left); 37115f39d397SChris Mason 371200ec4c51SChris Mason /* make room in the right data area */ 37135f39d397SChris Mason data_end = leaf_data_end(root, right); 37145f39d397SChris Mason memmove_extent_buffer(right, 37155f39d397SChris Mason btrfs_leaf_data(right) + data_end - push_space, 37165f39d397SChris Mason btrfs_leaf_data(right) + data_end, 37175f39d397SChris Mason BTRFS_LEAF_DATA_SIZE(root) - data_end); 37185f39d397SChris Mason 371900ec4c51SChris Mason /* copy from the left data area */ 37205f39d397SChris Mason copy_extent_buffer(right, left, btrfs_leaf_data(right) + 3721d6025579SChris Mason BTRFS_LEAF_DATA_SIZE(root) - push_space, 3722d6025579SChris Mason btrfs_leaf_data(left) + leaf_data_end(root, left), 3723d6025579SChris Mason push_space); 37245f39d397SChris Mason 37255f39d397SChris Mason memmove_extent_buffer(right, btrfs_item_nr_offset(push_items), 37265f39d397SChris Mason btrfs_item_nr_offset(0), 37270783fcfcSChris Mason right_nritems * sizeof(struct btrfs_item)); 37285f39d397SChris Mason 372900ec4c51SChris Mason /* copy the items from left to right */ 37305f39d397SChris Mason copy_extent_buffer(right, left, btrfs_item_nr_offset(0), 37315f39d397SChris Mason btrfs_item_nr_offset(left_nritems - push_items), 37320783fcfcSChris Mason push_items * sizeof(struct btrfs_item)); 373300ec4c51SChris Mason 373400ec4c51SChris Mason /* update the item pointers */ 37357518a238SChris Mason right_nritems += push_items; 37365f39d397SChris Mason btrfs_set_header_nritems(right, right_nritems); 3737123abc88SChris Mason push_space = BTRFS_LEAF_DATA_SIZE(root); 37387518a238SChris Mason for (i = 0; i < right_nritems; i++) { 3739dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 3740cfed81a0SChris Mason push_space -= btrfs_token_item_size(right, item, &token); 3741cfed81a0SChris Mason btrfs_set_token_item_offset(right, item, push_space, &token); 3742db94535dSChris Mason } 3743db94535dSChris Mason 37447518a238SChris Mason left_nritems -= push_items; 37455f39d397SChris Mason btrfs_set_header_nritems(left, left_nritems); 374600ec4c51SChris Mason 374734a38218SChris Mason if (left_nritems) 37485f39d397SChris Mason btrfs_mark_buffer_dirty(left); 3749f0486c68SYan, Zheng else 3750f0486c68SYan, Zheng clean_tree_block(trans, root, left); 3751f0486c68SYan, Zheng 37525f39d397SChris Mason btrfs_mark_buffer_dirty(right); 3753a429e513SChris Mason 37545f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 37555f39d397SChris Mason btrfs_set_node_key(upper, &disk_key, slot + 1); 3756d6025579SChris Mason btrfs_mark_buffer_dirty(upper); 375702217ed2SChris Mason 375800ec4c51SChris Mason /* then fixup the leaf pointer in the path */ 37597518a238SChris Mason if (path->slots[0] >= left_nritems) { 37607518a238SChris Mason path->slots[0] -= left_nritems; 3761925baeddSChris Mason if (btrfs_header_nritems(path->nodes[0]) == 0) 3762925baeddSChris Mason clean_tree_block(trans, root, path->nodes[0]); 3763925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 37645f39d397SChris Mason free_extent_buffer(path->nodes[0]); 37655f39d397SChris Mason path->nodes[0] = right; 376600ec4c51SChris Mason path->slots[1] += 1; 376700ec4c51SChris Mason } else { 3768925baeddSChris Mason btrfs_tree_unlock(right); 37695f39d397SChris Mason free_extent_buffer(right); 377000ec4c51SChris Mason } 377100ec4c51SChris Mason return 0; 3772925baeddSChris Mason 3773925baeddSChris Mason out_unlock: 3774925baeddSChris Mason btrfs_tree_unlock(right); 3775925baeddSChris Mason free_extent_buffer(right); 3776925baeddSChris Mason return 1; 377700ec4c51SChris Mason } 3778925baeddSChris Mason 377900ec4c51SChris Mason /* 378044871b1bSChris Mason * push some data in the path leaf to the right, trying to free up at 378174123bd7SChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 378244871b1bSChris Mason * 378344871b1bSChris Mason * returns 1 if the push failed because the other node didn't have enough 378444871b1bSChris Mason * room, 0 if everything worked out and < 0 if there were major errors. 378599d8f83cSChris Mason * 378699d8f83cSChris Mason * this will push starting from min_slot to the end of the leaf. It won't 378799d8f83cSChris Mason * push any slot lower than min_slot 378874123bd7SChris Mason */ 378944871b1bSChris Mason static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root 379099d8f83cSChris Mason *root, struct btrfs_path *path, 379199d8f83cSChris Mason int min_data_size, int data_size, 379299d8f83cSChris Mason int empty, u32 min_slot) 3793be0e5c09SChris Mason { 379444871b1bSChris Mason struct extent_buffer *left = path->nodes[0]; 379544871b1bSChris Mason struct extent_buffer *right; 379644871b1bSChris Mason struct extent_buffer *upper; 379744871b1bSChris Mason int slot; 379844871b1bSChris Mason int free_space; 379944871b1bSChris Mason u32 left_nritems; 380044871b1bSChris Mason int ret; 380144871b1bSChris Mason 380244871b1bSChris Mason if (!path->nodes[1]) 380344871b1bSChris Mason return 1; 380444871b1bSChris Mason 380544871b1bSChris Mason slot = path->slots[1]; 380644871b1bSChris Mason upper = path->nodes[1]; 380744871b1bSChris Mason if (slot >= btrfs_header_nritems(upper) - 1) 380844871b1bSChris Mason return 1; 380944871b1bSChris Mason 381044871b1bSChris Mason btrfs_assert_tree_locked(path->nodes[1]); 381144871b1bSChris Mason 381244871b1bSChris Mason right = read_node_slot(root, upper, slot + 1); 381391ca338dSTsutomu Itoh if (right == NULL) 381491ca338dSTsutomu Itoh return 1; 381591ca338dSTsutomu Itoh 381644871b1bSChris Mason btrfs_tree_lock(right); 381744871b1bSChris Mason btrfs_set_lock_blocking(right); 381844871b1bSChris Mason 381944871b1bSChris Mason free_space = btrfs_leaf_free_space(root, right); 382044871b1bSChris Mason if (free_space < data_size) 382144871b1bSChris Mason goto out_unlock; 382244871b1bSChris Mason 382344871b1bSChris Mason /* cow and double check */ 382444871b1bSChris Mason ret = btrfs_cow_block(trans, root, right, upper, 382544871b1bSChris Mason slot + 1, &right); 382644871b1bSChris Mason if (ret) 382744871b1bSChris Mason goto out_unlock; 382844871b1bSChris Mason 382944871b1bSChris Mason free_space = btrfs_leaf_free_space(root, right); 383044871b1bSChris Mason if (free_space < data_size) 383144871b1bSChris Mason goto out_unlock; 383244871b1bSChris Mason 383344871b1bSChris Mason left_nritems = btrfs_header_nritems(left); 383444871b1bSChris Mason if (left_nritems == 0) 383544871b1bSChris Mason goto out_unlock; 383644871b1bSChris Mason 38372ef1fed2SFilipe David Borba Manana if (path->slots[0] == left_nritems && !empty) { 38382ef1fed2SFilipe David Borba Manana /* Key greater than all keys in the leaf, right neighbor has 38392ef1fed2SFilipe David Borba Manana * enough room for it and we're not emptying our leaf to delete 38402ef1fed2SFilipe David Borba Manana * it, therefore use right neighbor to insert the new item and 38412ef1fed2SFilipe David Borba Manana * no need to touch/dirty our left leaft. */ 38422ef1fed2SFilipe David Borba Manana btrfs_tree_unlock(left); 38432ef1fed2SFilipe David Borba Manana free_extent_buffer(left); 38442ef1fed2SFilipe David Borba Manana path->nodes[0] = right; 38452ef1fed2SFilipe David Borba Manana path->slots[0] = 0; 38462ef1fed2SFilipe David Borba Manana path->slots[1]++; 38472ef1fed2SFilipe David Borba Manana return 0; 38482ef1fed2SFilipe David Borba Manana } 38492ef1fed2SFilipe David Borba Manana 385099d8f83cSChris Mason return __push_leaf_right(trans, root, path, min_data_size, empty, 385199d8f83cSChris Mason right, free_space, left_nritems, min_slot); 385244871b1bSChris Mason out_unlock: 385344871b1bSChris Mason btrfs_tree_unlock(right); 385444871b1bSChris Mason free_extent_buffer(right); 385544871b1bSChris Mason return 1; 385644871b1bSChris Mason } 385744871b1bSChris Mason 385844871b1bSChris Mason /* 385944871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 386044871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 386199d8f83cSChris Mason * 386299d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 386399d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us do all the 386499d8f83cSChris Mason * items 386544871b1bSChris Mason */ 386644871b1bSChris Mason static noinline int __push_leaf_left(struct btrfs_trans_handle *trans, 386744871b1bSChris Mason struct btrfs_root *root, 386844871b1bSChris Mason struct btrfs_path *path, int data_size, 386944871b1bSChris Mason int empty, struct extent_buffer *left, 387099d8f83cSChris Mason int free_space, u32 right_nritems, 387199d8f83cSChris Mason u32 max_slot) 387244871b1bSChris Mason { 38735f39d397SChris Mason struct btrfs_disk_key disk_key; 38745f39d397SChris Mason struct extent_buffer *right = path->nodes[0]; 3875be0e5c09SChris Mason int i; 3876be0e5c09SChris Mason int push_space = 0; 3877be0e5c09SChris Mason int push_items = 0; 38780783fcfcSChris Mason struct btrfs_item *item; 38797518a238SChris Mason u32 old_left_nritems; 388034a38218SChris Mason u32 nr; 3881aa5d6bedSChris Mason int ret = 0; 3882db94535dSChris Mason u32 this_item_size; 3883db94535dSChris Mason u32 old_left_item_size; 3884cfed81a0SChris Mason struct btrfs_map_token token; 3885cfed81a0SChris Mason 3886cfed81a0SChris Mason btrfs_init_map_token(&token); 3887be0e5c09SChris Mason 388834a38218SChris Mason if (empty) 388999d8f83cSChris Mason nr = min(right_nritems, max_slot); 389034a38218SChris Mason else 389199d8f83cSChris Mason nr = min(right_nritems - 1, max_slot); 389234a38218SChris Mason 389334a38218SChris Mason for (i = 0; i < nr; i++) { 3894dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 3895db94535dSChris Mason 389631840ae1SZheng Yan if (!empty && push_items > 0) { 389731840ae1SZheng Yan if (path->slots[0] < i) 389831840ae1SZheng Yan break; 389931840ae1SZheng Yan if (path->slots[0] == i) { 390031840ae1SZheng Yan int space = btrfs_leaf_free_space(root, right); 390131840ae1SZheng Yan if (space + push_space * 2 > free_space) 390231840ae1SZheng Yan break; 390331840ae1SZheng Yan } 390431840ae1SZheng Yan } 390531840ae1SZheng Yan 3906be0e5c09SChris Mason if (path->slots[0] == i) 390787b29b20SYan Zheng push_space += data_size; 3908db94535dSChris Mason 3909db94535dSChris Mason this_item_size = btrfs_item_size(right, item); 3910db94535dSChris Mason if (this_item_size + sizeof(*item) + push_space > free_space) 3911be0e5c09SChris Mason break; 3912db94535dSChris Mason 3913be0e5c09SChris Mason push_items++; 3914db94535dSChris Mason push_space += this_item_size + sizeof(*item); 3915be0e5c09SChris Mason } 3916db94535dSChris Mason 3917be0e5c09SChris Mason if (push_items == 0) { 3918925baeddSChris Mason ret = 1; 3919925baeddSChris Mason goto out; 3920be0e5c09SChris Mason } 3921fae7f21cSDulshani Gunawardhana WARN_ON(!empty && push_items == btrfs_header_nritems(right)); 39225f39d397SChris Mason 3923be0e5c09SChris Mason /* push data from right to left */ 39245f39d397SChris Mason copy_extent_buffer(left, right, 39255f39d397SChris Mason btrfs_item_nr_offset(btrfs_header_nritems(left)), 39265f39d397SChris Mason btrfs_item_nr_offset(0), 39275f39d397SChris Mason push_items * sizeof(struct btrfs_item)); 39285f39d397SChris Mason 3929123abc88SChris Mason push_space = BTRFS_LEAF_DATA_SIZE(root) - 39305f39d397SChris Mason btrfs_item_offset_nr(right, push_items - 1); 39315f39d397SChris Mason 39325f39d397SChris Mason copy_extent_buffer(left, right, btrfs_leaf_data(left) + 3933d6025579SChris Mason leaf_data_end(root, left) - push_space, 3934123abc88SChris Mason btrfs_leaf_data(right) + 39355f39d397SChris Mason btrfs_item_offset_nr(right, push_items - 1), 3936be0e5c09SChris Mason push_space); 39375f39d397SChris Mason old_left_nritems = btrfs_header_nritems(left); 393887b29b20SYan Zheng BUG_ON(old_left_nritems <= 0); 3939eb60ceacSChris Mason 3940db94535dSChris Mason old_left_item_size = btrfs_item_offset_nr(left, old_left_nritems - 1); 3941be0e5c09SChris Mason for (i = old_left_nritems; i < old_left_nritems + push_items; i++) { 39425f39d397SChris Mason u32 ioff; 3943db94535dSChris Mason 3944dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 3945db94535dSChris Mason 3946cfed81a0SChris Mason ioff = btrfs_token_item_offset(left, item, &token); 3947cfed81a0SChris Mason btrfs_set_token_item_offset(left, item, 3948cfed81a0SChris Mason ioff - (BTRFS_LEAF_DATA_SIZE(root) - old_left_item_size), 3949cfed81a0SChris Mason &token); 3950be0e5c09SChris Mason } 39515f39d397SChris Mason btrfs_set_header_nritems(left, old_left_nritems + push_items); 3952be0e5c09SChris Mason 3953be0e5c09SChris Mason /* fixup right node */ 395431b1a2bdSJulia Lawall if (push_items > right_nritems) 395531b1a2bdSJulia Lawall WARN(1, KERN_CRIT "push items %d nr %u\n", push_items, 3956d397712bSChris Mason right_nritems); 395734a38218SChris Mason 395834a38218SChris Mason if (push_items < right_nritems) { 39595f39d397SChris Mason push_space = btrfs_item_offset_nr(right, push_items - 1) - 3960123abc88SChris Mason leaf_data_end(root, right); 39615f39d397SChris Mason memmove_extent_buffer(right, btrfs_leaf_data(right) + 3962d6025579SChris Mason BTRFS_LEAF_DATA_SIZE(root) - push_space, 3963d6025579SChris Mason btrfs_leaf_data(right) + 3964123abc88SChris Mason leaf_data_end(root, right), push_space); 39655f39d397SChris Mason 39665f39d397SChris Mason memmove_extent_buffer(right, btrfs_item_nr_offset(0), 39675f39d397SChris Mason btrfs_item_nr_offset(push_items), 39685f39d397SChris Mason (btrfs_header_nritems(right) - push_items) * 39690783fcfcSChris Mason sizeof(struct btrfs_item)); 397034a38218SChris Mason } 3971eef1c494SYan right_nritems -= push_items; 3972eef1c494SYan btrfs_set_header_nritems(right, right_nritems); 3973123abc88SChris Mason push_space = BTRFS_LEAF_DATA_SIZE(root); 39745f39d397SChris Mason for (i = 0; i < right_nritems; i++) { 3975dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 3976db94535dSChris Mason 3977cfed81a0SChris Mason push_space = push_space - btrfs_token_item_size(right, 3978cfed81a0SChris Mason item, &token); 3979cfed81a0SChris Mason btrfs_set_token_item_offset(right, item, push_space, &token); 3980db94535dSChris Mason } 3981eb60ceacSChris Mason 39825f39d397SChris Mason btrfs_mark_buffer_dirty(left); 398334a38218SChris Mason if (right_nritems) 39845f39d397SChris Mason btrfs_mark_buffer_dirty(right); 3985f0486c68SYan, Zheng else 3986f0486c68SYan, Zheng clean_tree_block(trans, root, right); 3987098f59c2SChris Mason 39885f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 3989d6a0a126STsutomu Itoh fixup_low_keys(root, path, &disk_key, 1); 3990be0e5c09SChris Mason 3991be0e5c09SChris Mason /* then fixup the leaf pointer in the path */ 3992be0e5c09SChris Mason if (path->slots[0] < push_items) { 3993be0e5c09SChris Mason path->slots[0] += old_left_nritems; 3994925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 39955f39d397SChris Mason free_extent_buffer(path->nodes[0]); 39965f39d397SChris Mason path->nodes[0] = left; 3997be0e5c09SChris Mason path->slots[1] -= 1; 3998be0e5c09SChris Mason } else { 3999925baeddSChris Mason btrfs_tree_unlock(left); 40005f39d397SChris Mason free_extent_buffer(left); 4001be0e5c09SChris Mason path->slots[0] -= push_items; 4002be0e5c09SChris Mason } 4003eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 4004aa5d6bedSChris Mason return ret; 4005925baeddSChris Mason out: 4006925baeddSChris Mason btrfs_tree_unlock(left); 4007925baeddSChris Mason free_extent_buffer(left); 4008925baeddSChris Mason return ret; 4009be0e5c09SChris Mason } 4010be0e5c09SChris Mason 401174123bd7SChris Mason /* 401244871b1bSChris Mason * push some data in the path leaf to the left, trying to free up at 401344871b1bSChris Mason * least data_size bytes. returns zero if the push worked, nonzero otherwise 401499d8f83cSChris Mason * 401599d8f83cSChris Mason * max_slot can put a limit on how far into the leaf we'll push items. The 401699d8f83cSChris Mason * item at 'max_slot' won't be touched. Use (u32)-1 to make us push all the 401799d8f83cSChris Mason * items 401844871b1bSChris Mason */ 401944871b1bSChris Mason static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root 402099d8f83cSChris Mason *root, struct btrfs_path *path, int min_data_size, 402199d8f83cSChris Mason int data_size, int empty, u32 max_slot) 402244871b1bSChris Mason { 402344871b1bSChris Mason struct extent_buffer *right = path->nodes[0]; 402444871b1bSChris Mason struct extent_buffer *left; 402544871b1bSChris Mason int slot; 402644871b1bSChris Mason int free_space; 402744871b1bSChris Mason u32 right_nritems; 402844871b1bSChris Mason int ret = 0; 402944871b1bSChris Mason 403044871b1bSChris Mason slot = path->slots[1]; 403144871b1bSChris Mason if (slot == 0) 403244871b1bSChris Mason return 1; 403344871b1bSChris Mason if (!path->nodes[1]) 403444871b1bSChris Mason return 1; 403544871b1bSChris Mason 403644871b1bSChris Mason right_nritems = btrfs_header_nritems(right); 403744871b1bSChris Mason if (right_nritems == 0) 403844871b1bSChris Mason return 1; 403944871b1bSChris Mason 404044871b1bSChris Mason btrfs_assert_tree_locked(path->nodes[1]); 404144871b1bSChris Mason 404244871b1bSChris Mason left = read_node_slot(root, path->nodes[1], slot - 1); 404391ca338dSTsutomu Itoh if (left == NULL) 404491ca338dSTsutomu Itoh return 1; 404591ca338dSTsutomu Itoh 404644871b1bSChris Mason btrfs_tree_lock(left); 404744871b1bSChris Mason btrfs_set_lock_blocking(left); 404844871b1bSChris Mason 404944871b1bSChris Mason free_space = btrfs_leaf_free_space(root, left); 405044871b1bSChris Mason if (free_space < data_size) { 405144871b1bSChris Mason ret = 1; 405244871b1bSChris Mason goto out; 405344871b1bSChris Mason } 405444871b1bSChris Mason 405544871b1bSChris Mason /* cow and double check */ 405644871b1bSChris Mason ret = btrfs_cow_block(trans, root, left, 405744871b1bSChris Mason path->nodes[1], slot - 1, &left); 405844871b1bSChris Mason if (ret) { 405944871b1bSChris Mason /* we hit -ENOSPC, but it isn't fatal here */ 406079787eaaSJeff Mahoney if (ret == -ENOSPC) 406144871b1bSChris Mason ret = 1; 406244871b1bSChris Mason goto out; 406344871b1bSChris Mason } 406444871b1bSChris Mason 406544871b1bSChris Mason free_space = btrfs_leaf_free_space(root, left); 406644871b1bSChris Mason if (free_space < data_size) { 406744871b1bSChris Mason ret = 1; 406844871b1bSChris Mason goto out; 406944871b1bSChris Mason } 407044871b1bSChris Mason 407199d8f83cSChris Mason return __push_leaf_left(trans, root, path, min_data_size, 407299d8f83cSChris Mason empty, left, free_space, right_nritems, 407399d8f83cSChris Mason max_slot); 407444871b1bSChris Mason out: 407544871b1bSChris Mason btrfs_tree_unlock(left); 407644871b1bSChris Mason free_extent_buffer(left); 407744871b1bSChris Mason return ret; 407844871b1bSChris Mason } 407944871b1bSChris Mason 408044871b1bSChris Mason /* 408174123bd7SChris Mason * split the path's leaf in two, making sure there is at least data_size 408274123bd7SChris Mason * available for the resulting leaf level of the path. 408374123bd7SChris Mason */ 4084143bede5SJeff Mahoney static noinline void copy_for_split(struct btrfs_trans_handle *trans, 4085e02119d5SChris Mason struct btrfs_root *root, 408644871b1bSChris Mason struct btrfs_path *path, 408744871b1bSChris Mason struct extent_buffer *l, 408844871b1bSChris Mason struct extent_buffer *right, 408944871b1bSChris Mason int slot, int mid, int nritems) 4090be0e5c09SChris Mason { 4091be0e5c09SChris Mason int data_copy_size; 4092be0e5c09SChris Mason int rt_data_off; 4093be0e5c09SChris Mason int i; 4094d4dbff95SChris Mason struct btrfs_disk_key disk_key; 4095cfed81a0SChris Mason struct btrfs_map_token token; 4096cfed81a0SChris Mason 4097cfed81a0SChris Mason btrfs_init_map_token(&token); 4098be0e5c09SChris Mason 40995f39d397SChris Mason nritems = nritems - mid; 41005f39d397SChris Mason btrfs_set_header_nritems(right, nritems); 41015f39d397SChris Mason data_copy_size = btrfs_item_end_nr(l, mid) - leaf_data_end(root, l); 41025f39d397SChris Mason 41035f39d397SChris Mason copy_extent_buffer(right, l, btrfs_item_nr_offset(0), 41045f39d397SChris Mason btrfs_item_nr_offset(mid), 41055f39d397SChris Mason nritems * sizeof(struct btrfs_item)); 41065f39d397SChris Mason 41075f39d397SChris Mason copy_extent_buffer(right, l, 4108d6025579SChris Mason btrfs_leaf_data(right) + BTRFS_LEAF_DATA_SIZE(root) - 4109123abc88SChris Mason data_copy_size, btrfs_leaf_data(l) + 4110123abc88SChris Mason leaf_data_end(root, l), data_copy_size); 411174123bd7SChris Mason 41125f39d397SChris Mason rt_data_off = BTRFS_LEAF_DATA_SIZE(root) - 41135f39d397SChris Mason btrfs_item_end_nr(l, mid); 41145f39d397SChris Mason 41155f39d397SChris Mason for (i = 0; i < nritems; i++) { 4116dd3cc16bSRoss Kirk struct btrfs_item *item = btrfs_item_nr(i); 4117db94535dSChris Mason u32 ioff; 4118db94535dSChris Mason 4119cfed81a0SChris Mason ioff = btrfs_token_item_offset(right, item, &token); 4120cfed81a0SChris Mason btrfs_set_token_item_offset(right, item, 4121cfed81a0SChris Mason ioff + rt_data_off, &token); 41220783fcfcSChris Mason } 412374123bd7SChris Mason 41245f39d397SChris Mason btrfs_set_header_nritems(l, mid); 41255f39d397SChris Mason btrfs_item_key(right, &disk_key, 0); 4126143bede5SJeff Mahoney insert_ptr(trans, root, path, &disk_key, right->start, 4127c3e06965SJan Schmidt path->slots[1] + 1, 1); 41285f39d397SChris Mason 41295f39d397SChris Mason btrfs_mark_buffer_dirty(right); 41305f39d397SChris Mason btrfs_mark_buffer_dirty(l); 4131eb60ceacSChris Mason BUG_ON(path->slots[0] != slot); 41325f39d397SChris Mason 4133be0e5c09SChris Mason if (mid <= slot) { 4134925baeddSChris Mason btrfs_tree_unlock(path->nodes[0]); 41355f39d397SChris Mason free_extent_buffer(path->nodes[0]); 41365f39d397SChris Mason path->nodes[0] = right; 4137be0e5c09SChris Mason path->slots[0] -= mid; 4138be0e5c09SChris Mason path->slots[1] += 1; 4139925baeddSChris Mason } else { 4140925baeddSChris Mason btrfs_tree_unlock(right); 41415f39d397SChris Mason free_extent_buffer(right); 4142925baeddSChris Mason } 41435f39d397SChris Mason 4144eb60ceacSChris Mason BUG_ON(path->slots[0] < 0); 414544871b1bSChris Mason } 414644871b1bSChris Mason 414744871b1bSChris Mason /* 414899d8f83cSChris Mason * double splits happen when we need to insert a big item in the middle 414999d8f83cSChris Mason * of a leaf. A double split can leave us with 3 mostly empty leaves: 415099d8f83cSChris Mason * leaf: [ slots 0 - N] [ our target ] [ N + 1 - total in leaf ] 415199d8f83cSChris Mason * A B C 415299d8f83cSChris Mason * 415399d8f83cSChris Mason * We avoid this by trying to push the items on either side of our target 415499d8f83cSChris Mason * into the adjacent leaves. If all goes well we can avoid the double split 415599d8f83cSChris Mason * completely. 415699d8f83cSChris Mason */ 415799d8f83cSChris Mason static noinline int push_for_double_split(struct btrfs_trans_handle *trans, 415899d8f83cSChris Mason struct btrfs_root *root, 415999d8f83cSChris Mason struct btrfs_path *path, 416099d8f83cSChris Mason int data_size) 416199d8f83cSChris Mason { 416299d8f83cSChris Mason int ret; 416399d8f83cSChris Mason int progress = 0; 416499d8f83cSChris Mason int slot; 416599d8f83cSChris Mason u32 nritems; 41665a4267caSFilipe David Borba Manana int space_needed = data_size; 416799d8f83cSChris Mason 416899d8f83cSChris Mason slot = path->slots[0]; 41695a4267caSFilipe David Borba Manana if (slot < btrfs_header_nritems(path->nodes[0])) 41705a4267caSFilipe David Borba Manana space_needed -= btrfs_leaf_free_space(root, path->nodes[0]); 417199d8f83cSChris Mason 417299d8f83cSChris Mason /* 417399d8f83cSChris Mason * try to push all the items after our slot into the 417499d8f83cSChris Mason * right leaf 417599d8f83cSChris Mason */ 41765a4267caSFilipe David Borba Manana ret = push_leaf_right(trans, root, path, 1, space_needed, 0, slot); 417799d8f83cSChris Mason if (ret < 0) 417899d8f83cSChris Mason return ret; 417999d8f83cSChris Mason 418099d8f83cSChris Mason if (ret == 0) 418199d8f83cSChris Mason progress++; 418299d8f83cSChris Mason 418399d8f83cSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 418499d8f83cSChris Mason /* 418599d8f83cSChris Mason * our goal is to get our slot at the start or end of a leaf. If 418699d8f83cSChris Mason * we've done so we're done 418799d8f83cSChris Mason */ 418899d8f83cSChris Mason if (path->slots[0] == 0 || path->slots[0] == nritems) 418999d8f83cSChris Mason return 0; 419099d8f83cSChris Mason 419199d8f83cSChris Mason if (btrfs_leaf_free_space(root, path->nodes[0]) >= data_size) 419299d8f83cSChris Mason return 0; 419399d8f83cSChris Mason 419499d8f83cSChris Mason /* try to push all the items before our slot into the next leaf */ 419599d8f83cSChris Mason slot = path->slots[0]; 41965a4267caSFilipe David Borba Manana ret = push_leaf_left(trans, root, path, 1, space_needed, 0, slot); 419799d8f83cSChris Mason if (ret < 0) 419899d8f83cSChris Mason return ret; 419999d8f83cSChris Mason 420099d8f83cSChris Mason if (ret == 0) 420199d8f83cSChris Mason progress++; 420299d8f83cSChris Mason 420399d8f83cSChris Mason if (progress) 420499d8f83cSChris Mason return 0; 420599d8f83cSChris Mason return 1; 420699d8f83cSChris Mason } 420799d8f83cSChris Mason 420899d8f83cSChris Mason /* 420944871b1bSChris Mason * split the path's leaf in two, making sure there is at least data_size 421044871b1bSChris Mason * available for the resulting leaf level of the path. 421144871b1bSChris Mason * 421244871b1bSChris Mason * returns 0 if all went well and < 0 on failure. 421344871b1bSChris Mason */ 421444871b1bSChris Mason static noinline int split_leaf(struct btrfs_trans_handle *trans, 421544871b1bSChris Mason struct btrfs_root *root, 421644871b1bSChris Mason struct btrfs_key *ins_key, 421744871b1bSChris Mason struct btrfs_path *path, int data_size, 421844871b1bSChris Mason int extend) 421944871b1bSChris Mason { 42205d4f98a2SYan Zheng struct btrfs_disk_key disk_key; 422144871b1bSChris Mason struct extent_buffer *l; 422244871b1bSChris Mason u32 nritems; 422344871b1bSChris Mason int mid; 422444871b1bSChris Mason int slot; 422544871b1bSChris Mason struct extent_buffer *right; 422644871b1bSChris Mason int ret = 0; 422744871b1bSChris Mason int wret; 42285d4f98a2SYan Zheng int split; 422944871b1bSChris Mason int num_doubles = 0; 423099d8f83cSChris Mason int tried_avoid_double = 0; 423144871b1bSChris Mason 4232a5719521SYan, Zheng l = path->nodes[0]; 4233a5719521SYan, Zheng slot = path->slots[0]; 4234a5719521SYan, Zheng if (extend && data_size + btrfs_item_size_nr(l, slot) + 4235a5719521SYan, Zheng sizeof(struct btrfs_item) > BTRFS_LEAF_DATA_SIZE(root)) 4236a5719521SYan, Zheng return -EOVERFLOW; 4237a5719521SYan, Zheng 423844871b1bSChris Mason /* first try to make some room by pushing left and right */ 423933157e05SLiu Bo if (data_size && path->nodes[1]) { 42405a4267caSFilipe David Borba Manana int space_needed = data_size; 42415a4267caSFilipe David Borba Manana 42425a4267caSFilipe David Borba Manana if (slot < btrfs_header_nritems(l)) 42435a4267caSFilipe David Borba Manana space_needed -= btrfs_leaf_free_space(root, l); 42445a4267caSFilipe David Borba Manana 42455a4267caSFilipe David Borba Manana wret = push_leaf_right(trans, root, path, space_needed, 42465a4267caSFilipe David Borba Manana space_needed, 0, 0); 424744871b1bSChris Mason if (wret < 0) 424844871b1bSChris Mason return wret; 424944871b1bSChris Mason if (wret) { 42505a4267caSFilipe David Borba Manana wret = push_leaf_left(trans, root, path, space_needed, 42515a4267caSFilipe David Borba Manana space_needed, 0, (u32)-1); 425244871b1bSChris Mason if (wret < 0) 425344871b1bSChris Mason return wret; 425444871b1bSChris Mason } 425544871b1bSChris Mason l = path->nodes[0]; 425644871b1bSChris Mason 425744871b1bSChris Mason /* did the pushes work? */ 425844871b1bSChris Mason if (btrfs_leaf_free_space(root, l) >= data_size) 425944871b1bSChris Mason return 0; 426044871b1bSChris Mason } 426144871b1bSChris Mason 426244871b1bSChris Mason if (!path->nodes[1]) { 4263fdd99c72SLiu Bo ret = insert_new_root(trans, root, path, 1); 426444871b1bSChris Mason if (ret) 426544871b1bSChris Mason return ret; 426644871b1bSChris Mason } 426744871b1bSChris Mason again: 42685d4f98a2SYan Zheng split = 1; 426944871b1bSChris Mason l = path->nodes[0]; 427044871b1bSChris Mason slot = path->slots[0]; 427144871b1bSChris Mason nritems = btrfs_header_nritems(l); 427244871b1bSChris Mason mid = (nritems + 1) / 2; 427344871b1bSChris Mason 42745d4f98a2SYan Zheng if (mid <= slot) { 42755d4f98a2SYan Zheng if (nritems == 1 || 42765d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + data_size > 42775d4f98a2SYan Zheng BTRFS_LEAF_DATA_SIZE(root)) { 42785d4f98a2SYan Zheng if (slot >= nritems) { 42795d4f98a2SYan Zheng split = 0; 42805d4f98a2SYan Zheng } else { 42815d4f98a2SYan Zheng mid = slot; 42825d4f98a2SYan Zheng if (mid != nritems && 42835d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 42845d4f98a2SYan Zheng data_size > BTRFS_LEAF_DATA_SIZE(root)) { 428599d8f83cSChris Mason if (data_size && !tried_avoid_double) 428699d8f83cSChris Mason goto push_for_double; 42875d4f98a2SYan Zheng split = 2; 42885d4f98a2SYan Zheng } 42895d4f98a2SYan Zheng } 42905d4f98a2SYan Zheng } 42915d4f98a2SYan Zheng } else { 42925d4f98a2SYan Zheng if (leaf_space_used(l, 0, mid) + data_size > 42935d4f98a2SYan Zheng BTRFS_LEAF_DATA_SIZE(root)) { 42945d4f98a2SYan Zheng if (!extend && data_size && slot == 0) { 42955d4f98a2SYan Zheng split = 0; 42965d4f98a2SYan Zheng } else if ((extend || !data_size) && slot == 0) { 42975d4f98a2SYan Zheng mid = 1; 42985d4f98a2SYan Zheng } else { 42995d4f98a2SYan Zheng mid = slot; 43005d4f98a2SYan Zheng if (mid != nritems && 43015d4f98a2SYan Zheng leaf_space_used(l, mid, nritems - mid) + 43025d4f98a2SYan Zheng data_size > BTRFS_LEAF_DATA_SIZE(root)) { 430399d8f83cSChris Mason if (data_size && !tried_avoid_double) 430499d8f83cSChris Mason goto push_for_double; 43055d4f98a2SYan Zheng split = 2; 43065d4f98a2SYan Zheng } 43075d4f98a2SYan Zheng } 43085d4f98a2SYan Zheng } 43095d4f98a2SYan Zheng } 43105d4f98a2SYan Zheng 43115d4f98a2SYan Zheng if (split == 0) 43125d4f98a2SYan Zheng btrfs_cpu_key_to_disk(&disk_key, ins_key); 43135d4f98a2SYan Zheng else 43145d4f98a2SYan Zheng btrfs_item_key(l, &disk_key, mid); 43155d4f98a2SYan Zheng 43165d4f98a2SYan Zheng right = btrfs_alloc_free_block(trans, root, root->leafsize, 0, 431744871b1bSChris Mason root->root_key.objectid, 43185581a51aSJan Schmidt &disk_key, 0, l->start, 0); 4319f0486c68SYan, Zheng if (IS_ERR(right)) 432044871b1bSChris Mason return PTR_ERR(right); 4321f0486c68SYan, Zheng 4322f0486c68SYan, Zheng root_add_used(root, root->leafsize); 432344871b1bSChris Mason 432444871b1bSChris Mason memset_extent_buffer(right, 0, 0, sizeof(struct btrfs_header)); 432544871b1bSChris Mason btrfs_set_header_bytenr(right, right->start); 432644871b1bSChris Mason btrfs_set_header_generation(right, trans->transid); 43275d4f98a2SYan Zheng btrfs_set_header_backref_rev(right, BTRFS_MIXED_BACKREF_REV); 432844871b1bSChris Mason btrfs_set_header_owner(right, root->root_key.objectid); 432944871b1bSChris Mason btrfs_set_header_level(right, 0); 433044871b1bSChris Mason write_extent_buffer(right, root->fs_info->fsid, 43310a4e5586SRoss Kirk btrfs_header_fsid(), BTRFS_FSID_SIZE); 433244871b1bSChris Mason 433344871b1bSChris Mason write_extent_buffer(right, root->fs_info->chunk_tree_uuid, 4334b308bc2fSGeert Uytterhoeven btrfs_header_chunk_tree_uuid(right), 433544871b1bSChris Mason BTRFS_UUID_SIZE); 433644871b1bSChris Mason 43375d4f98a2SYan Zheng if (split == 0) { 433844871b1bSChris Mason if (mid <= slot) { 433944871b1bSChris Mason btrfs_set_header_nritems(right, 0); 4340143bede5SJeff Mahoney insert_ptr(trans, root, path, &disk_key, right->start, 4341c3e06965SJan Schmidt path->slots[1] + 1, 1); 434244871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 434344871b1bSChris Mason free_extent_buffer(path->nodes[0]); 434444871b1bSChris Mason path->nodes[0] = right; 434544871b1bSChris Mason path->slots[0] = 0; 434644871b1bSChris Mason path->slots[1] += 1; 434744871b1bSChris Mason } else { 434844871b1bSChris Mason btrfs_set_header_nritems(right, 0); 4349143bede5SJeff Mahoney insert_ptr(trans, root, path, &disk_key, right->start, 4350c3e06965SJan Schmidt path->slots[1], 1); 435144871b1bSChris Mason btrfs_tree_unlock(path->nodes[0]); 435244871b1bSChris Mason free_extent_buffer(path->nodes[0]); 435344871b1bSChris Mason path->nodes[0] = right; 435444871b1bSChris Mason path->slots[0] = 0; 4355143bede5SJeff Mahoney if (path->slots[1] == 0) 4356d6a0a126STsutomu Itoh fixup_low_keys(root, path, &disk_key, 1); 43575d4f98a2SYan Zheng } 435844871b1bSChris Mason btrfs_mark_buffer_dirty(right); 435944871b1bSChris Mason return ret; 436044871b1bSChris Mason } 436144871b1bSChris Mason 4362143bede5SJeff Mahoney copy_for_split(trans, root, path, l, right, slot, mid, nritems); 436344871b1bSChris Mason 43645d4f98a2SYan Zheng if (split == 2) { 4365cc0c5538SChris Mason BUG_ON(num_doubles != 0); 4366cc0c5538SChris Mason num_doubles++; 4367cc0c5538SChris Mason goto again; 43683326d1b0SChris Mason } 436944871b1bSChris Mason 4370143bede5SJeff Mahoney return 0; 437199d8f83cSChris Mason 437299d8f83cSChris Mason push_for_double: 437399d8f83cSChris Mason push_for_double_split(trans, root, path, data_size); 437499d8f83cSChris Mason tried_avoid_double = 1; 437599d8f83cSChris Mason if (btrfs_leaf_free_space(root, path->nodes[0]) >= data_size) 437699d8f83cSChris Mason return 0; 437799d8f83cSChris Mason goto again; 4378be0e5c09SChris Mason } 4379be0e5c09SChris Mason 4380ad48fd75SYan, Zheng static noinline int setup_leaf_for_split(struct btrfs_trans_handle *trans, 4381ad48fd75SYan, Zheng struct btrfs_root *root, 4382ad48fd75SYan, Zheng struct btrfs_path *path, int ins_len) 4383ad48fd75SYan, Zheng { 4384ad48fd75SYan, Zheng struct btrfs_key key; 4385ad48fd75SYan, Zheng struct extent_buffer *leaf; 4386ad48fd75SYan, Zheng struct btrfs_file_extent_item *fi; 4387ad48fd75SYan, Zheng u64 extent_len = 0; 4388ad48fd75SYan, Zheng u32 item_size; 4389ad48fd75SYan, Zheng int ret; 4390ad48fd75SYan, Zheng 4391ad48fd75SYan, Zheng leaf = path->nodes[0]; 4392ad48fd75SYan, Zheng btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 4393ad48fd75SYan, Zheng 4394ad48fd75SYan, Zheng BUG_ON(key.type != BTRFS_EXTENT_DATA_KEY && 4395ad48fd75SYan, Zheng key.type != BTRFS_EXTENT_CSUM_KEY); 4396ad48fd75SYan, Zheng 4397ad48fd75SYan, Zheng if (btrfs_leaf_free_space(root, leaf) >= ins_len) 4398ad48fd75SYan, Zheng return 0; 4399ad48fd75SYan, Zheng 4400ad48fd75SYan, Zheng item_size = btrfs_item_size_nr(leaf, path->slots[0]); 4401ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 4402ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 4403ad48fd75SYan, Zheng struct btrfs_file_extent_item); 4404ad48fd75SYan, Zheng extent_len = btrfs_file_extent_num_bytes(leaf, fi); 4405ad48fd75SYan, Zheng } 4406b3b4aa74SDavid Sterba btrfs_release_path(path); 4407ad48fd75SYan, Zheng 4408ad48fd75SYan, Zheng path->keep_locks = 1; 4409ad48fd75SYan, Zheng path->search_for_split = 1; 4410ad48fd75SYan, Zheng ret = btrfs_search_slot(trans, root, &key, path, 0, 1); 4411ad48fd75SYan, Zheng path->search_for_split = 0; 4412ad48fd75SYan, Zheng if (ret < 0) 4413ad48fd75SYan, Zheng goto err; 4414ad48fd75SYan, Zheng 4415ad48fd75SYan, Zheng ret = -EAGAIN; 4416ad48fd75SYan, Zheng leaf = path->nodes[0]; 4417ad48fd75SYan, Zheng /* if our item isn't there or got smaller, return now */ 4418ad48fd75SYan, Zheng if (ret > 0 || item_size != btrfs_item_size_nr(leaf, path->slots[0])) 4419ad48fd75SYan, Zheng goto err; 4420ad48fd75SYan, Zheng 4421109f6aefSChris Mason /* the leaf has changed, it now has room. return now */ 4422109f6aefSChris Mason if (btrfs_leaf_free_space(root, path->nodes[0]) >= ins_len) 4423109f6aefSChris Mason goto err; 4424109f6aefSChris Mason 4425ad48fd75SYan, Zheng if (key.type == BTRFS_EXTENT_DATA_KEY) { 4426ad48fd75SYan, Zheng fi = btrfs_item_ptr(leaf, path->slots[0], 4427ad48fd75SYan, Zheng struct btrfs_file_extent_item); 4428ad48fd75SYan, Zheng if (extent_len != btrfs_file_extent_num_bytes(leaf, fi)) 4429ad48fd75SYan, Zheng goto err; 4430ad48fd75SYan, Zheng } 4431ad48fd75SYan, Zheng 4432ad48fd75SYan, Zheng btrfs_set_path_blocking(path); 4433ad48fd75SYan, Zheng ret = split_leaf(trans, root, &key, path, ins_len, 1); 4434f0486c68SYan, Zheng if (ret) 4435f0486c68SYan, Zheng goto err; 4436ad48fd75SYan, Zheng 4437ad48fd75SYan, Zheng path->keep_locks = 0; 4438ad48fd75SYan, Zheng btrfs_unlock_up_safe(path, 1); 4439ad48fd75SYan, Zheng return 0; 4440ad48fd75SYan, Zheng err: 4441ad48fd75SYan, Zheng path->keep_locks = 0; 4442ad48fd75SYan, Zheng return ret; 4443ad48fd75SYan, Zheng } 4444ad48fd75SYan, Zheng 4445ad48fd75SYan, Zheng static noinline int split_item(struct btrfs_trans_handle *trans, 4446459931ecSChris Mason struct btrfs_root *root, 4447459931ecSChris Mason struct btrfs_path *path, 4448459931ecSChris Mason struct btrfs_key *new_key, 4449459931ecSChris Mason unsigned long split_offset) 4450459931ecSChris Mason { 4451459931ecSChris Mason struct extent_buffer *leaf; 4452459931ecSChris Mason struct btrfs_item *item; 4453459931ecSChris Mason struct btrfs_item *new_item; 4454459931ecSChris Mason int slot; 4455ad48fd75SYan, Zheng char *buf; 4456459931ecSChris Mason u32 nritems; 4457ad48fd75SYan, Zheng u32 item_size; 4458459931ecSChris Mason u32 orig_offset; 4459459931ecSChris Mason struct btrfs_disk_key disk_key; 4460459931ecSChris Mason 4461459931ecSChris Mason leaf = path->nodes[0]; 4462b9473439SChris Mason BUG_ON(btrfs_leaf_free_space(root, leaf) < sizeof(struct btrfs_item)); 4463b9473439SChris Mason 4464b4ce94deSChris Mason btrfs_set_path_blocking(path); 4465b4ce94deSChris Mason 4466dd3cc16bSRoss Kirk item = btrfs_item_nr(path->slots[0]); 4467459931ecSChris Mason orig_offset = btrfs_item_offset(leaf, item); 4468459931ecSChris Mason item_size = btrfs_item_size(leaf, item); 4469459931ecSChris Mason 4470459931ecSChris Mason buf = kmalloc(item_size, GFP_NOFS); 4471ad48fd75SYan, Zheng if (!buf) 4472ad48fd75SYan, Zheng return -ENOMEM; 4473ad48fd75SYan, Zheng 4474459931ecSChris Mason read_extent_buffer(leaf, buf, btrfs_item_ptr_offset(leaf, 4475459931ecSChris Mason path->slots[0]), item_size); 4476ad48fd75SYan, Zheng 4477459931ecSChris Mason slot = path->slots[0] + 1; 4478459931ecSChris Mason nritems = btrfs_header_nritems(leaf); 4479459931ecSChris Mason if (slot != nritems) { 4480459931ecSChris Mason /* shift the items */ 4481459931ecSChris Mason memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + 1), 4482459931ecSChris Mason btrfs_item_nr_offset(slot), 4483459931ecSChris Mason (nritems - slot) * sizeof(struct btrfs_item)); 4484459931ecSChris Mason } 4485459931ecSChris Mason 4486459931ecSChris Mason btrfs_cpu_key_to_disk(&disk_key, new_key); 4487459931ecSChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 4488459931ecSChris Mason 4489dd3cc16bSRoss Kirk new_item = btrfs_item_nr(slot); 4490459931ecSChris Mason 4491459931ecSChris Mason btrfs_set_item_offset(leaf, new_item, orig_offset); 4492459931ecSChris Mason btrfs_set_item_size(leaf, new_item, item_size - split_offset); 4493459931ecSChris Mason 4494459931ecSChris Mason btrfs_set_item_offset(leaf, item, 4495459931ecSChris Mason orig_offset + item_size - split_offset); 4496459931ecSChris Mason btrfs_set_item_size(leaf, item, split_offset); 4497459931ecSChris Mason 4498459931ecSChris Mason btrfs_set_header_nritems(leaf, nritems + 1); 4499459931ecSChris Mason 4500459931ecSChris Mason /* write the data for the start of the original item */ 4501459931ecSChris Mason write_extent_buffer(leaf, buf, 4502459931ecSChris Mason btrfs_item_ptr_offset(leaf, path->slots[0]), 4503459931ecSChris Mason split_offset); 4504459931ecSChris Mason 4505459931ecSChris Mason /* write the data for the new item */ 4506459931ecSChris Mason write_extent_buffer(leaf, buf + split_offset, 4507459931ecSChris Mason btrfs_item_ptr_offset(leaf, slot), 4508459931ecSChris Mason item_size - split_offset); 4509459931ecSChris Mason btrfs_mark_buffer_dirty(leaf); 4510459931ecSChris Mason 4511ad48fd75SYan, Zheng BUG_ON(btrfs_leaf_free_space(root, leaf) < 0); 4512459931ecSChris Mason kfree(buf); 4513ad48fd75SYan, Zheng return 0; 4514ad48fd75SYan, Zheng } 4515ad48fd75SYan, Zheng 4516ad48fd75SYan, Zheng /* 4517ad48fd75SYan, Zheng * This function splits a single item into two items, 4518ad48fd75SYan, Zheng * giving 'new_key' to the new item and splitting the 4519ad48fd75SYan, Zheng * old one at split_offset (from the start of the item). 4520ad48fd75SYan, Zheng * 4521ad48fd75SYan, Zheng * The path may be released by this operation. After 4522ad48fd75SYan, Zheng * the split, the path is pointing to the old item. The 4523ad48fd75SYan, Zheng * new item is going to be in the same node as the old one. 4524ad48fd75SYan, Zheng * 4525ad48fd75SYan, Zheng * Note, the item being split must be smaller enough to live alone on 4526ad48fd75SYan, Zheng * a tree block with room for one extra struct btrfs_item 4527ad48fd75SYan, Zheng * 4528ad48fd75SYan, Zheng * This allows us to split the item in place, keeping a lock on the 4529ad48fd75SYan, Zheng * leaf the entire time. 4530ad48fd75SYan, Zheng */ 4531ad48fd75SYan, Zheng int btrfs_split_item(struct btrfs_trans_handle *trans, 4532ad48fd75SYan, Zheng struct btrfs_root *root, 4533ad48fd75SYan, Zheng struct btrfs_path *path, 4534ad48fd75SYan, Zheng struct btrfs_key *new_key, 4535ad48fd75SYan, Zheng unsigned long split_offset) 4536ad48fd75SYan, Zheng { 4537ad48fd75SYan, Zheng int ret; 4538ad48fd75SYan, Zheng ret = setup_leaf_for_split(trans, root, path, 4539ad48fd75SYan, Zheng sizeof(struct btrfs_item)); 4540ad48fd75SYan, Zheng if (ret) 4541459931ecSChris Mason return ret; 4542ad48fd75SYan, Zheng 4543ad48fd75SYan, Zheng ret = split_item(trans, root, path, new_key, split_offset); 4544ad48fd75SYan, Zheng return ret; 4545ad48fd75SYan, Zheng } 4546ad48fd75SYan, Zheng 4547ad48fd75SYan, Zheng /* 4548ad48fd75SYan, Zheng * This function duplicate a item, giving 'new_key' to the new item. 4549ad48fd75SYan, Zheng * It guarantees both items live in the same tree leaf and the new item 4550ad48fd75SYan, Zheng * is contiguous with the original item. 4551ad48fd75SYan, Zheng * 4552ad48fd75SYan, Zheng * This allows us to split file extent in place, keeping a lock on the 4553ad48fd75SYan, Zheng * leaf the entire time. 4554ad48fd75SYan, Zheng */ 4555ad48fd75SYan, Zheng int btrfs_duplicate_item(struct btrfs_trans_handle *trans, 4556ad48fd75SYan, Zheng struct btrfs_root *root, 4557ad48fd75SYan, Zheng struct btrfs_path *path, 4558ad48fd75SYan, Zheng struct btrfs_key *new_key) 4559ad48fd75SYan, Zheng { 4560ad48fd75SYan, Zheng struct extent_buffer *leaf; 4561ad48fd75SYan, Zheng int ret; 4562ad48fd75SYan, Zheng u32 item_size; 4563ad48fd75SYan, Zheng 4564ad48fd75SYan, Zheng leaf = path->nodes[0]; 4565ad48fd75SYan, Zheng item_size = btrfs_item_size_nr(leaf, path->slots[0]); 4566ad48fd75SYan, Zheng ret = setup_leaf_for_split(trans, root, path, 4567ad48fd75SYan, Zheng item_size + sizeof(struct btrfs_item)); 4568ad48fd75SYan, Zheng if (ret) 4569ad48fd75SYan, Zheng return ret; 4570ad48fd75SYan, Zheng 4571ad48fd75SYan, Zheng path->slots[0]++; 4572afe5fea7STsutomu Itoh setup_items_for_insert(root, path, new_key, &item_size, 4573ad48fd75SYan, Zheng item_size, item_size + 4574ad48fd75SYan, Zheng sizeof(struct btrfs_item), 1); 4575ad48fd75SYan, Zheng leaf = path->nodes[0]; 4576ad48fd75SYan, Zheng memcpy_extent_buffer(leaf, 4577ad48fd75SYan, Zheng btrfs_item_ptr_offset(leaf, path->slots[0]), 4578ad48fd75SYan, Zheng btrfs_item_ptr_offset(leaf, path->slots[0] - 1), 4579ad48fd75SYan, Zheng item_size); 4580ad48fd75SYan, Zheng return 0; 4581459931ecSChris Mason } 4582459931ecSChris Mason 4583459931ecSChris Mason /* 4584d352ac68SChris Mason * make the item pointed to by the path smaller. new_size indicates 4585d352ac68SChris Mason * how small to make it, and from_end tells us if we just chop bytes 4586d352ac68SChris Mason * off the end of the item or if we shift the item to chop bytes off 4587d352ac68SChris Mason * the front. 4588d352ac68SChris Mason */ 4589afe5fea7STsutomu Itoh void btrfs_truncate_item(struct btrfs_root *root, struct btrfs_path *path, 4590179e29e4SChris Mason u32 new_size, int from_end) 4591b18c6685SChris Mason { 4592b18c6685SChris Mason int slot; 45935f39d397SChris Mason struct extent_buffer *leaf; 45945f39d397SChris Mason struct btrfs_item *item; 4595b18c6685SChris Mason u32 nritems; 4596b18c6685SChris Mason unsigned int data_end; 4597b18c6685SChris Mason unsigned int old_data_start; 4598b18c6685SChris Mason unsigned int old_size; 4599b18c6685SChris Mason unsigned int size_diff; 4600b18c6685SChris Mason int i; 4601cfed81a0SChris Mason struct btrfs_map_token token; 4602cfed81a0SChris Mason 4603cfed81a0SChris Mason btrfs_init_map_token(&token); 4604b18c6685SChris Mason 46055f39d397SChris Mason leaf = path->nodes[0]; 4606179e29e4SChris Mason slot = path->slots[0]; 4607179e29e4SChris Mason 4608179e29e4SChris Mason old_size = btrfs_item_size_nr(leaf, slot); 4609179e29e4SChris Mason if (old_size == new_size) 4610143bede5SJeff Mahoney return; 4611b18c6685SChris Mason 46125f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 4613b18c6685SChris Mason data_end = leaf_data_end(root, leaf); 4614b18c6685SChris Mason 46155f39d397SChris Mason old_data_start = btrfs_item_offset_nr(leaf, slot); 4616179e29e4SChris Mason 4617b18c6685SChris Mason size_diff = old_size - new_size; 4618b18c6685SChris Mason 4619b18c6685SChris Mason BUG_ON(slot < 0); 4620b18c6685SChris Mason BUG_ON(slot >= nritems); 4621b18c6685SChris Mason 4622b18c6685SChris Mason /* 4623b18c6685SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 4624b18c6685SChris Mason */ 4625b18c6685SChris Mason /* first correct the data pointers */ 4626b18c6685SChris Mason for (i = slot; i < nritems; i++) { 46275f39d397SChris Mason u32 ioff; 4628dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 4629db94535dSChris Mason 4630cfed81a0SChris Mason ioff = btrfs_token_item_offset(leaf, item, &token); 4631cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 4632cfed81a0SChris Mason ioff + size_diff, &token); 4633b18c6685SChris Mason } 4634db94535dSChris Mason 4635b18c6685SChris Mason /* shift the data */ 4636179e29e4SChris Mason if (from_end) { 46375f39d397SChris Mason memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) + 4638b18c6685SChris Mason data_end + size_diff, btrfs_leaf_data(leaf) + 4639b18c6685SChris Mason data_end, old_data_start + new_size - data_end); 4640179e29e4SChris Mason } else { 4641179e29e4SChris Mason struct btrfs_disk_key disk_key; 4642179e29e4SChris Mason u64 offset; 4643179e29e4SChris Mason 4644179e29e4SChris Mason btrfs_item_key(leaf, &disk_key, slot); 4645179e29e4SChris Mason 4646179e29e4SChris Mason if (btrfs_disk_key_type(&disk_key) == BTRFS_EXTENT_DATA_KEY) { 4647179e29e4SChris Mason unsigned long ptr; 4648179e29e4SChris Mason struct btrfs_file_extent_item *fi; 4649179e29e4SChris Mason 4650179e29e4SChris Mason fi = btrfs_item_ptr(leaf, slot, 4651179e29e4SChris Mason struct btrfs_file_extent_item); 4652179e29e4SChris Mason fi = (struct btrfs_file_extent_item *)( 4653179e29e4SChris Mason (unsigned long)fi - size_diff); 4654179e29e4SChris Mason 4655179e29e4SChris Mason if (btrfs_file_extent_type(leaf, fi) == 4656179e29e4SChris Mason BTRFS_FILE_EXTENT_INLINE) { 4657179e29e4SChris Mason ptr = btrfs_item_ptr_offset(leaf, slot); 4658179e29e4SChris Mason memmove_extent_buffer(leaf, ptr, 4659179e29e4SChris Mason (unsigned long)fi, 4660179e29e4SChris Mason offsetof(struct btrfs_file_extent_item, 4661179e29e4SChris Mason disk_bytenr)); 4662179e29e4SChris Mason } 4663179e29e4SChris Mason } 4664179e29e4SChris Mason 4665179e29e4SChris Mason memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) + 4666179e29e4SChris Mason data_end + size_diff, btrfs_leaf_data(leaf) + 4667179e29e4SChris Mason data_end, old_data_start - data_end); 4668179e29e4SChris Mason 4669179e29e4SChris Mason offset = btrfs_disk_key_offset(&disk_key); 4670179e29e4SChris Mason btrfs_set_disk_key_offset(&disk_key, offset + size_diff); 4671179e29e4SChris Mason btrfs_set_item_key(leaf, &disk_key, slot); 4672179e29e4SChris Mason if (slot == 0) 4673d6a0a126STsutomu Itoh fixup_low_keys(root, path, &disk_key, 1); 4674179e29e4SChris Mason } 46755f39d397SChris Mason 4676dd3cc16bSRoss Kirk item = btrfs_item_nr(slot); 46775f39d397SChris Mason btrfs_set_item_size(leaf, item, new_size); 46785f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 4679b18c6685SChris Mason 46805f39d397SChris Mason if (btrfs_leaf_free_space(root, leaf) < 0) { 46815f39d397SChris Mason btrfs_print_leaf(root, leaf); 4682b18c6685SChris Mason BUG(); 46835f39d397SChris Mason } 4684b18c6685SChris Mason } 4685b18c6685SChris Mason 4686d352ac68SChris Mason /* 46878f69dbd2SStefan Behrens * make the item pointed to by the path bigger, data_size is the added size. 4688d352ac68SChris Mason */ 46894b90c680STsutomu Itoh void btrfs_extend_item(struct btrfs_root *root, struct btrfs_path *path, 46905f39d397SChris Mason u32 data_size) 46916567e837SChris Mason { 46926567e837SChris Mason int slot; 46935f39d397SChris Mason struct extent_buffer *leaf; 46945f39d397SChris Mason struct btrfs_item *item; 46956567e837SChris Mason u32 nritems; 46966567e837SChris Mason unsigned int data_end; 46976567e837SChris Mason unsigned int old_data; 46986567e837SChris Mason unsigned int old_size; 46996567e837SChris Mason int i; 4700cfed81a0SChris Mason struct btrfs_map_token token; 4701cfed81a0SChris Mason 4702cfed81a0SChris Mason btrfs_init_map_token(&token); 47036567e837SChris Mason 47045f39d397SChris Mason leaf = path->nodes[0]; 47056567e837SChris Mason 47065f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 47076567e837SChris Mason data_end = leaf_data_end(root, leaf); 47086567e837SChris Mason 47095f39d397SChris Mason if (btrfs_leaf_free_space(root, leaf) < data_size) { 47105f39d397SChris Mason btrfs_print_leaf(root, leaf); 47116567e837SChris Mason BUG(); 47125f39d397SChris Mason } 47136567e837SChris Mason slot = path->slots[0]; 47145f39d397SChris Mason old_data = btrfs_item_end_nr(leaf, slot); 47156567e837SChris Mason 47166567e837SChris Mason BUG_ON(slot < 0); 47173326d1b0SChris Mason if (slot >= nritems) { 47183326d1b0SChris Mason btrfs_print_leaf(root, leaf); 4719efe120a0SFrank Holton btrfs_crit(root->fs_info, "slot %d too large, nritems %d", 4720d397712bSChris Mason slot, nritems); 47213326d1b0SChris Mason BUG_ON(1); 47223326d1b0SChris Mason } 47236567e837SChris Mason 47246567e837SChris Mason /* 47256567e837SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 47266567e837SChris Mason */ 47276567e837SChris Mason /* first correct the data pointers */ 47286567e837SChris Mason for (i = slot; i < nritems; i++) { 47295f39d397SChris Mason u32 ioff; 4730dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 4731db94535dSChris Mason 4732cfed81a0SChris Mason ioff = btrfs_token_item_offset(leaf, item, &token); 4733cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 4734cfed81a0SChris Mason ioff - data_size, &token); 47356567e837SChris Mason } 47365f39d397SChris Mason 47376567e837SChris Mason /* shift the data */ 47385f39d397SChris Mason memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) + 47396567e837SChris Mason data_end - data_size, btrfs_leaf_data(leaf) + 47406567e837SChris Mason data_end, old_data - data_end); 47415f39d397SChris Mason 47426567e837SChris Mason data_end = old_data; 47435f39d397SChris Mason old_size = btrfs_item_size_nr(leaf, slot); 4744dd3cc16bSRoss Kirk item = btrfs_item_nr(slot); 47455f39d397SChris Mason btrfs_set_item_size(leaf, item, old_size + data_size); 47465f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 47476567e837SChris Mason 47485f39d397SChris Mason if (btrfs_leaf_free_space(root, leaf) < 0) { 47495f39d397SChris Mason btrfs_print_leaf(root, leaf); 47506567e837SChris Mason BUG(); 47515f39d397SChris Mason } 47526567e837SChris Mason } 47536567e837SChris Mason 475474123bd7SChris Mason /* 475544871b1bSChris Mason * this is a helper for btrfs_insert_empty_items, the main goal here is 475644871b1bSChris Mason * to save stack depth by doing the bulk of the work in a function 475744871b1bSChris Mason * that doesn't call btrfs_search_slot 475874123bd7SChris Mason */ 4759afe5fea7STsutomu Itoh void setup_items_for_insert(struct btrfs_root *root, struct btrfs_path *path, 47609c58309dSChris Mason struct btrfs_key *cpu_key, u32 *data_size, 476144871b1bSChris Mason u32 total_data, u32 total_size, int nr) 4762be0e5c09SChris Mason { 47635f39d397SChris Mason struct btrfs_item *item; 47649c58309dSChris Mason int i; 47657518a238SChris Mason u32 nritems; 4766be0e5c09SChris Mason unsigned int data_end; 4767e2fa7227SChris Mason struct btrfs_disk_key disk_key; 476844871b1bSChris Mason struct extent_buffer *leaf; 476944871b1bSChris Mason int slot; 4770cfed81a0SChris Mason struct btrfs_map_token token; 4771cfed81a0SChris Mason 4772cfed81a0SChris Mason btrfs_init_map_token(&token); 4773e2fa7227SChris Mason 47745f39d397SChris Mason leaf = path->nodes[0]; 477544871b1bSChris Mason slot = path->slots[0]; 477674123bd7SChris Mason 47775f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 4778123abc88SChris Mason data_end = leaf_data_end(root, leaf); 4779eb60ceacSChris Mason 4780f25956ccSChris Mason if (btrfs_leaf_free_space(root, leaf) < total_size) { 47813326d1b0SChris Mason btrfs_print_leaf(root, leaf); 4782efe120a0SFrank Holton btrfs_crit(root->fs_info, "not enough freespace need %u have %d", 47839c58309dSChris Mason total_size, btrfs_leaf_free_space(root, leaf)); 4784be0e5c09SChris Mason BUG(); 4785d4dbff95SChris Mason } 47865f39d397SChris Mason 4787be0e5c09SChris Mason if (slot != nritems) { 47885f39d397SChris Mason unsigned int old_data = btrfs_item_end_nr(leaf, slot); 4789be0e5c09SChris Mason 47905f39d397SChris Mason if (old_data < data_end) { 47915f39d397SChris Mason btrfs_print_leaf(root, leaf); 4792efe120a0SFrank Holton btrfs_crit(root->fs_info, "slot %d old_data %d data_end %d", 47935f39d397SChris Mason slot, old_data, data_end); 47945f39d397SChris Mason BUG_ON(1); 47955f39d397SChris Mason } 4796be0e5c09SChris Mason /* 4797be0e5c09SChris Mason * item0..itemN ... dataN.offset..dataN.size .. data0.size 4798be0e5c09SChris Mason */ 4799be0e5c09SChris Mason /* first correct the data pointers */ 48000783fcfcSChris Mason for (i = slot; i < nritems; i++) { 48015f39d397SChris Mason u32 ioff; 4802db94535dSChris Mason 4803dd3cc16bSRoss Kirk item = btrfs_item_nr( i); 4804cfed81a0SChris Mason ioff = btrfs_token_item_offset(leaf, item, &token); 4805cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 4806cfed81a0SChris Mason ioff - total_data, &token); 48070783fcfcSChris Mason } 4808be0e5c09SChris Mason /* shift the items */ 48099c58309dSChris Mason memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + nr), 48105f39d397SChris Mason btrfs_item_nr_offset(slot), 48110783fcfcSChris Mason (nritems - slot) * sizeof(struct btrfs_item)); 4812be0e5c09SChris Mason 4813be0e5c09SChris Mason /* shift the data */ 48145f39d397SChris Mason memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) + 48159c58309dSChris Mason data_end - total_data, btrfs_leaf_data(leaf) + 4816be0e5c09SChris Mason data_end, old_data - data_end); 4817be0e5c09SChris Mason data_end = old_data; 4818be0e5c09SChris Mason } 48195f39d397SChris Mason 482062e2749eSChris Mason /* setup the item for the new data */ 48219c58309dSChris Mason for (i = 0; i < nr; i++) { 48229c58309dSChris Mason btrfs_cpu_key_to_disk(&disk_key, cpu_key + i); 48239c58309dSChris Mason btrfs_set_item_key(leaf, &disk_key, slot + i); 4824dd3cc16bSRoss Kirk item = btrfs_item_nr(slot + i); 4825cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 4826cfed81a0SChris Mason data_end - data_size[i], &token); 48279c58309dSChris Mason data_end -= data_size[i]; 4828cfed81a0SChris Mason btrfs_set_token_item_size(leaf, item, data_size[i], &token); 48299c58309dSChris Mason } 483044871b1bSChris Mason 48319c58309dSChris Mason btrfs_set_header_nritems(leaf, nritems + nr); 4832aa5d6bedSChris Mason 48335a01a2e3SChris Mason if (slot == 0) { 48345a01a2e3SChris Mason btrfs_cpu_key_to_disk(&disk_key, cpu_key); 4835d6a0a126STsutomu Itoh fixup_low_keys(root, path, &disk_key, 1); 48365a01a2e3SChris Mason } 4837b9473439SChris Mason btrfs_unlock_up_safe(path, 1); 4838b9473439SChris Mason btrfs_mark_buffer_dirty(leaf); 4839aa5d6bedSChris Mason 48405f39d397SChris Mason if (btrfs_leaf_free_space(root, leaf) < 0) { 48415f39d397SChris Mason btrfs_print_leaf(root, leaf); 4842be0e5c09SChris Mason BUG(); 48435f39d397SChris Mason } 484444871b1bSChris Mason } 484544871b1bSChris Mason 484644871b1bSChris Mason /* 484744871b1bSChris Mason * Given a key and some data, insert items into the tree. 484844871b1bSChris Mason * This does all the path init required, making room in the tree if needed. 484944871b1bSChris Mason */ 485044871b1bSChris Mason int btrfs_insert_empty_items(struct btrfs_trans_handle *trans, 485144871b1bSChris Mason struct btrfs_root *root, 485244871b1bSChris Mason struct btrfs_path *path, 485344871b1bSChris Mason struct btrfs_key *cpu_key, u32 *data_size, 485444871b1bSChris Mason int nr) 485544871b1bSChris Mason { 485644871b1bSChris Mason int ret = 0; 485744871b1bSChris Mason int slot; 485844871b1bSChris Mason int i; 485944871b1bSChris Mason u32 total_size = 0; 486044871b1bSChris Mason u32 total_data = 0; 486144871b1bSChris Mason 486244871b1bSChris Mason for (i = 0; i < nr; i++) 486344871b1bSChris Mason total_data += data_size[i]; 486444871b1bSChris Mason 486544871b1bSChris Mason total_size = total_data + (nr * sizeof(struct btrfs_item)); 486644871b1bSChris Mason ret = btrfs_search_slot(trans, root, cpu_key, path, total_size, 1); 486744871b1bSChris Mason if (ret == 0) 486844871b1bSChris Mason return -EEXIST; 486944871b1bSChris Mason if (ret < 0) 4870143bede5SJeff Mahoney return ret; 487144871b1bSChris Mason 487244871b1bSChris Mason slot = path->slots[0]; 487344871b1bSChris Mason BUG_ON(slot < 0); 487444871b1bSChris Mason 4875afe5fea7STsutomu Itoh setup_items_for_insert(root, path, cpu_key, data_size, 487644871b1bSChris Mason total_data, total_size, nr); 4877143bede5SJeff Mahoney return 0; 487862e2749eSChris Mason } 487962e2749eSChris Mason 488062e2749eSChris Mason /* 488162e2749eSChris Mason * Given a key and some data, insert an item into the tree. 488262e2749eSChris Mason * This does all the path init required, making room in the tree if needed. 488362e2749eSChris Mason */ 4884e089f05cSChris Mason int btrfs_insert_item(struct btrfs_trans_handle *trans, struct btrfs_root 4885e089f05cSChris Mason *root, struct btrfs_key *cpu_key, void *data, u32 4886e089f05cSChris Mason data_size) 488762e2749eSChris Mason { 488862e2749eSChris Mason int ret = 0; 48892c90e5d6SChris Mason struct btrfs_path *path; 48905f39d397SChris Mason struct extent_buffer *leaf; 48915f39d397SChris Mason unsigned long ptr; 489262e2749eSChris Mason 48932c90e5d6SChris Mason path = btrfs_alloc_path(); 4894db5b493aSTsutomu Itoh if (!path) 4895db5b493aSTsutomu Itoh return -ENOMEM; 48962c90e5d6SChris Mason ret = btrfs_insert_empty_item(trans, root, path, cpu_key, data_size); 489762e2749eSChris Mason if (!ret) { 48985f39d397SChris Mason leaf = path->nodes[0]; 48995f39d397SChris Mason ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); 49005f39d397SChris Mason write_extent_buffer(leaf, data, ptr, data_size); 49015f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 490262e2749eSChris Mason } 49032c90e5d6SChris Mason btrfs_free_path(path); 4904aa5d6bedSChris Mason return ret; 4905be0e5c09SChris Mason } 4906be0e5c09SChris Mason 490774123bd7SChris Mason /* 49085de08d7dSChris Mason * delete the pointer from a given node. 490974123bd7SChris Mason * 4910d352ac68SChris Mason * the tree should have been previously balanced so the deletion does not 4911d352ac68SChris Mason * empty a node. 491274123bd7SChris Mason */ 4913afe5fea7STsutomu Itoh static void del_ptr(struct btrfs_root *root, struct btrfs_path *path, 4914afe5fea7STsutomu Itoh int level, int slot) 4915be0e5c09SChris Mason { 49165f39d397SChris Mason struct extent_buffer *parent = path->nodes[level]; 49177518a238SChris Mason u32 nritems; 4918f3ea38daSJan Schmidt int ret; 4919be0e5c09SChris Mason 49205f39d397SChris Mason nritems = btrfs_header_nritems(parent); 4921be0e5c09SChris Mason if (slot != nritems - 1) { 49220e411eceSLiu Bo if (level) 4923f3ea38daSJan Schmidt tree_mod_log_eb_move(root->fs_info, parent, slot, 4924f3ea38daSJan Schmidt slot + 1, nritems - slot - 1); 49255f39d397SChris Mason memmove_extent_buffer(parent, 49265f39d397SChris Mason btrfs_node_key_ptr_offset(slot), 49275f39d397SChris Mason btrfs_node_key_ptr_offset(slot + 1), 4928d6025579SChris Mason sizeof(struct btrfs_key_ptr) * 4929d6025579SChris Mason (nritems - slot - 1)); 493057ba86c0SChris Mason } else if (level) { 493157ba86c0SChris Mason ret = tree_mod_log_insert_key(root->fs_info, parent, slot, 4932c8cc6341SJosef Bacik MOD_LOG_KEY_REMOVE, GFP_NOFS); 493357ba86c0SChris Mason BUG_ON(ret < 0); 4934be0e5c09SChris Mason } 4935f3ea38daSJan Schmidt 49367518a238SChris Mason nritems--; 49375f39d397SChris Mason btrfs_set_header_nritems(parent, nritems); 49387518a238SChris Mason if (nritems == 0 && parent == root->node) { 49395f39d397SChris Mason BUG_ON(btrfs_header_level(root->node) != 1); 4940eb60ceacSChris Mason /* just turn the root into a leaf and break */ 49415f39d397SChris Mason btrfs_set_header_level(root->node, 0); 4942bb803951SChris Mason } else if (slot == 0) { 49435f39d397SChris Mason struct btrfs_disk_key disk_key; 49445f39d397SChris Mason 49455f39d397SChris Mason btrfs_node_key(parent, &disk_key, 0); 4946d6a0a126STsutomu Itoh fixup_low_keys(root, path, &disk_key, level + 1); 4947be0e5c09SChris Mason } 4948d6025579SChris Mason btrfs_mark_buffer_dirty(parent); 4949be0e5c09SChris Mason } 4950be0e5c09SChris Mason 495174123bd7SChris Mason /* 4952323ac95bSChris Mason * a helper function to delete the leaf pointed to by path->slots[1] and 49535d4f98a2SYan Zheng * path->nodes[1]. 4954323ac95bSChris Mason * 4955323ac95bSChris Mason * This deletes the pointer in path->nodes[1] and frees the leaf 4956323ac95bSChris Mason * block extent. zero is returned if it all worked out, < 0 otherwise. 4957323ac95bSChris Mason * 4958323ac95bSChris Mason * The path must have already been setup for deleting the leaf, including 4959323ac95bSChris Mason * all the proper balancing. path->nodes[1] must be locked. 4960323ac95bSChris Mason */ 4961143bede5SJeff Mahoney static noinline void btrfs_del_leaf(struct btrfs_trans_handle *trans, 4962323ac95bSChris Mason struct btrfs_root *root, 49635d4f98a2SYan Zheng struct btrfs_path *path, 49645d4f98a2SYan Zheng struct extent_buffer *leaf) 4965323ac95bSChris Mason { 49665d4f98a2SYan Zheng WARN_ON(btrfs_header_generation(leaf) != trans->transid); 4967afe5fea7STsutomu Itoh del_ptr(root, path, 1, path->slots[1]); 4968323ac95bSChris Mason 49694d081c41SChris Mason /* 49704d081c41SChris Mason * btrfs_free_extent is expensive, we want to make sure we 49714d081c41SChris Mason * aren't holding any locks when we call it 49724d081c41SChris Mason */ 49734d081c41SChris Mason btrfs_unlock_up_safe(path, 0); 49744d081c41SChris Mason 4975f0486c68SYan, Zheng root_sub_used(root, leaf->len); 4976f0486c68SYan, Zheng 49773083ee2eSJosef Bacik extent_buffer_get(leaf); 49785581a51aSJan Schmidt btrfs_free_tree_block(trans, root, leaf, 0, 1); 49793083ee2eSJosef Bacik free_extent_buffer_stale(leaf); 4980323ac95bSChris Mason } 4981323ac95bSChris Mason /* 498274123bd7SChris Mason * delete the item at the leaf level in path. If that empties 498374123bd7SChris Mason * the leaf, remove it from the tree 498474123bd7SChris Mason */ 498585e21bacSChris Mason int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root, 498685e21bacSChris Mason struct btrfs_path *path, int slot, int nr) 4987be0e5c09SChris Mason { 49885f39d397SChris Mason struct extent_buffer *leaf; 49895f39d397SChris Mason struct btrfs_item *item; 499085e21bacSChris Mason int last_off; 499185e21bacSChris Mason int dsize = 0; 4992aa5d6bedSChris Mason int ret = 0; 4993aa5d6bedSChris Mason int wret; 499485e21bacSChris Mason int i; 49957518a238SChris Mason u32 nritems; 4996cfed81a0SChris Mason struct btrfs_map_token token; 4997cfed81a0SChris Mason 4998cfed81a0SChris Mason btrfs_init_map_token(&token); 4999be0e5c09SChris Mason 50005f39d397SChris Mason leaf = path->nodes[0]; 500185e21bacSChris Mason last_off = btrfs_item_offset_nr(leaf, slot + nr - 1); 500285e21bacSChris Mason 500385e21bacSChris Mason for (i = 0; i < nr; i++) 500485e21bacSChris Mason dsize += btrfs_item_size_nr(leaf, slot + i); 500585e21bacSChris Mason 50065f39d397SChris Mason nritems = btrfs_header_nritems(leaf); 5007be0e5c09SChris Mason 500885e21bacSChris Mason if (slot + nr != nritems) { 5009123abc88SChris Mason int data_end = leaf_data_end(root, leaf); 50105f39d397SChris Mason 50115f39d397SChris Mason memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) + 5012d6025579SChris Mason data_end + dsize, 5013123abc88SChris Mason btrfs_leaf_data(leaf) + data_end, 501485e21bacSChris Mason last_off - data_end); 50155f39d397SChris Mason 501685e21bacSChris Mason for (i = slot + nr; i < nritems; i++) { 50175f39d397SChris Mason u32 ioff; 5018db94535dSChris Mason 5019dd3cc16bSRoss Kirk item = btrfs_item_nr(i); 5020cfed81a0SChris Mason ioff = btrfs_token_item_offset(leaf, item, &token); 5021cfed81a0SChris Mason btrfs_set_token_item_offset(leaf, item, 5022cfed81a0SChris Mason ioff + dsize, &token); 50230783fcfcSChris Mason } 5024db94535dSChris Mason 50255f39d397SChris Mason memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot), 502685e21bacSChris Mason btrfs_item_nr_offset(slot + nr), 50270783fcfcSChris Mason sizeof(struct btrfs_item) * 502885e21bacSChris Mason (nritems - slot - nr)); 5029be0e5c09SChris Mason } 503085e21bacSChris Mason btrfs_set_header_nritems(leaf, nritems - nr); 503185e21bacSChris Mason nritems -= nr; 50325f39d397SChris Mason 503374123bd7SChris Mason /* delete the leaf if we've emptied it */ 50347518a238SChris Mason if (nritems == 0) { 50355f39d397SChris Mason if (leaf == root->node) { 50365f39d397SChris Mason btrfs_set_header_level(leaf, 0); 50379a8dd150SChris Mason } else { 5038f0486c68SYan, Zheng btrfs_set_path_blocking(path); 5039f0486c68SYan, Zheng clean_tree_block(trans, root, leaf); 5040143bede5SJeff Mahoney btrfs_del_leaf(trans, root, path, leaf); 50419a8dd150SChris Mason } 5042be0e5c09SChris Mason } else { 50437518a238SChris Mason int used = leaf_space_used(leaf, 0, nritems); 5044aa5d6bedSChris Mason if (slot == 0) { 50455f39d397SChris Mason struct btrfs_disk_key disk_key; 50465f39d397SChris Mason 50475f39d397SChris Mason btrfs_item_key(leaf, &disk_key, 0); 5048d6a0a126STsutomu Itoh fixup_low_keys(root, path, &disk_key, 1); 5049aa5d6bedSChris Mason } 5050aa5d6bedSChris Mason 505174123bd7SChris Mason /* delete the leaf if it is mostly empty */ 5052d717aa1dSYan Zheng if (used < BTRFS_LEAF_DATA_SIZE(root) / 3) { 5053be0e5c09SChris Mason /* push_leaf_left fixes the path. 5054be0e5c09SChris Mason * make sure the path still points to our leaf 5055be0e5c09SChris Mason * for possible call to del_ptr below 5056be0e5c09SChris Mason */ 50574920c9acSChris Mason slot = path->slots[1]; 50585f39d397SChris Mason extent_buffer_get(leaf); 50595f39d397SChris Mason 5060b9473439SChris Mason btrfs_set_path_blocking(path); 506199d8f83cSChris Mason wret = push_leaf_left(trans, root, path, 1, 1, 506299d8f83cSChris Mason 1, (u32)-1); 506354aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 5064aa5d6bedSChris Mason ret = wret; 50655f39d397SChris Mason 50665f39d397SChris Mason if (path->nodes[0] == leaf && 50675f39d397SChris Mason btrfs_header_nritems(leaf)) { 506899d8f83cSChris Mason wret = push_leaf_right(trans, root, path, 1, 506999d8f83cSChris Mason 1, 1, 0); 507054aa1f4dSChris Mason if (wret < 0 && wret != -ENOSPC) 5071aa5d6bedSChris Mason ret = wret; 5072aa5d6bedSChris Mason } 50735f39d397SChris Mason 50745f39d397SChris Mason if (btrfs_header_nritems(leaf) == 0) { 5075323ac95bSChris Mason path->slots[1] = slot; 5076143bede5SJeff Mahoney btrfs_del_leaf(trans, root, path, leaf); 50775f39d397SChris Mason free_extent_buffer(leaf); 5078143bede5SJeff Mahoney ret = 0; 50795de08d7dSChris Mason } else { 5080925baeddSChris Mason /* if we're still in the path, make sure 5081925baeddSChris Mason * we're dirty. Otherwise, one of the 5082925baeddSChris Mason * push_leaf functions must have already 5083925baeddSChris Mason * dirtied this buffer 5084925baeddSChris Mason */ 5085925baeddSChris Mason if (path->nodes[0] == leaf) 50865f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 50875f39d397SChris Mason free_extent_buffer(leaf); 5088be0e5c09SChris Mason } 5089d5719762SChris Mason } else { 50905f39d397SChris Mason btrfs_mark_buffer_dirty(leaf); 5091be0e5c09SChris Mason } 50929a8dd150SChris Mason } 5093aa5d6bedSChris Mason return ret; 50949a8dd150SChris Mason } 50959a8dd150SChris Mason 509697571fd0SChris Mason /* 5097925baeddSChris Mason * search the tree again to find a leaf with lesser keys 50987bb86316SChris Mason * returns 0 if it found something or 1 if there are no lesser leaves. 50997bb86316SChris Mason * returns < 0 on io errors. 5100d352ac68SChris Mason * 5101d352ac68SChris Mason * This may release the path, and so you may lose any locks held at the 5102d352ac68SChris Mason * time you call it. 51037bb86316SChris Mason */ 510416e7549fSJosef Bacik int btrfs_prev_leaf(struct btrfs_root *root, struct btrfs_path *path) 51057bb86316SChris Mason { 5106925baeddSChris Mason struct btrfs_key key; 5107925baeddSChris Mason struct btrfs_disk_key found_key; 5108925baeddSChris Mason int ret; 51097bb86316SChris Mason 5110925baeddSChris Mason btrfs_item_key_to_cpu(path->nodes[0], &key, 0); 5111925baeddSChris Mason 5112e8b0d724SFilipe David Borba Manana if (key.offset > 0) { 5113925baeddSChris Mason key.offset--; 5114e8b0d724SFilipe David Borba Manana } else if (key.type > 0) { 5115925baeddSChris Mason key.type--; 5116e8b0d724SFilipe David Borba Manana key.offset = (u64)-1; 5117e8b0d724SFilipe David Borba Manana } else if (key.objectid > 0) { 5118925baeddSChris Mason key.objectid--; 5119e8b0d724SFilipe David Borba Manana key.type = (u8)-1; 5120e8b0d724SFilipe David Borba Manana key.offset = (u64)-1; 5121e8b0d724SFilipe David Borba Manana } else { 51227bb86316SChris Mason return 1; 5123e8b0d724SFilipe David Borba Manana } 51247bb86316SChris Mason 5125b3b4aa74SDavid Sterba btrfs_release_path(path); 5126925baeddSChris Mason ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 5127925baeddSChris Mason if (ret < 0) 5128925baeddSChris Mason return ret; 5129925baeddSChris Mason btrfs_item_key(path->nodes[0], &found_key, 0); 5130925baeddSChris Mason ret = comp_keys(&found_key, &key); 5131925baeddSChris Mason if (ret < 0) 51327bb86316SChris Mason return 0; 5133925baeddSChris Mason return 1; 51347bb86316SChris Mason } 51357bb86316SChris Mason 51363f157a2fSChris Mason /* 51373f157a2fSChris Mason * A helper function to walk down the tree starting at min_key, and looking 5138de78b51aSEric Sandeen * for nodes or leaves that are have a minimum transaction id. 5139de78b51aSEric Sandeen * This is used by the btree defrag code, and tree logging 51403f157a2fSChris Mason * 51413f157a2fSChris Mason * This does not cow, but it does stuff the starting key it finds back 51423f157a2fSChris Mason * into min_key, so you can call btrfs_search_slot with cow=1 on the 51433f157a2fSChris Mason * key and get a writable path. 51443f157a2fSChris Mason * 51453f157a2fSChris Mason * This does lock as it descends, and path->keep_locks should be set 51463f157a2fSChris Mason * to 1 by the caller. 51473f157a2fSChris Mason * 51483f157a2fSChris Mason * This honors path->lowest_level to prevent descent past a given level 51493f157a2fSChris Mason * of the tree. 51503f157a2fSChris Mason * 5151d352ac68SChris Mason * min_trans indicates the oldest transaction that you are interested 5152d352ac68SChris Mason * in walking through. Any nodes or leaves older than min_trans are 5153d352ac68SChris Mason * skipped over (without reading them). 5154d352ac68SChris Mason * 51553f157a2fSChris Mason * returns zero if something useful was found, < 0 on error and 1 if there 51563f157a2fSChris Mason * was nothing in the tree that matched the search criteria. 51573f157a2fSChris Mason */ 51583f157a2fSChris Mason int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key, 5159de78b51aSEric Sandeen struct btrfs_path *path, 51603f157a2fSChris Mason u64 min_trans) 51613f157a2fSChris Mason { 51623f157a2fSChris Mason struct extent_buffer *cur; 51633f157a2fSChris Mason struct btrfs_key found_key; 51643f157a2fSChris Mason int slot; 51659652480bSYan int sret; 51663f157a2fSChris Mason u32 nritems; 51673f157a2fSChris Mason int level; 51683f157a2fSChris Mason int ret = 1; 51693f157a2fSChris Mason 5170934d375bSChris Mason WARN_ON(!path->keep_locks); 51713f157a2fSChris Mason again: 5172bd681513SChris Mason cur = btrfs_read_lock_root_node(root); 51733f157a2fSChris Mason level = btrfs_header_level(cur); 5174e02119d5SChris Mason WARN_ON(path->nodes[level]); 51753f157a2fSChris Mason path->nodes[level] = cur; 5176bd681513SChris Mason path->locks[level] = BTRFS_READ_LOCK; 51773f157a2fSChris Mason 51783f157a2fSChris Mason if (btrfs_header_generation(cur) < min_trans) { 51793f157a2fSChris Mason ret = 1; 51803f157a2fSChris Mason goto out; 51813f157a2fSChris Mason } 51823f157a2fSChris Mason while (1) { 51833f157a2fSChris Mason nritems = btrfs_header_nritems(cur); 51843f157a2fSChris Mason level = btrfs_header_level(cur); 51859652480bSYan sret = bin_search(cur, min_key, level, &slot); 51863f157a2fSChris Mason 5187323ac95bSChris Mason /* at the lowest level, we're done, setup the path and exit */ 5188323ac95bSChris Mason if (level == path->lowest_level) { 5189e02119d5SChris Mason if (slot >= nritems) 5190e02119d5SChris Mason goto find_next_key; 51913f157a2fSChris Mason ret = 0; 51923f157a2fSChris Mason path->slots[level] = slot; 51933f157a2fSChris Mason btrfs_item_key_to_cpu(cur, &found_key, slot); 51943f157a2fSChris Mason goto out; 51953f157a2fSChris Mason } 51969652480bSYan if (sret && slot > 0) 51979652480bSYan slot--; 51983f157a2fSChris Mason /* 5199de78b51aSEric Sandeen * check this node pointer against the min_trans parameters. 5200de78b51aSEric Sandeen * If it is too old, old, skip to the next one. 52013f157a2fSChris Mason */ 52023f157a2fSChris Mason while (slot < nritems) { 52033f157a2fSChris Mason u64 gen; 5204e02119d5SChris Mason 52053f157a2fSChris Mason gen = btrfs_node_ptr_generation(cur, slot); 52063f157a2fSChris Mason if (gen < min_trans) { 52073f157a2fSChris Mason slot++; 52083f157a2fSChris Mason continue; 52093f157a2fSChris Mason } 52103f157a2fSChris Mason break; 52113f157a2fSChris Mason } 5212e02119d5SChris Mason find_next_key: 52133f157a2fSChris Mason /* 52143f157a2fSChris Mason * we didn't find a candidate key in this node, walk forward 52153f157a2fSChris Mason * and find another one 52163f157a2fSChris Mason */ 52173f157a2fSChris Mason if (slot >= nritems) { 5218e02119d5SChris Mason path->slots[level] = slot; 5219b4ce94deSChris Mason btrfs_set_path_blocking(path); 5220e02119d5SChris Mason sret = btrfs_find_next_key(root, path, min_key, level, 5221de78b51aSEric Sandeen min_trans); 5222e02119d5SChris Mason if (sret == 0) { 5223b3b4aa74SDavid Sterba btrfs_release_path(path); 52243f157a2fSChris Mason goto again; 52253f157a2fSChris Mason } else { 52263f157a2fSChris Mason goto out; 52273f157a2fSChris Mason } 52283f157a2fSChris Mason } 52293f157a2fSChris Mason /* save our key for returning back */ 52303f157a2fSChris Mason btrfs_node_key_to_cpu(cur, &found_key, slot); 52313f157a2fSChris Mason path->slots[level] = slot; 52323f157a2fSChris Mason if (level == path->lowest_level) { 52333f157a2fSChris Mason ret = 0; 5234f7c79f30SChris Mason unlock_up(path, level, 1, 0, NULL); 52353f157a2fSChris Mason goto out; 52363f157a2fSChris Mason } 5237b4ce94deSChris Mason btrfs_set_path_blocking(path); 52383f157a2fSChris Mason cur = read_node_slot(root, cur, slot); 523979787eaaSJeff Mahoney BUG_ON(!cur); /* -ENOMEM */ 52403f157a2fSChris Mason 5241bd681513SChris Mason btrfs_tree_read_lock(cur); 5242b4ce94deSChris Mason 5243bd681513SChris Mason path->locks[level - 1] = BTRFS_READ_LOCK; 52443f157a2fSChris Mason path->nodes[level - 1] = cur; 5245f7c79f30SChris Mason unlock_up(path, level, 1, 0, NULL); 5246bd681513SChris Mason btrfs_clear_path_blocking(path, NULL, 0); 52473f157a2fSChris Mason } 52483f157a2fSChris Mason out: 52493f157a2fSChris Mason if (ret == 0) 52503f157a2fSChris Mason memcpy(min_key, &found_key, sizeof(found_key)); 5251b4ce94deSChris Mason btrfs_set_path_blocking(path); 52523f157a2fSChris Mason return ret; 52533f157a2fSChris Mason } 52543f157a2fSChris Mason 52557069830aSAlexander Block static void tree_move_down(struct btrfs_root *root, 52567069830aSAlexander Block struct btrfs_path *path, 52577069830aSAlexander Block int *level, int root_level) 52587069830aSAlexander Block { 525974dd17fbSChris Mason BUG_ON(*level == 0); 52607069830aSAlexander Block path->nodes[*level - 1] = read_node_slot(root, path->nodes[*level], 52617069830aSAlexander Block path->slots[*level]); 52627069830aSAlexander Block path->slots[*level - 1] = 0; 52637069830aSAlexander Block (*level)--; 52647069830aSAlexander Block } 52657069830aSAlexander Block 52667069830aSAlexander Block static int tree_move_next_or_upnext(struct btrfs_root *root, 52677069830aSAlexander Block struct btrfs_path *path, 52687069830aSAlexander Block int *level, int root_level) 52697069830aSAlexander Block { 52707069830aSAlexander Block int ret = 0; 52717069830aSAlexander Block int nritems; 52727069830aSAlexander Block nritems = btrfs_header_nritems(path->nodes[*level]); 52737069830aSAlexander Block 52747069830aSAlexander Block path->slots[*level]++; 52757069830aSAlexander Block 527674dd17fbSChris Mason while (path->slots[*level] >= nritems) { 52777069830aSAlexander Block if (*level == root_level) 52787069830aSAlexander Block return -1; 52797069830aSAlexander Block 52807069830aSAlexander Block /* move upnext */ 52817069830aSAlexander Block path->slots[*level] = 0; 52827069830aSAlexander Block free_extent_buffer(path->nodes[*level]); 52837069830aSAlexander Block path->nodes[*level] = NULL; 52847069830aSAlexander Block (*level)++; 52857069830aSAlexander Block path->slots[*level]++; 52867069830aSAlexander Block 52877069830aSAlexander Block nritems = btrfs_header_nritems(path->nodes[*level]); 52887069830aSAlexander Block ret = 1; 52897069830aSAlexander Block } 52907069830aSAlexander Block return ret; 52917069830aSAlexander Block } 52927069830aSAlexander Block 52937069830aSAlexander Block /* 52947069830aSAlexander Block * Returns 1 if it had to move up and next. 0 is returned if it moved only next 52957069830aSAlexander Block * or down. 52967069830aSAlexander Block */ 52977069830aSAlexander Block static int tree_advance(struct btrfs_root *root, 52987069830aSAlexander Block struct btrfs_path *path, 52997069830aSAlexander Block int *level, int root_level, 53007069830aSAlexander Block int allow_down, 53017069830aSAlexander Block struct btrfs_key *key) 53027069830aSAlexander Block { 53037069830aSAlexander Block int ret; 53047069830aSAlexander Block 53057069830aSAlexander Block if (*level == 0 || !allow_down) { 53067069830aSAlexander Block ret = tree_move_next_or_upnext(root, path, level, root_level); 53077069830aSAlexander Block } else { 53087069830aSAlexander Block tree_move_down(root, path, level, root_level); 53097069830aSAlexander Block ret = 0; 53107069830aSAlexander Block } 53117069830aSAlexander Block if (ret >= 0) { 53127069830aSAlexander Block if (*level == 0) 53137069830aSAlexander Block btrfs_item_key_to_cpu(path->nodes[*level], key, 53147069830aSAlexander Block path->slots[*level]); 53157069830aSAlexander Block else 53167069830aSAlexander Block btrfs_node_key_to_cpu(path->nodes[*level], key, 53177069830aSAlexander Block path->slots[*level]); 53187069830aSAlexander Block } 53197069830aSAlexander Block return ret; 53207069830aSAlexander Block } 53217069830aSAlexander Block 53227069830aSAlexander Block static int tree_compare_item(struct btrfs_root *left_root, 53237069830aSAlexander Block struct btrfs_path *left_path, 53247069830aSAlexander Block struct btrfs_path *right_path, 53257069830aSAlexander Block char *tmp_buf) 53267069830aSAlexander Block { 53277069830aSAlexander Block int cmp; 53287069830aSAlexander Block int len1, len2; 53297069830aSAlexander Block unsigned long off1, off2; 53307069830aSAlexander Block 53317069830aSAlexander Block len1 = btrfs_item_size_nr(left_path->nodes[0], left_path->slots[0]); 53327069830aSAlexander Block len2 = btrfs_item_size_nr(right_path->nodes[0], right_path->slots[0]); 53337069830aSAlexander Block if (len1 != len2) 53347069830aSAlexander Block return 1; 53357069830aSAlexander Block 53367069830aSAlexander Block off1 = btrfs_item_ptr_offset(left_path->nodes[0], left_path->slots[0]); 53377069830aSAlexander Block off2 = btrfs_item_ptr_offset(right_path->nodes[0], 53387069830aSAlexander Block right_path->slots[0]); 53397069830aSAlexander Block 53407069830aSAlexander Block read_extent_buffer(left_path->nodes[0], tmp_buf, off1, len1); 53417069830aSAlexander Block 53427069830aSAlexander Block cmp = memcmp_extent_buffer(right_path->nodes[0], tmp_buf, off2, len1); 53437069830aSAlexander Block if (cmp) 53447069830aSAlexander Block return 1; 53457069830aSAlexander Block return 0; 53467069830aSAlexander Block } 53477069830aSAlexander Block 53487069830aSAlexander Block #define ADVANCE 1 53497069830aSAlexander Block #define ADVANCE_ONLY_NEXT -1 53507069830aSAlexander Block 53517069830aSAlexander Block /* 53527069830aSAlexander Block * This function compares two trees and calls the provided callback for 53537069830aSAlexander Block * every changed/new/deleted item it finds. 53547069830aSAlexander Block * If shared tree blocks are encountered, whole subtrees are skipped, making 53557069830aSAlexander Block * the compare pretty fast on snapshotted subvolumes. 53567069830aSAlexander Block * 53577069830aSAlexander Block * This currently works on commit roots only. As commit roots are read only, 53587069830aSAlexander Block * we don't do any locking. The commit roots are protected with transactions. 53597069830aSAlexander Block * Transactions are ended and rejoined when a commit is tried in between. 53607069830aSAlexander Block * 53617069830aSAlexander Block * This function checks for modifications done to the trees while comparing. 53627069830aSAlexander Block * If it detects a change, it aborts immediately. 53637069830aSAlexander Block */ 53647069830aSAlexander Block int btrfs_compare_trees(struct btrfs_root *left_root, 53657069830aSAlexander Block struct btrfs_root *right_root, 53667069830aSAlexander Block btrfs_changed_cb_t changed_cb, void *ctx) 53677069830aSAlexander Block { 53687069830aSAlexander Block int ret; 53697069830aSAlexander Block int cmp; 53707069830aSAlexander Block struct btrfs_path *left_path = NULL; 53717069830aSAlexander Block struct btrfs_path *right_path = NULL; 53727069830aSAlexander Block struct btrfs_key left_key; 53737069830aSAlexander Block struct btrfs_key right_key; 53747069830aSAlexander Block char *tmp_buf = NULL; 53757069830aSAlexander Block int left_root_level; 53767069830aSAlexander Block int right_root_level; 53777069830aSAlexander Block int left_level; 53787069830aSAlexander Block int right_level; 53797069830aSAlexander Block int left_end_reached; 53807069830aSAlexander Block int right_end_reached; 53817069830aSAlexander Block int advance_left; 53827069830aSAlexander Block int advance_right; 53837069830aSAlexander Block u64 left_blockptr; 53847069830aSAlexander Block u64 right_blockptr; 53856baa4293SFilipe Manana u64 left_gen; 53866baa4293SFilipe Manana u64 right_gen; 53877069830aSAlexander Block 53887069830aSAlexander Block left_path = btrfs_alloc_path(); 53897069830aSAlexander Block if (!left_path) { 53907069830aSAlexander Block ret = -ENOMEM; 53917069830aSAlexander Block goto out; 53927069830aSAlexander Block } 53937069830aSAlexander Block right_path = btrfs_alloc_path(); 53947069830aSAlexander Block if (!right_path) { 53957069830aSAlexander Block ret = -ENOMEM; 53967069830aSAlexander Block goto out; 53977069830aSAlexander Block } 53987069830aSAlexander Block 53997069830aSAlexander Block tmp_buf = kmalloc(left_root->leafsize, GFP_NOFS); 54007069830aSAlexander Block if (!tmp_buf) { 54017069830aSAlexander Block ret = -ENOMEM; 54027069830aSAlexander Block goto out; 54037069830aSAlexander Block } 54047069830aSAlexander Block 54057069830aSAlexander Block left_path->search_commit_root = 1; 54067069830aSAlexander Block left_path->skip_locking = 1; 54077069830aSAlexander Block right_path->search_commit_root = 1; 54087069830aSAlexander Block right_path->skip_locking = 1; 54097069830aSAlexander Block 54107069830aSAlexander Block /* 54117069830aSAlexander Block * Strategy: Go to the first items of both trees. Then do 54127069830aSAlexander Block * 54137069830aSAlexander Block * If both trees are at level 0 54147069830aSAlexander Block * Compare keys of current items 54157069830aSAlexander Block * If left < right treat left item as new, advance left tree 54167069830aSAlexander Block * and repeat 54177069830aSAlexander Block * If left > right treat right item as deleted, advance right tree 54187069830aSAlexander Block * and repeat 54197069830aSAlexander Block * If left == right do deep compare of items, treat as changed if 54207069830aSAlexander Block * needed, advance both trees and repeat 54217069830aSAlexander Block * If both trees are at the same level but not at level 0 54227069830aSAlexander Block * Compare keys of current nodes/leafs 54237069830aSAlexander Block * If left < right advance left tree and repeat 54247069830aSAlexander Block * If left > right advance right tree and repeat 54257069830aSAlexander Block * If left == right compare blockptrs of the next nodes/leafs 54267069830aSAlexander Block * If they match advance both trees but stay at the same level 54277069830aSAlexander Block * and repeat 54287069830aSAlexander Block * If they don't match advance both trees while allowing to go 54297069830aSAlexander Block * deeper and repeat 54307069830aSAlexander Block * If tree levels are different 54317069830aSAlexander Block * Advance the tree that needs it and repeat 54327069830aSAlexander Block * 54337069830aSAlexander Block * Advancing a tree means: 54347069830aSAlexander Block * If we are at level 0, try to go to the next slot. If that's not 54357069830aSAlexander Block * possible, go one level up and repeat. Stop when we found a level 54367069830aSAlexander Block * where we could go to the next slot. We may at this point be on a 54377069830aSAlexander Block * node or a leaf. 54387069830aSAlexander Block * 54397069830aSAlexander Block * If we are not at level 0 and not on shared tree blocks, go one 54407069830aSAlexander Block * level deeper. 54417069830aSAlexander Block * 54427069830aSAlexander Block * If we are not at level 0 and on shared tree blocks, go one slot to 54437069830aSAlexander Block * the right if possible or go up and right. 54447069830aSAlexander Block */ 54457069830aSAlexander Block 54463f8a18ccSJosef Bacik down_read(&left_root->fs_info->commit_root_sem); 54477069830aSAlexander Block left_level = btrfs_header_level(left_root->commit_root); 54487069830aSAlexander Block left_root_level = left_level; 54497069830aSAlexander Block left_path->nodes[left_level] = left_root->commit_root; 54507069830aSAlexander Block extent_buffer_get(left_path->nodes[left_level]); 54517069830aSAlexander Block 54527069830aSAlexander Block right_level = btrfs_header_level(right_root->commit_root); 54537069830aSAlexander Block right_root_level = right_level; 54547069830aSAlexander Block right_path->nodes[right_level] = right_root->commit_root; 54557069830aSAlexander Block extent_buffer_get(right_path->nodes[right_level]); 54563f8a18ccSJosef Bacik up_read(&left_root->fs_info->commit_root_sem); 54577069830aSAlexander Block 54587069830aSAlexander Block if (left_level == 0) 54597069830aSAlexander Block btrfs_item_key_to_cpu(left_path->nodes[left_level], 54607069830aSAlexander Block &left_key, left_path->slots[left_level]); 54617069830aSAlexander Block else 54627069830aSAlexander Block btrfs_node_key_to_cpu(left_path->nodes[left_level], 54637069830aSAlexander Block &left_key, left_path->slots[left_level]); 54647069830aSAlexander Block if (right_level == 0) 54657069830aSAlexander Block btrfs_item_key_to_cpu(right_path->nodes[right_level], 54667069830aSAlexander Block &right_key, right_path->slots[right_level]); 54677069830aSAlexander Block else 54687069830aSAlexander Block btrfs_node_key_to_cpu(right_path->nodes[right_level], 54697069830aSAlexander Block &right_key, right_path->slots[right_level]); 54707069830aSAlexander Block 54717069830aSAlexander Block left_end_reached = right_end_reached = 0; 54727069830aSAlexander Block advance_left = advance_right = 0; 54737069830aSAlexander Block 54747069830aSAlexander Block while (1) { 54757069830aSAlexander Block if (advance_left && !left_end_reached) { 54767069830aSAlexander Block ret = tree_advance(left_root, left_path, &left_level, 54777069830aSAlexander Block left_root_level, 54787069830aSAlexander Block advance_left != ADVANCE_ONLY_NEXT, 54797069830aSAlexander Block &left_key); 54807069830aSAlexander Block if (ret < 0) 54817069830aSAlexander Block left_end_reached = ADVANCE; 54827069830aSAlexander Block advance_left = 0; 54837069830aSAlexander Block } 54847069830aSAlexander Block if (advance_right && !right_end_reached) { 54857069830aSAlexander Block ret = tree_advance(right_root, right_path, &right_level, 54867069830aSAlexander Block right_root_level, 54877069830aSAlexander Block advance_right != ADVANCE_ONLY_NEXT, 54887069830aSAlexander Block &right_key); 54897069830aSAlexander Block if (ret < 0) 54907069830aSAlexander Block right_end_reached = ADVANCE; 54917069830aSAlexander Block advance_right = 0; 54927069830aSAlexander Block } 54937069830aSAlexander Block 54947069830aSAlexander Block if (left_end_reached && right_end_reached) { 54957069830aSAlexander Block ret = 0; 54967069830aSAlexander Block goto out; 54977069830aSAlexander Block } else if (left_end_reached) { 54987069830aSAlexander Block if (right_level == 0) { 54997069830aSAlexander Block ret = changed_cb(left_root, right_root, 55007069830aSAlexander Block left_path, right_path, 55017069830aSAlexander Block &right_key, 55027069830aSAlexander Block BTRFS_COMPARE_TREE_DELETED, 55037069830aSAlexander Block ctx); 55047069830aSAlexander Block if (ret < 0) 55057069830aSAlexander Block goto out; 55067069830aSAlexander Block } 55077069830aSAlexander Block advance_right = ADVANCE; 55087069830aSAlexander Block continue; 55097069830aSAlexander Block } else if (right_end_reached) { 55107069830aSAlexander Block if (left_level == 0) { 55117069830aSAlexander Block ret = changed_cb(left_root, right_root, 55127069830aSAlexander Block left_path, right_path, 55137069830aSAlexander Block &left_key, 55147069830aSAlexander Block BTRFS_COMPARE_TREE_NEW, 55157069830aSAlexander Block ctx); 55167069830aSAlexander Block if (ret < 0) 55177069830aSAlexander Block goto out; 55187069830aSAlexander Block } 55197069830aSAlexander Block advance_left = ADVANCE; 55207069830aSAlexander Block continue; 55217069830aSAlexander Block } 55227069830aSAlexander Block 55237069830aSAlexander Block if (left_level == 0 && right_level == 0) { 55247069830aSAlexander Block cmp = btrfs_comp_cpu_keys(&left_key, &right_key); 55257069830aSAlexander Block if (cmp < 0) { 55267069830aSAlexander Block ret = changed_cb(left_root, right_root, 55277069830aSAlexander Block left_path, right_path, 55287069830aSAlexander Block &left_key, 55297069830aSAlexander Block BTRFS_COMPARE_TREE_NEW, 55307069830aSAlexander Block ctx); 55317069830aSAlexander Block if (ret < 0) 55327069830aSAlexander Block goto out; 55337069830aSAlexander Block advance_left = ADVANCE; 55347069830aSAlexander Block } else if (cmp > 0) { 55357069830aSAlexander Block ret = changed_cb(left_root, right_root, 55367069830aSAlexander Block left_path, right_path, 55377069830aSAlexander Block &right_key, 55387069830aSAlexander Block BTRFS_COMPARE_TREE_DELETED, 55397069830aSAlexander Block ctx); 55407069830aSAlexander Block if (ret < 0) 55417069830aSAlexander Block goto out; 55427069830aSAlexander Block advance_right = ADVANCE; 55437069830aSAlexander Block } else { 5544ba5e8f2eSJosef Bacik enum btrfs_compare_tree_result cmp; 5545ba5e8f2eSJosef Bacik 554674dd17fbSChris Mason WARN_ON(!extent_buffer_uptodate(left_path->nodes[0])); 55477069830aSAlexander Block ret = tree_compare_item(left_root, left_path, 55487069830aSAlexander Block right_path, tmp_buf); 5549ba5e8f2eSJosef Bacik if (ret) 5550ba5e8f2eSJosef Bacik cmp = BTRFS_COMPARE_TREE_CHANGED; 5551ba5e8f2eSJosef Bacik else 5552ba5e8f2eSJosef Bacik cmp = BTRFS_COMPARE_TREE_SAME; 55537069830aSAlexander Block ret = changed_cb(left_root, right_root, 55547069830aSAlexander Block left_path, right_path, 5555ba5e8f2eSJosef Bacik &left_key, cmp, ctx); 55567069830aSAlexander Block if (ret < 0) 55577069830aSAlexander Block goto out; 55587069830aSAlexander Block advance_left = ADVANCE; 55597069830aSAlexander Block advance_right = ADVANCE; 55607069830aSAlexander Block } 55617069830aSAlexander Block } else if (left_level == right_level) { 55627069830aSAlexander Block cmp = btrfs_comp_cpu_keys(&left_key, &right_key); 55637069830aSAlexander Block if (cmp < 0) { 55647069830aSAlexander Block advance_left = ADVANCE; 55657069830aSAlexander Block } else if (cmp > 0) { 55667069830aSAlexander Block advance_right = ADVANCE; 55677069830aSAlexander Block } else { 55687069830aSAlexander Block left_blockptr = btrfs_node_blockptr( 55697069830aSAlexander Block left_path->nodes[left_level], 55707069830aSAlexander Block left_path->slots[left_level]); 55717069830aSAlexander Block right_blockptr = btrfs_node_blockptr( 55727069830aSAlexander Block right_path->nodes[right_level], 55737069830aSAlexander Block right_path->slots[right_level]); 55746baa4293SFilipe Manana left_gen = btrfs_node_ptr_generation( 55756baa4293SFilipe Manana left_path->nodes[left_level], 55766baa4293SFilipe Manana left_path->slots[left_level]); 55776baa4293SFilipe Manana right_gen = btrfs_node_ptr_generation( 55786baa4293SFilipe Manana right_path->nodes[right_level], 55796baa4293SFilipe Manana right_path->slots[right_level]); 55806baa4293SFilipe Manana if (left_blockptr == right_blockptr && 55816baa4293SFilipe Manana left_gen == right_gen) { 55827069830aSAlexander Block /* 55837069830aSAlexander Block * As we're on a shared block, don't 55847069830aSAlexander Block * allow to go deeper. 55857069830aSAlexander Block */ 55867069830aSAlexander Block advance_left = ADVANCE_ONLY_NEXT; 55877069830aSAlexander Block advance_right = ADVANCE_ONLY_NEXT; 55887069830aSAlexander Block } else { 55897069830aSAlexander Block advance_left = ADVANCE; 55907069830aSAlexander Block advance_right = ADVANCE; 55917069830aSAlexander Block } 55927069830aSAlexander Block } 55937069830aSAlexander Block } else if (left_level < right_level) { 55947069830aSAlexander Block advance_right = ADVANCE; 55957069830aSAlexander Block } else { 55967069830aSAlexander Block advance_left = ADVANCE; 55977069830aSAlexander Block } 55987069830aSAlexander Block } 55997069830aSAlexander Block 56007069830aSAlexander Block out: 56017069830aSAlexander Block btrfs_free_path(left_path); 56027069830aSAlexander Block btrfs_free_path(right_path); 56037069830aSAlexander Block kfree(tmp_buf); 56047069830aSAlexander Block return ret; 56057069830aSAlexander Block } 56067069830aSAlexander Block 56073f157a2fSChris Mason /* 56083f157a2fSChris Mason * this is similar to btrfs_next_leaf, but does not try to preserve 56093f157a2fSChris Mason * and fixup the path. It looks for and returns the next key in the 5610de78b51aSEric Sandeen * tree based on the current path and the min_trans parameters. 56113f157a2fSChris Mason * 56123f157a2fSChris Mason * 0 is returned if another key is found, < 0 if there are any errors 56133f157a2fSChris Mason * and 1 is returned if there are no higher keys in the tree 56143f157a2fSChris Mason * 56153f157a2fSChris Mason * path->keep_locks should be set to 1 on the search made before 56163f157a2fSChris Mason * calling this function. 56173f157a2fSChris Mason */ 5618e7a84565SChris Mason int btrfs_find_next_key(struct btrfs_root *root, struct btrfs_path *path, 5619de78b51aSEric Sandeen struct btrfs_key *key, int level, u64 min_trans) 5620e7a84565SChris Mason { 5621e7a84565SChris Mason int slot; 5622e7a84565SChris Mason struct extent_buffer *c; 5623e7a84565SChris Mason 5624934d375bSChris Mason WARN_ON(!path->keep_locks); 5625e7a84565SChris Mason while (level < BTRFS_MAX_LEVEL) { 5626e7a84565SChris Mason if (!path->nodes[level]) 5627e7a84565SChris Mason return 1; 5628e7a84565SChris Mason 5629e7a84565SChris Mason slot = path->slots[level] + 1; 5630e7a84565SChris Mason c = path->nodes[level]; 56313f157a2fSChris Mason next: 5632e7a84565SChris Mason if (slot >= btrfs_header_nritems(c)) { 563333c66f43SYan Zheng int ret; 563433c66f43SYan Zheng int orig_lowest; 563533c66f43SYan Zheng struct btrfs_key cur_key; 563633c66f43SYan Zheng if (level + 1 >= BTRFS_MAX_LEVEL || 563733c66f43SYan Zheng !path->nodes[level + 1]) 5638e7a84565SChris Mason return 1; 563933c66f43SYan Zheng 564033c66f43SYan Zheng if (path->locks[level + 1]) { 564133c66f43SYan Zheng level++; 5642e7a84565SChris Mason continue; 5643e7a84565SChris Mason } 564433c66f43SYan Zheng 564533c66f43SYan Zheng slot = btrfs_header_nritems(c) - 1; 564633c66f43SYan Zheng if (level == 0) 564733c66f43SYan Zheng btrfs_item_key_to_cpu(c, &cur_key, slot); 564833c66f43SYan Zheng else 564933c66f43SYan Zheng btrfs_node_key_to_cpu(c, &cur_key, slot); 565033c66f43SYan Zheng 565133c66f43SYan Zheng orig_lowest = path->lowest_level; 5652b3b4aa74SDavid Sterba btrfs_release_path(path); 565333c66f43SYan Zheng path->lowest_level = level; 565433c66f43SYan Zheng ret = btrfs_search_slot(NULL, root, &cur_key, path, 565533c66f43SYan Zheng 0, 0); 565633c66f43SYan Zheng path->lowest_level = orig_lowest; 565733c66f43SYan Zheng if (ret < 0) 565833c66f43SYan Zheng return ret; 565933c66f43SYan Zheng 566033c66f43SYan Zheng c = path->nodes[level]; 566133c66f43SYan Zheng slot = path->slots[level]; 566233c66f43SYan Zheng if (ret == 0) 566333c66f43SYan Zheng slot++; 566433c66f43SYan Zheng goto next; 566533c66f43SYan Zheng } 566633c66f43SYan Zheng 5667e7a84565SChris Mason if (level == 0) 5668e7a84565SChris Mason btrfs_item_key_to_cpu(c, key, slot); 56693f157a2fSChris Mason else { 56703f157a2fSChris Mason u64 gen = btrfs_node_ptr_generation(c, slot); 56713f157a2fSChris Mason 56723f157a2fSChris Mason if (gen < min_trans) { 56733f157a2fSChris Mason slot++; 56743f157a2fSChris Mason goto next; 56753f157a2fSChris Mason } 5676e7a84565SChris Mason btrfs_node_key_to_cpu(c, key, slot); 56773f157a2fSChris Mason } 5678e7a84565SChris Mason return 0; 5679e7a84565SChris Mason } 5680e7a84565SChris Mason return 1; 5681e7a84565SChris Mason } 5682e7a84565SChris Mason 56837bb86316SChris Mason /* 5684925baeddSChris Mason * search the tree again to find a leaf with greater keys 56850f70abe2SChris Mason * returns 0 if it found something or 1 if there are no greater leaves. 56860f70abe2SChris Mason * returns < 0 on io errors. 568797571fd0SChris Mason */ 5688234b63a0SChris Mason int btrfs_next_leaf(struct btrfs_root *root, struct btrfs_path *path) 5689d97e63b6SChris Mason { 56903d7806ecSJan Schmidt return btrfs_next_old_leaf(root, path, 0); 56913d7806ecSJan Schmidt } 56923d7806ecSJan Schmidt 56933d7806ecSJan Schmidt int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path, 56943d7806ecSJan Schmidt u64 time_seq) 56953d7806ecSJan Schmidt { 5696d97e63b6SChris Mason int slot; 56978e73f275SChris Mason int level; 56985f39d397SChris Mason struct extent_buffer *c; 56998e73f275SChris Mason struct extent_buffer *next; 5700925baeddSChris Mason struct btrfs_key key; 5701925baeddSChris Mason u32 nritems; 5702925baeddSChris Mason int ret; 57038e73f275SChris Mason int old_spinning = path->leave_spinning; 5704bd681513SChris Mason int next_rw_lock = 0; 5705925baeddSChris Mason 5706925baeddSChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 5707d397712bSChris Mason if (nritems == 0) 5708925baeddSChris Mason return 1; 5709925baeddSChris Mason 57108e73f275SChris Mason btrfs_item_key_to_cpu(path->nodes[0], &key, nritems - 1); 57118e73f275SChris Mason again: 57128e73f275SChris Mason level = 1; 57138e73f275SChris Mason next = NULL; 5714bd681513SChris Mason next_rw_lock = 0; 5715b3b4aa74SDavid Sterba btrfs_release_path(path); 57168e73f275SChris Mason 5717a2135011SChris Mason path->keep_locks = 1; 57188e73f275SChris Mason path->leave_spinning = 1; 57198e73f275SChris Mason 57203d7806ecSJan Schmidt if (time_seq) 57213d7806ecSJan Schmidt ret = btrfs_search_old_slot(root, &key, path, time_seq); 57223d7806ecSJan Schmidt else 5723925baeddSChris Mason ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 5724925baeddSChris Mason path->keep_locks = 0; 5725925baeddSChris Mason 5726925baeddSChris Mason if (ret < 0) 5727925baeddSChris Mason return ret; 5728925baeddSChris Mason 5729a2135011SChris Mason nritems = btrfs_header_nritems(path->nodes[0]); 5730168fd7d2SChris Mason /* 5731168fd7d2SChris Mason * by releasing the path above we dropped all our locks. A balance 5732168fd7d2SChris Mason * could have added more items next to the key that used to be 5733168fd7d2SChris Mason * at the very end of the block. So, check again here and 5734168fd7d2SChris Mason * advance the path if there are now more items available. 5735168fd7d2SChris Mason */ 5736a2135011SChris Mason if (nritems > 0 && path->slots[0] < nritems - 1) { 5737e457afecSYan Zheng if (ret == 0) 5738168fd7d2SChris Mason path->slots[0]++; 57398e73f275SChris Mason ret = 0; 5740925baeddSChris Mason goto done; 5741925baeddSChris Mason } 5742d97e63b6SChris Mason 5743234b63a0SChris Mason while (level < BTRFS_MAX_LEVEL) { 57448e73f275SChris Mason if (!path->nodes[level]) { 57458e73f275SChris Mason ret = 1; 57468e73f275SChris Mason goto done; 57478e73f275SChris Mason } 57485f39d397SChris Mason 5749d97e63b6SChris Mason slot = path->slots[level] + 1; 5750d97e63b6SChris Mason c = path->nodes[level]; 57515f39d397SChris Mason if (slot >= btrfs_header_nritems(c)) { 5752d97e63b6SChris Mason level++; 57538e73f275SChris Mason if (level == BTRFS_MAX_LEVEL) { 57548e73f275SChris Mason ret = 1; 57558e73f275SChris Mason goto done; 57568e73f275SChris Mason } 5757d97e63b6SChris Mason continue; 5758d97e63b6SChris Mason } 57595f39d397SChris Mason 5760925baeddSChris Mason if (next) { 5761bd681513SChris Mason btrfs_tree_unlock_rw(next, next_rw_lock); 57625f39d397SChris Mason free_extent_buffer(next); 5763925baeddSChris Mason } 57645f39d397SChris Mason 57658e73f275SChris Mason next = c; 5766bd681513SChris Mason next_rw_lock = path->locks[level]; 57678e73f275SChris Mason ret = read_block_for_search(NULL, root, path, &next, level, 57685d9e75c4SJan Schmidt slot, &key, 0); 57698e73f275SChris Mason if (ret == -EAGAIN) 57708e73f275SChris Mason goto again; 57715f39d397SChris Mason 577276a05b35SChris Mason if (ret < 0) { 5773b3b4aa74SDavid Sterba btrfs_release_path(path); 577476a05b35SChris Mason goto done; 577576a05b35SChris Mason } 577676a05b35SChris Mason 57775cd57b2cSChris Mason if (!path->skip_locking) { 5778bd681513SChris Mason ret = btrfs_try_tree_read_lock(next); 5779d42244a0SJan Schmidt if (!ret && time_seq) { 5780d42244a0SJan Schmidt /* 5781d42244a0SJan Schmidt * If we don't get the lock, we may be racing 5782d42244a0SJan Schmidt * with push_leaf_left, holding that lock while 5783d42244a0SJan Schmidt * itself waiting for the leaf we've currently 5784d42244a0SJan Schmidt * locked. To solve this situation, we give up 5785d42244a0SJan Schmidt * on our lock and cycle. 5786d42244a0SJan Schmidt */ 5787cf538830SJan Schmidt free_extent_buffer(next); 5788d42244a0SJan Schmidt btrfs_release_path(path); 5789d42244a0SJan Schmidt cond_resched(); 5790d42244a0SJan Schmidt goto again; 5791d42244a0SJan Schmidt } 57928e73f275SChris Mason if (!ret) { 57938e73f275SChris Mason btrfs_set_path_blocking(path); 5794bd681513SChris Mason btrfs_tree_read_lock(next); 5795bd681513SChris Mason btrfs_clear_path_blocking(path, next, 5796bd681513SChris Mason BTRFS_READ_LOCK); 57978e73f275SChris Mason } 5798bd681513SChris Mason next_rw_lock = BTRFS_READ_LOCK; 5799bd681513SChris Mason } 5800d97e63b6SChris Mason break; 5801d97e63b6SChris Mason } 5802d97e63b6SChris Mason path->slots[level] = slot; 5803d97e63b6SChris Mason while (1) { 5804d97e63b6SChris Mason level--; 5805d97e63b6SChris Mason c = path->nodes[level]; 5806925baeddSChris Mason if (path->locks[level]) 5807bd681513SChris Mason btrfs_tree_unlock_rw(c, path->locks[level]); 58088e73f275SChris Mason 58095f39d397SChris Mason free_extent_buffer(c); 5810d97e63b6SChris Mason path->nodes[level] = next; 5811d97e63b6SChris Mason path->slots[level] = 0; 5812a74a4b97SChris Mason if (!path->skip_locking) 5813bd681513SChris Mason path->locks[level] = next_rw_lock; 5814d97e63b6SChris Mason if (!level) 5815d97e63b6SChris Mason break; 5816b4ce94deSChris Mason 58178e73f275SChris Mason ret = read_block_for_search(NULL, root, path, &next, level, 58185d9e75c4SJan Schmidt 0, &key, 0); 58198e73f275SChris Mason if (ret == -EAGAIN) 58208e73f275SChris Mason goto again; 58218e73f275SChris Mason 582276a05b35SChris Mason if (ret < 0) { 5823b3b4aa74SDavid Sterba btrfs_release_path(path); 582476a05b35SChris Mason goto done; 582576a05b35SChris Mason } 582676a05b35SChris Mason 58275cd57b2cSChris Mason if (!path->skip_locking) { 5828bd681513SChris Mason ret = btrfs_try_tree_read_lock(next); 58298e73f275SChris Mason if (!ret) { 58308e73f275SChris Mason btrfs_set_path_blocking(path); 5831bd681513SChris Mason btrfs_tree_read_lock(next); 5832bd681513SChris Mason btrfs_clear_path_blocking(path, next, 5833bd681513SChris Mason BTRFS_READ_LOCK); 58348e73f275SChris Mason } 5835bd681513SChris Mason next_rw_lock = BTRFS_READ_LOCK; 5836bd681513SChris Mason } 5837d97e63b6SChris Mason } 58388e73f275SChris Mason ret = 0; 5839925baeddSChris Mason done: 5840f7c79f30SChris Mason unlock_up(path, 0, 1, 0, NULL); 58418e73f275SChris Mason path->leave_spinning = old_spinning; 58428e73f275SChris Mason if (!old_spinning) 58438e73f275SChris Mason btrfs_set_path_blocking(path); 58448e73f275SChris Mason 58458e73f275SChris Mason return ret; 5846d97e63b6SChris Mason } 58470b86a832SChris Mason 58483f157a2fSChris Mason /* 58493f157a2fSChris Mason * this uses btrfs_prev_leaf to walk backwards in the tree, and keeps 58503f157a2fSChris Mason * searching until it gets past min_objectid or finds an item of 'type' 58513f157a2fSChris Mason * 58523f157a2fSChris Mason * returns 0 if something is found, 1 if nothing was found and < 0 on error 58533f157a2fSChris Mason */ 58540b86a832SChris Mason int btrfs_previous_item(struct btrfs_root *root, 58550b86a832SChris Mason struct btrfs_path *path, u64 min_objectid, 58560b86a832SChris Mason int type) 58570b86a832SChris Mason { 58580b86a832SChris Mason struct btrfs_key found_key; 58590b86a832SChris Mason struct extent_buffer *leaf; 5860e02119d5SChris Mason u32 nritems; 58610b86a832SChris Mason int ret; 58620b86a832SChris Mason 58630b86a832SChris Mason while (1) { 58640b86a832SChris Mason if (path->slots[0] == 0) { 5865b4ce94deSChris Mason btrfs_set_path_blocking(path); 58660b86a832SChris Mason ret = btrfs_prev_leaf(root, path); 58670b86a832SChris Mason if (ret != 0) 58680b86a832SChris Mason return ret; 58690b86a832SChris Mason } else { 58700b86a832SChris Mason path->slots[0]--; 58710b86a832SChris Mason } 58720b86a832SChris Mason leaf = path->nodes[0]; 5873e02119d5SChris Mason nritems = btrfs_header_nritems(leaf); 5874e02119d5SChris Mason if (nritems == 0) 5875e02119d5SChris Mason return 1; 5876e02119d5SChris Mason if (path->slots[0] == nritems) 5877e02119d5SChris Mason path->slots[0]--; 5878e02119d5SChris Mason 58790b86a832SChris Mason btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 5880e02119d5SChris Mason if (found_key.objectid < min_objectid) 5881e02119d5SChris Mason break; 58820a4eefbbSYan Zheng if (found_key.type == type) 58830a4eefbbSYan Zheng return 0; 5884e02119d5SChris Mason if (found_key.objectid == min_objectid && 5885e02119d5SChris Mason found_key.type < type) 5886e02119d5SChris Mason break; 58870b86a832SChris Mason } 58880b86a832SChris Mason return 1; 58890b86a832SChris Mason } 5890ade2e0b3SWang Shilong 5891ade2e0b3SWang Shilong /* 5892ade2e0b3SWang Shilong * search in extent tree to find a previous Metadata/Data extent item with 5893ade2e0b3SWang Shilong * min objecitd. 5894ade2e0b3SWang Shilong * 5895ade2e0b3SWang Shilong * returns 0 if something is found, 1 if nothing was found and < 0 on error 5896ade2e0b3SWang Shilong */ 5897ade2e0b3SWang Shilong int btrfs_previous_extent_item(struct btrfs_root *root, 5898ade2e0b3SWang Shilong struct btrfs_path *path, u64 min_objectid) 5899ade2e0b3SWang Shilong { 5900ade2e0b3SWang Shilong struct btrfs_key found_key; 5901ade2e0b3SWang Shilong struct extent_buffer *leaf; 5902ade2e0b3SWang Shilong u32 nritems; 5903ade2e0b3SWang Shilong int ret; 5904ade2e0b3SWang Shilong 5905ade2e0b3SWang Shilong while (1) { 5906ade2e0b3SWang Shilong if (path->slots[0] == 0) { 5907ade2e0b3SWang Shilong btrfs_set_path_blocking(path); 5908ade2e0b3SWang Shilong ret = btrfs_prev_leaf(root, path); 5909ade2e0b3SWang Shilong if (ret != 0) 5910ade2e0b3SWang Shilong return ret; 5911ade2e0b3SWang Shilong } else { 5912ade2e0b3SWang Shilong path->slots[0]--; 5913ade2e0b3SWang Shilong } 5914ade2e0b3SWang Shilong leaf = path->nodes[0]; 5915ade2e0b3SWang Shilong nritems = btrfs_header_nritems(leaf); 5916ade2e0b3SWang Shilong if (nritems == 0) 5917ade2e0b3SWang Shilong return 1; 5918ade2e0b3SWang Shilong if (path->slots[0] == nritems) 5919ade2e0b3SWang Shilong path->slots[0]--; 5920ade2e0b3SWang Shilong 5921ade2e0b3SWang Shilong btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 5922ade2e0b3SWang Shilong if (found_key.objectid < min_objectid) 5923ade2e0b3SWang Shilong break; 5924ade2e0b3SWang Shilong if (found_key.type == BTRFS_EXTENT_ITEM_KEY || 5925ade2e0b3SWang Shilong found_key.type == BTRFS_METADATA_ITEM_KEY) 5926ade2e0b3SWang Shilong return 0; 5927ade2e0b3SWang Shilong if (found_key.objectid == min_objectid && 5928ade2e0b3SWang Shilong found_key.type < BTRFS_EXTENT_ITEM_KEY) 5929ade2e0b3SWang Shilong break; 5930ade2e0b3SWang Shilong } 5931ade2e0b3SWang Shilong return 1; 5932ade2e0b3SWang Shilong } 5933